Class: Iudex::Worker::Agent

Inherits:
Object
  • Object
show all
Includes:
Gravitext::HTMap, Core, DA, Filter::KeyHelper, Iudex::Worker
Defined in:
lib/iudex-worker/agent.rb

Constant Summary

Constants included from Iudex::Worker

VERSION

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Agent

Returns a new instance of Agent.



38
39
40
41
42
43
 
# File 'lib/iudex-worker/agent.rb', line 38

def initialize
  @log = RJack::SLF4J[ self.class ]
  @http_manager = nil
  @raise_on_run = false
  Hooker.apply( [ :iudex, :worker ], self )
end

Instance Attribute Details

#raise_on_run ⇒ Object

Returns the value of attribute raise_on_run.



36
37
38
 
# File 'lib/iudex-worker/agent.rb', line 36

def raise_on_run
  @raise_on_run
end

Instance Method Details

#filter_chain_factory ⇒ Object

Note this can/is used to override factory in derived classes.



50
51
52
 
# File 'lib/iudex-worker/agent.rb', line 50

def filter_chain_factory
  FilterChainFactory.new( 'agent' )
end

#http_client(executor) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/iudex-worker/agent.rb', line 54

def http_client( executor )
  if defined?( JettyHTTPClient.create_client )
    @log.info "Setting up JettyHTTPClient"
    JettyHTTPClient.create_client.tap do |c|
      c.executor = executor
      c.start
    end
  elsif defined?( AsyncHTTPClient.create_client )
    @log.info "Setting up AsyncHTTPClient"
    opts = {}
    opts[ :executor_service ] = executor if executor
    AsyncHTTPClient.create_client( opts )
  else
    gem     'iudex-httpclient-3', '~> 1.2.b'
    require 'iudex-httpclient-3'
    @log.info "Setting up HTTPClient3"
    @http_manager = HTTPClient3.create_manager
    @http_manager.start
    HTTPClient3::HTTPClient3.new( @http_manager.client )
  end
end

#poll_keys ⇒ Object 



45
46
47
 
# File 'lib/iudex-worker/agent.rb', line 45

def poll_keys
  [ :url, :type, :priority, :next_visit_after, :last_visit, :etag ]
end

#run ⇒ Object 



92
93
94
95
96
 
# File 'lib/iudex-worker/agent.rb', line 92

def run
  Hooker.with( :iudex ) do
    run_safe
  end
end

#run_safe ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/iudex-worker/agent.rb', line 98

def run_safe
  dsf = PoolDataSourceFactory.new
  data_source = dsf.create

  wpoller = work_poller( data_source )
  vexec = visit_manager( wpoller )
  vexec.start_executor

  hclient = http_client( vexec.executor )

  fcf = filter_chain_factory
  fcf.http_client = hclient
  fcf.data_source = data_source
  fcf.visit_counter = vexec
  fcf.work_poller = wpoller

  # FilterChain's executor is the same executor, unless using
  # HTTPClient3, where executor is best not used
  fcf.executor = vexec.executor unless @http_manager

  Hooker.apply( [ :iudex, :filter_factory ], fcf )

  fcf.filter do |chain|
    vexec.filter_chain = chain

    Hooker.log_not_applied # All hooks should be used by now

    vexec.start
    veref, vexec = vexec, nil
    veref.join # Run until interrupted
  end # fcf closes

rescue => e
  if @raise_on_run
    raise e
  else
    @log.error( "On run: ", e )
  end
ensure
  hclient.close if hclient && hclient.respond_to?( :close )
  @http_manager.shutdown if @http_manager

  # Jetty 9 for example, uses executor thread on start, so
  # executor shutdown is required for early termination
  # (i.e. when fcf.filter raises)
  vexec.shutdown if vexec

  dsf.close if dsf
end

#visit_manager(wpoller) ⇒ Object 



76
77
78
79
 
# File 'lib/iudex-worker/agent.rb', line 76

def visit_manager( wpoller )
  vexec = VisitManager.new( wpoller )
  Hooker.apply( [ :iudex, :visit_manager ], vexec )
end

#work_poller(data_source) ⇒ Object 



81
82
83
84
85
86
87
88
89
90
 
# File 'lib/iudex-worker/agent.rb', line 81

def work_poller( data_source )
  cmapper = ContentMapper.new( keys( poll_keys ) )
  wpoller = WorkPoller.new( data_source, cmapper )

  visit_q = Hooker.apply( [ :iudex, :visit_queue ], VisitQueue.new )

  wpoller.visit_queue_factory = VisitQueueFactory.new( visit_q )

  Hooker.apply( [ :iudex, :work_poller ], wpoller )
end