Class: Kimurai::Runner

Inherits:
Object
  • Object
show all
Defined in:
lib/kimurai/runner.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(spiders, parallel_jobs) ⇒ Runner

Returns a new instance of Runner.



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/kimurai/runner.rb', line 7

def initialize(spiders, parallel_jobs)
  @jobs = parallel_jobs
  @spiders = spiders
  @start_time = Time.now

  @session_info = {
    id: @start_time.to_i,
    status: :processing,
    start_time: @start_time,
    stop_time: nil,
    environment: Kimurai.env,
    concurrent_jobs: @jobs,
    spiders: @spiders
  }

  if time_zone = Kimurai.configuration.time_zone
    Kimurai.time_zone = time_zone
  end

  ENV.store("SESSION_ID", @start_time.to_i.to_s)
  ENV.store("RBCAT_COLORIZER", "false")
end

Instance Attribute Details

#jobs ⇒ Object (readonly)

Returns the value of attribute jobs.



5
6
7
 
# File 'lib/kimurai/runner.rb', line 5

def jobs
  @jobs
end

#session_info ⇒ Object (readonly)

Returns the value of attribute session_info.



5
6
7
 
# File 'lib/kimurai/runner.rb', line 5

def session_info
  @session_info
end

#spiders ⇒ Object (readonly)

Returns the value of attribute spiders.



5
6
7
 
# File 'lib/kimurai/runner.rb', line 5

def spiders
  @spiders
end

Instance Method Details

#run!(exception_on_fail: true) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/kimurai/runner.rb', line 30

def run!(exception_on_fail: true)
  puts ">>> Runner: started: #{session_info}"
  if at_start_callback = Kimurai.configuration.runner_at_start_callback
    at_start_callback.call(session_info)
  end

  running = true
  spiders.peach_with_index(jobs) do |spider, i|
    next unless running

    puts "> Runner: started spider: #{spider}, index: #{i}"
    pid = spawn("bundle", "exec", "kimurai", "crawl", spider, [:out, :err] => "log/#{spider}.log")
    Process.wait pid

    puts "< Runner: stopped spider: #{spider}, index: #{i}"
  end
rescue StandardError, SignalException, SystemExit => e
  running = false

  session_info.merge!(status: :failed, error: e.inspect, stop_time: Time.now)
  exception_on_fail ? raise(e) : [session_info, e]
else
  session_info.merge!(status: :completed, stop_time: Time.now)
ensure
  if at_stop_callback = Kimurai.configuration.runner_at_stop_callback
    at_stop_callback.call(session_info)
  end
  puts "<<< Runner: stopped: #{session_info}"
end