Class: Wayfarer::CLI::Job

Inherits:
Base
  • Object
show all
Defined in:
lib/wayfarer/cli/job.rb

Instance Method Summary collapse

Methods inherited from Base

crawl_later, discard_on, #retry_job, retry_on, #task

Methods included from Middleware::Worker

included

Instance Method Details

#enqueue(job, url) ⇒ Object 



27
28
29
30
31
32
33
34
 
# File 'lib/wayfarer/cli/job.rb', line 27

def enqueue(job, url)
  load_environment
  mock_redis if options[:mock_redis]

  url = Addressable::URI.parse(url)
  job = job.classify.constantize
  job.crawl_later(url, batch: options[:batch])
end

#execute(job, url) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/wayfarer/cli/job.rb', line 42

def execute(job, url)
  load_environment
  mock_redis if options[:mock_redis]

  url = Addressable::URI.parse(url)
  job = job.classify.constantize

  job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
                                                                 max_threads: options[:max_threads])
  scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
  executor = scheduler.instance_variable_get(:@async_executor)

  job.crawl_later(url, batch: options[:batch])

  sleep(1) while executor.scheduled_task_count > executor.completed_task_count

  free_agent_pool
end

#perform(job, url) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/wayfarer/cli/job.rb', line 10

def perform(job, url)
  load_environment
  mock_redis if options[:mock_redis]

  url = Addressable::URI.parse(url)
  job = job.classify.constantize.new
  task = Wayfarer::Task.new(url, "tmp")
  job.arguments.push(task)
  job.perform(task)
  GC.new(job).run

  free_agent_pool
end