Class: Datahen::Client::ScraperJob

Inherits:
Base
  • Object
show all
Defined in:
lib/datahen/client/scraper_job.rb

Instance Method Summary collapse

Methods inherited from Base

#auth_token, #auth_token=, #env_api_url, env_auth_token, env_ignore_ssl, #ignore_ssl, #initialize

Constructor Details

This class inherits a constructor from Datahen::Client::Base

Instance Method Details

#all(scraper_name, opts = {}) ⇒ Object 



4
5
6
7
 
# File 'lib/datahen/client/scraper_job.rb', line 4

def all(scraper_name, opts={})
  params = @options.merge(opts)
  self.class.get("/scrapers/#{scraper_name}/jobs", params)
end

#cancel(scraper_name, opts = {}) ⇒ Object 



51
52
53
54
 
# File 'lib/datahen/client/scraper_job.rb', line 51

def cancel(scraper_name, opts={})
  opts[:status] = 'cancelled'
  update(scraper_name, opts)
end

#create(scraper_name, opts = {}) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/datahen/client/scraper_job.rb', line 9

def create(scraper_name, opts={})
  body = {}
  body[:standard_worker_count] = opts[:workers] if opts[:workers]
  body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
  body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
  body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
  if opts[:vars]
    if opts[:vars].is_a?(Array)
      body[:vars] = opts[:vars]
    elsif opts[:vars].is_a?(String)
      body[:vars] = JSON.parse(opts[:vars])
    end
  end
  params = @options.merge({body: body.to_json})
  self.class.post("/scrapers/#{scraper_name}/jobs", params)
end

#delete(scraper_name, opts = {}) ⇒ Object 



72
73
74
75
 
# File 'lib/datahen/client/scraper_job.rb', line 72

def delete(scraper_name, opts={})
  params = @options.merge(opts)
  self.class.delete("/scrapers/#{scraper_name}/current_job", params)
end

#find(scraper_name, opts = {}) ⇒ Object 



28
29
30
31
32
33
34
 
# File 'lib/datahen/client/scraper_job.rb', line 28

def find(scraper_name, opts={})
  if opts[:live]
    self.class.get("/scrapers/#{scraper_name}/current_job", @options)
  else
    self.class.get("/cached/scrapers/#{scraper_name}/current_job", @options)
  end
end

#pause(scraper_name, opts = {}) ⇒ Object 



61
62
63
64
 
# File 'lib/datahen/client/scraper_job.rb', line 61

def pause(scraper_name, opts={})
  opts[:status] = 'paused'
  update(scraper_name, opts)
end

#profile(scraper_name, opts = {}) ⇒ Object 



66
67
68
69
70
 
# File 'lib/datahen/client/scraper_job.rb', line 66

def profile(scraper_name, opts={})
  params = @options.merge(opts)

  self.class.get("/scrapers/#{scraper_name}/current_job/profile", params)
end

#resume(scraper_name, opts = {}) ⇒ Object 



56
57
58
59
 
# File 'lib/datahen/client/scraper_job.rb', line 56

def resume(scraper_name, opts={})
  opts[:status] = 'active'
  update(scraper_name, opts)
end

#update(scraper_name, opts = {}) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/datahen/client/scraper_job.rb', line 36

def update(scraper_name, opts={})
  body = {}
  body[:status] = opts[:status] if opts[:status]
  body[:standard_worker_count] = opts[:workers] if opts[:workers]
  body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
  body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
  body[:profile] = opts[:profile] if opts[:profile]
  body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
  params = @options.merge({body: body.to_json})

  self.class.put("/scrapers/#{scraper_name}/current_job", params)
end