Class: Datahen::Client::ScraperJobPage

Inherits:
Base
  • Object
show all
Defined in:
lib/datahen/client/scraper_job_page.rb

Constant Summary

Constants inherited from Base

Base::CHECK_EMPTY_BODY, Base::CHECK_NIL, Base::DEFAULT_RETRY_LIMIT

Instance Method Summary collapse

Methods inherited from Base

#auth_token, #auth_token=, #default_retry_limit, #env_api_url, env_auth_token, env_ignore_ssl, #ignore_ssl, #initialize, #left_merge, random_delay, #retry

Constructor Details

This class inherits a constructor from Datahen::Client::Base

Instance Method Details

#all(scraper_name, opts = {}) ⇒ Object 



8
9
10
11
 
# File 'lib/datahen/client/scraper_job_page.rb', line 8

def all(scraper_name, opts={})
  params = @options.merge(opts)
  self.class.get("/scrapers/#{scraper_name}/current_job/pages", params)
end

#enqueue(scraper_name, page, opts = {}) ⇒ Object 



52
53
54
55
56
 
# File 'lib/datahen/client/scraper_job_page.rb', line 52

def enqueue(scraper_name, page, opts={})
params = @options.merge(opts).merge({body: page.to_json})

  self.class.post("/scrapers/#{scraper_name}/current_job/pages", params)
end

#find(scraper_name, gid) ⇒ Object 



4
5
6
 
# File 'lib/datahen/client/scraper_job_page.rb', line 4

def find(scraper_name, gid)
  self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}", @options)
end

#find_content(scraper_name, gid) ⇒ Object 



65
66
67
 
# File 'lib/datahen/client/scraper_job_page.rb', line 65

def find_content(scraper_name, gid)
  self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/content", @options)
end

#find_failed_content(scraper_name, gid) ⇒ Object 



69
70
71
 
# File 'lib/datahen/client/scraper_job_page.rb', line 69

def find_failed_content(scraper_name, gid)
  self.class.get("/scrapers/#{scraper_name}/current_job/pages/#{gid}/failed_content", @options)
end

#get_gid(scraper_name, page, opts = {}) ⇒ Object 



58
59
60
61
62
63
 
# File 'lib/datahen/client/scraper_job_page.rb', line 58

def get_gid(scraper_name, page, opts={})

  params = @options.merge(opts).merge({body: page.to_json})

  self.class.post("/scrapers/#{scraper_name}/current_job/generate_gid", params)
end

#limbo(scraper_name, opts = {}) ⇒ Object 



47
48
49
50
 
# File 'lib/datahen/client/scraper_job_page.rb', line 47

def limbo(scraper_name, opts={})
  params = @options.merge(opts)
  self.class.put("/scrapers/#{scraper_name}/current_job/pages/limbo", params)
end

#refetch(scraper_name, opts = {}) ⇒ Object 



29
30
31
32
 
# File 'lib/datahen/client/scraper_job_page.rb', line 29

def refetch(scraper_name, opts={})
  params = @options.merge(opts)
  self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
end

#refetch_by_job(job_id, opts = {}) ⇒ Object

Note:

This method will be removed at some point in the future.

Deprecated, please use Datahen::Client::JobPage#refetch instead.



37
38
39
40
 
# File 'lib/datahen/client/scraper_job_page.rb', line 37

def refetch_by_job(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/refetch", params)
end

#reparse(scraper_name, opts = {}) ⇒ Object 



42
43
44
45
 
# File 'lib/datahen/client/scraper_job_page.rb', line 42

def reparse(scraper_name, opts={})
  params = @options.merge(opts)
  self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
end

#update(scraper_name, gid, opts = {}) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/datahen/client/scraper_job_page.rb', line 13

def update(scraper_name, gid, opts={})
  body = {}
  body[:page_type] = opts[:page_type] if opts[:page_type]
  body[:priority] = opts[:priority] if opts[:priority]
  body[:vars] = opts[:vars] if opts[:vars]
  body[:max_size] = opts[:max_size] if opts[:max_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
  body[:use_browserforge] = opts[:use_browserforge] if opts.has_key?("use_browserforge") || opts.has_key?(:use_browserforge)
  body[:browserforge_config] = opts[:browserforge_config] if opts.has_key?("browserforge_config") || opts.has_key?(:browserforge_config)

  params = @options.merge({body: body.to_json})

  self.class.put("/scrapers/#{scraper_name}/current_job/pages/#{gid}", params)
end