Class: Datahen::Client::JobPage

Inherits:
Base
  • Object
show all
Defined in:
lib/datahen/client/job_page.rb

Constant Summary

Constants inherited from Base

Base::CHECK_EMPTY_BODY, Base::CHECK_NIL, Base::DEFAULT_RETRY_LIMIT

Instance Method Summary collapse

Methods inherited from Base

#auth_token, #auth_token=, #default_retry_limit, #env_api_url, env_auth_token, env_ignore_ssl, #ignore_ssl, #initialize, #left_merge, random_delay, #retry

Constructor Details

This class inherits a constructor from Datahen::Client::Base

Instance Method Details

#all(job_id, opts = {}) ⇒ Object 



8
9
10
11
 
# File 'lib/datahen/client/job_page.rb', line 8

def all(job_id, opts={})
  params = @options.merge(opts)
  self.class.get("/jobs/#{job_id}/pages", params)
end

#dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {}) ⇒ Object 



46
47
48
49
50
51
52
53
54
 
# File 'lib/datahen/client/job_page.rb', line 46

def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
  body = {
    limit: limit,
    page_types: page_types,
    parse_fetching_failed: parse_fetching_failed
  }
  params = @options.merge(opts).merge({body: body.to_json})
  self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
end

#enqueue(job_id, page, opts = {}) ⇒ Object 



32
33
34
35
36
37
 
# File 'lib/datahen/client/job_page.rb', line 32

def enqueue(job_id, page, opts={})
  params = @options.merge(opts).merge({body: page.to_json})

  self.class.post("/jobs/#{job_id}/pages", params)
  
end

#find(job_id, gid) ⇒ Object 



4
5
6
 
# File 'lib/datahen/client/job_page.rb', line 4

def find(job_id, gid)
  self.class.get("/jobs/#{job_id}/pages/#{gid}", @options)
end

#find_content(job_id, gid) ⇒ Object 



77
78
79
 
# File 'lib/datahen/client/job_page.rb', line 77

def find_content(job_id, gid)
  self.class.get("/jobs/#{job_id}/pages/#{gid}/content", @options)
end

#find_failed_content(job_id, gid) ⇒ Object 



81
82
83
 
# File 'lib/datahen/client/job_page.rb', line 81

def find_failed_content(job_id, gid)
  self.class.get("/jobs/#{job_id}/pages/#{gid}/failed_content", @options)
end

#get_gid(job_id, page, opts = {}) ⇒ Object 



39
40
41
42
43
44
 
# File 'lib/datahen/client/job_page.rb', line 39

def get_gid(job_id, page, opts={})

  params = @options.merge(opts).merge({body: page.to_json})

  self.class.post("/jobs/#{job_id}/generate_gid", params)
end

#limbo(job_id, opts = {}) ⇒ Object 



95
96
97
98
 
# File 'lib/datahen/client/job_page.rb', line 95

def limbo(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/limbo", params)
end

#parsing_update(job_id, gid, opts = {}) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/datahen/client/job_page.rb', line 56

def parsing_update(job_id, gid, opts={})
  body = {}
  body[:outputs] = opts.fetch(:outputs) {[]}
  body[:pages] = opts.fetch(:pages) {[]}
  body[:parsing_status] = opts.fetch(:parsing_status){ nil }
  body[:log_error] = opts[:log_error] if opts[:log_error]
  body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)
  body[:parsing_try_limit] = opts[:parsing_try_limit] if opts.fetch(:parsing_try_limit){ nil }

  params = @options.merge({body: body.to_json})

  limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
  self.retry(limit, 5, "Error while updating the parser.", false, CHECK_EMPTY_BODY) do
    response = self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
    if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
      raise Error::CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
    end
    response
  end
end

#refetch(job_id, opts = {}) ⇒ Object 



90
91
92
93
 
# File 'lib/datahen/client/job_page.rb', line 90

def refetch(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/refetch", params)
end

#reparse(job_id, opts = {}) ⇒ Object 



85
86
87
88
 
# File 'lib/datahen/client/job_page.rb', line 85

def reparse(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/reparse", params)
end

#still_alive(job_id, gid, opts = {}) ⇒ Object 



100
101
102
103
 
# File 'lib/datahen/client/job_page.rb', line 100

def still_alive(job_id, gid, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/#{gid}/still_alive", params)
end

#update(job_id, gid, opts = {}) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/datahen/client/job_page.rb', line 13

def update(job_id, gid, opts={})
  body = {}
  body[:page_type] = opts[:page_type] if opts[:page_type]
  body[:priority] = opts[:priority] if opts[:priority]
  body[:vars] = opts[:vars] if opts[:vars]
  body[:max_size] = opts[:max_size] if opts[:max_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
  body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
  body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
  body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
  body[:use_browserforge] = opts[:use_browserforge] if opts.has_key?("use_browserforge") || opts.has_key?(:use_browserforge)
  body[:browserforge_config] = opts[:browserforge_config] if opts.has_key?("browserforge_config") || opts.has_key?(:browserforge_config)

  params = @options.merge({body: body.to_json})

  self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
end