Class: Datahen::Client::JobPage

Inherits:
Base
  • Object
show all
Defined in:
lib/datahen/client/job_page.rb

Instance Method Summary collapse

Methods inherited from Base

#auth_token, #auth_token=, #env_api_url, env_auth_token, env_ignore_ssl, #ignore_ssl, #initialize

Constructor Details

This class inherits a constructor from Datahen::Client::Base

Instance Method Details

#all(job_id, opts = {}) ⇒ Object 



8
9
10
11
 
# File 'lib/datahen/client/job_page.rb', line 8

def all(job_id, opts={})
  params = @options.merge(opts)
  self.class.get("/jobs/#{job_id}/pages", params)
end

#dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {}) ⇒ Object 



51
52
53
54
55
56
57
58
59
 
# File 'lib/datahen/client/job_page.rb', line 51

def dequeue(job_id, limit, page_types, parse_fetching_failed, opts = {})
  body = {
    limit: limit,
    page_types: page_types,
    parse_fetching_failed: parse_fetching_failed
  }
  params = @options.merge(opts).merge({body: body.to_json})
  self.class.put("/jobs/#{job_id}/pages/parse_dequeue", params)
end

#enqueue(job_id, method, url, opts = {}) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/datahen/client/job_page.rb', line 27

def enqueue(job_id, method, url, opts={})
  body = {}
  body[:method] =  method != "" ? method : "GET"
  body[:url] =  url
  body[:page_type] = opts[:page_type] if opts[:page_type]
  body[:priority] = opts[:priority] if opts[:priority]
  body[:fetch_type] = opts[:fetch_type] if opts[:fetch_type]
  body[:body] = opts[:body] if opts[:body]
  body[:headers] = opts[:headers] if opts[:headers]
  body[:vars] = opts[:vars] if opts[:vars]
  body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
  body[:freshness] = opts[:freshness] if opts[:freshness]
  body[:ua_type] = opts[:ua_type] if opts[:ua_type]
  body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
  body[:cookie] = opts[:cookie] if opts[:cookie]
  body[:max_size] = opts[:max_size] if opts[:max_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]

  params = @options.merge({body: body.to_json})

  self.class.post("/jobs/#{job_id}/pages", params)
end

#find(job_id, gid) ⇒ Object 



4
5
6
 
# File 'lib/datahen/client/job_page.rb', line 4

def find(job_id, gid)
  self.class.get("/jobs/#{job_id}/pages/#{gid}", @options)
end

#find_content(job_id, gid) ⇒ Object 



74
75
76
 
# File 'lib/datahen/client/job_page.rb', line 74

def find_content(job_id, gid)
  self.class.get("/jobs/#{job_id}/pages/#{gid}/content", @options)
end

#find_failed_content(job_id, gid) ⇒ Object 



78
79
80
 
# File 'lib/datahen/client/job_page.rb', line 78

def find_failed_content(job_id, gid)
  self.class.get("/jobs/#{job_id}/pages/#{gid}/failed_content", @options)
end

#limbo(job_id, opts = {}) ⇒ Object 



92
93
94
95
 
# File 'lib/datahen/client/job_page.rb', line 92

def limbo(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/limbo", params)
end

#parsing_update(job_id, gid, opts = {}) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/datahen/client/job_page.rb', line 61

def parsing_update(job_id, gid, opts={})
  body = {}
  body[:outputs] = opts.fetch(:outputs) {[]}
  body[:pages] = opts.fetch(:pages) {[]}
  body[:parsing_status] = opts.fetch(:parsing_status){ nil }
  body[:log_error] = opts[:log_error] if opts[:log_error]
  body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)

  params = @options.merge({body: body.to_json})

  self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
end

#refetch(job_id, opts = {}) ⇒ Object 



87
88
89
90
 
# File 'lib/datahen/client/job_page.rb', line 87

def refetch(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/refetch", params)
end

#reparse(job_id, opts = {}) ⇒ Object 



82
83
84
85
 
# File 'lib/datahen/client/job_page.rb', line 82

def reparse(job_id, opts={})
  params = @options.merge(opts)
  self.class.put("/jobs/#{job_id}/pages/reparse", params)
end

#update(job_id, gid, opts = {}) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/datahen/client/job_page.rb', line 13

def update(job_id, gid, opts={})
  body = {}
  body[:page_type] = opts[:page_type] if opts[:page_type]
  body[:priority] = opts[:priority] if opts[:priority]
  body[:vars] = opts[:vars] if opts[:vars]
  body[:max_size] = opts[:max_size] if opts[:max_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]

  params = @options.merge({body: body.to_json})

  self.class.put("/jobs/#{job_id}/pages/#{gid}", params)
end