Class: Datahen::CLI::Scraper

Inherits:
Thor
  • Object
show all
Defined in:
lib/datahen/cli/scraper.rb

Instance Method Summary collapse

Instance Method Details

#create(scraper_name, git_repository) ⇒ Object 



45
46
47
48
49
 
# File 'lib/datahen/cli/scraper.rb', line 45

def create(scraper_name, git_repository)
  # puts "options #{options}"
  client = Client::Scraper.new(options)
  puts "#{client.create(scraper_name, git_repository, options)}"
end

#delete(scraper_name) ⇒ Object 



92
93
94
95
 
# File 'lib/datahen/cli/scraper.rb', line 92

def delete(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.delete(scraper_name)}"
end

#deploy(scraper_name) ⇒ Object 



102
103
104
105
106
 
# File 'lib/datahen/cli/scraper.rb', line 102

def deploy(scraper_name)
  client = Client::ScraperDeployment.new()
  puts "Deploying scraper. This may take a while..."
  puts "#{client.deploy(scraper_name)}"
end

#history(scraper_name) ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/datahen/cli/scraper.rb', line 203

def history(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    json = JSON.parse(client.job_stats_history(options[:job], options).body)
    if json['error'] == ""
      puts "#{JSON.pretty_generate(json['data'])}"
    else 
      puts "#{JSON.pretty_generate(json['error'])}"
    end
  else
    json = JSON.parse(client.scraper_job_stats_history(scraper_name, options).body)
    if json['error'] == ""
      puts "#{JSON.pretty_generate(json['data'])}"
    else 
      puts "#{JSON.pretty_generate(json['error'])}"
    end
  end
end

#list ⇒ Object 



12
13
14
15
 
# File 'lib/datahen/cli/scraper.rb', line 12

def list
  client = Client::Scraper.new(options)
  puts "#{client.all}"
end

#log(scraper_name) ⇒ Object 



142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/datahen/cli/scraper.rb', line 142

def log(scraper_name)
  client = Client::JobLog.new(options)

  query = {}
  query["order"] = options.delete(:head) if options[:head]
  query["job_type"] = "parsing" if options[:parsing]
  query["job_type"] = "seeding" if options[:seeding]
  query["job_type"] = "finisher executing" if options[:finisher]
  query["page_token"] = options.delete(:more) if options[:more]
  query["per_page"] = options.delete(:per_page) if options[:per_page]

  puts "Fetching logs..."

  if options[:job]
    result = client.all_job_log(options[:job], {query: query})
  else
    result = client.scraper_all_job_log(scraper_name, {query: query})
  end

  if result['entries'].nil? || result["entries"].length == 0
    puts "No logs yet, please try again later."
  else
    more_token = result["more_token"]

    result["entries"].each do |entry|
      puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
    end

    unless more_token.nil?
      puts "-----------"
      puts "To see more entries, add: \"--more #{more_token}\""
    end
  end
end

#profile(scraper_name) ⇒ Object 



226
227
228
229
 
# File 'lib/datahen/cli/scraper.rb', line 226

def profile(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.profile(scraper_name)}"
end

#show(scraper_name) ⇒ Object 



86
87
88
89
 
# File 'lib/datahen/cli/scraper.rb', line 86

def show(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.find(scraper_name)}"
end

#start(scraper_name) ⇒ Object 



124
125
126
127
128
 
# File 'lib/datahen/cli/scraper.rb', line 124

def start(scraper_name)
  client = Client::ScraperJob.new(options)
  puts "Starting a scrape job..."
  puts "#{client.create(scraper_name, options)}"
end

#stats(scraper_name) ⇒ Object 



183
184
185
186
187
188
189
190
 
# File 'lib/datahen/cli/scraper.rb', line 183

def stats(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    puts "#{client.job_current_stats(options[:job], options)}"
  else
    puts "#{client.scraper_job_current_stats(scraper_name, options)}"
  end
end

#update(scraper_name) ⇒ Object 



79
80
81
82
 
# File 'lib/datahen/cli/scraper.rb', line 79

def update(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.update(scraper_name, options)}"
end