Class: Turbot::Command::Bots

Inherits:
Base
  • Object
show all
Defined in:
lib/turbot/command/bots.rb

Overview

Manage bots (generate template, validate data, submit code)

Constant Summary collapse

BOT_ID_RE = 
/\A[A-Za-z0-9_-]+\z/.freeze

Constants included from Helpers

Helpers::DEFAULT_HOST

Instance Method Summary collapse

Methods included from Helpers

#api, #ask, #ask_for_password, #ask_for_password_on_windows, #delete_netrc_entry, #email_address_and_api_key, #error, #format_error, #host, #netrc_exists?, #netrc_path, #open_netrc, #save_netrc_entry, #styled_error, #turbot_api, #turbot_api_parameters

Constructor Details

#initialize(*args) ⇒ Bots

Returns a new instance of Bots.



6
7
8
9
10
11
12
13
14
 
# File 'lib/turbot/command/bots.rb', line 6

def initialize(*args)
  super

  require 'turbot_runner'
  require 'turbot/handlers/base_handler'
  require 'turbot/handlers/dump_handler'
  require 'turbot/handlers/preview_handler'
  require 'turbot/handlers/validation_handler'
end

Instance Method Details

#dump ⇒ Object

bots:dump

Execute the bot locally and write the bot's output to STDOUT.

-q, --quiet # only output validation errors

Example:

$ turbot bots:dump 'bar' 'bar2'



284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/turbot/command/bots.rb', line 284

def dump
  validate_arguments!
  error_if_no_local_bot_found

  if options[:quiet]
    handler = Turbot::Handlers::BaseHandler.new
  else
    handler = Turbot::Handlers::DumpHandler.new
  end
  runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
  rc = runner.run

  if rc == TurbotRunner::Runner::RC_OK
    puts 'Bot ran successfully!'
  else
    puts 'Bot failed!'
  end
end

#generate ⇒ Object

bots:generate --bot BOT

Generate a bot template in the specified language.

-b, --bot BOT # a bot ID -l, --language LANGUAGE # ruby (default) or python

Example:

$ turbot bots:generate --bot my_amazing_bot --language ruby Created new bot template for my_amazing_bot!



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/turbot/command/bots.rb', line 85

def generate
  validate_arguments!
  error_if_bot_exists_in_turbot

  # Check the bot name.
  unless bot[BOT_ID_RE]
    error "The bot name #{bot} is invalid. Bot names must contain only lowercase letters (a-z), numbers (0-9), underscore (_) or hyphen (-)."
  end

  # Check collision with existing directory.
  bot_directory = File.join(working_directory, bot)
  if File.exists?(bot_directory)
    error "There's already a directory named #{bot}. Move it, delete it, change directory, or try another name."
  end

  language = (options[:language] || 'ruby').downcase
  scraper_template = File.expand_path("../../../../data/templates/#{language}", __FILE__)

  # Check language.
  unless File.exists?(scraper_template)
    error "The language #{language} is unsupported."
  end

  scraper_name = case language
  when 'ruby'
    'scraper.rb'
  when 'python'
    'scraper.py'
  end

  # Create the scraper.
  FileUtils.mkdir(bot_directory)
  FileUtils.cp(File.join(scraper_template, scraper_name), File.join(bot_directory, scraper_name))

  # Create the license.
  license_template = File.expand_path('../../../../data/templates/LICENSE.txt', __FILE__)
  FileUtils.cp(license_template, File.join(bot_directory, 'LICENSE.txt'))

  # Create the manifest.
  manifest_template = File.expand_path('../../../../data/templates/manifest.json', __FILE__)
  manifest = File.read(manifest_template).
    sub('{{bot_id}}', bot).
    sub('{{scraper_name}}', scraper_name).
    sub('{{language}}', language)
  File.open(File.join(bot_directory, 'manifest.json'), 'w') do |f|
    f.write(JSON.pretty_generate(JSON.load(manifest)))
  end

  puts "Created new bot template for #{bot}!"
end

#index ⇒ Object

bots

List your bots.

Example:

$ turbot bots example1 example2



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/turbot/command/bots.rb', line 26

def index
  validate_arguments!

  response = api.list_bots
  if response.is_a?(Turbot::API::SuccessResponse)
    if response.data.empty?
      puts 'You have no bots.'
    else
      response.data.each do |bot|
        puts bot[:bot_id]
      end
    end
  else
    error_message(response)
  end
end

#info ⇒ Object

bots:info [--bot BOT]

Show a bot's details.

-b, --bot BOT # a bot ID

Example:

$ turbot bots:info --bot example bot_id: example created_at: 2010-01-01T00:00:00.000Z updated_at: 2010-01-02T00:00:00.000Z state: scheduled



58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/turbot/command/bots.rb', line 58

def info
  validate_arguments!
  error_if_no_local_bot_found

  response = api.show_bot(bot)
  if response.is_a?(Turbot::API::SuccessResponse)
    response.data.each do |key,value|
      puts "#{key}: #{value}"
    end
  else
    error_message(response)
  end
end

#preview ⇒ Object

bots:preview

Send bot data to Turbot for remote previewing or sharing.

Example:

$ turbot bots:preview Sending to Turbot... Submitted 2 records to Turbot. View your records at http://turbot.opencorporates.com/..



314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
 
# File 'lib/turbot/command/bots.rb', line 314

def preview
  validate_arguments!
  error_if_no_local_bot_found

  response = api.update_bot(bot, parse_manifest)
  if response.is_a?(Turbot::API::FailureResponse)
    error_message(response)
  end

  response = api.destroy_draft_data(bot)
  if response.is_a?(Turbot::API::FailureResponse)
    error_message(response)
  end

  puts 'Sending to Turbot...'

  handler = Turbot::Handlers::PreviewHandler.new(bot, api)
  runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
  rc = runner.run

  if rc == TurbotRunner::Runner::RC_OK
    response = handler.submit_batch
    if response.is_a?(Turbot::API::SuccessResponse)
      if handler.count > 0
        puts "Submitted #{handler.count} records to Turbot.\nView your records at #{response.data[:url]}"
      else
        puts 'No records sent.'
      end
    else
      error_message(response)
    end
  else
    puts 'Bot failed!'
  end
end

#push ⇒ Object

bots:push

Push the bot's code to Turbot. Must be run from a bot directory containing a manifest.json file.

-y, --yes # skip confirmation

Example:

$ turbot bots:push This will submit your bot and its data for review. Are you happy your bot produces valid data (e.g. with turbot bots:validate)? [Y/n] Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/turbot/command/bots.rb', line 171

def push
  validate_arguments!
  error_if_no_local_bot_found

  unless options[:yes]
    puts 'This will submit your bot and its data for review.'
    puts 'Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]'
    answer = ask
    unless ['', 'y'].include?(answer.downcase.strip)
      error 'Aborted.'
    end
  end

  # TODO Validate the manifest.json file.

  manifest = parse_manifest
  tempfile = Tempfile.new(bot)
  tempfile.close # Windows will raise Errno::EACCES on Zip::File.open below
  archive_path = "#{tempfile.path}.zip"
  create_zip_archive(archive_path, manifest['files'] + ['manifest.json'])

  response = File.open(archive_path) do |f|
    api.update_code(bot, f)
  end
  if response.is_a?(Turbot::API::SuccessResponse)
    puts 'Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!'
  else
    error_message(response)
  end
end

#register ⇒ Object

bots:register

Register a bot with Turbot. Must be run from a bot directory containing a manifest.json file.

Example:

$ turbot bots:register Registered my_amazing_bot!



145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/turbot/command/bots.rb', line 145

def register
  validate_arguments!
  error_if_no_local_bot_found
  error_if_bot_exists_in_turbot

  response = api.create_bot(bot, parse_manifest)
  if response.is_a?(Turbot::API::SuccessResponse)
    puts "Registered #{bot}!"
  else
    error_message(response)
  end
end

#validate ⇒ Object

bots:validate

Validate the manifest.json file and validate the bot's output against its schema.

Example:

$ turbot bots:validate Validated 2 records!



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/turbot/command/bots.rb', line 213

def validate
  validate_arguments!
  error_if_no_local_bot_found

  manifest = parse_manifest

  { 'allow_duplicates' => 'duplicates_allowed',
    'author' => 'publisher',
    'incremental' => 'manually_end_run',
    'public_repository' => 'public_repo_url',
  }.each do |deprecated,field|
    if manifest[deprecated]
      puts %(WARNING: "#{deprecated}" is deprecated. Use "#{field}" instead.)
    end
  end

  schema = JSON.load(File.read(File.expand_path('../../../../data/schema.json', __FILE__)))
  validator = JSON::Validator.new(schema, manifest, {
    clear_cache: false,
    parse_data: false,
    record_errors: true,
    errors_as_objects: true,
  })

  errors = validator.validate
  if errors.any?
    messages = ['`manifest.json` is invalid. Please correct the errors:']
    errors.each do |error|
      messages << "* #{error.fetch(:message).sub(/ in schema \S+\z/, '')}"
    end
    error messages.join("\n")
  end

  if manifest['transformers']
    difference = manifest['transformers'].map { |transformer| transformer['file'] } - manifest['files']
    if difference.any?
      messages = ['`manifest.json` is invalid. Please correct the errors:']
      messages << "* Some transformer files are not listed in the top-level files: #{difference.join(', ')}"
      error messages.join("\n")
    end
  end

  handler = Turbot::Handlers::ValidationHandler.new
  runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
  begin
    rc = runner.run
  rescue TurbotRunner::InvalidDataType
    messages = ['`manifest.json` is invalid. Please correct the errors:']
    messages << %(* The property '#/data_type' value "#{manifest['data_type']}" is not a supported data type.)
    error messages.join("\n")
  end

  if rc == TurbotRunner::Runner::RC_OK
    puts "Validated #{handler.count} records!"
  else
    puts "Validated #{handler.count} records before bot failed!"
  end
end