Class: Zillabyte::Command::Components

Inherits:
Flows
  • Object
show all
Defined in:
lib/zillabyte/cli/components.rb

Overview

manage custom components

Constant Summary

Constants inherited from Flows

Flows::MAX_POLL_SECONDS, Flows::POLL_SLEEP

Constants inherited from Base

Base::META_COLUMNS

Instance Attribute Summary

Attributes inherited from Base

#args, #options

Instance Method Summary collapse

Methods inherited from Flows

#color_for, get_info

Methods inherited from Base

alias_command, #api, extract_banner, extract_description, extract_help, extract_help_from_caller, extract_options, extract_summary, inherited, #initialize, method_added, namespace

Methods included from Helpers

#add_git_remote, #app, #ask, #create_git_remote, #display, #error, #extract_app_from_git_config, #extract_app_in_dir, #format_with_bang, #get_flow_name, #git, #handle_downloading_manifest, #has_git?, #longest, #read_multiline, #with_tty

Constructor Details

This class inherits a constructor from Zillabyte::Command::Base

Instance Method Details

#delete ⇒ Object

components:delete ID

Deletes a component. If the rpc is currently running, this command will kill it.

-f, --force # Don't ask for confirmation --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



132
133
134
 
# File 'lib/zillabyte/cli/components.rb', line 132

def delete
  super
end

#errors ⇒ Object

components:errors ID

Show recent errors generated by the componeny rpc.

--output_type OUTPUT_TYPE # Specify an output type i.e. json HIDDEN



190
191
192
 
# File 'lib/zillabyte/cli/components.rb', line 190

def errors
  super
end

#index ⇒ Object

components

List custom components.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



16
17
18
 
# File 'lib/zillabyte/cli/components.rb', line 16

def index
  self.list
end

#info ⇒ Object

components:info [DIR]

Outputs the info for the component in the dir.

--pretty # Pretty prints the info output --directory DIR # Component directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



155
156
157
 
# File 'lib/zillabyte/cli/components.rb', line 155

def info
  super
end

#init ⇒ Object

components:init [NAME]

Initializes a new component.

--lang LANG # Specify which language to use (ruby, python) [default: 'ruby'] --dir DIR # Target directory of the app --remote REMOTE # The git remote name [default: 'zillabyte']

Examples:

$ zillabyte components:init domain_extractor --lang ruby



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/zillabyte/cli/components.rb', line 33

def init
  
  name = options[:name] || shift_argument
  error("name required") if name.nil?
    
  lang = options[:lang] || options[:language] || "ruby"
  dir = options[:dir] || options[:directory] || Dir.pwd
  remote = options[:remote] || "zillabyte"
  
  dir = File.expand_path(dir) unless dir.nil?
  type = options[:output_type]

  languages = ["ruby","python", "js"]

  error("Unsupported language #{lang}. Zillabyte currently supports #{languages.join(', ')}.", type) if not languages.include? lang

  display "initializing empty #{lang} component in #{dir}" if type.nil?
  erb_binding = binding
  FileUtils.mkdir_p dir
  
  Dir[File.join(File.expand_path("../templates/components/#{lang}", __FILE__), "*")].each do |source_file|
    
    next if File.directory?(source_file)
    erb = ERB.new(File.read(source_file))
    erb.filename = source_file
    
    dest_file = File.join(dir, File.basename(source_file).gsub(/\.erb$/, ""))
    File.open(dest_file, 'w') {|file| file.write(erb.result(erb_binding))}
    
  end
  
  # Add the git remote... 
  add_git_remote(name, remote)
  
end

#kill ⇒ Object

components:kill ID

Kills the given component rpc.

--config CONFIG_FILE # Use the given config file --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



143
144
145
 
# File 'lib/zillabyte/cli/components.rb', line 143

def kill
  super
end

#list ⇒ Object

components

List custom components.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/zillabyte/cli/components.rb', line 76

def list
  type = options[:output_type]
  
  headings = ["id", "name", "state", "inputs", "outputs", "scope"]
  rows = api.component.list.map do |row|
    if headings.size == 0
      headings = row.keys 
      headings.delete("rel_dir")
    end
    
    v = row.values_at *headings
    v
  end

  display "components:\n" if type.nil?
  display TableOutputBuilder.build_table(headings, rows, type)
  display "Total number of components: " + rows.length.to_s if type.nil?
end

#live_run ⇒ Object

components:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]

Runs a local component with live data.

--config CONFIG_FILE # Use the given config file --directory DIR # Component directory --output_type OUTPUT_TYPE # Specify an output type i.e. json HIDDEN

HIDDEN:



215
216
217
 
# File 'lib/zillabyte/cli/components.rb', line 215

def live_run
  super
end

#logs ⇒ Object

components:logs ID [OPERATION_NAME]

Streams logs for the component from our cluster.

--operation OPERATION_NAME # Specify the operation to show logs for -v, --verbose LEVEL # Sets the verbosity (error, info, debug) [default: info] --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



202
203
204
 
# File 'lib/zillabyte/cli/components.rb', line 202

def logs
  super
end

#prep ⇒ Object

components:prep [DIR]

Performs any necessary initialization for the component.

--directory DIR # Component directory --output_type OUTPUT_TYPE # Specify an output type i.e. json HIDDEN



180
181
182
 
# File 'lib/zillabyte/cli/components.rb', line 180

def prep
  super
end

#pull ⇒ Object

components:pull ID DIR

Pulls a component source to a directory.

--force # Pulls even if the directory exists --directory DIR # Component directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN

Examples:

$ zillabyte components:pull .



108
109
110
 
# File 'lib/zillabyte/cli/components.rb', line 108

def pull
  super
end

#push ⇒ Object

components:push [DIR]

Uploads a component.

--config CONFIG_FILE # Use the given config file --directory DIR # Component directory

Examples:

$ zillabyte components:push .

HIDDEN: This is only meant for local testing. You must be an admin.



232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
 
# File 'lib/zillabyte/cli/components.rb', line 232

def push
  
  since = Time.now.utc.to_s
  dir = options[:directory] || Dir.pwd
  dir = File.expand_path(dir)
  
  hash = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, session, options)
  name = hash['name']
  
  options[:directory] = dir
  options[:component_id] = dir
  options[:schema] = hash
  options[:meta] = hash
  options[:name] = name
  type = options[:output_type]

  # Push the component... 
  session.display("packaging directory... ") if session && type.nil?
  tar = Zillabyte::Common::Tar.tar(dir)
  
  # Create the component
  res = self.api.request(
    :expects  => 200,
    :method   => :post,
    :path     => "/components/",
    :body     => options.to_json
  ).body

  error("error: #{res['error_message']}", type) if res['error']
  comp_id = res['id']
  
  # Push the tar... 
  if(res['uri'])
    uri = URI(res['uri'])
    try_again = 1
    while(try_again)
      Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
        request = Net::HTTP::Put.new(uri.request_uri)
        request.body_stream = tar
        request['Content-Length'] = request.body_stream.size
        request['Content-Type'] = ''
        response = http.request(request)
        if response.code.to_i >= 300
          try_again = 1
          break
        end
        try_again = nil
      end
    end
    res.delete('uri')
    session.display( "pushed")
  else
    error "unable to push"
  end
  
  sleep 2
  hash = self.api.logs.get(comp_id, nil, options)
  fetch_logs(hash, "_ALL_", "Component registered")

  display "{}" if type == "json"

end

#results ⇒ Object

components:results [ID] [RUN_ID_1] [RUN_ID_2] ...

Gets the result of rpc requests corresponding to RUN_ID if it is done running, otherwise gets current run status. A list of RUN_IDs may also be specified in an input csv file like that generated by components:rpc with the --output_file flag.

--input_file INTPUT_FILE # Input csv file containing query parameters and run ids (this can be generated automatically by components:rpc using the --output_file flag)



416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
 
# File 'lib/zillabyte/cli/components.rb', line 416

def results
  component_id = options[:id] || shift_argument

  if component_id.nil?
    component_id = read_name_from_conf(options)
    options[:is_name] = true
  elsif !(component_id =~ /^\d*$/)
    options[:is_name] = true
  end

  type = options[:output_type]
  file = options[:input_file]

  run_ids = []
  if file
    CSV.foreach(file) do |line|
      run_ids << line[1]
    end
  else
    while(true) do
      next_arg = shift_argument
      break if next_arg.nil?
      run_ids << next_arg
    end
  end

  res = api.components.get_rpc_results(component_id, {:execute_ids => run_ids})
  if res['error']
    error("error: #{res['error_message']}", type)
  else
    if type.nil?
      res["results"].each do |id, hash|
        display "#{id}: #{hash["status"]}"
        if hash["status"] == "complete"
          if hash["data"].size == 0
            display "\t NO OUTPUT"
          else
            display "\t #{hash["data"]}"
          end
        end
      end
    end
  end
end

#rpc ⇒ Object

components:rpc [ID] [INPUT_1] [INPUT_2] ...

Submits a single query to the rpc. The input parameters MUST be listed in the same order as that given in the component inputs. TO SUBMIT MULTIPLE queries, use the --file switch to specify a csv file containing the queries without listing any inputs on the command line. Each line of the file should correspond to a unique query.

--async # Run the command asynchronously --input_file INTPUT_FILE # Input csv file containing parameters for multiple queries --output_file OUTPUT_FILE # Output csv file containing query parameters and run ids

Examples:

Single query: $ zillabyte components:rpc 'web_screenshot' 'google.com'

Multiple queries: $ zillabyte components:rpc 'web_screenshot' --file 'url_list.csv'



317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
 
# File 'lib/zillabyte/cli/components.rb', line 317

def rpc
  
  component_id = options[:id] || shift_argument
  async = options[:async] || false

  if component_id.nil?
    component_id = read_name_from_conf(options)
    options[:is_name] = true
  elsif !(component_id =~ /^\d*$/)
    options[:is_name] = true
  end

  type = options[:output_type]
  file = options[:input_file]
  out_file = options[:output_file]

  component_args = []
  if file
    CSV.foreach(file) {|line| component_args << line}
  else
    args = []
    while(true) do
      next_arg = shift_argument
      break if next_arg.nil?
      args << next_arg
    end
    component_args << args if !args.empty?
  end

  opts = {:rpc_inputs => component_args} if component_args.size != 0
  res = api.components.rpc(component_id, opts)
  
  input_par_for_id = {}
  if res['error']
    error("error: #{res['error_message']}", type)
  else
    if type.nil?
      display "Request submitted to component ##{res['id']}."
    end

    if res["execute_ids"]
      input_par_for_id = res["execute_ids"].invert
      if out_file
        CSV.open(out_file, "w") {|csv| res["execute_ids"].to_a.each {|elem| csv << elem}}
      end
    end
  end

  if async
    # Async -- return right away.. 
    display "The run ids are (query: run_id):"
    res["execute_ids"].each do |pars, id|
      display "\t #{pars}: #{id}"
    end
    display "Please use \"zillabyte components:results [run id]\" to check on the status of your query and to retrieve your results.\nYou may also wish to check \"zillabyte logs\" for errors."
  else
    # Sync -- poll until it's done...
    if res["execute_ids"] 
      run_ids = res["execute_ids"].values
      while true
        res = api.components.get_rpc_results(component_id, {:execute_ids => run_ids})
        if res['error']
          error("error: #{res['error_message']}", type)
        else
          if res["results"].detect{|id, res| ["running", "waiting"].member?(res["status"].downcase)}
            # poll again
            sleep(2)
          else
            if type.nil?
              # All results are done (or rather, not 'running). Display them
              res["results"].each do |id, hash|
                display "Query #{input_par_for_id[id]}:"
                if hash["data"].size == 0
                  display "\t NO OUTPUT"
                else
                  display "\t #{hash["data"].to_json()}"
                end
              end
            end
            break
          end
        end
      end
    end
  end
end

#status ⇒ Object

components:status [DIR]

Fetches detailed status of the component.

--directory DIR # Component directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



119
120
121
 
# File 'lib/zillabyte/cli/components.rb', line 119

def status
  super
end

#test ⇒ Object

components:test

Tests a local component with sample data.

--config CONFIG_FILE # Use the given config file --input INPUT_FILE # Uses a CSV for component input --output OUTPUT_FILE # Write output to a CSV --directory DIR # Component directory



169
170
171
 
# File 'lib/zillabyte/cli/components.rb', line 169

def test
  super
end