Class: Zillabyte::Command::Apps

Inherits:
Flows show all
Defined in:
lib/zillabyte/cli/apps.rb

Overview

manage custom apps

Constant Summary collapse

MAX_POLL_SECONDS = 
60 * 15
POLL_SLEEP = 
2.0

Constants inherited from Base

Base::META_COLUMNS

Instance Attribute Summary

Attributes inherited from Base

#args, #options

Instance Method Summary collapse

Methods inherited from Flows

#color_for, get_info

Methods inherited from Base

alias_command, #api, extract_banner, extract_description, extract_help, extract_help_from_caller, extract_options, extract_summary, inherited, #initialize, method_added, namespace

Methods included from Helpers

#add_git_remote, #app, #ask, #create_git_remote, #display, #error, #extract_app_from_git_config, #extract_app_in_dir, #format_with_bang, #get_flow_name, #git, #handle_downloading_manifest, #has_git?, #longest, #read_multiline, #with_tty

Constructor Details

This class inherits a constructor from Zillabyte::Command::Base

Instance Method Details

#cycles ⇒ Object

apps:cycles ID [OPTIONS]

Operations on the app's cycles. With no options, the command lists the apps cycles.

-n, --next # Request the app to move to the next cycle -f, --forever # Automatically start new cycles when previous cycles ends --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
 
# File 'lib/zillabyte/cli/apps.rb', line 350

def cycles
  app_id = options[:id] || shift_argument
  type = options[:output_type]

  trigger_next = options[:next] || false
  trigger_forever = options[:forever] || false

  if app_id.nil?
    app_id = read_name_from_conf(options)
    options[:is_name] = true
  elsif !(app_id =~ /^\d*$/)
    options[:is_name] = true
  end

  if trigger_next
    # Trigger the next app
    response = api.apps.create_cycle(app_id, options)
  elsif trigger_forever
    response = api.apps.run_forever(app_id, options)
  else
    # List the apps
    response = api.apps.list_cycles(app_id, options)
    # TODO List the sequence number for this app.
    display "Most recent cyles of the app:" if type.nil?
    headings = ["Cycle_id", "State", "Start", "End"]
    rows = response["cycles"] 
    rows = rows.map do |row|
      start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
      end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
      [row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
    end

    display TableOutputBuilder.build_table(headings, rows, type)
    display "Total number of cycles executed: #{response['total']}" if type.nil?
    return
  end

  if response["job_id"]
    options[:job_id] = response["job_id"]
    app_id = response["app_id"]
    options.delete :is_name

    start = Time.now.utc
    display "Next cycle request sent. If your app was RETIRED this may take a few minutes. Please wait or check 'zillabyte logs' for progress." if type.nil?

    while(Time.now.utc < start + MAX_POLL_SECONDS) do

      # Poll
      res = self.api.apps.cycles_poll(app_id, options)

      case res['status']
      when 'completed'
        if res['return']
          if type == "json"
            return "{}"
          else
            display res['return']
          end
        else
          throw "something is wrong: #{res}"
        end
        # success! continue below
        break
      when 'running'
        sleep(POLL_SLEEP)
     #   display ".", false
      else
        throw "unknown status: #{res}"
      end

    end
  elsif response["error"]
    error(response["error"], type)
  else
    error("remote server error (a380)", type) 
  end
end

#delete ⇒ Object

apps:delete ID

Deletes an app. If the app is running, this command will kill it.

-f, --force # Don't ask for confirmation --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



252
253
254
 
# File 'lib/zillabyte/cli/apps.rb', line 252

def delete
  super
end

#details ⇒ Object

apps:details ID

List details for an app, including percentage complete, if the source is an existing dataset.



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/zillabyte/cli/apps.rb', line 62

def details
  
  app_id = options[:id] || shift_argument
  type = options[:output_type]
  
  if app_id.nil?
    app_id = read_name_from_conf(options)
    options[:is_name] = true
  elsif !(app_id =~ /^\d*$/)
    options[:is_name] = true
  end
      
  res = api.request(
    :expects => 200,
    :method => :get,
    :path => "/apps/#{app_id}/details",
    :body => options.to_json
  )
  res.body
  
  # a nested hash so can't map directly
  # instead i'll map separately and then merge the arrays
  heading1 = ["type", "name"]
  
  # a level deeper
  heading2 = ["consumed", "emitted", "errors"]
  
  # for the table output
  headings = heading1 + heading2 
  
  # get the values for the headings
  type_name = res.body.map do |instance|

    heading1.map do |heading|
      instance[heading]
    end  

  end
  
  # get the values for the headings
  stats = res.body.map do |instance|

    heading2.map do |heading|
      instance["stats"][heading]
    end  

  end
  
  # combine the arrays
  rows = type_name.zip(stats).map{|x,y| x.concat y}
  
  # check if it's from a dataset.
  # if the source is a dataset, there's a sharder. 
  # remove source types, and hardcode sharder-named instances as source
  
  from_dataset = false
      
  rows.each do | row |
    if row[1].scan("sharder").length != 0
      from_dataset = true
      break
    end
  end

  if from_dataset == true
    
    rows = rows.each do |row|
      if row[0].scan("source").length != 0 
        rows.delete(row)
      end
      
    end
    
  
    rows = rows.each do |row|
      if row[1].scan("sharder").length != 0
        row[0] = "source"
      end
    end
      
  end    
  
  # rows to be output to the CLI
  rows = rows.group_by{ |type, name, consumed, emitted, errors| name }.map{ |hashkey, array| [array[0][0], hashkey, array.inject(0){ |sum, i | sum + i[2].to_i }, array.inject(0){ |sum, i | sum + i[3].to_i },array.inject(0){ |sum, i | sum + i[4].to_i }]    }
  # example sums: [["sink", "has_facebook_likebox_and_vwo", 73, 0, 0], ["source", "sharder.1", 16, 10367872, 1], ["each", "each_3", 10367872, 73, 0]]
  
  # calculate percentage for the user to see
  
  attempts = 0
  total = 0

  if from_dataset == true

    rows.each do |instance|
      if instance[0] == "source"
        total = instance[3]
      end
    end


    rows.each do |instance|
      if instance[0] == "each"
        attempts = instance[2] 
      end
    end
  end
  
  if attempts > 0 && total > 0 
    completion = attempts/total*100
  else 
    completion = 0
  end

  # output on the CLI
  display "details:"
  display TableOutputBuilder.build_table(headings, rows, type)
  if from_dataset == true
    display "percent complete: #{completion}%\n"
  end    
end

#errors ⇒ Object

apps:errors ID

Show recent errors generated by the app.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



324
325
326
 
# File 'lib/zillabyte/cli/apps.rb', line 324

def errors
  super
end

#index ⇒ Object

apps

List custom apps.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



26
27
28
 
# File 'lib/zillabyte/cli/apps.rb', line 26

def index
  self.list
end

#info ⇒ Object

apps:info [DIR]

Outputs the info for the app in the dir.

--pretty # Pretty prints the info output --directory DIR # App directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



491
492
493
 
# File 'lib/zillabyte/cli/apps.rb', line 491

def info
  super
end

#init ⇒ Object

apps:init [NAME]

Initializes a new app.

--lang LANG # Specify which language to use (ruby, python) [default: 'ruby'] --dir DIR # Target directory of the app

Examples:

$ zillabyte apps:init contact_extractor --lang python



281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
 
# File 'lib/zillabyte/cli/apps.rb', line 281

def init
  
  name = options[:name] || shift_argument
  error("name required") if name.nil? 
  
  lang = options[:lang] || options[:language] || "ruby"
  dir = options[:dir] || options[:directory] || Dir.pwd
  remote = options[:remote] || "zillabyte"
  
  dir = File.expand_path(dir) unless dir.nil?
  type = options[:output_type]
  
  languages = ["ruby","python", "js"]

  error("Unsupported language #{lang}. Zillabyte currently supports #{languages.join(', ')}.", type) if not languages.include? lang

  display "initializing empty #{lang} app in #{dir}" if type.nil?
  erb_binding = binding
  FileUtils.mkdir_p dir
  
  Dir[File.join(File.expand_path("../templates/apps/#{lang}", __FILE__), "*")].each do |source_file|
    
    next if File.directory?(source_file)
    erb = ERB.new(File.read(source_file))
    erb.filename = source_file
    
    dest_file = File.join(dir, File.basename(source_file).gsub(/\.erb$/, ""))
    File.open(dest_file, 'w') {|file| file.write(erb.result(erb_binding))}
    
  end
  
  # Add the git remote... 
  add_git_remote(name, remote)
  
end

#kill ⇒ Object

apps:kill ID

Kills the given app.

--config CONFIG_FILE # Use the given config file --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



451
452
453
 
# File 'lib/zillabyte/cli/apps.rb', line 451

def kill
  super
end

#list ⇒ Object

apps

List custom apps.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/zillabyte/cli/apps.rb', line 38

def list
  type = options[:output_type]
  
  headings = ["id", "name", "state", "cycles"]
  rows = api.app.list.map do |row|
    if headings.size == 0
      headings = row.keys 
      headings.delete("rel_dir")
    end
    
    v = row.values_at *headings
    v
  end

  display "apps:\n" if type.nil?
  display TableOutputBuilder.build_table(headings, rows, type)
  display "Total number of apps: " + rows.length.to_s if type.nil?
end

#live_run ⇒ Object

apps:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]

Runs a local app with live data.

--config CONFIG_FILE # Use the given config file --directory DIR # App directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN

HIDDEN:



465
466
467
 
# File 'lib/zillabyte/cli/apps.rb', line 465

def live_run
  super
end

#logs ⇒ Object

apps:logs ID [OPERATION_NAME]

Streams logs for the app from our cluster.

--operation OPERATION_NAME # Specify the operation to show logs for -v, --verbose LEVEL # Sets the verbosity (error, info, debug) [default: info] --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



336
337
338
 
# File 'lib/zillabyte/cli/apps.rb', line 336

def logs
  super
end

#prep ⇒ Object

apps:prep [DIR]

Performs any necessary initialization for the app.

--directory DIR # App directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN --mode MODE # Specify development or production mode for dependencies #HIDDEN



265
266
267
 
# File 'lib/zillabyte/cli/apps.rb', line 265

def prep
  super
end

#pull ⇒ Object

apps:pull ID DIR

Pulls an app source to a directory.

--force # Pulls even if the directory exists --directory DIR # App directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN

Examples:

$ zillabyte apps:pull .



240
241
242
 
# File 'lib/zillabyte/cli/apps.rb', line 240

def pull
  super
end

#push ⇒ Object

apps:push [DIR]

Uploads an app.

--config CONFIG_FILE # Use the given config file --directory DIR # App directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN

Examples:

$ zillabyte apps:push .



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/zillabyte/cli/apps.rb', line 196

def push
  
  since = Time.now.utc.to_s
  dir = options[:directory]
  if dir.nil?
    dir = Dir.pwd
  else
    dir = File.expand_path(dir)
  end
  options[:directory] = dir
  type = options[:output_type]

  res = api.apps.push_directory dir, session, options
  
  if res['error']
    error("error: #{res['error_message']}", type)
  else
    display "app ##{res['id']} #{res['action']}" if type.nil?
  end

  display "Starting up your app...please wait..." if type.nil?
  sleep(2) # wait for kill command 
  
  hash = self.api.logs.get(res['id'], nil, options)
  fetch_logs(hash, "_ALL_", "App deployed")

  display "{}" if type == "json"

end

#status ⇒ Object

apps:status [DIR]

Fetches detailed status of the app.

--directory DIR # App directory --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



477
478
479
 
# File 'lib/zillabyte/cli/apps.rb', line 477

def status
  super
end

#test ⇒ Object

apps:test

Tests a local app with sample data.

--config CONFIG_FILE # Use the given config file --output OUTPUT_FILE # Writes sink output to a file --cycles CYCLES # Number of cycles to emit [default: 1] --directory DIR # App directory -i, --interactive # Interactively control input and read output



439
440
441
 
# File 'lib/zillabyte/cli/apps.rb', line 439

def test
  super
end