Class: Zillabyte::Command::Data

Inherits:
Base
  • Object
show all
Defined in:
lib/zillabyte/cli/data.rb

Overview

manage custom datasets

Constant Summary collapse

MAX_POLL_SECONDS = 
60 * 5
POLL_SLEEP = 
1
APPENDS_ROWS_SLICE = 
5_000

Constants inherited from Base

Base::META_COLUMNS

Instance Attribute Summary

Attributes inherited from Base

#args, #options

Instance Method Summary collapse

Methods inherited from Base

alias_command, #api, extract_banner, extract_description, extract_help, extract_help_from_caller, extract_options, extract_summary, inherited, #initialize, method_added, namespace

Methods included from Helpers

#add_git_remote, #app, #ask, #create_git_remote, #display, #error, #extract_app_from_git_config, #extract_app_in_dir, #format_with_bang, #get_flow_name, #git, #handle_downloading_manifest, #has_git?, #longest, #read_multiline, #with_tty

Constructor Details

This class inherits a constructor from Zillabyte::Command::Base

Instance Method Details

#append ⇒ Object

data:append ID FILE

Adds data to an existing dataset.

--filetype FILETYPE # Input File format type, defaults to csv --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/zillabyte/cli/data.rb', line 161

def append

  id = options[:id] || shift_argument
  file = options[:file] || shift_argument
  type = options[:output_type]

  filetype = options[:filetype]
  filetype ||= File.extname(file || "").gsub(".", "")
  error("no id given", type) if id.nil?
  error("no file given", type) if file.nil?
  
  dataset = self.api.data.get(id, options)
  columns = dataset["columns"].map{|col| {col["index"] => col["type"]}}
  raw_rows = sanity_check_file(file,filetype,{"columns" => columns}, type)
 
  total_rows = 0 
  display("uploading content.", false)
  raw_rows.each_slice(APPENDS_ROWS_SLICE) do |rows|
    
    
    # TODO: post to direct signed s3 (http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/S3/PresignedPost.html)
    display(".", false)
    res = self.api.data.append(id, {:gzip_rows => Base64.encode64(gzip(rows.to_json()))})
    # res = self.api.data.append(id, {:rows => rows})
    
    break unless res["size"]
    total_rows += res["size"]
  end
  
  if type == "json"
    display({:rows => total_rows}.to_json)
  else
    display "dataset ##{id} appended #{total_rows} rows"
  end

end

#create ⇒ Object

data:create NAME

Creates a new dataset.

--schema SCHEMA # Column names and types in the format "field_1:output_type_1,field_2:output_type_2,..." --public SCOPE # Make the dataset public --file FILE # A data file --filetype FILETYPE # File format type, defaults to csv --description DESCRIPTION # Description of dataset contents --aliases ALIASES # Dataset name aliases in the format "alias_1,alias_2,..." --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/zillabyte/cli/data.rb', line 114

def create

  name = options[:name] || shift_argument
  file = options[:file] || nil
  filetype = options[:filetype] || nil
  type = options[:output_type]

  error("no name given", type) if name.nil?

  schema = options[:schema] if options[:schema]
  is_public = options[:public] || false
  description = options[:description] || nil
  aliases = options[:aliases] || nil

  if type.nil?
    hash = get_dataset_properties(schema,is_public,description,aliases)
  else
    hash = hash_dataset_properties(schema,is_public,description,aliases, type)
  end

  if file
    filetype ||= File.extname(file).gsub(".", "")
    rows = sanity_check_file(file,filetype, {"columns" => hash[:schema]}, type)
    hash[:rows] = rows
  end

  res = api.data.create name, hash
  if res['error']
    error("#{res['error_message']}", type)
  else
    if type == "json"
      display "{}"
    else
      display "dataset ##{res['id']} #{res['action']}. size: #{res['size'] || 0} rows."
    end
  end

end

#delete ⇒ Object

data:delete ID

Deletes a dataset.

-f, --force # Delete without asking for confirmation --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/zillabyte/cli/data.rb', line 66

def delete 
  id = options[:id] || shift_argument
  forced = options[:force]
  type = options[:output_type] || nil

  if !forced

    if !type.nil?
      error("specify -f, --force to confirm deletion", type)
    end

    while true
      display "This operation cannot be undone. Are you sure you want to delete this dataset? (yes/no):", false
      confirm = ask
      break if confirm == "yes" || confirm == "no"
      display "Please enter 'yes' to delete the dataset or 'no' to exit"
    end
  end

  confirmed = forced || confirm == "yes"
  if confirmed
    res = api.data.delete(id, options)

    if res['error']
      error(res['error'], type)
    else
      if type == "json"
        display "{}"
      else
        display res["body"]
      end
    end
  end
end

#index ⇒ Object

data

Lists your custom datasets.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



23
24
25
 
# File 'lib/zillabyte/cli/data.rb', line 23

def index
  self.list
end

#list ⇒ Object

data

Lists your custom datasets.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/zillabyte/cli/data.rb', line 34

def list
  type = options[:output_type] || nil

  response = api.request(
    :expects  => 200,
    :method   => :get,
    :path     => "/relations"
  )

  headings = []
  rows = response.body.map do |row|
    headings = row.keys if headings.size == 0
    row["columns"] = row["columns"].map{|c| c['type']}.join(',')
    row["aliases"] = row["aliases"].map{|a| a['name']}.join(',')
    vals = row.values_at *headings
    vals 
  end

  display "datasets\n" if type.nil? && rows.size > 0
  display TableOutputBuilder.build_table(headings, rows, type)
  display "Total number of datasets: "+rows.length.to_s if type.nil?

end

#pull ⇒ Object

data:pull ID OUTPUT

Pulls dataset into OUTPUT.gz.

--cycle_id [cycle_id] # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle] --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/zillabyte/cli/data.rb', line 205

def pull

  id = options[:id] || shift_argument
  file = options[:file] || shift_argument
  type = options[:output_type]
  error("no id given", type) if id.nil?
  error("no file given", type) if file.nil?

  file = "#{file}.gz" unless File.extname(file) == ".gz"

  res = self.api.data.pull(id, options)

  handle_downloading_manifest(file, res, type)

  if type == "json"
    display "{}"
  else
    display "finished pulling dataset ##{id} to file"
  end

end

#pull_to_s3 ⇒ Object

data:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY

Pulls dataset to S3_BUCKET/FILE_KEY/part***.gz.

--cycle_id [cycle_id] # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle] --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
 
# File 'lib/zillabyte/cli/data.rb', line 234

def pull_to_s3

  id = options[:id] || shift_argument
  type = options[:output_type]
  error("no id given", type) if id.nil?

  user_s3_access_key = options[:s3_access_key] || shift_argument
  user_s3_secret = options[:s3_secret] || shift_argument
  user_s3_bucket = options[:s3_bucket] || shift_argument
  user_s3_file_key = options[:s3_file_key] || shift_argument
  error("no s3 access key provided", type) if user_s3_access_key.nil?
  error("no s3 access secret provided", type) if user_s3_secret.nil?
  error("no s3 access bucket provided", type) if user_s3_bucket.nil?
  error("no s3 file path provided", type) if user_s3_file_key.nil?

  s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret, 
               :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
  s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]

  res = self.api.data.pull_to_s3(id, s3_params)

  if type == "json"
    display "{}"
  else
    display "downloading dataset to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
    display "if the dataset is large, this may take a while, please check your s3 account after a few minutes"
  end
end

#show ⇒ Object

data:show ID

Shows a sample of the dataset. See 'zillabyte queries' for more elaborate functionality.

--cycle_id [cycle_id] # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle] --no_truncation # Don't truncate long strings --meta # Show metadata columns (since, confidence, source) --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
 
# File 'lib/zillabyte/cli/data.rb', line 275

def show
  name = options[:name] || shift_argument
  type = options[:output_type]
  show_meta = options[:meta] || false
  error "no id given" if name.nil?
  
  # Initial request..
  res = self.api.data.show(name, :post, options)

  if res['job_id']
    job_id = res['job_id']
    options[:job_id] = job_id

    # Poll until the results are ready... 
    start = Time.now.utc

    display "Fetching your data, please wait..." if type.nil?

    while(Time.now.utc < start + MAX_POLL_SECONDS) do

      # Poll
      res = self.api.data.show(name, :get, options)

      # Status?
      case res['status']
      when 'completed'
        if res['return']
          res = res['return']
        else
          throw "something is wrong: #{res}"
        end
        # success! continue below
        break
      when 'running'
        sleep(POLL_SLEEP)
     #   display ".", false
      else
        throw "unknown status: #{res}"
      end
      
    end
  else
    if res['error']
      error(res['error_message'] || res['error'], type)
    else
      error("remote server error (r256)", type)
    end
  end 

  # We only reach here after polling is complete...
  if res["rows"].size > 0
    headings = []
    concrete_headings = res["rows"].first.keys
    concrete_headings.delete("id")
    META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
    concrete_headings.each do |ch|
      has_alias = false
      (res['column_aliases'] || []).each do |al|
        if(al["concrete_name"] == ch)
          headings << al["alias"]
          has_alias = true
        end
      end
      headings << ch if !has_alias
    end

    rows = []
    res["rows"].each do |obj|
      new_row = concrete_headings.map do |heading|
        if options[:no_truncation]
          obj[heading]
        else
          if obj[heading].to_s.size > 30
            obj[heading].to_s[0..30] + "..."
          else
            obj[heading]
          end
        end
      end
      rows << new_row
    end

    display TableOutputBuilder.build_table(headings, rows, type)
  else
    if type == "json"
      display "{}"
    else
      display "empty dataset"
    end
  end
  
end