Class: Zillabyte::Command::Query

Inherits:
Base
  • Object
show all
Defined in:
lib/zillabyte/cli/query.rb

Overview

executes queries

Constant Summary collapse

MAX_POLL_SECONDS = 
60 * 5
POLL_SLEEP = 
0.5

Constants inherited from Base

Base::META_COLUMNS

Instance Attribute Summary

Attributes inherited from Base

#args, #options

Instance Method Summary collapse

Methods inherited from Base

alias_command, #api, extract_banner, extract_description, extract_help, extract_help_from_caller, extract_options, extract_summary, inherited, #initialize, method_added, namespace

Methods included from Helpers

#add_git_remote, #app, #ask, #create_git_remote, #display, #error, #extract_app_from_git_config, #extract_app_in_dir, #format_with_bang, #get_flow_name, #git, #handle_downloading_manifest, #has_git?, #longest, #read_multiline, #with_tty

Constructor Details

This class inherits a constructor from Zillabyte::Command::Base

Instance Method Details

#pull ⇒ Object

query:pull QUERY FILE

Executes a query and downloads the results to FILE.



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
 
# File 'lib/zillabyte/cli/query.rb', line 253

def pull

  query = options[:query] || shift_argument
  file = options[:file] || shift_argument
  type = options[:output_type]
  error("no query given", type) if query.nil?
  error("no file given", type) if file.nil?
  file = "#{file}.gz" unless File.extname(file) == ".gz"

  options[:query] = query

  res = api.request(
    :expects  => 200,
    :method   => :post,
    :path     => "/query_pull",
    :body     => options.to_json 
  )
  res = res.body
  
  handle_downloading_manifest(file, res, type)

  if type == "json"
    display "{}"
  else
    display "finished downloading results to file"
  end
  
end

#pull_to_s3 ⇒ Object

query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY

Executes a query and pulls data to S3_BUCKET/FILE_KEY/part***.gz.

--output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN



291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/zillabyte/cli/query.rb', line 291

def pull_to_s3

  query = options[:query] || shift_argument
  error("query cannot be empty", type) if query.nil?

  type = options [:output_type]

  user_s3_access_key = options[:s3_access_key] || shift_argument
  user_s3_secret = options[:s3_secret] || shift_argument
  user_s3_bucket = options[:s3_bucket] || shift_argument
  user_s3_file_key = options[:s3_file_key] || shift_argument
  error("no s3 access key provided", type) if user_s3_access_key.nil?
  error("no s3 secret provided", type) if user_s3_secret.nil?
  error("no s3 bucket provided", type) if user_s3_bucket.nil?
  error("no s3 file path specified", type) if user_s3_file_key.nil?

  s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret, 
               :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}

  res = self.api.queries.pull_to_s3(query, s3_params)

  if type == "json"
    display {}.to_json
  else
    display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
    display "if the dataset is large, this may take a while, please check your s3 account after a few minutes"
  end
end

#sql ⇒ Object

query:sql EXPRESSION

Executes SQL queries against the zillabyte corpus.

-o, --offset OFFSET # Skips to the offset [default: 0] -l, --limit LIMIT # Sets the result limit [default: 20] -t, --tail TAIL # Continuously watches for new results -s, --since SINCE # Grab newer records since SINCE --no_truncation # Doesn't truncate long strings --meta # Show meta columns (since, confidence, source) --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN

Examples:

$ zillabyte query:sql "select * from company" --limit 100



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/zillabyte/cli/query.rb', line 130

def sql
  
  opts = {}
  opts[:offset] = options[:offset] || 0
  opts[:limit] = options[:limit] || 10

  tail = options[:tail] || false
  expression = options[:expression] || shift_argument 
  opts[:since] = options[:since]
  show_meta = options[:meta] || false
  type = options[:output_type]
  seen = {}
  
  if expression.nil?
    error("no expression given", type)
  end
  
  begin
  
    validate_arguments!
    res = api.query.sql(expression, :post, opts)

    if res['job_id']
      job_id = res['job_id']
      opts[:job_id] = job_id

      start = Time.now.utc

      col_aliases = res['column_aliases']
      while(Time.now.utc < start + MAX_POLL_SECONDS) do

        res = api.query.sql(expression, :get, opts)
        case res['status']
        when 'completed'
          if(res['return'])
            res = res['return']
          else
            throw "something is wrong: #{res}"
          end
        # success! continue below
          break
        when 'running'
          sleep(POLL_SLEEP)
     #   display ".", false
        else
          throw "unknown status: #{res}"
        end

      end
    else
      if res['error']
        error(res['error_message'] || res['error'], type)
      else
        error("remote server error(r256)", type)
      end
    end

    headings = []
    filtered = []
    if res['rows'].first
      concrete_headings = res['rows'].first.keys
      META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
      concrete_headings.each do |ch|
        has_alias = false
        col_aliases.each do |al|
          if(al["concrete_name"] == ch)
            headings << al["alias"]
            has_alias = true
          end
        end
        headings << ch if !has_alias
      end
      rows = res['rows'].each do |obj|
        # if obj['since']
        #   opts[:since] = [obj['since'], opts[:since]].compact.max
        # end
        
        new_row = concrete_headings.map do |heading|
          if options[:no_truncation]
            obj[heading]
          else
            if obj[heading].to_s.size > 30
              obj[heading].to_s[0..30] + "..."
            else
              obj[heading]
            end
          end
        end
        
        unless seen[new_row.to_s]
          filtered << new_row
          seen[new_row.to_s] = true
        end
      end
    end
  
    if filtered.size > 0
      opts[:since] = Time.now.utc
      display TableOutputBuilder.build_table(headings, filtered, type)

    else
      unless tail
        if type == "json"
          display {}.to_json
        else
          display "no results"
        end
      end
    end
    
        
  end while (tail && sleep(5))

end

#sxp ⇒ Object

query:sxp EXPRESSION

Executes SXP queries against the zillabyte corpus.

-o, --offset OFFSET # Skips to the offset [default: 0] -l, --limit LIMIT # Sets the result limit [default: 20] -t, --tail TAIL # Continuously watches for new results --meta # Show meta columns (since, confidence, source) --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN

Examples:

$ zillabyte query:sxp "(uses company 'web_css')" --limit 100



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/zillabyte/cli/query.rb', line 24

def sxp
  
  opts = {}
  opts[:offset] = options[:offset] || 0
  opts[:limit] = options[:limit] || 10
  type = options[:output_type]
  show_meta = options[:meta] || false
  tail = options[:tail] || false
  expression = options[:expression] || shift_argument 
  opts[:since] = options[:since]
  
  if expression.nil?
    error("no expression given", type)
  end
  
  seen = {}
  
  begin
  
    validate_arguments!
    res = api.query.sxp(expression, :post, opts)
  
    if res['job_id']
      job_id = res['job_id']
      opts[:job_id] = job_id

      start = Time.now.utc
      display "Fetching your data, please wait..." if type.nil?

      while(Time.now.utc < start + MAX_POLL_SECONDS) do

        res = api.query.sxp(expression, :get, opts)
        case res['status']
        when 'completed'
          if(res['return'])
            res = res['return']
          else
            error("something is wrong #{res}", type)
          end
        # success! continue below
          break
        when 'running'
          sleep(POLL_SLEEP)
        else
          error("unknown status: #{res}", type)
        end

      end
    else
      if res['error']
        error(res['error_message'] || res['error'], type)
      else
        error("remote server error(r256)", type)
      end
    end

    filtered = []
    headings = []
    if res['results'].first
      headings = res['results'].first.keys
      META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
      rows = res['results'].each do |obj|
        new_row = headings.map do |heading|
          obj[heading]
        end
        unless seen[new_row.to_s]
          filtered << new_row
          seen[new_row.to_s] = true
        end
      end
    end
  
    if filtered.size > 0
      display TableOutputBuilder.build_table(headings, filtered)
    else
      unless tail
        display "no results" if type.nil?
      end
    end
    
    opts[:since] = DateTime.now
        
  end while (tail && sleep(5))

end