Class: Dag::CLI::Job

Inherits:
SubCommand show all
Defined in:
lib/dag/cli/sub_command/job.rb

Instance Method Summary collapse

Methods inherited from SubCommand

banner, #help

Methods included from Utils::TimeFormat

#time_format

Methods included from Utils::TerminalTable

#terminal_table

Instance Method Details

#info(job_id) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/dag/cli/sub_command/job.rb', line 66

def info(job_id)
  headers = ["item", "info"]
  rows = []

  job = handle_api_failure do
    client.job(job_id)
  end

  rows << ["id", job.id]
  rows << ["status", job.status]
  rows << ["start_at", time_format(job.start_at)]
  rows << ["type", job.type]
  rows << ["dsl", job.dsl]
  rows << ["cluster", job.cluster]
  rows << ["cluster_rebooted", job.cluster_rebooted]
  rows << ["label", job.label]
  rows << ["stage", job.stage]
  rows << ["progress", job.progress]
  rows << ["access_key_id", job.access_key_id]

  case job.dsl
    when "hive"
      rows << ["output_format", job.output_format]
      rows << ["output_resource_path", job.output_resource_path]
      query = sql_format(job.query).split("\n")
      query.each_with_index do |line, i|
        rows << ['', line]
        rows.last[0] = 'query' if i == 0
      end
    when "mapreduce"
      rows << ["job_id", job.job_id]
      rows << ["schema", job.schema]
      rows << ["input_object_keys", job.input_object_keys]
      rows << ["input_format", job.input_format]
      rows << ["output_database", job.output_database]
      rows << ["output_table", job.output_table]
  end

  # vertical, unicode, tab, markdown, simple
  terminal_table(rows, headers: headers, max_width: Float::INFINITY)
end

#kill!(job_id) ⇒ Object 



279
280
281
282
283
284
285
 
# File 'lib/dag/cli/sub_command/job.rb', line 279

def kill!(job_id)
  handle_api_failure do
    client.job(job_id).kill
  end

  say_status("Cancelling job", job_id)
end

#list ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/dag/cli/sub_command/job.rb', line 15

def list
  headers = %w(id status start_at type cluster query)
  rows = []

  if options[:filter] && cluster_rebooted = options[:filter]['cluster_rebooted']
    options[:filter]['cluster_rebooted'] = case cluster_rebooted
      when "true"
        true
      when "false"
        false
    end
  end

  handle_api_failure do
    jobs = client.jobs.where(cluster_name: open_client.cluster_name)

    if options[:filter]
      jobs = jobs.where(options[:filter].symbolize_keys)
    end

    if options[:limit]
      jobs = jobs.limit(options[:limit])
    end

    iterator = :each
    order = if options[:order]
              options[:order]
            else
              if [:limit]
                iterator = :reverse_each
                'desc'
              else
                'asc'
              end
            end

    jobs.order(order).send(iterator) do |job|
      row = [ job.id ]
      row += [ job.status ]
      row += [ time_format(job.start_at) ]
      row += [ job.type ]
      row += [ job.cluster ]
      row += [ job.query ]
      rows << row
    end
  end

  terminal_table(rows, headers: headers)
end

#log(job_id) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/dag/cli/sub_command/job.rb', line 168

def log(job_id)
  if options[:tail]
    mark=nil
    while true
      jst, log = handle_api_failure {
        j = client.job(job_id)
        [j.running?, j.log]
      }
      lines = log.split("\n")
      if mark
        n = lines.rindex(mark)
        lines = lines[(n+1)..-1] if n
      end
      unless lines.empty?
        mark=lines[-1]
        puts lines.join("\n")
      end
      break unless jst
      sleep 1
    end
  else
    log = handle_api_failure do
      client.job(job_id).log
    end

    puts log
  end
end

#progress(job_id = nil) ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
 
# File 'lib/dag/cli/sub_command/job.rb', line 288

def progress(job_id=nil)
  if job_id
    jobs = [job_id]
  else
    jobs = handle_api_failure{
      client.jobs.select{|j| j.running? }.map{|j| j.id }
    }
  end
  return unless jobs.length==1
  progress_bar = ProgressBar.create(total: 100, format: '%t %a <%B> %p%% %e', autofinish: false)
  stage = nil
  while true
    job = handle_api_failure{ client.job(jobs[0]) }
    if ! job.running?
      progress_bar.finish
      puts "#{job.status}"
      break
    end
    if job.stage && stage!=job.stage
      progress_bar.finish if stage
      progress_bar = ProgressBar.create(title: "Stage-#{job.stage}/Job-#{jobs[0]}", total: 100, format: '%t %a <%B> %p%% %e', autofinish: false)
      stage = job.stage
    end
    progress_bar.progress = job.progress.to_f if job.progress && stage == job.stage
    sleep 1
  end
end

#result(job_id) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/dag/cli/sub_command/job.rb', line 199

def result(job_id)
  if options[:format] == "url"
    download_url = handle_api_failure do
      client.job(job_id).download_urls
    end
    puts download_url
    return
  end

  unless job_id.include? "/"
    job = handle_api_failure do
      client.job(job_id)
    end
    target = job.output_resource_path
  else
    target = job_id
  end

  unless target
    if job.try(:type) == 'split'
      say_status 'ERROR', 'Invalid job type', :red
    else
      say_status 'ERROR', 'Invalid job_id or dag://bucket/prefix/', :red
    end
    exit 1
  end

  unless job && job.finished?
    say_status 'ERROR', "Job is not finished (status=#{job.status})", :red
    return
  end

  uri = URI.parse(target)
  bucket = uri.host
  prefix = uri.path[1..-1]
  bucket = client.buckets[bucket]
  rsm = handle_api_failure do
    bucket.objects[File.join(prefix, ".resultsetmetadata")].read
  end

  if rsm.headers['Last-Modified'].present?
    finishedat = Time.parse(rsm.headers['Last-Modified'].first)
  end

  if %w(table time).include? options[:format]
    h = []
    h << "job start at #{time_format(job.start_at)}" if job
    h << "finished #{time_format(finishedat)}" if finishedat
    h << "#{(finishedat-job.start_at).to_i} seconds." if job && finishedat
    puts h.join(", ")
    return if options[:format]=="time"
  end

  field_separator = ","
  field_separator = "\t" if job.output_format == "tsv"

  hdr = Oj.load(rsm)
  hdrs=[]
  hdr["columns"].each{|e| hdrs[e["position"].to_i-1] = e["name"] }
  rows = handle_api_failure do
    bucket.objects.where(prefix: prefix)
     .select{|s| s.name.include? "/0" }
     .map{|e| CSV.parse(bucket.objects[e.name].read, col_sep: field_separator) }
  end

  rows.each{|obj|
    case options[:format]
    when 'csv'
      puts hdrs.join(",")
      puts obj.map{|m| m.join(",") }.join("\n")
    when 'tsv'
      puts hdrs.join("\t")
      puts obj.map{|m| m.join("\t") }.join("\n")
    else
      terminal_table(obj, headers: hdrs)
    end
  }
end

#reuse(job_id) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/dag/cli/sub_command/job.rb', line 114

def reuse(job_id)
  reuse_job = handle_api_failure do
    client.job(job_id)
  end

  if reuse_job.type == "select"
    query_string = options[:query]
    if query_string
      if File.exists? query_string
        query_string = open(query_string).read
        puts "Query: #{query_string}"
      end
    else
      query_string = reuse_job.query
    end

    output_format = options[:format]
    unless output_format
      output_format = reuse_job.output_format
    end

    output_resource_path = options[:output]
    unless output_resource_path
      output_resource_path = reuse_job.output_resource_path
    end

    label = options[:label]
    unless label
      label = reuse_job.label
    end

    params = {
        query: query_string,
        output_format: output_format,
        output_resource_path: output_resource_path,
        label: label
    }

    query = Query.new(open_client, params)
    job = handle_api_failure { query.execute }

    say_status "accepted job", "job_id: #{job.id}"
    if options[:wait]
      CLI::Command.start(["job", "log", job.id.to_s, "-t"])
      CLI::Command.start(["job", "result", job.id.to_s])
    end
  else
    say_status "invalid job type", "job_type: #{reuse_job.type}"
    exit 1
  end
end

#wait(job_id = nil) ⇒ Object 



317
318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/dag/cli/sub_command/job.rb', line 317

def wait(job_id=nil)
  while true
    if job_id
      job = handle_api_failure{ client.job(job_id) }
      break unless job.running?
    else
      jobs = handle_api_failure{
        client.jobs.select{|j| j.running? }
      }
      break if jobs.empty?
    end
    sleep 1
  end
end