Class: Tumugi::Plugin::BigqueryLoadTask

Inherits:
Task
  • Object
show all
Defined in:
lib/tumugi/plugin/task/bigquery_load.rb

Instance Method Summary collapse

Instance Method Details

#output ⇒ Object 



27
28
29
30
31
32
33
 
# File 'lib/tumugi/plugin/task/bigquery_load.rb', line 27

def output
  return @output if @output

  opts = { dataset_id: dataset_id, table_id: table_id }
  opts[:project_id] = project_id if project_id
  @output = Tumugi::Plugin::BigqueryTableTarget.new(opts)
end

#run ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/tumugi/plugin/task/bigquery_load.rb', line 35

def run
  if mode != 'append'
    raise Tumugi::ParameterError.new("Parameter 'schema' is required when 'mode' is 'truncate' or 'empty'") if schema.nil?
  end

  src_uri = "gs://#{bucket}#{normalize_path(key)}"
  log "Source: #{src_uri}"
  log "Destination: #{output}"

  bq_client = output.client
  opts = {
    schema: schema,
    field_delimiter: field_delimiter,
    mode: mode.to_sym,
    allow_jagged_rows: allow_jagged_rows,
    max_bad_records: max_bad_records,
    ignore_unknown_values: ignore_unknown_values,
    allow_quoted_newlines: allow_quoted_newlines,
    quote: quote,
    skip_leading_rows: skip_leading_rows,
    source_format: source_format,
    project_id: output.project_id,
    wait: wait
  }
  bq_client.load(output.dataset_id, output.table_id, src_uri, opts)
end