Class: Samidare::BigQueryUtility

Inherits:
Object
  • Object
show all
Defined in:
lib/samidare/bigquery_utility.rb

Constant Summary collapse

CONTENTS = 
<<-EOS.unindent
in:
  type: mysql
  user: <%= user %>
  password: <%= password %>
  database: <%= database %>
  host: <%= host %>
  query: |
    <%= query %>
out:
  type: bigquery
  project: <%= project %>
  p12_keyfile_path: <%= p12_keyfile_path %>
  service_account_email: <%= service_account_email %>
  dataset: <%= dataset %>
  table: <%= table_name %>
  schema_path: <%= schema_path %>
  auto_create_table: 1
  path_prefix: <%= path_prefix %>
  source_format: NEWLINE_DELIMITED_JSON
  file_ext: .json.gz
  delete_from_local_when_job_end: 1
  formatter:
    type: jsonl
  encoders:
  - {type: gzip}
EOS

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ BigQueryUtility

Returns a new instance of BigQueryUtility.



36
37
38
 
# File 'lib/samidare/bigquery_utility.rb', line 36

def initialize(config)
  @config = config.dup
end

Class Method Details

.generate_schema(column_infos) ⇒ Object 



40
41
42
43
 
# File 'lib/samidare/bigquery_utility.rb', line 40

def self.generate_schema(column_infos)
  json_body = column_infos.map { |column_info| column_info.to_json }.join(",\n")
  "[\n" + json_body + "\n]\n"
end

.generate_sql(table_name, column_infos) ⇒ Object 



45
46
47
48
49
 
# File 'lib/samidare/bigquery_utility.rb', line 45

def self.generate_sql(table_name, column_infos)
  columns = column_infos.map { |column_info| column_info.converted_value }
  sql = "SELECT " + columns.join(",")
  sql << " FROM #{table_name}\n"
end

Instance Method Details

#delete_table(dataset, table_name) ⇒ Object 



68
69
70
71
72
73
 
# File 'lib/samidare/bigquery_utility.rb', line 68

def delete_table(dataset, table_name)
  @config['dataset'] = dataset

  bq = BigQuery::Client.new(@config)
  bq.delete_table(table_name)
end

#generate_embulk_config(db_name, db_info, table_info, column_infos) ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/samidare/bigquery_utility.rb', line 51

def generate_embulk_config(db_name, db_info, table_info, column_infos)
  host = db_info['host']
  user = db_info['username']
  password = db_info['password']
  database = db_info['database']
  query = Samidare::BigQueryUtility.generate_sql(table_info.name, column_infos)
  project = @config['project_id']
  p12_keyfile_path = @config['key']
   = @config['service_email']
  dataset = db_info['bq_dataset']
  table_name = table_info.name
  schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_info.name}.json"
  path_prefix = "/var/tmp/embulk_#{db_name}_#{table_info.name}"

  ERB.new(CONTENTS).result(binding)
end