Class: Samidare::BigQueryUtility

Inherits:
Object
  • Object
show all
Defined in:
lib/samidare/bigquery_utility.rb

Constant Summary collapse

CONTENTS = 
"in:\n  type: mysql\n  user: <%= user %>\n  password: <%= password %>\n  database: <%= database %>\n  host: <%= host %>\n  query: |\n    <%= query %>\nout:\n  type: bigquery\n  project: <%= project %>\n  p12_keyfile_path: <%= p12_keyfile_path %>\n  service_account_email: <%= service_account_email %>\n  dataset: <%= dataset %>\n  table: <%= table_name %>\n  schema_path: <%= schema_path %>\n  auto_create_table: 1\n  path_prefix: <%= path_prefix %>\n  source_format: NEWLINE_DELIMITED_JSON\n  file_ext: .json.gz\n  delete_from_local_when_job_end: 1\n  formatter:\n    type: jsonl\n  encoders:\n  - {type: gzip}\n".unindent

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ BigQueryUtility

Returns a new instance of BigQueryUtility.



37
38
39
40
 
# File 'lib/samidare/bigquery_utility.rb', line 37

def initialize(config)
  @config = config.dup
  @current_date = Date.today
end

Class Method Details

.generate_schema(columns) ⇒ Object 



42
43
44
45
 
# File 'lib/samidare/bigquery_utility.rb', line 42

def self.generate_schema(columns)
  json_body = columns.map { |column| column.to_json }.join(",\n")
  "[\n" + json_body + "\n]\n"
end

.generate_sql(table_config, columns) ⇒ Object 



47
48
49
50
51
52
53
54
 
# File 'lib/samidare/bigquery_utility.rb', line 47

def self.generate_sql(table_config, columns)
  columns = columns.map { |column| column.converted_value }
  sql = "SELECT " + columns.join(",")
  sql << " FROM #{table_config.name}"
  sql << " WHERE #{table_config.condition}" if table_config.condition
  sql << "\n"
  sql
end

Instance Method Details

#actual_table_name(table_name, daily_snapshot) ⇒ Object 



80
81
82
83
 
# File 'lib/samidare/bigquery_utility.rb', line 80

def actual_table_name(table_name, daily_snapshot)
  return table_name unless daily_snapshot
  table_name + @current_date.strftime('%Y%m%d')
end

#delete_table(dataset, table_name) ⇒ Object 



73
74
75
76
77
78
 
# File 'lib/samidare/bigquery_utility.rb', line 73

def delete_table(dataset, table_name)
  @config['dataset'] = dataset

  bq = BigQuery::Client.new(@config)
  bq.delete_table(table_name)
end

#generate_embulk_config(db_name, database_config, table_config, columns) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/samidare/bigquery_utility.rb', line 56

def generate_embulk_config(db_name, database_config, table_config, columns)
  host = database_config['host']
  user = database_config['username']
  password = database_config['password']
  database = database_config['database']
  query = Samidare::BigQueryUtility.generate_sql(table_config, columns)
  project = @config['project_id']
  p12_keyfile_path = @config['key']
   = @config['service_email']
  dataset = database_config['bq_dataset']
  table_name = actual_table_name(table_config.name, database_config['daily_snapshot'] || table_config.daily_snapshot)
  schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_config.name}.json"
  path_prefix = "/var/tmp/embulk_#{db_name}_#{table_config.name}"

  ERB.new(CONTENTS).result(binding)
end