Class: ExportToGcloud::Exporter

Inherits:
Object
  • Object
show all
Defined in:
lib/export_to_gcloud/exporters/exporter.rb

Direct Known Subclasses

CSVExporter, PGExporter

Defined Under Namespace

Classes: Context, Definition

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(definition, context) ⇒ Exporter

Returns a new instance of Exporter.



6
7
8
9
10
11
12
13
14
15
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 6

def initialize definition, context
  @definition = definition
  @context = context

  @parts = []
  case definition.parts
    when Array then definition.parts.each{|label, *part_args| add_data_part *part_args, label: label}
    when Proc  then definition.parts.call self
  end
end

Class Method Details

.define(**kwargs, &block) ⇒ Object 



87
88
89
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 87

def self.define **kwargs, &block
  ::ExportToGcloud::Exporter::Definition.set_last_definition self, kwargs, &block
end

Instance Method Details

#add_data_part(*args, label: nil) ⇒ Object 



26
27
28
29
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 26

def add_data_part *args, label:nil
  args.unshift(label ? label.to_s : (@parts.length+1).to_s)
  @parts << args
end

#bq_table ⇒ Object 



62
63
64
65
66
67
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 62

def bq_table
  unless defined? @bq_table
    @bq_table = @context.dataset.table @definition.get_bq_table_name
  end
  @bq_table
end

#get_storage_files ⇒ Object 



56
57
58
59
60
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 56

def get_storage_files
  @parts.map do |label, *_|
    @context.bucket.file storage_file_path(label)
  end.compact
end

#local_file_path(label) ⇒ Object 



17
18
19
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 17

def local_file_path label
  @context.dump_path.join "#{@definition.name}#{prepend_underscore label}.csv"
end

#process_all_parts!(recreate_table = true) ⇒ Object 



31
32
33
34
35
36
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 31

def process_all_parts! recreate_table=true
  add_data_part label: 'all' if @parts.empty?
  recreate_bq_table! if recreate_table

  @parts.map{|args| process_part! *args}
end

#process_part!(label, *part_args) ⇒ Object 



38
39
40
41
42
43
44
45
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 38

def process_part! label, *part_args
  file = local_file_path label
  create_data_file! file, *part_args

  storage_name = storage_file_path label
  gcloud_file = upload_file! file, storage_name
  start_load_job gcloud_file
end

#recreate_bq_table! ⇒ Object 



69
70
71
72
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 69

def recreate_bq_table!
  bq_table.delete if bq_table
  @bq_table = @context.dataset.create_table @definition.get_bq_table_name, &@definition.bq_schema
end

#start_load_job(gcloud_file, **_load_settings) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 74

def start_load_job gcloud_file, **_load_settings
  load_settings = {
      format: 'csv',
      quote: '"',
      delimiter: ';',
      create: 'never',
      write: 'append',
      max_bad_records: 0
  }
  load_settings.merge! _load_settings unless _load_settings.empty?
  bq_table.load gcloud_file, **load_settings
end

#storage_file_path(label) ⇒ Object 



21
22
23
24
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 21

def storage_file_path label
  prefix = @definition.storage_prefix || @context.storage_prefix
  "#{prefix}#{@definition.name}#{prepend_underscore label}.csv"
end

#upload_file!(file, storage_name) ⇒ Object 



49
50
51
52
53
54
 
# File 'lib/export_to_gcloud/exporters/exporter.rb', line 49

def upload_file!(file, storage_name)
  file = compress_file! file
  gcloud_file = @context.bucket.create_file file, storage_name, chunk_size: 2**21 # 2MB
  file.delete
  gcloud_file
end