Module: ExportToGcloud

Defined in:
lib/export_to_gcloud.rb,
lib/export_to_gcloud/library.rb,
lib/export_to_gcloud/version.rb,
lib/export_to_gcloud/exporters/exporter.rb,
lib/export_to_gcloud/exporters/pg_exporter.rb,
lib/export_to_gcloud/exporters/csv_exporter.rb

Defined Under Namespace

Classes: CSVExporter, Exporter, PGExporter

Constant Summary collapse

VERSION = 
'1.0.1'

Class Method Summary collapse

Class Method Details

.client ⇒ Object 



12
13
14
 
# File 'lib/export_to_gcloud.rb', line 12

def self.client
  @client || raise('Gcloud client not present. call ExportToGcloud#setup first.')
end

.create_context(**opts) ⇒ Object 



43
44
45
 
# File 'lib/export_to_gcloud/library.rb', line 43

def self.create_context **opts
  ::ExportToGcloud::Exporter::Context.new client, opts
end

.definitions_resolver=(proc) ⇒ Object 



4
5
6
 
# File 'lib/export_to_gcloud/library.rb', line 4

def self.definitions_resolver= proc
  @definitions_resolver = proc
end

.get_exporter(name, context) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/export_to_gcloud/library.rb', line 31

def self.get_exporter name, context
  name = name.to_s

  @definitions ||= {}
  unless @definitions.has_key? name
    @definitions[name] = ::ExportToGcloud::Exporter::Definition.load_definition name, @definitions_resolver
  end

  definition = @definitions[name]
  definition.type.new definition, context
end

.setup(project_name:, config_file:, definitions_resolver: nil) ⇒ Object 



5
6
7
8
9
10
 
# File 'lib/export_to_gcloud.rb', line 5

def self.setup project_name:, config_file:, definitions_resolver:nil
  require_relative 'export_to_gcloud/library'

  self.definitions_resolver = definitions_resolver if definitions_resolver
  @client = ::Gcloud.new project_name, config_file
end

.wait_for_load_jobs(jobs, &block) ⇒ Object

waits for BigQuery jobs

  • send a block to do something with failed



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/export_to_gcloud/library.rb', line 10

def self.wait_for_load_jobs(jobs, &block)
  jobs_left = jobs.dup
  failed = []
  sleeper = ->(_retries) {sleep 2 * _retries + 5}
  retries = 0

  until jobs_left.empty?
    sleeper.call retries
    retries += 1
    jobs_left.each &:reload!
    jobs_left.delete_if do |j|
      if j.done?
        failed << {id: j.job_id, error: j.error, sources: j.sources} if j.failed?
        true
      end
    end
  end

  block.call failed unless failed.empty?
end