Class: RedshiftConnector::AbstractDataFileBundle

Inherits:
Object
  • Object
show all
Defined in:
lib/redshift-connector/data_file/abstract_data_file_bundle.rb

Direct Known Subclasses

UrlDataFileBundle

Constant Summary collapse

REPORT_SIZE = 
10_0000

Instance Method Summary collapse

Instance Method Details

#all_data_objects ⇒ Object 



18
19
20
 
# File 'lib/redshift-connector/data_file/abstract_data_file_bundle.rb', line 18

def all_data_objects
  data_files.select {|obj| obj.data_object? }
end

#each_batch(report: true) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/redshift-connector/data_file/abstract_data_file_bundle.rb', line 24

def each_batch(report: true)
  @logger.info "reader: #{@reader_class}"
  n = 0
  reported = 0
  do_each_batch(@batch_size) do |rows|
    yield rows
    n += rows.size
    if n / REPORT_SIZE > reported
      @logger.info "#{n} rows processed" if report
      reported = n / REPORT_SIZE
    end
  end
  @logger.info "total #{n} rows processed" if report
end

#each_object(&block) ⇒ Object 



11
12
13
14
15
16
 
# File 'lib/redshift-connector/data_file/abstract_data_file_bundle.rb', line 11

def each_object(&block)
  all_data_objects.each do |obj|
    @logger.info "processing s3 object: #{obj.key}"
    yield obj
  end
end

#each_row(&block) ⇒ Object Also known as: each 



3
4
5
6
7
 
# File 'lib/redshift-connector/data_file/abstract_data_file_bundle.rb', line 3

def each_row(&block)
  each_object do |obj|
    obj.each_row(&block)
  end
end