Module: InstDataShipper::DataSources::LocalTables

Includes:
Base
Defined in:
lib/inst_data_shipper/data_sources/local_tables.rb

Overview

This module contains the logic for processing local AR tables

Instance Method Summary collapse

Methods included from Base

included, #process_raw_data_enumerator

Instance Method Details

#acceptable_row_counts_mismatch?(expected_row_count, processed_row_count, variation: 0.01) ⇒ Boolean

The count of rows in the database and the exported files may not always match. We accept a small range difference

Returns:



49
50
51
52
53
54
 
# File 'lib/inst_data_shipper/data_sources/local_tables.rb', line 49

def acceptable_row_counts_mismatch?(expected_row_count, processed_row_count, variation: 0.01)
  return true if expected_row_count.nil?

  expected_row_count = expected_row_count.to_f
  expected_row_count * (1 - variation) <= processed_row_count && processed_row_count <= expected_row_count * (1 + variation)
end

#build_query(model, table_def) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/inst_data_shipper/data_sources/local_tables.rb', line 25

def build_query(model, table_def)
  query = model.all
  query = _resolve_model_query(query, table_def[:query])

  if table_is_incremental?(table_def)
    query = _resolve_model_query(
      query,
      table_def.dig(:incremental, :scope),
      string: ->(query, column) { query.where("#{column} > ?", incremental_since) },
      default: "updated_at",
    )
  end
  query
end

#iterate_query(query, table_def, file) ⇒ Object 



40
41
42
43
44
45
46
 
# File 'lib/inst_data_shipper/data_sources/local_tables.rb', line 40

def iterate_query(query, table_def, file)
  processed_row_count = 0
  expected_row_count = query.count

  process_raw_data_enumerator(table_def, query.find_each.lazy.map { |x| processed_row_count += 1; x }, file)
  [expected_row_count, processed_row_count]
end

#make_data_factory(model, table_def) ⇒ Object 



17
18
19
20
21
22
23
 
# File 'lib/inst_data_shipper/data_sources/local_tables.rb', line 17

def make_data_factory(model, table_def)
  ->(file) {
    query = build_query(model, table_def)
    expected_row_count, processed_row_count = iterate_query(query, table_def, file)
    raise MismatchingRowCounts.new(table_def, expected_row_count, processed_row_count) if !acceptable_row_counts_mismatch?(expected_row_count, processed_row_count)
  }
end