Module: NdrImport::UniversalImporterHelper

Defined in:
lib/ndr_import/universal_importer_helper.rb

Overview

This mixin provides file importer helper methods that abstract away some of the complexity of enumerating over files and tables (which should be universally useful). It is assumed that the host module/class defines unzip_path.

Defined Under Namespace

Classes: TableEnumProxy

Instance Method Summary collapse

Instance Method Details

#extract(source_file, &block) ⇒ Object

Iterate through the file(s) line by line, yielding each one in turn, using get_table_mapping to select the mapping relevant to this file.



48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 48

def extract(source_file, &block)
  return enum_for(:extract, source_file) unless block

  NdrImport::File::Registry.files(source_file, 'unzip_path' => unzip_path).each do |filename|
    # now at the individual file level, can we find the table mapping?
    table_mapping = get_table_mapping(filename, nil)
    options       = table_options_from(table_mapping).merge { 'unzip_path' => unzip_path }

    tables = NdrImport::File::Registry.tables(filename, table_mapping.try(:format), options)
    yield_tables_and_their_content(filename, tables, &block)
  end
end

#filename_augmented_rows(table, rows, filename) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 93

def filename_augmented_rows(table, rows, filename)
  return enum_for(:filename_augmented_rows, table, rows, filename) unless block_given?

  filename_column_position = table.columns.index { |c| c['column'] == :filename }

  rows.each.with_index do |row, i|
    if filename_column_position
      if i < table.header_lines
        row.insert(filename_column_position, :filename)
      else
        row.insert(filename_column_position, ::File.basename(filename))
      end
    end

    yield row
  end
end

#get_notifier(_total_records) ⇒ Object

This method needs to be implemented where this mixin is used.

Raises:

  • (NotImplementedError) 


116
117
118
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 116

def get_notifier(_total_records)
  raise NotImplementedError, 'get_notifier must be defined!'
end

#get_table_mapping(filename, tablename) ⇒ Object

This method returns the correct NdrImport::,NonTabular::Table for the given filename/tablename. It requires all the mappings to be stored in the table_mappings instance variable.



42
43
44
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 42

def get_table_mapping(filename, tablename)
  @table_mappings.find { |mapping| mapping.match(filename, tablename) }
end

#mapped_tables(filename) ⇒ Object 



111
112
113
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 111

def mapped_tables(filename)
  @mapped_tables ||= table_enumerators(filename)
end

#record_total(filename, table_content) ⇒ Object 



120
121
122
123
124
125
126
127
128
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 120

def record_total(filename, table_content)
  if '.csv' == ::File.extname(filename).downcase
    return `wc -l #{Shellwords.escape(filename)}`.strip.match(/\A(\d+)/)[1].to_i
  elsif table_content.is_a?(Enumerator)
    nil # Avoid slurping
  else
    table_content.size
  end
end

#table_enumerators(filename) ⇒ Object 



29
30
31
32
33
34
35
36
37
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 29

def table_enumerators(filename)
  table_enumerators = Hash.new { |hash, key| hash[key] = TableEnumProxy.new }

  extract(filename).each do |table, rows|
    table_enumerators[table.canonical_name].add_table_enum table.transform(rows)
  end

  table_enumerators
end

#table_options_from(table_mapping) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 61

def table_options_from(table_mapping)
  { 'col_sep'                    => table_mapping.try(:delimiter),
    'file_password'              => table_mapping.try(:file_password),
    'liberal_parsing'            => table_mapping.try(:liberal_parsing),
    'xml_record_xpath'           => table_mapping.try(:xml_record_xpath),
    'slurp'                      => table_mapping.try(:slurp),
    'yield_xml_record'           => table_mapping.try(:yield_xml_record),
    'pattern_match_record_xpath' => table_mapping.try(:pattern_match_record_xpath),
    'xml_file_metadata'          => table_mapping.try(:xml_file_metadata),
    'vcf_file_metadata'          => table_mapping.try(:vcf_file_metadata) }
end

#yield_tables_and_their_content(filename, tables) ⇒ Object

This method does the table row yielding for the extract method, setting the notifier so that we can monitor progress



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/ndr_import/universal_importer_helper.rb', line 75

def yield_tables_and_their_content(filename, tables, &)
  return enum_for(:yield_tables_and_their_content, filename, tables) unless block_given?

  tables.each do |tablename, table_content, file_metadata|
    mapping = get_table_mapping(filename, tablename)
    next if mapping.nil?

    mapping.notifier = get_notifier(record_total(filename, table_content))
    # Merge file_metadata (from VCF or XML) with directly assigned table_metadata
    # to form `table_metadata`
    mapping.table_metadata = (mapping.table_metadata || {}).merge(file_metadata || {})

    augmented_table_content = filename_augmented_rows(mapping, table_content, filename)

    yield(mapping, augmented_table_content)
  end
end