Module: ProcessData

Included in:
IndexManager
Defined in:
lib/process_data.rb

Instance Method Summary collapse

Instance Method Details

#createFromFile(data, dataspec) ⇒ Object

Processes and creates items in single file


3
4
5
6
7
8
9
# File 'lib/process_data.rb', line 3

def createFromFile(data, dataspec)
  unique_id = 0
  data.each do |item|
    createItem(processItem(item, dataspec), unique_id)
    unique_id += 1
  end
end

#createItem(item, unique_id, dataspec, doc_class) ⇒ Object

Creates a new item in the index


24
25
26
27
28
29
30
31
32
# File 'lib/process_data.rb', line 24

def createItem(item, unique_id, dataspec, doc_class)
  begin
    if deduplicate(item, dataspec, doc_class)
      doc_class.create item.merge(id: getID(item, dataspec)), index: dataspec.index_name
    end
  rescue
    doc_class.create item.merge(id: getID(item, dataspec)), index: dataspec.index_name
  end
end

#processItem(item, dataspec) ⇒ Object

Preprocesses documents for loading into ES


12
13
14
15
16
17
18
19
20
21
# File 'lib/process_data.rb', line 12

def processItem(item, dataspec)
  # Go through each field in item
  dataspec.field_info.each do |f|
    item = process_date(f, item)
    item = process_pic(f, item, dataspec)
    item = make_facet_version(f, item, dataspec)
  end
  
  return item
end