Class: DataShift::LoaderBase

Inherits:
Object
  • Object
show all
Extended by:
Forwardable
Includes:
Logging, Querying
Defined in:
lib/datashift/loaders/loader_base.rb

Instance Attribute Summary collapse

Attributes included from Delimiters

#attribute_list_end, #attribute_list_start, #csv_delimiter, #key_value_sep, #text_delim

Instance Method Summary collapse

Methods included from Querying

#find_or_new, #get_record_by, #get_record_by!, #search_for_record, where_field_and_values

Methods included from Logging

#logdir, #logdir=, #logger, #verbose

Methods included from Delimiters

#column_delim, #column_delim=, #eol, #multi_assoc_delim, #multi_assoc_delim=, #multi_facet_delim, #multi_value_delim, #multi_value_delim=, #name_value_delim, #name_value_delim=, #setmulti_facet_delim

Constructor Details

#initialize ⇒ LoaderBase

Returns a new instance of LoaderBase.



36
37
38
39
40
41
 
# File 'lib/datashift/loaders/loader_base.rb', line 36

def initialize
  @file_name = ''

  @doc_context = DocContext.new(Object)
  @binder      = Binder.new
end

Instance Attribute Details

#binder ⇒ Object

Returns the value of attribute binder.



24
25
26
 
# File 'lib/datashift/loaders/loader_base.rb', line 24

def binder
  @binder
end

#doc_context ⇒ Object

Returns the value of attribute doc_context.



26
27
28
 
# File 'lib/datashift/loaders/loader_base.rb', line 26

def doc_context
  @doc_context
end

#file_name ⇒ Object

Returns the value of attribute file_name.



23
24
25
 
# File 'lib/datashift/loaders/loader_base.rb', line 23

def file_name
  @file_name
end

Instance Method Details

#abort_on_failure? ⇒ Boolean

Returns:

  • (Boolean) 


64
65
66
 
# File 'lib/datashift/loaders/loader_base.rb', line 64

def abort_on_failure?
  !! DataShift::Configuration.call.abort_on_failure
end

#bind_headers(headers) ⇒ Object Also known as: bind_fields

Core API

Returns an instance of DataShift::Binder

Given a list of free text column names from inbound headers, map all headers to a domain model containing details on operator, look ups etc.



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/datashift/loaders/loader_base.rb', line 88

def bind_headers( headers )

  logger.info("Binding #{headers.size} inbound headers to #{load_object_class.name}")

  @binder ||= DataShift::Binder.new

  begin
    binder.map_inbound_headers(load_object_class, headers)
  rescue => e
    logger.error("Failed to map header row to set of database operators : #{e.inspect}")
    logger.error( e.backtrace )
    raise MappingDefinitionError, 'Failed to map header row to set of database operators'
  end

  unless binder.missing_bindings.empty?
    logger.warn("Following headings couldn't be mapped to #{load_object_class}:")
    binder.missing_bindings.each { |m| logger.warn("Heading [#{m.source}] - Index (#{m.index})") }

    if DataShift::Configuration.call.strict_inbound_mapping
      raise MappingDefinitionError, "Missing mappings for columns : #{binder.missing_bindings.join(',')}"
    end

  end

  mandatory = DataShift::Mandatory.new(DataShift::Configuration.call.mandatory)

  unless mandatory.contains_all?(binder)
    mandatory.missing_columns.each do |er|
      logger.error "Mandatory column missing - expected column '#{er}'"
    end

    raise MissingMandatoryError, 'Mandatory columns missing  - see logs - please fix and retry.'
  end

  binder
end

#configure_from(yaml_file, klass = nil, locale_key = 'data_flow_schema') ⇒ Object

Any Config under key ‘LoaderBase’ is merged over existing options - taking precedence.

Any Config under a key equal to the full name of the Loader class (e.g DataShift::SpreeEcom::ImageLoader) is merged over existing options - taking precedence.

Format :

  LoaderClass:
   option: value



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/datashift/loaders/loader_base.rb', line 138

def configure_from(yaml_file, klass = nil, locale_key = 'data_flow_schema')

  setup_load_class(klass) if(klass)

  logger.info("Reading Datashift loader config from: #{yaml_file.inspect}")

  data = YAML.load( ERB.new( IO.read(yaml_file) ).result )

  logger.info("Read Datashift config: #{data.inspect}")

  @config.merge!(data['LoaderBase']) if data['LoaderBase']

  @config.merge!(data[self.class.name]) if data[self.class.name]

  @binder ||= DataShift::Binder.new

  data_flow_schema = DataShift::DataFlowSchema.new

  # Includes configuring DataShift::Transformation
  nodes = data_flow_schema.prepare_from_file(yaml_file, locale_key)

  @binder.add_bindings_from_nodes( nodes )

  PopulatorFactory.configure(load_object_class, yaml_file)

  logger.info("Loader Options : #{@config.inspect}")
end

#load_object_class ⇒ Object 



68
69
70
 
# File 'lib/datashift/loaders/loader_base.rb', line 68

def load_object_class
  doc_context.klass
end

#report ⇒ Object 



77
78
79
 
# File 'lib/datashift/loaders/loader_base.rb', line 77

def report
  reporters.each(&:report)
end

#reset(object = nil) ⇒ Object

Reset the loader, including database object to be populated, and load counts



60
61
62
 
# File 'lib/datashift/loaders/loader_base.rb', line 60

def reset(object = nil)
  doc_context.reset(object)
end

#run(file_name, load_class) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
 
# File 'lib/datashift/loaders/loader_base.rb', line 47

def run(file_name, load_class)
  @file_name = file_name

  setup_load_class(load_class)

  logger.info("Loading objects of type #{load_object_class}")

  # no implementation - derived classes must implement
  perform_load
end

#set_headers(headings) ⇒ Object 



72
73
74
75
 
# File 'lib/datashift/loaders/loader_base.rb', line 72

def set_headers(headings)
  logger.info("Setting parsed headers to [#{headings.inspect}]")
  doc_context.headers = headings
end

#setup_load_class(load_class) ⇒ Object 



43
44
45
 
# File 'lib/datashift/loaders/loader_base.rb', line 43

def setup_load_class(load_class)
  @doc_context = DocContext.new( MapperUtils.ensure_class(load_class) )
end