Class: MercatorIcecat::Metadatum

Inherits:
ActiveRecord::Base
  • Object
show all
Defined in:
app/models/mercator_icecat/metadatum.rb

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.assign_products(only_missing: true) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'app/models/mercator_icecat/metadatum.rb', line 87

def self.assign_products(only_missing: true)
  if only_missing
    products = Product.without_icecat_metadata
    ::JobLogger.warn(products.count.to_s + " products without metadata.")
  else
    products = Product.all
  end

  products.each do |product|
    metadata = self.where(prod_id: product.icecat_article_number)
    amount = metadata.count
    metadata.each_with_index do |metadatum, index|
      if metadatum.update(product_id: product.id)
        ::JobLogger.info("Product " + product.number.to_s + " assigned to " + metadatum.id.to_s + " (" + index.to_s + "/" + amount.to_s + ")")
      else
        ::JobLogger.error("Product " + product.number.to_s + " assigned to " + metadatum.id.to_s)
      end
    end
  end

  products = Product.without_icecat_metadata
  ::JobLogger.warn(products.count.to_s + " products without metadata.")
end

.download(overwrite: false, from_today: true) ⇒ Object 



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'app/models/mercator_icecat/metadatum.rb', line 111

def self.download(overwrite: false, from_today: true)
  if from_today
    @twentyfivehours = Time.now - 25.hours
    metadata = self.where{ (product_id != nil) & (updated_at > my{@twentyfivehours})}
  else
    metadata = self.where{ product_id != nil }
  end

  amount = metadata.count
  metadata.each_with_index do |metadatum, index|
    if metadatum.download(overwrite: overwrite)
      ::JobLogger.info("XML Metadatum " + metadatum.prod_id.to_s + " downloaded. (" + index.to_s + "/" + amount.to_s + ")")
    else
      ::JobLogger.info("XML Metadatum " + metadatum.prod_id.to_s + " exists (no overwrite)!")
    end
  end
end

.import(full: false, date: Date.today) ⇒ Object

— Class Methods — #



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'app/models/mercator_icecat/metadatum.rb', line 51

def self.import(full: false, date: Date.today)
  if full
    file = File.open(Rails.root.join("vendor","catalogs","files.index.xml"), "r")
  else
    file = File.open(Rails.root.join("vendor","catalogs",date.to_s + "-index.xml"), "r")
  end

  parser = Saxerator.parser(file) do |config|
    config.put_attributes_in_hash!
  end

  # Hewlett Packard has Supplier_id = 1
  parser.for_tag("file").with_attribute("Supplier_id", "1").each do |product|
    metadatum = self.find_or_create_by_icecat_product_id(product["Product_ID"])
    mode = Time.now - metadatum.created_at > 5 ? " updated." : " created."

    model_name = product["Model_Name"].fix_utf8 if product["Model_Name"].present?

    if metadatum.update(path:              product["path"],
                        cat_id:            product["Catid"],
                        icecat_product_id: product["Product_ID"],
                        icecat_updated_at: product["Updated"],
                        quality:           product["Quality"],
                        supplier_id:       product["Supplier_id"],
                        prod_id:           product["Prod_ID"],
                        on_market:         product["On_Market"],
                        model_name:        model_name,
                        product_view:      product["Product_View"])
      ::JobLogger.info("Metadatum " + product["Prod_ID"].to_s + mode)
    else
      ::JobLogger.error("Metadatum " + product["Prod_ID"].to_s + " could not be saved: " + metadatum.errors.first )
    end
  end
  file.close
end

.import_missing_images ⇒ Object 



160
161
162
163
164
165
166
 
# File 'app/models/mercator_icecat/metadatum.rb', line 160

def self.import_missing_images
  metadata = self.includes(:product).where{product.id != nil}
                 .where{product.photo_file_name == nil}.references(:product).order(id: :asc)
  metadata.each do |metadatum|
    metadatum.import_missing_image
  end
end

.update_product_relations(from_today: true) ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'app/models/mercator_icecat/metadatum.rb', line 145

def self.update_product_relations(from_today: true)
  if from_today
    @twentyfivehours = Time.now - 25.hours
    metadata = self.where{ (product_id != nil) & (updated_at > my{@twentyfivehours})}.order(id: :asc)
  else
    metadata = self.where{ product_id != nil }.order(id: :asc)
  end

  amount = metadata.count
  metadata.each_with_index do |metadatum, index|
    metadatum.update_product_relations
    ::JobLogger.info("(" + index.to_s + "/" + amount.to_s + ")")
  end
end

.update_products(from_today: true) ⇒ Object 



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'app/models/mercator_icecat/metadatum.rb', line 129

def self.update_products(from_today: true)
  if from_today
    @twentyfivehours = Time.now - 25.hours
    metadata = self.where{ (product_id != nil) & (updated_at > my{@twentyfivehours})}.order(id: :asc)
  else
    metadata = self.where{ product_id != nil }.order(id: :asc)
  end

  amount = metadata.count
  metadata.each_with_index do |metadatum, index|
    # if-clause is handy for resume after dump
    metadatum.update_product # if metadatum.id >= 109279
    ::JobLogger.info("(" + index.to_s + "/" + amount.to_s + ")")
  end
end

Instance Method Details

#create_permitted? ⇒ Boolean

— Permissions — #

Returns:

  • (Boolean) 


33
34
35
 
# File 'app/models/mercator_icecat/metadatum.rb', line 33

def create_permitted?
  acting_user.administrator?
end

#delete_relations ⇒ Object 



310
311
312
313
314
315
316
317
318
319
 
# File 'app/models/mercator_icecat/metadatum.rb', line 310

def delete_relations
  product = self.product
  relations_count = product.productrelations.count
  product.productrelations.destroy_all
  supplies_count = product.supplyrelations.count
  product.supplyrelations.destroy_all
  ::JobLogger.info(relations_count.to_s + " Productrel., " +
                   supplies_count.to_s + " Supplyrel. deleted for Product " + product.id.to_s +
                   " Metadatum " + self.id.to_s)
end

#destroy_permitted? ⇒ Boolean

Returns:

  • (Boolean) 


41
42
43
 
# File 'app/models/mercator_icecat/metadatum.rb', line 41

def destroy_permitted?
  acting_user.administrator?
end

#download(overwrite: false) ⇒ Object

— Instance Methods — #



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
 
# File 'app/models/mercator_icecat/metadatum.rb', line 171

def download(overwrite: false)
  unless overwrite
    return false if File.exist?(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml"))
  end

  unless self.path
    return false
  end

  # force_encoding fixes: Encoding::UndefinedConversionError: "\xC3" from ASCII-8BIT to UTF-8
  begin
    io = open(Access::BASE_URL + "/" + self.path, Access.open_uri_options).read.force_encoding('UTF-8')
    file = File.new(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml"), "w")
    io.each_line do |line|
      file.write line
    end

    file.close
    return true
  rescue
    ::JobLogger.error("Download error: " + Access::BASE_URL + "/" + self.path)
    return false
  end
end

#import_missing_image ⇒ Object 



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
 
# File 'app/models/mercator_icecat/metadatum.rb', line 360

def import_missing_image
  file = open(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml")).read
  product_nodeset = Nokogiri::XML(file).xpath("//ICECAT-interface/Product")[0]

  product = self.product
  return nil if product.photo_file_name # no overwriting intended

  path = product_nodeset["HighPic"]

  if path.empty?
    ::JobLogger.warn("no Image available for Product " + product.id.to_s)
    return nil # no image available
  end

  begin
    io = StringIO.new(open(path, Access.open_uri_options).read)
    io.class.class_eval { attr_accessor :original_filename }
    io.original_filename = path.split("/").last

    product.photo = io

    if product.save(validate: false) # FIXME!: This time without validations ...
      ::JobLogger.info("Image  " + path.split("/").last + " for Product " + product.id.to_s + " saved." )
    else
      ::JobLogger.error("Image  " + path.split("/").last + " for Product " + product.id.to_s + " could not be saved!" )
    end
  rescue Exception => e
    ::JobLogger.warn("Image  " + path + " could not be loaded!" )
    ::JobLogger.warn(e)
  end
end

#update_permitted? ⇒ Boolean

Returns:

  • (Boolean) 


37
38
39
 
# File 'app/models/mercator_icecat/metadatum.rb', line 37

def update_permitted?
  acting_user.administrator?
end

#update_product ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
 
# File 'app/models/mercator_icecat/metadatum.rb', line 196

def update_product
  # :en => lang_id = 1, :de => lang_id = 4
  file = open(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml")).read
  product_nodeset = Nokogiri::XML(file).xpath("//ICECAT-interface/Product")[0]
  product = self.product

  description_de = try_to { product_nodeset.xpath("ProductDescription[@langid='4']")[0]["ShortDesc"].fix_utf8 }
  description_en = try_to { product_nodeset.xpath("ProductDescription[@langid='1']")[0]["ShortDesc"].fix_utf8 }
  long_description_de = try_to { product_nodeset.xpath("ProductDescription[@langid='4']")[0]["LongDesc"].fix_utf8 }
  long_description_en = try_to { product_nodeset.xpath("ProductDescription[@langid='1']")[0]["LongDesc"].fix_utf8 }
  warranty_de = try_to { product_nodeset.xpath("ProductDescription[@langid='4']")[0]["WarrantyInfo"].fix_utf8 }
  warranty_en = try_to { product_nodeset.xpath("ProductDescription[@langid='1']")[0]["WarrantyInfo"].fix_utf8 }

  product.update(# title_de: product_nodeset["Title"],
                 # title_en: product_nodeset["Title"],
                 description_de: description_de,
                 description_en: description_en,
                 long_description_de: long_description_de,
                 long_description_en: long_description_en,
                 warranty_de: warranty_de,
                 warranty_en: warranty_en)

  property_groups_nodeset = product_nodeset.xpath("CategoryFeatureGroup")
  property_groups_nodeset.each do |property_group_nodeset|
    icecat_id = property_group_nodeset["ID"]
    name_en = try_to { property_group_nodeset.xpath("FeatureGroup/Name[@langid='1']")[0]["Value"].fix_utf8 }
    name_de = try_to { property_group_nodeset.xpath("FeatureGroup/Name[@langid='4']")[0]["Value"].fix_utf8 }
    name_de ||= name_en # English, if German not available
    name_de ||= try_to { property_group_nodeset.xpath("FeatureGroup/Name")[0]["Value"].fix_utf8 }
                # anything if neither German nor English available

    property_group = ::PropertyGroup.find_by_icecat_id(icecat_id)
    unless property_group
      property_group = ::PropertyGroup.new(icecat_id: icecat_id,
                                           name_de: name_de,
                                           name_en: name_en,
                                           position: icecat_id) # no better idea ...
      if property_group.save
        ::JobLogger.info("PropertyGroup " + icecat_id.to_s + " created.")
      else
        ::JobLogger.error("PropertyGroup " + icecat_id.to_s + " could not be created: " + property_group.errors.first.to_s)
      end
    end
  end

  product.values.destroy_all

  features_nodeset = product_nodeset.xpath("ProductFeature")
  features_nodeset.each do |feature|
    # icecat_presentation_value = feature.xpath("Presentation_Value") # not used here
    icecat_feature_id = feature.xpath("Feature")[0]["ID"].to_i
    icecat_value = feature["Value"]
    icecat_feature_group_id = feature["CategoryFeatureGroup_ID"]

    name_en = try_to { feature.xpath("Feature/Name[@langid='1']")[0]["Value"].fix_utf8 }
    name_de = try_to { feature.xpath("Feature/Name[@langid='4']")[0]["Value"].fix_utf8 }
    name_de ||= name_en # English, if German not available
    name_de ||= try_to { feature.xpath("Feature/Name")[0]["Value"].fix_utf8 } # anything if neither German nor English available

    unit_en = try_to { feature.xpath("Feature/Measure/Signs/Sign[@langid='1']")[0].content.fix_utf8 }
    unit_de = try_to { feature.xpath("Feature/Measure/Signs/Sign[@langid='4']")[0].content.fix_utf8 }
    unit_de ||= unit_en # English, if German not available
    unit_de ||= try_to { feature.xpath("Feature/Measure/Signs/Sign")[0].content.fix_utf8 }
                # anything if neither German nor English available

    property_group = PropertyGroup.find_by_icecat_id(icecat_feature_group_id)

    property = Property.where(icecat_id: icecat_feature_id).first
    unless property
      property = Property.new(icecat_id: icecat_feature_id,
                              position: icecat_feature_id,
                              name_de: name_de,
                              name_en: name_en,
                              datatype: icecat_value.icecat_datatype)
      if property.save
        ::JobLogger.info("Property " + property.id.to_s + " saved.")
      else
        ::JobLogger.error("Property could not be saved:" + property.errors.first.to_s)
      end
    end

    value = Value.where(property_group_id: property_group.id, property_id: property.id,
                        product_id: product.id, state: icecat_value.icecat_datatype).first
    unless value
      value = Value.new(property_group_id: property_group.id, property_id: property.id, product_id: product.id)
      value.state = icecat_value.icecat_datatype
    end

    if icecat_value.icecat_datatype == "flag"
      value.flag = ( icecat_value == "Y" )
    end

    if icecat_value.icecat_datatype == "numeric"
      value.amount = icecat_value.to_f
      value.unit_de = try_to { unit_de.fix_utf8 }
      value.unit_en = try_to { unit_en.fix_utf8 }
    end

    if icecat_value.icecat_datatype == "textual"
      value.title_de = try_to { icecat_value.truncate(252).fix_utf8 }
      value.title_en = try_to { icecat_value.truncate(252).fix_utf8 }
      value.unit_de = try_to { unit_de.fix_utf8 }
      value.unit_en = try_to { unit_en.fix_utf8 }
    end

    if value.save
      ::JobLogger.info("Value " + value.id.to_s + " saved.")
    else
      ::JobLogger.error("Value could not be saved:" + value.errors.first)
    end
  end
  ::JobLogger.info("=== Metadatum " + id.to_s + " updated Product " + product_id.to_s + " ===")
end

#update_product_relations ⇒ Object 



321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
 
# File 'app/models/mercator_icecat/metadatum.rb', line 321

def update_product_relations
  file = open(Rails.root.join("vendor","xml",icecat_product_id.to_s + ".xml")).read
  product_nodeset = Nokogiri::XML(file).xpath("//ICECAT-interface/Product")[0]
  product = self.product
  cat_id = self.cat_id

  self.delete_relations

  unknown_products = 0
  icecat_ids = []
  product_nodeset.xpath("ProductRelated").each do |relation|
    icecat_ids << try_to { relation.xpath("Product")[0]["ID"].to_i }
  end

  related_metadata = Metadatum.where(icecat_product_id: icecat_ids)
  related_metadata.each do |related_metadatum|
    related_product_id = try_to { related_metadatum.product_id.to_i }

    if related_product_id > 0
      if related_metadatum.cat_id == cat_id
        product.productrelations.new(related_product_id: related_product_id)
      else
        product.supplyrelations.new(supply_id: related_product_id)
      end
    else
      unknown_products += 1
    end
  end

  if product.save(validate: false) # FIXME!: This time without validations ...
    ::JobLogger.info("Product " + product.id.to_s + ": " +
                     product.productrelations.count.to_s + " Productrel. " +
                     product.supplyrelations.count.to_s + ", Supplyrel. created, " +
                     unknown_products.to_s + " unknown.")
  else
    ::JobLogger.error("Product " + product.id.to_s + " could not be updated")
  end
end

#view_permitted?(field) ⇒ Boolean

Returns:

  • (Boolean) 


45
46
47
 
# File 'app/models/mercator_icecat/metadatum.rb', line 45

def view_permitted?(field)
  true
end