Class: Bulkrax::Importer

Inherits:
ApplicationRecord show all
Includes:
ImporterExporterBehavior, StatusInfo
Defined in:
app/models/bulkrax/importer.rb

Overview

rubocop:disable Metrics/ClassLength

Constant Summary collapse

DEFAULT_OBJECT_TYPES = 
%w[collection work file_set relationship].freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from StatusInfo

#current_status, #failed?, #last_error, #set_status_info, #skipped?, #status_at, #succeeded?

Methods included from ImporterExporterBehavior

#file?, #increment_counters, #key_without_numbers, #keys_without_numbers, #last_imported_at, #next_import_at, #parser, #parser_class, #zip?

Instance Attribute Details

#current_run(skip_counts: false) ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'app/models/bulkrax/importer.rb', line 104

def current_run(skip_counts: false)
  return @current_run if @current_run.present?

  @current_run = self.importer_runs.create!
  return @current_run if file? && zip?
  return @current_run if skip_counts

  entry_counts = {
    total_work_entries: self.limit || parser.works_total,
    total_collection_entries: parser.collections_total,
    total_file_set_entries: parser.file_sets_total
  }
  @current_run.update!(entry_counts)

  @current_run
end

#file ⇒ Object

Returns the value of attribute file.



21
22
23
 
# File 'app/models/bulkrax/importer.rb', line 21

def file
  @file
end

#file_style ⇒ Object

Returns the value of attribute file_style.



21
22
23
 
# File 'app/models/bulkrax/importer.rb', line 21

def file_style
  @file_style
end

#only_updates ⇒ Object

Returns the value of attribute only_updates.



21
22
23
 
# File 'app/models/bulkrax/importer.rb', line 21

def only_updates
  @only_updates
end

Class Method Details

.frequency_enums ⇒ Object 



84
85
86
87
88
89
 
# File 'app/models/bulkrax/importer.rb', line 84

def self.frequency_enums
  # these duration values use ISO 8601 Durations (https://en.wikipedia.org/wiki/ISO_8601#Durations)
  # TLDR; all durations are prefixed with 'P' and the parts are a number with the type of duration.
  # i.e. P1Y2M3W4DT5H6M7S == 1 Year, 2 Months, 3 Weeks, 4 Days, 5 Hours, 6 Minutes, 7 Seconds
  [['Daily', 'P1D'], ['Monthly', 'P1M'], ['Yearly', 'P1Y'], ['Once (on save)', 'PT0S']]
end

.safe_uri_filename(uri) ⇒ Object 



24
25
26
27
28
29
30
31
 
# File 'app/models/bulkrax/importer.rb', line 24

def self.safe_uri_filename(uri)
  r = Faraday.head(uri.to_s)
  return CGI.parse(r.headers['content-disposition'])["filename"][0].delete("\"")
rescue
  filename = File.basename(uri.to_s)
  filename.delete!('/')
  filename.presence || SecureRandom.uuid
end

Instance Method Details

#completed_statuses ⇒ Object 



142
143
144
145
146
 
# File 'app/models/bulkrax/importer.rb', line 142

def completed_statuses
  @completed_statuses ||= Bulkrax::Status.latest_by_statusable
                                         .includes(:statusable)
                                         .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Complete')
end

#default_field_mapping ⇒ Object 



70
71
72
73
74
75
76
77
78
 
# File 'app/models/bulkrax/importer.rb', line 70

def default_field_mapping
  return self.field_mapping if parser.import_fields.nil?

  ActiveSupport::HashWithIndifferentAccess.new(
    parser.import_fields.reject(&:nil?).map do |m|
      Bulkrax.default_field_mapping.call(m)
    end.inject(:merge)
  )
end

#errored_entries_csv_path ⇒ Object 



229
230
231
 
# File 'app/models/bulkrax/importer.rb', line 229

def errored_entries_csv_path
  @errored_entries_csv_path ||= File.join(parser.base_path, "import_#{path_string}_errored_entries.csv")
end

#existing_entries? ⇒ Boolean

Returns:

  • (Boolean) 


168
169
170
 
# File 'app/models/bulkrax/importer.rb', line 168

def existing_entries?
  parser.parser_fields['file_style']&.match(/Existing Entries/)
end

#failed_entries? ⇒ Boolean

Returns:

  • (Boolean) 


125
126
127
 
# File 'app/models/bulkrax/importer.rb', line 125

def failed_entries?
  entries.failed.any?
end

#failed_messages ⇒ Object 



135
136
137
138
139
140
 
# File 'app/models/bulkrax/importer.rb', line 135

def failed_messages
  failed_statuses.each_with_object({}) do |e, i|
    i[e.error_message] ||= []
    i[e.error_message] << e.id
  end
end

#failed_statuses ⇒ Object 



129
130
131
132
133
 
# File 'app/models/bulkrax/importer.rb', line 129

def failed_statuses
  @failed_statuses ||= Bulkrax::Status.latest_by_statusable
                                      .includes(:statusable)
                                      .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Failed')
end

#frequency ⇒ Object 



95
96
97
98
 
# File 'app/models/bulkrax/importer.rb', line 95

def frequency
  f = self[:frequency] || "PT0S"
  ISO8601::Duration.new(f)
end

#frequency=(frequency) ⇒ Object 



91
92
93
 
# File 'app/models/bulkrax/importer.rb', line 91

def frequency=(frequency)
  self[:frequency] = ISO8601::Duration.new(frequency).to_s
end

#import_collections ⇒ Object 



176
177
178
 
# File 'app/models/bulkrax/importer.rb', line 176

def import_collections
  import_objects(['collection'])
end

#import_file_sets ⇒ Object 



180
181
182
 
# File 'app/models/bulkrax/importer.rb', line 180

def import_file_sets
  import_objects(['file_set'])
end

#import_metadata_format ⇒ Object

The format for metadata for the incoming import; corresponds to an Entry class



215
216
217
 
# File 'app/models/bulkrax/importer.rb', line 215

def import_metadata_format
  [['CSV', 'Bulkrax::CsvEntry'], ['RDF (N-Triples)', 'Bulkrax::RdfEntry']]
end

#import_objects(types_array = nil) ⇒ Object 



190
191
192
193
194
195
196
197
198
 
# File 'app/models/bulkrax/importer.rb', line 190

def import_objects(types_array = nil)
  self.only_updates ||= false
  self.save if self.new_record? # Object needs to be saved for statuses
  types = types_array || DEFAULT_OBJECT_TYPES
  existing_entries? ? parser.rebuild_entries(types) : parser.create_objects(types)
  mark_unseen_as_skipped
rescue StandardError => e
  set_status_info(e)
end

#import_relationships ⇒ Object 



184
185
186
 
# File 'app/models/bulkrax/importer.rb', line 184

def import_relationships
  import_objects(['relationship'])
end

#import_works ⇒ Object 



172
173
174
 
# File 'app/models/bulkrax/importer.rb', line 172

def import_works
  import_objects(['work'])
end

#importer_unzip_path ⇒ Object

If the import data is zipped, unzip it to this path



225
226
227
 
# File 'app/models/bulkrax/importer.rb', line 225

def importer_unzip_path
  @importer_unzip_path ||= File.join(parser.base_path, "import_#{path_string}")
end

#last_run ⇒ Object 



121
122
123
 
# File 'app/models/bulkrax/importer.rb', line 121

def last_run
  @last_run ||= self.importer_runs.last
end

#mapping ⇒ Object

If field_mapping is empty, setup a default based on the export_properties



57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'app/models/bulkrax/importer.rb', line 57

def mapping
  # rubocop:disable Style/IfUnlessModifier
  @mapping ||= if self.field_mapping.blank? || self.field_mapping == [{}]
                 if parser.import_fields.present? || self.field_mapping == [{}]
                   default_field_mapping
                 end
               else
                 default_field_mapping.merge(self.field_mapping)
               end

  # rubocop:enable Style/IfUnlessModifier
end

#mark_unseen_as_skipped ⇒ Object

After an import any entries we did not touch are skipped. They are not really pending, complete for the last run, or failed



202
203
204
205
206
 
# File 'app/models/bulkrax/importer.rb', line 202

def mark_unseen_as_skipped
  entries.where.not(identifier: seen.keys).find_each do |entry|
    entry.set_status_info('Skipped')
  end
end

#metadata_only? ⇒ Boolean

Returns:

  • (Boolean) 


164
165
166
 
# File 'app/models/bulkrax/importer.rb', line 164

def metadata_only?
  parser.parser_fields['metadata_only'] == true
end

#parser_fields ⇒ Object 



80
81
82
 
# File 'app/models/bulkrax/importer.rb', line 80

def parser_fields
  self[:parser_fields] || {}
end

#path_string ⇒ Object 



233
234
235
236
237
 
# File 'app/models/bulkrax/importer.rb', line 233

def path_string
  "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}_#{self.importer_runs.last.id}"
rescue
  "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
end

#record_status ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'app/models/bulkrax/importer.rb', line 41

def record_status
  importer_run = ImporterRun.find(current_run.id) # make sure fresh
  return if importer_run.enqueued_records.positive? # still processing
  if importer_run.failed_records.positive?
    if importer_run.invalid_records.present?
      e = Bulkrax::ImportFailed.new('Failed with Invalid Records', importer_run.invalid_records.split("\n"))
      importer_run.importer.set_status_info(e)
    else
      importer_run.importer.set_status_info('Complete (with failures)')
    end
  else
    importer_run.importer.set_status_info('Complete')
  end
end

#remove_and_rerun ⇒ Object 



160
161
162
 
# File 'app/models/bulkrax/importer.rb', line 160

def remove_and_rerun
  self.parser_fields['remove_and_rerun']
end

#replace_files ⇒ Object 



152
153
154
 
# File 'app/models/bulkrax/importer.rb', line 152

def replace_files
  self.parser_fields['replace_files']
end

#schedulable? ⇒ Boolean

Returns:

  • (Boolean) 


100
101
102
 
# File 'app/models/bulkrax/importer.rb', line 100

def schedulable?
  frequency.to_seconds != 0
end

#seen ⇒ Object 



148
149
150
 
# File 'app/models/bulkrax/importer.rb', line 148

def seen
  @seen ||= {}
end

#status ⇒ Object 



33
34
35
36
37
38
39
 
# File 'app/models/bulkrax/importer.rb', line 33

def status
  if self.validate_only
    'Validated'
  else
    super
  end
end

#unique_collection_identifier(id) ⇒ Object

TODO:

  • move to parser, as this is OAI specific

Prepend the base_url to ensure unique set identifiers



210
211
212
 
# File 'app/models/bulkrax/importer.rb', line 210

def unique_collection_identifier(id)
  "#{self.parser_fields['base_url'].split('/')[2]}_#{id}"
end

#update_files ⇒ Object 



156
157
158
 
# File 'app/models/bulkrax/importer.rb', line 156

def update_files
  self.parser_fields['update_files']
end