Class: Bulkrax::ApplicationParser
Abstract
- Inherits:
-
Object
- Object
- Bulkrax::ApplicationParser
show all
- Defined in:
- app/parsers/bulkrax/application_parser.rb
Overview
This class is abstract.
Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
An abstract class that establishes the API for Bulkrax’s import and export parsing.
Instance Attribute Summary collapse
Class Method Summary
collapse
Instance Method Summary
collapse
-
#base_path(type = 'import') ⇒ String
Base path for imported and exported files.
-
#collection_entry_class ⇒ Object
abstract
-
#collections_total ⇒ Object
-
#create_collections ⇒ Object
abstract
-
#create_file_sets ⇒ Object
abstract
-
#create_objects(types = []) ⇒ Object
-
#create_relationships ⇒ Object
abstract
-
#create_works ⇒ Object
abstract
-
#entry_class ⇒ Object
abstract
-
#exporter? ⇒ TrueClass, FalseClass
-
#file_set_entry_class ⇒ Object
abstract
-
#file_sets_total ⇒ Object
-
#find_or_create_entry(entryclass, identifier, type, raw_metadata = nil) ⇒ Object
-
#generated_metadata_mapping ⇒ String
-
#get_field_mapping_hash_for(key) ⇒ Object
private
-
#import_file_path ⇒ String
-
#importer? ⇒ TrueClass, FalseClass
-
#initialize(importerexporter) ⇒ ApplicationParser
constructor
A new instance of ApplicationParser.
-
#invalid_record(message) ⇒ Object
rubocop:disable Rails/SkipsModelValidations.
-
#limit_reached?(limit, index) ⇒ TrueClass, FalseClass
-
#model_field_mappings ⇒ Array<String>
-
#new_entry(entryclass, type) ⇒ Object
-
#path_for_import ⇒ String
Path where we’ll store the import metadata and files this is used for uploaded and cloud files.
-
#perform_method ⇒ String
-
#record(identifier, _opts = {}) ⇒ Object
-
#record_has_source_identifier(record, index) ⇒ TrueClass, FalseClass
-
#records(_opts = {}) ⇒ Object
abstract
-
#related_children_parsed_mapping ⇒ String
-
#related_children_raw_mapping ⇒ String, NilClass
-
#related_parents_parsed_mapping ⇒ String
-
#related_parents_raw_mapping ⇒ String, NilClass
-
#required_elements ⇒ Array<String>
-
#retrieve_cloud_files(files) ⇒ Object
Optional, define if using browse everything for file upload.
-
#setup_export_file ⇒ Object
abstract
-
#source_identifier ⇒ Symbol
importing (e.g. is not this application that mounts this Bulkrax engine).
-
#total ⇒ Object
-
#unzip(file_to_unzip) ⇒ Object
-
#valid_import? ⇒ TrueClass, FalseClass
Override to add specific validations.
-
#visibility ⇒ String
The visibility of the record.
-
#work_identifier ⇒ Symbol
The name of the identifying property for the system which we’re importing into (e.g. the application that mounts this Bulkrax engine).
-
#write ⇒ Object
-
#write_files ⇒ Object
abstract
-
#write_import_file(file) ⇒ Object
-
#zip ⇒ Object
Constructor Details
Returns a new instance of ApplicationParser.
39
40
41
42
|
# File 'app/parsers/bulkrax/application_parser.rb', line 39
def initialize(importerexporter)
@importerexporter = importerexporter
@headers = []
end
|
Instance Attribute Details
rubocop:disable Metrics/ClassLength
10
11
12
|
# File 'app/parsers/bulkrax/application_parser.rb', line 10
def
@headers
end
|
#importerexporter ⇒ Object
Also known as:
importer, exporter
rubocop:disable Metrics/ClassLength
10
11
12
|
# File 'app/parsers/bulkrax/application_parser.rb', line 10
def importerexporter
@importerexporter
end
|
Class Method Details
.export_supported? ⇒ TrueClass, FalseClass
TODO:
Convert to ‘class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
Returns this parser does or does not support exports.
28
29
30
|
# File 'app/parsers/bulkrax/application_parser.rb', line 28
def self.export_supported?
false
end
|
.import_supported? ⇒ TrueClass, FalseClass
TODO:
Convert to ‘class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
Returns this parser does or does not support imports.
35
36
37
|
# File 'app/parsers/bulkrax/application_parser.rb', line 35
def self.import_supported?
true
end
|
.parser_fields ⇒ Object
TODO:
Convert to ‘class_attribute :parser_fiels, default: {}`
21
22
23
|
# File 'app/parsers/bulkrax/application_parser.rb', line 21
def self.parser_fields
{}
end
|
Instance Method Details
#base_path(type = 'import') ⇒ String
Base path for imported and exported files
205
206
207
208
209
|
# File 'app/parsers/bulkrax/application_parser.rb', line 205
def base_path(type = 'import')
is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
end
|
#collection_entry_class ⇒ Object
52
53
54
|
# File 'app/parsers/bulkrax/application_parser.rb', line 52
def collection_entry_class
raise NotImplementedError, 'must be defined'
end
|
#collections_total ⇒ Object
324
325
326
|
# File 'app/parsers/bulkrax/application_parser.rb', line 324
def collections_total
0
end
|
#create_collections ⇒ Object
169
170
171
|
# File 'app/parsers/bulkrax/application_parser.rb', line 169
def create_collections
raise NotImplementedError, 'must be defined' if importer?
end
|
#create_file_sets ⇒ Object
179
180
181
|
# File 'app/parsers/bulkrax/application_parser.rb', line 179
def create_file_sets
raise NotImplementedError, 'must be defined' if importer?
end
|
#create_objects(types = []) ⇒ Object
162
163
164
165
166
|
# File 'app/parsers/bulkrax/application_parser.rb', line 162
def create_objects(types = [])
types.each do |object_type|
send("create_#{object_type.pluralize}")
end
end
|
#create_relationships ⇒ Object
184
185
186
|
# File 'app/parsers/bulkrax/application_parser.rb', line 184
def create_relationships
raise NotImplementedError, 'must be defined' if importer?
end
|
#create_works ⇒ Object
This method is abstract.
Subclass and override #create_works to implement behavior for the parser.
174
175
176
|
# File 'app/parsers/bulkrax/application_parser.rb', line 174
def create_works
raise NotImplementedError, 'must be defined' if importer?
end
|
#entry_class ⇒ Object
This method is abstract.
Subclass and override #entry_class to implement behavior for the parser.
46
47
48
|
# File 'app/parsers/bulkrax/application_parser.rb', line 46
def entry_class
raise NotImplementedError, 'must be defined'
end
|
#exporter? ⇒ TrueClass, FalseClass
236
237
238
|
# File 'app/parsers/bulkrax/application_parser.rb', line 236
def exporter?
importerexporter.is_a?(Bulkrax::Exporter)
end
|
#file_set_entry_class ⇒ Object
58
59
60
|
# File 'app/parsers/bulkrax/application_parser.rb', line 58
def file_set_entry_class
raise NotImplementedError, 'must be defined'
end
|
#file_sets_total ⇒ Object
328
329
330
|
# File 'app/parsers/bulkrax/application_parser.rb', line 328
def file_sets_total
0
end
|
#find_or_create_entry(entryclass, identifier, type, raw_metadata = nil) ⇒ Object
300
301
302
303
304
305
306
307
308
309
|
# File 'app/parsers/bulkrax/application_parser.rb', line 300
def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
entry = entryclass.where(
importerexporter_id: importerexporter.id,
importerexporter_type: type,
identifier: identifier
).first_or_create!
entry.raw_metadata = raw_metadata
entry.save!
entry
end
|
85
86
87
|
# File 'app/parsers/bulkrax/application_parser.rb', line 85
def generated_metadata_mapping
@generated_metadata_mapping ||= 'generated'
end
|
#get_field_mapping_hash_for(key) ⇒ Object
This method is part of a private API.
You should avoid using this method if possible, as it may be removed or be changed in the future.
114
115
116
117
118
119
120
121
122
123
124
125
|
# File 'app/parsers/bulkrax/application_parser.rb', line 114
def get_field_mapping_hash_for(key)
return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
instance_variable_set(
"@#{key}_hash",
mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
)
raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
instance_variable_get("@#{key}_hash")
end
|
#import_file_path ⇒ String
364
365
366
|
# File 'app/parsers/bulkrax/application_parser.rb', line 364
def import_file_path
@import_file_path ||= real_import_file_path
end
|
#importer? ⇒ TrueClass, FalseClass
231
232
233
|
# File 'app/parsers/bulkrax/application_parser.rb', line 231
def importer?
importerexporter.is_a?(Bulkrax::Importer)
end
|
#invalid_record(message) ⇒ Object
rubocop:disable Rails/SkipsModelValidations
269
270
271
272
273
274
275
|
# File 'app/parsers/bulkrax/application_parser.rb', line 269
def invalid_record(message)
current_run.invalid_records ||= ""
current_run.invalid_records += message
current_run.save
ImporterRun.increment_counter(:failed_records, current_run.id)
ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 end
|
#limit_reached?(limit, index) ⇒ TrueClass, FalseClass
243
244
245
246
|
# File 'app/parsers/bulkrax/application_parser.rb', line 243
def limit_reached?(limit, index)
return false if limit.nil? || limit.zero? index >= limit
end
|
#model_field_mappings ⇒ Array<String>
128
129
130
131
132
133
|
# File 'app/parsers/bulkrax/application_parser.rb', line 128
def model_field_mappings
model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
model_mappings |= ['model']
model_mappings
end
|
#new_entry(entryclass, type) ⇒ Object
293
294
295
296
297
298
|
# File 'app/parsers/bulkrax/application_parser.rb', line 293
def new_entry(entryclass, type)
entryclass.new(
importerexporter_id: importerexporter.id,
importerexporter_type: type
)
end
|
#path_for_import ⇒ String
Path where we’ll store the import metadata and files
this is used for uploaded and cloud files
214
215
216
217
218
|
# File 'app/parsers/bulkrax/application_parser.rb', line 214
def path_for_import
@path_for_import = File.join(base_path, importerexporter.path_string)
FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
@path_for_import
end
|
136
137
138
139
140
141
142
|
# File 'app/parsers/bulkrax/application_parser.rb', line 136
def perform_method
if self.validate_only
'perform_now'
else
'perform_later'
end
end
|
#record(identifier, _opts = {}) ⇒ Object
312
313
314
315
316
317
318
|
# File 'app/parsers/bulkrax/application_parser.rb', line 312
def record(identifier, _opts = {})
return @record if @record
@record = entry_class.new(self, identifier)
@record.build
return @record
end
|
#record_has_source_identifier(record, index) ⇒ TrueClass, FalseClass
255
256
257
258
259
260
261
262
263
264
265
266
|
# File 'app/parsers/bulkrax/application_parser.rb', line 255
def record_has_source_identifier(record, index)
if record[source_identifier].blank?
if Bulkrax.fill_in_blank_source_identifiers.present?
record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
else
invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
false
end
else
true
end
end
|
#records(_opts = {}) ⇒ Object
This method is abstract.
Subclass and override #records to implement behavior for the parser.
64
65
66
|
# File 'app/parsers/bulkrax/application_parser.rb', line 64
def records(_opts = {})
raise NotImplementedError, 'must be defined'
end
|
109
110
111
|
# File 'app/parsers/bulkrax/application_parser.rb', line 109
def related_children_parsed_mapping
@related_children_parsed_mapping ||= (get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children')
end
|
103
104
105
|
# File 'app/parsers/bulkrax/application_parser.rb', line 103
def related_children_raw_mapping
@related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
end
|
97
98
99
|
# File 'app/parsers/bulkrax/application_parser.rb', line 97
def related_parents_parsed_mapping
@related_parents_parsed_mapping ||= (get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents')
end
|
91
92
93
|
# File 'app/parsers/bulkrax/application_parser.rb', line 91
def related_parents_raw_mapping
@related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
end
|
#required_elements ⇒ Array<String>
279
280
281
282
283
284
285
286
287
288
289
290
291
|
# File 'app/parsers/bulkrax/application_parser.rb', line 279
def required_elements
matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
unless matched_elements.count == Bulkrax.required_elements.count
missing_elements = Bulkrax.required_elements - matched_elements
error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
raise StandardError, error_alert
end
if Bulkrax.fill_in_blank_source_identifiers
Bulkrax.required_elements
else
Bulkrax.required_elements + [source_identifier]
end
end
|
#retrieve_cloud_files(files) ⇒ Object
Optional, define if using browse everything for file upload
189
|
# File 'app/parsers/bulkrax/application_parser.rb', line 189
def retrieve_cloud_files(files); end
|
#setup_export_file ⇒ Object
221
222
223
|
# File 'app/parsers/bulkrax/application_parser.rb', line 221
def setup_export_file
raise NotImplementedError, 'must be defined' if exporter?
end
|
#source_identifier ⇒ Symbol
importing (e.g. is not this application that mounts this Bulkrax engine).
73
74
75
|
# File 'app/parsers/bulkrax/application_parser.rb', line 73
def source_identifier
@source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
end
|
#total ⇒ Object
320
321
322
|
# File 'app/parsers/bulkrax/application_parser.rb', line 320
def total
0
end
|
#unzip(file_to_unzip) ⇒ Object
337
338
339
340
341
342
343
344
345
|
# File 'app/parsers/bulkrax/application_parser.rb', line 337
def unzip(file_to_unzip)
Zip::File.open(file_to_unzip) do |zip_file|
zip_file.each do |entry|
entry_path = File.join(importer_unzip_path, entry.name)
FileUtils.mkdir_p(File.dirname(entry_path))
zip_file.(entry, entry_path) unless File.exist?(entry_path)
end
end
end
|
#valid_import? ⇒ TrueClass, FalseClass
Override to add specific validations
250
251
252
|
# File 'app/parsers/bulkrax/application_parser.rb', line 250
def valid_import?
true
end
|
#visibility ⇒ String
The visibility of the record. Acceptable values are: “open”, “embaro”, “lease”, “authenticated”, “restricted”. The default is “open”
149
150
151
|
# File 'app/parsers/bulkrax/application_parser.rb', line 149
def visibility
@visibility ||= self.parser_fields['visibility'] || 'open'
end
|
#work_identifier ⇒ Symbol
Returns the name of the identifying property for the system which we’re importing into (e.g. the application that mounts this Bulkrax engine).
80
81
82
|
# File 'app/parsers/bulkrax/application_parser.rb', line 80
def work_identifier
@work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
end
|
#write ⇒ Object
332
333
334
335
|
# File 'app/parsers/bulkrax/application_parser.rb', line 332
def write
write_files
zip
end
|
#write_files ⇒ Object
This method is abstract.
Subclass and override #write_files to implement behavior for the parser.
226
227
228
|
# File 'app/parsers/bulkrax/application_parser.rb', line 226
def write_files
raise NotImplementedError, 'must be defined' if exporter?
end
|
#write_import_file(file) ⇒ Object
193
194
195
196
197
198
199
200
|
# File 'app/parsers/bulkrax/application_parser.rb', line 193
def write_import_file(file)
path = File.join(path_for_import, file.original_filename)
FileUtils.mv(
file.path,
path
)
path
end
|
#zip ⇒ Object
347
348
349
350
351
352
353
354
355
356
357
358
359
360
|
# File 'app/parsers/bulkrax/application_parser.rb', line 347
def zip
FileUtils.mkdir_p(exporter_export_zip_path)
Dir["#{exporter_export_path}/**"].each do |folder|
zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
Dir["#{folder}/**/**"].each do |file|
zip_file.add(file.sub("#{folder}/", ''), file)
end
end
end
end
|