Class: DS::Manifest::ManifestValidator

Inherits:
Object
  • Object
show all
Includes:
Constants
Defined in:
lib/ds/manifest/manifest_validator.rb

Overview

TODO:

Add test for live URLs

Validate a DS input manifest.

Validation does the following:

- Confirms all required columns are present
- Confirms all all required values are present
- Confirms all column values are the correct type
- Confirms all listed input files are present
- Confirms all listed input files match the record
identifier provided in the manifest

Constant Summary collapse

URI_REGEXP = 
URI::DEFAULT_PARSER.make_regexp %w{http https}
QID_REGEXP = 
%r{\AQ\d+\z}

Constants included from Constants

Constants::CALL_NUMBER, Constants::DATED, Constants::DATE_TIME_COLUMNS, Constants::DS_ID, Constants::FILENAME, Constants::IIIF_MANIFEST_URL, Constants::INSTITUTIONAL_ID, Constants::INSTITUTIONAL_ID_LOCATION_IN_SOURCE, Constants::INSTITUTION_DS_QID, Constants::INSTITUTION_WIKIDATA_LABEL, Constants::LINK_TO_INSTITUTIONAL_RECORD, Constants::MANIFEST_COLUMNS, Constants::MANIFEST_GENERATED_AT, Constants::QID_COLUMNS, Constants::RECORD_LAST_UPDATED, Constants::REQUIRED_VALUES, Constants::SOURCE_TYPE, Constants::TITLE, Constants::URI_COLUMNS

Constants included from DS

VERSION

Constants included from Constants

Constants::DS_CSV, Constants::DS_METS, Constants::HEADINGS, Constants::INSTITUTIONS, Constants::INSTITUTION_DS_IDS, Constants::MARC_XML, Constants::MAX_WIKIBASE_FIELD_LENGTH, Constants::NESTED_COLUMNS, Constants::TEI_XML, Constants::TRAILING_PUNCTUATION_RE, Constants::VALID_SOURCE_TYPES, Constants::XML_NAMESPACES

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from DS

configure!, data_dir, env, env=, normalize_key, root

Methods included from ClassMethods

#adjust_for_century, #calculate_century, #configure_logger_for, #logger, #logger_for, #lookup_century, #mark_long, #timestamp, #transform_centuries_to_aat, #transform_dates_to_centuries

Constructor Details

#initialize(manifest) ⇒ DS::ManifestValidator

Parameters:



36
37
38
39
40
 
# File 'lib/ds/manifest/manifest_validator.rb', line 36

def initialize manifest
  @manifest = manifest
  @errors   = []
  @id_validators = {}
end

Instance Attribute Details

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



28
29
30
 
# File 'lib/ds/manifest/manifest_validator.rb', line 28

def errors
  @errors
end

#manifest ⇒ Object (readonly)

Returns the value of attribute manifest.



26
27
28
 
# File 'lib/ds/manifest/manifest_validator.rb', line 26

def manifest
  @manifest
end

#source_dir ⇒ Object (readonly)

Returns the value of attribute source_dir.



27
28
29
 
# File 'lib/ds/manifest/manifest_validator.rb', line 27

def source_dir
  @source_dir
end

Instance Method Details

#add_error(message) ⇒ void

This method returns an undefined value.

Adds an error message to the list of errors.

Parameters:

  • message (String) —

    the error message to add



220
221
222
 
# File 'lib/ds/manifest/manifest_validator.rb', line 220

def add_error message
  @errors << message
end

#get_id_validator(source_type) ⇒ DS::Manifest::BaseIdValidator

Retrieves the appropriate ID validator for the given source type.

Parameters:

  • source_type (Symbol) —

    the type of the source

Returns:

Raises:

  • (NotImplementedError) —

    if the source type is not implemented



236
237
238
239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/ds/manifest/manifest_validator.rb', line 236

def get_id_validator source_type
  case source_type
  when DS::Constants::MARC_XML
    @id_validators[source_type] ||= SimpleXmlIdValidator.new(DS::Source::MarcXML.new)
  when DS::Constants::DS_METS
    @id_validators[source_type] ||= SimpleXmlIdValidator.new(DS::Source::DSMetsXML.new)
  when DS::Constants::TEI_XML
    @id_validators[source_type] ||= SimpleXmlIdValidator.new(DS::Source::TeiXML.new)
  when DS::Constants::DS_CSV
    @id_validators[source_type] ||= DsCsvIdValidator.new(DS::Source::DSCSV.new)
  else
    raise NotImplementedError, "validate_ids not implemented for: #{source_type}"
  end
end

#handle_count_error(count, inst_id, location_in_source) ⇒ nil

Handles the error when the count of records found for a given inst_id and location_in_source is 0 or more than 1.

Parameters:

  • count (Integer) —

    the number of records found for the given inst_id and location_in_source

  • inst_id (String) —

    the identifier of the record

  • location_in_source (String) —

    the location in the source where the record is found

Returns:

  • (nil) 


156
157
158
159
160
161
162
163
164
165
 
# File 'lib/ds/manifest/manifest_validator.rb', line 156

def handle_count_error count, inst_id, location_in_source
  return if count == 1

  if count > 1
    add_error "ERROR: Multiple records (#{count}) found for id: #{inst_id} (location: #{location_in_source})"
  elsif count == 0
    add_error "ERROR: No records found for id: #{inst_id} (location: #{location_in_source})"
  end
  nil
end

#has_errors? ⇒ Boolean

Checks if there are any errors in the errors collection.

Returns:

  • (Boolean) —

    true if there are errors, false otherwise



227
228
229
 
# File 'lib/ds/manifest/manifest_validator.rb', line 227

def has_errors?
  errors.any?
end

#source_types ⇒ Object 



251
252
253
 
# File 'lib/ds/manifest/manifest_validator.rb', line 251

def source_types
  VALID_SOURCE_TYPES
end

#valid? ⇒ boolean

Returns true if the manifest is valid.

Returns:

  • (boolean) —

    true if the manifest is valid



44
45
46
47
48
49
50
51
52
 
# File 'lib/ds/manifest/manifest_validator.rb', line 44

def valid?
  return false unless validate_columns
  return false unless validate_required_values
  return false unless validate_ids_unique
  return false unless validate_data_types
  return false unless validate_files_exist
  return false unless validate_records_present
  true
end

#validate_columns ⇒ boolean

Returns true if all required columns are present.

Returns:

  • (boolean) —

    true if all required columns are present



56
57
58
59
60
61
62
 
# File 'lib/ds/manifest/manifest_validator.rb', line 56

def validate_columns
  found_columns = manifest.headers
  diff          = MANIFEST_COLUMNS - found_columns
  return true if diff.blank?
  add_error "Manifest missing required columns: #{diff.join ', '}" if diff.present?
  false
end

#validate_data_types ⇒ boolean

Returns true if all data types are valid.

Returns:

  • (boolean) —

    true if all data types are valid



81
82
83
84
85
86
87
88
89
 
# File 'lib/ds/manifest/manifest_validator.rb', line 81

def validate_data_types
  is_valid = true
  manifest.each_with_index do |entry, row_num|
    is_valid = false unless validate_urls entry, row_num
    is_valid = false unless validate_qids entry, row_num
    is_valid = false unless validate_dates entry, row_num
  end
  is_valid
end

#validate_dates(entry, row_num) ⇒ Object 



202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/ds/manifest/manifest_validator.rb', line 202

def validate_dates entry, row_num
  is_valid = true
  DATE_TIME_COLUMNS.each do |col|
    next if entry[col].blank?
    begin
      Date.parse entry[col]
    rescue Date::Error
      is_valid = false
      add_error "Invalid date in row: #{row_num+1}, col.: #{col}: '#{entry[col]}'"
    end
  end
  is_valid
end

#validate_files_exist ⇒ boolean

Returns true if all list input files are present.

Returns:

  • (boolean) —

    true if all list input files are present



93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/ds/manifest/manifest_validator.rb', line 93

def validate_files_exist
  is_valid = true
  manifest.each_with_index do |entry, row_num|
    file_path = File.join manifest.source_dir, entry.filename
    unless File.exist? file_path
      is_valid = false
      add_error "Source file not found row: #{row_num+1}; source directory: #{source_dir}; file: #{entry.filename}"
    end
  end
  is_valid
end

#validate_ids_unique ⇒ Object

Validates the uniqueness of all IDs in the manifest.

This method collects the count of all IDs in the manifest and selects those with a count greater than 1. It then iterates over the multiples and adds an error for each duplicate ID found.

Returns:

  • true if no duplicate IDs are found.
  • false if duplicate IDs are found.


113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/ds/manifest/manifest_validator.rb', line 113

def validate_ids_unique
  # collect the count of all ids and select those with a count > 1
  multiples = manifest.inject({}) { |h, id|
    h[id] ||= 0; h[id] += 1; h
  }.filter_map { |id, count|
    [id, count] if count > 1
  }

  return true if multiples.blank?

  multiples.each do |id, count|
    add_error "Duplicate ID found in manifest: ID '#{id}' found in #{count} rows"
  end
  false
end

#validate_qids(entry, row_num) ⇒ Object 



191
192
193
194
195
196
197
198
199
200
 
# File 'lib/ds/manifest/manifest_validator.rb', line 191

def validate_qids entry, row_num
  is_valid = true
  QID_COLUMNS.each do |col|
    unless entry[col].to_s =~ QID_REGEXP
      is_valid = false
      add_error "Invalid QID in row: #{row_num+1}; col.: #{col}: '#{entry[col]}'"
    end
  end
  is_valid
end

#validate_records_present ⇒ boolean

Returns true if all holding_institution_institutional_id values match source file.

Returns:

  • (boolean) —

    true if all holding_institution_institutional_id values match source file



132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/ds/manifest/manifest_validator.rb', line 132

def validate_records_present
  is_valid = true
  manifest.each_with_index do |entry, row_num|
    file_path = File.join manifest.source_dir, entry.filename

    inst_id      = entry.institutional_id
    id_validator = get_id_validator entry.source_type
    found        = id_validator.valid? file_path, inst_id, entry.institutional_id_location_in_source

    unless found
      is_valid = false
      id_validator.errors.each { |error| add_error error }
    end
  end
  is_valid
end

#validate_required_values ⇒ boolean

Returns true if all require values present.

Returns:

  • (boolean) —

    true if all require values present



66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/ds/manifest/manifest_validator.rb', line 66

def validate_required_values
  is_valid = true
  manifest.each_with_index do |row, ndx|
    REQUIRED_VALUES.each do |col|
      if row[col].blank?
        add_error "Required value missing in row: #{ndx+1}, col.: #{col}"
        is_valid = false
      end
    end
  end
  is_valid
end

#validate_source_type(entry, row_num) ⇒ Object

Type validations



170
171
172
173
174
175
176
177
178
 
# File 'lib/ds/manifest/manifest_validator.rb', line 170

def validate_source_type entry, row_num
  is_valid = true

  unless source_types.include? entry.source_type
    add_error "Invalid source type in row: #{row_num+1}; expected one of #{VALID_SOURCE_TYPES.join ', '}; got: '#{entry.source_type}'"
    is_valid = false
  end
  is_valid
end

#validate_urls(entry, row_num) ⇒ Object 



180
181
182
183
184
185
186
187
188
189
 
# File 'lib/ds/manifest/manifest_validator.rb', line 180

def validate_urls entry, row_num
  is_valid = true
  URI_COLUMNS.each do |col|
    if entry[col].present? && entry[col].to_s !~ URI_REGEXP
      add_error "Invalid URL in row: #{row_num+1}; col.: #{col}: '#{entry[col]}'"
      is_valid = false
    end
  end
  is_valid
end