Class: UDAMUtils::WorkflowPublishMapProcessor

Inherits:
BasePublishMapProcessor show all
Defined in:
lib/udam_utils/publish_map_processor.rb

Instance Attribute Summary collapse

Attributes inherited from BasePublishMapProcessor

#logger

Instance Method Summary collapse

Methods inherited from BasePublishMapProcessor

#load_configuration_from_file, process, publish_map, publish_map=, search_hash, #search_hash

Constructor Details

#initialize(params = { }) ⇒ WorkflowPublishMapProcessor

Returns a new instance of WorkflowPublishMapProcessor.

Options Hash (params):

  • :logger (Object)
  • :uu_executable (String) — default: [UDAMUtils.bin_dir]/uu
  • :confirm_filtered_vents (Boolean) —

    NOT CURRENTLY USED

  • :publish_map (Hash)
  • :event_id_field_name (Symbol, String) — default: :id
  • :event_type_field_name (Symbol, String) — default: :type
  • :entity_path_field_name (Symbol, String) — default: :path 


114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/udam_utils/publish_map_processor.rb', line 114

def initialize(params = { })
  super(params)

  logger.debug { "Initializing Workflow Publish Map Processor. #{params}" }

  options = params
  @uu_bin_dir = '/usr/bin' #|| UDAMUtils.get_bin_dir

  @udam_utils_exec = options[:uu_executable] ||= File.join(@uu_bin_dir, 'uu')
  raise "UDAM Utils Executable Not Found. File Not Found: '#{@udam_utils_exec}'" unless File.exist?(@udam_utils_exec)
  logger.debug { "UDAM Utils Executable: #{@udam_utils_exec}" }

  #@get_events_exec = options[:get_events_exec] ||= File.join(mre_bin_dir, 'get_events_with_metadata.rb')
  #raise "Get Events Executable Not Found. File Not Found: '#{@get_events_exec}'" unless File.exist?(@get_events_exec)
  #@logger.debug { "Get Events Exec: #{@get_events_exec}" }

  #@confirm_event_exec = options[:confirm_event_exec] ||= File.join(mre_bin_dir, 'confirm_event.rb')
  #raise "Confirm Events Executable Not Found. File Not Found: '#{@confirm_event_exec}'" unless File.exist?(@confirm_event_exec)
  #@logger.debug { "Confirm Event Exec: #{@confirm_event_exec}" }

  # Determines if events that are set to not be published still get confirmed by default
  @confirm_filtered_events ||= params[:confirm_filtered_events] ||= false
  logger.debug { "Confirm Filtered Events: #{@confirm_filtered_events}" }

  @event_id_field_name ||= params[:event_id_field_name] || :id
  @event_id_field_name = @event_id_field_name.to_sym if @event_id_field_name.is_a?(String)

  @event_type_field_name ||= params[:event_type_field_name] || :type
  @event_type_field_name = @event_type_field_name.to_sym if @event_type_field_name.is_a?(String)

  @entity_field_name ||= params[:entity_field_name] || :entity
  @entity_field_name = @entity_field_name.to_sym if @entity_field_name.is_a?(String)

  @entity_path_field_name ||= params[:entity_path_field_name] || :path
  @entity_path_field_name = @entity_path_field_name.to_sym if @entity_path_field_name.is_a?(String)
end

Instance Attribute Details

#event ⇒ Object

Returns the value of attribute event.



104
105
106
 
# File 'lib/udam_utils/publish_map_processor.rb', line 104

def event
  @event
end

Instance Method Details

#confirm_event(*args) ⇒ Object

search_global_publish_map



520
521
522
 
# File 'lib/udam_utils/publish_map_processor.rb', line 520

def confirm_event(*args)
  { :stdout => '', :stderr => '', :status => '', :success => true }
end

#eval_workflow_parameters(parameters, event = @event) ⇒ Hash 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/udam_utils/publish_map_processor.rb', line 154

def eval_workflow_parameters(parameters, event = @event)
  workflow_parameter_values = { }
  parameters.each { |pname, param|
    logger.debug { "Processing Workflow Parameter: #{pname} -> #{param}" }
    case param
      when String
        pv = param
        eval_pv = true
      when Hash
        pv = param.fetch(:value, nil)
        eval_pv = param.fetch(:eval, false)
      else
        pv = nil
        eval_pv = false
    end
    begin
      pv = eval(pv) if eval_pv && pv.is_a?(String)
      logger.debug { "Processed Workflow Parameter: #{pname} -> #{pv}" }
      workflow_parameter_values[pname] = pv
    rescue => e
      logger.error { "Failed to evaluate parameter. #{e.message}\nName: #{pname}\nValue: #{param}" }
    end
  }
  workflow_parameter_values
end

#execute(cmd_line) ⇒ Hash 



212
213
214
215
216
217
218
219
220
221
 
# File 'lib/udam_utils/publish_map_processor.rb', line 212

def execute(cmd_line)
  begin
    stdout_str, stderr_str, status = Open3.capture3(cmd_line)
    logger.error { "Error Executing #{cmd_line}. Stdout: #{stdout_str} Stderr: #{stderr_str}" } unless status.success?
    return { :stdout => stdout_str, :stderr => stderr_str, :status => status, :success => status.success? }
  rescue
    logger.error { "Error Executing '#{cmd_line}'. Exception: #{$!} @ #{$@} STDOUT: '#{stdout_str}' STDERR: '#{stderr_str}' Status: #{status.inspect} " }
    return { :stdout => stdout_str, :stderr => stderr_str, :status => status, :success => false }
  end
end

#init_publish_params(params_by_event_type) ⇒ Object

process_publish_maps



401
402
403
404
405
406
407
408
 
# File 'lib/udam_utils/publish_map_processor.rb', line 401

def init_publish_params(params_by_event_type)
  return false unless params_by_event_type

  @publish_params = params_by_event_type[@event_type.to_sym] || params_by_event_type[@event_type.to_s]
  @publish_params ||= params_by_event_type[:any] || params_by_event_type['any']
  @publish_params ||= params_by_event_type[:all] || params_by_event_type['all']
  @publish_params
end

#match_found_in_publish_maps?(publish_maps = @publish_maps) ⇒ Boolean 



375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
 
# File 'lib/udam_utils/publish_map_processor.rb', line 375

def match_found_in_publish_maps?(publish_maps = @publish_maps)
  matched = false
  publish_maps.each { |current_publish_map|
    current_publish_map = { :type => :glob, :map => current_publish_map } if current_publish_map.is_a?(Array)

    map_type = current_publish_map.fetch(:type, :unknown)
    map = current_publish_map.fetch(:map, false)
    logger.warn { "Mapping with no map detected. #{current_publish_map}"} and next unless map

    case map_type
      when :eval
        matched = search_eval_publish_map(current_publish_map)
      when :media_type
        matched = search_media_type_publish_map(current_publish_map)
      when :glob
        matched = search_glob_publish_map(current_publish_map)
      when :global
        matched = search_global_publish_map(current_publish_map)
      else
        logger.warn { "Unknown map type '#{map_type}'."}
    end
    break if matched
  }
  matched
end

#parse_event(event = @event) ⇒ Object 



224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/udam_utils/publish_map_processor.rb', line 224

def parse_event(event = @event)
  # Since the event came in and was converted from JSON all of the keys are strings instead of symbols

  begin
    @object = @event
    @event_id = event[@event_id_field_name] || event[@event_id_field_name.to_s]
    @event_type = event[@event_type_field_name] || event[@event_type_field_name.to_s]

    entity = event.fetch(@entity_field_name, event)
    @full_file_path = entity.fetch(@entity_path_field_name)

    @metadata_sources = entity.fetch(:metadata_sources, { })
    @exiftool = @metadata_sources[:exiftool] ||= { }
    @mediainfo = @metadata_sources[:mediainfo] ||= { }
    @ffmpeg = @metadata_sources[:ffmpeg] ||= { }
    @filemagic = @metadata_sources[:filemagic] ||= { }
    @media = @metadata_sources[:filemagic] ||= { }
    @common_media_info = @metadata_sources[:common] ||= { }

    #media = entity.fetch('media', { })
    @media_type = @media[:type] || @media['type']
    @media_subtype = @media[:subtype] || @media['subtype']
  rescue => e
    logger.error "Error parsing event.\n\tEvent: #{event.inspect}\n\n\tException #{e.message}\n\tBacktrace #{e.backtrace}"
    raise
  end
end

#process_event(event = @event) ⇒ Object 



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/udam_utils/publish_map_processor.rb', line 253

def process_event(event = @event)
  @event = event
  logger.debug { "Processing Event: \n\n #{PP.pp(event, '')}" }
  ignore_publish_error = false

  parse_event(event) #rescue return

  if match_found_in_publish_maps?

    # Determines if the event is to be published
    # Defaults to true so that that it doesn't have to be defined for every workflow map
    map_publish_event = @publish_params.fetch(:publish_event, true)

    # Determines if the event is to be confirmed
    # Defaults to true so that that it doesn't have to be defined for every workflow map
    map_confirm_event = @publish_params.fetch(:confirm_event, true)

    # Determines if the event will still get confirmed if there is an error during publishing
    ignore_publish_error = @publish_params.fetch(:ignore_publish_error, false)
  else
    map_publish_event = nil
    map_confirm_event = nil
  end

  to_publish = map_publish_event
  to_confirm = (map_confirm_event or @confirm_filtered_events)

  if to_publish
    publish_response = publish_event
    publish_successful = publish_response[:success]
    confirm_response = confirm_event(@event_id) if (publish_successful or ignore_publish_error) and map_confirm_event
  elsif to_confirm
    logger.debug { "Event is being confirmed but not published. Included Event: #{!map_publish_event.nil?} Map Publish Content: #{map_publish_event} Confirm Filtered Events: #{@confirm_filtered_events} Map Confirm Event: #{map_confirm_event} Event: #{event.inspect}"}
    confirm_response = confirm_event(@event_id)
  else
    publish_successful = publish_response = confirm_successful = confirm_response = nil
  end
  confirm_successful = confirm_response[:success] if confirm_response

  {
      :to_publish       => map_publish_event,
      :to_confirm       => to_confirm,
      :published        => (publish_successful or ignore_publish_error),
      :confirmed        => confirm_successful,
      :publish_response => publish_response,
      :confirm_response => confirm_response,
      :success          => (
        (!to_publish or (to_publish and publish_successful)) and
          (!to_confirm or (to_confirm and confirm_successful))
      )
  }
end

#process_events(events) ⇒ Object

# @param [Array(Hash)] events



308
309
310
311
312
313
314
315
316
317
318
 
# File 'lib/udam_utils/publish_map_processor.rb', line 308

def process_events(events)
  @event = nil
  events.each do |event|
    @event = event
    begin
      process_event
    rescue StandardError, ScriptError => e
      logger.error "Error processing event.\n\tEvent: #{event.inspect}\n\n\tException #{e.inspect}"
    end
  end
end

#publish_event(event = @event, params = @publish_params) ⇒ Boolean

Options Hash (params):

  • :publish_event_exec (String|nil)
  • (false) (Boolean|nil) —

    :eval_publish_event_exec

  • :publish_event_arguments (String|nil)
  • (true) (Boolean|nil) —

    :eval_publish_event_exec



328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/udam_utils/publish_map_processor.rb', line 328

def publish_event(event = @event, params = @publish_params)

  workflow = params.fetch(:workflow, false)
  return publish_event_to_workflow(:workflow => workflow) if workflow

  exec = params.fetch(:publish_event_exec, nil)
  eval_publish_event_exec = params.fetch(:eval_publish_event_exec, false)

  arguments = params.fetch(:publish_event_arguments, nil)
  eval_publish_event_arguments = params.fetch(:eval_publish_event_arguments, true)

  logger.debug { "Evaluating exec: #{exec}" } and exec = eval(exec) if eval_publish_event_exec and exec
  logger.debug { "Evaluating arguments: #{arguments}" } and arguments = eval(arguments) if eval_publish_event_arguments and arguments
  cmd_line = "#{exec} #{arguments}" if arguments
  logger.debug { "Publishing event using command line. #{cmd_line}" }
  response = execute(cmd_line)
  logger.debug { "Publish event command response: #{response}" }
  response
end

#publish_event_to_workflow(params = { }) ⇒ Boolean 



351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
 
# File 'lib/udam_utils/publish_map_processor.rb', line 351

def publish_event_to_workflow(params = { })
  event = params.fetch(:event, @event)
  workflow = params.fetch(:workflow, @publish_params.fetch(:workflow, nil))


  logger.debug { "Publishing Event To Workflow. Workflow: #{workflow}" }
  unless (workflow_name = workflow.fetch(:name, false))
    logger.error "No Workflow Name Specified. Event: #{event} Workflow: #{workflow}"
    return false
  end

  workflow_parameters = workflow.fetch(:parameters, false)
  workflow_parameter_values = eval_workflow_parameters(workflow_parameters, event) if workflow_parameters
  workflow_parameter_values ||= { }

  cmd_line = [ @udam_utils_exec, 'job', '--workflow', workflow_name, '--workflow-parameters', workflow_parameter_values.to_json].shelljoin
  logger.debug { "Publishing event using command line. #{cmd_line}" }
  response = execute(cmd_line)
  logger.debug { "Publish event command response: #{response}" }
  response
end

#publish_to_workflow(params = { }) ⇒ Boolean

Options Hash (params):

  • :event (Hash)
  • :workflow (Hash) —

    A hash containing the :name and optionally the :parameters key for the workflow to execute

  • :mq_connection_uri (String) —

    The connection URI to use when publishing the workflow.



185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
 
# File 'lib/udam_utils/publish_map_processor.rb', line 185

def publish_to_workflow(params = { })
  event = params.fetch(:event, @event)
  workflow = params.fetch(:workflow, @publish_params.fetch(:workflow, nil))
  mq_connection_uri = params.fetch(:mq_connection_uri, nil)

  logger.debug { "Publishing To Workflow. Workflow: #{workflow}" }
  unless (workflow_name = workflow.fetch(:name, false))
    logger.error "No Workflow Name Specified. Event: #{event} Workflow: #{workflow}"
    return false
  end

  workflow_parameters = workflow.fetch(:parameters, false)
  workflow_parameter_values = eval_workflow_parameters(workflow_parameters, event) if workflow_parameters
  workflow_parameter_values ||= { }

  cmd_line = [ @udam_utils_exec, 'job', '--workflow', workflow_name, '--workflow-parameters', workflow_parameter_values.to_json]
  cmd_line << '--mq-connection-uri' << mq_connection_uri if mq_connection_uri
  cmd_line = cmd_line.shelljoin

  logger.debug { "Publishing event using command line. #{cmd_line}" }
  response = execute(cmd_line)
  logger.debug { "Publish event command response: #{response}" }
  response[:success]
end

#search_eval_publish_map(params = { }) ⇒ Object

Options Hash (params):

  • :map (Hash) 


412
413
414
415
416
417
418
419
420
421
422
423
424
425
 
# File 'lib/udam_utils/publish_map_processor.rb', line 412

def search_eval_publish_map(params = { })
  logger.debug { 'Starting Eval Search.' }
  map = params.fetch(:map, false)
  return false unless map

  match_found = false
  map.each { |expressions, map_params|
    @logger.debug { "Testing expressions. #{expressions} #{map_params}" }
    next unless init_publish_params(map_params)
    [*expressions].each { |expression| logger.debug { "Matched expression: #{expression}" } and match_found = true and break if eval(expression) }
    break if match_found
  }
  match_found
end

#search_glob_publish_map(params = { }) ⇒ Object

Options Hash (params):

  • :map (Hash)
  • :options (Integer) 


458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
 
# File 'lib/udam_utils/publish_map_processor.rb', line 458

def search_glob_publish_map(params = { })
  logger.debug { 'Starting Glob Search.' }
  map = params.fetch(:map, false)
  return false unless map

  logger.debug { "MAP: #{map}" }

  full_file_path = @full_file_path
  logger.warn("Full file path is empty. #{@object}") and return false unless full_file_path

  event_type = @event_type

  match_found = false

  default_glob_options = params.fetch(:options, 0)
  default_glob_options = 0 unless default_glob_options.is_a? Integer

  map.each { |patterns, map_params|
    next unless init_publish_params(map_params)

    if patterns.is_a?(Hash)
      globs = patterns[:globs] || patterns[:glob] || { }
      glob_options = patterns.fetch(:options, default_glob_options)
      glob_options = 0 unless glob_options.is_a? Integer
    else
      globs = patterns
      glob_options = default_glob_options
    end

    [*globs].each do |pattern|
      logger.debug { "Testing #{full_file_path} against #{pattern} with options #{glob_options}" }
      if File.fnmatch(pattern, full_file_path, glob_options)
        logger.debug { "Matched pattern: #{full_file_path} -> #{pattern}" }
        match_found = true
        break
      end
    end
    break if match_found
  }
  match_found
end

#search_global_publish_map(params = { }) ⇒ Object

Processes a catch all publish map

{ type: :global, map: { anything: all: { publish_event: false, confirm_event: false } } } Where :anything is ignored and :all can be anyone of :all, :created, :modified, :deleted

Options Hash (params):

  • :map (Hash) 


507
508
509
510
511
512
513
514
515
516
517
518
 
# File 'lib/udam_utils/publish_map_processor.rb', line 507

def search_global_publish_map(params = { })
  logger.debug { 'Starting Global Search.' }
  return false unless (map = params[:map])

  match_found = false
  map.each { |ignored, params_by_event_type|
    next unless init_publish_params(params_by_event_type)
    match_found = true
    break
  }
  match_found
end

#search_media_type_publish_map(params = { }) ⇒ Object

Options Hash (params):

  • :map (Hash) 


429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
 
# File 'lib/udam_utils/publish_map_processor.rb', line 429

def search_media_type_publish_map(params = { })
  logger.debug { 'Starting Media Type Search.' }
  map = params.fetch(:map, false)
  return false unless map

  logger.warn("Asset media type is empty. #{@object}") and return false unless @media_type
  logger.warn("Asset media subtype is empty. #{@object}") and return false unless @media_subtype

  match_found = false
  map.each { |media_types, media_subtypes_with_params|
    [*media_types].each { |media_type|
      next unless media_type.match(@media_type) || @media_type == '*'
      media_subtypes_with_params.each { |media_subtypes, map_params|
        next unless init_publish_params(map_params)
        [*media_subtypes].each { |media_subtype|
          logger.debug { "Matched media type: #{media_type.to_s}/#{media_subtype.to_s} -> #{@media_type}/#{@media_subtype}" } and match_found = true and break if (media_subtype.match(@media_subtype) || media_subtype == '*')
        }
      }
      break if match_found
    }
    break if match_found
  }
  logger.debug { "Media Type Search Completed. Match Found: #{match_found}" }
  match_found
end