Class: Envoi::Mam::Cantemo::Agent::WatchFolderManager

Inherits:
Object
  • Object
show all
Defined in:
lib/envoi/mam/cantemo/agent/watch_folder_manager.rb

Defined Under Namespace

Classes: MultiLogger

Constant Summary collapse

LWF =

AWF = Envoi::Aspera::WatchService::WatchFolder # Aspera Watch Folder

Envoi::WatchFolderUtility::WatchFolder::Handler::Listen
DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT =

Listen Watch Folder

10

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(args = {}) ⇒ WatchFolderManager 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 33

def initialize(args = {})
  initialize_logger(args)

  logger.info { 'Initializing Agent Watch Folder Manager.' }
  args[:default_preserve_file_path] = args.fetch(:default_preserve_file_path, false)

  @config = Envoi::Mam::Cantemo::Agent.load_config_from_file(args)
  initialize_logger_from_config
  args[:logger] = @logger

  @ignored_file_paths_by_watch_folder = Hash.new { |h, k| h[k] = [] }
  @ignored_file_paths_lock            = Mutex.new

  @threaded = args.fetch(:threaded, config.fetch(:threaded, config.fetch('threaded', true)))

  @default_maximum_active_processors = DEFAULT_WATCH_FOLDER_PROCESSOR_LIMIT
  @processors_by_watch_folder = Hash.new { |h, k| h[k] = {} }

  @watch_folder_defs = config[:watch_folders] || config['watch_folders']

  @default_agent_class = Envoi::Mam::Cantemo::Agent

  cantemo_config = config[:cantemo] || config['cantemo']
  if cantemo_config
    logger.debug { 'Initializing Default Cantemo Portal Agent.' }
    @default_agent = @default_agent_class.new(args.merge({ config: cantemo_config }))
    logger.debug { 'Default Cantemo Portal Agent Initialized.' }
    @default_storages = @default_agent.agent_config_storages

    if !@watch_folder_defs
      @watch_folder_defs = cantemo_config[:watch_folders] || cantemo_config['watch_folders']
    end
  else
    @default_agent = nil
  end

  process_watch_folder_defs
end

Instance Attribute Details

#config ⇒ Object

Returns the value of attribute config.



31
32
33
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 31

def config
  @config
end

#default_agent ⇒ Object

Returns the value of attribute default_agent.



31
32
33
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 31

def default_agent
  @default_agent
end

#default_agent_class ⇒ Object

Returns the value of attribute default_agent_class.



31
32
33
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 31

def default_agent_class
  @default_agent_class
end

#logger ⇒ Object

Returns the value of attribute logger.



31
32
33
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 31

def logger
  @logger
end

#watch_folder_defs ⇒ Object

Returns the value of attribute watch_folder_defs.



31
32
33
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 31

def watch_folder_defs
  @watch_folder_defs
end

#watch_folders ⇒ Object

Returns the value of attribute watch_folders.



31
32
33
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 31

def watch_folders
  @watch_folders
end

Class Method Details

.run(args) ⇒ Object 



374
375
376
377
378
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 374

def self.run(args)
  w = self.new(args)
  w.run
  w
end

Instance Method Details

#add_to_ignore(file) ⇒ Object 



144
145
146
147
148
149
150
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 144

def add_to_ignore(file)
  logger.debug { "Adding File to Ignore Cache: '#{file.path}'" }
  @ignored_file_paths_lock.synchronize do
    @ignored_file_paths_by_watch_folder[file.watch_folder] << file.path
    file.ignore if file.respond_to?(:ignore)
  end
end

#find_in_patterns(patterns, file) ⇒ Object

Used to compare file to patterns



219
220
221
222
223
224
225
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 219

def find_in_patterns(patterns, file)
  patterns.find do |pattern|
      matched = pattern.is_a?(Regexp) ? pattern.match(file.path) : File.fnmatch(pattern, file.path)
      logger.debug { "#{pattern} #{matched ? 'matched' : "didn't match"} #{file.path}" }
      matched
  end
end

#initialize_logger(args = {}) ⇒ Object 



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 72

def initialize_logger(args = {})
  @logger = MultiLogger.new(Logger.new(STDOUT))

  _logger = args[:logger] ||= begin
    log_to = args[:log_to]
    log_age = args.fetch(:log_age, 'daily')

    log_to ? Logger.new(log_to, log_age) : nil
  end
  @logger.targets << _logger if _logger

  log_level     = args[:log_level] ||= Logger::INFO
  @logger.level = log_level if log_level
  @logger
end

#initialize_logger_from_config(_config = @config) ⇒ Object 



88
89
90
91
92
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 88

def initialize_logger_from_config(_config = @config)
  logger_args = { }
  [ :log_to, :log_level, :log_age ].each { |k| v = _config[k] || _config[k.to_s]; logger_args[k] = v if v }
  initialize_logger(logger_args) unless logger_args.empty?
end

#pause ⇒ Object 



344
345
346
347
348
349
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 344

def pause
  if @should_run
    logger.info { 'Pausing...' }
    watch_folders.each { |wf| wf.pause if wf.respond_to?(:pause) }
  end
end

#poll ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 288

def poll
  stable_files_by_watch_folder = {} # Hash.new { |h, k| h[k] = [] }
  watch_folders.each do |watch_folder|
    next unless watch_folder.poll_interval_elapsed?

    logger.debug { "Polling Watch Folder: #{watch_folder.name}" }
    watch_folder.poll

    stable_files = watch_folder.stable_files
    stable_files_by_watch_folder[watch_folder] = stable_files
  end

  stable_files_by_watch_folder.each do |watch_folder, stable_files|
    process_watch_folder_stable_files(watch_folder)
  end
end

#process_file(file) ⇒ Hash 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 154

def process_file(file)
  file.processing = true
  file_name       = file.name || file.path
  logger.debug { "Processing File '#{file_name}'" }
  watch_folder = file.watch_folder
  agent = watch_folder.respond_to?(:agent) ? watch_folder.agent : default_agent
  agent ||= default_agent

  storage_id                = watch_folder.definition['upload_to_storage_id']

  unless storage_id
    logger.warn { "Skipping processing of file because of missing storage ID." }
    return { success: false, message: 'Missing storage ID.' }
  end

  quarantine_directory_path = watch_folder.definition['quarantine_directory_path']
  completed_directory_path  = watch_folder.definition['completed_directory_path']
  watch_folder_upload_args  = watch_folder.definition['upload_args']

  # full_file_path = File.join(watch_folder.path, file.path)
  full_file_path = file.path

  upload_args = {
    file_path: full_file_path,
    storage_id: storage_id
  }
  upload_args.merge!(watch_folder_upload_args) if watch_folder_upload_args.is_a?(Hash)

  logger.debug { "Executing Upload. #{upload_args}" }
  _response = agent.upload(upload_args)
  _response = { success: _response } if _response == true || _response == false

  if _response[:success]
    if completed_directory_path
      if Dir.exist?(completed_directory_path)
        logger.debug { "Moving '#{full_file_path}' to completed directory path '#{completed_directory_path}'" }
        FileUtils.mv full_file_path, completed_directory_path
      else
        logger.warn { "Completed directory path not found: '#{completed_directory_path}'" }
        add_to_ignore(file)
      end
    else
      FileUtils.rm full_file_path
    end
  else
    if quarantine_directory_path && Dir.exist?(quarantine_directory_path)
      logger.warn { "Moving '#{full_file_path}' to quarantine directory path '#{quarantine_directory_path}'" }
      FileUtils.mv full_file_path, quarantine_directory_path
    else
      logger.warn { "Adding '#{full_file_path}' to the temporary ignore list." }
      add_to_ignore(file)
    end
  end

  file.processed = true

  _response
rescue => e
  file.exception = e
  raise e
ensure
  file.processing = false
end

#process_watch_folder_def(watch_folder_def) ⇒ Object

Options Hash (watch_folder_def):

  • path (String)
  • upload_to_storage_id (String)
  • name (String) — default: path
  • paths (Array<String>) — default: [path]
  • exclude (String) — default: '**/.*'
  • excludes (Array<string>) — default: [exclude]
  • include (String)
  • includes (Array<String>) — default: [include]
  • quarantine_directory_path (String)
  • completed_directory_path (String)
  • maximum_active_processors (Integer|False) — default: @default_maximum_active_processors
  • logging (Hash) 


107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 107

def process_watch_folder_def(watch_folder_def)
  args_out              = {}
  logging               = watch_folder_def['logging'] || watch_folder_def
  log_to                = logging['log_to']
  log_level             = logging['log_level']
  # args_out[:log_to]     = log_to if log_to && !log_to.empty?
  # args_out[:log_level]  = log_level if log_level && !log_level.empty?
  args_out[:logger]     = logger # unless log_to
  args_out[:default_agent] = default_agent
  args_out[:default_agent_class] = default_agent_class
  args_out[:definition] = watch_folder_def

  Envoi::WatchFolderUtility::WatchFolder.new(args_out)
end

#process_watch_folder_defs ⇒ Object

Iterates through watch_folder_defs and populates @watch_folders with watch folders initialized from the watch folder definitions

Supports both array and hash formats, the hash format will use the key as the mame if the definition doesn’t already have a name set



127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 127

def process_watch_folder_defs
  logger.debug { 'Processing watch folder definitions.' }
  if watch_folder_defs.is_a?(Array)
    @watch_folders = watch_folder_defs.map { |watch_folder_def| process_watch_folder_def(watch_folder_def) }
  elsif watch_folder_defs.is_a?(Hash)
    @watch_folders = watch_folder_defs.map do |name, watch_folder_def|
      watch_folder_def['name'] ||= name
      process_watch_folder_def(watch_folder_def)
    end
  else
    raise "Unhandled format: #{watch_folder_defs.class.name}"
  end
  @watch_folders.keep_if { |wf| wf }
  logger.debug { 'Processing of watch folder definitions completed.' }
end

#process_watch_folder_stable_files(wf) ⇒ Object

This should be part of the watch folder but it is here to track active processors globally



228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 228

def process_watch_folder_stable_files(wf)
  stable_files = wf.stable_files
  active_processors         = @processors_by_watch_folder[wf]
  maximum_active_processors = wf.definition['maximum_active_processors']

  includes = wf.definition['includes']
  excludes = wf.definition['excludes']

  ignored_file_paths = wf.ignored_file_paths if wf.respond_to?(:ingored_file_paths)
  unless ignored_file_paths
    ignored_file_paths = @ignored_file_paths_by_watch_folder[wf]
  end

  stable_files.each do |file|
    file.watch_folder ||= wf
    next if file.respond_to?(:ignore?) ? file.ignore? : ignored_file_paths.include?(file.path)
    next if file.processing || file.processed

    if includes && !includes.empty?
      should_include = find_in_patterns(includes, file)
      unless should_include
        add_to_ignore(file)
        next
      end
    end

    should_exclude = find_in_patterns(excludes, file)
    if should_exclude
      add_to_ignore(file)
      next
    end

    if @threaded
      active_processors.keep_if { |k, v| k.processing }
      if active_processors.length >= maximum_active_processors
        logger.debug { "Maximum number of active processors reached for watch folder. #{wf.name || wf.paths}" }
        break
      end
      t = Thread.new(file) do |file|
        wf = file.watch_folder
        begin
          process_file(file)
        rescue => e
          logger.error { "Exception '#{e.message}' in thread for `#{wf.name || wf.paths}` `#{file.path}`. " }
          raise e
        ensure
          file.processing = false rescue nil
        end
      end
      t.join
      active_processors[file] = t if file.processing
    else
      process_file(file)
    end

  end

end

#resume ⇒ Object 



351
352
353
354
355
356
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 351

def resume
  if @should_run
    logger.info { 'Resuming...' }
    watch_folders.each { |wf| wf.resume if wf.respond_to?(:resume) }
  end
end

#run ⇒ Object

The main execution method



310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 310

def run
  @should_run = true

  # AWF.run_once(watch_folders) { |wf| pp wf }
  # AWF.run(watch_folders) { |wf| process_watch_folder(wf) }
  logger.info { 'Running...' }
  watch_folders.map { |wf| wf.respond_to?(:run) ? wf.run : logger.debug { "Skip run for #{wf}" } }
  logger.debug { 'Initial Run Complete.' }
  while(@should_run) do
    begin
      poll
      sleep 1
    rescue Interrupt, SystemExit => e
      logger.debug { "Received Signal: #{e.class.name}" }
      break
    end
  end
rescue => e
  logger.debug { "EXCEPTION #{e.message} \n#{e.backtrace.join("\n")}\n" }
  logger.error { "An error occurred. #{e.message}" }
  raise e
ensure
  stop
  logger.info { 'Exiting...' }
end

#run_once ⇒ Object 



305
306
307
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 305

def run_once
  # AWF.run_once(watch_folders) { |wf| process_watch_folder(wf) }
end

#stop ⇒ Object 



336
337
338
339
340
341
342
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 336

def stop
  if @should_run
    @should_run = false
    logger.info { 'Stopping...' }
    watch_folders.each { |wf| wf.stop if wf.respond_to?(:stop) }
  end
end

#symbolize_keys(value, recursive = true) ⇒ Object

Converts hash keys to symbols



362
363
364
365
366
367
368
369
370
371
372
 
# File 'lib/envoi/mam/cantemo/agent/watch_folder_manager.rb', line 362

def symbolize_keys (value, recursive = true)
  case value
  when Hash
    Hash[value.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym : k, recursive ? symbolize_keys(v, true) : v ] }]
  when Array
    value.map { |v| symbolize_keys(v, recursive) }
  else
    value
  end
  # symbolize_keys
end