Class: At_email::Tasks::Imap_To_Fs

Inherits:
Object
  • Object
show all
Defined in:
lib/at_email/tasks/imap_to_fs.rb

Constant Summary collapse

METADATA_ATTRIBUTES = 
[ 'ENVELOPE', 'FLAGS', 'RFC822.SIZE' ].freeze
REQUESTED_ATTRIBUTES = 
[ 'INTERNALDATE', 'RFC822', 'RFC822.SIZE' ].freeze

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Imap_To_Fs

Returns a new instance of Imap_To_Fs.



15
16
17
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 15

def initialize
  @dir_base_path = $config[:output_base_dir] + '/' + $config[:account_id]
end

Instance Method Details

#bootstrap_thread(folder_name, uid) ⇒ Object 



359
360
361
362
363
364
365
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 359

def bootstrap_thread(folder_name, uid)
  message_config = get_message_config(folder_name, uid)
  set_thread_property(folder_name, uid, 'File Name', File.basename(message_config['file_path']))
  set_thread_property(folder_name, uid, 'File Path', message_config['file_path'])
  set_thread_property(folder_name, uid, 'Server Size', message_config['size'])
  set_thread_property(folder_name, uid, 'Retry Count', 0)
end

#check_folder_included(folder_name) ⇒ Object 



614
615
616
617
618
619
620
621
622
623
624
625
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 614

def check_folder_included(folder_name)
  ignore_list = []
  ignore_list << "[Gmail]"
  ignore_list << "[Gmail]/All Mail"
  ignore_list << "[Gmail]/Important"
  ignore_list << "[Gmail]/Starred"
  if ignore_list.include?(folder_name)
    return false
  else
    return true
  end
end

#create_output_directory(folder_name) ⇒ Object 



482
483
484
485
486
487
488
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 482

def create_output_directory(folder_name)
  dir_path = @dir_base_path + '/' + folder_name
  if !Dir.exists?(dir_path)
    $logger.debug folder_name + ' - Creating Directory: ' + dir_path
    FileUtils.mkdir_p dir_path
  end
end

#decrease_thread_count(folder_name) ⇒ Object 



269
270
271
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 269

def decrease_thread_count(folder_name)
  @account_metadata[:folders][folder_name]['process']['threads']['count'] -= 1
end

#delete_directories(folder_name) ⇒ Object 



171
172
173
174
175
176
177
178
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 171

def delete_directories(folder_name)
  server_dirs = @account_metadata[:folders][folder_name]['directories']['server']
  @account_metadata[:folders][folder_name]['directories']['local'].each do |local_dir|
    if !server_dirs[local_dir]
      $logger.info( folder_name + ' - Deleting Directory: ' + local_dir )
    end
  end
end

#delete_files(folder_name) ⇒ Object 



205
206
207
208
209
210
211
212
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 205

def delete_files(folder_name)
  @account_metadata[:folders][folder_name]['files']['deletes'].each do |file_path|
    if File.exists?(file_path)
      $logger.info( folder_name + ' -  Deleting File: ' + file_path )
      File.delete(file_path)
    end
  end
end

#execute ⇒ Object 



627
628
629
630
631
632
633
634
635
636
637
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 627

def execute
  @connection = At_email::Account::ImapConnection.new()
  @connection.
  get_imap
  
  @account_metadata[:task][:start_time] = Time.now
  get_data_for_all_folders
  @connection.disconnect
  @account_metadata[:task][:end_time] = Time.now
  task_report
end

#format_file_size(file_size) ⇒ Object 



746
747
748
749
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 746

def format_file_size(file_size)
  output = format_number(file_size) + ' bytes'
  return output
end

#format_number(number) ⇒ Object 



751
752
753
754
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 751

def format_number(number)
  output = number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
  return output
end

#generate_metadata_for_account ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 82

def ()
  @account_metadata = {}
  @account_metadata[:account] = {}
  @account_metadata[:account]['directories'] = {}
  @account_metadata[:account]['directories']['server'] = get_server_dir_list()
  @account_metadata[:account]['directories']['local'] = get_local_dir_list()
  @account_metadata[:account]['messages'] = {}
  @account_metadata[:account]['messages']['server'] = {}
  @account_metadata[:account]['messages']['server']['folders'] = get_server_message_folder_list()
  @account_metadata[:task] = {}
  @account_metadata[:task] = {}
  @account_metadata[:folders] = {}
end

#generate_metadata_for_folder(folder_name) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 96

def generate_metadata_for_folder(folder_name)
  @account_metadata[:folders][folder_name] = {}
  @account_metadata[:folders][folder_name]['messages'] = {}
  @account_metadata[:folders][folder_name]['messages']['server'] = {}
  @account_metadata[:folders][folder_name]['messages']['server']['uids'] = get_uids()
  @account_metadata[:folders][folder_name]['messages']['server']['metadata'] = get_imap_metadata(@account_metadata[:folders][folder_name]['messages']['server']['uids'])
  @account_metadata[:folders][folder_name]['messages']['server']['config'] = update_message_config(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['metadata'])
  @account_metadata[:folders][folder_name]['files'] = {}
  @account_metadata[:folders][folder_name]['files']['server'] = get_server_file_list(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['config'])
  @account_metadata[:folders][folder_name]['files']['local'] = get_local_file_list(folder_name)
  @account_metadata[:folders][folder_name]['files']['deletes'] = get_delete_list(folder_name)
  @account_metadata[:folders][folder_name]['process'] = {}
end

#get_data_for_all_folders ⇒ Object 



595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 595

def get_data_for_all_folders
  folder_list = @account_metadata[:account]['messages']['server']['folders']
  $logger.debug ''
  $logger.debug 'Folder Count: ' + folder_list.count.to_s
  folder_list.each do |folder_name|
    if check_folder_included(folder_name)
      $logger.debug ' ' + folder_name
    else
      $logger.debug ' ' + folder_name + ' - SKIPPED'
    end
  end
  folder_list.each do |folder_name|
    if check_folder_included(folder_name)
      write_data_for_folder(folder_name)
    end
  end
  $logger.debug ''
end

#get_delete_list(folder_name) ⇒ Object 



192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 192

def get_delete_list(folder_name)
  output = []
  server_files = @account_metadata[:folders][folder_name]['files']['server']
  @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
    if !server_files[file_path]
      if File.exists?(file_path)
        output << file_path
      end
    end
  end
  return output
end

#get_download_list(folder_name) ⇒ Object 



180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 180

def get_download_list(folder_name)
  output = []
  local_file_list = @account_metadata[:folders][folder_name]['files']['local']
  @account_metadata[:folders][folder_name]['messages']['server']['config'].each do |uid, message_config|
    file_path = message_config['file_path']
    if !local_file_list.include?(file_path)
      output << uid
    end
  end
  return output
end

#get_duration(start_time, end_time) ⇒ Object 



723
724
725
726
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 723

def get_duration(start_time, end_time)
  output = end_time.to_f - start_time.to_f
  return output
end

#get_duration_formatted(start_time, end_time) ⇒ Object 



728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 728

def get_duration_formatted(start_time, end_time)
  duration_seconds = get_duration(start_time, end_time)
  if duration_seconds
    if duration_seconds <= 60
      output = sprintf("%.6f", duration_seconds) + ' Sec(s)'
    else
      nanoseconds = (sprintf("%.6f", duration_seconds).to_f * 1000000) % 1000000
      seconds = duration_seconds % 60
      minutes = (duration_seconds / 60) % 60
      hours = duration_seconds / (60 * 60)
      output = format("%02d:%02d:%02d.%6d", hours, minutes, seconds, nanoseconds)
    end
  else
    return false
  end
  return output
end

#get_file_path_for_message(folder_name, message_config) ⇒ Object 



72
73
74
75
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 72

def get_file_path_for_message(folder_name, message_config)
  output = get_file_path_prefix(folder_name, message_config)+'.'+$config[:storage_file_extension]
  return output
end

#get_file_path_prefix(folder_name, message_config) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 54

def get_file_path_prefix(folder_name, message_config)
  if $config[:storage_file_name_uid_padding]
    if $config[:storage_file_name_uid_padding_length] <= 1
      storage_file_name_uid_padding = 2
    else
      storage_file_name_uid_padding = $config[:storage_file_name_uid_padding_length]
    end
    uid_based_prefix = message_config['uid'] + (10 ** (storage_file_name_uid_padding - 1))
  else
    uid_based_prefix = message_config['uid']
  end
  hash_length = $config[:storage_file_name_hash_length]
  dir_path = @dir_base_path + '/' + folder_name
  file_name_hash = Digest::MD5.hexdigest(Digest::MD5.hexdigest(YAML.dump(message_config['envelope']))+'-'+Digest::MD5.hexdigest(YAML.dump(message_config['flags'])))[0...hash_length]
  output = dir_path+'/'+uid_based_prefix.to_s+'-'+file_name_hash
  return output
end

#get_formatted_thread_property(folder_name, uid, property_name) ⇒ Object 



248
249
250
251
252
253
254
255
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 248

def get_formatted_thread_property(folder_name, uid, property_name)
  ### getting property
  name = property_name
  value = get_thread_property(folder_name, uid, name) || 'Unable to get property: ' + name
  #####
  output = name + ': ' + value.to_s
  return output
end

#get_imap ⇒ Object 



19
20
21
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 19

def get_imap
  @imap = @connection.imap
end

#get_imap_metadata(uid_list) ⇒ Object 



34
35
36
37
38
39
40
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 34

def get_imap_metadata(uid_list)
  output = {}
  if uid_list.count > 0
    output = @imap.uid_fetch(uid_list, METADATA_ATTRIBUTES)
  end
  return output
end

#get_local_dir_list ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 148

def get_local_dir_list()
  output = []
  if Dir.exists?(@dir_base_path)
    Dir.chdir(@dir_base_path)
    file_list = Dir.glob("**/*/")
    file_list.each do |object_name|
      if File.directory?(object_name)
        output << @dir_base_path + '/' + object_name
      end
    end
  end
  return output
end

#get_local_file_list(folder_name) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 124

def get_local_file_list(folder_name)
  output = []
  dir_path = @dir_base_path + '/' + folder_name
  if Dir.exists?(dir_path)
    Dir.chdir(dir_path)
    file_list = Dir.glob("*."+$config[:storage_file_extension]).sort_by{|f| f.split(" ")[0].to_i }
    file_list.each do |file_name|
      if File.file?(file_name)
        output << dir_path+'/'+file_name
      end
    end
  end
  return output
end

#get_message_config(folder_name, uid) ⇒ Object 



350
351
352
353
354
355
356
357
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 350

def get_message_config(folder_name, uid)
  if @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
    output = @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
    return output
  else
    raise '***** ERROR ***** - Message Config: Cannot get message config for folder: ' + folder_name + ' - UID:' + uid
  end
end

#get_message_data(uid) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 42

def get_message_data(uid)
  data_raw = @imap.uid_fetch(uid.to_i, REQUESTED_ATTRIBUTES)
  if data_raw
    if data_raw[0]
      if data_raw[0].attr
        output = data_raw[0].attr
        return output
      end
    end
  end
end

#get_ratio_formatted(a, b, delimiter = '/') ⇒ Object 



214
215
216
217
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 214

def get_ratio_formatted(a, b, delimiter = '/')
  output = a.to_s + delimiter + b.to_s
  return output
end

#get_server_dir_list ⇒ Object 



162
163
164
165
166
167
168
169
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 162

def get_server_dir_list()
  output = []
  message_folder_list = get_server_message_folder_list()
  message_folder_list.each do |message_folder|
    output << @dir_base_path + '/' + message_folder
  end
  return output
end

#get_server_file_list(folder_name, server_message_config) ⇒ Object 



139
140
141
142
143
144
145
146
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 139

def get_server_file_list(folder_name, server_message_config)
  output = {}
  server_message_config.each do |uid, message_config|
    file_path = get_file_path_for_message(folder_name, message_config)
    output[file_path] = uid
  end
  return output
end

#get_server_message_folder_list ⇒ Object 



23
24
25
26
27
28
29
30
31
32
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 23

def get_server_message_folder_list()
  output = []
  root_path = @imap.list("", "")[0].name
  folder_data = @imap.list(root_path, "*")
  output = []
  folder_data.map do |this_folder_data|
    output << this_folder_data.name
  end
  return output
end

#get_task_stats ⇒ Object 



690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 690

def get_task_stats()
  output = {}
  output[:account] = {}
  output[:folders] = {}
  @account_metadata[:folders].each do |folder_name, folder_data|
    output[:folders][folder_name] = {}
    output[:folders][folder_name]['messages'] = {}
    output[:folders][folder_name]['messages']['size'] = 0
    folder_data['messages']['server']['config'].each do |uid, message_config|
      output[:folders][folder_name]['messages']['size'] += message_config['size']
    end
    output[:folders][folder_name]['threads'] = {}
    output[:folders][folder_name]['threads']['Total'] = 0
    output[:folders][folder_name]['threads']['Success'] = 0
    output[:folders][folder_name]['threads']['Failed'] = 0
    if folder_data['process']
      if folder_data['process']['threads']
        if folder_data['process']['threads']['uids']
          folder_data['process']['threads']['uids'].each do |uid, thread_data|
            output[:folders][folder_name]['threads']['Total'] += 1
            if thread_data['State'] === 'Successful'
              output[:folders][folder_name]['threads']['Success'] += 1
            else
              output[:folders][folder_name]['threads']['Failed'] += 1
            end
          end
        end
      end
    end
  end
  return output
end

#get_thread_id_string(uid, msg_ratio) ⇒ Object 



273
274
275
276
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 273

def get_thread_id_string(uid, msg_ratio)
  uid_string = get_uid_string_formatted(uid)
  output = msg_ratio + ' - ' + uid_string
end

#get_thread_property(folder_name, uid, property) ⇒ Object 



234
235
236
237
238
239
240
241
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 234

def get_thread_property(folder_name, uid, property)
  if @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
    output = @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
    return output
  else
    return false
  end
end

#get_uid_string_formatted(uid) ⇒ Object 



219
220
221
222
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 219

def get_uid_string_formatted(uid)
  output = 'UID:' + uid.to_s
  return output
end

#get_uids ⇒ Object 



77
78
79
80
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 77

def get_uids()
  uid_list = @imap.uid_search(["ALL"]).sort
  return uid_list
end

#increase_thread_count(folder_name) ⇒ Object 



265
266
267
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 265

def increase_thread_count(folder_name)
  @account_metadata[:folders][folder_name]['process']['threads']['count'] += 1
end

#increase_thread_property(folder_name, uid, property) ⇒ Object 



257
258
259
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 257

def increase_thread_property(folder_name, uid, property)
  @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] += 1
end

#kill_thread(folder_name, uid) ⇒ Object 



500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 500

def kill_thread(folder_name, uid)
  log_folder_event(folder_name, 'WARN', 'ERROR', 'Thread Failure - ' + get_uid_string_formatted(uid))
  log_folder_event(folder_name, 'WARN', 'ERROR', ' Killing Thread')
  Thread.kill(get_thread_property(folder_name, uid, 'Thread'))
  ### getting formatted thread property
  property_name = 'Queue Time'
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
  ### getting formatted thread property
  property_name = 'End Time'
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
  $logger.error folder_name + ' -  Duration: ' + get_duration_formatted(get_thread_property(folder_name, uid, 'Queue Time'), get_thread_property(folder_name, uid, 'End Time'))
  ### getting formatted thread property
  property_name = 'File Name'
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
end

#log_folder_event(folder_name, event_level, event_tag, event_data) ⇒ Object 



345
346
347
348
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 345

def log_folder_event(folder_name, event_level, event_tag, event_data)
  log_string = folder_name + ' - ' + event_data
  $logger.event(event_level, event_tag, log_string)
end

#log_folder_info(folder_name) ⇒ Object 



471
472
473
474
475
476
477
478
479
480
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 471

def log_folder_info(folder_name)
  $logger.debug folder_name + ' - Server Files: ' + @account_metadata[:folders][folder_name]['files']['server'].count.to_s
  @account_metadata[:folders][folder_name]['files']['server'].each do |file_path, uid|
    $logger.debug folder_name + ' -  ' + File.basename(file_path)
  end
  $logger.debug folder_name + ' - Local Files: ' + @account_metadata[:folders][folder_name]['files']['local'].count.to_s
  @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
    $logger.debug folder_name + ' -  ' + File.basename(file_path)
  end
end

#log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag = '', event_data) ⇒ Object 



338
339
340
341
342
343
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 338

def log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data)
  uid_string = get_uid_string_formatted(uid) || 'Unable to get UID string'
  thread_id_string = get_thread_id_string(uid, msg_ratio) || 'Unable to get thread ID string'
  log_string = thread_id_string + ' - ' + event_data
  log_folder_event(folder_name, event_level, event_tag, log_string)
end

#log_thread_on_failure(folder_name, uid, msg_ratio, event_data) ⇒ Object 



306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 306

def log_thread_on_failure(folder_name, uid, msg_ratio, event_data)
  ### getting property
  property_name = 'Queue Time'
  queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'End Time'
  end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Server Size'
  server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Local Size'
  local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting data
  duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
  #####
  ### getting data
  size_formatted = server_size + ' Bytes'
  #####
  log_string = 'Duration: ' + duration_formatted + ' -  Size: ' + size_formatted
  log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'E', log_string)
  log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'ERROR', event_data)
end

#log_thread_on_retry(folder_name, uid, msg_ratio, event_data) ⇒ Object 



334
335
336
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 334

def log_thread_on_retry(folder_name, uid, msg_ratio, event_data)
  log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', event_data)
end

#log_thread_on_success(folder_name, uid, msg_ratio) ⇒ Object 



278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 278

def log_thread_on_success(folder_name, uid, msg_ratio)
  ### getting property
  property_name = 'Queue Time'
  queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'End Time'
  end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Server Size'
  server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Local Size'
  local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting data
  duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
  #####
  ### getting data
  size_ratio_formatted = get_ratio_formatted(local_size, server_size, ' / ') || 'Unable to get data'
  size_formatted = size_ratio_formatted + ' Bytes'
  #####
  log_string = 'Duration: ' + duration_formatted + ' -  Size: ' + size_formatted
  log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
end

#log_thread_property(folder_name, uid, msg_ratio, property_name) ⇒ Object 



243
244
245
246
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 243

def log_thread_property(folder_name, uid, msg_ratio, property_name)
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
end

#process_deletes(folder_name) ⇒ Object 



490
491
492
493
494
495
496
497
498
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 490

def process_deletes(folder_name)
  if $config[:delete_on_sync]
    if @account_metadata[:folders][folder_name]['files']['deletes'].count > 0
      $logger.info folder_name + ' - Delete Count: ' + @account_metadata[:folders][folder_name]['files']['deletes'].count.to_s
      $logger.info folder_name + ' - Executing Deletes'
      delete_files(folder_name)
    end
  end
end

#process_downloads(folder_name, download_list) ⇒ Object 



522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 522

def process_downloads(folder_name, download_list)
  process_healthy = true
  if download_list.count > 0
    $logger.info( folder_name + ' - ' + download_list.count.to_s + ' Message(s) to download - Thread Limit: ' + $config[:threads].to_s )
    message_count = 0
    ### Creating / resetting active thread data structure
    @account_metadata[:folders][folder_name]['process']['threads'] = {}
    @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
    @account_metadata[:folders][folder_name]['process']['threads']['uids'] = {}
    message_total = download_list.count
    download_list.each do |uid|
      if process_healthy
        message_count += 1
        uid_string = get_uid_string_formatted(uid)
        msg_ratio = get_ratio_formatted(message_count, message_total)
        last_queue_time = Time.now
        if @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads]
          $logger.debug folder_name + ' - Thread Count: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + ' - Waiting...'
        end
        while @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] && process_healthy
          if (Time.now.to_i - last_queue_time.to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
            if (Time.now.to_i - last_queue_time.to_i) >= $config[:thread_timeout]
              $logger.error folder_name + ' - *E* Thread queuing has stopped - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
              process_healthy = false
            else
              $logger.warn folder_name + ' - *W* Thread queuing has stopped - Timeout: ' + (Time.now.to_i - last_queue_time.to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
              sleep rand(10..20)
            end
          else
            sleep 0.2
          end
        end
        if process_healthy
          @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid] = {}
          queue_thread(folder_name, uid, msg_ratio)
          sleep 0.1
          while get_thread_property(folder_name, uid, 'State') === 'Queued'
            if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
              if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= $config[:thread_timeout]
                $logger.error folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *E* Thread state - Moving on... - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
                set_thread_state(folder_name, uid, msg_ratio, 'Failed')
                kill_thread(folder_name, uid)
              else
                $logger.warn folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *W* Thread state - Timeout: ' + (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
                sleep rand(10..20)
              end
            else
              sleep 0.2
            end
          end
        end
      end
    end
    if process_healthy
      sleep 2
      process_shutdown(folder_name)
    end
    process_thread_data_before_finish(folder_name)
  end
end

#process_shutdown(folder_name) ⇒ Object 



457
458
459
460
461
462
463
464
465
466
467
468
469
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 457

def process_shutdown(folder_name)
  time_started_waiting = Time.now.to_i
  while @account_metadata[:folders][folder_name]['process']['threads']['count'] > 0
    time_since_started_waiting = Time.now.to_i - time_started_waiting
    if time_since_started_waiting >= $config[:thread_timeout]
      $logger.warn folder_name + ' - *E* - Thread pool shutdown - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
      @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
    else
      $logger.info folder_name + ' - Waiting for threads - Remaining: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + '/' + $config[:threads].to_s + ' - Timeout: ' + time_since_started_waiting.to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
      sleep 10
    end
  end
end

#process_thread_data_before_finish(folder_name) ⇒ Object 



441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 441

def process_thread_data_before_finish(folder_name)
  has_failed = false
  @account_metadata[:folders][folder_name]['process']['threads']['uids'].each do |uid, thread_data|
    if thread_data['State'] != 'Successful'
      thread_data['State'] = 'Failed'
    end
    if thread_data['State'] === 'Failed'
      kill_thread(folder_name, uid)
      has_failed = true
    end
  end
  if has_failed
    sleep 5
  end
end

#queue_thread(folder_name, uid, msg_ratio) ⇒ Object 



381
382
383
384
385
386
387
388
389
390
391
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 381

def queue_thread(folder_name, uid, msg_ratio)
  set_thread_state(folder_name, uid, msg_ratio, 'Queued')
  bootstrap_thread(folder_name, uid)
  set_thread_property(folder_name, uid, 'Queue Time', Time.now)
  increase_thread_count(folder_name)
  Thread.abort_on_exception = true
  @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid]['Thread'] = Thread.new do
    sleep 0.1
    write_data_for_uid(folder_name, uid, msg_ratio)
  end
end

#retry_check(folder_name, uid, msg_ratio) ⇒ Object 



367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 367

def retry_check(folder_name, uid, msg_ratio)
  if get_thread_property(folder_name, uid, 'Retry Count') <= 3
    log_thread_on_retry(folder_name, uid, msg_ratio, 'Queuing for a retry')
    increase_thread_property(folder_name, uid, 'Retry Count')
    set_thread_state(folder_name, uid, msg_ratio, 'Retrying')
    queue_thread(folder_name, uid, msg_ratio)
    return true
  else
    set_thread_property(folder_name, uid, 'End Time', Time.now)
    set_thread_state(folder_name, uid, msg_ratio, 'Failed')
    return false
  end
end

#set_thread_property(folder_name, uid, property, value) ⇒ Object 



230
231
232
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 230

def set_thread_property(folder_name, uid, property, value)
  @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] = value
end

#set_thread_state(folder_name, uid, msg_ratio, state) ⇒ Object 



224
225
226
227
228
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 224

def set_thread_state(folder_name, uid, msg_ratio, state)
  uid_string = get_uid_string_formatted(uid)
  set_thread_property(folder_name, uid, 'State', state.to_s)
  log_thread_property(folder_name, uid, msg_ratio, 'State')
end

#task_report ⇒ Object 



639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 639

def task_report
  stats = get_task_stats()
  $logger.info ''
  $logger.info '##### Task Report #####'
  $logger.info ''
  $logger.info ' Task: ' + $config[:task]
  $logger.info ''
  $logger.info ' Server: ' + $config[:server]
  $logger.info ' Username: ' + $config[:username]
  $logger.info ' Output Directory: ' + @dir_base_path
  $logger.info ''
  $logger.info ' Start Time: ' + @account_metadata[:task][:start_time].to_s
  $logger.info ' End Time: ' + @account_metadata[:task][:end_time].to_s
  $logger.info ' Duration: ' + get_duration_formatted(@account_metadata[:task][:start_time], @account_metadata[:task][:end_time])
  $logger.info ''
  $logger.info ' Folders Processed: ' + @account_metadata[:folders].count.to_s
  $logger.info ''
  @account_metadata[:folders].each do |folder_name, folder_data|
    $logger.info '  ' + folder_name
    $logger.info ''
    $logger.info '   Message Stats'
    $logger.info '    Total: ' + folder_data['messages']['server']['config'].count.to_s
    $logger.info '    Total Size: ' + stats[:folders][folder_name]['messages']['size'].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + ' bytes'
    $logger.info ''
    $logger.info '   Thread Stats'
    $logger.info '    Total: ' + stats[:folders][folder_name]['threads']['Total'].to_s
    $logger.info '    Success: ' + stats[:folders][folder_name]['threads']['Success'].to_s
    $logger.info '    Failed: ' + stats[:folders][folder_name]['threads']['Failed'].to_s
    if folder_data['process']
      if folder_data['process']['threads']
        if folder_data['process']['threads']['uids']
          $logger.info '   Threads'
          folder_data['process']['threads']['uids'].each do |uid, thread_data|
            if thread_data['State'] === 'Successful'
              $logger.debug '    UID: ' + uid.to_s
              $logger.debug '     Path: ' + folder_data['messages']['server']['config'][uid]['file_path']
              $logger.debug '     Thread State: ' + thread_data['State']
            else
              $logger.info '    UID: ' + uid.to_s
              $logger.info '     Path: ' + message_config['file_path']
              $logger.info '     Thread State: ' + thread_data['State']
            end
          end
        end
      end
    end
    $logger.info ''
  end
  $logger.info ''
end

#update_message_config(folder_name, metadata) ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 110

def update_message_config(folder_name, metadata)
  output = {}
  metadata.each do |message_metadata|
    uid = message_metadata.attr['UID']
    output[uid] = {}
    output[uid]['uid'] = uid
    output[uid]['envelope'] = message_metadata.attr['ENVELOPE'].to_h
    output[uid]['flags'] = message_metadata.attr['FLAGS']
    output[uid]['size'] = message_metadata.attr['RFC822.SIZE']
    output[uid]['file_path'] = get_file_path_for_message(folder_name, output[uid])
  end
  return output
end

#write_data_for_folder(folder_name) ⇒ Object 



583
584
585
586
587
588
589
590
591
592
593
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 583

def write_data_for_folder(folder_name)
  $logger.debug ''
  $logger.info 'Checking ' + folder_name
  @imap.examine(folder_name)
  generate_metadata_for_folder(folder_name)
  log_folder_info(folder_name)
  download_list = get_download_list(folder_name)
  create_output_directory(folder_name)
  process_deletes(folder_name)
  process_downloads(folder_name, download_list)
end

#write_data_for_uid(folder_name, uid, msg_ratio) ⇒ Object 



393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
 
# File 'lib/at_email/tasks/imap_to_fs.rb', line 393

def write_data_for_uid(folder_name, uid, msg_ratio)
  uid_string = get_uid_string_formatted(uid)
  if get_thread_property(folder_name, uid, 'State') === 'Queued'
    set_thread_state(folder_name, uid, msg_ratio, 'Active')
    log_thread_property(folder_name, uid, msg_ratio, 'File Name')
    file_path = get_thread_property(folder_name, uid, 'File Path')
    file_path_tmp = file_path + '.tmp'
    if message_data = get_message_data(uid)
      File.write(file_path_tmp, Zlib::Deflate.deflate(YAML.dump(message_data)))
      FileUtils.mv(file_path_tmp, file_path)
      set_thread_property(folder_name, uid, 'Local Size', File.size(get_thread_property(folder_name, uid, 'File Path')))
      if get_thread_property(folder_name, uid, 'Local Size') > (get_thread_property(folder_name, uid, 'Server Size') * 3)
        if !retry_check(folder_name, uid, msg_ratio)
          log_thread_on_failure(folder_name, uid, msg_ratio, 'File size mismatch after download')
        end
        if File.exists?(file_path)
          ### getting property
          property_name = 'File Name'
          file_name = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
          #####
          log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'File size mismatch detected, cleaning up')
          log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'Deleting File: ' + file_name)
          File.delete(file_path)
        end
      else
        set_thread_property(folder_name, uid, 'End Time', Time.now)
        set_thread_state(folder_name, uid, msg_ratio, 'Successful')
        log_thread_on_success(folder_name, uid, msg_ratio)
      end
    else
      if !retry_check(folder_name, uid, msg_ratio)
        log_thread_on_failure(folder_name, uid, msg_ratio, 'Unable to get message data')
      end
    end
    decrease_thread_count(folder_name)
  else
    ### getting property
    property_name = 'State'
    formatted_property = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
    #####
    log_string = 'Thread state incorrect - ' + formatted_property
    log_folder_event(folder_name, 'ERROR', 'E', log_string)
    raise log_string
  end
end