Class: Remi::Extractor::FileSystem

Inherits:
Object
  • Object
show all
Defined in:
lib/remi/extractor/file_system.rb

Direct Known Subclasses

S3File, SftpFileNew

Defined Under Namespace

Classes: FileNotFoundError

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block) ⇒ FileSystem

Returns a new instance of FileSystem.



24
25
26
27
28
29
30
31
32
 
# File 'lib/remi/extractor/file_system.rb', line 24

def initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block)
  @remote_path = Pathname.new(remote_path)
  @pattern = pattern
  @local_path = Pathname.new(local_path)
  @most_recent_only = most_recent_only
  @group_by = group_by
  @most_recent_by = most_recent_by
  @logger = logger
end

Instance Attribute Details

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



34
35
36
 
# File 'lib/remi/extractor/file_system.rb', line 34

def logger
  @logger
end

Instance Method Details

#all_entries ⇒ Object

Public: Returns an array of all FileSystemEntry instances that are in the remote_path. NOTE: all_entries is responsible for matching the path using @remote_path

Raises:

  • (NoMethodError) 


45
46
47
 
# File 'lib/remi/extractor/file_system.rb', line 45

def all_entries
  raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
end

#entries ⇒ Object

Public: Returns just the entries that are to be extracted.



50
51
52
53
54
55
56
57
58
 
# File 'lib/remi/extractor/file_system.rb', line 50

def entries
  if @group_by
    most_recent_matching_entry_in_group
  elsif @most_recent_only
    Array(most_recent_matching_entry)
  else
    matching_entries
  end
end

#extract ⇒ Object

Public: Called to extract files from the source filesystem.

Returns an array with containing the paths to all files extracted.

Raises:

  • (NoMethodError) 


39
40
41
 
# File 'lib/remi/extractor/file_system.rb', line 39

def extract
  raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
end

#matching_entries ⇒ Object 



60
61
62
 
# File 'lib/remi/extractor/file_system.rb', line 60

def matching_entries
  all_entries.select { |e| @pattern.match e.name }
end

#most_recent_matching_entry ⇒ Object 



64
65
66
 
# File 'lib/remi/extractor/file_system.rb', line 64

def most_recent_matching_entry
  matching_entries.sort_by { |e| e.send(@most_recent_by) }.reverse.first
end

#most_recent_matching_entry_in_group ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/remi/extractor/file_system.rb', line 68

def most_recent_matching_entry_in_group
  entries_with_group = matching_entries.map do |entry|
    match = entry.name.match(@group_by)
    next unless match

    group = match.to_a[1..-1]
    { group: group, entry: entry }
  end.compact
  sorted_entries_with_group = entries_with_group.sort_by { |e| [e[:group], e[:entry].send(@most_recent_by)] }.reverse

  last_group = nil
  sorted_entries_with_group.map do |entry|
    next unless entry[:group] != last_group
    last_group = entry[:group]
    entry[:entry]
  end.compact
end