Class: Remi::Extractor::FileSystem

Inherits:
Object
  • Object
show all
Defined in:
lib/remi/extractor/file_system.rb

Direct Known Subclasses

LocalFile, S3File, SftpFile

Defined Under Namespace

Classes: FileNotFoundError

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block) ⇒ FileSystem

Returns a new instance of FileSystem.



23
24
25
26
27
28
29
30
31
 
# File 'lib/remi/extractor/file_system.rb', line 23

def initialize(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, logger: Remi::Settings.logger, **kargs, &block)
  @remote_path = Pathname.new(remote_path)
  @pattern = pattern
  @local_path = Pathname.new(local_path)
  @most_recent_only = most_recent_only
  @group_by = group_by
  @most_recent_by = most_recent_by
  @logger = logger
end

Instance Attribute Details

#group_by ⇒ Object (readonly)

Returns the value of attribute group_by.



37
38
39
 
# File 'lib/remi/extractor/file_system.rb', line 37

def group_by
  @group_by
end

#local_path ⇒ Object (readonly)

Returns the value of attribute local_path.



35
36
37
 
# File 'lib/remi/extractor/file_system.rb', line 35

def local_path
  @local_path
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



39
40
41
 
# File 'lib/remi/extractor/file_system.rb', line 39

def logger
  @logger
end

#most_recent_by ⇒ Object (readonly)

Returns the value of attribute most_recent_by.



38
39
40
 
# File 'lib/remi/extractor/file_system.rb', line 38

def most_recent_by
  @most_recent_by
end

#most_recent_only ⇒ Object (readonly)

Returns the value of attribute most_recent_only.



36
37
38
 
# File 'lib/remi/extractor/file_system.rb', line 36

def most_recent_only
  @most_recent_only
end

#pattern ⇒ Object (readonly)

Returns the value of attribute pattern.



34
35
36
 
# File 'lib/remi/extractor/file_system.rb', line 34

def pattern
  @pattern
end

#remote_path ⇒ Object (readonly)

Returns the value of attribute remote_path.



33
34
35
 
# File 'lib/remi/extractor/file_system.rb', line 33

def remote_path
  @remote_path
end

Instance Method Details

#all_entries ⇒ Object

Public: Returns an array of all FileSystemEntry instances that are in the remote_path. NOTE: all_entries is responsible for matching the path using @remote_path

Raises:

  • (NoMethodError) 


50
51
52
 
# File 'lib/remi/extractor/file_system.rb', line 50

def all_entries
  raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
end

#entries ⇒ Object

Public: Returns just the entries that are to be extracted.



55
56
57
58
59
60
61
62
63
 
# File 'lib/remi/extractor/file_system.rb', line 55

def entries
  if @group_by
    most_recent_matching_entry_in_group
  elsif @most_recent_only
    Array(most_recent_matching_entry)
  else
    matching_entries
  end
end

#extract ⇒ Object

Public: Called to extract files from the source filesystem.

Returns an array with containing the paths to all files extracted.

Raises:

  • (NoMethodError) 


44
45
46
 
# File 'lib/remi/extractor/file_system.rb', line 44

def extract
  raise NoMethodError, "#{__method__} not defined for#{self.class.name}"
end

#matching_entries ⇒ Object 



65
66
67
 
# File 'lib/remi/extractor/file_system.rb', line 65

def matching_entries
  all_entries.select { |e| @pattern.match e.name }
end

#most_recent_matching_entry ⇒ Object 



69
70
71
 
# File 'lib/remi/extractor/file_system.rb', line 69

def most_recent_matching_entry
  matching_entries.sort_by { |e| e.send(@most_recent_by) }.reverse.first
end

#most_recent_matching_entry_in_group ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/remi/extractor/file_system.rb', line 73

def most_recent_matching_entry_in_group
  entries_with_group = matching_entries.map do |entry|
    match = entry.name.match(@group_by)
    next unless match

    group = match.to_a[1..-1]
    { group: group, entry: entry }
  end.compact
  sorted_entries_with_group = entries_with_group.sort_by { |e| [e[:group], e[:entry].send(@most_recent_by)] }.reverse

  last_group = nil
  sorted_entries_with_group.map do |entry|
    next unless entry[:group] != last_group
    last_group = entry[:group]
    entry[:entry]
  end.compact
end