Class: Remi::Extractor::S3File

Inherits:
FileSystem show all
Defined in:
lib/remi/extractor/s3_file.rb

Instance Attribute Summary

Attributes inherited from FileSystem

#group_by, #local_path, #logger, #most_recent_by, #most_recent_only, #pattern, #remote_path

Instance Method Summary collapse

Methods inherited from FileSystem

#entries, #matching_entries, #most_recent_matching_entry, #most_recent_matching_entry_in_group

Constructor Details

#initialize(*args, **kargs, &block) ⇒ S3File

Returns a new instance of S3File.



6
7
8
9
 
# File 'lib/remi/extractor/s3_file.rb', line 6

def initialize(*args, **kargs, &block)
  super
  init_s3_file(*args, **kargs, &block)
end

Instance Method Details

#all_entries ⇒ Object

Public: Returns an array of all FileSystemEntry instances that are in the remote_path.



24
25
26
 
# File 'lib/remi/extractor/s3_file.rb', line 24

def all_entries
  @all_entries ||= all_entries!
end

#all_entries! ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/remi/extractor/s3_file.rb', line 28

def all_entries!
  # S3 does not track anything like a create time, so use last modified for both
  bucket.objects(prefix: @remote_path.to_s).map do |entry|
    FileSystemEntry.new(
      pathname: entry.key,
      create_time: entry.last_modified,
      modified_time: entry.last_modified,
      raw: entry
    )
  end
end

#extract ⇒ Object

Public: Called to extract files from the source filesystem.

Returns an array with containing the paths to all files extracted.



14
15
16
17
18
19
20
21
 
# File 'lib/remi/extractor/s3_file.rb', line 14

def extract
  entries.map do |entry|
    local_file = File.join(@local_path, entry.name)
    @logger.info "Downloading #{entry.pathname} from S3 to #{local_file}"
    File.open(local_file, 'wb') { |file| entry.raw.get(response_target: file) }
    local_file
  end
end

#s3_client ⇒ Object 



40
41
42
 
# File 'lib/remi/extractor/s3_file.rb', line 40

def s3_client
  @s3_client ||= Aws::S3::Client.new
end