Class: QueueryClient::S3DataFileBundle

Inherits:
DataFileBundle show all
Defined in:
lib/queuery_client/s3_data_file_bundle.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods inherited from DataFileBundle

#each_row

Constructor Details

#initialize(bucket, prefix, s3_client: nil, logger: Logger.new($stderr)) ⇒ S3DataFileBundle

Returns a new instance of S3DataFileBundle.



9
10
11
12
13
14
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 9

def initialize(bucket, prefix, s3_client: nil, logger: Logger.new($stderr))
  @s3_client = s3_client || Aws::S3::Client.new   # Use env to inject credentials
  @bucket = bucket
  @prefix = prefix
  @logger = logger
end

Instance Attribute Details

#bucket ⇒ Object (readonly)

Returns the value of attribute bucket.



16
17
18
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 16

def bucket
  @bucket
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



18
19
20
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 18

def logger
  @logger
end

#prefix ⇒ Object (readonly)

Returns the value of attribute prefix.



17
18
19
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 17

def prefix
  @prefix
end

Instance Method Details

#data_files ⇒ Object 



24
25
26
27
28
29
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 24

def data_files
  b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
  b.objects(prefix: @prefix)
    .select {|obj| obj.key.include?('_part_') }
    .map {|obj| S3DataFile.new(obj) }
end

#has_manifest? ⇒ Boolean

Returns:

  • (Boolean) 


41
42
43
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 41

def has_manifest?
  !manifest_file.nil?
end

#manifest_file ⇒ Object 



31
32
33
34
35
36
37
38
39
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 31

def manifest_file
  b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
  obj = b.object("#{@prefix}manifest")
  if obj.exists?
    S3ManifestFile.new(obj)
  else
    nil
  end
end

#url ⇒ Object 



20
21
22
 
# File 'lib/queuery_client/s3_data_file_bundle.rb', line 20

def url
  "s3://#{@bucket}/#{@prefix}"
end