Class: Bricolage::S3DataSource

Inherits:
DataSource show all
Defined in:
lib/bricolage/psqldatasource.rb,
lib/bricolage/s3datasource.rb

Overview

reopen

Constant Summary

Constants inherited from DataSource

DataSource::CLASSES

Instance Attribute Summary collapse

Attributes inherited from DataSource

#context, #logger, #name

Instance Method Summary collapse

Methods inherited from DataSource

get_class, new_for_type, #open, #open_for_batch

Constructor Details

#initialize(endpoint: 's3-ap-northeast-1.amazonaws.com', region: 'ap-northeast-1', bucket: nil, prefix: nil, access_key_id: nil, secret_access_key: nil, master_symmetric_key: nil, encryption: nil, s3cfg: nil) ⇒ S3DataSource

Returns a new instance of S3DataSource.



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/bricolage/s3datasource.rb', line 11

def initialize(
    endpoint: 's3-ap-northeast-1.amazonaws.com',
    region: 'ap-northeast-1',
    bucket: nil, prefix: nil,
    access_key_id: nil, secret_access_key: nil, master_symmetric_key: nil,
    encryption: nil,
    s3cfg: nil)
  @endpoint = (/\Ahttps?:/ =~ endpoint) ? endpoint : "https://#{endpoint}"
  @region = region
  @bucket_name = bucket
  @prefix = (prefix && prefix.empty?) ? nil : prefix
  @access_key_id = access_key_id
  @secret_access_key = secret_access_key
  @master_symmetric_key = master_symmetric_key
  @encryption = encryption
  @s3cfg = s3cfg
  @configurations = @s3cfg ? load_configurations(@s3cfg) : nil
end

Instance Attribute Details

#bucket_name ⇒ Object (readonly)

Returns the value of attribute bucket_name.



32
33
34
 
# File 'lib/bricolage/s3datasource.rb', line 32

def bucket_name
  @bucket_name
end

#encryption ⇒ Object (readonly)

Returns the value of attribute encryption.



73
74
75
 
# File 'lib/bricolage/s3datasource.rb', line 73

def encryption
  @encryption
end

#endpoint ⇒ Object (readonly)

Returns the value of attribute endpoint.



30
31
32
 
# File 'lib/bricolage/s3datasource.rb', line 30

def endpoint
  @endpoint
end

#prefix ⇒ Object (readonly)

Returns the value of attribute prefix.



33
34
35
 
# File 'lib/bricolage/s3datasource.rb', line 33

def prefix
  @prefix
end

#region ⇒ Object (readonly)

Returns the value of attribute region.



31
32
33
 
# File 'lib/bricolage/s3datasource.rb', line 31

def region
  @region
end

Instance Method Details

#access_key ⇒ Object 



48
49
50
 
# File 'lib/bricolage/s3datasource.rb', line 48

def access_key
  @access_key_id || get_config('access_key')
end

#bucket ⇒ Object 



87
88
89
90
 
# File 'lib/bricolage/s3datasource.rb', line 87

def bucket
  @resource ||= Aws::S3::Resource.new(client: client)
  @bucket ||= @resource.bucket(@bucket_name)
end

#client ⇒ Object

Ruby Interface



83
84
85
 
# File 'lib/bricolage/s3datasource.rb', line 83

def client
  @client ||= Aws::S3::Client.new(region: @region, endpoint: @endpoint, access_key_id: access_key, secret_access_key: secret_key)
end

#credential_string ⇒ Object

For Redshift



40
41
42
43
44
45
46
 
# File 'lib/bricolage/s3datasource.rb', line 40

def credential_string
  [
    "aws_access_key_id=#{access_key}",
    "aws_secret_access_key=#{secret_key}",
    (@master_symmetric_key && "master_symmetric_key=#{@master_symmetric_key}")
  ].compact.join(';')
end

#encrypted? ⇒ Boolean

Returns:

  • (Boolean) 


75
76
77
 
# File 'lib/bricolage/s3datasource.rb', line 75

def encrypted?
  !!(@master_symmetric_key or @encryption)
end

#get_config(key) ⇒ Object 



56
57
58
 
# File 'lib/bricolage/s3datasource.rb', line 56

def get_config(key)
  @configurations[key] or raise ParameterError, "missing s3cfg entry: #{key}"
end

#load_configurations(path) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/bricolage/s3datasource.rb', line 60

def load_configurations(path)
  h = {}
  File.foreach(path) do |line|
    case line
    when /\A\s*\w+\s*=\s*/
      key, value = line.split('=', 2)
      val = value.strip
      h[key.strip] = val.empty? ? nil : val
    end
  end
  h
end

#new_task ⇒ Object 



35
36
37
 
# File 'lib/bricolage/s3datasource.rb', line 35

def new_task
  S3Task.new(self)
end

#object(rel, no_prefix: false) ⇒ Object 



92
93
94
 
# File 'lib/bricolage/s3datasource.rb', line 92

def object(rel, no_prefix: false)
  bucket.object(path(rel, no_prefix: no_prefix))
end

#path(rel, no_prefix: false) ⇒ Object 



100
101
102
103
 
# File 'lib/bricolage/s3datasource.rb', line 100

def path(rel, no_prefix: false)
  path = (no_prefix || !@prefix) ? rel.to_s : "#{@prefix}/#{rel}"
  path.sub(%r<\A/>, '').gsub(%r<//>, '/')
end

#redshift_loader_source? ⇒ Boolean

Returns:

  • (Boolean) 


151
152
153
 
# File 'lib/bricolage/psqldatasource.rb', line 151

def redshift_loader_source?
  true
end

#secret_key ⇒ Object 



52
53
54
 
# File 'lib/bricolage/s3datasource.rb', line 52

def secret_key
  @secret_access_key || get_config('secret_key')
end

#traverse(rel, no_prefix: false) ⇒ Object 



105
106
107
 
# File 'lib/bricolage/s3datasource.rb', line 105

def traverse(rel, no_prefix: false)
  bucket.objects(prefix: path(rel, no_prefix: no_prefix))
end

#url(rel, no_prefix: false) ⇒ Object 



96
97
98
 
# File 'lib/bricolage/s3datasource.rb', line 96

def url(rel, no_prefix: false)
  "s3://#{@bucket_name}/#{path(rel, no_prefix: no_prefix)}"
end