Class: Pupistry::StorageAWS

Inherits:
Object
  • Object
show all
Defined in:
lib/pupistry/storage_aws.rb

Overview

Pupistry::StorageAWS

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(mode) ⇒ StorageAWS

Returns a new instance of StorageAWS.



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/pupistry/storage_aws.rb', line 13

def initialize(mode)
  # mode is either "build" or "agent", depending which we load a different
  # set of permissions. Awareness of both is intentional, since we want the
  # build machines to known the agent creds so we can generate bootstrap
  # template files.

  unless defined? $config['general']['s3_bucket']
    $logger.fatal 'You must set the AWS s3_bucket'
    exit 0
  end

  # Define AWS configuration
  if defined? $config[mode]['access_key_id']
    if $config[mode]['access_key_id'] == ''
      $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery'
      $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.'
    else
      $logger.debug 'Loading AWS credentials from configuration file'

      AWS.config(
        access_key_id:     $config[mode]['access_key_id'],
        secret_access_key: $config[mode]['secret_access_key'],
        region:            $config[mode]['region'],
        proxy_uri:         $config[mode]['proxy_uri']
      )
    end
  else
    $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery'
    $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.'
  end

  # Setup S3 bucket
  if defined? $config['general']['s3_endpoint'] and $config['general']['s3_endpoint'] != nil
    $logger.debug 'Connecting to alternative endpoint ' + $config['general']['s3_endpoint']
    @s3 = AWS::S3.new(
     s3_endpoint: $config['general']['s3_endpoint'],
     s3_force_path_style: true,
   )
  else
    @s3 = AWS::S3.new
  end
  @bucket = @s3.buckets[$config[mode]['s3_bucket']]
end

Instance Attribute Details

#bucket ⇒ Object

Returns the value of attribute bucket.



11
12
13
 
# File 'lib/pupistry/storage_aws.rb', line 11

def bucket
  @bucket
end

#s3 ⇒ Object

Returns the value of attribute s3.



10
11
12
 
# File 'lib/pupistry/storage_aws.rb', line 10

def s3
  @s3
end

Instance Method Details

#download(src, dest = 'stream') ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/pupistry/storage_aws.rb', line 96

def download(src, dest = 'stream')
  $logger.debug "Downloading file s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{src} to #{dest}"

  begin
    # Generate the object name/key based on the relative file name and path.
    s3_obj_name = "#{$config['general']['s3_prefix']}#{src}"
    s3_obj      = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name]

    # Download the file
    if dest == 'stream'
      # Return the contents rather than writing to disk. We assume stream mode
      # if the dest filename was unspecified
      return s3_obj.read
    else
      # Download to an ondisk file
      File.open(dest, 'wb') do |file|
        s3_obj.read do |chunk|
          file.write(chunk)
        end
      end
    end

  rescue AWS::S3::Errors::NoSuchKey
    $logger.debug 'No such file exists for download, this is normal at times.'
    return false

  rescue AWS::S3::Errors::NoSuchBucket
    $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist"
    exit 0

  rescue AWS::S3::Errors::AccessDenied
    $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied"
    exit 0

  rescue AWS::S3::Errors::InvalidObjectState
    $logger.warn "Unable to download \"#{src}\", it has been archived off into Glacier and would need to be recovered first."

    # Do we need to restore it?
    begin
      if s3_obj.restore_in_progress?
        $logger.warn "A restore of this file is currently in progress, but can take up to 4 hours - please re-try later."
      else
        # Not being restored currently, let's file a request. This allows
        # us to cater for situations where a bunch of servers need to get
        # an old manifest/file, however the fastest solution is to simply
        # do a new `pupistry push` from a workstation to upload a new
        # manifest file.
        if s3_obj.restore(:days => 30)
          $logger.warn "Recover request has been issued, this could take up to 4 hours to complete."
          $logger.warn "Note that doing a `pupistry push` from the workstation would solve this faster."
        end
      end
    rescue StandardError => e
      $logger.error "Glacier restore request for #{src} failed. (#{e.class}), best option is to push the latest manifest from a workstation with `pupistry push`."
    end

    return false

  rescue AWS::S3::Errors::PermanentRedirect => e
    $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}."
    raise e

  rescue AWS::S3::Errors::SignatureDoesNotMatch => e
    $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials"
    raise e

  rescue AWS::S3::Errors::MissingCredentialsError
    $logger.error 'AWS credentials not supplied. You must either:'
    $logger.error 'a) Specify them in the config file for Pupistry'
    $logger.error 'b) Use IAM roles with an EC2 instance.'
    $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY'
    return false

  rescue StandardError => e
    raise e
  end
end

#upload(src, dest) ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/pupistry/storage_aws.rb', line 57

def upload(src, dest)
  $logger.debug "Pushing file #{src} to s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{dest}"

  begin
    # Generate the object name/key based on the relative file name and path.
    s3_obj_name = "#{$config['general']['s3_prefix']}#{dest}"
    s3_obj      = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name]

    # Perform S3 upload
    s3_obj.write(file: src)

  rescue AWS::S3::Errors::NoSuchBucket
    $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist"
    exit 0

  rescue AWS::S3::Errors::AccessDenied
    $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied"
    exit 0

  rescue AWS::S3::Errors::PermanentRedirect => e
    $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}."
    raise e

  rescue AWS::S3::Errors::SignatureDoesNotMatch => e
    $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials"
    raise e

  rescue AWS::S3::Errors::MissingCredentialsError
    $logger.error 'AWS credentials not supplied. You must either:'
    $logger.error 'a) Specify them in the config file for Pupistry'
    $logger.error 'b) Use IAM roles with an EC2 instance.'
    $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY'
    return false

  rescue StandardError => e
    raise e
  end
end