Module: Buckler

Extended by:
Logging
Defined in:
lib/buckler/aws.rb,
lib/buckler/heroku.rb,
lib/buckler/actions.rb,
lib/buckler/logging.rb,
lib/buckler/regions.rb,
lib/buckler/version.rb,
lib/buckler/thread_dispatch.rb,
lib/buckler/actions/empty_bucket.rb,
lib/buckler/actions/list_buckets.rb,
lib/buckler/actions/list_regions.rb,
lib/buckler/actions/sync_buckets.rb,
lib/buckler/actions/create_bucket.rb,
lib/buckler/actions/destroy_bucket.rb

Defined Under Namespace

Modules: Commands, Logging, VERSION Classes: ThreadDispatch

Constant Summary collapse

S3_BUCKET_REGIONS = 
{
  "us-east-1"      => "🇺🇸  US East (Virginia)",
  "us-west-1"      => "🇺🇸  US West (California)",
  "us-west-2"      => "🇺🇸  US West (Oregon)",
  "sa-east-1"      => "🇧🇷  South America (São Paulo)",
  "eu-central-1"   => "🇩🇪  Europe (Frankfurt)",
  "eu-west-1"      => "🇮🇪  Europe (Ireland)",
  "ap-northeast-1" => "🇯🇵  Asia Pacific (Tokyo)",
  "ap-northeast-2" => "🇰🇷  Asia Pacific (Seoul)",
  "ap-south-1"     => "🇮🇳  Asia Pacific (Mumbai)",
  "ap-southeast-1" => "🇸🇬  Asia Pacific (Singapore)",
  "ap-southeast-2" => "🇦🇺  Asia Pacific (Sydney)",
  "cn-north-1"     => "🇨🇳  China (Beijing)",
}.freeze

Class Method Summary collapse

Methods included from Logging

alert, log, verbose

Class Method Details

.aws_access_key_id ⇒ Object

Returns the discovered AWS Access Key ID Prerequisite: Buckler.discover_aws_credentials!



5
6
7
 
# File 'lib/buckler/aws.rb', line 5

def self.aws_access_key_id
  @aws_access_key_id
end

.connect_to_s3!(region: "us-east-1") ⇒ Object

Returns an Aws::S3::Client in the given region Prerequisite: Buckler.discover_aws_credentials!



11
12
13
14
15
16
17
18
19
 
# File 'lib/buckler/aws.rb', line 11

def self.connect_to_s3!(region:"us-east-1")
  return @s3 if @s3.present?
  @s3 = Aws::S3::Client.new(
    region: region,
    access_key_id: @aws_access_key_id,
    secret_access_key: @aws_secret_access_key,
  )
  return @s3
end

.create_bucket!(name: nil, region: nil) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/buckler/actions/create_bucket.rb', line 3

def self.create_bucket!(name:nil, region:nil)

  unless name.present?
    alert "No bucket name provided."
    alert "Usage: bucket create <bucket-name> --region <region>"
    exit false
  end

  region ||= "us-east-1"
  unless valid_region?(region)
    log "Invalid region “#{region}”"
    log "Use `bucket regions` to see a list of all S3 regions"
    exit false
  end

  connect_to_s3!(region:region)
  @bucket = Aws::S3::Bucket.new(name, client:@s3)

  if @bucket.exists?
    alert "Bucket #{@bucket.name} already exists"
    exit false
  end

  log "Creating bucket #{name.bucketize} on #{region}…"

  options = {
    acl: "private"
  }

  unless region.eql?("us-east-1")
    options[:create_bucket_configuration] = {
      location_constraint: region
    }
  end

  @bucket.create(options)
  @bucket.wait_until_exists

  log "Bucket #{name.bucketize} is how available for use ✔"
  exit true

rescue Aws::S3::Errors::BucketAlreadyExists

  alert "The bucket name “#{name}” is already taken."
  alert "Bucket names must be unique across the entire AWS ecosystem."
  alert "Select a different bucket name and re-run your command."
  exit false

end

.destroy_bucket!(name:, confirmation: nil) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/buckler/actions/destroy_bucket.rb', line 3

def self.destroy_bucket!(name:, confirmation:nil)

  connect_to_s3!
  @bucket = get_bucket!(name)
  require_confirmation!(name_required:name, confirmation:confirmation)

  if @bucket.versioning.status == "Enabled"
    log "The bucket #{name.bucketize} has versioning enabled, it cannot be deleted."
    log "You must disable versioning in the AWS Mangement Console."
    exit false
  end

  log "Destroying bucket #{name.bucketize}…"
  @bucket.delete!(max_attempts:3)
  log "Bucket #{name.bucketize} was destroyed ✔"
  exit true

end

.discover_aws_credentials!(key_id: nil, key: nil) ⇒ Object

Attempts to find the AWS Access Key ID and Secret Access Key by searching the command line paramters, the environment, the .env, and Heroku in that order. The parameters are the values of --id and --secret on the command line. The program ends if credentials cannot be discovered.



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/buckler/aws.rb', line 25

def self.discover_aws_credentials!(key_id:nil, key:nil)

  verbose "Attempting to find AWS credentials…"

  # Try to find keys as command line parameters, if the invoker has set them directly
  if key_id.present? && key.present?
    verbose "The Access Key ID and Secret Access Key were set as command line options ✔"
    @aws_access_key_id = key_id
    @aws_secret_access_key = key
    return true
  end

  # Try to find keys in the current environment, if the invoker has set them directly
  key_id = ENV["AWS_ACCESS_KEY_ID"]
  key = ENV["AWS_SECRET_ACCESS_KEY"]
  if key_id.present? && key.present?
    verbose "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY found as environment variables ✔"
    @aws_access_key_id = key_id
    @aws_secret_access_key = key
    return true
  end

  # Try to find keys in a .env file in this directory
  Dotenv.load
  key_id = ENV["AWS_ACCESS_KEY_ID"]
  key = ENV["AWS_SECRET_ACCESS_KEY"]
  if key_id.present? && key.present?
    verbose "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY found in the .env file ✔"
    @aws_access_key_id = key_id
    @aws_secret_access_key = key
    return true
  end

  # Try to find keys by asking Heroku about the project in this directory
  if heroku_available?
    key_id = heroku_config_get("AWS_ACCESS_KEY_ID")
    key = heroku_config_get("AWS_SECRET_ACCESS_KEY")
    if key_id.present? && key.present?
      verbose "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY found on your Heroku application ✔"
      @aws_access_key_id = key_id
      @aws_secret_access_key = key
      return true
    end
  end

  alert "Could not discover any AWS credentials."
  alert "Set command line options --id and --secret"
  alert "Or, set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as environment variables."
  alert "Or, set them in a .env file in this directory."
  if heroku_available?
    alert "Or, set them on a Heroku application in this directory with `heroku config:set`."
  end
  exit false

end

.empty_bucket!(name:, confirmation: nil) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
 
# File 'lib/buckler/actions/empty_bucket.rb', line 3

def self.empty_bucket!(name:, confirmation:nil)

  connect_to_s3!
  @bucket = get_bucket!(name)
  require_confirmation!(name_required:name, confirmation:confirmation)

  log "Deleting all objects in bucket #{name.bucketize}…"
  @bucket.clear!
  log "Bucket #{name.bucketize} is now empty ✔"
  exit true

end

.get_bucket!(name) ⇒ Object

Generate an Aws::S3::Bucket for the given name Also checks that the bucket is real and we have access to it. The only way to truly test bucket access is to try to read an item from it. Exit with a message if there is no access. Prerequisite: Buckler.discover_aws_credentials!



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/buckler/actions.rb', line 9

def self.get_bucket!(name)

  unless name.present?
    alert "No bucket name provided"
    exit false
  end

  @bucket = Aws::S3::Bucket.new(name, client:@s3)

  unless @bucket.exists?
    alert "No such bucket “#{name}”"
    exit false
  end

  @bucket.objects(max_keys:1).first # Tests bucket access
  return @bucket

rescue Aws::S3::Errors::NoSuchBucket
  
  alert "No such bucket “#{name}”"
  exit false

rescue Aws::S3::Errors::AccessDenied

  alert "Access denied for bucket #{name}"
  exit false

end

.heroku_available? ⇒ Boolean

True if the user has a Heroku and Ruby executable

Returns:

  • (Boolean) 


14
15
16
 
# File 'lib/buckler/heroku.rb', line 14

def self.heroku_available?
  ruby_cmd.present? && heroku_cmd.present?
end

.heroku_cmd ⇒ Object

Returns the Heroku executable on the user’s $PATH



9
10
11
 
# File 'lib/buckler/heroku.rb', line 9

def self.heroku_cmd
  @heroku_cmd ||= find_executable0("heroku")
end

.heroku_config_get(variable_name) ⇒ Object

Fetches the given environment variable_name from the user’s Heroku project



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/buckler/heroku.rb', line 19

def self.heroku_config_get(variable_name)

  command_output, command_intake = IO.pipe

  pid = Kernel.spawn(
    "#{ruby_cmd} #{heroku_cmd} config:get #{variable_name}",
    STDOUT => command_intake,
    STDERR => command_intake
  )

  command_intake.close

  _, status = Process.wait2(pid)

  if status.exitstatus == 0
    results = command_output.read.to_s.chomp
    verbose %{`heroku config:get #{variable_name}` returned "#{results}"}
    return results
  else
    verbose %{`heroku config:get #{variable_name}` returned a nonzero exit status}
    return false
  end

end

.list_buckets! ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/buckler/actions/list_buckets.rb', line 3

def self.list_buckets!

  connect_to_s3!

  verbose "Fetching buckets visible to #{@aws_access_key_id}…"
  table = [["NAME", "REGION", "VERSIONING"]]

  @s3.list_buckets.buckets.each do |bucket|
    region = @s3.get_bucket_location(bucket:bucket.name).location_constraint.presence || "us-east-1"
    versioning = @s3.get_bucket_versioning(bucket:bucket.name).status.presence || "Not Configured"
    table << [bucket.name, region, versioning]
  end

  puts_table!(table)
  exit true

end

.list_regions! ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/buckler/actions/list_regions.rb', line 3

def self.list_regions!

  table = [["REGION", "NAME", nil]]

  S3_BUCKET_REGIONS.each do |name, human_name|
    case name
    when "us-east-1"
      table << [name, human_name, "Default region"]
    when "cn-north-1"
      table << [name, human_name, "Requires chinese account"]
    else
      table << [name, human_name, nil]
    end
  end

  puts_table!(table)
  exit true

end

.puts_table!(table_array) ⇒ Object

Prints a table neatly to the screen. The given table_array must be an Array of Arrays of Strings. Each inner array is a single row of the table, strings are cells of the row.



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/buckler/actions.rb', line 72

def self.puts_table!(table_array)

  column_sizes = []

  table_array.first.count.times do |column_index|
    column_sizes << table_array.collect{ |row| row[column_index] }.collect(&:to_s).collect(&:length).max + 3
  end

  table_array.each do |line|
    chart_row = ""
    line.each_with_index do |column, index|
      chart_row << column.to_s.ljust(column_sizes[index])
    end
    log chart_row
  end

end

.require_confirmation!(name_required:, confirmation: nil, additional_lines: []) ⇒ Object

Prints a warning message about irreversable changes to the screen. The user is required to confirm by typing the given name_required additional_lines are printed before the warning. If confirmation matches name_required, this method is a no-op. The program ends if the confirmation is not provided.



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/buckler/actions.rb', line 44

def self.require_confirmation!(name_required:, confirmation:nil, additional_lines:[])

  return true if confirmation == name_required

  alert "WARNING: Destructive Action"
  additional_lines.each do |line|
    log line
  end
  log "Depending on your S3 settings, this command may permanently"
  log "delete objects from the bucket #{name_required.bucketize}."
  log "To proceed, type “#{name_required}” or re-run this command with --confirm #{name_required}"
  print "> ".dangerize

  confirmation = STDIN.gets.chomp

  if confirmation == name_required
    return true
  else
    alert "Invalid confirmation “#{name_required}”, aborting"
    exit false
  end

end

.ruby_cmd ⇒ Object

Returns the Ruby executable on the user’s $PATH



4
5
6
 
# File 'lib/buckler/heroku.rb', line 4

def self.ruby_cmd
  @ruby_cmd ||= find_executable0("ruby")
end

.sync_buckets!(source_name:, target_name:, confirmation: nil) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/buckler/actions/sync_buckets.rb', line 3

def self.sync_buckets!(source_name:, target_name:, confirmation:nil)

  unless source_name.present? && target_name.present?
    alert "You must provide both a source bucket and a target bucket"
    alert "Usage: bucket sync <source-bucket> <target-bucket>"
    exit false
  end

  unless source_name != target_name
    alert "The source bucket name and target bucket name must be different"
    exit false
  end

  connect_to_s3!
  @source_bucket = get_bucket!(source_name)
  @target_bucket = get_bucket!(target_name)

  source_name = @source_bucket.name.bucketize(:pink).freeze
  target_name = @target_bucket.name.bucketize.freeze

  require_confirmation!(name_required:@target_bucket.name, confirmation:confirmation, additional_lines:[
    "The contents of #{source_name} will be synced into #{target_name}.",
    "Objects in #{target_name} that aren’t in the source bucket will be removed.",
  ])

  log "Syncing #{source_name} into #{target_name}…"
  log "Fetching bucket file lists…"

  @source_bucket_keys = @source_bucket.objects.collect(&:key)
  @target_bucket_keys = @target_bucket.objects.collect(&:key)

  # -------------------------------------------------------------------------
  # Delete bucket differences
  # -------------------------------------------------------------------------

  @keys_to_delete = @target_bucket_keys - @source_bucket_keys

  @dispatch = Buckler::ThreadDispatch.new

  log "Deleting unshared objects from target bucket…"

  @keys_to_delete.lazy.each do |key|
    @dispatch.queue(lambda {
      log "Deleting #{target_name}/#{key}"
      @target_bucket.object(key).delete
    })
  end

  time_elapsed = @dispatch.perform_and_wait
  log "Unshared objects deleted from target bucket (#{time_elapsed} seconds) ✔"

  # -------------------------------------------------------------------------
  # Sync files
  # -------------------------------------------------------------------------

  @dispatch = Buckler::ThreadDispatch.new

  @source_bucket_keys.lazy.each do |object_key|

    @dispatch.queue(lambda {

      source_object = Aws::S3::Object.new(@source_bucket.name, object_key, client:@s3)
      target_object = Aws::S3::Object.new(@target_bucket.name, object_key, client:@s3)

      options = {
        storage_class: source_object.storage_class,
        metadata: source_object.metadata,
        content_encoding: source_object.content_encoding,
        content_language: source_object.content_language,
        content_type: source_object.content_type,
        cache_control: source_object.cache_control,
        expires: source_object.expires,
      }

      if source_object.content_disposition.present?
        options[:content_disposition] = ActiveSupport::Inflector.transliterate(source_object.content_disposition, "")
      end

      if source_object.content_length > 5242882 # 5 megabytes + 2 bytes
        options[:multipart_copy] = true
        options[:content_length] = source_object.content_length
      end

      target_object.copy_from(source_object, options)
      target_object.acl.put({
        access_control_policy: {
          grants: source_object.acl.grants,
          owner: source_object.acl.owner,
        }
      })

      log "Copied #{source_name} → #{target_name}/#{object_key}"

    })

  end

  time_elapsed = @dispatch.perform_and_wait
  log "#{@source_bucket_keys.count} objects synced in #{target_name} (#{time_elapsed} seconds) ✔"
  exit true

end

.valid_region?(name) ⇒ Boolean

True if the given name is a valid AWS region.

Returns:

  • (Boolean) 


19
20
21
 
# File 'lib/buckler/regions.rb', line 19

def self.valid_region?(name)
  S3_BUCKET_REGIONS.keys.include?(name)
end

.version ⇒ Object

Returns Buckler’s version number



4
5
6
 
# File 'lib/buckler/version.rb', line 4

def self.version
  Gem::Version.new("1.0.1")
end