Class: Manage::S3

Inherits:
Object
  • Object
show all
Defined in:
lib/manage_s3_bucket.rb

Instance Method Summary collapse

Constructor Details

#initialize(aws_access_key_id, aws_secret_key_id) ⇒ S3

nodoc



22
23
24
25
 
# File 'lib/manage_s3_bucket.rb', line 22

def initialize(aws_access_key_id, aws_secret_key_id)
  @aws_access_key_id = aws_access_key_id
  @aws_secret_key_id = aws_secret_key_id
end

Instance Method Details

#copy_key(_s3, source_path, dest_path, key) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/manage_s3_bucket.rb', line 74

def copy_key(_s3, source_path, dest_path, key)
  copied = false
  i = 1
  source_bucket, source_directory = extract_path(source_path)
  dest_bucket, dest_directory     = extract_path(dest_path)
  new_key = "#{dest_directory}#{key.gsub(source_directory, '')}"
  while !copied
    begin
      _s3.CopyObject(
        'SourceBucket'      => source_bucket,
        'SourceObject'      => key,
        'DestinationBucket' => dest_bucket,
        'DestinationObject' => new_key
      )
      copied = true
      pp "SUCCESS: Copyed #{key} to #{new_key}"
    rescue
      i += 1
      pp $!
      sleep 10
      pp '#'*200
      pp "ERROR: Trying #{i} to copy #{key} to #{new_key}"
      _s3 = s3
      copy_key(_s3, source_path, dest_path, key)
    end
  end
end

#copy_path(source_path, dest_path) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/manage_s3_bucket.rb', line 102

def copy_path(source_path, dest_path)
  # It's not too fast as s3mirror (java tool)
  max_threads = 500
  wq = WorkQueue.new max_threads
  execute_in_path source_path do |keys|
    wq.enqueue_b do
      _s3 = s3
      keys.each do |key|
        copy_key(_s3, source_path, dest_path, key)
      end
    end
  end
end

#execute_in_path(path, marker = nil, &block) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/manage_s3_bucket.rb', line 48

def execute_in_path(path, marker = nil, &block)
  marker ||= nil
  bucket, directory = extract_path(path)
  list = list_directory(path, marker)
  objects = list['Contents']
  if objects
    keys = objects.map { |c| c['Key'] }
    yield keys
    if list["IsTruncated"] == "true"
      marker = keys.last
      p "Next Page: #{marker}"
      execute_in_path path, marker do |keys|
        yield keys
      end
    end
  end
end

#extract_path(path) ⇒ Object 



31
32
33
34
 
# File 'lib/manage_s3_bucket.rb', line 31

def extract_path(path)
  slice_path = path.split('/')
  [slice_path[0], slice_path[1..-1].join('/')]
end

#list_directory(path, marker = nil) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/manage_s3_bucket.rb', line 36

def list_directory(path, marker = nil)
  bucket, directory = extract_path(path)
  opt_list = {
    'Bucket' => bucket,
    'prefix' => directory,
    "max-keys" => 1000
  }
  opt_list['marker'] = marker if marker
  list = s3.ListObjects(opt_list)['ListBucketResult']
  return list
end

#remove_path(path) ⇒ Object 



66
67
68
69
70
71
72
 
# File 'lib/manage_s3_bucket.rb', line 66

def remove_path(path)
  bucket, directory = extract_path(path)
  p "Deleting #{path}"
  execute_in_path path do |keys|
    s3.DeleteMultipleObjects('Bucket' => bucket, 'Object' => keys)
  end
end

#s3 ⇒ Object 



27
28
29
 
# File 'lib/manage_s3_bucket.rb', line 27

def s3
  @s3 ||= RightScale::CloudApi::AWS::S3::Manager::new(@aws_access_key_id, @aws_secret_key_id, 'https://s3.amazonaws.com')
end