Class: Mortar::S3::S3

Inherits:
Object
  • Object
show all
Includes:
Helpers
Defined in:
lib/mortar/s3.rb

Instance Method Summary collapse

Methods included from Helpers

#action, #ask, #confirm, #copy_if_not_present_at_dest, #default_host, #deprecate, #display, #display_header, #display_object, #display_row, #display_table, #display_with_indent, #download_to_file, #ensure_dir_exists, #error, error_with_failure, error_with_failure=, extended, extended_into, #format_bytes, #format_date, #format_with_bang, #full_host, #get_terminal_environment, #home_directory, #host, #hprint, #hputs, included, included_into, #installed_with_omnibus?, #json_decode, #json_encode, #line_formatter, #longest, #output_with_bang, #pending_github_team_state_message, #quantify, #redisplay, #retry_on_exception, #running_on_a_mac?, #running_on_windows?, #set_buffer, #shell, #spinner, #status, #string_distance, #styled_array, #styled_error, #styled_hash, #styled_header, #suggestion, #test_name, #ticking, #time_ago, #truncate, #warning, #with_tty, #write_to_file

Instance Method Details

#check_bucket(s3, bucket) ⇒ Object

checks s3 bucket to see if bucket and key exists for given aws keys



107
108
109
 
# File 'lib/mortar/s3.rb', line 107

def check_bucket(s3, bucket)
  s3.buckets[bucket].exists?
end

#do_download(bucket, key, output_path, concat = nil) ⇒ Object

Implementation of the download command



160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/mortar/s3.rb', line 160

def do_download(bucket, key, output_path, concat=nil)
  s3 = get_s3
  # check and messages for concatting file
  unless check_bucket(s3, bucket)
   error("Requested S3 bucket #{bucket} in path, s3://#{bucket}/#{key}, does not exist.")
  end
  # creates output directory
  unless File.directory?(output_path)
    FileUtils.mkdir_p(output_path)
  end
  download_s3(s3, bucket, key, output_path, concat)

  display "All done! "
end

#download_s3(s3, bucket, key, output_path, concat = nil) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/mortar/s3.rb', line 139

def download_s3(s3, bucket, key, output_path, concat = nil)
    s3_objects = get_s3_objects(s3, bucket,key)
    concat_file_name = "output"
    concat_target = "#{output_path}/#{concat_file_name}"
    if concat
      remove_file(concat_target)
    end
    if !s3_objects.empty?
      s3_objects.each do |s3_obj|
        key_str = s3_obj.key
        out_file = concat ? concat_file_name : get_file_name(key_str)
        output_target = "#{output_path}/#{out_file}"
        display "Writing s3://#{bucket}/#{key_str} to #{output_target}"
        write_s3_to_file(s3_obj, output_target, concat ? "a" : "w+")
      end
    else
      error("No contents were found at path s3://#{bucket}/#{key}.  Please specify again.")
    end
end

#get_bucket_and_key(s3_path) ⇒ Object

returns bucket and key from s3 path



42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/mortar/s3.rb', line 42

def get_bucket_and_key(s3_path)
  uri = ''
  begin
    uri = URI.parse(s3_path)
  rescue URI::InvalidURIError => msg
    error("#{msg}.\nIt is strongly suggested that your bucket name does not contain an underscore character.\nPlease see http://blog.mortardata.com/post/58920122308/s3-hadoop-performance at tip #4.") 
  end


  unless uri != '' && is_valid_s3_path(uri)
    error("Requested S3 path, #{s3_path}, is invalid. Please ensure that your S3 path begins with 's3://'.  Example: s3://my-bucket/my-key.")
  end
  return uri.host,uri.path[1, uri.path.length]
end

#get_file_name(key_str) ⇒ Object 



71
72
73
74
75
76
77
 
# File 'lib/mortar/s3.rb', line 71

def get_file_name(key_str)
    if !key_str.rindex("/") 
      return key_str
    else
      return Pathname.new(key_str).basename.to_s
    end
end

#get_s3 ⇒ Object

gets aws::s3 object



33
34
35
36
37
38
39
 
# File 'lib/mortar/s3.rb', line 33

def get_s3
  ctrl = Mortar::Local::Controller.new
  ctrl.require_aws_keys
  return AWS::S3.new(
    :access_key_id => ENV['AWS_ACCESS_KEY'],
    :secret_access_key => ENV['AWS_SECRET_KEY'])
end

#get_s3_objects(s3, bucket, key) ⇒ Object

gets s3 object, where each item is a file in bucket and key



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/mortar/s3.rb', line 113

def get_s3_objects(s3, bucket, key)
  buck = s3.buckets[bucket]
  # removes slash at end if it exists
  key = remove_slash(key) 
  if buck.objects[key].exists?
    [buck.objects[key]]
  else
    valid_items = Array.new
    buck.objects.with_prefix(key).each do |obj|
      if is_file(obj.key, key) 
        valid_items.push(obj)
      end
    end
    return valid_items
  end
end

#is_file(path, key) ⇒ Object 



79
80
81
82
83
84
85
86
87
 
# File 'lib/mortar/s3.rb', line 79

def is_file(path, key)
  if is_not_folder(path,key)
    res = Pathname.new(path).basename.to_s
    if !is_hidden_file(res)
      return true 
    end
  end 
  return false 
end

#is_hidden_file(str) ⇒ Object 



96
97
98
 
# File 'lib/mortar/s3.rb', line 96

def is_hidden_file(str)
  str[0,1] == "." || str[0,1] == "_"
end

#is_not_folder(path, key) ⇒ Object 



89
90
91
92
93
94
 
# File 'lib/mortar/s3.rb', line 89

def is_not_folder(path, key)
  suffix = ["/", "_$folder$"]
  # not a folder
  path[-1,1] != suffix[0]  && !path.index(suffix[1]) and !path[key.length+1, path.length].index("/")
  #path[-1,1] != suffix[0] && !path.index(suffix[1]) and Pathname.new(path).basename.to_s != '' 
end

#is_valid_s3_path(uri) ⇒ Object

checks if string is a valid s3 path



59
60
61
 
# File 'lib/mortar/s3.rb', line 59

def is_valid_s3_path(uri)
  uri.scheme == 's3' && uri.host && !uri.path[1, uri.path.length].to_s.empty?
end

#remove_file(file_path) ⇒ Object 



100
101
102
103
104
 
# File 'lib/mortar/s3.rb', line 100

def remove_file(file_path)
  if File.file?(file_path)
    FileUtils.remove(file_path)
  end
end

#remove_slash(str) ⇒ Object 



64
65
66
67
68
69
 
# File 'lib/mortar/s3.rb', line 64

def remove_slash(str)
  if str[-1, 1] == "/"
    str = str[0, str.length-1]
  end
  return str
end

#write_s3_to_file(s3_object, file_name, write_mode) ⇒ Object 



130
131
132
133
134
135
136
 
# File 'lib/mortar/s3.rb', line 130

def write_s3_to_file(s3_object, file_name, write_mode)
  File.open(file_name, write_mode) do |file|
    s3_object.read do |chunk|
      file.write(chunk)
    end
  end
end