Class: S3Helper

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_helper.rb

Defined Under Namespace

Classes: SettingMissing

Constant Summary collapse

FIFTEEN_MEGABYTES =
15 * 1024 * 1024

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(s3_bucket_name, tombstone_prefix = "", options = {}) ⇒ S3Helper

Returns a new instance of S3Helper.



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# File 'lib/s3_helper.rb', line 28

def initialize(s3_bucket_name, tombstone_prefix = "", options = {})
  @s3_client = options.delete(:client)
  @s3_options = default_s3_options.merge(options)

  @s3_bucket_name, @s3_bucket_folder_path =
    begin
      raise Discourse::InvalidParameters.new("s3_bucket_name") if s3_bucket_name.blank?
      self.class.get_bucket_and_folder_path(s3_bucket_name)
    end

  @tombstone_prefix =
    if @s3_bucket_folder_path
      File.join(@s3_bucket_folder_path, tombstone_prefix)
    else
      tombstone_prefix
    end
end

Instance Attribute Details

#s3_bucket_folder_pathObject (readonly)

Returns the value of attribute s3_bucket_folder_path.



11
12
13
# File 'lib/s3_helper.rb', line 11

def s3_bucket_folder_path
  @s3_bucket_folder_path
end

#s3_bucket_nameObject (readonly)

Returns the value of attribute s3_bucket_name.



11
12
13
# File 'lib/s3_helper.rb', line 11

def s3_bucket_name
  @s3_bucket_name
end

Class Method Details

.build_from_config(use_db_s3_config: false, for_backup: false, s3_client: nil) ⇒ Object



46
47
48
49
50
51
52
53
54
55
56
57
58
59
# File 'lib/s3_helper.rb', line 46

def self.build_from_config(use_db_s3_config: false, for_backup: false, s3_client: nil)
  setting_klass = use_db_s3_config ? SiteSetting : GlobalSetting
  options = S3Helper.s3_options(setting_klass)
  options[:client] = s3_client if s3_client.present?

  bucket =
    if for_backup
      setting_klass.s3_backup_bucket
    else
      use_db_s3_config ? SiteSetting.s3_upload_bucket : GlobalSetting.s3_bucket
    end

  S3Helper.new(bucket.downcase, "", options)
end

.get_bucket_and_folder_path(s3_bucket_name) ⇒ Object



61
62
63
# File 'lib/s3_helper.rb', line 61

def self.get_bucket_and_folder_path(s3_bucket_name)
  s3_bucket_name.downcase.split("/", 2)
end

.s3_options(obj) ⇒ Object



260
261
262
263
264
265
266
267
268
269
270
271
272
# File 'lib/s3_helper.rb', line 260

def self.s3_options(obj)
  opts = { region: obj.s3_region }

  opts[:endpoint] = SiteSetting.s3_endpoint if SiteSetting.s3_endpoint.present?
  opts[:http_continue_timeout] = SiteSetting.s3_http_continue_timeout

  unless obj.s3_use_iam_profile
    opts[:access_key_id] = obj.s3_access_key_id
    opts[:secret_access_key] = obj.s3_secret_access_key
  end

  opts
end

Instance Method Details

#abort_multipart(key:, upload_id:) ⇒ Object



289
290
291
# File 'lib/s3_helper.rb', line 289

def abort_multipart(key:, upload_id:)
  s3_client.abort_multipart_upload(bucket: s3_bucket_name, key: key, upload_id: upload_id)
end

#complete_multipart(upload_id:, key:, parts:) ⇒ Object



338
339
340
341
342
343
344
345
346
347
# File 'lib/s3_helper.rb', line 338

def complete_multipart(upload_id:, key:, parts:)
  s3_client.complete_multipart_upload(
    bucket: s3_bucket_name,
    key: key,
    upload_id: upload_id,
    multipart_upload: {
      parts: parts,
    },
  )
end

#copy(source, destination, options: {}) ⇒ Object



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
# File 'lib/s3_helper.rb', line 112

def copy(source, destination, options: {})
  if options[:apply_metadata_to_destination]
    options = options.except(:apply_metadata_to_destination).merge(metadata_directive: "REPLACE")
  end

  destination = get_path_for_s3_upload(destination)
  source_object =
    if !Rails.configuration.multisite || source.include?(multisite_upload_path) ||
         source.include?(@tombstone_prefix)
      s3_bucket.object(source)
    elsif @s3_bucket_folder_path
      folder, filename = source.split("/", 2)
      s3_bucket.object(File.join(folder, multisite_upload_path, filename))
    else
      s3_bucket.object(File.join(multisite_upload_path, source))
    end

  if source_object.size > FIFTEEN_MEGABYTES
    options[:multipart_copy] = true
    options[:content_length] = source_object.size
  end

  destination_object = s3_bucket.object(destination)

  # Note for small files that do not use multipart copy: Any options for metadata
  # (e.g. content_disposition, content_type) will not be applied unless the
  # metadata_directive = "REPLACE" option is passed in. If this is not passed in,
  # the source object's metadata will be used.
  # For larger files it copies the metadata from the source file and merges it
  # with values from the copy call.
  response = destination_object.copy_from(source_object, options)

  etag =
    if response.respond_to?(:copy_object_result)
      # small files, regular copy
      response.copy_object_result.etag
    else
      # larger files, multipart copy
      response.data.etag
    end

  [destination, etag.gsub('"', "")]
end

#create_multipart(key, content_type, metadata: {}) ⇒ Object



293
294
295
296
297
298
299
300
301
302
303
# File 'lib/s3_helper.rb', line 293

def create_multipart(key, content_type, metadata: {})
  response =
    s3_client.create_multipart_upload(
      acl: SiteSetting.s3_use_acls ? "private" : nil,
      bucket: s3_bucket_name,
      key: key,
      content_type: content_type,
      metadata: ,
    )
  { upload_id: response.upload_id, key: key }
end

#delete_object(key) ⇒ Object



103
104
105
106
# File 'lib/s3_helper.rb', line 103

def delete_object(key)
  s3_bucket.object(key).delete
rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NotFound
end

#delete_objects(keys) ⇒ Object



108
109
110
# File 'lib/s3_helper.rb', line 108

def delete_objects(keys)
  s3_bucket.delete_objects({ delete: { objects: keys.map { |k| { key: k } }, quiet: true } })
end

#download_file(filename, destination_path, failure_message = nil) ⇒ Object



274
275
276
277
278
279
# File 'lib/s3_helper.rb', line 274

def download_file(filename, destination_path, failure_message = nil)
  object(filename).download_file(destination_path)
rescue => err
  raise failure_message&.to_s ||
          "Failed to download #{filename} because #{err.message.length > 0 ? err.message : err.class.to_s}"
end

#ensure_cors!(rules = nil) ⇒ Object

Several places in the application need certain CORS rules to exist inside an S3 bucket so requests to the bucket can be made directly from the browser. The s3:ensure_cors_rules rake task is used to ensure these rules exist for assets, S3 backups, and direct S3 uploads, depending on configuration.



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
# File 'lib/s3_helper.rb', line 161

def ensure_cors!(rules = nil)
  return unless SiteSetting.s3_install_cors_rule
  rules = [rules] if !rules.is_a?(Array)
  existing_rules = fetch_bucket_cors_rules

  new_rules = rules - existing_rules
  return false if new_rules.empty?

  final_rules = existing_rules + new_rules

  begin
    s3_resource.client.put_bucket_cors(
      bucket: @s3_bucket_name,
      cors_configuration: {
        cors_rules: final_rules,
      },
    )
  rescue Aws::S3::Errors::AccessDenied
    Rails.logger.info(
      "Could not PutBucketCors rules for #{@s3_bucket_name}, rules: #{final_rules}",
    )
    return false
  end

  true
end

#list(prefix = "", marker = nil) ⇒ Object



237
238
239
240
241
# File 'lib/s3_helper.rb', line 237

def list(prefix = "", marker = nil)
  options = { prefix: get_path_for_s3_upload(prefix) }
  options[:marker] = marker if marker.present?
  s3_bucket.objects(options)
end

#list_multipart_parts(upload_id:, key:, max_parts: 1000, start_from_part_number: nil) ⇒ Object

Important note from the S3 documentation:

This request returns a default and maximum of 1000 parts. You can restrict the number of parts returned by specifying the max_parts argument. If your multipart upload consists of more than 1,000 parts, the response returns an IsTruncated field with the value of true, and a NextPartNumberMarker element.

In subsequent ListParts requests you can include the part_number_marker arg using the NextPartNumberMarker the field value from the previous response to get more parts.

See docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_parts-instance_method



330
331
332
333
334
335
336
# File 'lib/s3_helper.rb', line 330

def list_multipart_parts(upload_id:, key:, max_parts: 1000, start_from_part_number: nil)
  options = { bucket: s3_bucket_name, key: key, upload_id: upload_id, max_parts: max_parts }

  options[:part_number_marker] = start_from_part_number if start_from_part_number.present?

  s3_client.list_parts(options)
end

#object(path) ⇒ Object



256
257
258
# File 'lib/s3_helper.rb', line 256

def object(path)
  s3_bucket.object(get_path_for_s3_upload(path))
end

#path_from_url(url) ⇒ Object



85
86
87
# File 'lib/s3_helper.rb', line 85

def path_from_url(url)
  URI.parse(url).path.delete_prefix("/")
end

#presign_multipart_part(upload_id:, key:, part_number:) ⇒ Object



305
306
307
308
309
310
311
312
313
314
315
# File 'lib/s3_helper.rb', line 305

def presign_multipart_part(upload_id:, key:, part_number:)
  presigned_url(
    key,
    method: :upload_part,
    expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS,
    opts: {
      part_number: part_number,
      upload_id: upload_id,
    },
  )
end

#presigned_request(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) ⇒ Object

Returns url, headers in a tuple which is needed in some cases.



357
358
359
360
361
362
363
364
365
366
367
# File 'lib/s3_helper.rb', line 357

def presigned_request(
  key,
  method:,
  expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS,
  opts: {}
)
  Aws::S3::Presigner.new(client: s3_client).presigned_request(
    method,
    { bucket: s3_bucket_name, key: key, expires_in: expires_in }.merge(opts),
  )
end

#presigned_url(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) ⇒ Object



349
350
351
352
353
354
# File 'lib/s3_helper.rb', line 349

def presigned_url(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {})
  Aws::S3::Presigner.new(client: s3_client).presigned_url(
    method,
    { bucket: s3_bucket_name, key: key, expires_in: expires_in }.merge(opts),
  )
end

#remove(s3_filename, copy_to_tombstone = false) ⇒ Object



89
90
91
92
93
94
95
96
97
98
99
100
101
# File 'lib/s3_helper.rb', line 89

def remove(s3_filename, copy_to_tombstone = false)
  s3_filename = s3_filename.dup

  # copy the file in tombstone
  if copy_to_tombstone && @tombstone_prefix.present?
    self.copy(get_path_for_s3_upload(s3_filename), File.join(@tombstone_prefix, s3_filename))
  end

  # delete the file
  s3_filename.prepend(multisite_upload_path) if Rails.configuration.multisite
  delete_object(get_path_for_s3_upload(s3_filename))
rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NotFound
end

#s3_clientObject



281
282
283
# File 'lib/s3_helper.rb', line 281

def s3_client
  @s3_client ||= Aws::S3::Client.new(@s3_options)
end

#s3_inventory_path(path = "inventory") ⇒ Object



285
286
287
# File 'lib/s3_helper.rb', line 285

def s3_inventory_path(path = "inventory")
  get_path_for_s3_upload(path)
end

#tag_file(key, tags) ⇒ Object



243
244
245
246
247
248
249
250
251
252
253
254
# File 'lib/s3_helper.rb', line 243

def tag_file(key, tags)
  tag_array = []
  tags.each { |k, v| tag_array << { key: k.to_s, value: v.to_s } }

  s3_resource.client.put_object_tagging(
    bucket: @s3_bucket_name,
    key: key,
    tagging: {
      tag_set: tag_array,
    },
  )
end

#update_lifecycle(id, days, prefix: nil, tag: nil) ⇒ Object



188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
# File 'lib/s3_helper.rb', line 188

def update_lifecycle(id, days, prefix: nil, tag: nil)
  filter = {}

  if prefix
    filter[:prefix] = prefix
  elsif tag
    filter[:tag] = tag
  end

  # cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
  rule = { id: id, status: "Enabled", expiration: { days: days }, filter: filter }

  rules = []

  begin
    rules = s3_resource.client.get_bucket_lifecycle_configuration(bucket: @s3_bucket_name).rules
  rescue Aws::S3::Errors::NoSuchLifecycleConfiguration
    # skip trying to merge
  end

  # in the past we has a rule that was called purge-tombstone vs purge_tombstone
  # just go ahead and normalize for our bucket
  rules.delete_if { |r| r.id.gsub("_", "-") == id.gsub("_", "-") }

  rules << rule

  # normalize filter in rules, due to AWS library bug
  rules =
    rules.map do |r|
      r = r.to_h
      prefix = r.delete(:prefix)
      r[:filter] = { prefix: prefix } if prefix
      r
    end

  s3_resource.client.put_bucket_lifecycle_configuration(
    bucket: @s3_bucket_name,
    lifecycle_configuration: {
      rules: rules,
    },
  )
end

#update_tombstone_lifecycle(grace_period) ⇒ Object



231
232
233
234
235
# File 'lib/s3_helper.rb', line 231

def update_tombstone_lifecycle(grace_period)
  return if !SiteSetting.s3_configure_tombstone_policy
  return if @tombstone_prefix.blank?
  update_lifecycle("purge_tombstone", grace_period, prefix: @tombstone_prefix)
end

#upload(file, path, options = {}) ⇒ Object



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
# File 'lib/s3_helper.rb', line 65

def upload(file, path, options = {})
  path = get_path_for_s3_upload(path)
  obj = s3_bucket.object(path)

  etag =
    begin
      if File.size(file.path) >= FIFTEEN_MEGABYTES
        options[:multipart_threshold] = FIFTEEN_MEGABYTES
        obj.upload_file(file, options)
        obj.load
        obj.etag
      else
        options[:body] = file
        obj.put(options).etag
      end
    end

  [path, etag.gsub('"', "")]
end