Class: S3Helper
- Inherits:
-
Object
- Object
- S3Helper
- Defined in:
- lib/s3_helper.rb
Defined Under Namespace
Classes: SettingMissing
Constant Summary collapse
- FIFTEEN_MEGABYTES =
15 * 1024 * 1024
Instance Attribute Summary collapse
-
#s3_bucket_folder_path ⇒ Object
readonly
Returns the value of attribute s3_bucket_folder_path.
-
#s3_bucket_name ⇒ Object
readonly
Returns the value of attribute s3_bucket_name.
Class Method Summary collapse
- .build_from_config(use_db_s3_config: false, for_backup: false, s3_client: nil) ⇒ Object
- .get_bucket_and_folder_path(s3_bucket_name) ⇒ Object
- .s3_options(obj) ⇒ Object
Instance Method Summary collapse
- #abort_multipart(key:, upload_id:) ⇒ Object
- #complete_multipart(upload_id:, key:, parts:) ⇒ Object
- #copy(source, destination, options: {}) ⇒ Object
- #create_multipart(key, content_type, metadata: {}) ⇒ Object
- #delete_object(key) ⇒ Object
- #delete_objects(keys) ⇒ Object
- #download_file(filename, destination_path, failure_message = nil) ⇒ Object
-
#ensure_cors!(rules = nil) ⇒ Object
Several places in the application need certain CORS rules to exist inside an S3 bucket so requests to the bucket can be made directly from the browser.
-
#initialize(s3_bucket_name, tombstone_prefix = "", options = {}) ⇒ S3Helper
constructor
A new instance of S3Helper.
- #list(prefix = "", marker = nil) ⇒ Object
-
#list_multipart_parts(upload_id:, key:, max_parts: 1000, start_from_part_number: nil) ⇒ Object
Important note from the S3 documentation:.
- #object(path) ⇒ Object
- #path_from_url(url) ⇒ Object
- #presign_multipart_part(upload_id:, key:, part_number:) ⇒ Object
-
#presigned_request(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) ⇒ Object
Returns url, headers in a tuple which is needed in some cases.
- #presigned_url(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) ⇒ Object
- #remove(s3_filename, copy_to_tombstone = false) ⇒ Object
- #s3_client ⇒ Object
- #s3_inventory_path(path = "inventory") ⇒ Object
- #tag_file(key, tags) ⇒ Object
- #update_lifecycle(id, days, prefix: nil, tag: nil) ⇒ Object
- #update_tombstone_lifecycle(grace_period) ⇒ Object
- #upload(file, path, options = {}) ⇒ Object
Constructor Details
#initialize(s3_bucket_name, tombstone_prefix = "", options = {}) ⇒ S3Helper
Returns a new instance of S3Helper.
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
# File 'lib/s3_helper.rb', line 28 def initialize(s3_bucket_name, tombstone_prefix = "", = {}) @s3_client = .delete(:client) @s3_options = .merge() @s3_bucket_name, @s3_bucket_folder_path = begin raise Discourse::InvalidParameters.new("s3_bucket_name") if s3_bucket_name.blank? self.class.get_bucket_and_folder_path(s3_bucket_name) end @tombstone_prefix = if @s3_bucket_folder_path File.join(@s3_bucket_folder_path, tombstone_prefix) else tombstone_prefix end end |
Instance Attribute Details
#s3_bucket_folder_path ⇒ Object (readonly)
Returns the value of attribute s3_bucket_folder_path.
11 12 13 |
# File 'lib/s3_helper.rb', line 11 def s3_bucket_folder_path @s3_bucket_folder_path end |
#s3_bucket_name ⇒ Object (readonly)
Returns the value of attribute s3_bucket_name.
11 12 13 |
# File 'lib/s3_helper.rb', line 11 def s3_bucket_name @s3_bucket_name end |
Class Method Details
.build_from_config(use_db_s3_config: false, for_backup: false, s3_client: nil) ⇒ Object
46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
# File 'lib/s3_helper.rb', line 46 def self.build_from_config(use_db_s3_config: false, for_backup: false, s3_client: nil) setting_klass = use_db_s3_config ? SiteSetting : GlobalSetting = S3Helper.(setting_klass) [:client] = s3_client if s3_client.present? bucket = if for_backup setting_klass.s3_backup_bucket else use_db_s3_config ? SiteSetting.s3_upload_bucket : GlobalSetting.s3_bucket end S3Helper.new(bucket.downcase, "", ) end |
.get_bucket_and_folder_path(s3_bucket_name) ⇒ Object
61 62 63 |
# File 'lib/s3_helper.rb', line 61 def self.get_bucket_and_folder_path(s3_bucket_name) s3_bucket_name.downcase.split("/", 2) end |
.s3_options(obj) ⇒ Object
260 261 262 263 264 265 266 267 268 269 270 271 272 |
# File 'lib/s3_helper.rb', line 260 def self.(obj) opts = { region: obj.s3_region } opts[:endpoint] = SiteSetting.s3_endpoint if SiteSetting.s3_endpoint.present? opts[:http_continue_timeout] = SiteSetting.s3_http_continue_timeout unless obj.s3_use_iam_profile opts[:access_key_id] = obj.s3_access_key_id opts[:secret_access_key] = obj.s3_secret_access_key end opts end |
Instance Method Details
#abort_multipart(key:, upload_id:) ⇒ Object
289 290 291 |
# File 'lib/s3_helper.rb', line 289 def abort_multipart(key:, upload_id:) s3_client.abort_multipart_upload(bucket: s3_bucket_name, key: key, upload_id: upload_id) end |
#complete_multipart(upload_id:, key:, parts:) ⇒ Object
338 339 340 341 342 343 344 345 346 347 |
# File 'lib/s3_helper.rb', line 338 def complete_multipart(upload_id:, key:, parts:) s3_client.complete_multipart_upload( bucket: s3_bucket_name, key: key, upload_id: upload_id, multipart_upload: { parts: parts, }, ) end |
#copy(source, destination, options: {}) ⇒ Object
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
# File 'lib/s3_helper.rb', line 112 def copy(source, destination, options: {}) if [:apply_metadata_to_destination] = .except(:apply_metadata_to_destination).merge(metadata_directive: "REPLACE") end destination = get_path_for_s3_upload(destination) source_object = if !Rails.configuration.multisite || source.include?(multisite_upload_path) || source.include?(@tombstone_prefix) s3_bucket.object(source) elsif @s3_bucket_folder_path folder, filename = source.split("/", 2) s3_bucket.object(File.join(folder, multisite_upload_path, filename)) else s3_bucket.object(File.join(multisite_upload_path, source)) end if source_object.size > FIFTEEN_MEGABYTES [:multipart_copy] = true [:content_length] = source_object.size end destination_object = s3_bucket.object(destination) # Note for small files that do not use multipart copy: Any options for metadata # (e.g. content_disposition, content_type) will not be applied unless the # metadata_directive = "REPLACE" option is passed in. If this is not passed in, # the source object's metadata will be used. # For larger files it copies the metadata from the source file and merges it # with values from the copy call. response = destination_object.copy_from(source_object, ) etag = if response.respond_to?(:copy_object_result) # small files, regular copy response.copy_object_result.etag else # larger files, multipart copy response.data.etag end [destination, etag.gsub('"', "")] end |
#create_multipart(key, content_type, metadata: {}) ⇒ Object
293 294 295 296 297 298 299 300 301 302 303 |
# File 'lib/s3_helper.rb', line 293 def create_multipart(key, content_type, metadata: {}) response = s3_client.create_multipart_upload( acl: SiteSetting.s3_use_acls ? "private" : nil, bucket: s3_bucket_name, key: key, content_type: content_type, metadata: , ) { upload_id: response.upload_id, key: key } end |
#delete_object(key) ⇒ Object
103 104 105 106 |
# File 'lib/s3_helper.rb', line 103 def delete_object(key) s3_bucket.object(key).delete rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NotFound end |
#delete_objects(keys) ⇒ Object
108 109 110 |
# File 'lib/s3_helper.rb', line 108 def delete_objects(keys) s3_bucket.delete_objects({ delete: { objects: keys.map { |k| { key: k } }, quiet: true } }) end |
#download_file(filename, destination_path, failure_message = nil) ⇒ Object
274 275 276 277 278 279 |
# File 'lib/s3_helper.rb', line 274 def download_file(filename, destination_path, = nil) object(filename).download_file(destination_path) rescue => err raise &.to_s || "Failed to download #{filename} because #{err..length > 0 ? err. : err.class.to_s}" end |
#ensure_cors!(rules = nil) ⇒ Object
Several places in the application need certain CORS rules to exist inside an S3 bucket so requests to the bucket can be made directly from the browser. The s3:ensure_cors_rules rake task is used to ensure these rules exist for assets, S3 backups, and direct S3 uploads, depending on configuration.
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 |
# File 'lib/s3_helper.rb', line 161 def ensure_cors!(rules = nil) return unless SiteSetting.s3_install_cors_rule rules = [rules] if !rules.is_a?(Array) existing_rules = fetch_bucket_cors_rules new_rules = rules - existing_rules return false if new_rules.empty? final_rules = existing_rules + new_rules begin s3_resource.client.put_bucket_cors( bucket: @s3_bucket_name, cors_configuration: { cors_rules: final_rules, }, ) rescue Aws::S3::Errors::AccessDenied Rails.logger.info( "Could not PutBucketCors rules for #{@s3_bucket_name}, rules: #{final_rules}", ) return false end true end |
#list(prefix = "", marker = nil) ⇒ Object
237 238 239 240 241 |
# File 'lib/s3_helper.rb', line 237 def list(prefix = "", marker = nil) = { prefix: get_path_for_s3_upload(prefix) } [:marker] = marker if marker.present? s3_bucket.objects() end |
#list_multipart_parts(upload_id:, key:, max_parts: 1000, start_from_part_number: nil) ⇒ Object
Important note from the S3 documentation:
This request returns a default and maximum of 1000 parts. You can restrict the number of parts returned by specifying the max_parts argument. If your multipart upload consists of more than 1,000 parts, the response returns an IsTruncated field with the value of true, and a NextPartNumberMarker element.
In subsequent ListParts requests you can include the part_number_marker arg using the NextPartNumberMarker the field value from the previous response to get more parts.
See docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_parts-instance_method
330 331 332 333 334 335 336 |
# File 'lib/s3_helper.rb', line 330 def list_multipart_parts(upload_id:, key:, max_parts: 1000, start_from_part_number: nil) = { bucket: s3_bucket_name, key: key, upload_id: upload_id, max_parts: max_parts } [:part_number_marker] = start_from_part_number if start_from_part_number.present? s3_client.list_parts() end |
#object(path) ⇒ Object
256 257 258 |
# File 'lib/s3_helper.rb', line 256 def object(path) s3_bucket.object(get_path_for_s3_upload(path)) end |
#path_from_url(url) ⇒ Object
85 86 87 |
# File 'lib/s3_helper.rb', line 85 def path_from_url(url) URI.parse(url).path.delete_prefix("/") end |
#presign_multipart_part(upload_id:, key:, part_number:) ⇒ Object
305 306 307 308 309 310 311 312 313 314 315 |
# File 'lib/s3_helper.rb', line 305 def presign_multipart_part(upload_id:, key:, part_number:) presigned_url( key, method: :upload_part, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: { part_number: part_number, upload_id: upload_id, }, ) end |
#presigned_request(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) ⇒ Object
Returns url, headers in a tuple which is needed in some cases.
357 358 359 360 361 362 363 364 365 366 367 |
# File 'lib/s3_helper.rb', line 357 def presigned_request( key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {} ) Aws::S3::Presigner.new(client: s3_client).presigned_request( method, { bucket: s3_bucket_name, key: key, expires_in: expires_in }.merge(opts), ) end |
#presigned_url(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) ⇒ Object
349 350 351 352 353 354 |
# File 'lib/s3_helper.rb', line 349 def presigned_url(key, method:, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, opts: {}) Aws::S3::Presigner.new(client: s3_client).presigned_url( method, { bucket: s3_bucket_name, key: key, expires_in: expires_in }.merge(opts), ) end |
#remove(s3_filename, copy_to_tombstone = false) ⇒ Object
89 90 91 92 93 94 95 96 97 98 99 100 101 |
# File 'lib/s3_helper.rb', line 89 def remove(s3_filename, copy_to_tombstone = false) s3_filename = s3_filename.dup # copy the file in tombstone if copy_to_tombstone && @tombstone_prefix.present? self.copy(get_path_for_s3_upload(s3_filename), File.join(@tombstone_prefix, s3_filename)) end # delete the file s3_filename.prepend(multisite_upload_path) if Rails.configuration.multisite delete_object(get_path_for_s3_upload(s3_filename)) rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NotFound end |
#s3_client ⇒ Object
281 282 283 |
# File 'lib/s3_helper.rb', line 281 def s3_client @s3_client ||= Aws::S3::Client.new(@s3_options) end |
#s3_inventory_path(path = "inventory") ⇒ Object
285 286 287 |
# File 'lib/s3_helper.rb', line 285 def s3_inventory_path(path = "inventory") get_path_for_s3_upload(path) end |
#tag_file(key, tags) ⇒ Object
243 244 245 246 247 248 249 250 251 252 253 254 |
# File 'lib/s3_helper.rb', line 243 def tag_file(key, ) tag_array = [] .each { |k, v| tag_array << { key: k.to_s, value: v.to_s } } s3_resource.client.put_object_tagging( bucket: @s3_bucket_name, key: key, tagging: { tag_set: tag_array, }, ) end |
#update_lifecycle(id, days, prefix: nil, tag: nil) ⇒ Object
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 |
# File 'lib/s3_helper.rb', line 188 def update_lifecycle(id, days, prefix: nil, tag: nil) filter = {} if prefix filter[:prefix] = prefix elsif tag filter[:tag] = tag end # cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html rule = { id: id, status: "Enabled", expiration: { days: days }, filter: filter } rules = [] begin rules = s3_resource.client.get_bucket_lifecycle_configuration(bucket: @s3_bucket_name).rules rescue Aws::S3::Errors::NoSuchLifecycleConfiguration # skip trying to merge end # in the past we has a rule that was called purge-tombstone vs purge_tombstone # just go ahead and normalize for our bucket rules.delete_if { |r| r.id.gsub("_", "-") == id.gsub("_", "-") } rules << rule # normalize filter in rules, due to AWS library bug rules = rules.map do |r| r = r.to_h prefix = r.delete(:prefix) r[:filter] = { prefix: prefix } if prefix r end s3_resource.client.put_bucket_lifecycle_configuration( bucket: @s3_bucket_name, lifecycle_configuration: { rules: rules, }, ) end |
#update_tombstone_lifecycle(grace_period) ⇒ Object
231 232 233 234 235 |
# File 'lib/s3_helper.rb', line 231 def update_tombstone_lifecycle(grace_period) return if !SiteSetting.s3_configure_tombstone_policy return if @tombstone_prefix.blank? update_lifecycle("purge_tombstone", grace_period, prefix: @tombstone_prefix) end |
#upload(file, path, options = {}) ⇒ Object
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
# File 'lib/s3_helper.rb', line 65 def upload(file, path, = {}) path = get_path_for_s3_upload(path) obj = s3_bucket.object(path) etag = begin if File.size(file.path) >= FIFTEEN_MEGABYTES [:multipart_threshold] = FIFTEEN_MEGABYTES obj.upload_file(file, ) obj.load obj.etag else [:body] = file obj.put().etag end end [path, etag.gsub('"', "")] end |