Class: ObjectStorage::DirectUpload

Inherits:
Object
  • Object
show all
Includes:
Gitlab::Utils::StrongMemoize
Defined in:
lib/object_storage/direct_upload.rb

Overview

The DirectUpload class generates a set of presigned URLs that can be used to upload data to object storage from untrusted component: Workhorse, Runner?

For Google it assumes that the platform supports variable Content-Length.

For AWS it initiates Multipart Upload and presignes a set of part uploads.

Class calculates the best part size to be able to upload up to asked maximum size.
The number of generated parts will never go above 100,
but we will always try to reduce amount of generated parts.
The part size is rounded-up to 5MB.

Constant Summary collapse

TIMEOUT = 
4.hours
EXPIRE_OFFSET = 
15.minutes
MAXIMUM_MULTIPART_PARTS = 
100
MINIMUM_MULTIPART_SIZE = 
5.megabytes

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config, object_name, has_length:, maximum_size: nil, skip_delete: false) ⇒ DirectUpload

Returns a new instance of DirectUpload.



28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/object_storage/direct_upload.rb', line 28

def initialize(config, object_name, has_length:, maximum_size: nil, skip_delete: false)
  unless has_length
    raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size
  end

  @config = config
  @credentials = config.credentials
  @bucket_name = config.bucket
  @object_name = object_name
  @has_length = has_length
  @maximum_size = maximum_size
  @skip_delete = skip_delete
end

Instance Attribute Details

#bucket_name ⇒ Object (readonly)

Returns the value of attribute bucket_name.



25
26
27
 
# File 'lib/object_storage/direct_upload.rb', line 25

def bucket_name
  @bucket_name
end

#config ⇒ Object (readonly)

Returns the value of attribute config.



25
26
27
 
# File 'lib/object_storage/direct_upload.rb', line 25

def config
  @config
end

#credentials ⇒ Object (readonly)

Returns the value of attribute credentials.



25
26
27
 
# File 'lib/object_storage/direct_upload.rb', line 25

def credentials
  @credentials
end

#has_length ⇒ Object (readonly)

Returns the value of attribute has_length.



26
27
28
 
# File 'lib/object_storage/direct_upload.rb', line 26

def has_length
  @has_length
end

#maximum_size ⇒ Object (readonly)

Returns the value of attribute maximum_size.



26
27
28
 
# File 'lib/object_storage/direct_upload.rb', line 26

def maximum_size
  @maximum_size
end

#object_name ⇒ Object (readonly)

Returns the value of attribute object_name.



25
26
27
 
# File 'lib/object_storage/direct_upload.rb', line 25

def object_name
  @object_name
end

#skip_delete ⇒ Object (readonly)

Returns the value of attribute skip_delete.



26
27
28
 
# File 'lib/object_storage/direct_upload.rb', line 26

def skip_delete
  @skip_delete
end

Instance Method Details

#azure_gocloud_url ⇒ Object 



112
113
114
115
116
 
# File 'lib/object_storage/direct_upload.rb', line 112

def azure_gocloud_url
  url = "azblob://#{bucket_name}"
  url += "?domain=#{config.azure_storage_domain}" if config.azure_storage_domain.present?
  url
end

#delete_url ⇒ Object

Implements docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html



166
167
168
 
# File 'lib/object_storage/direct_upload.rb', line 166

def delete_url
  connection.delete_object_url(bucket_name, object_name, expire_at)
end

#get_url ⇒ Object

Implements docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html



157
158
159
160
161
162
163
 
# File 'lib/object_storage/direct_upload.rb', line 157

def get_url
  if config.google?
    connection.get_object_https_url(bucket_name, object_name, expire_at)
  else
    connection.get_object_url(bucket_name, object_name, expire_at)
  end
end

#google_gocloud_url ⇒ Object 



131
132
133
 
# File 'lib/object_storage/direct_upload.rb', line 131

def google_gocloud_url
  "gs://#{bucket_name}"
end

#multipart_abort_url ⇒ Object

Implements docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html



204
205
206
207
208
209
210
211
 
# File 'lib/object_storage/direct_upload.rb', line 204

def multipart_abort_url
  connection.signed_url({
    method: 'DELETE',
    bucket_name: bucket_name,
    object_name: object_name,
    query: { 'uploadId' => upload_id }
  }, expire_at)
end

#multipart_complete_url ⇒ Object

Implements docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html



193
194
195
196
197
198
199
200
201
 
# File 'lib/object_storage/direct_upload.rb', line 193

def multipart_complete_url
  connection.signed_url({
    method: 'POST',
    bucket_name: bucket_name,
    object_name: object_name,
    query: { 'uploadId' => upload_id },
    headers: { 'Content-Type' => 'application/xml' }
  }, expire_at)
end

#multipart_part_upload_url(part_number) ⇒ Object

Implements docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html



182
183
184
185
186
187
188
189
190
 
# File 'lib/object_storage/direct_upload.rb', line 182

def multipart_part_upload_url(part_number)
  connection.signed_url({
    method: 'PUT',
    bucket_name: bucket_name,
    object_name: object_name,
    query: { 'uploadId' => upload_id, 'partNumber' => part_number },
    headers: upload_options
  }, expire_at)
end

#multipart_part_urls ⇒ Object 



175
176
177
178
179
 
# File 'lib/object_storage/direct_upload.rb', line 175

def multipart_part_urls
  Array.new(number_of_multipart_parts) do |part_index|
    multipart_part_upload_url(part_index + 1)
  end
end

#multipart_upload_hash ⇒ Object 



55
56
57
58
59
60
61
62
63
64
 
# File 'lib/object_storage/direct_upload.rb', line 55

def multipart_upload_hash
  return unless requires_multipart_upload?

  {
    PartSize: rounded_multipart_part_size,
    PartURLs: multipart_part_urls,
    CompleteURL: multipart_complete_url,
    AbortURL: multipart_abort_url
  }
end

#provider ⇒ Object 



152
153
154
 
# File 'lib/object_storage/direct_upload.rb', line 152

def provider
  credentials[:provider].to_s
end

#store_url ⇒ Object

Implements docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html



171
172
173
 
# File 'lib/object_storage/direct_upload.rb', line 171

def store_url
  connection.put_object_url(bucket_name, object_name, expire_at, upload_options)
end

#to_hash ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/object_storage/direct_upload.rb', line 42

def to_hash
  {
    Timeout: TIMEOUT,
    GetURL: get_url,
    StoreURL: store_url,
    DeleteURL: delete_url,
    SkipDelete: skip_delete,
    MultipartUpload: multipart_upload_hash,
    CustomPutHeaders: true,
    PutHeaders: upload_options
  }.merge(workhorse_client_hash).compact
end

#use_workhorse_google_client? ⇒ Boolean

Returns:

  • (Boolean) 


143
144
145
146
147
148
149
150
 
# File 'lib/object_storage/direct_upload.rb', line 143

def use_workhorse_google_client?
  return false unless config.consolidated_settings?
  return true if credentials[:google_application_default]
  return true if credentials[:google_json_key_location]
  return true if credentials[:google_json_key_string]

  false
end

#use_workhorse_s3_client? ⇒ Boolean

Returns:

  • (Boolean) 


135
136
137
138
139
140
141
 
# File 'lib/object_storage/direct_upload.rb', line 135

def use_workhorse_s3_client?
  return false unless config.use_iam_profile? || config.consolidated_settings?
  # The Golang AWS SDK does not support V2 signatures
  return false unless credentials.fetch(:aws_signature_version, 4).to_i >= 4

  true
end

#workhorse_aws_hash ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/object_storage/direct_upload.rb', line 78

def workhorse_aws_hash
  {
    UseWorkhorseClient: use_workhorse_s3_client?,
    RemoteTempObjectID: object_name,
    ObjectStorage: {
      Provider: 'AWS',
      S3Config: {
        Bucket: bucket_name,
        Region: credentials[:region] || ::Fog::AWS::Storage::DEFAULT_REGION,
        Endpoint: credentials[:endpoint],
        PathStyle: config.use_path_style?,
        UseIamProfile: config.use_iam_profile?,
        ServerSideEncryption: config.server_side_encryption,
        SSEKMSKeyID: config.server_side_encryption_kms_key_id
      }.compact
    }
  }
end

#workhorse_azure_hash ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/object_storage/direct_upload.rb', line 97

def workhorse_azure_hash
  {
    # Azure requires Workhorse client because direct uploads can't
    # use pre-signed URLs without buffering the whole file to disk.
    UseWorkhorseClient: true,
    RemoteTempObjectID: object_name,
    ObjectStorage: {
      Provider: 'AzureRM',
      GoCloudConfig: {
        URL: azure_gocloud_url
      }
    }
  }
end

#workhorse_client_hash ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/object_storage/direct_upload.rb', line 66

def workhorse_client_hash
  if config.aws?
    workhorse_aws_hash
  elsif config.azure?
    workhorse_azure_hash
  elsif Feature.enabled?(:workhorse_google_client) && config.google?
    workhorse_google_hash
  else
    {}
  end
end

#workhorse_google_hash ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/object_storage/direct_upload.rb', line 118

def workhorse_google_hash
  {
    UseWorkhorseClient: use_workhorse_google_client?,
    RemoteTempObjectID: object_name,
    ObjectStorage: {
      Provider: 'Google',
      GoCloudConfig: {
        URL: google_gocloud_url
      }
    }
  }
end