Class: Fog::Backblaze::Storage::Real

Inherits:
Object
  • Object
show all
Defined in:
lib/fog/backblaze/storage/real.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ Real

Returns a new instance of Real.



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/fog/backblaze/storage/real.rb', line 4

def initialize(options = {})
  @options = options
  @logger = @options[:logger] || begin
    require 'logger'
    Logger.new("/dev/null")
  end

  @token_cache = if options[:token_cache].nil? || options[:token_cache] == :memory
    Fog::Backblaze::TokenCache.new
  elsif options[:token_cache] === false
    Fog::Backblaze::TokenCache::NullTokenCache.new
  elsif token_cache.is_a?(Fog::Backblaze::TokenCache)
    token_cache
  else
    Fog::Backblaze::TokenCache::FileTokenCache.new(options[:token_cache])
  end
end

Instance Attribute Details

#options ⇒ Object (readonly)

Returns the value of attribute options.



2
3
4
 
# File 'lib/fog/backblaze/storage/real.rb', line 2

def options
  @options
end

#token_cache ⇒ Object (readonly)

Returns the value of attribute token_cache.



2
3
4
 
# File 'lib/fog/backblaze/storage/real.rb', line 2

def token_cache
  @token_cache
end

Instance Method Details

#_cached_buchets_hash(force_fetch: false) ⇒ Object 



390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
 
# File 'lib/fog/backblaze/storage/real.rb', line 390

def _cached_buchets_hash(force_fetch: false)

  if !force_fetch && cached = @token_cache.buckets
    cached
  end

  buckets_hash = {}
  list_buckets.json['buckets'].each do |bucket|
    buckets_hash[bucket['bucketName']] = bucket
  end

  @token_cache.buckets = buckets_hash

  buckets_hash
end

#_esc_file(file_name) ⇒ Object 



496
497
498
 
# File 'lib/fog/backblaze/storage/real.rb', line 496

def _esc_file(file_name)
  CGI.escape(file_name).gsub('%2F', '/')
end

#_get_bucket_id(bucket_name) ⇒ Object 



366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/fog/backblaze/storage/real.rb', line 366

def _get_bucket_id(bucket_name)
  if @options[:b2_bucket_name] == bucket_name && @options[:b2_bucket_id]
    return @options[:b2_bucket_id]
  else
    cached = @token_cache && @token_cache.buckets

    if cached && cached[bucket_name]
      return cached[bucket_name]['bucketId']
    else
      fetched = _cached_buchets_hash(force_fetch: !!cached)
      return fetched[bucket_name] && fetched[bucket_name]['bucketId']
    end
  end
end

#_get_bucket_id!(bucket_name) ⇒ Object 



381
382
383
384
385
386
387
388
 
# File 'lib/fog/backblaze/storage/real.rb', line 381

def _get_bucket_id!(bucket_name)
  bucket_id = _get_bucket_id(bucket_name)
  unless bucket_id
    raise Fog::Errors::NotFound, "Can not find bucket \"#{bucket_name}\""
  end

  return bucket_id
end

#_get_object_version_ids(bucket_name, file_name) ⇒ Object 



341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
 
# File 'lib/fog/backblaze/storage/real.rb', line 341

def _get_object_version_ids(bucket_name, file_name)
  response = b2_command(:b2_list_file_versions,
    body: {
      startFileName: file_name,
      prefix: file_name,
      bucketId: _get_bucket_id!(bucket_name),
      maxFileCount: 1000
    }
  )

  if response.status >= 400
    raise Fog::Errors::Error, "Fetch error: #{response.json['message']} (status = #{response.status})"
  end

  if response.json['files']
    version_ids = []
    response.json['files'].map do |file_version|
      version_ids << file_version['fileId'] if file_version['fileName'] == file_name
    end
    version_ids
  else
    []
  end
end

#auth_response ⇒ Object 



406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
 
# File 'lib/fog/backblaze/storage/real.rb', line 406

def auth_response
  #return @auth_response.json if @auth_response

  if cached = @token_cache.auth_response
    logger.info("get token from cache")
    return cached
  end

  auth_string = if @options[:b2_account_id] && @options[:b2_account_token]
    "#{@options[:b2_account_id]}:#{@options[:b2_account_token]}"
  elsif @options[:b2_key_id] && @options[:b2_key_token]
    "#{@options[:b2_key_id]}:#{@options[:b2_key_token]}"
  else
    raise Fog::Errors::Error, "B2 credentials are required, "\
                              "please use b2_account_id and b2_account_token or "\
                              "b2_key_id and b2_key_token"
  end

  @auth_response = json_req(:get, "https://api.backblazeb2.com/b2api/v1/b2_authorize_account",
    headers: {
      "Authorization" => "Basic " + Base64.strict_encode64(auth_string)
    },
    persistent: false
  )

  if @auth_response.status >= 400
    raise Fog::Errors::Error, "Authentication error: #{@auth_response.json['message']} (status = #{@auth_response.status})\n#{@auth_response.body}"
  end

  @token_cache.auth_response = @auth_response.json

  @auth_response.json
end

#b2_account_id ⇒ Object

return @options or call b2_authorize_account when using application key



501
502
503
504
505
506
507
 
# File 'lib/fog/backblaze/storage/real.rb', line 501

def 
  return @options[:b2_account_id] if @options[:b2_account_id]

  auth = auth_response
  p auth
  auth['accountId']
end

#b2_command(command, options = {}) ⇒ Object 



440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/fog/backblaze/storage/real.rb', line 440

def b2_command(command, options = {})
  auth_response = self.auth_response
  options[:headers] ||= {}
  options[:headers]['Authorization'] ||= auth_response['authorizationToken']

  if options[:body] && !options[:body].is_a?(String)
    options[:body] = ::JSON.generate(options[:body])
  end

  request_url = options.delete(:url) || "#{auth_response['apiUrl']}/b2api/v1/#{command}"

  #pp [:b2_command, request_url, options]

  json_req(options.delete(:method) || :post, request_url, options)
end

#create_key(name, capabilities: nil, bucket_id: nil, name_prefix: nil) ⇒ Object

TODO TEST call b2_create_key



299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
 
# File 'lib/fog/backblaze/storage/real.rb', line 299

def create_key(name, capabilities: nil, bucket_id: nil, name_prefix: nil)
  capabilities ||= [
    'listKeys',
    'writeKeys',
    'deleteKeys',
    'listBuckets',
    'writeBuckets',
    'deleteBuckets',
    'listFiles',
    'readFiles',
    'shareFiles',
    'writeFiles',
    'deleteFiles'
  ]

  response = b2_command(:b2_create_key,
    body: {
      accountId: ,
      keyName: name,
      capabilities: capabilities,
      bucketId: bucket_id,
      namePrefix: name_prefix
    }
  )
end

#delete_bucket(bucket_name, options = {}) ⇒ Object

call b2_delete_bucket



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/fog/backblaze/storage/real.rb', line 105

def delete_bucket(bucket_name, options = {})
  bucket_id = _get_bucket_id!(bucket_name)

  response = b2_command(:b2_delete_bucket,
    body: {
      bucketId: bucket_id,
      accountId: 
    }
  )

  if !options[:is_retrying]
    if response.status == 400 && response.json['message'] =~ /Bucket .+ does not exist/
      logger.info("Try drop cache and try again")
      @token_cache.buckets = nil
      return delete_bucket(bucket_name, is_retrying: true)
    end
  end

  if response.status >= 400
    raise Fog::Errors::Error, "Failed delete_bucket, status = #{response.status} #{response.body}"
  end

  if cached = @token_cache.buckets
    #cached.delete(bucket_name)
    #@token_cache.buckets = cached
    @token_cache.buckets = nil
  end

  response
end

#delete_object(bucket_name, file_name) ⇒ Object

call b2_delete_file_version



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/fog/backblaze/storage/real.rb', line 277

def delete_object(bucket_name, file_name)
  version_ids = _get_object_version_ids(bucket_name, file_name)

  if version_ids.size == 0
    raise Fog::Errors::NotFound, "Can not find #{file_name} in in bucket #{bucket_name}"
  end

  logger.info("Deleting #{version_ids.size} versions of #{file_name}")

  last_response = nil
  version_ids.each do |version_id|
    last_response = b2_command(:b2_delete_file_version, body: {
      fileName: file_name,
      fileId: version_id
    })
  end

  last_response
end

#get_bucket(bucket_name) ⇒ Object

call b2_list_buckets



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/fog/backblaze/storage/real.rb', line 88

def get_bucket(bucket_name)
  response = list_buckets(bucketName: bucket_name)

  bucket = response.json['buckets'].detect do |bucket|
    bucket['bucketName'] == bucket_name
  end

  unless bucket
    raise Fog::Errors::NotFound, "No bucket with name: #{bucket_name}"
  end

  response.body = bucket
  response.json = bucket
  return response
end

#get_object(bucket_name, file_name) ⇒ Object 



257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/fog/backblaze/storage/real.rb', line 257

def get_object(bucket_name, file_name)
  file_url = get_object_url(bucket_name, file_name)

  response = b2_command(nil,
    method: :get,
    url: file_url
  )

  if response.status == 404
    raise Fog::Errors::NotFound, "Can not find #{file_name.inspect} in bucket #{bucket_name}"
  end

  if response.status > 400
    raise Fog::Errors::Error, "Failed get_object, status = #{response.status} #{response.body}"
  end

  return response
end

#get_object_url(bucket_name, file_path) ⇒ Object Also known as: get_object_https_url

generates url regardless if bucket is private or not



230
231
232
 
# File 'lib/fog/backblaze/storage/real.rb', line 230

def get_object_url(bucket_name, file_path)
  "#{auth_response['downloadUrl']}/file/#{CGI.escape(bucket_name)}/#{_esc_file(file_path)}"
end

#get_public_object_url(bucket_name, file_path, options = {}) ⇒ Object

call b2_get_download_authorization



237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/fog/backblaze/storage/real.rb', line 237

def get_public_object_url(bucket_name, file_path, options = {})
  bucket_id = _get_bucket_id!(bucket_name)

  result = b2_command(:b2_get_download_authorization, body: {
    bucketId: bucket_id,
    fileNamePrefix: file_path,
    validDurationInSeconds: 604800
  }.merge(options))

  if result.status == 404
    raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
  end

  if result.status >= 400
    raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
  end

  "#{get_object_url(bucket_name, file_path)}?Authorization=#{result.json['authorizationToken']}"
end

#head_object(bucket_name, file_path, options = {}) ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/fog/backblaze/storage/real.rb', line 148

def head_object(bucket_name, file_path, options = {})
  file_url = get_object_url(bucket_name, file_path)

  result = b2_command(nil,
    method: :head,
    url: file_url
  )

  if result.status == 404
    raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
  end

  if result.status >= 400
    raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
  end

  result
end

#json_req(method, url, options = {}) ⇒ Object 



456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
 
# File 'lib/fog/backblaze/storage/real.rb', line 456

def json_req(method, url, options = {})
  start_time = Time.now.to_f
  logger.info("Req #{method.to_s.upcase} #{url}")
  if options[:body] && options[:body].size > 300
    logger.debug(options.merge(body: "-- Body #{options[:body].size} bytes --").to_s)
  else
    logger.debug(options.to_s)
  end

  if !options.has_key?(:persistent) || options[:persistent] == true
    @connections ||= {}
    full_path = [URI.parse(url).request_uri, URI.parse(url).fragment].compact.join("#")
    host_url = url.sub(full_path, "")
    connection = @connections[host_url] ||= Excon.new(host_url, persistent: true)
    http_response = connection.send(method, options.merge(path: full_path, idempotent: true))
  else
    http_response = Excon.send(method, url, options)
  end

  http_response.extend(Fog::Backblaze::JSONResponse)
  if http_response.json_response? && http_response.body.size > 0
    http_response.assign_json_body!
  end

  http_response
ensure
  status = http_response && http_response.status
  logger.info("    Done #{method.to_s.upcase} #{url} = #{status} (#{(Time.now.to_f - start_time).round(3)} sec)")
  if http_response
    logger.debug("    Headers: #{http_response.headers}")
    if method != :head && http_response.headers['Content-Type'].to_s !~ %r{^image/}
      logger.debug("    Body: #{http_response.body}")
    end
  end
end

#list_buckets(options = {}) ⇒ Object

call b2_list_buckets



79
80
81
82
83
84
85
 
# File 'lib/fog/backblaze/storage/real.rb', line 79

def list_buckets(options = {})
  response = b2_command(:b2_list_buckets, body: {
    accountId: 
  }.merge(options))

  response
end

#list_keys ⇒ Object

call b2_list_keys



326
327
328
329
330
331
332
333
334
335
336
337
338
339
 
# File 'lib/fog/backblaze/storage/real.rb', line 326

def list_keys
  response = b2_command(:b2_list_keys,
    body: {
      accountId: ,
      maxKeyCount: 1000
    }
  )

  if response.status > 400
    raise Fog::Errors::Error, "Failed get_object, status = #{response.status} #{response.body}"
  end

  response
end

#list_objects(bucket_name, options = {}) ⇒ Object

call b2_list_file_names



139
140
141
142
143
144
145
146
 
# File 'lib/fog/backblaze/storage/real.rb', line 139

def list_objects(bucket_name, options = {})
  bucket_id = _get_bucket_id!(bucket_name)

  b2_command(:b2_list_file_names, body: {
    bucketId: bucket_id,
    maxFileCount: 10_000
  }.merge(options))
end

#logger ⇒ Object 



22
23
24
 
# File 'lib/fog/backblaze/storage/real.rb', line 22

def logger
  @logger
end

#put_bucket(bucket_name, extra_options = {}) ⇒ Object

call b2_create_bucket



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/fog/backblaze/storage/real.rb', line 29

def put_bucket(bucket_name, extra_options = {})
  options = {
    accountId: ,
    bucketType: extra_options.delete(:public) ? 'allPublic' : 'allPrivate',
    bucketName: bucket_name,
  }.merge(extra_options)

  response = b2_command(:b2_create_bucket, body: options)

  if response.status >= 400
    raise Fog::Errors::Error, "Failed put_bucket, status = #{response.status} #{response.body}"
  end

  if cached = @token_cache.buckets
    @token_cache.buckets = cached.merge(bucket_name => response.json)
  else
    @token_cache.buckets = {bucket_name => response.json}
  end

  response
end

#put_object(bucket_name, file_path, content, options = {}) ⇒ Object

call b2_get_upload_url

connection.put_object("a-bucket", "/some_file.txt", string_or_io, options)

Possible options:

  • content_type

  • last_modified - time object or number of miliseconds

  • content_disposition

  • extra_headers - hash, list of custom headers



176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/fog/backblaze/storage/real.rb', line 176

def put_object(bucket_name, file_path, content, options = {})
  upload_url = @token_cache.fetch("upload_url/#{bucket_name}") do
    bucket_id = _get_bucket_id!(bucket_name)
    result = b2_command(:b2_get_upload_url, body: {bucketId: bucket_id})
    result.json
  end

  if content.is_a?(IO)
    content = content.read
  end

  extra_headers = {}
  if options[:content_type]
    extra_headers['Content-Type'] = options[:content_type]
  end

  if options[:last_modified]
    value = if options[:last_modified].is_a?(::Time)
      (options[:last_modified].to_f * 1000).round
    else
      value
    end
    extra_headers['X-Bz-Info-src_last_modified_millis'] = value
  end

  if options[:content_disposition]
    extra_headers['X-Bz-Info-b2-content-disposition'] = options[:content_disposition]
  end

  if options[:extra_headers]
    options[:extra_headers].each do |key, value|
      extra_headers["X-Bz-Info-#{key}"] = value
    end
  end

  response = b2_command(nil,
    url: upload_url['uploadUrl'],
    body: content,
    headers: {
      'Authorization': upload_url['authorizationToken'],
      'Content-Type': 'b2/x-auto',
      'X-Bz-File-Name': "#{_esc_file(file_path)}",
      'X-Bz-Content-Sha1': Digest::SHA1.hexdigest(content)
    }.merge(extra_headers)
  )

  if response.json['fileId'] == nil
    raise Fog::Errors::Error, "Failed put_object, status = #{response.status} #{response.body}"
  end

  response
end

#reset_token_cache ⇒ Object 



492
493
494
 
# File 'lib/fog/backblaze/storage/real.rb', line 492

def reset_token_cache
  @token_cache.reset
end

#update_bucket(bucket_name, extra_options) ⇒ Object

call b2_update_bucket if options presents, then bucket_name is option



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/fog/backblaze/storage/real.rb', line 53

def update_bucket(bucket_name, extra_options)
  options = {
    accountId: ,
    bucketId: extra_options[:bucketId] || _get_bucket_id!(bucket_name),
  }
  if extra_options.has_key?(:public)
    options[:bucketType] = extra_options.delete(:public) ? 'allPublic' : 'allPrivate'
  end
  options.merge!(extra_options)

  response = b2_command(:b2_update_bucket, body: options)

  if response.status >= 400
    raise Fog::Errors::Error, "Failed update_bucket, status = #{response.status} #{response.body}"
  end

  if cached = @token_cache.buckets
    @token_cache.buckets = cached.merge(bucket_name => response.json)
  else
    @token_cache.buckets = {bucket_name => response.json}
  end

  response
end