Class: Backup::Backblaze::UploadLargeFile

Inherits:
Object
  • Object
show all
Extended by:
ApiImporter
Defined in:
lib/backup/backblaze/upload_large_file.rb

Overview

Upload a large file in several parts.

Constant Summary collapse

MAX_PARTS =

10000 is backblaze specified max number of parts

10000

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from ApiImporter

import_endpoint

Constructor Details

#initialize(account:, src:, bucket_id:, dst:, url_token: nil, part_size:, content_type: nil) ⇒ UploadLargeFile

src is a Pathname dst is a String



12
13
14
15
16
17
18
19
20
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 12

def initialize account:, src:, bucket_id:, dst:, url_token: nil, part_size:, content_type: nil
  @account = 
  @src = src
  @dst = dst
  @bucket_id = bucket_id
  @content_type = content_type
  @url_token = url_token
  @part_size = part_size
end

Instance Attribute Details

#account ⇒ Object (readonly)

Returns the value of attribute account.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def 
  @account
end

#bucket_id ⇒ Object (readonly)

Returns the value of attribute bucket_id.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def bucket_id
  @bucket_id
end

#content_type ⇒ Object (readonly)

Returns the value of attribute content_type.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def content_type
  @content_type
end

#dst ⇒ Object (readonly)

Returns the value of attribute dst.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def dst
  @dst
end

#part_size ⇒ Object (readonly)

Returns the value of attribute part_size.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def part_size
  @part_size
end

#src ⇒ Object (readonly)

Returns the value of attribute src.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def src
  @src
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



22
23
24
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 22

def url
  @url
end

Instance Method Details

#auth_headers ⇒ Object

same as account



25
26
27
28
29
30
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 25

def auth_headers
  # only cos the double {{}} is a quite ugly :-p
  Hash headers: {
    'Authorization' => .authorization_token,
  }.merge(TEST_HEADERS)
end

#b2_authorize_account(retries:, backoff:) ⇒ Object

needed for retry logic



54
55
56
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 54

def (retries:, backoff:)
  . retries: retries, backoff: backoff
end

#call ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 133

def call
  if src.size > part_size * MAX_PARTS
    raise Error, "File #{src.to_s} has size #{src.size} which is larger than part_size * MAX_PARTS #{part_size * MAX_PARTS}. Try increasing part_size in model."
  end

  Logger.info "uploading '#{src}' to #{dst}' of #{src.size} in #{part_count} parts"

  b2_start_large_file # not really necessary, but makes the flow clearer
  url_token # try to re-use existing url token if there is one
  shas = upload_parts

  # finish up, log and return the response
  hash_wrap = b2_finish_large_file shas
  Backup::Logger.info "#{src} finished"
  hash_wrap
end

#content_disposition ⇒ Object

No idea what has to be in here



37
38
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 37

def content_disposition
end

#content_length ⇒ Object 



40
41
42
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 40

def content_length
  src.size
end

#file_id ⇒ Object 



75
76
77
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 75

def file_id
  @file_id or b2_start_large_file
end

#last_modified_millis ⇒ Object 



44
45
46
47
48
49
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 44

def last_modified_millis
  @last_modified_millis ||= begin
    time = File.lstat(src).mtime
    time.tv_sec * 1000 + time.tv_usec / 1000
  end
end

#part_count ⇒ Object 



88
89
90
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 88

def part_count
  @part_count ||= (src.size / part_size.to_r).ceil
end

#upload_parts ⇒ Object 



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 116

def upload_parts
  (0...MAX_PARTS).each_with_object [] do |sequence, shas|
    # read length, offset
    bytes = src.read part_size, part_size * sequence

    if bytes.nil? || bytes.empty?
      # no more file to send
      break shas
    else
      sha = Digest::SHA1.hexdigest bytes
      b2_upload_part sequence, bytes, sha
      Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
      shas << sha
    end
  end
end

#url_token ⇒ Object 



79
80
81
 
# File 'lib/backup/backblaze/upload_large_file.rb', line 79

def url_token
  @url_token or b2_get_upload_part_url
end