Class: Backup::CloudIO::CloudFiles

Inherits:
Base
  • Object
show all
Defined in:
lib/backup/cloud_io/cloud_files.rb

Defined Under Namespace

Classes: Error, Object

Constant Summary collapse

MAX_FILE_SIZE =

5 GiB

1024**3 * 5
MAX_SLO_SIZE =

1000 segments @ 5 GiB

1024**3 * 5000
SEGMENT_BUFFER =

1 MiB

1024**2

Instance Attribute Summary collapse

Attributes inherited from Base

#max_retries, #retry_waitsec

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ CloudFiles

Returns a new instance of CloudFiles.



19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 19

def initialize(options = {})
  super

  @username           = options[:username]
  @api_key            = options[:api_key]
  @auth_url           = options[:auth_url]
  @region             = options[:region]
  @servicenet         = options[:servicenet]
  @container          = options[:container]
  @segments_container = options[:segments_container]
  @segment_size       = options[:segment_size]
  @days_to_keep       = options[:days_to_keep]
  @fog_options        = options[:fog_options]
end

Instance Attribute Details

#api_key ⇒ Object (readonly)

Returns the value of attribute api_key.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def api_key
  @api_key
end

#auth_url ⇒ Object (readonly)

Returns the value of attribute auth_url.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def auth_url
  @auth_url
end

#container ⇒ Object (readonly)

Returns the value of attribute container.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def container
  @container
end

#days_to_keep ⇒ Object (readonly)

Returns the value of attribute days_to_keep.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def days_to_keep
  @days_to_keep
end

#fog_options ⇒ Object (readonly)

Returns the value of attribute fog_options.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def fog_options
  @fog_options
end

#region ⇒ Object (readonly)

Returns the value of attribute region.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def region
  @region
end

#segment_size ⇒ Object (readonly)

Returns the value of attribute segment_size.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def segment_size
  @segment_size
end

#segments_container ⇒ Object (readonly)

Returns the value of attribute segments_container.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def segments_container
  @segments_container
end

#servicenet ⇒ Object (readonly)

Returns the value of attribute servicenet.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def servicenet
  @servicenet
end

#username ⇒ Object (readonly)

Returns the value of attribute username.



15
16
17
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 15

def username
  @username
end

Instance Method Details

#delete(objects_or_names) ⇒ Object

Delete non-SLO object(s) from the container.

  • Called by the Storage (with objects) and the Syncer (with names)

  • Deletes 10,000 objects per request.

  • Missing objects will be ignored.



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 104

def delete(objects_or_names)
  names = Array(objects_or_names).dup
  names.map!(&:name) if names.first.is_a?(Object)

  until names.empty?
    _names = names.slice!(0, 10000)
    with_retries('DELETE Multiple Objects') do
      resp = connection.delete_multiple_objects(container, _names)
      resp_status = resp.body['Response Status']
      raise Error, <<-EOS unless resp_status == '200 OK'
        #{ resp_status }
        The server returned the following:
        #{ resp.body.inspect }
      EOS
    end
  end
end

#delete_slo(objects) ⇒ Object

Delete an SLO object(s) from the container.

  • Used only by the Storage. The Syncer cannot use SLOs.

  • Removes the SLO manifest object and all associated segments.

  • Missing segments will be ignored.



127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 127

def delete_slo(objects)
  Array(objects).each do |object|
    with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
      resp = connection.delete_static_large_object(container, object.name)
      resp_status = resp.body['Response Status']
      raise Error, <<-EOS unless resp_status == '200 OK'
        #{ resp_status }
        The server returned the following:
        #{ resp.body.inspect }
      EOS
    end
  end
end

#head_object(object) ⇒ Object

Used by Object to fetch metadata if needed.



91
92
93
94
95
96
97
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 91

def head_object(object)
  resp = nil
  with_retries("HEAD '#{ container }/#{ object.name }'") do
    resp = connection.head_object(container, object.name)
  end
  resp
end

#objects(prefix) ⇒ Object

Returns all objects in the container with the given prefix.

  • #get_container returns a max of 10000 objects per request.

  • Returns objects sorted using a sqlite binary collating function.

  • If marker is given, only objects after the marker are in the response.



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 69

def objects(prefix)
  objects = []
  resp = nil
  prefix = prefix.chomp('/')
  opts = { :prefix => prefix + '/' }

  create_containers

  while resp.nil? || resp.body.count == 10000
    opts.merge!(:marker => objects.last.name) unless objects.empty?
    with_retries("GET '#{ container }/#{ prefix }/*'") do
      resp = connection.get_container(container, opts)
    end
    resp.body.each do |obj_data|
      objects << Object.new(self, obj_data)
    end
  end

  objects
end

#upload(src, dest) ⇒ Object

The Syncer may call this method in multiple threads, but #objects is always called before this occurs.



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/backup/cloud_io/cloud_files.rb', line 36

def upload(src, dest)
  create_containers

  file_size = File.size(src)
  segment_bytes = segment_size * 1024**2
  if segment_bytes > 0 && file_size > segment_bytes
    raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
      File Too Large
      File: #{ src }
      Size: #{ file_size }
      Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
    EOS

    segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
    segments = upload_segments(src, dest, segment_bytes, file_size)
    upload_manifest(dest, segments)
  else
    raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
      File Too Large
      File: #{ src }
      Size: #{ file_size }
      Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
    EOS

    put_object(src, dest)
  end
end