Class: Tumugi::Plugin::GoogleCloudStorage::FileSystem

Inherits:
FileSystem
  • Object
show all
Defined in:
lib/tumugi/plugin/google_cloud_storage/file_system.rb

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ FileSystem

Returns a new instance of FileSystem.



14
15
16
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 14

def initialize(config)
  save_config(config)
end

Instance Method Details

#bucket_exist?(bucket) ⇒ Boolean

Returns:

  • (Boolean) 


211
212
213
214
215
216
217
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 211

def bucket_exist?(bucket)
  client.get_bucket(bucket, options: request_options)
  true
rescue => e
  return false if e.status_code == 404
  process_error(e)
end

#copy(src_path, dest_path, raise_if_exist: false) ⇒ Object 



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 153

def copy(src_path, dest_path, raise_if_exist: false)
  if raise_if_exist && exist?(dest_path)
    raise Tumugi::FileAlreadyExistError.new("Path #{dest_path} is already exist")
  end

  src_bucket, src_key = path_to_bucket_and_key(src_path)
  dest_bucket, dest_key = path_to_bucket_and_key(dest_path)

  if directory?(src_path)
    src_prefix = add_path_delimiter(src_key)
    dest_prefix = add_path_delimiter(dest_key)

    src_path = add_path_delimiter(src_path)
    copied_objs = []
    entries(src_path).each do |entry|
      suffix = entry.name[src_prefix.length..-1]
      client.copy_object(src_bucket, src_prefix + suffix,
                          dest_bucket, dest_prefix + suffix, options: request_options)
      copied_objs << (dest_prefix + suffix)
    end
    wait_until { copied_objs.all? {|obj| obj_exist?(dest_bucket, obj)} }
  else
    client.copy_object(src_bucket, src_key, dest_bucket, dest_key, options: request_options)
    wait_until { obj_exist?(dest_bucket, dest_key) }
  end
rescue
  process_error($!)
end

#create_bucket(bucket) ⇒ Object 



188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 188

def create_bucket(bucket)
  unless bucket_exist?(bucket)
    b = Google::Apis::StorageV1::Bucket.new(name: bucket)
    client.insert_bucket(@project_id, b, options: request_options)
    true
  else
    false
  end
rescue
  process_error($!)
end

#directory?(path) ⇒ Boolean

Returns:

  • (Boolean) 


75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 75

def directory?(path)
  bucket, key = path_to_bucket_and_key(path)
  if root?(key)
    bucket_exist?(bucket)
  else
    obj = add_path_delimiter(key)
    if obj_exist?(bucket, obj)
      true
    else
      # Any objects with this prefix
      objects = client.list_objects(bucket, prefix: obj, max_results: 20, options: request_options)
      !!(objects.items && objects.items.size > 0)
    end
  end
rescue
  process_error($!)
end

#download(path, download_path: nil, mode: 'r', &block) ⇒ Object 



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 131

def download(path, download_path: nil, mode: 'r', &block)
  bucket, key = path_to_bucket_and_key(path)
  if download_path.nil?
    download_path = Tempfile.new('tumugi_gcs_file_system').path
  end
  client.get_object(bucket, key, download_dest: download_path, options: request_options)
  wait_until { File.exist?(download_path) }

  if block_given?
    File.open(download_path, mode, &block)
  else
    File.open(download_path, mode)
  end
rescue
  process_error($!)
end

#entries(path) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 93

def entries(path)
  bucket, key = path_to_bucket_and_key(path)
  obj = add_path_delimiter(key)
  results = []
  next_page_token = ''

  until next_page_token.nil?
    objects = client.list_objects(bucket, prefix: obj, page_token: next_page_token, options: request_options)
    if objects && objects.items
      results.concat(objects.items)
      next_page_token = objects.next_page_token
    else
      next_page_token = nil
    end
  end
  results
rescue
  process_error($!)
end

#exist?(path) ⇒ Boolean

FileSystem interfaces

Returns:

  • (Boolean) 


22
23
24
25
26
27
28
29
30
31
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 22

def exist?(path)
  bucket, key = path_to_bucket_and_key(path)
  if obj_exist?(bucket, key)
    true
  else
    directory?(path)
  end
rescue
  process_error($!)
end

#mkdir(path, parents: true, raise_if_exist: false) ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 59

def mkdir(path, parents: true, raise_if_exist: false)
  if exist?(path)
    if raise_if_exist
      raise Tumugi::FileAlreadyExistError.new("Path #{path} is already exist")
    elsif !directory?(path)
      raise Tumugi::NotADirectoryError.new("Path #{path} is not a directory")
    end
    false
  else
    put_string("", add_path_delimiter(path))
    true
  end
rescue
  process_error($!)
end

#move(src_path, dest_path, raise_if_exist: false) ⇒ Object 



113
114
115
116
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 113

def move(src_path, dest_path, raise_if_exist: false)
  copy(src_path, dest_path, raise_if_exist: raise_if_exist)
  remove(src_path)
end

#path_to_bucket_and_key(path) ⇒ Object

Raises:

  • (Tumugi::FileSystemError) 


182
183
184
185
186
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 182

def path_to_bucket_and_key(path)
  uri = URI.parse(path)
  raise Tumugi::FileSystemError.new("URI scheme must be 'gs' but '#{uri.scheme}'") unless uri.scheme == 'gs'
  [ uri.host, uri.path[1..-1] ]
end

#put_string(contents, path, content_type: 'text/plain') ⇒ Object 



148
149
150
151
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 148

def put_string(contents, path, content_type: 'text/plain')
  media = StringIO.new(contents)
  upload(media, path, content_type: content_type)
end

#remove(path, recursive: true) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 33

def remove(path, recursive: true)
  bucket, key = path_to_bucket_and_key(path)
  raise Tumugi::FileSystemError.new("Cannot delete root of bucket at path '#{path}'") if root?(key)

  if obj_exist?(bucket, key)
    client.delete_object(bucket, key, options: request_options)
    wait_until { !obj_exist?(bucket, key) }
    true
  elsif directory?(path)
    raise Tumugi::FileSystemError.new("Path '#{path}' is a directory. Must use recursive delete") if !recursive

    objs = entries(path).map(&:name)
    client.batch do |client|
      objs.each do |obj|
        client.delete_object(bucket, obj, options: request_options)
      end
    end
    wait_until { !directory?(path) }
    true
  else
    false
  end
rescue
  process_error($!)
end

#remove_bucket(bucket) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 200

def remove_bucket(bucket)
  if bucket_exist?(bucket)
    client.delete_bucket(bucket, options: request_options)
    true
  else
    false
  end
rescue
  process_error($!)
end

#upload(media, path, content_type: nil) ⇒ Object

Specific methods



122
123
124
125
126
127
128
129
 
# File 'lib/tumugi/plugin/google_cloud_storage/file_system.rb', line 122

def upload(media, path, content_type: nil)
  bucket, key = path_to_bucket_and_key(path)
  obj = Google::Apis::StorageV1::Object.new(bucket: bucket, name: key)
  client.insert_object(bucket, obj, upload_source: media, content_type: content_type, options: request_options)
  wait_until { obj_exist?(bucket, key) }
rescue
  process_error($!)
end