Class: Smash::CloudPowers::Storage::Bucket

Inherits:
Resource
  • Object
show all
Includes:
Smash::CloudPowers::Storage
Defined in:
lib/cloud_powers/storage/bucket.rb

Instance Attribute Summary collapse

Attributes inherited from Resource

#call_name, #client, #linked, #meta, #name, #remote_id, #tags, #type

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Smash::CloudPowers::Storage

#all_storage, #build_storage, #create_storage, #existing_storage, #local_job_file_exists?, #search, #send_logs_to_s3, #source_job, #storage_select

Methods included from Helpers

#create_logger, #log_file, #logger

Methods included from PathHelp

#common_delimiter, #expand_path, #file_exists?, #file_search, #filename?, #job_exist?, #job_path, #job_require_path, #path_search, #paths_gcd, #paths_lcd, #to_path, #to_pathname, #to_realpath, #touch, #zlib_path

Methods included from LogicHelp

#attr_map, #called_from, #i_var_hash, #instance_attr_accessor, #smart_retry, #update_message_body, #wait_until

Methods included from LangHelp

#deep_modify_keys_with, #extract!, #find_and_remove, #format_error_message, #from_json, #modify_keys_with, #to_basic_hash, #to_camel, #to_hyph, #to_i_var, #to_pascal, #to_ruby_file_name, #to_snake, #valid_json?, #valid_url?

Methods included from AwsResources

#ec2, #image, #kinesis, #queue_poller, #region, #sns, #sqs

Methods included from Zenv

#env_vars, #i_vars, #lsof_cwd, #pid, #proc_cwd, #process_search, #project_root, #project_root=, #ps_cwd, #system_vars, #zfind, #zselect

Methods included from Auth

creds, region

Methods included from Creatable

included

Constructor Details

#initialize(name:, client: s3, **config) ⇒ Bucket

Returns a new instance of Bucket.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/cloud_powers/storage/bucket.rb', line 12

def initialize(name:, client: s3, **config)
  super
  @s3 = client
  @bucket = Aws::S3::Bucket.new(name: name, client: client)
  @origin = config[:origin] || project_root.split.last
  build_storage(
    name: 'local',
    named_type: 'storage',
    project_root: @project_root
  ).link
  @delimiter = config[:delimiter] || common_delimiter
  # should be the 'lowest level' of storage this object is aware of
  @tie_in_path = local_storage.tie_in_path
end

Instance Attribute Details

#delimiter ⇒ Object

Returns the value of attribute delimiter.



10
11
12
 
# File 'lib/cloud_powers/storage/bucket.rb', line 10

def delimiter
  @delimiter
end

#origin ⇒ Object

Returns the value of attribute origin.



10
11
12
 
# File 'lib/cloud_powers/storage/bucket.rb', line 10

def origin
  @origin
end

#s3 ⇒ Object

Returns the value of attribute s3.



10
11
12
 
# File 'lib/cloud_powers/storage/bucket.rb', line 10

def s3
  @s3
end

#tie_in_path ⇒ Object

Returns the value of attribute tie_in_path.



10
11
12
 
# File 'lib/cloud_powers/storage/bucket.rb', line 10

def tie_in_path
  @tie_in_path
end

Class Method Details

.tie_in_config ⇒ Object 



27
28
29
30
 
# File 'lib/cloud_powers/storage/bucket.rb', line 27

def self.tie_in_config
  # TODO make this able to use #to_snake so it can be dynamic
  { name: 'bucket', client: 's3' }
end

Instance Method Details

#create_resource {|_self| ... } ⇒ Object

def bucket_select(pattern, **config)

client.list_buckets.buckets.select { |b| %r"#{name}" =~ b.name }

end

Yields:

  • (_self)

Yield Parameters:



40
41
42
43
44
45
46
 
# File 'lib/cloud_powers/storage/bucket.rb', line 40

def create_resource
  # TODO! Use Seahorse #on_complete, #on_success etc. callbacks
  @response = s3.create_bucket(bucket: name).data
  @remote_origin = @response.location.gsub(%r"^\/",'')
  yield self if block_given?
  self
end

#exists? ⇒ Boolean

Returns:

  • (Boolean) 


48
49
50
51
 
# File 'lib/cloud_powers/storage/bucket.rb', line 48

def exists?
  # TODO nil.exists? will asplode
  !!(@bucket.exists? rescue nil)
end

#find(*patterns, location: origin, **opts) ⇒ Object

Find the closest match to the request you’re looking for with those params. If you pass this method a block, a file will be created and passed to the block, along with the data from the file that was found in the bucket.

Parameters

  • patterns String|Regexp - names of files or buckets to look for

  • location String - narrow down the scope of the search

  • opts KeywordArgument(s) - pass more specific configuration along to the search method(s) and object(s) used

Returns

  • String:

    • if no block was passed, only the name of the object found is returned

    • if a block is passed to this method, you will have a file and data object to use as params. The file is created at the location of this Storage object’s origin, which is defaulted to project_root/zlib/+@name+_bucket. The data object is the data from the object in the bucket



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/cloud_powers/storage/bucket.rb', line 73

def find(*patterns, location: origin, **opts)
  path = select(*patterns, location: location, **opts).first
  file_name = path.kind_of?(Pathname) ? path.split.last.to_s : path
  file_permissions = opts[:file_permissions] || 'a+'
  path = local_storage.tie_in_path + path

  if block_given?
    begin
      to_realpath(path)
      File.open(path, file_permissions) do |file|
        data = get_file(file_name, location: location).get.body.read
        yield file, data
      end
    rescue Exception => e
      byebug
      if e =~ /no implicit conversion of nil into String/
        logger.info "no file found from #{path} called #{file}"
        nil
      else
        raise
      end
    end
  else
    path
  end
end

#job_file_exists?(file_name, location = job_path) ⇒ Boolean

Find out if a job exists in the bucket

Parameters

  • file_name Pathname|String - the name of the file you want to pass

  • location Pathname|String (optional) - this helps speed up the process by limiting any searching for the file

Returns IO -

Returns:

  • (Boolean) 


109
110
111
112
 
# File 'lib/cloud_powers/storage/bucket.rb', line 109

def job_file_exists?(file_name, location = job_path)
  normalized_location = paths_gcd(project_root, location)
  !get_file_names(file_name, location: normalized_location).empty?
end

#link ⇒ Object 



114
115
116
117
118
119
120
121
122
123
 
# File 'lib/cloud_powers/storage/bucket.rb', line 114

def link
  if exists?
    byebug
    # do stuff
  else
    save!
  end

  @linked = @response.nil?
end

#put(file_name, location = local_storage.tie_in_path, **config) ⇒ Object

Notes:

  • See https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/ Client.html#put_object-instance_method



186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/cloud_powers/storage/bucket.rb', line 186

def put(file_name, location = local_storage.tie_in_path, **config)
  body = local_storage.find(
    file_name,
    location: location,
    file_rights: 'r'
  ) { |data| data }

  return if body.nil?

  client.put_object(
    put_config(body: body, key: file_name, **config)
  )#.on_success { Thread.new { update_registry } }
  true
end

#select(*patterns, location: @tie_in_path, **opts) ⇒ Object

names = get_file_names(pattern, location: location)

if block_given?
  names.map do |file_name|
    new_file = to_realpath(local_storage.find(file_name))
    File.open(new_file, 'r+') do |file|
      data = get_file(file_name, location: location).get.body.read
      yield file, data
    end
  end
else
  names
end

end



149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/cloud_powers/storage/bucket.rb', line 149

def select(*patterns, location: @tie_in_path, **opts)
  file_rights = opts[:file_rights] || 'a+'
  names = patterns.map do |pattern|
    pattern = pattern.kind_of?(Regexp) ? pattern : %r"#{pattern}"

    # if filename?(pattern.source)
    #   get_file_names(pattern, location: location)
    # else
    #   get_bucket_names(pattern, location: location)
    # end
    files_thread = Thread.new do
      get_file_names(pattern, location: location)
    end
    bucket_thread = Thread.new do
      get_bucket_names(pattern, location: location)
    end
    files_thread.join
    bucket_thread.join

    [files_thread.value, bucket_thread.value]
  end.flatten
  if block_given?
    names.map do |file_name|
      new_file = local_storage.touch(file_name)
      File.open(new_file, file_rights) do |file|
        data = get_file(file_name, location: location).get.body.read
        yield file, data
      end
    end
  else
    names
  end
end