Class: Bricolage::StreamingLoad::ManifestFile

Inherits:
Object
  • Object
show all
Defined in:
lib/bricolage/streamingload/manifest.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(ds, job_id, object_urls, logger:, noop: false) ⇒ ManifestFile

Returns a new instance of ManifestFile.



19
20
21
22
23
24
25
 
# File 'lib/bricolage/streamingload/manifest.rb', line 19

def initialize(ds, job_id, object_urls, logger:, noop: false)
  @ds = ds
  @job_id = job_id
  @object_urls = object_urls
  @logger = logger
  @noop = noop
end

Class Method Details

.create(ds:, job_id:, object_urls:, logger:, noop: false, &block) ⇒ Object 



9
10
11
12
13
14
15
16
17
 
# File 'lib/bricolage/streamingload/manifest.rb', line 9

def ManifestFile.create(ds:, job_id:, object_urls:, logger:, noop: false, &block)
  manifest = new(ds, job_id, object_urls, logger: logger, noop: noop)
  if block
    manifest.create_temporary(&block)
  else
    manifest.put
    return manifest
  end
end

Instance Method Details

#content ⇒ Object 



41
42
43
44
45
46
47
48
49
 
# File 'lib/bricolage/streamingload/manifest.rb', line 41

def content
  @content ||= begin
    ents = @object_urls.map {|url|
      { "url" => url, "mandatory" => true }
    }
    obj = { "entries" => ents }
    JSON.pretty_generate(obj)
  end
end

#create_temporary {|_self| ... } ⇒ Object

Yields:

  • (_self)

Yield Parameters:



67
68
69
70
71
 
# File 'lib/bricolage/streamingload/manifest.rb', line 67

def create_temporary
  put
  yield self
  delete
end

#credential_string ⇒ Object 



27
28
29
 
# File 'lib/bricolage/streamingload/manifest.rb', line 27

def credential_string
  @ds.credential_string
end

#delete ⇒ Object 



59
60
61
62
63
64
65
 
# File 'lib/bricolage/streamingload/manifest.rb', line 59

def delete
  @logger.info "s3: delete: #{url}"
  @ds.object(name).delete unless @noop
rescue Aws::S3::Errors::ServiceError => ex
  @logger.exception ex
  raise S3Exception.wrap(ex)
end

#name ⇒ Object 



31
32
33
34
35
 
# File 'lib/bricolage/streamingload/manifest.rb', line 31

def name
  return @name if @name
  now =Time.now
  "#{now.strftime('%Y/%m/%d')}/manifest-#{now.strftime('%H%M%S')}-#{@job_id}.json"
end

#put ⇒ Object 



51
52
53
54
55
56
57
 
# File 'lib/bricolage/streamingload/manifest.rb', line 51

def put
  @logger.info "s3: put: #{url}"
  @ds.object(name).put(body: content) unless @noop
rescue Aws::S3::Errors::ServiceError => ex
  @logger.exception ex
  raise S3Exception.wrap(ex)
end

#url ⇒ Object 



37
38
39
 
# File 'lib/bricolage/streamingload/manifest.rb', line 37

def url
  @url ||= @ds.url(name)
end