Class: Etna::Clients::Magma::MaterializeDataWorkflow

Inherits:
Struct
  • Object
show all
Defined in:
lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(**kwds) ⇒ MaterializeDataWorkflow

Returns a new instance of MaterializeDataWorkflow.



15
16
17
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 15

def initialize(**kwds)
  super(**({filesystem: Etna::Filesystem.new}.update(kwds)))
end

Instance Attribute Details

#filesystem ⇒ Object

Returns the value of attribute filesystem

Returns:

  • (Object) —

    the current value of filesystem



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def filesystem
  @filesystem
end

#logger ⇒ Object

Returns the value of attribute logger

Returns:

  • (Object) —

    the current value of logger



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def logger
  @logger
end

#magma_client ⇒ Object

Returns the value of attribute magma_client

Returns:

  • (Object) —

    the current value of magma_client



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def magma_client
  @magma_client
end

#metis_client ⇒ Object

Returns the value of attribute metis_client

Returns:

  • (Object) —

    the current value of metis_client



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def metis_client
  @metis_client
end

#model_attributes_mask ⇒ Object

Returns the value of attribute model_attributes_mask

Returns:

  • (Object) —

    the current value of model_attributes_mask



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def model_attributes_mask
  @model_attributes_mask
end

#model_filters ⇒ Object

Returns the value of attribute model_filters

Returns:

  • (Object) —

    the current value of model_filters



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def model_filters
  @model_filters
end

#model_name ⇒ Object

Returns the value of attribute model_name

Returns:

  • (Object) —

    the current value of model_name



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def model_name
  @model_name
end

#project_name ⇒ Object

Returns the value of attribute project_name

Returns:

  • (Object) —

    the current value of project_name



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def project_name
  @project_name
end

#skip_tmpdir ⇒ Object

Returns the value of attribute skip_tmpdir

Returns:

  • (Object) —

    the current value of skip_tmpdir



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def skip_tmpdir
  @skip_tmpdir
end

#stub_files ⇒ Object

Returns the value of attribute stub_files

Returns:

  • (Object) —

    the current value of stub_files



9
10
11
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 9

def stub_files
  @stub_files
end

Instance Method Details

#each_file(template, record, &block) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 56

def each_file(template, record, &block)
  results = []

  template.attributes.all.each do |attribute|
    if attribute.attribute_type == AttributeType::FILE_COLLECTION
      record[attribute.name]&.each_with_index do |file, i|
        results << [attribute, file, i]
      end
    elsif attribute.attribute_type == AttributeType::FILE
      results << [attribute, record[attribute.name], 0]
    end
  end

  results.each do |attr, file, idx|
    next if file.nil?
    next unless file.is_a?(Hash)
    next unless file['url']
    yield attr.name, file['url'], (file['original_filename'] || File.basename(file['path'])), idx
  end
end

#each_root_record ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 44

def each_root_record
  request = RetrievalRequest.new(project_name: project_name, model_name: model_name, record_names: "all",
      filter: filter, page_size: 100, page: 1)
  magma_crud.page_records(model_name, request) do |response|
    model = response.models.model(model_name)
    template = model.template
    model.documents.document_keys.each do |key|
      yield template, model.documents.document(key)
    end
  end
end

#magma_crud ⇒ Object 



19
20
21
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 19

def magma_crud
  @magma_crud ||= Etna::Clients::Magma::MagmaCrudWorkflow.new(magma_client: magma_client, project_name: project_name)
end

#materialize_all(dest) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 27

def materialize_all(dest)
  tmpdir = skip_tmpdir ? nil : filesystem.tmpdir

  begin
    model_walker.walk_from(
        model_name,
        model_attributes_mask: model_attributes_mask,
        model_filters: model_filters,
    ) do |template, document|
      logger&.info("Materializing #{template.name}##{document[template.identifier]}")
      materialize_record(dest, tmpdir, template, document)
    end
  ensure
    filesystem.rm_rf(tmpdir) unless skip_tmpdir
  end
end

#materialize_record(dest_dir, tmpdir, template, record) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 85

def materialize_record(dest_dir, tmpdir, template, record)
  record_to_serialize = record.dup

  each_file(template, record) do |attr_name, url, filename, idx|
    if idx == 0
      record_to_serialize[attr_name] = []
    end

    dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
    sync_metis_data_workflow.copy_file(bin_root_dir: dest_dir, tmpdir: tmpdir, dest: dest_file, url: url, stub: stub_files)
    record_to_serialize[attr_name] << { file: dest_file, original_filename: filename }
  end

  dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
  filesystem.mkdir_p(File.dirname(dest_file))
  json = record_to_serialize.to_json
  filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
    io.write(json)
  end
end

#metadata_file_name(record_name:, record_model_name:, ext:) ⇒ Object 



106
107
108
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 106

def metadata_file_name(record_name:, record_model_name:, ext:)
  "#{record_model_name}/#{record_name.gsub(/\s/, '_')}#{ext}"
end

#model_walker ⇒ Object 



23
24
25
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 23

def model_walker
  @model_walker ||= WalkModelTreeWorkflow.new(magma_crud: magma_crud, logger: logger)
end

#sync_metis_data_workflow ⇒ Object 



77
78
79
80
81
82
83
 
# File 'lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb', line 77

def sync_metis_data_workflow
  @sync_metis_data_workflow ||= Etna::Clients::Metis::SyncMetisDataWorkflow.new(
      metis_client: metis_client,
      logger: logger,
      skip_tmpdir: skip_tmpdir,
      filesystem: filesystem)
end