Class: Etna::Clients::Metis::SyncMetisDataWorkflow

Inherits:
Struct
  • Object
show all
Defined in:
lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#bucket_name ⇒ Object

Returns the value of attribute bucket_name

Returns:

  • (Object) —

    the current value of bucket_name



9
10
11
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 9

def bucket_name
  @bucket_name
end

#filesystem ⇒ Object

Returns the value of attribute filesystem

Returns:

  • (Object) —

    the current value of filesystem



9
10
11
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 9

def filesystem
  @filesystem
end

#logger ⇒ Object

Returns the value of attribute logger

Returns:

  • (Object) —

    the current value of logger



9
10
11
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 9

def logger
  @logger
end

#metis_client ⇒ Object

Returns the value of attribute metis_client

Returns:

  • (Object) —

    the current value of metis_client



9
10
11
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 9

def metis_client
  @metis_client
end

#project_name ⇒ Object

Returns the value of attribute project_name

Returns:

  • (Object) —

    the current value of project_name



9
10
11
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 9

def project_name
  @project_name
end

Instance Method Details

#bin_file_name(etag:) ⇒ Object 



32
33
34
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 32

def bin_file_name(etag:)
  "bin/#{etag}"
end

#copy_directory(src, dest, root = dest, tmpdir = nil) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 10

def copy_directory(src, dest, root = dest, tmpdir = nil)
  own_tmpdir = tmpdir.nil?
  if own_tmpdir
    tmpdir = filesystem.tmpdir
  end

  begin
    response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))

    response.files.all.each do |file|
      logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
      copy_file(bin_root_dir: root, tmpdir: tmpdir, dest: ::File.join(dest, file.file_name), url: file.download_url)
    end

    response.folders.all.each do |folder|
      copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root, tmpdir)
    end
  ensure
    filesystem.rm_rf(tmpdir) if own_tmpdir
  end
end

#copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 36

def copy_file(bin_root_dir:, tmpdir:, dest:, url:, stub: false)
  metadata = metis_client.file_metadata(url)
  etag = metadata[:etag]
  size = metadata[:size]

  dest_bin_file = ::File.join(bin_root_dir, bin_file_name(etag: etag))
  # Already materialized, continue
  if filesystem.exist?(dest_bin_file)
    return
  end

  tmp_file = ::File.join(tmpdir, etag)


  upload_timings = []
  upload_amount = 0
  last_rate = 0.00001

  filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
    if stub
      io.write("(stub) #{size} bytes")
    else
      metis_client.download_file(url) do |chunk|
        io.write(chunk)

        upload_timings << [chunk.length, Time.now.to_f]
        upload_amount += chunk.length

        if upload_timings.length > 150
          s, _ = upload_timings.shift
          upload_amount -= s
        end

        _, start_time = upload_timings.first
        _, end_time = upload_timings.last

        if start_time == end_time
          next
        end

        rate = upload_amount / (end_time - start_time)

        if rate / last_rate > 1.3 || rate / last_rate < 0.7
          logger&.info("Uploading #{Etna::Formatting.as_size(rate)} per second")

          if rate == 0
            last_rate = 0.0001
          else
            last_rate = rate
          end
        end
      end
    end
  end

  filesystem.mkdir_p(::File.dirname(dest))
  filesystem.mv(tmp_file, dest)

  filesystem.mkdir_p(::File.dirname(dest_bin_file))
  filesystem.with_writeable(dest_bin_file, 'w', size_hint: 0) do |io|
    # empty file
  end
end