Class: Etna::Clients::Metis::SyncMetisDataWorkflow

Inherits:
Struct
  • Object
show all
Defined in:
lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

Constant Summary collapse

DOWNLOAD_REGEX = 
/^https:\/\/[^\/]*\/(?<project_name>.*)\/download\/(?<bucket_name>.*)\/(?<file_path>[^\?]*).*$/

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#bucket_name ⇒ Object

Returns the value of attribute bucket_name

Returns:

  • (Object) —

    the current value of bucket_name



8
9
10
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 8

def bucket_name
  @bucket_name
end

#filesystem ⇒ Object

Returns the value of attribute filesystem

Returns:

  • (Object) —

    the current value of filesystem



8
9
10
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 8

def filesystem
  @filesystem
end

#logger ⇒ Object

Returns the value of attribute logger

Returns:

  • (Object) —

    the current value of logger



8
9
10
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 8

def logger
  @logger
end

#metis_client ⇒ Object

Returns the value of attribute metis_client

Returns:

  • (Object) —

    the current value of metis_client



8
9
10
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 8

def metis_client
  @metis_client
end

#project_name ⇒ Object

Returns the value of attribute project_name

Returns:

  • (Object) —

    the current value of project_name



8
9
10
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 8

def project_name
  @project_name
end

Instance Method Details

#copy_directory(src, dest, root = dest) ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 11

def copy_directory(src, dest, root = dest)
  response = metis_client.list_folder(ListFolderRequest.new(project_name: project_name, bucket_name: bucket_name, folder_path: src))

  response.files.all.each do |file|
    logger&.info("Copying file #{file.file_path} (#{Etna::Formatting.as_size(file.size)})")
    copy_file(dest: ::File.join(dest, file.file_name), url: file.download_url)
  end

  response.folders.all.each do |folder|
    copy_directory(::File.join(src, folder.folder_name), ::File.join(dest, folder.folder_name), root)
  end
end

#copy_file(dest:, url:, stub: false) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb', line 24

def copy_file(dest:, url:, stub: false)
  url_match = DOWNLOAD_REGEX.match(url)

  if filesystem.instance_of?(Etna::Filesystem::Metis) && !url_match.nil?
    bucket_name = url_match[:bucket_name]
    project_name = url_match[:project_name]
    file_path = url_match[:file_path]

    # ensure target parent directory exists
    metis_client.ensure_parent_folder_exists(
      project_name: filesystem.project_name,
      bucket_name: filesystem.bucket_name,
      path: dest
    )

    metis_client.copy_files(
      Etna::Clients::Metis::CopyFilesRequest.new(
        project_name: project_name,
        revisions: [
          Etna::Clients::Metis::CopyRevision.new(
            source: "metis://#{project_name}/#{bucket_name}/#{file_path}",
            dest: "metis://#{filesystem.project_name}/#{filesystem.bucket_name}/#{dest}",
          )
        ]
      )
    )

    return
  end

  metadata = metis_client.file_metadata(url)
  size = metadata[:size]

  begin
    if filesystem.exist?(dest) && filesystem.stat(dest).size == size
      logger&.info "Already downloaded #{dest}"
      return
    end
  rescue Etna::Filesystem::Error => e
    unless e.message =~ /stat not supported/
      raise e
    end
  end

  tmp_file = dest
  upload_timings = []
  upload_amount = 0
  last_rate = 0.00001
  remaining = size

  logger&.info "Downloading #{dest} - #{Etna::Formatting.as_size(size)}"
  filesystem.with_writeable(tmp_file, "w", size_hint: size) do |io|
    if stub
      io.write("(stub) #{size} bytes")
    else
      metis_client.download_file(url) do |chunk|
        io.write(chunk)

        upload_timings << [chunk.length, Time.now.to_f]
        upload_amount += chunk.length
        remaining -= chunk.length unless remaining.nil?

        if upload_timings.length > 150
          s, _ = upload_timings.shift
          upload_amount -= s
        end

        _, start_time = upload_timings.first
        _, end_time = upload_timings.last

        if start_time == end_time
          next
        end

        rate = upload_amount / (end_time - start_time)

        if rate / last_rate > 1.3 || rate / last_rate < 0.7
          logger&.debug("Uploading #{Etna::Formatting.as_size(rate)} per second, #{Etna::Formatting.as_size(remaining)} remaining")

          if rate == 0
            last_rate = 0.0001
          else
            last_rate = rate
          end
        end
      end
    end
  end
end