Class: Nsrr::Models::File

Inherits:
Object
  • Object
show all
Defined in:
lib/nsrr/models/file.rb

Overview

Models a downloadable file or folder

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(json = {}) ⇒ File 



14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/nsrr/models/file.rb', line 14

def initialize(json = {})
  @dataset_slug = json["dataset"]
  @full_path = json["full_path"]
  @folder = json["folder"]
  @file_name = json["file_name"]
  @is_file = json["is_file"]
  @file_size = json["file_size"]
  @file_checksum_md5 = json["file_checksum_md5"]
  @archived = json["archived"]
  @latest_checksum = ""
  @latest_file_size = -1
end

Instance Attribute Details

#archived ⇒ Object (readonly)

Returns the value of attribute archived.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def archived
  @archived
end

#dataset_slug ⇒ Object (readonly)

Returns the value of attribute dataset_slug.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def dataset_slug
  @dataset_slug
end

#file_checksum_md5 ⇒ Object (readonly)

Returns the value of attribute file_checksum_md5.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def file_checksum_md5
  @file_checksum_md5
end

#file_name ⇒ Object (readonly)

Returns the value of attribute file_name.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def file_name
  @file_name
end

#file_size ⇒ Object (readonly)

Returns the value of attribute file_size.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def file_size
  @file_size
end

#folder ⇒ Object (readonly)

Returns the value of attribute folder.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def folder
  @folder
end

#full_path ⇒ Object (readonly)

Returns the value of attribute full_path.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def full_path
  @full_path
end

#is_file ⇒ Object (readonly)

Returns the value of attribute is_file.



12
13
14
 
# File 'lib/nsrr/models/file.rb', line 12

def is_file
  @is_file
end

Instance Method Details

#did_download_succeed?(method, path) ⇒ Boolean 



128
129
130
131
132
133
134
 
# File 'lib/nsrr/models/file.rb', line 128

def did_download_succeed?(method, path)
  if method == "fast"
    file_size_matches?(path)
  else
    md5_matches?(path)
  end
end

#download(method, path, token) ⇒ Object

method:

"md5"       => [default] Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
"fresh"     => Downloads every file without checking if it was already downloaded
"fast"      => Only checks if a download file exists with the same file size as the online version, if so, skips that file



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/nsrr/models/file.rb', line 31

def download(method, path, token)
  do_md5_check = (method == "md5")
  redownload_all = (method == "fresh")

  if do_md5_check
    if md5_matches?(path)
      skip
    else
      force_download(path, token, method)
    end
  else
    if redownload_all
      force_download(path, token, method)
    else
      if file_size_matches?(path)
        skip
      else
        force_download(path, token, method)
      end
    end
  end
end

#file_size_matches?(path) ⇒ Boolean 



114
115
116
 
# File 'lib/nsrr/models/file.rb', line 114

def file_size_matches?(path)
  @file_size == local_filesize(path)
end

#force_download(path, token, method) ⇒ Object

MD5 or file size checks are now performed after a file is downloaded without error. If the file check fails, the file is downloaded a second time and rechecked. If the second download and check fail, the file is marked as failed, and the downloader continues to the subsequent file.



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/nsrr/models/file.rb', line 57

def force_download(path, token, method)
  download_folder = ::File.join(Dir.pwd, path.to_s, @file_name.to_s)
  auth_section = (token.to_s == "" ? "" : "/a/#{token}")
  download_url = "#{Nsrr::WEBSITE}/datasets/#{@dataset_slug}/files#{auth_section}/m/nsrr-gem-v#{Nsrr::VERSION::STRING.gsub(".", "-")}/#{@full_path.to_s}"
  download_request = Nsrr::Helpers::DownloadRequest.new(download_url, download_folder)
  download_request.get
  download_success = false
  if download_request.error.to_s == ""
    # Check to see if the file downloaded correctly
    # If the file size does not match, attempt one additional download
    download_success = did_download_succeed?(method, path)
    unless download_success
      download_request = Nsrr::Helpers::DownloadRequest.new(download_url, download_folder)
      download_request.get
      download_success = did_download_succeed?(method, path)
    end
  end
  if download_request.error.to_s == "" and download_success
    puts "  downloaded".green + " #{@file_name}"
    download_request.file_size
  elsif download_request.error.to_s == ""
    puts "      failed".red + " #{@file_name}"
    if method == "fast"
      puts "             File size mismatch, expected: #{@file_size}"
      puts "                                   actual: #{@latest_file_size}"
    else
      puts "             File checksum mismatch, expected: #{@file_checksum_md5}"
      puts "                                       actual: #{@latest_checksum}"
    end
    ::File.delete(download_folder) if ::File.exist?(download_folder)
    "fail"
  else
    puts "      failed".red + " #{@file_name}"
    puts "             #{download_request.error}"
    "fail"
  end
end

#local_checksum(path) ⇒ Object 



104
105
106
107
108
109
110
111
112
 
# File 'lib/nsrr/models/file.rb', line 104

def local_checksum(path)
  download_folder = ::File.join(Dir.pwd, path.to_s, file_name.to_s)
  @latest_checksum = if ::File.exist?(download_folder)
                       Digest::MD5.file(download_folder).hexdigest
                     else
                       ""
                     end
  @latest_checksum
end

#local_filesize(path) ⇒ Object 



118
119
120
121
122
123
124
125
126
 
# File 'lib/nsrr/models/file.rb', line 118

def local_filesize(path)
  download_folder = ::File.join(Dir.pwd, path.to_s, file_name.to_s)
  @latest_file_size = if ::File.exist?(download_folder)
                        ::File.size(download_folder)
                      else
                        -1
                      end
  @latest_file_size
end

#md5_matches?(path) ⇒ Boolean 



100
101
102
 
# File 'lib/nsrr/models/file.rb', line 100

def md5_matches?(path)
  @file_checksum_md5 == local_checksum(path)
end

#skip ⇒ Object 



95
96
97
98
 
# File 'lib/nsrr/models/file.rb', line 95

def skip
  puts "   identical".blue + " #{file_name}"
  "skip"
end