Class: Downloader

Inherits:
Object show all
Defined in:
lib/more/facets/downloader.rb

Overview

Downloader

Tool module for downloading files and extracting archive files. Currently this is console-based but in the future will have SOC for any interface.

Defined Under Namespace

Classes: Checksum, Mirror

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(destination, mirrors = nil, region = nil, &config) ⇒ Downloader

Returns a new instance of Downloader.



64
65
66
67
68
69
70
 
# File 'lib/more/facets/downloader.rb', line 64

def initialize( destination, mirrors=nil, region=nil, &config )
  @destination = destination
  @mirrors = []
  add_mirrors( mirrors ) if mirrors
  @region = (region || 'US').to_s
  config.call(self) if config
end

Instance Attribute Details

#destination ⇒ Object

Returns the value of attribute destination.



72
73
74
 
# File 'lib/more/facets/downloader.rb', line 72

def destination
  @destination
end

#region ⇒ Object

Returns the value of attribute region.



72
73
74
 
# File 'lib/more/facets/downloader.rb', line 72

def region
  @region
end

Instance Method Details

#add_mirror(url, region = nil) ⇒ Object 



74
75
76
 
# File 'lib/more/facets/downloader.rb', line 74

def add_mirror( url, region=nil )
  @mirrors << Mirror.new( url, region )
end

#add_mirrors(mirrors) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/more/facets/downloader.rb', line 78

def add_mirrors( mirrors )
  mirrors.each { |mirror|
    case mirror
    when String
      url, rgn = *mirror.strip.split(' ')
      add_mirror( url, rgn.unbracket )
    when Array
      add_mirror( *mirror )
    when Hash
      add_mirror( mirror[:url], mirror[:region] )
    when Mirror
      @mirrors << mirror
    else
      raise "unrecogized mirror definition #{mirror.inspect}"
    end
  }
end

#compute_checksum(local_path) ⇒ Object

compute_checksum



238
239
240
241
242
243
244
 
# File 'lib/more/facets/downloader.rb', line 238

def compute_checksum( local_path )
  if File.exists?( local_path )
    File.open( local_path ) do |local_file|
      return Digest::MD5.new( local_file.read ).hexdigest
    end
  end
end

#download(url, filepath, checksum = '', est_size = nil) ⇒ Object

currently can only download a single compressed file does not handle downloading an uncompressed directory tree (should it? doubt it)

currently this displays progress to STDOUT; either their should be a way to activate/deactivate or preferably use ducktype singletons (more on that later, see google://_whytheluckystiff if interested) of course I prefer chain messaging but matz said no :(



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/more/facets/downloader.rb', line 177

def download( url, filepath, checksum='', est_size=nil )

  checksum = checksum.to_s.strip
  est_size = nil if est_size == 0

  download_complete = nil

  if interface
    interface.preparing_to_download( File.basename( filepath ), url, est_size )
  end

  progress_total = est_size ? est_size : 100000000  # pretend 100MB if no size
  pbar = Console::ProgressBar.new( "Status", progress_total, STDOUT )
  pbar.bar_mark = "="
  pbar.format = "%-6s %3d%% %s %s"
  pbar.file_transfer_mode if est_size

  progress_proc = proc { |posit| pbar.set(posit) }

  STDOUT.sync = true
  begin
    local_file = File.open( filepath, 'wb' )
    remote_file = open( url, :progress_proc => progress_proc )
    local_file << remote_file.read
  rescue
    pbar.halt
    download_complete = nil
    raise
  else
    pbar.finish
    download_complete = filepath
  ensure
    remote_file.close unless remote_file.nil?
    local_file.close unless local_file.nil?
    STDOUT.sync = false
  end

  unless checksum.empty?
    raise ChecksumError if compute_checksum(filepath) != checksum
  end

  if interface
    if checksum.empty?
      interface.lacks_checksum( compute_checksum(filepath), :md5 )
    end
    unless est_size
      interface.lacks_size( File.size(filepath) )
    end
  end

  if download_complete
    if interface
      interface.downloaded( filepath )
    end
  end

  return download_complete
end

#extract(local_path) ⇒ Object

extract



248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/more/facets/downloader.rb', line 248

def extract( local_path )
  success = false
  local_dir = File.dirname(local_path)
  local_file = File.basename(local_path)
  current_dir = Dir.getwd
  begin
    Dir.chdir(local_dir)
    case local_file
      when /.*gz$/
        system "tar -xzf #{local_file}"
      when /.*bz2$/
        system "tar -xjf #{local_file}"
      when /.zip$/
        system "unzip #{local_file}"
      else
        success = false
    end
  rescue
    success = false
  else
    success = true
  ensure
    Dir.chdir(current_dir)
  end
  if interface
    interface.extracted( local_path)
  end
  return success
end

#fetch(file, checksum = 0, est_size = 0, force = false) ⇒ Object

fetch



102
103
104
105
106
107
 
# File 'lib/more/facets/downloader.rb', line 102

def fetch( file, checksum=0, est_size=0, force=false )
  urls = mirrors.collect { |m| "#{m.url.chomp('/')}/#{file}" }
  urls = prioritize_urls( urls )
  filepath = "#{destination.chomp('/')}/#{file}"
  monitored_download( urls, filepath, checksum, est_size, force )
end

#interface ⇒ Object

delegate to an interface (under development)



59
 
# File 'lib/more/facets/downloader.rb', line 59

def interface ; @interface; end

#interface=(iface) ⇒ Object 



60
61
62
 
# File 'lib/more/facets/downloader.rb', line 60

def interface=(iface)
  @interface = iface
end

#mirrors ⇒ Object 



96
97
98
 
# File 'lib/more/facets/downloader.rb', line 96

def mirrors
  @mirrors.sort{ |a,b| a.region == region ? 1 : ( b.region == region ? -1 : 0 ) }
end

#monitored_download(urls, filepath, checksum, est_size = 0, force = false) ⇒ Object

regional_urls - array of arrays of [ url, region, md5, expected_size ] local_region - region of the user’s system to_dir - where to store downloaded file (full path) force - download even if file already exists locally



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/more/facets/downloader.rb', line 123

def monitored_download( urls, filepath, checksum, est_size=0, force=false )

  checksum = checksum.to_s.strip
  est_size = nil if est_size == 0

  success=nil

  # source file exists and passes checksum then we need not fetch
  #file_path = File.join(to_dir,File.basename(url[0]))
  if File.exists?(filepath)
    if compute_checksum(filepath) == checksum and ! force
      interface.report("File has already been fetched and passes checksum.")
      success = filepath
    else
      File.delete(filepath)
    end
  end

  unless success
    # download
    urls.each do |url|
      begin
        #file_path = File.join(to_dir,File.basename(url[0]))
        #file_checksum = url[2].to_s.strip
        #file_size = url[3].to_i
        success = self.download( url, filepath, checksum, est_size )
        break if success
      rescue
        next
      end
    end
  end

  return success
end

#prioritize_urls(urls) ⇒ Object

In the future we may test each connection for fastest download



161
162
163
164
165
166
167
 
# File 'lib/more/facets/downloader.rb', line 161

def prioritize_urls( urls )
  urls
  # put local region first
  #prioritized_urls = regional_urls.find_all { |a| a[1] == local_region }
  #prioritized_urls.concat regional_urls.find_all { |a| a[1] != local_region }
  #return prioritized_urls
end