21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
# File 'lib/download_files.rb', line 21
def self.download_files(
page_address,
pattern,
dir,
ssl_verify,
is_verbose,
logger
)
logger.debug(
"DownloadFiles.download_files: " +
{
page_address: page_address,
pattern: pattern,
dir: dir,
ssl_verify: ssl_verify,
is_verbose: is_verbose,
logger: logger
}.inspect
)
abs_dir = File.absolute_path(dir)
logger.info(
"Downloading files from \"#{page_address}\" to \"#{abs_dir}\" that match /#{pattern}/ " +
(if ssl_verify then 'requiring' else 'without requiring' end + ' SSL verification') +
'...'
)
Mechanize.go(page_address, ssl_verify) do |agent, page|
logger.debug page.inspect
if is_verbose
logger.info "On #{page.uri}"
end
agent.pluggable_parser.default = Mechanize::Download
page.links.each do |link|
if is_verbose
logger.info "Checking link #{link.href}"
end
next unless link.href && link.href.match(pattern)
full_file_name = File.expand_path(File.basename(link.href), abs_dir)
if is_verbose
logger.info "Downloading #{link.click.uri} to #{full_file_name}"
end
agent.get(link.click.uri).save(full_file_name)
end
end
end
|