Class: Retriever::FetchFiles

Inherits:
Fetch
  • Object
show all
Defined in:
lib/retriever/fetchfiles.rb

Overview

recieves target url and RR options returns an array of all unique files (based on given filetype) found on the target site

Instance Attribute Summary

Attributes inherited from Fetch

#max_pages, #t

Instance Method Summary collapse

Methods inherited from Fetch

#async_crawl_and_collect, #dump, #errlog, #good_response?, #lg, #process_link_stack, #write

Constructor Details

#initialize(url, options) ⇒ FetchFiles

Returns a new instance of FetchFiles.



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/retriever/fetchfiles.rb', line 6

def initialize(url, options)
  super
  @data = []
  page_one = Retriever::Page.new(@t.source, @t)
  @link_stack = page_one.parse_internal_visitable
  lg("URL Crawled: #{@t.target}")
  lg("#{@link_stack.size - 1} new links found")

  temp_file_collection = page_one.parse_files
  @data.concat(tempFileCollection) if temp_file_collection.size > 0
  lg("#{@data.size} new files found")
  errlog("Bad URL -- #{@t.target}") unless @link_stack
  @link_stack.delete(@t.target)

  async_crawl_and_collect

  @data.sort_by! { |x| x.length }
  @data.uniq!
end

Instance Method Details

#autodownload ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/retriever/fetchfiles.rb', line 39

def autodownload
  # go through the fetched file URL collection and download each one.
  lenny = @data.count
  puts '###################'
  puts '### Initiating Autodownload...'
  puts '###################'
  puts "#{lenny} - #{@file_ext}'s Located"
  puts '###################'
  if File.directory?('rr-downloads')
    Dir.chdir('rr-downloads')
  else
    puts 'creating rr-downloads Directory'
    Dir.mkdir('rr-downloads')
    Dir.chdir('rr-downloads')
  end
  file_counter = 0
  @data.each do |entry|
    begin
      download_file(entry)
      file_counter += 1
      lg('    File [#{file_counter} of #{lenny}]')
      puts
    rescue StandardError => e
      puts 'ERROR: failed to download - #{entry}'
      puts e.message
      puts
    end
  end
  Dir.chdir('..')
end

#download_file(path) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/retriever/fetchfiles.rb', line 26

def download_file(path)
  # given valid url, downloads file to current directory in /rr-downloads/
  arr = path.split('/')
  shortname = arr.pop
  puts "Initiating Download to: '/rr-downloads/' + #{shortname}"
  File.open(shortname, 'wb') do |saved_file|
    open(path) do |read_file|
      saved_file.write(read_file.read)
    end
  end
  puts '  SUCCESS: Download Complete'
end