Class: Aquatone::Collectors::WaybackMachine

Inherits:
Aquatone::Collector show all
Defined in:
lib/aquatone/collectors/wayback_machine.rb

Constant Summary collapse

DEFAULT_TIMEOUT = 
30.freeze

Constants inherited from Aquatone::Collector

Aquatone::Collector::DEFAULT_PRIORITY

Instance Attribute Summary

Attributes inherited from Aquatone::Collector

#domain, #hosts

Instance Method Summary collapse

Methods inherited from Aquatone::Collector

cli_options, descendants, #execute!, #initialize, meta, meta=, priority, sluggified_name

Constructor Details

This class inherits a constructor from Aquatone::Collector

Instance Method Details

#run ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/aquatone/collectors/wayback_machine.rb', line 17

def run
  response = nil
  Timeout::timeout(timeout) do
    response = get_request("http://web.archive.org/cdx/search/cdx?url=*.#{url_escape(domain.name)}&output=json&fl=original&collapse=urlkey")
  end
  response.parsed_response.each do |page|
    if page[0] != "original"
      begin
        add_host(URI.parse(page[0]).host)
      rescue URI::Error; end
    end
  end
end