Class: WebCrawler::Follower

Inherits:
Object
  • Object
show all
Defined in:
lib/web_crawler/follower.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*responses) ⇒ Follower

Returns a new instance of Follower.



5
6
7
8
 
# File 'lib/web_crawler/follower.rb', line 5

def initialize(*responses)
  @options   = responses.extract_options!
  @responses = responses.flatten
end

Instance Attribute Details

#options ⇒ Object (readonly)

Returns the value of attribute options.



3
4
5
 
# File 'lib/web_crawler/follower.rb', line 3

def options
  @options
end

Instance Method Details

#collect(&block) ⇒ Object 



19
20
21
22
23
24
25
26
 
# File 'lib/web_crawler/follower.rb', line 19

def collect(&block)
  urls = @responses.map do |response|
    parser = WebCrawler::Parsers::Url.new(response.url.host, url: response.url.request_uri, same_host: @options[:same_host])
    parser.parse(response.body, &block)
  end.flatten
  urls = urls.select { |url| url =~ @options[:only] } if @options[:only]
  urls
end

#follow(response) ⇒ Object 



14
15
16
17
 
# File 'lib/web_crawler/follower.rb', line 14

def follow(response)
  @responses += Array.wrap(response)
  self
end

#process(options = { }) ⇒ Object 



10
11
12
 
# File 'lib/web_crawler/follower.rb', line 10

def process(options = { })
  WebCrawler::BatchRequest.new(collect, options).process
end