Class: Caule::Bot

Inherits:
Object
  • Object
show all
Defined in:
lib/caule/bot.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(*urls) ⇒ Bot

Returns a new instance of Bot.



7
8
9
10
11
12
13
14
 
# File 'lib/caule/bot.rb', line 7

def initialize(*urls)
  @urls = urls.flatten
  @agent = Mechanize.new do |a|
    a.user_agent_alias = "Windows IE 7"
  end

  @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
end

Instance Attribute Details

#agent ⇒ Object (readonly)

Returns the value of attribute agent.



5
6
7
 
# File 'lib/caule/bot.rb', line 5

def agent
  @agent
end

#urls ⇒ Object (readonly)

Returns the value of attribute urls.



5
6
7
 
# File 'lib/caule/bot.rb', line 5

def urls
  @urls
end

Instance Method Details

#focus_crawl(&block) ⇒ Object 



47
48
49
 
# File 'lib/caule/bot.rb', line 47

def focus_crawl(&block)
  @focus_crawl_block = block
end

#on_every_page(&block) ⇒ Object 



16
17
18
 
# File 'lib/caule/bot.rb', line 16

def on_every_page(&block)
  on_pages_like(/.+/, &block)
end

#on_pages_like(*patterns, &block) ⇒ Object 



20
21
22
23
24
25
26
 
# File 'lib/caule/bot.rb', line 20

def on_pages_like(*patterns, &block)
  if patterns && !patterns.empty?
    patterns.each do |pattern|
      @on_pages_like_blocks[pattern] << block
    end
  end
end

#run ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/caule/bot.rb', line 28

def run
  urls.each do |url|
    page = agent.get(url)
    do_page_blocks(page)

    stack = links_to_follow(page)
    while l = stack.shift
      begin
        page = l.click
        next unless Mechanize::Page === page
        do_page_blocks(page)
        stack.push(*links_to_follow(page))
      rescue Mechanize::ResponseCodeError => ex
        puts ex.message.red
      end
    end
  end
end