Module: EmailCrawler::MechanizeHelper

Included in:
PageLinks, Scraper
Defined in:
lib/email_crawler/mechanize_helper.rb

Instance Method Summary collapse

Instance Method Details

#get(url) ⇒ Object 



15
16
17
18
19
20
 
# File 'lib/email_crawler/mechanize_helper.rb', line 15

def get(url)
  begin
    page = agent.get(url)
    page if page.is_a?(Mechanize::Page)
  rescue Mechanize::Error, Net::OpenTimeout; end
end

#new_agent ⇒ Object 



5
6
7
8
9
10
11
12
13
 
# File 'lib/email_crawler/mechanize_helper.rb', line 5

def new_agent
  Thread.current[:agent] ||= Mechanize.new do |agent|
    agent.user_agent_alias = "Mac Safari"
    agent.open_timeout = agent.read_timeout = 30
    agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
    agent.history.max_size = 1
    yield(agent) if block_given?
  end
end