Class: Polipus::HTTP

Inherits:
Object
  • Object
show all
Defined in:
lib/polipus/http.rb

Constant Summary collapse

REDIRECT_LIMIT =

Maximum number of redirects to follow on each get_response

5

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ HTTP

Returns a new instance of HTTP.



11
12
13
14
 
# File 'lib/polipus/http.rb', line 11

def initialize(opts = {})
  @connections = {}
  @opts = opts
end

Instance Method Details

#accept_cookies? ⇒ Boolean

Does this HTTP client accept cookies from the server?

Returns:

  • (Boolean) 


96
97
98
 
# File 'lib/polipus/http.rb', line 96

def accept_cookies?
  @opts[:accept_cookies]
end

#cookie_jar ⇒ Object 



100
101
102
103
 
# File 'lib/polipus/http.rb', line 100

def cookie_jar
  @opts[:cookie_jar] ||= ::HTTP::CookieJar.new
  @opts[:cookie_jar]
end

#fetch_page(url, referer = nil, depth = nil) ⇒ Object

Fetch a single Page from the response of an HTTP request to url. Just gets the final destination page.



20
21
22
 
# File 'lib/polipus/http.rb', line 20

def fetch_page(url, referer = nil, depth = nil)
  fetch_pages(url, referer, depth).last
end

#fetch_pages(url, referer = nil, depth = nil) ⇒ Object

Create new Pages from the response of an HTTP request to url, including redirects



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/polipus/http.rb', line 28

def fetch_pages(url, referer = nil, depth = nil)
  begin
    url = URI(url) unless url.is_a?(URI)
    pages = []
    get(url, referer) do |response, code, location, redirect_to, response_time|
      body = response.body.dup
      if response.to_hash.fetch('content-encoding', [])[0] == 'gzip'
        gzip = Zlib::GzipReader.new(StringIO.new(body))    
        body = gzip.read
      end
      pages << Page.new(location, :body => response.body.dup,
                                  :code => code,
                                  :headers => response.to_hash,
                                  :referer => referer,
                                  :depth => depth,
                                  :redirect_to => redirect_to,
                                  :response_time => response_time)
    end

    return pages
  rescue StandardError => e
    if verbose?
      puts e.inspect
      puts e.backtrace
    end
    return [Page.new(url, :error => e)]
  end
end

#proxy_host ⇒ Object

The proxy address string



76
77
78
 
# File 'lib/polipus/http.rb', line 76

def proxy_host
  @opts[:proxy_host]
end

#proxy_port ⇒ Object

The proxy port



83
84
85
 
# File 'lib/polipus/http.rb', line 83

def proxy_port
  @opts[:proxy_port]
end

#read_timeout ⇒ Object

HTTP read timeout in seconds



90
91
92
 
# File 'lib/polipus/http.rb', line 90

def read_timeout
  @opts[:read_timeout]
end

#redirect_limit ⇒ Object

The maximum number of redirects to follow



60
61
62
 
# File 'lib/polipus/http.rb', line 60

def redirect_limit
  @opts[:redirect_limit] || REDIRECT_LIMIT
end

#user_agent ⇒ Object

The user-agent string which will be sent with each request, or nil if no such option is set



68
69
70
 
# File 'lib/polipus/http.rb', line 68

def user_agent
  @opts[:user_agent]
end