Class: Sunbro::HTTP

Inherits:
Object
  • Object
show all
Defined in:
lib/sunbro/http.rb

Constant Summary collapse

REDIRECT_LIMIT =

Maximum number of redirects to follow on each get_response

5

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ HTTP

Returns a new instance of HTTP.



6
7
8
9
 
# File 'lib/sunbro/http.rb', line 6

def initialize(opts = {})
  @connections = {}
  @opts = opts
end

Instance Method Details

#accept_cookies? ⇒ Boolean

Does this HTTP client accept cookies from the server?

Returns:

  • (Boolean) 


95
96
97
 
# File 'lib/sunbro/http.rb', line 95

def accept_cookies?
  @opts[:accept_cookies]
end

#close ⇒ Object 



11
12
13
14
15
16
17
 
# File 'lib/sunbro/http.rb', line 11

def close
  @connections.each do |host, ports|
    ports.each do |port, connection|
      connection.finish
    end
  end
end

#convert_to_uri(url) ⇒ Object

Convert the link to a valid URI if possible



71
72
73
74
75
 
# File 'lib/sunbro/http.rb', line 71

def convert_to_uri(url)
  URI(url)
rescue URI::InvalidURIError
  URI(URI.escape(url))
end

#fetch_page(url, opts = {}) ⇒ Object

Fetch a single Page from the response of an HTTP request to url. Just gets the final destination page.



23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/sunbro/http.rb', line 23

def fetch_page(url, opts={})
  original_url = url.dup
  pages = fetch_pages(url, opts)
  if pages.count == 1
    page = pages.first
    page.url = original_url
    page
  else
    page = pages.last
    page.redirect_from = original_url
    page
  end
end

#fetch_pages(url, opts = {}) ⇒ Object

Create new Pages from the response of an HTTP request to url, including redirects



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/sunbro/http.rb', line 41

def fetch_pages(url, opts={})
  referer, depth = opts[:referer], opts[:depth]
  force_format = opts[:force_format] || default_page_format
  begin
    url = convert_to_uri(url) unless url.is_a?(URI)
    pages = []
    get(url, referer) do |response, code, location, redirect_to, response_time|
      pages << Page.new(location, :body => response.body.dup,
                                  :code => code,
                                  :headers => response.to_hash,
                                  :referer => referer,
                                  :depth => depth,
                                  :redirect_to => redirect_to,
                                  :response_time => response_time,
                                  :force_format => force_format)
    end

    return pages
  rescue Exception => e
    if verbose?
      puts e.inspect
      puts e.backtrace
    end
    return [Page.new(url, :error => e)]
  end
end

#proxy_host ⇒ Object

The proxy address string



102
103
104
 
# File 'lib/sunbro/http.rb', line 102

def proxy_host
  @opts[:proxy_host]
end

#proxy_port ⇒ Object

The proxy port



109
110
111
 
# File 'lib/sunbro/http.rb', line 109

def proxy_port
  @opts[:proxy_port]
end

#read_timeout ⇒ Object

HTTP read timeout in seconds



116
117
118
 
# File 'lib/sunbro/http.rb', line 116

def read_timeout
  @opts[:read_timeout]
end

#redirect_limit ⇒ Object

The maximum number of redirects to follow



80
81
82
 
# File 'lib/sunbro/http.rb', line 80

def redirect_limit
  @opts[:redirect_limit] || REDIRECT_LIMIT
end

#user_agent ⇒ Object

The user-agent string which will be sent with each request, or nil if no such option is set



88
89
90
 
# File 'lib/sunbro/http.rb', line 88

def user_agent
  @opts[:agent] || Settings.user_agent
end