Class: HTTPClient

Inherits:
Object
  • Object
show all
Includes:
Util
Defined in:
lib/httpclient.rb

Overview

DESCRIPTION

HTTPClient -- Client to retrieve web resources via HTTP.

How to create your client.

1. Create simple client.
  clnt = HTTPClient.new

2. Accessing resources through HTTP proxy.
  clnt = HTTPClient.new("http://myproxy:8080")

3. Set User-Agent and From in HTTP request header.(nil means "No proxy")
  clnt = HTTPClient.new(nil, "MyAgent", "[email protected]")

How to retrieve web resources.

1. Get content of specified URL.
  puts clnt.get_content("http://www.ruby-lang.org/en/")

2. Do HEAD request.
  res = clnt.head(uri)

3. Do GET request with query.
  res = clnt.get(uri)

4. Do POST request.
  res = clnt.post(uri)
  res = clnt.get|post|head(uri, proxy)

Direct Known Subclasses

HTTPAccess2::Client

Defined Under Namespace

Modules: DebugSocket, SocketWrap, Util Classes: AuthFilterBase, BasicAuth, Connection, DigestAuth, LoopBackSocket, NegotiateAuth, ProxyAuth, RetryableResponse, SSLConfig, SSLSocketWrap, SSPINegotiateAuth, Session, SessionManager, Site, WWWAuth

Constant Summary collapse

VERSION = 
'2.1.2'
RUBY_VERSION_STRING = 
"ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
SSLEnabled = 
begin
  require 'openssl'
  true
rescue LoadError
  false
end
NTLMEnabled = 
begin
  require 'net/ntlm'
  true
rescue LoadError
  false
end
SSPIEnabled = 
begin
  require 'win32/sspi'
  true
rescue LoadError
  false
end
DEBUG_SSL = 
true

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Util

hash_find_value, parse_challenge_param, uri_dirname, uri_part_of, #urify

Constructor Details

#initialize(proxy = nil, agent_name = nil, from = nil) ⇒ HTTPClient

SYNOPSIS

Client.new(proxy = nil, agent_name = nil, from = nil)

ARGS

proxy             A String of HTTP proxy URL. ex. "http://proxy:8080".
agent_name        A String for "User-Agent" HTTP request header.
from              A String for "From" HTTP request header.

DESCRIPTION

Create an instance.
SSLConfig cannot be re-initialized.  Create new client.



1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
 
# File 'lib/httpclient.rb', line 1730

def initialize(proxy = nil, agent_name = nil, from = nil)
  @proxy = nil        # assigned later.
  @no_proxy = nil
  @agent_name = agent_name
  @from = from
  @www_auth = WWWAuth.new
  @proxy_auth = ProxyAuth.new
  @request_filter = [@proxy_auth, @www_auth]
  @debug_dev = nil
  @redirect_uri_callback = method(:default_redirect_uri_callback)
  @test_loopback_response = []
  @session_manager = SessionManager.new
  @session_manager.agent_name = @agent_name
  @session_manager.from = @from
  @session_manager.ssl_config = @ssl_config = SSLConfig.new(self)
  @cookie_manager = WebAgent::CookieManager.new
  load_environment
  self.proxy = proxy if proxy
end

Instance Attribute Details

#agent_name ⇒ Object (readonly)

Returns the value of attribute agent_name.



1696
1697
1698
 
# File 'lib/httpclient.rb', line 1696

def agent_name
  @agent_name
end

#cookie_manager ⇒ Object

Returns the value of attribute cookie_manager.



1699
1700
1701
 
# File 'lib/httpclient.rb', line 1699

def cookie_manager
  @cookie_manager
end

#from ⇒ Object (readonly)

Returns the value of attribute from.



1697
1698
1699
 
# File 'lib/httpclient.rb', line 1697

def from
  @from
end

#proxy_auth ⇒ Object (readonly)

Returns the value of attribute proxy_auth.



1702
1703
1704
 
# File 'lib/httpclient.rb', line 1702

def proxy_auth
  @proxy_auth
end

#request_filter ⇒ Object (readonly)

Returns the value of attribute request_filter.



1701
1702
1703
 
# File 'lib/httpclient.rb', line 1701

def request_filter
  @request_filter
end

#ssl_config ⇒ Object (readonly)

Returns the value of attribute ssl_config.



1698
1699
1700
 
# File 'lib/httpclient.rb', line 1698

def ssl_config
  @ssl_config
end

#test_loopback_response ⇒ Object (readonly)

Returns the value of attribute test_loopback_response.



1700
1701
1702
 
# File 'lib/httpclient.rb', line 1700

def test_loopback_response
  @test_loopback_response
end

#www_auth ⇒ Object (readonly)

Returns the value of attribute www_auth.



1703
1704
1705
 
# File 'lib/httpclient.rb', line 1703

def www_auth
  @www_auth
end

Instance Method Details

#connect_timeout ⇒ Object 



1769
1770
1771
 
# File 'lib/httpclient.rb', line 1769

def connect_timeout
  @session_manager.connect_timeout
end

#connect_timeout=(connect_timeout) ⇒ Object 



1773
1774
1775
1776
 
# File 'lib/httpclient.rb', line 1773

def connect_timeout=(connect_timeout)
  reset_all
  @session_manager.connect_timeout = connect_timeout
end

#debug_dev ⇒ Object 



1750
1751
1752
 
# File 'lib/httpclient.rb', line 1750

def debug_dev
  @debug_dev
end

#debug_dev=(dev) ⇒ Object 



1754
1755
1756
1757
1758
 
# File 'lib/httpclient.rb', line 1754

def debug_dev=(dev)
  @debug_dev = dev
  reset_all
  @session_manager.debug_dev = dev
end

#default_redirect_uri_callback(uri, res) ⇒ Object 



1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
 
# File 'lib/httpclient.rb', line 1907

def default_redirect_uri_callback(uri, res)
  newuri = URI.parse(res.header['location'][0])
  unless newuri.is_a?(URI::HTTP)
    newuri = uri + newuri
    STDERR.puts(
      "could be a relative URI in location header which is not recommended")
    STDERR.puts(
      "'The field value consists of a single absolute URI' in HTTP spec")
  end
  puts "Redirect to: #{newuri}" if $DEBUG
  newuri
end

#delete(uri, extheader = {}, &block) ⇒ Object 



1936
1937
1938
 
# File 'lib/httpclient.rb', line 1936

def delete(uri, extheader = {}, &block)
  request('DELETE', uri, nil, nil, extheader, &block)
end

#delete_async(uri, extheader = {}) ⇒ Object 



1986
1987
1988
 
# File 'lib/httpclient.rb', line 1986

def delete_async(uri, extheader = {})
  request_async('DELETE', uri, nil, nil, extheader)
end

#get(uri, query = nil, extheader = {}, &block) ⇒ Object 



1924
1925
1926
 
# File 'lib/httpclient.rb', line 1924

def get(uri, query = nil, extheader = {}, &block)
  request('GET', uri, query, nil, extheader, &block)
end

#get_async(uri, query = nil, extheader = {}) ⇒ Object 



1974
1975
1976
 
# File 'lib/httpclient.rb', line 1974

def get_async(uri, query = nil, extheader = {})
  request_async('GET', uri, query, nil, extheader)
end

#get_content(uri, query = nil, extheader = {}, &block) ⇒ Object

SYNOPSIS

Client#get_content(uri, query = nil, extheader = {}, &block = nil)

ARGS

uri       an_URI or a_string of uri to connect.
query     a_hash or an_array of query part.  e.g. { "a" => "b" }.
          Give an array to pass multiple value like
          [["a" => "b"], ["a" => "c"]].
extheader a_hash of extra headers like { "SOAPAction" => "urn:foo" }.
&block    Give a block to get chunked message-body of response like
          get_content(uri) { |chunked_body| ... }
          Size of each chunk may not be the same.

DESCRIPTION

Get a_sring of message-body of response.



1889
1890
1891
1892
1893
 
# File 'lib/httpclient.rb', line 1889

def get_content(uri, query = nil, extheader = {}, &block)
  follow_redirect(uri, query) { |uri, query|
    get(uri, query, extheader, &block)
  }.content
end

#head(uri, query = nil, extheader = {}) ⇒ Object 



1920
1921
1922
 
# File 'lib/httpclient.rb', line 1920

def head(uri, query = nil, extheader = {})
  request('HEAD', uri, query, nil, extheader)
end

#head_async(uri, query = nil, extheader = {}) ⇒ Object

Async interface.



1970
1971
1972
 
# File 'lib/httpclient.rb', line 1970

def head_async(uri, query = nil, extheader = {})
  request_async('HEAD', uri, query, nil, extheader)
end

#no_proxy ⇒ Object 



1819
1820
1821
 
# File 'lib/httpclient.rb', line 1819

def no_proxy
  @no_proxy
end

#no_proxy=(no_proxy) ⇒ Object 



1823
1824
1825
1826
 
# File 'lib/httpclient.rb', line 1823

def no_proxy=(no_proxy)
  @no_proxy = no_proxy
  reset_all
end

#options(uri, extheader = {}, &block) ⇒ Object 



1940
1941
1942
 
# File 'lib/httpclient.rb', line 1940

def options(uri, extheader = {}, &block)
  request('OPTIONS', uri, nil, nil, extheader, &block)
end

#options_async(uri, extheader = {}) ⇒ Object 



1990
1991
1992
 
# File 'lib/httpclient.rb', line 1990

def options_async(uri, extheader = {})
  request_async('OPTIONS', uri, nil, nil, extheader)
end

#post(uri, body = nil, extheader = {}, &block) ⇒ Object 



1928
1929
1930
 
# File 'lib/httpclient.rb', line 1928

def post(uri, body = nil, extheader = {}, &block)
  request('POST', uri, nil, body, extheader, &block)
end

#post_async(uri, body = nil, extheader = {}) ⇒ Object 



1978
1979
1980
 
# File 'lib/httpclient.rb', line 1978

def post_async(uri, body = nil, extheader = {})
  request_async('POST', uri, nil, body, extheader)
end

#post_content(uri, body = nil, extheader = {}, &block) ⇒ Object 



1895
1896
1897
1898
1899
 
# File 'lib/httpclient.rb', line 1895

def post_content(uri, body = nil, extheader = {}, &block)
  follow_redirect(uri, nil) { |uri, query|
    post(uri, body, extheader, &block)
  }.content
end

#protocol_version ⇒ Object 



1760
1761
1762
 
# File 'lib/httpclient.rb', line 1760

def protocol_version
  @session_manager.protocol_version
end

#protocol_version=(protocol_version) ⇒ Object 



1764
1765
1766
1767
 
# File 'lib/httpclient.rb', line 1764

def protocol_version=(protocol_version)
  reset_all
  @session_manager.protocol_version = protocol_version
end

#proxy ⇒ Object 



1796
1797
1798
 
# File 'lib/httpclient.rb', line 1796

def proxy
  @proxy
end

#proxy=(proxy) ⇒ Object 



1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
 
# File 'lib/httpclient.rb', line 1800

def proxy=(proxy)
  if proxy.nil?
    @proxy = nil
    @proxy_auth.reset_challenge
  else
    @proxy = urify(proxy)
    if @proxy.scheme == nil or @proxy.scheme.downcase != 'http' or
        @proxy.host == nil or @proxy.port == nil
      raise ArgumentError.new("unsupported proxy `#{proxy}'")
    end
    @proxy_auth.reset_challenge
    if @proxy.user || @proxy.password
      @proxy_auth.set_auth(@proxy.user, @proxy.password)
    end
  end
  reset_all
  @proxy
end

#put(uri, body = nil, extheader = {}, &block) ⇒ Object 



1932
1933
1934
 
# File 'lib/httpclient.rb', line 1932

def put(uri, body = nil, extheader = {}, &block)
  request('PUT', uri, nil, body, extheader, &block)
end

#put_async(uri, body = nil, extheader = {}) ⇒ Object 



1982
1983
1984
 
# File 'lib/httpclient.rb', line 1982

def put_async(uri, body = nil, extheader = {})
  request_async('PUT', uri, nil, body, extheader)
end

#receive_timeout ⇒ Object 



1787
1788
1789
 
# File 'lib/httpclient.rb', line 1787

def receive_timeout
  @session_manager.receive_timeout
end

#receive_timeout=(receive_timeout) ⇒ Object 



1791
1792
1793
1794
 
# File 'lib/httpclient.rb', line 1791

def receive_timeout=(receive_timeout)
  reset_all
  @session_manager.receive_timeout = receive_timeout
end

#redirect_uri_callback=(redirect_uri_callback) ⇒ Object 



1865
1866
1867
 
# File 'lib/httpclient.rb', line 1865

def redirect_uri_callback=(redirect_uri_callback)
  @redirect_uri_callback = redirect_uri_callback
end

#request(method, uri, query = nil, body = nil, extheader = {}, &block) ⇒ Object 



1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
 
# File 'lib/httpclient.rb', line 1948

def request(method, uri, query = nil, body = nil, extheader = {}, &block)
  uri = urify(uri)
  conn = Connection.new
  res = nil
  retry_count = 5
  while retry_count > 0
    begin
      prepare_request(method, uri, query, body, extheader) do |req, proxy|
        do_get_block(req, proxy, conn, &block)
      end
      res = conn.pop
      break
    rescue RetryableResponse
      res = conn.pop
      retry_count -= 1
    end
  end
  res
end

#request_async(method, uri, query = nil, body = nil, extheader = {}) ⇒ Object 



1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
 
# File 'lib/httpclient.rb', line 1998

def request_async(method, uri, query = nil, body = nil, extheader = {})
  uri = urify(uri)
  conn = Connection.new
  t = Thread.new(conn) { |tconn|
    prepare_request(method, uri, query, body, extheader) do |req, proxy|
      do_get_stream(req, proxy, tconn)
    end
  }
  conn.async_thread = t
  conn
end

#reset(uri) ⇒ Object

Management interface.



2018
2019
2020
2021
 
# File 'lib/httpclient.rb', line 2018

def reset(uri)
  uri = urify(uri)
  @session_manager.reset(uri)
end

#reset_all ⇒ Object 



2023
2024
2025
 
# File 'lib/httpclient.rb', line 2023

def reset_all
  @session_manager.reset_all
end

#save_cookie_store ⇒ Object 



1861
1862
1863
 
# File 'lib/httpclient.rb', line 1861

def save_cookie_store
  @cookie_manager.save_cookies
end

#send_timeout ⇒ Object 



1778
1779
1780
 
# File 'lib/httpclient.rb', line 1778

def send_timeout
  @session_manager.send_timeout
end

#send_timeout=(send_timeout) ⇒ Object 



1782
1783
1784
1785
 
# File 'lib/httpclient.rb', line 1782

def send_timeout=(send_timeout)
  reset_all
  @session_manager.send_timeout = send_timeout
end

#set_auth(uri, user, passwd) ⇒ Object 



1834
1835
1836
1837
1838
 
# File 'lib/httpclient.rb', line 1834

def set_auth(uri, user, passwd)
  uri = urify(uri)
  @www_auth.set_auth(uri, user, passwd)
  reset_all
end

#set_basic_auth(uri, user, passwd) ⇒ Object

for backward compatibility



1841
1842
1843
1844
1845
 
# File 'lib/httpclient.rb', line 1841

def set_basic_auth(uri, user, passwd)
  uri = urify(uri)
  @www_auth.basic_auth.set(uri, user, passwd)
  reset_all
end

#set_cookie_store(filename) ⇒ Object 



1853
1854
1855
1856
1857
1858
1859
 
# File 'lib/httpclient.rb', line 1853

def set_cookie_store(filename)
  if @cookie_manager.cookies_file
    raise RuntimeError.new("overriding cookie file location")
  end
  @cookie_manager.cookies_file = filename
  @cookie_manager.load_cookies if filename
end

#set_proxy_auth(user, passwd) ⇒ Object 



1847
1848
1849
1850
1851
 
# File 'lib/httpclient.rb', line 1847

def set_proxy_auth(user, passwd)
  uri = urify(uri)
  @proxy_auth.set_auth(user, passwd)
  reset_all
end

#socket_sync=(socket_sync) ⇒ Object

if your ruby is older than 2005-09-06, do not set socket_sync = false to avoid an SSL socket blocking bug in openssl/buffering.rb.



1830
1831
1832
 
# File 'lib/httpclient.rb', line 1830

def socket_sync=(socket_sync)
  @session_manager.socket_sync = socket_sync
end

#strict_redirect_uri_callback(uri, res) ⇒ Object 



1901
1902
1903
1904
1905
 
# File 'lib/httpclient.rb', line 1901

def strict_redirect_uri_callback(uri, res)
  newuri = URI.parse(res.header['location'][0])
  puts "Redirect to: #{newuri}" if $DEBUG
  newuri
end

#test_loopback_http_response ⇒ Object 



1869
1870
1871
 
# File 'lib/httpclient.rb', line 1869

def test_loopback_http_response
  @session_manager.test_loopback_http_response
end

#trace(uri, query = nil, body = nil, extheader = {}, &block) ⇒ Object 



1944
1945
1946
 
# File 'lib/httpclient.rb', line 1944

def trace(uri, query = nil, body = nil, extheader = {}, &block)
  request('TRACE', uri, query, body, extheader, &block)
end

#trace_async(uri, query = nil, body = nil, extheader = {}) ⇒ Object 



1994
1995
1996
 
# File 'lib/httpclient.rb', line 1994

def trace_async(uri, query = nil, body = nil, extheader = {})
  request_async('TRACE', uri, query, body, extheader)
end