Class: EventMachine::HttpFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/em/http-fetcher/fetcher.rb

Defined Under Namespace

Classes: RequestPool

Constant Summary collapse

VERSION =
"0.1.1"

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ HttpFetcher

Returns a new instance of HttpFetcher.



91
92
93
94
95
96
97
98
99
100
101
102
# File 'lib/em/http-fetcher/fetcher.rb', line 91

def initialize(opts = {})
  @concurrency       = opts[:concurrency]       || 40
  @host_concurrency  = opts[:host_concurrency]  || 2
  @host_request_wait = opts[:host_request_wait] || 0.2
  @request_pool      = nil
  @default_callbacks = []
  @default_errbacks  = []
  @req_opts = {}.merge(opts)
  @req_opts.delete :concurrency
  @req_opts.delete :host_concurrency
  @req_opts.delete :host_request_wait
end

Instance Method Details

#callback(&block) ⇒ Object



108
109
110
# File 'lib/em/http-fetcher/fetcher.rb', line 108

def callback(&block)
  @default_callbacks << block
end

#errback(&block) ⇒ Object



112
113
114
# File 'lib/em/http-fetcher/fetcher.rb', line 112

def errback(&block)
 @default_errbacks << block
end

#request(*args) ⇒ Object



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# File 'lib/em/http-fetcher/fetcher.rb', line 116

def request(*args)
  if args.first.kind_of? Hash
    opts = args[0]
    uri = opts.delete(:uri)
  else
    uri = args.first
    opts = args[1].kind_of?(Hash) ? args[1] : {}
  end

  uri.kind_of?(Addressable::URI) or
    uri = Addressable::URI.parse(uri.to_s)
  opts = {
    :keepalive => true,
    :redirects => 20,
    :path => uri.path || '/',
  }.merge(opts)
  method = opts.delete(:method) || :get
  uri.query and otps[:query] = uri.query

  df = nil
  request_pool.perform("#{uri.scheme}://#{uri.host}") do |conn|
    df = req = conn.__send__(method, opts)
    @default_callbacks.each do |cb|
      req.callback(&cb)
    end
    @default_errbacks.each do |cb|
      req.errback(&cb)
    end
    req
  end
  df
end

#request_poolObject



104
105
106
# File 'lib/em/http-fetcher/fetcher.rb', line 104

def request_pool
  @request_pool ||= RequestPool.new(@concurrency, @host_concurrency, @host_request_wait, @req_opts)
end