Class: EventMachine::HttpClient

Inherits:
Connection
  • Object
show all
Includes:
Deferrable, HttpEncoding
Defined in:
lib/em-http/client.rb

Direct Known Subclasses

MockHttpRequest::FakeHttpClient

Constant Summary collapse

TRANSFER_ENCODING = 
"TRANSFER_ENCODING"
CONTENT_ENCODING = 
"CONTENT_ENCODING"
CONTENT_LENGTH = 
"CONTENT_LENGTH"
LAST_MODIFIED = 
"LAST_MODIFIED"
KEEP_ALIVE = 
"CONNECTION"
"SET_COOKIE"
LOCATION = 
"LOCATION"
HOST = 
"HOST"
ETAG = 
"ETAG"
CRLF = 
"\r\n"

Constants included from HttpEncoding

EventMachine::HttpEncoding::FIELD_ENCODING, EventMachine::HttpEncoding::HTTP_REQUEST_HEADER

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from HttpEncoding

#bytesize, #encode_auth, #encode_cookie, #encode_field, #encode_headers, #encode_host, #encode_param, #encode_query, #encode_request, #escape, #munge_header_keys, #unescape

Instance Attribute Details

#error ⇒ Object (readonly)

Returns the value of attribute error.



211
212
213
 
# File 'lib/em-http/client.rb', line 211

def error
  @error
end

#last_effective_url ⇒ Object (readonly)

Returns the value of attribute last_effective_url.



211
212
213
 
# File 'lib/em-http/client.rb', line 211

def last_effective_url
  @last_effective_url
end

#method ⇒ Object

Returns the value of attribute method.



210
211
212
 
# File 'lib/em-http/client.rb', line 210

def method
  @method
end

#options ⇒ Object

Returns the value of attribute options.



210
211
212
 
# File 'lib/em-http/client.rb', line 210

def options
  @options
end

#redirects ⇒ Object (readonly)

Returns the value of attribute redirects.



211
212
213
 
# File 'lib/em-http/client.rb', line 211

def redirects
  @redirects
end

#response ⇒ Object (readonly)

Returns the value of attribute response.



211
212
213
 
# File 'lib/em-http/client.rb', line 211

def response
  @response
end

#response_header ⇒ Object (readonly)

Returns the value of attribute response_header.



211
212
213
 
# File 'lib/em-http/client.rb', line 211

def response_header
  @response_header
end

#uri ⇒ Object

Returns the value of attribute uri.



210
211
212
 
# File 'lib/em-http/client.rb', line 210

def uri
  @uri
end

Instance Method Details

#connect_proxy? ⇒ Boolean

determines if a http-proxy should be used with the CONNECT verb

Returns:

  • (Boolean) 


334
 
# File 'lib/em-http/client.rb', line 334

def connect_proxy?; http_proxy? && (@options[:proxy][:use_connect] == true); end

#connection_completed ⇒ Object

start HTTP request once we establish connection to host



231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/em-http/client.rb', line 231

def connection_completed
  # if a socks proxy is specified, then a connection request
  # has to be made to the socks server and we need to wait
  # for a response code
  if socks_proxy? and @state == :response_header
    @state = :connect_socks_proxy
    send_socks_handshake

  # if we need to negotiate the proxy connection first, then
  # issue a CONNECT query and wait for 200 response
  elsif connect_proxy? and @state == :response_header
    @state = :connect_http_proxy
    send_request_header

    # if connecting via proxy, then state will be :proxy_connected,
    # indicating successful tunnel. from here, initiate normal http
    # exchange

  else
    @state = :response_header
    ssl = @options[:tls] || @options[:ssl] || {}
    start_tls(ssl) if @uri.scheme == "https" or @uri.port == 443
    send_request_header
    send_request_body
  end
end

#disconnect(&blk) ⇒ Object

assign disconnect callback for websocket



285
286
287
 
# File 'lib/em-http/client.rb', line 285

def disconnect(&blk)
  @disconnect = blk
end

#dispatch ⇒ Object

Response processing



488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
 
# File 'lib/em-http/client.rb', line 488

def dispatch
  while case @state
      when :connect_socks_proxy
        parse_socks_response
      when :connect_http_proxy
        parse_response_header
      when :response_header
        parse_response_header
      when :chunk_header
        parse_chunk_header
      when :chunk_body
        process_chunk_body
      when :chunk_footer
        process_chunk_footer
      when :response_footer
        process_response_footer
      when :body
        process_body
      when :websocket
        process_websocket
      when :finished, :invalid
        break
      else raise RuntimeError, "invalid state: #{@state}"
    end
  end
end

#has_bytes?(num) ⇒ Boolean

determines if there is enough data in the buffer

Returns:

  • (Boolean) 


319
320
321
 
# File 'lib/em-http/client.rb', line 319

def has_bytes?(num)
  @data.size >= num
end

#headers(&blk) ⇒ Object

assign a headers parse callback



290
291
292
 
# File 'lib/em-http/client.rb', line 290

def headers(&blk)
  @headers = blk
end

#http_proxy? ⇒ Boolean

determines if a proxy should be used that uses http-headers as proxy-mechanism

this is the default proxy type if none is specified

Returns:

  • (Boolean) 


330
 
# File 'lib/em-http/client.rb', line 330

def http_proxy?; proxy? && [nil, :http].include?(@options[:proxy][:type]); end

#normalize_body ⇒ Object 



308
309
310
311
312
313
314
315
316
 
# File 'lib/em-http/client.rb', line 308

def normalize_body
  @normalized_body ||= begin
    if @options[:body].is_a? Hash
      @options[:body].to_params
    else
      @options[:body]
    end
  end
end

#on_body_data(data) ⇒ Object

Called when part of the body has been read



434
435
436
437
438
439
440
441
442
443
444
 
# File 'lib/em-http/client.rb', line 434

def on_body_data(data)
  if @content_decoder
    begin
      @content_decoder << data
    rescue HttpDecoders::DecoderError
      on_error "Content-decoder error"
    end
  else
    on_decoded_body_data(data)
  end
end

#on_decoded_body_data(data) ⇒ Object 



446
447
448
449
450
451
452
 
# File 'lib/em-http/client.rb', line 446

def on_decoded_body_data(data)
  if @stream
    @stream.call(data)
  else
    @response << data
  end
end

#on_error(msg, dns_error = false) ⇒ Object Also known as: close

request failed, invoke errback



270
271
272
273
274
275
276
 
# File 'lib/em-http/client.rb', line 270

def on_error(msg, dns_error = false)
  @error = msg

  # no connection signature on DNS failures
  # fail the connection directly
  dns_error == true ? fail(self) : unbind
end

#on_request_complete ⇒ Object

request is done, invoke the callback



259
260
261
262
263
264
265
266
267
 
# File 'lib/em-http/client.rb', line 259

def on_request_complete
  begin
    @content_decoder.finalize! if @content_decoder
  rescue HttpDecoders::DecoderError
    on_error "Content-decoder error"
  end

  close_connection
end

#parse_chunk_header ⇒ Object 



730
731
732
733
734
735
736
737
738
 
# File 'lib/em-http/client.rb', line 730

def parse_chunk_header
  return false unless parse_header(@chunk_header)

  @bytes_remaining = @chunk_header.chunk_size
  @chunk_header = HttpChunkHeader.new

  @state = @bytes_remaining > 0 ? :chunk_body : :response_footer
  true
end

#parse_header(header) ⇒ Object 



515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
 
# File 'lib/em-http/client.rb', line 515

def parse_header(header)
  return false if @data.empty?

  begin
    @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
  rescue EventMachine::HttpClientParserError
    @state = :invalid
    on_error "invalid HTTP format, parsing fails"
  end

  return false unless @parser.finished?

  # Clear parsed data from the buffer
  @data.read(@parser_nbytes)
  @parser.reset
  @parser_nbytes = 0

  true
end

#parse_response_header ⇒ Object 



535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
 
# File 'lib/em-http/client.rb', line 535

def parse_response_header
  return false unless parse_header(@response_header)

  # invoke headers callback after full parse if one
  # is specified by the user
  @headers.call(@response_header) if @headers

  unless @response_header.http_status and @response_header.http_reason
    @state = :invalid
    on_error "no HTTP response"
    return false
  end

  if @state == :connect_http_proxy
    # when a successfull tunnel is established, the proxy responds with a
    # 200 response code. from here, the tunnel is transparent.
    if @response_header.http_status.to_i == 200
      @response_header = HttpResponseHeader.new
      connection_completed
      return true
    else
      @state = :invalid
      on_error "proxy not accessible"
      return false
    end
  end

  # correct location header - some servers will incorrectly give a relative URI
  if @response_header.location
    begin
      location = Addressable::URI.parse(@response_header.location)

      if location.relative?
        location = @uri.join(location)
        @response_header[LOCATION] = location.to_s
      else
        # if redirect is to an absolute url, check for correct URI structure
        raise if location.host.nil?
      end

      # store last url on any sign of redirect
      @last_effective_url = location

    rescue
      on_error "Location header format error"
      return false
    end
  end

  # Fire callbacks immediately after recieving header requests
  # if the request method is HEAD. In case of a redirect, terminate
  # current connection and reinitialize the process.
  if @method == "HEAD"
    @state = :finished
    close_connection
    return false
  end

  if websocket?
    if @response_header.status == 101
      @state = :websocket
      succeed
    else
      fail "websocket handshake failed"
    end

  elsif @response_header.chunked_encoding?
    @state = :chunk_header
  elsif @response_header.content_length
    @state = :body
    @bytes_remaining = @response_header.content_length
  else
    @state = :body
    @bytes_remaining = nil
  end

  if decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
    begin
      @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
    rescue HttpDecoders::DecoderError
      on_error "Content-decoder error"
    end
  end

  true
end

#parse_socks_response ⇒ Object

parses socks 5 server responses as specified on www.faqs.org/rfcs/rfc1928.html



639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
 
# File 'lib/em-http/client.rb', line 639

def parse_socks_response
  if @socks_state == :method_negotiation
    return false unless has_bytes? 2

    _, method = @data.read(2).unpack('CC')

    if socks_methods.include?(method)
      if method == 0
        @socks_state = :connecting

        return send_socks_connect_request

      elsif method == 2
        @socks_state = :authenticating

        credentials = @options[:proxy][:authorization]
        if credentials.size < 2
          @state = :invalid
          on_error "username and password are not supplied"
          return false
        end

        username, password = credentials

        send_data [5, username.length, username, password.length, password].pack('CCA*CA*')
      end

    else
      @state = :invalid
      on_error "proxy did not accept method"
      return false
    end

  elsif @socks_state == :authenticating
    return false unless has_bytes? 2

    _, status_code = @data.read(2).unpack('CC')

    if status_code == 0
      # success
      @socks_state = :connecting

      return send_socks_connect_request

    else
      # error
      @state = :invalid
      on_error "access denied by proxy"
      return false
    end

  elsif @socks_state == :connecting
    return false unless has_bytes? 10

    _, response_code, _, address_type, _, _ = @data.read(10).unpack('CCCCNn')

    if response_code == 0
      # success
      @socks_state = :connected
      @state = :proxy_connected

      @response_header = HttpResponseHeader.new

      # connection_completed will invoke actions to
      # start sending all http data transparently
      # over the socks connection
      connection_completed

    else
      # error
      @state = :invalid

      error_messages = {
        1 => "general socks server failure",
        2 => "connection not allowed by ruleset",
        3 => "network unreachable",
        4 => "host unreachable",
        5 => "connection refused",
        6 => "TTL expired",
        7 => "command not supported",
        8 => "address type not supported"
      }
      error_message = error_messages[response_code] || "unknown error (code: #{response_code})"
      on_error "socks5 connect error: #{error_message}"
      return false
    end
  end

  true
end

#post_init ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/em-http/client.rb', line 213

def post_init
  @parser = HttpClientParser.new
  @data = EventMachine::Buffer.new
  @chunk_header = HttpChunkHeader.new
  @response_header = HttpResponseHeader.new
  @parser_nbytes = 0
  @redirects = 0
  @response = ''
  @error = ''
  @last_effective_url = nil
  @content_decoder = nil
  @stream = nil
  @disconnect = nil
  @state = :response_header
  @socks_state = nil
end

#process_body ⇒ Object 



786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
 
# File 'lib/em-http/client.rb', line 786

def process_body
  if @bytes_remaining.nil?
    on_body_data @data.read
    return false
  end

  if @bytes_remaining.zero?
    @state = :finished
    on_request_complete
    return false
  end

  if @data.size < @bytes_remaining
    @bytes_remaining -= @data.size
    on_body_data @data.read
    return false
  end

  on_body_data @data.read(@bytes_remaining)
  @bytes_remaining = 0

  # If Keep-Alive is enabled, the server may be pushing more data to us
  # after the first request is complete. Hence, finish first request, and
  # reset state.
  if @response_header.keep_alive?
    @data.clear # hard reset, TODO: add support for keep-alive connections!
    @state = :finished
    on_request_complete

  else
    if @data.empty?
      @state = :finished
      on_request_complete
    else
      @state = :invalid
      on_error "garbage at end of body"
    end
  end

  false
end

#process_chunk_body ⇒ Object 



740
741
742
743
744
745
746
747
748
749
750
751
752
 
# File 'lib/em-http/client.rb', line 740

def process_chunk_body
  if @data.size < @bytes_remaining
    @bytes_remaining -= @data.size
    on_body_data @data.read
    return false
  end

  on_body_data @data.read(@bytes_remaining)
  @bytes_remaining = 0

  @state = :chunk_footer
  true
end

#process_chunk_footer ⇒ Object 



754
755
756
757
758
759
760
761
762
763
764
765
 
# File 'lib/em-http/client.rb', line 754

def process_chunk_footer
  return false if @data.size < 2

  if @data.read(2) == CRLF
    @state = :chunk_header
  else
    @state = :invalid
    on_error "non-CRLF chunk footer"
  end

  true
end

#process_response_footer ⇒ Object 



767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
 
# File 'lib/em-http/client.rb', line 767

def process_response_footer
  return false if @data.size < 2

  if @data.read(2) == CRLF
    if @data.empty?
      @state = :finished
      on_request_complete
    else
      @state = :invalid
      on_error "garbage at end of chunked response"
    end
  else
    @state = :invalid
    on_error "non-CRLF response footer"
  end

  false
end

#process_websocket ⇒ Object 



828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
 
# File 'lib/em-http/client.rb', line 828

def process_websocket
  return false if @data.empty?

  # slice the message out of the buffer and pass in
  # for processing, and buffer data otherwise
  buffer = @data.read
  while msg = buffer.slice!(/\000([^\377]*)\377/n)
    msg.gsub!(/\A\x00|\xff\z/n, '')
    @stream.call(msg)
  end

  # store remainder if message boundary has not yet
  # been received
  @data << buffer if not buffer.empty?

  false
end

#proxy? ⇒ Boolean

Returns:

  • (Boolean) 


324
 
# File 'lib/em-http/client.rb', line 324

def proxy?; !@options[:proxy].nil?; end

#receive_data(data) ⇒ Object 



428
429
430
431
 
# File 'lib/em-http/client.rb', line 428

def receive_data(data)
  @data << data
  dispatch
end

#send(data) ⇒ Object

raw data push from the client (WebSocket) should only be invoked after handshake, otherwise it will inject data into the header exchange

frames need to start with 0x00-0x7f byte and end with an 0xFF byte. Per spec, we can also set the first byte to a value betweent 0x80 and 0xFF, followed by a leading length indicator



302
303
304
305
306
 
# File 'lib/em-http/client.rb', line 302

def send(data)
  if @state == :websocket
    send_data("\x00#{data}\xff")
  end
end

#send_request_body ⇒ Object 



418
419
420
421
422
423
424
425
426
 
# File 'lib/em-http/client.rb', line 418

def send_request_body
  if @options[:body]
    body = normalize_body
    send_data body
    return
  elsif @options[:file]
    stream_file_data @options[:file], :http_chunks => false
  end
end

#send_request_header ⇒ Object 



357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
 
# File 'lib/em-http/client.rb', line 357

def send_request_header
  query   = @options[:query]
  head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
  file    = @options[:file]
  proxy   = @options[:proxy]
  body    = normalize_body

  request_header = nil

  if http_proxy?
    # initialize headers for the http proxy
    head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
    head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]

    # if we need to negotiate the tunnel connection first, then
    # issue a CONNECT query to the proxy first. This is an optional
    # flag, by default we will provide full URIs to the proxy
    if @state == :connect_http_proxy
      request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
    end
  end

  if websocket?
    head['upgrade'] = 'WebSocket'
    head['connection'] = 'Upgrade'
    head['origin'] = @options[:origin] || @uri.host

  else
    # Set the Content-Length if file is given
    head['content-length'] = File.size(file) if file

    # Set the Content-Length if body is given
    head['content-length'] =  body.bytesize if body

    # Set the cookie header if provided
    if cookie = head.delete('cookie')
      head['cookie'] = encode_cookie(cookie)
    end

    # Set content-type header if missing and body is a Ruby hash
    if not head['content-type'] and options[:body].is_a? Hash
      head['content-type'] = "application/x-www-form-urlencoded"
    end
  end

  # Set the Host header if it hasn't been specified already
  head['host'] ||= encode_host

  # Set the User-Agent if it hasn't been specified
  head['user-agent'] ||= "EventMachine HttpClient"

  # Record last seen URL
  @last_effective_url = @uri

  # Build the request headers
  request_header ||= encode_request(@method, @uri, query, proxy)
  request_header << encode_headers(head)
  request_header << CRLF
  send_data request_header
end

#send_socks_connect_request ⇒ Object 



622
623
624
625
626
627
628
629
630
631
632
633
634
635
 
# File 'lib/em-http/client.rb', line 622

def send_socks_connect_request
  # TO-DO: Implement address types for IPv6 and Domain
  begin
    ip_address = Socket.gethostbyname(@uri.host).last
    send_data [5, 1, 0, 1, ip_address, @uri.port].flatten.pack('CCCCA4n')

  rescue
    @state = :invalid
    on_error "could not resolve host", true
    return false
  end

  true
end

#send_socks_handshake ⇒ Object 



347
348
349
350
351
352
353
354
355
 
# File 'lib/em-http/client.rb', line 347

def send_socks_handshake
  # Method Negotiation as described on
  # http://www.faqs.org/rfcs/rfc1928.html Section 3

  @socks_state = :method_negotiation

  methods = socks_methods
  send_data [5, methods.size].pack('CC') + methods.pack('C*')
end

#socks_methods ⇒ Object 



339
340
341
342
343
344
345
 
# File 'lib/em-http/client.rb', line 339

def socks_methods
  methods = []
  methods << 2 if !options[:proxy][:authorization].nil? # 2 => Username/Password Authentication
  methods << 0 # 0 => No Authentication Required

  methods
end

#socks_proxy? ⇒ Boolean

determines if a SOCKS5 proxy should be used

Returns:

  • (Boolean) 


337
 
# File 'lib/em-http/client.rb', line 337

def socks_proxy?; proxy? && (@options[:proxy][:type] == :socks); end

#stream(&blk) ⇒ Object

assign a stream processing block



280
281
282
 
# File 'lib/em-http/client.rb', line 280

def stream(&blk)
  @stream = blk
end

#unbind ⇒ Object 



454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
 
# File 'lib/em-http/client.rb', line 454

def unbind
  if (@state == :finished) && (@last_effective_url != @uri) && (@redirects < @options[:redirects])
    begin
      # update uri to redirect location if we're allowed to traverse deeper
      @uri = @last_effective_url

      # keep track of the depth of requests we made in this session
      @redirects += 1

      # swap current connection and reassign current handler
      req = HttpOptions.new(@method, @uri, @options)
      reconnect(req.host, req.port)

      @response_header = HttpResponseHeader.new
      @state = :response_header
      @response = ''
      @data.clear
    rescue EventMachine::ConnectionError => e
      on_error(e.message, true)
    end
  else
    if @state == :finished || (@state == :body && @bytes_remaining.nil?)
      succeed(self)
    else
      @disconnect.call(self) if @state == :websocket and @disconnect
      fail(self)
    end
  end
end

#websocket? ⇒ Boolean

Returns:

  • (Boolean) 


323
 
# File 'lib/em-http/client.rb', line 323

def websocket?; @uri.scheme == 'ws'; end