Class: Mu::Xtractr::Stream::HTTP

Inherits:
Processor show all
Defined in:
lib/mu/xtractr/stream/http.rb,
lib/mu/xtractr/test/stream/tc_http.rb

Overview

:nodoc:

Defined Under Namespace

Classes: Test

Class Method Summary collapse

Methods inherited from Processor

inherited, processors

Class Method Details

.dechunk(text) ⇒ Object 



72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/mu/xtractr/stream/http.rb', line 72

def self.dechunk text
    ios = StringIO.new text
    body = ''
    while true
        line = ios.readline rescue nil
        break if not line or line.empty?
        
        chunksz = line.to_i(16)
        break if chunksz.zero?
        
        body << ios.read(chunksz)
    end
    return body
end

.decompress(text, method) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/mu/xtractr/stream/http.rb', line 87

def self.decompress text, method
    if method == 'gzip'
        ios = StringIO.new text
        reader = Zlib::GzipReader.new ios
        begin
            return reader.read
        ensure
            reader.close
        end
    elsif method == 'deflate'
        return Zlib::Inflate.inflate(text)
    end
end

.extract(stream) ⇒ Object

Pull out the response body from the messages in the stream and convert them into contents on the stream.



29
30
31
 
# File 'lib/mu/xtractr/stream/http.rb', line 29

def self.extract stream
    stream.messages.each { |message| process stream, message }
end

.matches?(stream) ⇒ Boolean

Check to see if this is a HTTP stream.

Returns:

  • (Boolean) 


23
24
25
 
# File 'lib/mu/xtractr/stream/http.rb', line 23

def self.matches? stream
    not stream.messages.select { |message| message.bytes =~ /HTTP\/1\./ }.empty?
end

.process(stream, message) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/mu/xtractr/stream/http.rb', line 33

def self.process stream, message
    content = Content.new message
    chunked = false
    length = nil
    ios = StringIO.new message.bytes
    while true
        line = ios.readline.chomp rescue nil
        break if not line or line.empty?
        
        case line
        when /Content-Length:\s*(\d+)/i
            length = $1.to_i
        when /Content-Type:\s*(.*)/i
            content.type = $1
        when /Content-Encoding:\s*(.*)/i
            content.encoding = $1
        when /Transfer-Encoding:\s*chunked/i
            chunked = true
        end
    end
    
    # Read the content
    bytes = ios.read(length)
    return if not bytes or bytes.empty?
                    
    # Handle chunked encoding, if necessary
    bytes = dechunk(bytes) if chunked
    
    # And then decompress, if necessary
    if ['gzip','deflate'].member? content.encoding
        bytes = decompress(bytes, content.encoding)
    end
    
    if bytes
        content.body = bytes
        stream.contents << content
    end
end