Class: NOMS::Command::UserAgent

Inherits:
Base
  • Object
show all
Defined in:
lib/noms/command/useragent.rb,
lib/noms/command/useragent/cache.rb,
lib/noms/command/useragent/response.rb,
lib/noms/command/useragent/requester.rb,
lib/noms/command/useragent/response/typhoeus.rb,
lib/noms/command/useragent/requester/typhoeus.rb,
lib/noms/command/useragent/response/httpclient.rb,
lib/noms/command/useragent/requester/httpclient.rb

Defined Under Namespace

Classes: Cache, Requester, Response

Instance Attribute Summary collapse

Attributes inherited from Base

#logger

Instance Method Summary collapse

Methods inherited from Base

#default_logger

Constructor Details

#initialize(origin, attrs = {}) ⇒ UserAgent 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/noms/command/useragent.rb', line 27

def initialize(origin, attrs={})
    @origin = origin.respond_to?(:scheme) ? origin : URI.parse(origin)
    # httpclient
    # TODO Replace with TOFU implementation
    @log = attrs[:logger] || default_logger
    cookies = attrs.has_key?(:cookies) ? attrs[:cookies] : true

    @client = NOMS::Command::UserAgent::Requester.new :logger => @log, :cookies => cookies

    @redirect_checks = [ ]
    @plaintext_identity = attrs[:plaintext_identity] || nil

    @cache = attrs.has_key?(:cache) ? attrs[:cache] : true
    @max_age = attrs[:max_age] || 3600

    if @cache
        @cacher = NOMS::Command::UserAgent::Cache.new
    end

    @auth = attrs[:auth] || NOMS::Command::Auth.new(:logger => @log,
                                                    :specified_identities =>
                                                    (attrs[:specified_identities] || []))
end

Instance Attribute Details

#auth ⇒ Object

Returns the value of attribute auth.



25
26
27
 
# File 'lib/noms/command/useragent.rb', line 25

def auth
  @auth
end

#cache ⇒ Object

Returns the value of attribute cache.



25
26
27
 
# File 'lib/noms/command/useragent.rb', line 25

def cache
  @cache
end

#cacher ⇒ Object

Returns the value of attribute cacher.



25
26
27
 
# File 'lib/noms/command/useragent.rb', line 25

def cacher
  @cacher
end

Instance Method Details

#absolute_url(url) ⇒ Object 



67
68
69
70
71
72
73
74
75
 
# File 'lib/noms/command/useragent.rb', line 67

def absolute_url(url)
    begin
        url = URI.parse url unless url.respond_to? :scheme
        url = URI.join(@origin, url) unless url.absolute?
        url
    rescue StandardError => e
        raise NOMS::Command::Error.new "Error parsing URL #{url} in context of #{@origin} (#{e.class}): #{e.message}"
    end
end

#add_redirect_check(&block) ⇒ Object 



198
199
200
 
# File 'lib/noms/command/useragent.rb', line 198

def add_redirect_check(&block)
    @redirect_checks << block
end

#check_redirect(url) ⇒ Object 



59
60
61
 
# File 'lib/noms/command/useragent.rb', line 59

def check_redirect(url)
    @redirect_checks.all? { |check| check.call(url) }
end

#clear_cache! ⇒ Object 



51
52
53
 
# File 'lib/noms/command/useragent.rb', line 51

def clear_cache!
    @cacher.clear! unless @cacher.nil?
end

#clear_redirect_checks ⇒ Object 



202
203
204
 
# File 'lib/noms/command/useragent.rb', line 202

def clear_redirect_checks
    @redirect_checks = [ ]
end

#get(url, headers = {}) ⇒ Object 



188
189
190
 
# File 'lib/noms/command/useragent.rb', line 188

def get(url, headers={})
    request('GET', url, nil, headers)
end

#origin=(new_origin) ⇒ Object 



63
64
65
 
# File 'lib/noms/command/useragent.rb', line 63

def origin=(new_origin)
    @origin = new_origin
end

#pop_redirect_check ⇒ Object 



206
207
208
209
210
 
# File 'lib/noms/command/useragent.rb', line 206

def pop_redirect_check
    unless @redirect_checks.empty?
        @redirect_checks.pop
    end
end

#request(method, url, data = nil, headers = {}, tries = 10, identity = nil, cached = nil) ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/noms/command/useragent.rb', line 82

def request(method, url, data=nil, headers={}, tries=10, identity=nil, cached=nil)
    req_url = absolute_url(url)
    @log.debug "#{method} #{req_url}" + (headers.empty? ? '' : headers.inspect)

    # TODO: check Vary
    if @cache and cached.nil? and method.to_s.upcase == 'GET'
        key = request_key('GET', req_url)
        cached_response = NOMS::Command::UserAgent::Response.from_cache(@cacher.get(key), :logger => @log)
        if cached_response and cached_response.is_a? NOMS::Command::UserAgent::Response
            cached_response.logger = @log

            if cached_response.age < @max_age
                if (cached_response.auth_hash.nil? or (identity and identity.auth_verify? cached_response.auth_hash))
                    if cached_response.current?
                        return [cached_response, req_url]
                    else
                        # Maybe we can revalidate it
                        if cached_response.etag
                            headers = { 'If-None-Match' => cached_response.etag }.merge headers
                            return self.request(method, url, data, headers, tries, identity, cached_response)
                        elsif cached_response.last_modified
                            headers = { 'If-Modified-Since' => cached_response.last_modified.httpdate }.merge headers
                            return self.request(method, url, data, headers, tries, identity, cached_response)
                        end
                    end
                end
            end
        end
    end

    begin
        response = @client.request :method => method,
                                   :url => req_url,
                                   :body => data,
                                   :headers => headers
    rescue StandardError => e
        @log.debug e.backtrace.join("\n")
        raise NOMS::Command::Error.new "Couldn't retrieve #{req_url} (#{e.class}): #{e.message}"
    end
    @log.debug "-> #{response.statusText} (#{response.body.size} bytes of #{response.content_type})"
    @log.debug { JSON.pretty_generate(response.header) }

    case response.status
    when 401
        if identity
            # The identity we got was no good, try again
            identity.clear
            if tries > 0
                identity = @auth.load(req_url, response)
                # httpclient
                @client.set_auth(identity['domain'], identity['username'], identity['password'])
                return self.request(method, url, data, headers, tries - 1, identity)
            end
        else
            identity = @auth.load(req_url, response)
            # httpclient
            if identity
                @client.set_auth(identity['domain'], identity['username'], identity['password'])
                return self.request(method, url, data, headers, 2, identity)
            end
        end
        identity = nil
    when 304
        # The cached response has been revalidated
        if cached
            # TODO: Update Date: and Expires:/Cache-Control: headers
            # in cached response and re-cache, while maintaining
            # original_date
            key = request_key(method, req_url)
            @cacher.freshen key
            response, req_url = [cached, req_url]
        else
            raise NOMS::Command::Error.new "Server returned 304 Not Modified for #{new_url}, " +
                "but we were not revalidating a cached copy"
        end
    when 302, 301
        new_url = response.header('Location')
        if check_redirect new_url
            raise NOMS::Command::Error.new "Can't follow redirect to #{new_url}: too many redirects" if tries <= 0
            return self.request(method, new_url, data, headers, tries - 1, identity)
        end
    end

    if identity and response.success?
        @log.debug "Login succeeded, saving #{identity['username']} @ #{identity}"
        if @plaintext_identity
            identity.save :file => @plaintext_identity, :encrypt => false
        else
            identity.save :encrypt => true
        end
    end

    if @cache and method.to_s.upcase == 'GET' and response.cacheable?
        cache_object = response.cacheable_copy
        cache_object.cached!
        if identity
            cache_object.auth_hash = identity.verification_hash
        end
        key = request_key(method, req_url)
        @cacher.set(key, cache_object.to_cache)
    end

    @log.debug "<- #{response.statusText} <- #{req_url}"
    [response, req_url]
end

#request_key(method, url, opt = {}) ⇒ Object

Calculate a key for caching based on the method and URL



78
79
80
 
# File 'lib/noms/command/useragent.rb', line 78

def request_key(method, url, opt={})
    OpenSSL::Digest::SHA1.new([method, url.to_s].join(' ')).hexdigest
end

#wait(on = nil) ⇒ Object

Wait for all asynchronous requests to complete. A stub while these are simulated



194
195
196
 
# File 'lib/noms/command/useragent.rb', line 194

def wait(on=nil)
    []
end