Class: Arachni::HTTP::Client

Inherits:
Object
  • Object
show all
Includes:
Support::Mixins::Observable, UI::Output, Utilities, Singleton
Defined in:
lib/arachni/http/client.rb,
lib/arachni/http/client/dynamic_404_handler.rb

Overview

Provides a system-wide, simple and high-performance HTTP client.

Author:

Defined Under Namespace

Classes: Dynamic404Handler, Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Support::Mixins::Observable

included

Methods included from Utilities

#available_port, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_document, #cookies_from_file, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_document, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_document, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Client

Returns a new instance of Client.



119
120
121
122
 
# File 'lib/arachni/http/client.rb', line 119

def initialize
    super
    reset
end

Instance Attribute Details

#burst_response_count ⇒ Integer (readonly)

Returns Amount of responses received for the running requests (of the current burst).

Returns:

  • (Integer) —

    Amount of responses received for the running requests (of the current burst).



114
115
116
 
# File 'lib/arachni/http/client.rb', line 114

def burst_response_count
  @burst_response_count
end

#burst_response_time_sum ⇒ Integer (readonly)

Returns Sum of the response times for the running requests (of the current burst).

Returns:

  • (Integer) —

    Sum of the response times for the running requests (of the current burst).



110
111
112
 
# File 'lib/arachni/http/client.rb', line 110

def burst_response_time_sum
  @burst_response_time_sum
end

#dynamic_404_handler ⇒ Dynamic404Handler (readonly)

Returns:



117
118
119
 
# File 'lib/arachni/http/client.rb', line 117

def dynamic_404_handler
  @dynamic_404_handler
end

#headers ⇒ Hash (readonly)

Returns Default headers for requests.

Returns:



94
95
96
 
# File 'lib/arachni/http/client.rb', line 94

def headers
  @headers
end

#request_count ⇒ Integer (readonly)

Returns Amount of performed requests.

Returns:

  • (Integer) —

    Amount of performed requests.



98
99
100
 
# File 'lib/arachni/http/client.rb', line 98

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly)

Returns Amount of received responses.

Returns:

  • (Integer) —

    Amount of received responses.



102
103
104
 
# File 'lib/arachni/http/client.rb', line 102

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly)

Returns Amount of timed-out requests.

Returns:

  • (Integer) —

    Amount of timed-out requests.



106
107
108
 
# File 'lib/arachni/http/client.rb', line 106

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly)

Returns Framework target URL, used as reference.

Returns:

  • (String) —

    Framework target URL, used as reference.



90
91
92
 
# File 'lib/arachni/http/client.rb', line 90

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



452
453
454
 
# File 'lib/arachni/http/client.rb', line 452

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#abort ⇒ Object

Aborts the running requests on a best effort basis.



253
254
255
 
# File 'lib/arachni/http/client.rb', line 253

def abort
    exception_jail { client_abort }
end

#after_each_run(&block) ⇒ Arachni::HTTP

Returns self.

Parameters:

  • block (Block) —

    Called after each #run.

Returns:



58
 
# File 'lib/arachni/http/client.rb', line 58

advertise :after_each_run

#after_run(&block) ⇒ Arachni::HTTP::Client

Returns ‘self`.

Parameters:

  • block (Block) —

    Called after the next #run.

Returns:



50
 
# File 'lib/arachni/http/client.rb', line 50

advertise :after_run

#burst_average_response_time ⇒ Float

Returns Average response time for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Average response time for the running requests (i.e. the current burst).



288
289
290
291
 
# File 'lib/arachni/http/client.rb', line 288

def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end

#burst_responses_per_second ⇒ Float

Returns Responses/second for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Responses/second for the running requests (i.e. the current burst).



295
296
297
298
299
300
 
# File 'lib/arachni/http/client.rb', line 295

def burst_responses_per_second
    if @burst_response_count > 0 && burst_runtime > 0
        return @burst_response_count / burst_runtime
    end
    0
end

#burst_runtime ⇒ Float

Returns Amount of time (in seconds) that the current burst has been running.

Returns:

  • (Float) —

    Amount of time (in seconds) that the current burst has been running.



281
282
283
284
 
# File 'lib/arachni/http/client.rb', line 281

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#cookie(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the cookies in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



409
410
411
412
 
# File 'lib/arachni/http/client.rb', line 409

def cookie( url = @url, options = {}, &block )
    options[:cookies] = (options.delete( :parameters ) || {}).dup
    request( url, options, &block )
end

#cookie_jar ⇒ CookieJar

Returns:



185
186
187
 
# File 'lib/arachni/http/client.rb', line 185

def cookie_jar
    State.http.cookie_jar
end

#cookies ⇒ Array<Arachni::Element::Cookie>

Returns All cookies in the jar.

Returns:



316
317
318
 
# File 'lib/arachni/http/client.rb', line 316

def cookies
    cookie_jar.cookies
end

#get(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



377
378
379
 
# File 'lib/arachni/http/client.rb', line 377

def get( url = @url, options = {}, &block )
    request( url, options, &block )
end

#header(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the headers in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



420
421
422
423
424
 
# File 'lib/arachni/http/client.rb', line 420

def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end

#inspect ⇒ Object 



456
457
458
459
460
 
# File 'lib/arachni/http/client.rb', line 456

def inspect
    s = "#<#{self.class} "
    statistics.each { |k, v| s << "@#{k}=#{v.inspect} " }
    s << '>'
end

#max_concurrency ⇒ Integer

Returns Current maximum concurrency of HTTP requests.

Returns:

  • (Integer) —

    Current maximum concurrency of HTTP requests.



310
311
312
 
# File 'lib/arachni/http/client.rb', line 310

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object

Parameters:

  • concurrency (Integer) —

    Sets the maximum concurrency of HTTP requests.



304
305
306
 
# File 'lib/arachni/http/client.rb', line 304

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#on_complete(&block) ⇒ Object 



70
 
# File 'lib/arachni/http/client.rb', line 70

advertise :on_complete

#on_new_cookies(&block) ⇒ Object

Parameters:

  • block (Block) —

    To be passed the new cookies and the response that set them



67
 
# File 'lib/arachni/http/client.rb', line 67

advertise :on_new_cookies

#on_queue(&block) ⇒ Object 



61
 
# File 'lib/arachni/http/client.rb', line 61

advertise :on_queue

#parse_and_set_cookies(response) ⇒ Object

Note:

Runs #on_new_cookies callbacks.

Parameters:

  • response (Response) —

    Extracts cookies from ‘response` and updates the cookie-jar.



445
446
447
448
449
450
 
# File 'lib/arachni/http/client.rb', line 445

def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end

#post(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘POST` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



387
388
389
390
 
# File 'lib/arachni/http/client.rb', line 387

def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end

#queue(request) ⇒ Object

Parameters:

  • request (Request) —

    Request to queue.



428
429
430
431
 
# File 'lib/arachni/http/client.rb', line 428

def queue( request )
    notify_on_queue( request )
    forward_request( request )
end

#request(url = @url, options = {}, &block) ⇒ Request, Response

Queues/performs a generic request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Options Hash (options):

  • :cookies (Hash) — default: {} —

    Extra cookies to use for this request.

  • :no_cookie_jar (Hash) — default: false —

    Do not include cookies from the #cookie_jar.

Returns:



335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
 
# File 'lib/arachni/http/client.rb', line 335

def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options = options.dup
    cookies = options.delete( :cookies ) || {}

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            cookies = begin
                cookie_jar.for_url( url ).inject({}) do |h, c|
                    h[c.name] = c.value
                    h
                end.merge( cookies )
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                cookies
            end
        end

        request = Request.new( options.merge(
            url:     url,
            headers: headers.merge( options.delete( :headers ) || {} ),
            cookies: cookies
        ))

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP

Returns Reset ‘self`.

Returns:



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/arachni/http/client.rb', line 126

def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    @url = Options.url.to_s
    @url = nil if @url.empty?

    client_initialize

    headers.merge!(
        'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent' => Options.http.user_agent
    )
    headers['From'] = Options.authorized_by if Options.authorized_by
    headers.merge!( Options.http.request_headers )

    cookie_jar.load( Options.http.cookie_jar_filepath ) if Options.http.cookie_jar_filepath
    update_cookies( Options.http.cookies )
    update_cookies( Options.http.cookie_string ) if Options.http.cookie_string

    reset_burst_info

    @request_count  = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @dynamic_404_handler = Dynamic404Handler.new

    self
end

#run ⇒ Object

Runs all queued requests



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/arachni/http/client.rb', line 194

def run
    exception_jail false do
        @burst_runtime = nil

        begin
            run_and_update_statistics

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0 || observers_for( :after_run ).any?

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        @dynamic_404_handler.prune

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end

#sandbox(&block) ⇒ Object

Note:

Cookies or new callbacks set as a result of the block won’t affect the HTTP singleton.

Return value of the block.

Parameters:

  • block (Block) —

    Block to executes inside a sandbox.

Returns:

  • (Object) —

    Return value of the block.



226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/arachni/http/client.rb', line 226

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end

#statistics ⇒ Hash

Returns Hash including HTTP client statistics including:

Returns:



176
177
178
179
180
181
182
 
# File 'lib/arachni/http/client.rb', line 176

def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency].inject({}) { |h, k| h[k] = send(k); h }
end

#total_average_response_time ⇒ Float

Returns Average response time for all requests.

Returns:

  • (Float) —

    Average response time for all requests.



266
267
268
269
 
# File 'lib/arachni/http/client.rb', line 266

def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end

#total_responses_per_second ⇒ Float

Returns Responses/second.

Returns:

  • (Float) —

    Responses/second.



272
273
274
275
276
277
 
# File 'lib/arachni/http/client.rb', line 272

def total_responses_per_second
    if @response_count > 0 && total_runtime > 0
        return @response_count / Float( total_runtime )
    end
    0
end

#total_runtime ⇒ Integer

Returns Amount of time (in seconds) that has been devoted to performing requests and getting responses.

Returns:

  • (Integer) —

    Amount of time (in seconds) that has been devoted to performing requests and getting responses.



260
261
262
 
# File 'lib/arachni/http/client.rb', line 260

def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end

#trace(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘TRACE` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



398
399
400
 
# File 'lib/arachni/http/client.rb', line 398

def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies

Parameters:



435
436
437
438
 
# File 'lib/arachni/http/client.rb', line 435

def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end