Class: Arachni::HTTP::Client

Inherits:
Object
  • Object
show all
Includes:
Support::Mixins::Observable, UI::Output, Utilities, Singleton
Defined in:
lib/arachni/http/client.rb

Overview

Provides a system-wide, simple and high-performance HTTP client.

Author:

Defined Under Namespace

Classes: Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000
CUSTOM_404_CACHE_SIZE =

Maximum size of the cache that holds 404 signatures.

50
CUSTOM_404_SIGNATURE_THRESHOLD =

Maximum allowed difference ratio when comparing custom 404 signatures. The fact that we refine the signatures allows us to set this threshold really low and still maintain good accuracy.

0.1

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Support::Mixins::Observable

included

Methods included from Utilities

#available_port, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_document, #cookies_from_file, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_document, #forms_from_response, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_document, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Client

Returns a new instance of Client.



123
124
125
126
 
# File 'lib/arachni/http/client.rb', line 123

def initialize
    super
    reset
end

Instance Attribute Details

#burst_response_count ⇒ Integer (readonly)

Returns Amount of responses received for the running requests (of the current burst).

Returns:

  • (Integer) —

    Amount of responses received for the running requests (of the current burst).



121
122
123
 
# File 'lib/arachni/http/client.rb', line 121

def burst_response_count
  @burst_response_count
end

#burst_response_time_sum ⇒ Integer (readonly)

Returns Sum of the response times for the running requests (of the current burst).

Returns:

  • (Integer) —

    Sum of the response times for the running requests (of the current burst).



117
118
119
 
# File 'lib/arachni/http/client.rb', line 117

def burst_response_time_sum
  @burst_response_time_sum
end

#headers ⇒ Hash (readonly)

Returns Default headers for requests.

Returns:



101
102
103
 
# File 'lib/arachni/http/client.rb', line 101

def headers
  @headers
end

#request_count ⇒ Integer (readonly)

Returns Amount of performed requests.

Returns:

  • (Integer) —

    Amount of performed requests.



105
106
107
 
# File 'lib/arachni/http/client.rb', line 105

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly)

Returns Amount of received responses.

Returns:

  • (Integer) —

    Amount of received responses.



109
110
111
 
# File 'lib/arachni/http/client.rb', line 109

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly)

Returns Amount of timed-out requests.

Returns:

  • (Integer) —

    Amount of timed-out requests.



113
114
115
 
# File 'lib/arachni/http/client.rb', line 113

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly)

Returns Framework target URL, used as reference.

Returns:

  • (String) —

    Framework target URL, used as reference.



97
98
99
 
# File 'lib/arachni/http/client.rb', line 97

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



575
576
577
 
# File 'lib/arachni/http/client.rb', line 575

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#_404_cache ⇒ Object 



580
581
582
 
# File 'lib/arachni/http/client.rb', line 580

def _404_cache
    @_404
end

#abort ⇒ Object

Aborts the running requests on a best effort basis.



258
259
260
 
# File 'lib/arachni/http/client.rb', line 258

def abort
    exception_jail { client_abort }
end

#after_each_run(&block) ⇒ Arachni::HTTP

Returns self.

Parameters:

  • block (Block) —

    Called after each #run.

Returns:



57
 
# File 'lib/arachni/http/client.rb', line 57

advertise :after_each_run

#after_run(&block) ⇒ Arachni::HTTP::Client

Returns ‘self`.

Parameters:

  • block (Block) —

    Called after the next #run.

Returns:



49
 
# File 'lib/arachni/http/client.rb', line 49

advertise :after_run

#burst_average_response_time ⇒ Float

Returns Average response time for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Average response time for the running requests (i.e. the current burst).



293
294
295
296
 
# File 'lib/arachni/http/client.rb', line 293

def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end

#burst_responses_per_second ⇒ Float

Returns Responses/second for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Responses/second for the running requests (i.e. the current burst).



300
301
302
303
304
305
 
# File 'lib/arachni/http/client.rb', line 300

def burst_responses_per_second
    if @burst_response_count > 0 && burst_runtime > 0
        return @burst_response_count / burst_runtime
    end
    0
end

#burst_runtime ⇒ Float

Returns Amount of time (in seconds) that the current burst has been running.

Returns:

  • (Float) —

    Amount of time (in seconds) that the current burst has been running.



286
287
288
289
 
# File 'lib/arachni/http/client.rb', line 286

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#checked_but_not_custom_404?(url) ⇒ Bool

Returns ‘true` if the `url` has been checked for the existence of a custom-404 handler but none was identified, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    ‘true` if the `url` has been checked for the existence of a custom-404 handler but none was identified, `false` otherwise.



561
562
563
 
# File 'lib/arachni/http/client.rb', line 561

def checked_but_not_custom_404?( url )
    @with_regular_404_handler.include?( url_for_custom_404( url ) )
end

#checked_for_custom_404?(url) ⇒ Bool

Returns ‘true` if the `url` has been checked for the existence of a custom-404 handler, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    ‘true` if the `url` has been checked for the existence of a custom-404 handler, `false` otherwise.



551
552
553
 
# File 'lib/arachni/http/client.rb', line 551

def checked_for_custom_404?( url )
    _404_data_for_url( url )[:analyzed]
end

#cookie(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the cookies in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



414
415
416
417
 
# File 'lib/arachni/http/client.rb', line 414

def cookie( url = @url, options = {}, &block )
    options[:cookies] = (options.delete( :parameters ) || {}).dup
    request( url, options, &block )
end

#cookie_jar ⇒ CookieJar

Returns:



190
191
192
 
# File 'lib/arachni/http/client.rb', line 190

def cookie_jar
    State.http.cookie_jar
end

#cookies ⇒ Array<Arachni::Element::Cookie>

Returns All cookies in the jar.

Returns:



321
322
323
 
# File 'lib/arachni/http/client.rb', line 321

def cookies
    cookie_jar.cookies
end

#custom_404?(response, &block) ⇒ Boolean

Parameters:

  • response (Response) —

    Checks whether or not the provided response is a custom 404 page.

  • block (Block) —

    To be passed true or false depending on the result.

Returns:

  • (Boolean) 


461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
 
# File 'lib/arachni/http/client.rb', line 461

def custom_404?( response, &block )
    url = response.url

    if checked_for_custom_404?( url )
        result = is_404?( url, response.body )
        print_debug "#{__method__} [cached]: #{block} #{url} #{result}"
        return block.call( result )
    end

    # If someone else is already checking that resource don't bother
    # duplicating the effort, just let them know that we're waiting on the
    # results too.
    if _404_data_for_url( url )[:in_progress]
        print_debug "#{__method__} [waiting]: #{url} #{block}"
        _404_data_for_url( url )[:waiting] << [url, response.body, block]
        return
    end

    # Call dibs on fingerprinting this url.
    _404_data_for_url( url )[:in_progress] = true

    print_debug "#{__method__} [checking]: #{url} #{block}"

    precision  = 2
    generators = custom_404_probe_generators( url, precision )

    real_404s          = 0
    gathered_responses = 0
    expected_responses = generators.size * precision

    generators.each.with_index do |generator, i|
        _404_signatures_for_url( url )[i] ||= {}

        precision.times do
            get( generator.call,
                 follow_location: true,
                 # This is important, helps us reduce waiting callers.
                 high_priority: true
            ) do |c_res|

                gathered_responses += 1
                real_404s += 1 if c_res.code == 404

                if _404_signatures_for_url( url )[i][:body]
                    _404_signatures_for_url( url )[i][:rdiff] =
                        _404_signatures_for_url( url )[i][:body].
                            refine( c_res.body )

                    next if gathered_responses != expected_responses

                    # If we get real 404s flag that there's no handler.
                    if real_404s == expected_responses
                        @with_regular_404_handler << url_for_custom_404( url )
                    end

                    checked_for_custom_404( url )

                    result = is_404?( url, response.body )
                    print_debug "#{__method__} [checked]: #{block} #{url} #{result}"
                    block.call result

                    # Process other's request too.
                    while (waiting = _404_data_for_url( url )[:waiting].pop)
                        url, body, callback = waiting
                        result = is_404?( url, body )

                        print_debug "#{__method__} [notify]: #{callback} #{url} #{result}"
                        callback.call result
                    end

                    _404_data_for_url( url )[:in_progress] = false
                else
                    _404_signatures_for_url( url )[i][:body] =
                        Support::Signature.new(
                            c_res.body, threshold: CUSTOM_404_SIGNATURE_THRESHOLD
                        )
                end
            end
        end
    end

    nil
end

#get(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



382
383
384
 
# File 'lib/arachni/http/client.rb', line 382

def get( url = @url, options = {}, &block )
    request( url, options, &block )
end

#header(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the headers in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



425
426
427
428
429
 
# File 'lib/arachni/http/client.rb', line 425

def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end

#max_concurrency ⇒ Integer

Returns Current maximum concurrency of HTTP requests.

Returns:

  • (Integer) —

    Current maximum concurrency of HTTP requests.



315
316
317
 
# File 'lib/arachni/http/client.rb', line 315

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object

Parameters:

  • concurrency (Integer) —

    Sets the maximum concurrency of HTTP requests.



309
310
311
 
# File 'lib/arachni/http/client.rb', line 309

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#needs_custom_404_check?(url) ⇒ Bool

Returns ‘true` if the `url` needs to be checked for a #custom_404?, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    ‘true` if the `url` needs to be checked for a #custom_404?, `false` otherwise.



571
572
573
 
# File 'lib/arachni/http/client.rb', line 571

def needs_custom_404_check?( url )
    !checked_for_custom_404?( url ) || !checked_but_not_custom_404?( url )
end

#on_complete(&block) ⇒ Object 



69
 
# File 'lib/arachni/http/client.rb', line 69

advertise :on_complete

#on_new_cookies(&block) ⇒ Object

Parameters:

  • block (Block) —

    To be passed the new cookies and the response that set them



66
 
# File 'lib/arachni/http/client.rb', line 66

advertise :on_new_cookies

#on_queue(&block) ⇒ Object 



60
 
# File 'lib/arachni/http/client.rb', line 60

advertise :on_queue

#parse_and_set_cookies(response) ⇒ Object

Note:

Runs #on_new_cookies callbacks.

Parameters:

  • response (Response) —

    Extracts cookies from ‘response` and updates the cookie-jar.



450
451
452
453
454
455
 
# File 'lib/arachni/http/client.rb', line 450

def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end

#post(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘POST` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



392
393
394
395
 
# File 'lib/arachni/http/client.rb', line 392

def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end

#queue(request) ⇒ Object

Parameters:

  • request (Request) —

    Request to queue.



433
434
435
436
 
# File 'lib/arachni/http/client.rb', line 433

def queue( request )
    notify_on_queue( request )
    forward_request( request )
end

#request(url = @url, options = {}, &block) ⇒ Request, Response

Queues/performs a generic request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Options Hash (options):

  • :cookies (Hash) — default: {} —

    Extra cookies to use for this request.

  • :no_cookie_jar (Hash) — default: false —

    Do not include cookies from the #cookie_jar.

Returns:



340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
 
# File 'lib/arachni/http/client.rb', line 340

def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options = options.dup
    cookies = options.delete( :cookies ) || {}

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            cookies = begin
                cookie_jar.for_url( url ).inject({}) do |h, c|
                    h[c.name] = c.value
                    h
                end.merge( cookies )
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                cookies
            end
        end

        request = Request.new( options.merge(
            url:     url,
            headers: headers.merge( options.delete( :headers ) || {} ),
            cookies: cookies
        ))

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP

Returns Reset ‘self`.

Returns:



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/arachni/http/client.rb', line 130

def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    @url = Options.url.to_s
    @url = nil if @url.empty?

    client_initialize

    headers.merge!(
        'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent' => Options.http.user_agent
    )
    headers['From'] = Options.authorized_by if Options.authorized_by
    headers.merge!( Options.http.request_headers )

    cookie_jar.load( Options.http.cookie_jar_filepath ) if Options.http.cookie_jar_filepath
    update_cookies( Options.http.cookies )
    update_cookies( Options.http.cookie_string ) if Options.http.cookie_string

    reset_burst_info

    @request_count  = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @with_regular_404_handler = Support::LookUp::HashSet.new
    @_404  = Hash.new

    self
end

#run ⇒ Object

Runs all queued requests



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/arachni/http/client.rb', line 199

def run
    exception_jail false do
        @burst_runtime = nil

        begin
            run_and_update_statistics

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0 || observers_for( :after_run ).any?

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        prune_custom_404_cache

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end

#sandbox(&block) ⇒ Object

Note:

Cookies or new callbacks set as a result of the block won’t affect the HTTP singleton.

Return value of the block.

Parameters:

  • block (Block) —

    Block to executes inside a sandbox.

Returns:

  • (Object) —

    Return value of the block.



231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/arachni/http/client.rb', line 231

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end

#statistics ⇒ Hash

Returns Hash including HTTP client statistics including:

Returns:



181
182
183
184
185
186
187
 
# File 'lib/arachni/http/client.rb', line 181

def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency].inject({}) { |h, k| h[k] = send(k); h }
end

#total_average_response_time ⇒ Float

Returns Average response time for all requests.

Returns:

  • (Float) —

    Average response time for all requests.



271
272
273
274
 
# File 'lib/arachni/http/client.rb', line 271

def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end

#total_responses_per_second ⇒ Float

Returns Responses/second.

Returns:

  • (Float) —

    Responses/second.



277
278
279
280
281
282
 
# File 'lib/arachni/http/client.rb', line 277

def total_responses_per_second
    if @response_count > 0 && total_runtime > 0
        return @response_count / Float( total_runtime )
    end
    0
end

#total_runtime ⇒ Integer

Returns Amount of time (in seconds) that has been devoted to performing requests and getting responses.

Returns:

  • (Integer) —

    Amount of time (in seconds) that has been devoted to performing requests and getting responses.



265
266
267
 
# File 'lib/arachni/http/client.rb', line 265

def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end

#trace(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘TRACE` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



403
404
405
 
# File 'lib/arachni/http/client.rb', line 403

def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies

Parameters:



440
441
442
443
 
# File 'lib/arachni/http/client.rb', line 440

def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end

#url_for_custom_404(url) ⇒ Object 



584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
 
# File 'lib/arachni/http/client.rb', line 584

def url_for_custom_404( url )
    parsed = Arachni::URI( url )

    # If we're dealing with a file resource, then its parent directory will
    # be the applicable custom-404 handler...
    if parsed.resource_extension
        trv_back = Arachni::URI( parsed.up_to_path ).path

    # ...however, if we're dealing with a directory, the applicable handler
    # will be its parent directory.
    else
        trv_back = File.dirname( Arachni::URI( parsed.up_to_path ).path )
    end

    trv_back += '/' if trv_back[-1] != '/'

    parsed = parsed.dup
    parsed.path = trv_back
    parsed.to_s
end