Class: Arachni::HTTP::Client

Inherits:
Object
  • Object
show all
Includes:
Support::Mixins::Observable, UI::Output, Utilities, Singleton
Defined in:
lib/arachni/http/client.rb,
lib/arachni/http/client/dynamic_404_handler.rb

Overview

Provides a system-wide, simple and high-performance HTTP client.

Author:

Defined Under Namespace

Classes: Dynamic404Handler, Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000
SEED_HEADER_NAME = 
'X-Arachni-Scan-Seed'

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Support::Mixins::Observable

included

Methods included from Utilities

#available_port, available_port_mutex, #bytes_to_kilobytes, #bytes_to_megabytes, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_file, #cookies_from_parser, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_parser, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_parser, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_level_1?, #debug_level_2?, #debug_level_3?, #debug_level_4?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_debug_level_4, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Client 



123
124
125
126
 
# File 'lib/arachni/http/client.rb', line 123

def initialize
    super
    reset
end

Instance Attribute Details

#burst_response_count ⇒ Integer (readonly) 



116
117
118
 
# File 'lib/arachni/http/client.rb', line 116

def burst_response_count
  @burst_response_count
end

#burst_response_time_sum ⇒ Integer (readonly) 



112
113
114
 
# File 'lib/arachni/http/client.rb', line 112

def burst_response_time_sum
  @burst_response_time_sum
end

#dynamic_404_handler ⇒ Dynamic404Handler (readonly) 



119
120
121
 
# File 'lib/arachni/http/client.rb', line 119

def dynamic_404_handler
  @dynamic_404_handler
end

#headers ⇒ Hash (readonly) 



96
97
98
 
# File 'lib/arachni/http/client.rb', line 96

def headers
  @headers
end

#original_max_concurrency ⇒ Object (readonly)

Returns the value of attribute original_max_concurrency.



121
122
123
 
# File 'lib/arachni/http/client.rb', line 121

def original_max_concurrency
  @original_max_concurrency
end

#request_count ⇒ Integer (readonly) 



100
101
102
 
# File 'lib/arachni/http/client.rb', line 100

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly) 



104
105
106
 
# File 'lib/arachni/http/client.rb', line 104

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly) 



108
109
110
 
# File 'lib/arachni/http/client.rb', line 108

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly) 



92
93
94
 
# File 'lib/arachni/http/client.rb', line 92

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



501
502
503
 
# File 'lib/arachni/http/client.rb', line 501

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#abort ⇒ Object

Aborts the running requests on a best effort basis.



278
279
280
 
# File 'lib/arachni/http/client.rb', line 278

def abort
    exception_jail { client_abort }
end

#after_each_run(&block) ⇒ Arachni::HTTP 



58
 
# File 'lib/arachni/http/client.rb', line 58

advertise :after_each_run

#after_run(&block) ⇒ Arachni::HTTP::Client 



50
 
# File 'lib/arachni/http/client.rb', line 50

advertise :after_run

#burst_average_response_time ⇒ Float 



313
314
315
316
 
# File 'lib/arachni/http/client.rb', line 313

def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end

#burst_responses_per_second ⇒ Float 



320
321
322
323
324
325
 
# File 'lib/arachni/http/client.rb', line 320

def burst_responses_per_second
    if @async_burst_response_count > 0 && burst_runtime > 0
        return @async_burst_response_count / burst_runtime
    end
    0
end

#burst_runtime ⇒ Float 



306
307
308
309
 
# File 'lib/arachni/http/client.rb', line 306

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#cookie(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the cookies in `:parameters`.

See Also:



458
459
460
461
 
# File 'lib/arachni/http/client.rb', line 458

def cookie( url = @url, options = {}, &block )
    options[:cookies] = (options.delete( :parameters ) || {}).dup
    request( url, options, &block )
end

#cookie_jar ⇒ CookieJar 



210
211
212
 
# File 'lib/arachni/http/client.rb', line 210

def cookie_jar
    State.http.cookie_jar
end

#cookies ⇒ Array<Arachni::Element::Cookie> 



341
342
343
 
# File 'lib/arachni/http/client.rb', line 341

def cookies
    cookie_jar.cookies
end

#get(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request.

See Also:



426
427
428
 
# File 'lib/arachni/http/client.rb', line 426

def get( url = @url, options = {}, &block )
    request( url, options, &block )
end

#header(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘GET` request sending the headers in `:parameters`.

See Also:



469
470
471
472
473
 
# File 'lib/arachni/http/client.rb', line 469

def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end

#inspect ⇒ Object 



505
506
507
508
509
 
# File 'lib/arachni/http/client.rb', line 505

def inspect
    s = "#<#{self.class} "
    statistics.each { |k, v| s << "@#{k}=#{v.inspect} " }
    s << '>'
end

#max_concurrency ⇒ Integer 



335
336
337
 
# File 'lib/arachni/http/client.rb', line 335

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object 



329
330
331
 
# File 'lib/arachni/http/client.rb', line 329

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#on_complete(&block) ⇒ Object 



70
 
# File 'lib/arachni/http/client.rb', line 70

advertise :on_complete

#on_new_cookies(&block) ⇒ Object 



67
 
# File 'lib/arachni/http/client.rb', line 67

advertise :on_new_cookies

#on_queue(&block) ⇒ Object 



61
 
# File 'lib/arachni/http/client.rb', line 61

advertise :on_queue

#parse_and_set_cookies(response) ⇒ Object

Note:

Runs #on_new_cookies callbacks.



494
495
496
497
498
499
 
# File 'lib/arachni/http/client.rb', line 494

def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end

#post(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘POST` request.

See Also:



436
437
438
439
 
# File 'lib/arachni/http/client.rb', line 436

def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end

#queue(request) ⇒ Object 



477
478
479
480
 
# File 'lib/arachni/http/client.rb', line 477

def queue( request )
    notify_on_queue( request )
    forward_request( request )
end

#request(url = @url, options = {}, &block) ⇒ Request, Response

Queues/performs a generic request.

Options Hash (options):

  • :cookies (Hash) — default: {} —

    Extra cookies to use for this request.

  • :no_cookie_jar (Hash) — default: false —

    Do not include cookies from the #cookie_jar.



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
 
# File 'lib/arachni/http/client.rb', line 360

def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options     = options.dup
    cookies     = options.delete( :cookies )     || {}
    raw_cookies = options.delete( :raw_cookies ) || []
    raw_cookie_names = Set.new( raw_cookies.map(&:name) )

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            raw_cookies |= begin
                cookie_jar.for_url( url ).reject do |c|
                    cookies.include?( c.name ) ||
                        raw_cookie_names.include?( c.name )
                end
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                []
            end
        end

        on_headers    = options.delete(:on_headers)
        on_body       = options.delete(:on_body)
        on_body_line  = options.delete(:on_body_line)
        on_body_lines = options.delete(:on_body_lines)

        request = Request.new( options.merge(
            url:         url,
            headers:     headers.merge( options.delete( :headers ) || {}, false ),
            cookies:     cookies,
            raw_cookies: raw_cookies
        ))

        if on_headers
            request.on_headers( &on_headers )
        end

        if on_body
            request.on_body( &on_body )
        end

        if on_body_line
            request.on_body_line( &on_body_line )
        end

        if on_body_lines
            request.on_body_lines( &on_body_lines )
        end

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/arachni/http/client.rb', line 145

def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    @url = Options.url.to_s
    @url = nil if @url.empty?

    client_initialize

    reset_options

    if Options.http.cookie_jar_filepath
        cookie_jar.load( Options.http.cookie_jar_filepath )
    end

    Options.http.cookies.each do |name, value|
        update_cookies( name => value )
    end

    if Options.http.cookie_string
        update_cookies( Options.http.cookie_string )
    end

    reset_burst_info

    @request_count  = 0
    @async_response_count = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @dynamic_404_handler = Dynamic404Handler.new

    self
end

#reset_options ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/arachni/http/client.rb', line 128

def reset_options
    @original_max_concurrency = Options.http.request_concurrency || MAX_CONCURRENCY
    self.max_concurrency      = @original_max_concurrency

    headers.clear
    headers.merge!(
        'Accept'              => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent'          => Options.http.user_agent,
        'Accept-Language'     => 'en-US,en;q=0.8,he;q=0.6',
        SEED_HEADER_NAME      => Arachni::Utilities.random_seed
    )
    headers['From'] = Options.authorized_by if Options.authorized_by
    headers.merge!( Options.http.request_headers, false )
end

#run ⇒ Object

Runs all queued requests



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/arachni/http/client.rb', line 219

def run
    exception_jail false do
        @burst_runtime = nil

        begin
            run_and_update_statistics

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0 || observers_for( :after_run ).any?

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        @dynamic_404_handler.prune

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end

#sandbox(&block) ⇒ Object

Note:

Cookies or new callbacks set as a result of the block won’t affect the HTTP singleton.

Return value of the block.



251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/arachni/http/client.rb', line 251

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end

#statistics ⇒ Hash 



200
201
202
203
204
205
206
207
 
# File 'lib/arachni/http/client.rb', line 200

def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency, :original_max_concurrency].
       inject({}) { |h, k| h[k] = send(k); h }
end

#total_average_response_time ⇒ Float 



291
292
293
294
 
# File 'lib/arachni/http/client.rb', line 291

def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end

#total_responses_per_second ⇒ Float 



297
298
299
300
301
302
 
# File 'lib/arachni/http/client.rb', line 297

def total_responses_per_second
    if @async_response_count > 0 && total_runtime > 0
        return @async_response_count / Float( total_runtime )
    end
    0
end

#total_runtime ⇒ Integer 



285
286
287
 
# File 'lib/arachni/http/client.rb', line 285

def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end

#trace(url = @url, options = {}, &block) ⇒ Request, Response

Performs a ‘TRACE` request.

See Also:



447
448
449
 
# File 'lib/arachni/http/client.rb', line 447

def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies 



484
485
486
487
 
# File 'lib/arachni/http/client.rb', line 484

def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end