Class: Arachni::Framework

Inherits:
Object show all
Includes:
Mixins::Observable, Module::Utilities, UI::Output
Defined in:
lib/arachni/framework.rb

Overview

The Framework class ties together all the components.

It should be wrapped by a UI class.

It’s the brains of the operation, it bosses the rest of the classes around.

It runs the audit, loads modules and reports and runs them according to user options.

@author: Tasos “Zapotek” Laskos

<[email protected]>
<[email protected]>

@version: 0.2.5

Direct Known Subclasses

RPC::Server::Framework

Constant Summary collapse

REVISION =

the version of this class

'0.2.5'

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Mixins::Observable

#method_missing

Methods included from Module::Utilities

#exception_jail, #get_path, #hash_keys_to_str, #normalize_url, #read_file, #seed, #uri_decode, #uri_encode, #uri_parse, #uri_parser, #url_sanitize

Methods included from UI::Output

#buffer, #debug!, #debug?, #flush_buffer, #mute!, #muted?, #only_positives!, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_pp, #print_error, #print_error_backtrace, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, #uncap_buffer!, #unmute!, #verbose!, #verbose?

Constructor Details

#initialize(opts) ⇒ Framework

Initializes system components.

Parameters:



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/arachni/framework.rb', line 141

def initialize( opts )

    Encoding.default_external = "BINARY"
    Encoding.default_internal = "BINARY"

    @opts = opts

    @modules = Arachni::Module::Manager.new( @opts )
    @reports = Arachni::Report::Manager.new( @opts )
    @plugins = Arachni::Plugin::Manager.new( self )

    # will store full-fledged pages generated by the Trainer since these
    # may not be be accessible simply by their URL
    # @page_queue = ::Arachni::Database::Queue.new
    @page_queue = Queue.new
    @page_queue_total_size = 0

    # will hold paths found by the spider in order to be converted to pages
    # and ultimately audited by the modules
    @url_queue = Queue.new
    @url_queue_total_size = 0

    prepare_cookie_jar( )
    prepare_user_agent( )

    # deep clone the redundancy rules to preserve their counter
    # for the reports
    @orig_redundant = @opts.redundant.deep_clone

    @running = false
    @paused  = []

    @plugin_store = {}
    @store = nil

    @auditmap = []
    @sitemap  = []

    @current_url = ''
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method in the class Arachni::Mixins::Observable

Instance Attribute Details

#auditmap ⇒ Array (readonly)

Array of URLs that have been audited

Returns:



105
106
107
 
# File 'lib/arachni/framework.rb', line 105

def auditmap
  @auditmap
end

#modules ⇒ Arachni::Module::Manager (readonly)

Returns module manager.

Returns:



81
82
83
 
# File 'lib/arachni/framework.rb', line 81

def modules
  @modules
end

#opts ⇒ Options (readonly)

Instance options

Returns:



71
72
73
 
# File 'lib/arachni/framework.rb', line 71

def opts
  @opts
end

#page_queue_size ⇒ Integer (readonly)

Current amount of pages in the audit queue

Returns:

  • (Integer) 


119
120
121
 
# File 'lib/arachni/framework.rb', line 119

def page_queue_size
  @page_queue_size
end

#page_queue_total_size ⇒ Integer (readonly)

Total number of pages added to their audit queue

Returns:

  • (Integer) 


112
113
114
 
# File 'lib/arachni/framework.rb', line 112

def page_queue_total_size
  @page_queue_total_size
end

#plugins ⇒ Arachni::Plugin::Manager (readonly)

Returns plugin manager.

Returns:



86
87
88
 
# File 'lib/arachni/framework.rb', line 86

def plugins
  @plugins
end

#reports ⇒ Arachni::Report::Manager (readonly)

Returns report manager.

Returns:



76
77
78
 
# File 'lib/arachni/framework.rb', line 76

def reports
  @reports
end

#sitemap ⇒ Array (readonly)

URLs of all discovered pages

Returns:



98
99
100
 
# File 'lib/arachni/framework.rb', line 98

def sitemap
  @sitemap
end

#spider ⇒ Arachni::Spider (readonly)

Returns spider.

Returns:



91
92
93
 
# File 'lib/arachni/framework.rb', line 91

def spider
  @spider
end

#url_queue_size ⇒ Integer (readonly)

Current amount of urls in the audit queue

Returns:

  • (Integer) 


133
134
135
 
# File 'lib/arachni/framework.rb', line 133

def url_queue_size
  @url_queue_size
end

#url_queue_total_size ⇒ Integer (readonly)

Total number of urls added to their audit queue

Returns:

  • (Integer) 


126
127
128
 
# File 'lib/arachni/framework.rb', line 126

def url_queue_total_size
  @url_queue_total_size
end

Instance Method Details

#audit ⇒ Object

Performs the audit

Runs the spider, pushes each page or url to their respective audit queue, calls #audit_queue, runs the timeout attacks (Module::Auditor.timeout_audit_run) and finally re-runs #audit_queue in case the timing attacks uncovered a new page.



404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
 
# File 'lib/arachni/framework.rb', line 404

def audit
    wait_if_paused

    @spider = Arachni::Spider.new( @opts )

    # if we're restricted to a given list of paths there's no reason to run the spider
    if @opts.restrict_paths && !@opts.restrict_paths.empty?
        @sitemap = @opts.restrict_paths
        @sitemap.each {
            |url|
            push_to_url_queue( url_sanitize( url ) )
        }
    else
        # initiates the crawl
        @spider.run( false ) {
            |response|
            @sitemap |= @spider.sitemap
            push_to_url_queue( url_sanitize( response.effective_url ) )
        }
    end

    audit_queue

    exception_jail {
        if !Arachni::Module::Auditor.timeout_audit_blocks.empty?
            print_line
            print_status( 'Running timing attacks.' )
            print_info( '---------------------------------------' )
            Arachni::Module::Auditor.on_timing_attacks {
                |res, elem|
                @current_url = elem.action if !elem.action.empty?
            }
            Arachni::Module::Auditor.timeout_audit_run
        end

        audit_queue
    }

end

#audit_page_queue ⇒ Object

Audits the page queue



485
486
487
488
489
490
491
492
493
 
# File 'lib/arachni/framework.rb', line 485

def audit_page_queue
    # this will run until no new elements appear for the given page
    while( !@page_queue.empty? && page = @page_queue.pop )

        # audit the page
        exception_jail{ run_mods( page ) }
        harvest_http_responses if !@opts.http_harvest_last
    end
end

#audit_queue ⇒ Object

Audits the URL and Page queues



447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
 
# File 'lib/arachni/framework.rb', line 447

def audit_queue

    # goes through the URLs discovered by the spider, repeats the request
    # and parses the responses into page objects
    #
    # yes...repeating the request is wasteful but we can't store the
    # responses of the spider to consume them here because there's no way
    # of knowing how big the site will be.
    #
    while( !@url_queue.empty? && url = @url_queue.pop )

        http.get( url, :remove_id => true ).on_complete {
            |res|

            page = Arachni::Parser::Page.from_http_response( res, @opts )

            # audit the page
            exception_jail{ run_mods( page ) }

            # don't let the page queue build up,
            # consume it as soon as possible because the pages are stored
            # in the FS and thus take up precious system resources
            audit_page_queue
        }

        harvest_http_responses if !@opts.http_harvest_last
    end

    harvest_http_responses if( @opts.http_harvest_last )

    audit_page_queue

    harvest_http_responses if( @opts.http_harvest_last )
end

#audit_store(fresh = true) ⇒ AuditStore Also known as: auditstore

Returns the results of the audit as an AuditStore instance

Returns:

See Also:



503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
 
# File 'lib/arachni/framework.rb', line 503

def audit_store( fresh = true )

    # restore the original redundancy rules and their counters
    @opts.redundant = @orig_redundant
    opts = @opts.to_h
    opts['mods'] = @modules.keys

    if( !fresh && @store )
        return @store
    else
        return @store = AuditStore.new( {
            :version  => version( ),
            :revision => REVISION,
            :options  => opts,
            :sitemap  => audit_store_sitemap || [],
            :issues   => @modules.results( ).deep_clone,
            :plugins  => @plugin_store
        })
     end
end

#audit_store_sitemap ⇒ Array

Special sitemap for the auditstore.

Used only under special circumstances, will usually return the #sitemap but can be overridden by the RPC::Framework.

Returns:



533
534
535
 
# File 'lib/arachni/framework.rb', line 533

def audit_store_sitemap
    @override_sitemap && !@override_sitemap.empty? ? @override_sitemap : @sitemap
end

#clean_up!(skip_audit_queue = false) ⇒ True

Cleans up the framework; should be called after running the audit or after canceling a running scan.

It stops the clock, waits for the plugins to finish up, register their results and also refreshes the auditstore.

It also runs #audit_queue in case any new pages have been added by the plugins.

Parameters:

  • skip_audit_queue (Bool) (defaults to: false) —

    skips running #audit_queue, set to true if you don’t want any delays.

Returns:

  • (True) 


698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
 
# File 'lib/arachni/framework.rb', line 698

def clean_up!( skip_audit_queue = false )
    @opts.finish_datetime = Time.now
    @opts.delta_time = @opts.finish_datetime - @opts.start_datetime

    # make sure this is disabled or it'll break report output
    @@only_positives = false

    @running = false

    # wait for the plugins to finish
    @plugins.block!

    # a plug-in may have updated the page queue, rock it!
    audit_queue if !skip_audit_queue

    # refresh the audit store
    audit_store( true )

    return true
end

#http ⇒ Arachni::HTTP

Returns HTTP instance.

Returns:



185
186
187
 
# File 'lib/arachni/framework.rb', line 185

def http
    Arachni::HTTP.instance
end

#lsmod ⇒ Array<Hash>

Returns an array of hashes with information about all available modules

Returns:



570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
 
# File 'lib/arachni/framework.rb', line 570

def lsmod
    @modules.available.map {
        |name|

        path = @modules.name_to_path( name )
        next if !lsmod_match?( path )

        @modules[name].info.merge(
            :mod_name => name,
            :author   => [@modules[name].info[:author]].flatten.map { |a| a.strip },
            :path     => path.strip
        )
    }.compact
ensure
    @modules.clear
end

#lsplug ⇒ Array<Hash>

Returns an array of hashes with information about all available reports

Returns:



616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
 
# File 'lib/arachni/framework.rb', line 616

def lsplug
    @plugins.available.map {
        |plugin|

        path = @plugins.name_to_path( plugin )
        next if !lsplug_match?( path )

        @plugins[plugin].info.merge(
            :plug_name => plugin,
            :path      => path,
            :author    => [@plugins[plugin].info[:author]].flatten.map { |a| a.strip }
        )
    }.compact
ensure
    @plugins.clear
end

#lsrep ⇒ Array<Hash>

Returns an array of hashes with information about all available reports

Returns:



593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
 
# File 'lib/arachni/framework.rb', line 593

def lsrep
    @reports.available.map {
        |report|

        path = @reports.name_to_path( report )
        next if !lsrep_match?( path )

        @reports[report].info.merge(
            :rep_name => report,
            :path     => path,
            :author   => [@reports[report].info[:author]].flatten.map { |a| a.strip }
        )
    }.compact
ensure
    @reports.clear
end

#pause! ⇒ True

Returns pauses the framework on a best effort basis, might take a while to take effect.

Returns:

  • (True) —

    pauses the framework on a best effort basis, might take a while to take effect



651
652
653
654
655
 
# File 'lib/arachni/framework.rb', line 651

def pause!
    @spider.pause! if @spider
    @paused << caller
    return true
end

#paused? ⇒ Bool

Returns true if the framework is paused or in the process of.

Returns:

  • (Bool) —

    true if the framework is paused or in the process of



643
644
645
 
# File 'lib/arachni/framework.rb', line 643

def paused?
    !@paused.empty?
end

#plugin_store(plugin, obj) ⇒ Object

Adds an object to the plugin store.

Should only be called once, if an entry for a plugin already exists it will just return.

Parameters:

  • plugin (String) —

    plugin/owner name

  • obj (Object) —

    object to store



546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
 
# File 'lib/arachni/framework.rb', line 546

def plugin_store( plugin, obj )
    name = ''
    @plugins.each_pair {
        |k, v|

        if plugin.class.name == v.name
            name = k
            break
        end
    }

    return if @plugin_store[name]

    @plugin_store[name] = {
        :results => obj
    }.merge( plugin.class.info )
end

#prepare ⇒ Object

Prepares the framework for the audit.

Sets the status to ‘running’, starts the clock and runs the plugins.

Must be called just before calling #audit.



196
197
198
199
200
201
202
 
# File 'lib/arachni/framework.rb', line 196

def prepare
    @running = true
    @opts.start_datetime = Time.now

    # run all plugins
    @plugins.run
end

#push_to_page_queue(page) ⇒ Object

Pushes a page to the page audit queue and updates #page_queue_total_size



384
385
386
387
 
# File 'lib/arachni/framework.rb', line 384

def push_to_page_queue( page )
    @page_queue << page
    @page_queue_total_size += 1
end

#push_to_url_queue(url) ⇒ Object

Pushes a URL to the URL audit queue and updates #url_queue_total_size



392
393
394
395
 
# File 'lib/arachni/framework.rb', line 392

def push_to_url_queue( url )
    @url_queue << url
    @url_queue_total_size += 1
end

#resume! ⇒ True

Returns resumes the scan/audit.

Returns:

  • (True) —

    resumes the scan/audit



660
661
662
663
664
 
# File 'lib/arachni/framework.rb', line 660

def resume!
    @paused.delete( caller )
    @spider.resume! if @spider
    return true
end

#revision ⇒ String

Returns the revision of the Arachni::Framework (this) class

Returns:



680
681
682
 
# File 'lib/arachni/framework.rb', line 680

def revision
    REVISION
end

#run(&block) ⇒ Object

Runs the system

It parses the instance options, #prepare, runs the #audit and #clean_up!.

Parameters:

  • &block (Block) —

    a block to call after the audit has finished but before running the reports



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'lib/arachni/framework.rb', line 212

def run( &block )
    prepare

    # catch exceptions so that if something breaks down or the user opted to
    # exit the reports will still run with whatever results
    # Arachni managed to gather
    begin
        # start the audit
        exception_jail{ audit( ) }
    rescue Exception => e
        # ap e
        # ap e.backtrace
    end

    clean_up!
    begin
        block.call if block
    rescue Exception
    end

    # run reports
    if( @opts.reports && !@opts.reports.empty? )
        exception_jail{ @reports.run( audit_store( ) ) }
    end

    return true
end

#running? ⇒ Bool

Returns true if the framework is running.

Returns:

  • (Bool) —

    true if the framework is running



636
637
638
 
# File 'lib/arachni/framework.rb', line 636

def running?
    @running
end

#stats(refresh_time = false, override_refresh = false) ⇒ Hash

Returns the following framework stats:

  • :requests – HTTP request count

  • :responses – HTTP response count

  • :time_out_count – Amount of timed-out requests

  • :time – Amount of running time

  • :avg – Average requests per second

  • :sitemap_size – Number of discovered pages

  • :auditmap_size – Number of audited pages

  • :progress – Progress percentage

  • :curr_res_time – Average response time for the current burst of requests

  • :curr_res_cnt – Amount of responses for the current burst

  • :curr_avg – Average requests per second for the current burst

  • :average_res_time – Average response time

  • :max_concurrency – Current maximum concurrency of HTTP requests

  • :current_page – URL of the currently audited page

  • :eta – Estimated time of arrival i.e. estimated remaining time

Parameters:

  • refresh_time (Bool) (defaults to: false) —

    updates the running time of the audit (usefully when you want stats while paused without messing with the clocks)

  • override_refresh (Bool) (defaults to: false)

Returns:

  • (Hash) 


266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/arachni/framework.rb', line 266

def stats( refresh_time = false, override_refresh = false )
    req_cnt = http.request_count
    res_cnt = http.response_count

    @opts.start_datetime = Time.now if !@opts.start_datetime

    sitemap_sz  = @url_queue_total_size + @page_queue_total_size
    auditmap_sz = @auditmap.size

    if( !refresh_time || auditmap_sz == sitemap_sz ) && !override_refresh
        @opts.delta_time ||= Time.now - @opts.start_datetime
    else
        @opts.delta_time = Time.now - @opts.start_datetime
    end

    curr_avg = 0
    if http.curr_res_cnt > 0 && http.curr_res_time > 0
        curr_avg = (http.curr_res_cnt / http.curr_res_time).to_i
    end

    avg = 0
    if res_cnt > 0
        avg = ( res_cnt / @opts.delta_time ).to_i
    end

    # we need to remove URLs that lead to redirects from the sitemap
    # when calculating the progress %.
    #
    # this is because even though these URLs are valid webapp paths
    # they are not actual pages and thus can't be audited;
    # so the sitemap and auditmap will never match and the progress will
    # never get to 100% which may confuse users.
    #
    if @spider
        redir_sz = @spider.redirects.size
    else
        redir_sz = 0
    end

    #
    # There are 2 audit phases:
    #  * regular analysis attacks
    #  * timing attacks
    #
    # When calculating the progress % we have to take both into account,
    # however each is calculated using different criteria.
    #
    # Progress of regular attacks is calculated as:
    #     amount of audited pages / amount of all discovered pages
    #
    # However, the progress of the timing attacks is calculated as:
    #     amount of called timeout blocks / amount of total blocks
    #
    # The timing attack modules are run with the regular ones however
    # their procedures are piled up into an array of Procs
    # which are called after the regular attacks.
    #
    # So when we reach the point of needing to include their progress in
    # the overall progress percentage we'll be working with accurate
    # data regarding the total blocks, etc.
    #

    #
    # If we have timing attacks then each phase must account for half
    # of the progress.
    #
    # This is not very granular but it's good enough for now...
    #
    if Arachni::Module::Auditor.timeout_loaded_modules.size > 0
        multi = 50
    else
        multi = 100
    end

    progress = (Float( auditmap_sz ) /
        ( sitemap_sz - redir_sz ) ) * multi

    if Arachni::Module::Auditor.running_timeout_attacks?

        called_blocks = Arachni::Module::Auditor.timeout_audit_operations_cnt -
            Arachni::Module::Auditor.current_timeout_audit_operations_cnt

        progress += ( Float( called_blocks ) /
            Arachni::Module::Auditor.timeout_audit_operations_cnt ) * multi
    end

    begin
        progress = Float( sprintf( "%.2f", progress ) )
    rescue
        progress = 0.0
    end

    # sometimes progress may slightly exceed 100%
    # which can cause a few strange stuff to happen
    progress = 100.0 if progress > 100.0

    return {
        :requests   => req_cnt,
        :responses  => res_cnt,
        :time_out_count  => http.time_out_count,
        :time       => audit_store.delta_time,
        :avg        => avg,
        :sitemap_size  => @sitemap.size,
        :auditmap_size => auditmap_sz,
        :progress      => progress,
        :curr_res_time => http.curr_res_time,
        :curr_res_cnt  => http.curr_res_cnt,
        :curr_avg      => curr_avg,
        :average_res_time => http.average_res_time,
        :max_concurrency  => http.max_concurrency,
        :current_page     => @current_url,
        :eta           => ::Arachni::Mixins::ProgressBar.eta( progress, @opts.start_datetime )
    }
end

#version ⇒ String

Returns the version of the framework

Returns:



671
672
673
 
# File 'lib/arachni/framework.rb', line 671

def version
    Arachni::VERSION
end