Class: NexposeTicketing::TicketService

Inherits:
Object
  • Object
show all
Defined in:
lib/nexpose_ticketing/ticket_service.rb

Overview

WARNING! This code is still rough and going through substantive changes. While

you can build tools using this library today, keep in mind that
method names and parameters may change in the future.

Constant Summary collapse

TICKET_SERVICE_CONFIG_PATH = 
File.join(File.dirname(__FILE__), '/config/ticket_service.config')
LOGGER_FILE = 
File.join(File.dirname(__FILE__), '/logs/ticket_service.log')

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#first_time ⇒ Object

Returns the value of attribute first_time.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def first_time
  @first_time
end

#helper_data ⇒ Object

Returns the value of attribute helper_data.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def helper_data
  @helper_data
end

#nexpose_data ⇒ Object

Returns the value of attribute nexpose_data.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def nexpose_data
  @nexpose_data
end

#options ⇒ Object

Returns the value of attribute options.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def options
  @options
end

#ticket_repository ⇒ Object

Returns the value of attribute ticket_repository.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def ticket_repository
  @ticket_repository
end

Instance Method Details

#all_site_report(ticket_repository, options) ⇒ Object

Generates a full site(s) report ticket(s).



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 150

def all_site_report(ticket_repository, options)
  group = "#{options[:scan_mode]}s"

  log_message("Generating full vulnerability report on user entered #{group}.")
  items_to_query = Array(options[group.to_sym])
  log_message("Generating full vulnerability report on the following #{group}: #{items_to_query.join(', ')}")
  
  items_to_query.each do |item|
    log_message("Running full vulnerability report on item #{item}")
    all_vulns_file = ticket_repository.all_new_vulns(options, item)
    log_message('Preparing tickets.')

    ticket_rate_limiter(all_vulns_file, 'create', item)

    post_scan(item_id: item, generate_asset_list: true)
  end

  log_message('Finished processing all vulnerabilities.')
end

#delta_new_scan(item_id, options, helper, scan_histories) ⇒ Object

Performs a delta scan



332
333
334
335
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 332

def delta_new_scan(item_id, options, helper, scan_histories)
  delta_func = "delta_#{options[:scan_mode]}_new_scan"
  self.send(delta_func, item_id, options, helper, scan_histories)
end

#delta_scan(scan_histories) ⇒ Object 



314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 314

def delta_scan(scan_histories)
  log_message('Obtaining last scan information.')
  @latest_scans = @ticket_repository.last_scans(@options)

  # Scan states can change during our processing. Store the state we are
  # about to process and move this to the historical file if we
  # successfully process.
  log_message('Calculated deltas, storing current scan state.')
  @current_scan_state = ticket_repository.load_last_scans(@options)

  # Only run if a scan has been ran ever in Nexpose.
  return if @latest_scans.empty?

  delta_site_report(@ticket_repository, @options, @helper, scan_histories)
  log_message('Historical CSV file updated.')
end

#delta_site_new_scan(nexpose_item, options, helper, file_site_histories, tag_id = nil) ⇒ Object

There’s a new scan with possibly new vulnerabilities.



338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 338

def delta_site_new_scan(nexpose_item, options, helper, file_site_histories, tag_id=nil)
  log_message("New scan detected for nexpose id: #{nexpose_item}. Generating report.")
  
  format_method = "format_#{options[:scan_mode]}_id"
  nexpose_id = self.send(format_method, tag_id || nexpose_item)
  
  scan_options = { scan_id: file_site_histories[nexpose_item],
                   nexpose_item: nexpose_item,
                   severity: options[:severity],
                   ticket_mode: options[:ticket_mode],
                   riskScore: options[:riskScore],
                   vulnerabilityCategories: options[:vulnerabilityCategories],
                   tag_run: options[:tag_run],
                   tag: tag_id }

  log_message("Scan id for new scan: #{file_site_histories[nexpose_item]}.")

  if @mode.updates_supported?
    helper_method = 'update'
    new_vulns_query = 'all_vulns_since_scan'
    old_vulns_query = 'old_tickets'
  else
    helper_method = 'create'
    new_vulns_query = 'new_vulns_since_scan'
    old_vulns_query = 'old_vulns_since_scan'
  end

  all_scan_vuln_file = ticket_repository.send(new_vulns_query, scan_options)
  ticket_rate_limiter(all_scan_vuln_file, helper_method, nexpose_id)
    
  return unless options[:close_old_tickets_on_update] == 'Y'

  tickets_to_close_file = ticket_repository.send(old_vulns_query, scan_options)
  ticket_rate_limiter(tickets_to_close_file, 'close', nexpose_id)
end

#delta_site_report(ticket_repository, options, helper, scan_histories) ⇒ Object

There’s possibly a new scan with new data.



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 171

def delta_site_report(ticket_repository, options, helper, scan_histories)
  # Compares the scan information from file && Nexpose.
  no_processing = true
  @latest_scans.each do |item_id, last_scan_id|
    prev_scan_id = scan_histories[item_id]

    # There's no entry in the file, so it's either a new item in Nexpose or a new item we have to monitor.
    if prev_scan_id.nil? || prev_scan_id == -1
      options[:nexpose_item] = item_id
      full_new_site_report(item_id, ticket_repository, options, helper)
      options[:nexpose_item] = nil
      post_scan(item_id: item_id, generate_asset_list: true)
      no_processing = false
    # Site has been scanned since last seen according to the file.
    elsif prev_scan_id.to_s != @latest_scans[item_id].to_s
      delta_new_scan(item_id, options, helper, scan_histories)
      post_scan item_id: item_id
      no_processing = false
    end
  end

  log_name = @options["#{@options[:scan_mode]}_file_name".to_sym]
  # Done processing, update the CSV to the latest scan info.
  if no_processing
    log_message("Nothing new to process, historical CSV file has not been updated: #{options[:file_name]}.") 
  else
    log_message("Done processing, historical CSV file has been updated: #{options[:file_name]}.")
  end
  no_processing
end

#delta_tag_new_scan(nexpose_item, options, helper, file_site_histories, tag_id = nil) ⇒ Object 



374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 374

def delta_tag_new_scan(nexpose_item, options, helper, file_site_histories, tag_id=nil)
  # It's a tag run and something has changed (new/removed asset or new scan ID for an asset). To find out what, we must compare
  # All tag assets and their scan IDs. Firstly we fetch all the assets in the tags
  # in the configuration file and store them temporarily
  item_id = nexpose_item
  tag_assets_tmp_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.tmp")
  tag_assets_historic_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.csv")
  ticket_repository.generate_tag_asset_list(tags: item_id,
                          csv_file: tag_assets_tmp_file)
  new_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_tmp_file)
  historic_tag_config = ticket_repository.read_tag_asset_list(tag_assets_historic_file)
  #Compare the assets within the tags and their scan histories to find the ones we need to query
  changed_assets = Hash[*(historic_tag_config.to_a - new_tag_configuration.to_a).flatten]
  new_assets = Hash[*(new_tag_configuration.to_a - historic_tag_config.to_a).flatten]
  new_assets.delete_if {|asset_id, scan_id| historic_tag_config.has_key?(asset_id.to_s)}

  #all_assets_changed = new_assets.merge(changed_assets)
  changed_assets.each do |asset_id, scan_id|
    delta_site_new_scan(asset_id, options,
                        helper, changed_assets, item_id)
  end

  new_assets.each do |asset_id, scan_id|
    #Since no previous scan IDs - we generate a full report.
    options[:nexpose_item] = asset_id
    full_new_site_report(item_id, ticket_repository, options, helper)
    options.delete(:nexpose_item)
  end

   #Update the historic file
  new_tag_asset_list = historic_tag_config.merge(new_tag_configuration)
  trimmed_csv = []
  trimmed_csv << 'asset_id, last_scan_id'
  new_tag_asset_list.each do |asset_id, last_scan_id|
    trimmed_csv << "#{asset_id},#{last_scan_id}"
  end
  ticket_repository.save_to_file(tag_assets_historic_file, trimmed_csv)
  File.delete(tag_assets_tmp_file)
end

#format_id(item_id) ⇒ Object

Formats the Nexpose item ID according to the asset grouping mode



471
472
473
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 471

def format_id(item_id)
  self.send("format_#{options[:scan_mode]}_id", item_id)
end

#format_site_id(item_id) ⇒ Object 



475
476
477
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 475

def format_site_id(item_id)
  item_id
end

#format_tag_id(item_id) ⇒ Object 



479
480
481
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 479

def format_tag_id(item_id)
  "T#{item_id}"
end

#full_new_site_report(nexpose_item, ticket_repository, options, helper) ⇒ Object

There’s a new site we haven’t seen before.



203
204
205
206
207
208
209
210
211
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 203

def full_new_site_report(nexpose_item, ticket_repository, options, helper)
  log_message("New nexpose id: #{nexpose_item} detected. Generating report.")
  options[:scan_id] = 0
  new_item_vuln_file = ticket_repository.all_new_vulns(options, nexpose_item)
  log_message('Report generated, preparing tickets.')

  nexpose_id = format_id(nexpose_item)
  ticket_rate_limiter(new_item_vuln_file, 'create', nexpose_id)
end

#full_scan ⇒ Object 



304
305
306
307
308
309
310
311
312
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 304

def full_scan
  log_message('Storing current scan state before obtaining all vulnerabilities.')
  @current_scan_state = ticket_repository.load_last_scans(@options)
  
  all_site_report(@ticket_repository, @options)

  #Generate historical CSV file after completing the fist query.
  log_message('Historical CSV file generated.')
end

#full_scan_required?(histories) ⇒ Boolean

Determines whether all assets must be scanned



484
485
486
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 484

def full_scan_required?(histories)
  self.send("full_#{@options[:scan_mode]}_scan_required?", histories)
end

#full_site_scan_required?(scan_histories) ⇒ Boolean 



488
489
490
491
492
493
494
495
496
497
498
499
500
501
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 488

def full_site_scan_required?(scan_histories)
  is_full_run = false

  if @options[:sites].nil? || @options[:sites].empty?
    is_full_run = true
    
    all_site_details = @ticket_repository.all_site_details
    @options[:sites] = all_site_details.map { |s| s.id.to_s }
   
    log_message("List of sites is now <#{@options[:sites]}>")
  end

  is_full_run || scan_histories.nil?
end

#full_tag_scan_required?(scan_histories) ⇒ Boolean 



503
504
505
506
507
508
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 503

def full_tag_scan_required?(scan_histories)
  if @options[:tags].nil? || @options[:tags].empty?
    fail 'No tags specified within the configuration.'  
  end
  return scan_histories.nil?
end

#get_scan_mode ⇒ Object 



281
282
283
284
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 281

def get_scan_mode
  return 'tag' unless @options[:tags].nil? || @options[:tags].empty?
  return 'site'
end

#get_site_file_header ⇒ Object 



462
463
464
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 462

def get_site_file_header
  ['site_id,last_scan_id,finished']
end

#get_tag_file_header ⇒ Object 



466
467
468
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 466

def get_tag_file_header
  ['tag_id,last_scan_fingerprint']
end

#load_class(type, name) ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 99

def load_class(type, name)
  name.gsub!(type.capitalize, '')
  path = "#{type}s/#{name}_#{type}.rb".downcase
  
  log_message("Loading #{type} dependency: #{path}.")
  begin 
    require_relative path
  rescue => e
    error = "#{type.capitalize} dependency '#{path}' could not be loaded."
    @log.error e.to_s
    @log.error error
    fail error
  end

  eval("#{name}#{type.capitalize}")
end

#log_message(message) ⇒ Object

Logs a message if logging is enabled.



132
133
134
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 132

def log_message(message)
  @log.info(message) if @options[:logging_enabled]
end

#post_scan(**modifiers) ⇒ Object

Methods to run after a scan



415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 415

def post_scan(**modifiers)
  self.send("post_#{@options[:scan_mode]}_scan", modifiers)
  scan_history = self.send("get_#{@options[:scan_mode]}_file_header")

  item_id = modifiers[:item_id]
  historic_data = nil
  if File.exists?(@historical_file)
    log_message("Updating historical CSV file: #{@historical_file}.")
    historic_data = []
    CSV.foreach(@historical_file, headers: true) { |r| historic_data << r }
  end

  updated_row = [@current_scan_state.find { |row| row[0].eql?(item_id) }]

  if historic_data.nil?
    log_message('No historical CSV file found. Generating.')
    scan_history.concat(updated_row)
  else
    index = historic_data.find_index { |id| id[0] == item_id }
    if index.nil?
      historic_data.concat(updated_row)
      historic_data.sort! { |x,y| x[0].to_i <=> y[0].to_i }
    else
      historic_data[index] = updated_row
      historic_data.flatten!
    end
    scan_history.concat(historic_data)
  end
  
  log_message('Updated historical CSV file for ' \
              "#{@options[:scan_mode]}: #{item_id}.")
  @ticket_repository.save_to_file(@historical_file, scan_history)
end

#post_site_scan(**modifiers) ⇒ Object 



449
450
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 449

def post_site_scan(**modifiers)
end

#post_tag_scan(**modifiers) ⇒ Object 



452
453
454
455
456
457
458
459
460
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 452

def post_tag_scan(**modifiers)
  return unless modifiers[:generate_asset_list]      
  file_name = "#{@options[:tag_file_name]}_#{modifiers[:item_id]}.csv"
  historic_file = File.join(File.dirname(__FILE__), 'tag_assets', file_name)

  log_message("Generating current tag asset file: #{historic_file}.")
  ticket_repository.generate_tag_asset_list(tags: modifiers[:item_id],
                                            csv_file: historic_file)
end

#prepare_historical_data(ticket_repository, options) ⇒ Object

Prepares all the local and nexpose historical data.



137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 137

def prepare_historical_data(ticket_repository, options)
  historical_scan_file = @historical_file

  file_site_histories = nil
  if File.exists?(historical_scan_file)
    log_message("Reading historical CSV file: #{historical_scan_file}.")
    file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
  end

  file_site_histories
end

#setup(helper_data) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 64

def setup(helper_data)
  # Gets the Ticket Service configuration.
  service_data = begin
    YAML.load_file(TICKET_SERVICE_CONFIG_PATH)
  rescue ArgumentError => e
    raise "Could not parse YAML #{TICKET_SERVICE_CONFIG_PATH} : #{e.message}"
  end

  @helper_data = helper_data
  @nexpose_data = service_data[:nexpose_data]
  @options = service_data[:options]
  @options[:file_name] = @options[:file_name].to_s
  @options[:scan_mode] = get_scan_mode
  
  #Temporary - this should be refactored out e.g. to include DAGs
  @options[:tag_run] = @options[:scan_mode] == 'tag'

  file_name = @options["#{@options[:scan_mode]}_file_name".to_sym]
  @historical_file = File.join(File.dirname(__FILE__), file_name)

  # Sets logging up, if enabled.
  setup_logging(@options[:logging_enabled])

  mode_class = load_class 'mode', @options[:ticket_mode]
  @mode = mode_class.new(@options)
  @options[:query_suffix] = @mode.get_query_suffix

  helper_class = load_class 'helper', @helper_data[:helper_name]
  @helper = helper_class.new(@helper_data, @options, @mode)

  log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.")
  @ticket_repository = NexposeTicketing::TicketRepository.new(options)
  @ticket_repository.(@nexpose_data)
end

#setup_logging(enabled = false) ⇒ Object 



116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 116

def setup_logging(enabled = false)
  helper_log = NexposeTicketing::NxLogger.instance
  helper_log.setup_logging(@options[:logging_enabled],
                           @options[:log_level],
                           @options[:log_console])

  return unless enabled
  require 'logger'
  directory = File.dirname(LOGGER_FILE)
  FileUtils.mkdir_p(directory) unless File.directory?(directory)
  @log = Logger.new(LOGGER_FILE, 'monthly')
  @log.level = Logger::INFO
  log_message('Logging enabled, starting service.')
end

#start ⇒ Object

Starts the Ticketing Service.



287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 287

def start
  # Checks if the csv historical file already exists and reads it, otherwise create it and assume first time run.
  scan_histories = prepare_historical_data(@ticket_repository, @options)

  # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
  @options[:initial_run] = full_scan_required?(scan_histories)

  if @options[:initial_run]
    full_scan
  else
    delta_scan(scan_histories)
  end

  @helper.finish
  log_message('Exiting ticket service.')
end

#ticket_rate_limiter(query_results_file, ticket_method, nexpose_item) ⇒ Object 



235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 235

def ticket_rate_limiter(query_results_file, ticket_method, nexpose_item)
  batch_size_max = @options[:batch_size]
  log_message("Batching tickets in sizes: #{@options[:batch_size]}")
  matching_fields = @mode.get_matching_fields
  current_ids = Hash[*matching_fields.collect { |k| [k, nil] }.flatten]

  # Start the batching
  query_results_file.rewind
  csv_header = query_results_file.readline
  ticket_batch = []
  
  begin
    CSV.foreach(query_results_file, headers: csv_header) do |row|
      ticket_batch << row

      # Store the current 
      if ticket_batch.size < batch_size_max
        matching_fields.each { |id| current_ids[id] = row[id] }
        next
      end

      # Ensure that all rows associated with a ticket are captured.
      # This potentially ignores the batch limit.
      next if current_ids.all? { |id, val| val == row[id] }
    
      #Last row is mismatch/independent
      leftover_row = ticket_batch.pop

      log_message('Batch size reached. Sending tickets.')
      ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item)
      
      ticket_batch.clear
      ticket_batch << csv_header
      ticket_batch << leftover_row
      current_ids.each { |k,v| k = nil }
    end
  ensure
    log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.')

    ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item)

    query_results_file.close
    query_results_file.unlink
  end 
end

#ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item) ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 213

def ticket_rate_limiter_processor(ticket_batch, ticket_method, nexpose_item)
  #Just the header (no tickets).
  if ticket_batch.size == 1
    log_message('Received empty batch. Not sending tickets.')
    return
  end

  nexpose_item = format_id(nexpose_item)

  # Prep the batch of tickets
  log_message("Preparing to #{ticket_method} tickets.")
  tickets = @helper.send("prepare_#{ticket_method}_tickets", 
                         ticket_batch.join(''), 
                         nexpose_item)
  log_message("Parsed rows: #{ticket_batch.size}")

  # Send them off
  log_message('Sending tickets.')
  @helper.send("#{ticket_method}_tickets", tickets)
  log_message('Returning for next batch.')
end