Class: NexposeTicketing::TicketService

Inherits:
Object
  • Object
show all
Defined in:
lib/nexpose_ticketing/ticket_service.rb

Overview

WARNING! This code is still rough and going through substantive changes. While

you can build tools using this library today, keep in mind that
method names and parameters may change in the future.

Constant Summary collapse

TICKET_SERVICE_CONFIG_PATH = 
File.join(File.dirname(__FILE__), '/config/ticket_service.config')
LOGGER_FILE = 
File.join(File.dirname(__FILE__), '/logs/ticket_service.log')

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#first_time ⇒ Object

Returns the value of attribute first_time.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def first_time
  @first_time
end

#helper_data ⇒ Object

Returns the value of attribute helper_data.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def helper_data
  @helper_data
end

#nexpose_data ⇒ Object

Returns the value of attribute nexpose_data.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def nexpose_data
  @nexpose_data
end

#nexpose_item_histories ⇒ Object

Returns the value of attribute nexpose_item_histories.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def nexpose_item_histories
  @nexpose_item_histories
end

#options ⇒ Object

Returns the value of attribute options.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def options
  @options
end

#ticket_repository ⇒ Object

Returns the value of attribute ticket_repository.



62
63
64
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 62

def ticket_repository
  @ticket_repository
end

Instance Method Details

#all_site_report(ticket_repository, options, helper) ⇒ Object

Generates a full site(s) report ticket(s).



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 130

def all_site_report(ticket_repository, options, helper)
  if(options[:tag_run])
    log_message('Generating full vulnerability report on user entered tags.')
    items_to_query = Array(options[:tags])
    log_message("Generating full vulnerability report on the following tags: #{items_to_query}")
  else
    log_message('Generating full vulnerability report on user entered sites.')
    items_to_query =  Array(options[:sites])
    log_message("Generating full vulnerability report on the following sites: #{items_to_query.join(', ')}")
  end
  items_to_query.each { |item|
    log_message("Running full vulnerability report on item #{item}")
    all_vulns_file = ticket_repository.all_vulns(options, item)
    log_message('Preparing tickets.')
    ticket_rate_limiter(options, all_vulns_file, Proc.new { |ticket_batch| helper.prepare_create_tickets(ticket_batch, options[:tag_run] ? "T#{item}" : item) }, Proc.new { |tickets| helper.create_tickets(tickets) })
  }

  if(options[:tag_run])
    items_to_query. each { |item_id|
    tag_assets_historic_file = File.join(File.dirname(__FILE__), 'tag_assets', "#{options[:tag_file_name]}_#{item_id}.csv")
    ticket_repository.generate_tag_asset_list(tags: item_id,
                            csv_file: tag_assets_historic_file)
    }
  end
  log_message('Finished process all vulnerabilities.')
end

#delta_site_new_scan(ticket_repository, nexpose_item, options, helper, file_site_histories, tag_id = nil) ⇒ Object

There’s a new scan with possibly new vulnerabilities.



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 229

def delta_site_new_scan(ticket_repository, nexpose_item, options, helper, file_site_histories, tag_id=nil)
  log_message("New scan detected for nexpose id: #{nexpose_item}. Generating report.")
  item = options[:tag_run] ? 'asset' : 'site'
  
  if options[:ticket_mode] == 'I' || options[:ticket_mode] == 'V'
    # I-mode and V-mode tickets require updating the tickets in the target system.
    log_message("Scan id for new scan: #{file_site_histories[nexpose_item]}.")
    all_scan_vuln_file = ticket_repository.all_vulns_since(scan_id: file_site_histories[nexpose_item],
                                                           nexpose_item: nexpose_item,
                                                           severity: options[:severity],
                                                           ticket_mode: options[:ticket_mode],
                                                           riskScore: options[:riskScore],
                                                           vulnerabilityCategories: options[:vulnerabilityCategories],
                                                           tag_run: options[:tag_run],
                                                           tag: tag_id)

    if helper.respond_to?('prepare_update_tickets') && helper.respond_to?('update_tickets')
      ticket_rate_limiter(options, all_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_update_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.update_tickets(tickets)})
    else
      log_message('Helper does not implement update methods')
      fail "Helper using 'I' or 'V' mode must implement prepare_updates and update_tickets"
    end

    if options[:close_old_tickets_on_update] == 'Y'
      tickets_to_close_file = ticket_repository.tickets_to_close(scan_id: file_site_histories[nexpose_item],
                                                                 nexpose_item: nexpose_item,
                                                                 severity: options[:severity],
                                                                 ticket_mode: options[:ticket_mode],
                                                                 riskScore: options[:riskScore],
                                                                 vulnerabilityCategories: options[:vulnerabilityCategories],
                                                                 tag_run: options[:tag_run],
                                                                 tag: tag_id)

      if helper.respond_to?('prepare_close_tickets') && helper.respond_to?('close_tickets')
        ticket_rate_limiter(options, tickets_to_close_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.close_tickets(tickets)})
      else
        log_message('Helper does not implement close methods')
        fail 'Helper using \'I\' or \'V\' mode must implement prepare_close_tickets and close_tickets'
      end
    end
  else
    # D-mode tickets require creating new tickets and closing old tickets.
    new_scan_vuln_file = ticket_repository.new_vulns(scan_id: file_site_histories[nexpose_item],
                                                           nexpose_item: nexpose_item,
                                                           severity: options[:severity],
                                                           ticket_mode: options[:ticket_mode],
                                                           riskScore: options[:riskScore],
                                                           vulnerabilityCategories: options[:vulnerabilityCategories],
                                                           tag_run: options[:tag_run],
                                                           tag: tag_id)

    preparse = CSV.open(new_scan_vuln_file.path, headers: :first_row)
    empty_report = preparse.shift.nil?
    preparse.close

    
    log_message("No new vulnerabilities found in new scan for #{item}: #{nexpose_item}.") if empty_report
    log_message("New vulnerabilities found in new scan for #{item} #{nexpose_item}, preparing tickets.") unless empty_report
    unless empty_report
      ticket_rate_limiter(options, new_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.create_tickets(tickets)})
    end
    
    if helper.respond_to?('prepare_close_tickets') && helper.respond_to?('close_tickets')
      old_scan_vuln_file = ticket_repository.old_vulns(scan_id: file_site_histories[nexpose_item],
                                                             nexpose_item: nexpose_item,
                                                             site_id: nexpose_item,
                                                             severity: options[:severity],
                                                             riskScore: options[:riskScore],
                                                             vulnerabilityCategories: options[:vulnerabilityCategories],
                                                             tag_run: options[:tag_run],
                                                             tag: tag_id)

      preparse = CSV.open(old_scan_vuln_file.path, headers: :first_row, :skip_blanks => true)
      empty_report = preparse.shift.nil?
      preparse.close
      log_message("No old (closed) vulnerabilities found in new scan for #{item}: #{nexpose_item}.") if empty_report
      log_message("Old vulnerabilities found in new scan for #{item} #{nexpose_item}, preparing closures.") unless empty_report
      unless empty_report
        ticket_rate_limiter(options, old_scan_vuln_file, Proc.new {|ticket_batch| helper.prepare_close_tickets(ticket_batch, tag_id.nil? ? nexpose_item : "T#{tag_id}")}, Proc.new {|tickets| helper.close_tickets(tickets)})
      end
    else
      # Create a log message but do not halt execution of the helper if ticket closing is not
      # supported to allow legacy code to execute normally.
      log_message('Helper does not implement close methods.')
    end
  end
end

#delta_site_report(ticket_repository, options, helper, file_site_histories) ⇒ Object

There’s possibly a new scan with new data.



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 158

def delta_site_report(ticket_repository, options, helper, file_site_histories)
  # Compares the scan information from file && Nexpose.
  no_processing = true
  @nexpose_item_histories.each do |item_id, last_scan_id|
    # There's no entry in the file, so it's either a new item in Nexpose or a new item we have to monitor.
    if file_site_histories[item_id].nil? || file_site_histories[item_id] == -1
      full_new_site_report(item_id, ticket_repository, options, helper)
      if(options[:tag_run])
        tag_assets_historic_file = File.join(File.dirname(__FILE__), 'tag_assets', "#{options[:tag_file_name]}_#{item_id}.csv")
        ticket_repository.generate_tag_asset_list(tags: item_id,
                                                  csv_file: tag_assets_historic_file)
      end
      no_processing = false
      # Site has been scanned since last seen according to the file.
    elsif file_site_histories[item_id].to_s != nexpose_item_histories[item_id].to_s
      if(options[:tag_run])
        # It's a tag run and something has changed (new/removed asset or new scan ID for an asset). To find out what, we must compare
        # All tag assets and their scan IDs. Firstly we fetch all the assets in the tags
        # in the configuration file and store them temporarily
        tag_assets_tmp_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.tmp")
        tag_assets_historic_file = File.join(File.dirname(__FILE__), "/tag_assets/#{options[:tag_file_name]}_#{item_id}.csv")
        ticket_repository.generate_tag_asset_list(tags: item_id,
                                csv_file: tag_assets_tmp_file)
        new_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_tmp_file)
        historic_tag_configuration = ticket_repository.read_tag_asset_list(tag_assets_historic_file)
        #Compare the assets within the tags and their scan histories to find the ones we need to query
        changed_assets = Hash[*(historic_tag_configuration.to_a - new_tag_configuration.to_a).flatten]
        new_assets = Hash[*(new_tag_configuration.to_a - historic_tag_configuration.to_a).flatten]
        new_assets.delete_if {|asset_id, scan_id| historic_tag_configuration.has_key?(asset_id.to_s)}
        #all_assets_changed = new_assets.merge(changed_assets)
        changed_assets.each do |asset_id, scan_id|
          delta_site_new_scan(ticket_repository, asset_id, options, helper, changed_assets, item_id)
        end
        new_assets.each do |asset_id, scan_id|
          #Since no previous scan IDs - we generate a full report.
          options[:nexpose_item] = asset_id
          full_new_site_report(item_id, ticket_repository, options, helper)
          options.delete(:nexpose_item)
        end
      else
        delta_site_new_scan(ticket_repository, item_id, options, helper, file_site_histories)
      end
      if(options[:tag_run])
        #Update the historic file
        new_tag_asset_list = historic_tag_configuration.merge(new_tag_configuration)
        trimmed_csv = []
        trimmed_csv << 'asset_id, last_scan_id'
        new_tag_asset_list.each do |asset_id, last_scan_id|
          trimmed_csv << "#{asset_id},#{last_scan_id}"
        end
        ticket_repository.save_to_file(tag_assets_historic_file, trimmed_csv)
        File.delete(tag_assets_tmp_file)
      end
      no_processing = false
    end
  end
  # Done processing, update the CSV to the latest scan info.
  log_message("Nothing new to process, updating historical CSV file #{options[:file_name]}.") if no_processing
  log_message("Done processing, updating historical CSV file #{options[:file_name]}.") unless no_processing
  no_processing
end

#full_new_site_report(nexpose_item, ticket_repository, options, helper) ⇒ Object

There’s a new site we haven’t seen before.



221
222
223
224
225
226
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 221

def full_new_site_report(nexpose_item, ticket_repository, options, helper)
  log_message("New nexpose id: #{nexpose_item} detected. Generating report.")
  new_item_vuln_file = ticket_repository.all_vulns(options, nexpose_item)
  log_message('Report generated, preparing tickets.')
  ticket_rate_limiter(options, new_item_vuln_file, Proc.new {|ticket_batch| helper.prepare_create_tickets(ticket_batch, options[:tag_run] ? "T#{nexpose_item}" : nexpose_item)}, Proc.new {|tickets| helper.create_tickets(tickets)})
end

#log_message(message) ⇒ Object

Logs a message if logging is enabled.



110
111
112
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 110

def log_message(message)
  @log.info(message) if @options[:logging_enabled]
end

#prepare_historical_data(ticket_repository, options) ⇒ Object

Prepares all the local and nexpose historical data.



115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 115

def prepare_historical_data(ticket_repository, options)
  (options[:tag_run]) ?
      historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:tag_file_name]}") :
      historical_scan_file = File.join(File.dirname(__FILE__), "#{options[:file_name]}")

  if File.exists?(historical_scan_file)
    log_message("Reading historical CSV file: #{historical_scan_file}.")
    file_site_histories = ticket_repository.read_last_scans(historical_scan_file)
  else
    file_site_histories = nil
  end
  file_site_histories
end

#setup(helper_data) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 64

def setup(helper_data)
  # Gets the Ticket Service configuration.
  service_data = begin
    YAML.load_file(TICKET_SERVICE_CONFIG_PATH)
  rescue ArgumentError => e
    raise "Could not parse YAML #{TICKET_SERVICE_CONFIG_PATH} : #{e.message}"
  end
  @helper_data = helper_data
  @nexpose_data = service_data[:nexpose_data]
  @options = service_data[:options]
  @options[:file_name] = "#{@options[:file_name]}"

  # Setups logging if enabled.
  setup_logging(@options[:logging_enabled])

  # Loads all the helpers.
  log_message('Loading helpers.')
  Dir[File.join(File.dirname(__FILE__), '/helpers/*.rb')].each do |file|
    log_message("Loading helper: #{file}")
    require_relative file
  end
  log_message("Ticket mode: #{@options[:ticket_mode]}.")

  log_message("Enabling helper: #{@helper_data[:helper_name]}.")
  @helper = eval(@helper_data[:helper_name]).new(@helper_data, @options)

  log_message("Creating ticketing repository with timeout value: #{@options[:timeout]}.")
  @ticket_repository = NexposeTicketing::TicketRepository.new(options)
  @ticket_repository.(@nexpose_data)
end

#setup_logging(enabled = false) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 95

def setup_logging(enabled = false)
  helper_log = NexposeTicketing::NxLogger.instance
  helper_log.setup_logging(@options[:logging_enabled],
                           @options[:log_level])

  return unless enabled
  require 'logger'
  directory = File.dirname(LOGGER_FILE)
  FileUtils.mkdir_p(directory) unless File.directory?(directory)
  @log = Logger.new(LOGGER_FILE, 'monthly')
  @log.level = Logger::INFO
  log_message('Logging enabled, starting service.')
end

#start ⇒ Object

Starts the Ticketing Service.



391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 391

def start
  #Decide if this is a tag run (tags always override sites as the API does not allow for the combination of the two)
  @options[:tag_run] = !@options[:tags].nil? && !@options[:tags].empty?

  # Checks if the csv historical file already exists && reads it, otherwise create it && assume first time run.
  file_site_histories = prepare_historical_data(@ticket_repository, @options)
  historical_scan_file = File.join(File.dirname(__FILE__), "#{@options[:file_name]}")
  historical_tag_file = File.join(File.dirname(__FILE__), "#{@options[:tag_file_name]}")

  # If we didn't specify a site || first time run (no scan history), then it gets all the vulnerabilities.
  if (((@options[:sites].nil? || @options[:sites].empty? || file_site_histories.nil?) && !@options[:tag_run]) || (@options[:tag_run] && file_site_histories.nil?))
  log_message('Storing current scan state before obtaining all vulnerabilities.')
    current_scan_state = ticket_repository.load_last_scans(@options)

    if (options[:sites].nil? || options[:sites].empty?) && (!@options[:tag_run])
      log_message('No site(s) specified, generating for all sites.')
      @ticket_repository.all_site_details.each { |site|  (@options[:sites] ||= []) << site.id.to_s }
      log_message("List of sites is now <#{@options[:sites]}>")
    end

    all_site_report(@ticket_repository, @options, @helper)

    #Generate historical CSV file after completing the fist query.
    log_message('No historical CSV file found. Generating.')
    @options[:tag_run] ?
        @ticket_repository.save_to_file(historical_tag_file, current_scan_state) :
        @ticket_repository.save_to_file(historical_scan_file, current_scan_state)
    log_message('Historical CSV file generated.')
  else
    log_message('Obtaining last scan information.')
    @nexpose_item_histories = @ticket_repository.last_scans(@options)

    # Scan states can change during our processing. Store the state we are
    # about to process and move this to the historical file if we
    # successfully process.
    log_message('Calculated deltas, storing current scan state.')
    current_scan_state = ticket_repository.load_last_scans(options)

    # Only run if a scan has been ran ever in Nexpose.
    unless @nexpose_item_histories.empty?
      delta_site_report(@ticket_repository, @options, @helper, file_site_histories)
      # Processing completed successfully. Update historical scan file.
      @options[:tag_run] ?
          @ticket_repository.save_to_file(historical_tag_file, current_scan_state) :
          @ticket_repository.save_to_file(historical_scan_file, current_scan_state)
    end
  end
  log_message('Exiting ticket service.')
end

#ticket_rate_limiter(options, query_results_file, ticket_prepare_method, ticket_send_method) ⇒ Object 



318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 318

def ticket_rate_limiter(options, query_results_file, ticket_prepare_method, ticket_send_method)      
  batch_size_max = (options[:batch_size] + 1)
  log_message("Batching tickets in sizes: #{options[:batch_size]}")

  #Vulnerability mode is batched by vulnerability_id, IP mode is batched by IP address.
  current_id = -1
  id_field = if @options[:ticket_mode] == 'V'
               'vulnerability_id'
             else
               'ip_address'
             end
  
  # Start the batching
  query_results_file.rewind
  csv_header = query_results_file.readline
  ticket_batch = []
  current_csv_row = nil

  begin
    CSV.foreach(query_results_file, headers: csv_header) do |row|
      ticket_batch << row

      #Keep updating the ID until we hit the batch 'limit'
      #Should this be <=??? What happens if value n-1 is different?

      if ticket_batch.size < batch_size_max
        current_id = row[id_field]
        next
      end

      #Keep adding rows to get all information on an asset/vulnerability
      next if @options[:ticket_mode] != 'D' && row[id_field] == current_id
      
      #Last row is mismatch/independent
      leftover_row = ticket_batch.pop

      log_message('Batch size reached. Sending tickets.')
      ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
      
      ticket_batch.clear
      ticket_batch << csv_header
      ticket_batch << leftover_row
      current_id = -1
    end
  ensure
    log_message('Finished reading report. Sending any remaining tickets and cleaning up file system.')

    ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)

    query_results_file.close
    query_results_file.unlink
  end 
end

#ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method) ⇒ Object 



372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
 
# File 'lib/nexpose_ticketing/ticket_service.rb', line 372

def ticket_rate_limiter_processor(ticket_batch, ticket_prepare_method, ticket_send_method)
  #Just the header (no tickets).
  if ticket_batch.size == 1
    log_message('Received empty batch. Not sending tickets.')
    return
  end

  # Prep the batch of tickets
  log_message('Creating tickets.')
  tickets = ticket_prepare_method.call(ticket_batch.join(''))
  log_message("Parsed rows: #{ticket_batch.size}")
  # Sent them off
  log_message('Sending tickets.')
  ticket_send_method.call(tickets)
  log_message('Returning for next batch.')
end