Class: Recog::Nizer

Inherits:
Object
  • Object
show all
Defined in:
lib/recog/nizer.rb

Constant Summary collapse

DEFAULT_OS_CERTAINTY =

Default certainty ratings where none are specified in the fingerprint itself

0.85
DEFAULT_SERVICE_CERTAINTY =

Most frequent weights are 0.9, 1.0, and 0.5

0.85
HOST_ATTRIBUTES =

Non-weighted host attributes that can be extracted from fingerprint matches

%W{
  host.domain
  host.id
  host.ip
  host.mac
  host.name
  host.time
  hw.device
  hw.family
  hw.product
  hw.vendor
}
@@db_manager = 
nil
@@db_sorted = 
false

Class Method Summary collapse

Class Method Details

.best_os_match(matches) ⇒ Object

Consider an array of match outputs, choose the best result, taking into account the granularity of OS vs Version vs SP vs Language. Only consider fields relevant to the host (OS, name, mac address, etc).



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
 
# File 'lib/recog/nizer.rb', line 140

def self.best_os_match(matches)

  # The result hash we return to the caller
  result = {}

  # Certain attributes should be evaluated separately
  host_attrs  = {}

  # Bucket matches into matched OS product names
  os_products = {}

  matches.each do |m|
    # Count how many times each host attribute value is asserted
    (HOST_ATTRIBUTES & m.keys).each do |ha|
      host_attrs[ha]        ||= {}
      host_attrs[ha][m[ha]] ||= 0
      host_attrs[ha][m[ha]]  += 1
    end

    next unless m.has_key?('os.product')

    # Group matches by OS product and normalize certainty
    cm = m.dup
    cm['os.certainty'] = ( m['os.certainty'] || DEFAULT_OS_CERTAINTY ).to_f
    os_products[ cm['os.product'] ] ||= []
    os_products[ cm['os.product'] ]  << cm
  end

  #
  # Select the best host attribute value by highest frequency
  #
  host_attrs.keys.each do |hk|
    ranked_attr = host_attrs[hk].keys.sort do |a,b|
      host_attrs[hk][b] <=> host_attrs[hk][a]
    end
    result[hk] = ranked_attr.first
  end

  # Unable to guess the OS without OS matches
  unless os_products.keys.length > 0
    return result
  end

  #
  # Select the best operating system name by combined certainty of all
  # matches within an os.product group. Multiple weak matches can
  # outweigh a single strong match by design.
  #
  ranked_os = os_products.keys.sort do |a,b|
    os_products[b].map{ |r| r['os.certainty'] }.inject(:+) <=>
    os_products[a].map{ |r| r['os.certainty'] }.inject(:+)
  end

  # Within the best match group, try to fill in missing attributes
  os_name = ranked_os.first

  # Find the best match within the winning group
  ranked_os_matches = os_products[os_name].sort do |a,b|
    b['os.certainty'] <=> a['os.certainty']
  end

  # Fill in missing result values in descending order of best match
  ranked_os_matches.each do |rm|
    rm.each_pair do |k,v|
      result[k] ||= v
    end
  end

  result
end

.best_service_match(matches) ⇒ Object

Consider an array of match outputs, choose the best result, taking into account the granularity of service. Only consider fields relevant to the service.



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
 
# File 'lib/recog/nizer.rb', line 216

def self.best_service_match(matches)

  # The result hash we return to the caller
  result = {}

  # Bucket matches into matched service product names
  service_products = {}

  matches.select{ |m| m.has_key?('service.product') }.each do |m|
    # Group matches by product and normalize certainty
    cm = m.dup
    cm['service.certainty'] = ( m['service.certainty'] || DEFAULT_SERVICE_CERTAINTY ).to_f
    service_products[ cm['service.product'] ] ||= []
    service_products[ cm['service.product'] ]  << cm
  end

  # Unable to guess the service without service matches
  unless service_products.keys.length > 0
    return result
  end

  #
  # Select the best service name by combined certainty of all matches
  # within an service.product group. Multiple weak matches can
  # outweigh a single strong match by design.
  #
  ranked_service = service_products.keys.sort do |a,b|
    service_products[b].map{ |r| r['service.certainty'] }.inject(:+) <=>
    service_products[a].map{ |r| r['service.certainty'] }.inject(:+)
  end

  # Within the best match group, try to fill in missing attributes
  service_name = ranked_service.first

  # Find the best match within the winning group
  ranked_service_matches = service_products[service_name].sort do |a,b|
    b['service.certainty'] <=> a['service.certainty']
  end

  # Fill in missing service values in descending order of best match
  ranked_service_matches.each do |rm|
    rm.keys.select{ |k| k.index('service.') == 0 }.each do |k|
      result[k] ||= rm[k]
    end
  end

  result
end

.display_db_order ⇒ Object

Display the fingerprint databases in the order in which they will be used to match banners. This is useful for fingerprint tuning and debugging.



53
54
55
56
57
58
59
60
61
 
# File 'lib/recog/nizer.rb', line 53

def self.display_db_order
  self.load_db unless @@db_manager

  puts format('%s  %-22s  %-8s %s', 'Preference', 'Database', 'Type', 'Protocol')
  @@db_manager.databases.each do |db|
    puts format('%10.3f  %-22s  %-8s %s', db.preference, db.match_key,
                db.database_type, db.protocol)
  end
end

.load_db(path = nil) ⇒ Object

Load fingerprints from a specific file or directory This will not preserve any fingerprints that have already been loaded

Parameters:

  • path (String) (defaults to: nil) —

    Path to file or directory of XML fingerprints



30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/recog/nizer.rb', line 30

def self.load_db(path = nil)
  if path
    @@db_manager = Recog::DBManager.new(path)
  else
    @@db_manager = Recog::DBManager.new
  end

  # Sort the databases, no behavior or result change for those calling
  # Nizer.match or Nizer.multi_match as they have a single DB
  @@db_manager.databases.sort! { |a, b| b.preference <=> a.preference }
  @@db_sorted = true
end

.match(match_key, match_string) ⇒ see Fingerprint#match

2016.11 - Rewritten to be wrapper around #match_db_all, functionality and results must remain unchanged.

Locate a database that corresponds with the match_key and attempt to find a matching fingerprint, stopping at the first hit. Returns nil when no matching database or fingerprint is found.

Parameters:

  • match_key (String) —

    Fingerprint DB name, e.g. 'smb.native_os'

  • match_string (String) —

    String to match

Returns:



74
75
76
77
78
79
 
# File 'lib/recog/nizer.rb', line 74

def self.match(match_key, match_string)
  filter = { match_key: match_key, multi_match: false }
  matches = self.match_all_db(match_string, filter)

  matches[0] 
end

.match_all_db(match_string, filters = {}) ⇒ Array

Search all fingerprint dbs and attempt to find matching fingerprints. It will return the first match found unless the :multi_match option is used to request all matches. Returns an array of all matching fingerprints or an empty array.

Parameters:

  • match_string (String) —

    Service banner to match

  • filters (Hash) (defaults to: {}) —

    This hash contains filters used to limit the results to just those from specific types of fingerprints. The values that these filters match come from the 'fingerprints' top level element in the fingerprint DB XML or, in the case of 'protocol', this value can be overridden at the individual fingerprint level by setting a value for 'service.protocol'

    With the exception of 'match_key', the filters below match the 'fingerprints' attributes with the same name.

Options Hash (filters):

  • :match_key (String) —

    Value from XML 'matches' or file name

  • :database_type (String) —

    fprint db type: service, util.os, etc.

  • :protocol (String) —

    Protocol (ftp, smtp, etc.)

  • :multi_match (Boolean) —

    Return all matches instead of first

Returns:

  • (Array) —

    Array of Fingerprint#match or empty array



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/recog/nizer.rb', line 111

def self.match_all_db(match_string, filters = {})
  match_string = match_string.to_s.unpack('C*').pack('C*')
  matches = Array.new # array to hold all fingerprint matches

  self.load_db unless @@db_manager

  @@db_manager.databases.each do |db|
    next if filters[:match_key] && !filters[:match_key].eql?(db.match_key)
    next if filters[:database_type] && !filters[:database_type].eql?(db.database_type)
    db.fingerprints.each do |fp|
      m = fp.match(match_string)
      if m
        # Filter on protocol after match since each individual fp
        # can contain its own 'protocol' value that overrides the
        # one set at the DB level.
        matches.push(m) unless filters[:protocol] && !filters[:protocol].eql?(m['service.protocol'])
        return matches unless filters[:multi_match]
      end
    end
  end

  matches
end

.multi_match(match_key, match_string) ⇒ Array

Returns Array of Fingerprint#match or empty array.

Parameters:

  • match_key (String) —

    Fingerprint DB name, e.g. 'smb.native_os'

  • match_string (String) —

    String to match

Returns:

  • (Array) —

    Array of Fingerprint#match or empty array



85
86
87
88
 
# File 'lib/recog/nizer.rb', line 85

def self.multi_match(match_key, match_string)
  filter = { match_key: match_key, multi_match: true }
  self.match_all_db(match_string, filter)
end

.unload_db ⇒ Object

Destroy the current DBManager object



45
46
47
48
 
# File 'lib/recog/nizer.rb', line 45

def self.unload_db
  @@db_manager = nil
  @@db_sorted = false
end