Class: LogStash::Codecs::Nmap

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/codecs/nmap.rb

Overview

This codec is used to parse nmap.org/[namp] output data which is serialized in XML format. Nmap (“Network Mapper”) is a free and open source utility for network discovery and security auditing. For more information on nmap, see nmap.org/.

This codec can only be used for decoding data.

Event types are listed below

nmap_scan_metadata: An object containing top level information about the scan, including how many hosts were up, and how many were down. Useful for the case where you need to check if a DNS based hostname does not resolve, where both those numbers will be zero. nmap_host: One event is created per host. The full data covering an individual host, including open ports and traceroute information as a nested structure. nmap_port: One event is created per host/port. This duplicates data already in nmap_host: This was put in for the case where you want to model ports as separate documents in Elasticsearch (which Kibana prefers). nmap_traceroute_link: One of these is output per traceroute ‘connection’, with a from and a to object describing each hop. Note that traceroute hop data is not always correct due to the fact that each tracing ICMP packet may take a different route. Also very useful for Kibana visualizations.

Constant Summary collapse

EPOCH = 
LogStash::Timestamp.new(Time.at(0))

Instance Method Summary collapse

Instance Method Details

#de_keyword(value) ⇒ Object 



252
253
254
 
# File 'lib/logstash/codecs/nmap.rb', line 252

def de_keyword(value)
  value.is_a?(Symbol) ? value.to_s : value
end

#decode(data) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/logstash/codecs/nmap.rb', line 35

def decode(data)
  xml = Nmap::XML.parse(data)
  scan_id = SecureRandom.uuid

  base = {}
  base['arguments'] = xml.scanner.arguments
  base['version'] = xml.scanner.version
  base['scan_id'] = scan_id

  # This really needs to be put into ruby-nmap
  scan_host_stats = Hash[xml.instance_variable_get(:@doc).xpath('/nmaprun[@scanner="nmap"]/runstats/hosts').first.attributes.map {|k,v| [k,v.value.to_i]}]

  finished_info = Hash[xml.instance_variable_get(:@doc).xpath('/nmaprun[@scanner="nmap"]/runstats/finished').first.attributes.map {|k,v| [k,v.value] }]
  finished_info["elapsed"] = finished_info["elapsed"].to_f
  finished_info["time"] = timeify(Time.at(finished_info["time"].to_i))

  run_stats = hashify_struct(xml.run_stats.first)
  run_stats["finished"] = finished_info

  if @emit_scan_metadata
      yield LogStash::Event.new(base.merge({
        'type' => 'nmap_scan_metadata',
        'host_stats' => scan_host_stats,
        'run_stats' =>  run_stats,
        'start_time' => timeify(xml.scanner.start_time),
        'end_time' => run_stats["finished"]["time"]
      }))
  end

  xml.hosts.each_with_index do |host,idx|
    # Convert the host to a 'host_base' host event
    # This will be used for the later port/hop types
    host_base = hashify_host(host, xml).merge(base)


    # Pull out the detail
    ports = host.ports.map {|p| hashify_port(p)}
    traceroute = hashify_traceroute(host.traceroute)
    scan_host_id = scan_id + "-h#{idx}"

    if @emit_ports && ports
      ports.each.with_index do |port,idx|
        yield LogStash::Event.new(host_base.merge(
          'type' => 'nmap_port',
          'port' => port,
          'scan_host_id' => scan_host_id,
          'id' => scan_host_id+"-p#{idx}"
        ))
      end
    end

    if @emit_traceroute_links && traceroute && (hops = traceroute['hops'])
      hops.each_with_index do |hop,idx|
        next_hop = hops[idx+1]
        yield LogStash::Event.new(host_base.merge(
          'type' =>'nmap_traceroute_link',
          'from' => hop,
          'to' => next_hop,
          'rtt_diff' => (next_hop ? next_hop['rtt'] - hop['rtt'] : nil),
          'scan_host_id' => scan_host_id,
          'id' => scan_host_id+"-tl#{idx}"
        ))
      end
    end

    if @emit_hosts
      yield LogStash::Event.new(host_base.merge(
        'type' => 'nmap_host',
        'ports' => ports,
        'traceroute' => traceroute,
        'id' => scan_host_id
      ))
    end
  end
rescue StandardError => e
  raise e
  @logger.warn("An unexpected error occurred parsing nmap XML",
               :input => data,
               :message => e.message,
               :class => e.class.name,
               :backtrace => e.backtrace)
end

#dequote(str) ⇒ Object

Some strings have quoted values, we may want to remove leading/trailing quotes



268
269
270
271
 
# File 'lib/logstash/codecs/nmap.rb', line 268

def dequote(str)
  return nil unless str
  str.gsub(/\A"|"\Z/, '')
end

#hashify_host(host, xml) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/logstash/codecs/nmap.rb', line 118

def hashify_host(host, xml)
  scan_start = timeify(xml.scanner.start_time)

  h = {}
  h['start_time'] = timeify(host.start_time, scan_start)
  h['end_time'] = timeify(host.end_time, scan_start)

  # These two are actually different.
  # Address may contain a MAC, addresses will not AFAICT
  h['addresses'] = hashify_structs(host.addresses)
  h['address'] = host.address # str

  h['ip'] = host.ip # str
  h['ipv4'] = host.ipv4 # str
  h['ipv6'] = host.ipv6 # str
  h['mac'] = host.mac # str
  h['status'] = hashify_status(host.status)
  h['hostname'] = hashify_hostname(host.hostname)
  h['uptime'] = hashify_uptime(host.uptime)
  h['os'] = hashify_os(host.os)

  h
end

#hashify_hostname(hostname) ⇒ Object 



151
152
153
154
155
156
157
158
 
# File 'lib/logstash/codecs/nmap.rb', line 151

def hashify_hostname(hostname)
  return unless hostname

  {
    'name' => hostname.name, # str
    'type' => hostname.type, # str
  }
end

#hashify_os(os) ⇒ Object 



160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/logstash/codecs/nmap.rb', line 160

def hashify_os(os)
  return unless os

  # we need this nil guard here till https://github.com/sophsec/ruby-nmap/pull/41 is accepted
  fingerprint = os.fingerprint rescue nil
  {
    'ports_used' => os.ports_used,
    'fingerprint' => fingerprint,
    'classes' => hashify_os_classes(os.classes),
    'matches' => hashify_structs(os.matches)
  }
end

#hashify_os_classes(classes) ⇒ Object 



173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/logstash/codecs/nmap.rb', line 173

def hashify_os_classes(classes)
  return if !classes || classes.empty?

  classes.map do |klass|
    {
      'type' => klass.type.to_s, # returned as sym originally
      'vendor' => klass.vendor.to_s,
      'family' => klass.family.to_s,
      'gen' => klass.gen.to_s,
      'accuracy' => klass.accuracy # int
    }
  end
end

#hashify_port(port) ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/logstash/codecs/nmap.rb', line 213

def hashify_port(port)
  return unless port

  {
    'number' => port.number,
    'reason' => port.reason,
    'protocol' => port.protocol.to_s,
    'service' => hashify_service(port.service),
    'state' => port.state.to_s
  }
end

#hashify_service(service) ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/logstash/codecs/nmap.rb', line 196

def hashify_service(service)
  return unless service

  protocol = service.protocol rescue nil
  {
    'name' => service.name,
    'ssl' => service.ssl?,
    'protocol' => protocol,
    'product' => service.product,
    'hostname' => service.hostname, # This is just a string
    'device_type' => service.device_type,
    'fingerprint_method' => service.fingerprint_method.to_s,
    'fingerprint' => service.fingerprint,
    'confidence' => service.confidence
  }
end

#hashify_status(status) ⇒ Object 



142
143
144
145
146
147
148
149
 
# File 'lib/logstash/codecs/nmap.rb', line 142

def hashify_status(status)
  return unless status

  {
    'state' => status.state.to_s, # str
    'reason' => status.reason # str
  }
end

#hashify_struct(struct) ⇒ Object 



248
249
250
 
# File 'lib/logstash/codecs/nmap.rb', line 248

def hashify_struct(struct)
  Hash[struct.each_pair.map {|k,v| [k, de_keyword(v)]}]
end

#hashify_structs(structs) ⇒ Object 



244
245
246
 
# File 'lib/logstash/codecs/nmap.rb', line 244

def hashify_structs(structs)
  structs.map {|s| hashify_struct(s)}
end

#hashify_traceroute(traceroute) ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/logstash/codecs/nmap.rb', line 225

def hashify_traceroute(traceroute)
  return unless traceroute

  protocol = traceroute.protocol rescue nil
  {
    'port' => traceroute.port, # int
    'protocol' => protocol,
    'hops' => traceroute.map.with_index do |hop, idx|
      {
        'address' => hop.addr, # str
        'hostname' => hop.host, # str
        'ttl' => hop.ttl.to_i, # int
        'rtt' => hop.rtt.to_i, # int
        'index' => idx # int (for searching by distance)
      }
    end
  }
end

#hashify_uptime(uptime) ⇒ Object 



187
188
189
190
191
192
193
194
 
# File 'lib/logstash/codecs/nmap.rb', line 187

def hashify_uptime(uptime)
  return unless uptime

  {
    'seconds' => uptime.seconds,
    'last_boot' => timeify(uptime.last_boot)
  }
end

#register ⇒ Object 



31
32
 
# File 'lib/logstash/codecs/nmap.rb', line 31

def register
end

#timeify(time, default = nil) ⇒ Object 



257
258
259
260
261
262
263
264
265
 
# File 'lib/logstash/codecs/nmap.rb', line 257

def timeify(time, default=nil)
  timestamp = time ? LogStash::Timestamp.new(time) : nil
  # Sometimes the nmap parser returns the epoch when there's no time...
  if (!timestamp || timestamp <= EPOCH)
    default
  else
    timestamp
  end
end