Class: LogStash::Codecs::Nmap

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/codecs/nmap.rb

Overview

This codec may be used to decode only

Event types are listed below

nmap_scan_metadata: An object containing top level information about the scan, including how many hosts were up, and how many were down. Useful for the case where you need to check if a DNS based hostname does not resolve, where both those numbers will be zero. nmap_host: One event is created per host. The full data covering an individual host, including open ports and traceroute information as a nested structure. nmap_port: One event is created per host/port. This duplicates data already in nmap_host: This was put in for the case where you want to model ports as separate documents in Elasticsearch (which Kibana prefers). nmap_traceroute_link: One of these is output per traceroute ‘connection’, with a from and a to object describing each hop. Note that traceroute hop data is not always correct due to the fact that each tracing ICMP packet may take a different route. Also very useful for Kibana visualizations.

Constant Summary collapse

EPOCH = 
LogStash::Timestamp.new(Time.at(0))

Instance Method Summary collapse

Instance Method Details

#de_keyword(value) ⇒ Object 



240
241
242
 
# File 'lib/logstash/codecs/nmap.rb', line 240

def de_keyword(value)
  value.is_a?(Symbol) ? value.to_s : value
end

#decode(data) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/logstash/codecs/nmap.rb', line 32

def decode(data)
  xml = Nmap::XML.parse(data)
  scan_id = SecureRandom.uuid

  base = {}
  base['arguments'] = xml.scanner.arguments
  base['version'] = xml.scanner.version
  base['scan_id'] = scan_id

  # This really needs to be put into ruby-nmap
  scan_host_stats = Hash[xml.instance_variable_get(:@doc).xpath('/nmaprun[@scanner="nmap"]/runstats/hosts').first.attributes.map {|k,v| [k,v.value.to_i]}]

  if @emit_scan_metadata
      yield LogStash::Event.new(base.merge({
        'type' => 'nmap_scan_metadata',
        'host_stats' => scan_host_stats,
        'run_stats' =>  xml.run_stats.first
      }))
  end

  xml.hosts.each_with_index do |host,idx|
    # Convert the host to a 'host_base' host event
    # This will be used for the later port/hop types
    host_base = hashify_host(host, xml).merge(base)


    # Pull out the detail
    ports = host.ports.map {|p| hashify_port(p)}
    traceroute = hashify_traceroute(host.traceroute)
    scan_host_id = scan_id + "-h#{idx}"

    if @emit_ports && ports
      ports.each.with_index do |port,idx|
        yield LogStash::Event.new(host_base.merge(
          'type' => 'nmap_port',
          'port' => port,
          'scan_host_id' => scan_host_id,
          'id' => scan_host_id+"-p#{idx}"
        ))
      end
    end

    if @emit_traceroute_links && traceroute && (hops = traceroute['hops'])
      hops.each_with_index do |hop,idx|
        next_hop = hops[idx+1]
        yield LogStash::Event.new(host_base.merge(
          'type' =>'nmap_traceroute_link',
          'from' => hop,
          'to' => next_hop,
          'rtt_diff' => (next_hop ? next_hop['rtt'] - hop['rtt'] : nil),
          'scan_host_id' => scan_host_id,
          'id' => scan_host_id+"-tl#{idx}"
        ))
      end
    end

    if @emit_hosts
      yield LogStash::Event.new(host_base.merge(
        'type' => 'nmap_host',
        'ports' => ports,
        'traceroute' => traceroute,
        'id' => scan_host_id
      ))
    end
  end
rescue StandardError => e
  raise e
  @logger.warn("An unexpected error occurred parsing nmap XML",
               :input => data,
               :message => e.message,
               :class => e.class.name,
               :backtrace => e.backtrace)
end

#dequote(str) ⇒ Object

Some strings have quoted values, we may want to remove leading/trailing quotes



256
257
258
259
 
# File 'lib/logstash/codecs/nmap.rb', line 256

def dequote(str)
  return nil unless str
  str.gsub(/\A"|"\Z/, '')
end

#hashify_host(host, xml) ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/logstash/codecs/nmap.rb', line 106

def hashify_host(host, xml)
  scan_start = timeify(xml.scanner.start_time)

  h = {}
  h['start_time'] = timeify(host.start_time, scan_start)
  h['end_time'] = timeify(host.end_time, scan_start)

  # These two are actually different.
  # Address may contain a MAC, addresses will not AFAICT
  h['addresses'] = hashify_structs(host.addresses)
  h['address'] = host.address # str

  h['ip'] = host.ip # str
  h['ipv4'] = host.ipv4 # str
  h['ipv6'] = host.ipv6 # str
  h['mac'] = host.mac # str
  h['status'] = hashify_status(host.status)
  h['hostname'] = hashify_hostname(host.hostname)
  h['uptime'] = hashify_uptime(host.uptime)
  h['os'] = hashify_os(host.os)

  h
end

#hashify_hostname(hostname) ⇒ Object 



139
140
141
142
143
144
145
146
 
# File 'lib/logstash/codecs/nmap.rb', line 139

def hashify_hostname(hostname)
  return unless hostname

  {
    'name' => hostname.name, # str
    'type' => hostname.type, # str
  }
end

#hashify_os(os) ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/logstash/codecs/nmap.rb', line 148

def hashify_os(os)
  return unless os

  # we need this nil guard here till https://github.com/sophsec/ruby-nmap/pull/41 is accepted
  fingerprint = os.fingerprint rescue nil
  {
    'ports_used' => os.ports_used,
    'fingerprint' => fingerprint,
    'classes' => hashify_os_classes(os.classes),
    'matches' => hashify_structs(os.matches)
  }
end

#hashify_os_classes(classes) ⇒ Object 



161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/logstash/codecs/nmap.rb', line 161

def hashify_os_classes(classes)
  return if !classes || classes.empty?

  classes.map do |klass|
    {
      'type' => klass.type.to_s, # returned as sym originally
      'vendor' => klass.vendor.to_s,
      'family' => klass.family.to_s,
      'gen' => klass.gen.to_s,
      'accuracy' => klass.accuracy # int
    }
  end
end

#hashify_port(port) ⇒ Object 



201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/logstash/codecs/nmap.rb', line 201

def hashify_port(port)
  return unless port

  {
    'number' => port.number,
    'reason' => port.reason,
    'protocol' => port.protocol.to_s,
    'service' => hashify_service(port.service),
    'state' => port.state.to_s
  }
end

#hashify_service(service) ⇒ Object 



184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/logstash/codecs/nmap.rb', line 184

def hashify_service(service)
  return unless service

  protocol = service.protocol rescue nil
  {
    'name' => service.name,
    'ssl' => service.ssl?,
    'protocol' => protocol,
    'product' => service.product,
    'hostname' => service.hostname, # This is just a string
    'device_type' => service.device_type,
    'fingerprint_method' => service.fingerprint_method.to_s,
    'fingerprint' => service.fingerprint,
    'confidence' => service.confidence
  }
end

#hashify_status(status) ⇒ Object 



130
131
132
133
134
135
136
137
 
# File 'lib/logstash/codecs/nmap.rb', line 130

def hashify_status(status)
  return unless status

  {
    'state' => status.state.to_s, # str
    'reason' => status.reason # str
  }
end

#hashify_struct(struct) ⇒ Object 



236
237
238
 
# File 'lib/logstash/codecs/nmap.rb', line 236

def hashify_struct(struct)
  Hash[struct.each_pair.map {|k,v| [k, de_keyword(v)]}]
end

#hashify_structs(structs) ⇒ Object 



232
233
234
 
# File 'lib/logstash/codecs/nmap.rb', line 232

def hashify_structs(structs)
  structs.map {|s| hashify_struct(s)}
end

#hashify_traceroute(traceroute) ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/logstash/codecs/nmap.rb', line 213

def hashify_traceroute(traceroute)
  return unless traceroute

  protocol = traceroute.protocol rescue nil
  {
    'port' => traceroute.port, # int
    'protocol' => protocol,
    'hops' => traceroute.map.with_index do |hop, idx|
      {
        'address' => hop.addr, # str
        'hostname' => hop.host, # str
        'ttl' => hop.ttl.to_i, # int
        'rtt' => hop.rtt.to_i, # int
        'index' => idx # int (for searching by distance)
      }
    end
  }
end

#hashify_uptime(uptime) ⇒ Object 



175
176
177
178
179
180
181
182
 
# File 'lib/logstash/codecs/nmap.rb', line 175

def hashify_uptime(uptime)
  return unless uptime

  {
    'seconds' => uptime.seconds,
    'last_boot' => timeify(uptime.last_boot)
  }
end

#register ⇒ Object 



28
29
 
# File 'lib/logstash/codecs/nmap.rb', line 28

def register
end

#timeify(time, default = nil) ⇒ Object 



245
246
247
248
249
250
251
252
253
 
# File 'lib/logstash/codecs/nmap.rb', line 245

def timeify(time, default=nil)
  timestamp = time ? LogStash::Timestamp.new(time) : nil
  # Sometimes the nmap parser returns the epoch when there's no time...
  if (!timestamp || timestamp <= EPOCH)
    default
  else
    timestamp
  end
end