Module: GreenHat::Formatters

Included in:
Thing
Defined in:
lib/greenhat/thing/formatters/raw.rb,
lib/greenhat/thing/formatters/json.rb,
lib/greenhat/thing/formatters/nginx.rb,
lib/greenhat/thing/formatters/table.rb,
lib/greenhat/thing/formatters/dotenv.rb,
lib/greenhat/thing/formatters/format.rb,
lib/greenhat/thing/formatters/free_m.rb,
lib/greenhat/thing/formatters/syslog.rb,
lib/greenhat/thing/formatters/api_json.rb,
lib/greenhat/thing/formatters/clean_raw.rb,
lib/greenhat/thing/formatters/time_json.rb,
lib/greenhat/thing/formatters/shellwords.rb,
lib/greenhat/thing/formatters/time_space.rb,
lib/greenhat/thing/formatters/bracket_log.rb,
lib/greenhat/thing/formatters/gitlab_status.rb,
lib/greenhat/thing/formatters/multiline_json.rb,
lib/greenhat/thing/formatters/gitlab_ctl_tail.rb,
lib/greenhat/thing/formatters/json_shellwords.rb,
lib/greenhat/thing/formatters/time_shellwords.rb,
lib/greenhat/thing/formatters/colon_split_strip.rb

Overview

Log

Instance Method Summary collapse

Instance Method Details

#dmesg_split(raw) ⇒ Object 



34
35
36
37
38
39
40
41
42
 
# File 'lib/greenhat/thing/formatters/bracket_log.rb', line 34

def dmesg_split(raw)
  Shellwords.split(raw).each_with_object({}) do |x, h|
    key, value = x.split('=')
    next if value.nil?

    h[key] = value.numeric? ? value.to_f : value
    h[key] = 0.0 if h[key].numeric? && h[key].zero?
  end
end

#flatten_hash(param, prefix = nil) ⇒ Object 



27
28
29
30
31
 
# File 'lib/greenhat/thing/formatters/api_json.rb', line 27

def flatten_hash(param, prefix = nil)
  param.each_pair.reduce({}) do |a, (k, v)|
    v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v)
  end
end

#format_api_json ⇒ Object

Formatters

Primarily for gitlab-rails/api_json.log



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/greenhat/thing/formatters/api_json.rb', line 9

def format_api_json
  self.result = raw.map do |row|
    result = Oj.load row

    # Parsing Time
    format_json_traverse result

    flatten_hash(result).sort.to_h
  rescue StandardError => e
    # TODO: Background Logger?
    e.message
    LogBot.warn('JSON Parse', e.message)
    next
  end

  :ok
end

#format_bracket_log ⇒ Object

Formatters for bracket logs (dmesg, sos)



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/greenhat/thing/formatters/bracket_log.rb', line 6

def format_bracket_log
  self.result = raw.map do |row|
    next if row.empty? || row == "\n"

    result = {}
    time, output = row.split(']', 2)
    result[:time] = Time.parse time.split('[', 2).last.strip

    if output.include? ': '
      category, raw = output.split(': ', 2)
      result[:category] = category.strip

      result.merge! dmesg_split(raw) if raw.include? '='

      result[:message] = raw.strip
    else
      result[:message] = output
    end

    result
  rescue StandardError => e
    # TODO: Background logger
    LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}")
  end

  self.result.compact!
end

#format_clean_raw ⇒ Object

Remove Comments / Empty Lines



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/greenhat/thing/formatters/clean_raw.rb', line 6

def format_clean_raw
  staging = raw.clone

  # Empty
  staging.reject!(&:empty?)

  # Commented
  staging.reject! { |x| x =~ /#.*$/ }

  self.result = if staging.empty?
                  raw
                else
                  staging
                end
end

#format_colon_split_strip ⇒ Object

Formatters for single json blobs in entire file



6
7
8
9
10
 
# File 'lib/greenhat/thing/formatters/colon_split_strip.rb', line 6

def format_colon_split_strip
  self.result = raw.map do |row|
    row.split(':', 2).map(&:strip)
  end.to_h
end

#format_dotenv ⇒ Object

Formatters for Dmesg



6
7
8
 
# File 'lib/greenhat/thing/formatters/dotenv.rb', line 6

def format_dotenv
  self.result = Dotenv::Parser.new(raw.join("\n")).call
end

#format_free_m ⇒ Object

Get Split Memory Table



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/greenhat/thing/formatters/free_m.rb', line 6

def format_free_m
  # Headers to Readable Symbol
  headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
    x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
  end.map(&:to_sym)

  # Add Kind
  headers.unshift(:kind)

  final = []

  # Put fields into a Hash based on Location/Key
  raw[1..].map(&:split).each do |row|
    result = {}
    row.each_with_index do |detail, i|
      result[headers[i]] = detail.split(':').first
    end
    final.push result
  end

  self.result = final
end

#format_gitlab_status ⇒ Object

Remove Comments / Empty Lines



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/greenhat/thing/formatters/gitlab_status.rb', line 6

def format_gitlab_status
  final = {}
  raw.each do |row|
    list = row.split('; ').map do |entry|
      status, service, pid_uptime = entry.split(': ')

      {
        status: status,
        name: service,
        pid_uptime: pid_uptime
      }
    end

    final[list.first.name] = list
  end

  self.result = final
end

#format_gitlab_tail ⇒ Object

Gitlab Tail Formatter



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/greenhat/thing/formatters/gitlab_ctl_tail.rb', line 8

def format_gitlab_tail
  # Revert to raw for cats
  self.kind = :raw

  output = {}
  current_log = nil

  raw.each do |line|
    next if line.blank?

    if line.include? '==>'
      current_log = /==> (.+?) <==/.match(line).captures.first
    else
      output[current_log] ||= []
      output[current_log].push line
    end
  end

  # Remove Empty Entries
  output.reject { |_k, v| v.empty? }

  # Root Dir
  root_dir = "#{$TMP}/#{name}"
  Dir.mkdir(root_dir)

  # Write Files / Create Things
  output.each do |k, v|
    file_name = k.gsub('/var/log/gitlab/', '')

    dir = "#{root_dir}/#{file_name.split('/').first}"
    Dir.mkdir(dir) unless File.exist?(dir)

    File.write("#{root_dir}/#{file_name}", v.join("\n"))

    # Thing Setup
    archive.things_create(file: "#{root_dir}/#{file_name}").setup
  end

  # Link
  self.result = raw
end

#format_json ⇒ Object

Formatters Fallback to String



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/greenhat/thing/formatters/json.rb', line 9

def format_json
  self.result = raw.map do |row|
    result = begin
      Oj.load row
    rescue EncodingError
      { message: row }
    end

    # Parsing Time
    format_json_traverse result

    result.sort.to_h
  rescue StandardError => e
    # TODO: Background Logger?
    e.message
    LogBot.warn('JSON Parse', e.message)
    next
  end

  :ok
end

#format_json_shell ⇒ Object

Formatters



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/greenhat/thing/formatters/json_shellwords.rb', line 8

def format_json_shell
  self.result = raw.map do |row|
    result = begin
      Oj.load row
    rescue EncodingError
      json_shellwords_fallback row
    end

    # Parsing Time
    format_json_traverse result

    result.sort.to_h
  rescue StandardError => e
    LogBot.warn('JSON Parse', e.message)
    next
  end

  :ok
end

#format_json_time(result) ⇒ Object

Check for Common Fields



44
45
46
47
48
49
50
51
 
# File 'lib/greenhat/thing/formatters/json.rb', line 44

def format_json_time(result)
  result.time = format_time_parse(result.time) if result.key? :time
  result.created_at = format_time_parse(result.created_at) if result.key? :created_at
  result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
rescue StandardError => e
  LogBot.warn('JSON Time Parse', e.message)
  true
end

#format_json_traverse(result) ⇒ Object

Recursively Navigate



33
34
35
36
37
38
39
40
41
 
# File 'lib/greenhat/thing/formatters/json.rb', line 33

def format_json_traverse(result)
  format_json_time(result)

  result.each do |_key, value|
    next unless value.instance_of? Hash

    format_json_traverse(value)
  end
end

#format_multiline_json ⇒ Object

Formatters for single json blobs in entire file



6
7
8
 
# File 'lib/greenhat/thing/formatters/multiline_json.rb', line 6

def format_multiline_json
  self.result = Oj.load raw.join
end

#format_nginx ⇒ Object

Formatters



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/greenhat/thing/formatters/nginx.rb', line 8

def format_nginx
  self.result = raw.map do |row|
    ip, _sym, remote_user, rest = row.split(' ', 4)

    time, rest = rest.split(']', 2)
    time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')

    verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)

    method, path, http_version = verb.split

    {
      remote_addr: ip,
      remote_user: remote_user,
      method: method,
      path: path,
      status: status,
      bytes: bytes,
      http_version: http_version,
      http_referer: http_referer,
      http_user_agent: http_user_agent,
      gzip_ratio: gzip_ratio,
      time: time
    }
  end

  :ok
end

#format_raw ⇒ Object 



13
14
15
 
# File 'lib/greenhat/thing/formatters/raw.rb', line 13

def format_raw
  self.result = raw
end

#format_shellwords ⇒ Object

Formatters Not Handled



8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/greenhat/thing/formatters/shellwords.rb', line 8

def format_shellwords
  self.result = raw.map do |row|
    result = Shellwords.split(row).each_with_object({}) do |x, h|
      key, value = x.split('=')
      next if value.nil?

      h[key] = value.numeric? ? value.to_f : value
    end
    result.time = Time.parse result.time if result.key? 'time'

    result
  end
end

#format_syslog ⇒ Object

Formatters for bracket logs (dmesg, sos)



6
7
8
9
10
11
12
13
14
 
# File 'lib/greenhat/thing/formatters/syslog.rb', line 6

def format_syslog
  self.result = raw.map do |row|
    next if row.empty? || row == "\n"

    format_syslog_row(row)
  end

  result.compact!
end

#format_syslog_row(row) ⇒ Object

Split / Parse Time TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/greenhat/thing/formatters/syslog.rb', line 18

def format_syslog_row(row)
  month, day, time, device, service, message = row.split(' ', 6)
  service.gsub!(':', '')
  pid = service.match(/\[(.*?)\]/)&.captures&.first

  {
    time: format_time_parse("#{month} #{day} #{time}"),
    device: device,
    service: service,
    message: message,
    pid: pid
  }

# Return everything incase of error
rescue StandardError => e
  LogBot.warn('SysLog Parse', e.message)
  {
    message: row
  }
end

#format_table ⇒ Object

Formatters for Dmesg



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/greenhat/thing/formatters/table.rb', line 6

def format_table
  # Headers to Readable Symbol
  headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
    x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
  end.map(&:to_sym)

  final = []

  # Put fields into a Hash based on Location/Key
  raw[1..].map(&:split).each do |row|
    result = {}
    row.each_with_index do |detail, i|
      result[headers[i]] = detail
    end
    final.push result
  end

  self.result = final
end

#format_time_json ⇒ Object

Time Space JSON 2021-05-04_18:29:28.66542 Ctrl-C to stop”



9
10
11
12
13
14
15
16
17
18
 
# File 'lib/greenhat/thing/formatters/time_json.rb', line 9

def format_time_json
  self.result = raw.map do |row|
    time, msg = row.split(' ', 2)

    result =  Oj.load msg
    result.time = time

    result
  end
end

#format_time_parse(time) ⇒ Object

Handle Epoch Timestamps as well as string timestamps



54
55
56
57
58
59
60
 
# File 'lib/greenhat/thing/formatters/json.rb', line 54

def format_time_parse(time)
  if time.numeric?
    Time.at time
  else
    Time.parse time
  end
end

#format_time_shellwords ⇒ Object

Formatters Not Handled



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/greenhat/thing/formatters/time_shellwords.rb', line 8

def format_time_shellwords
  self.result = raw.map do |row|
    time, msg = row.split(' ', 2)

    result = Shellwords.split(msg).each_with_object({}) do |x, h|
      key, value = x.split('=')
      next if value.nil?

      h[key] = value.numeric? ? value.to_f : value
    end

    # Timestamp Parsing
    result.ts = Time.parse result.ts if result.key? 'ts'
    result.time = Time.parse time

    result
  end
end

#format_time_space ⇒ Object

Formatters Not Handled



8
9
10
11
12
13
14
15
16
17
 
# File 'lib/greenhat/thing/formatters/time_space.rb', line 8

def format_time_space
  self.result = raw.map do |row|
    time, msg = row.split(' ', 2)

    {
      time: Time.parse(time),
      msg: msg
    }
  end
end

#json_shellwords_fallback(row) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/greenhat/thing/formatters/json_shellwords.rb', line 28

def json_shellwords_fallback(row)
  result = Shellwords.split(row).each_with_object({}) do |x, h|
    key, value = x.split('=')
    next if value.nil?

    h[key] = value.numeric? ? value.to_f : value
  end
  result.time = Time.parse result.time if result.key? 'time'

  result
end

#log_format ⇒ Object 



4
5
6
 
# File 'lib/greenhat/thing/formatters/format.rb', line 4

def log_format
  self.result = send(SuperLog.type?(path)) if log_type
end

#log_type ⇒ Object 



8
9
10
 
# File 'lib/greenhat/thing/formatters/format.rb', line 8

def log_type
  SuperLog.type?(path)
end

#mia ⇒ Object

Formatters Not Handled



8
9
10
11
 
# File 'lib/greenhat/thing/formatters/raw.rb', line 8

def mia
  # TODO: Background Logger
  LogBot.warn('Log Format', "No Formatter for #{name}::#{path}")
end