Class: Trainspotter::Ingest::Parser

Inherits:
Object
  • Object
show all
Defined in:
app/jobs/trainspotter/ingest/parser.rb

Constant Summary collapse

TAG_PATTERN =

Pattern to extract request ID tag from tagged logger output e.g., "[5de6cb4c-4a8e-4d87-bafd-3ce2281e26f4] Started GET..." or " [req-id] Post Load (0.5ms)..." (tag after leading whitespace)

/^(?<leading_space>\s*)\[(?<request_id>[^\]]+)\]\s*/
PATTERNS =

Regex patterns for Rails log formats

{
  # Started GET "/posts" for 127.0.0.1 at 2024-01-06 10:00:00 +0000
  request_start: /^Started (?<method>GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) "(?<path>[^"]+)" for (?<ip>[\d.]+) at (?<timestamp>.+)$/,

  # Processing by PostsController#index as HTML
  # Also handles namespaced controllers like Trainspotter::LogsController
  processing: /^Processing by (?<controller>[\w:]+)#(?<action>\w+) as (?<format>\w+|\*\/\*)/,

  # Parameters: {"session"=>{"email"=>"[email protected]", "password"=>"[FILTERED]"}}
  params: /^\s*Parameters: (?<params_string>.+)$/,

  # Post Load (0.5ms)  SELECT "posts".* FROM "posts"
  sql: /^\s*(?<name>[\w\s]+) \((?<duration>[\d.]+)ms\)\s+(?<query>.+)$/,

  # Rendered posts/index.html.erb within layouts/application (Duration: 5.0ms | GC: 0.0ms)
  render: /^\s*Rendered (?<template>[^\s]+)(?: within (?<layout>[^\s]+))? \(Duration: (?<duration>[\d.]+)ms/,

  # Completed 200 OK in 50ms (Views: 40.0ms | ActiveRecord: 5.0ms | Allocations: 1234)
  request_end: /^Completed (?<status>\d+) .+ in (?<duration>[\d.]+)ms/
}.freeze

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Parser

Returns a new instance of Parser.



31
32
33
34
35
 
# File 'app/jobs/trainspotter/ingest/parser.rb', line 31

def initialize
  @groups_by_id = {}
  @current_untagged_group = nil
  @groups = []
end

Instance Method Details

#groups ⇒ Object 



74
75
76
 
# File 'app/jobs/trainspotter/ingest/parser.rb', line 74

def groups
  @groups.dup
end

#parse_file(path, limit: nil) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
 
# File 'app/jobs/trainspotter/ingest/parser.rb', line 53

def parse_file(path, limit: nil)
  reset_state

  File.foreach(path).with_index do |line, index|
    break if limit && index >= limit
    parse_line(line)
  end

  finalize_all_groups
  @groups
end

#parse_line(line) ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'app/jobs/trainspotter/ingest/parser.rb', line 37

def parse_line(line)
  line = sanitize_encoding(line.chomp)
  return nil if line.strip.empty?

  request_id, content = extract_tag(line)
  entry = identify_entry(content)

  if request_id
    handle_tagged_entry(request_id, entry)
  else
    handle_untagged_entry(entry)
  end

  entry
end

#parse_lines(lines) ⇒ Object 



65
66
67
68
69
70
71
72
 
# File 'app/jobs/trainspotter/ingest/parser.rb', line 65

def parse_lines(lines)
  reset_state

  lines.each { |line| parse_line(line) }

  finalize_all_groups
  @groups
end