Class: DecisionAgent::Monitoring::AlertManager

Inherits:
Object
  • Object
show all
Includes:
MonitorMixin
Defined in:
lib/decision_agent/monitoring/alert_manager.rb

Overview

Alert manager for anomaly detection and notifications

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(metrics_collector:) ⇒ AlertManager

Returns a new instance of AlertManager.



13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 13

def initialize(metrics_collector:)
  super()
  @metrics_collector = metrics_collector
  @rules = []
  @alerts = []
  @alert_handlers = []
  @check_interval = 60 # seconds
  @monitoring_thread = nil
  @rule_counter = 0
  freeze_config
end

Instance Attribute Details

#alerts ⇒ Object (readonly)

Returns the value of attribute alerts.



11
12
13
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 11

def alerts
  @alerts
end

#rules ⇒ Object (readonly)

Returns the value of attribute rules.



11
12
13
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 11

def rules
  @rules
end

Class Method Details

.error_spike(threshold: 10, time_window: 300) ⇒ Object 



175
176
177
178
179
180
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 175

def self.error_spike(threshold: 10, time_window: 300)
  lambda do |stats|
    recent_errors = stats.dig(:errors, :total) || 0
    recent_errors > threshold
  end
end

.high_error_rate(threshold: 0.1) ⇒ Object

Built-in alert conditions



151
152
153
154
155
156
157
158
159
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 151

def self.high_error_rate(threshold: 0.1)
  lambda do |stats|
    total_ops = stats.dig(:performance, :total_operations) || 0
    return false if total_ops.zero?

    success_rate = stats.dig(:performance, :success_rate) || 1.0
    (1.0 - success_rate) > threshold
  end
end

.high_latency(threshold_ms: 1000) ⇒ Object 



168
169
170
171
172
173
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 168

def self.high_latency(threshold_ms: 1000)
  lambda do |stats|
    p95 = stats.dig(:performance, :p95_duration_ms)
    p95 && p95 > threshold_ms
  end
end

.low_confidence(threshold: 0.5) ⇒ Object 



161
162
163
164
165
166
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 161

def self.low_confidence(threshold: 0.5)
  lambda do |stats|
    avg_confidence = stats.dig(:decisions, :avg_confidence)
    avg_confidence && avg_confidence < threshold
  end
end

Instance Method Details

#acknowledge_alert(alert_id, acknowledged_by: "system") ⇒ Object

Acknowledge alert



119
120
121
122
123
124
125
126
127
128
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 119

def acknowledge_alert(alert_id, acknowledged_by: "system")
  synchronize do
    alert = @alerts.find { |a| a[:id] == alert_id }
    if alert
      alert[:status] = :acknowledged
      alert[:acknowledged_by] = acknowledged_by
      alert[:acknowledged_at] = Time.now.utc
    end
  end
end

#active_alerts ⇒ Object

Get active alerts



105
106
107
108
109
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 105

def active_alerts
  synchronize do
    @alerts.select { |a| a[:status] == :active }
  end
end

#add_handler(&block) ⇒ Object

Register alert handler



61
62
63
64
65
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 61

def add_handler(&block)
  synchronize do
    @alert_handlers << block
  end
end

#add_rule(name:, condition:, severity: :warning, threshold: nil, message: nil, cooldown: 300) ⇒ Object

Define an alert rule



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 26

def add_rule(name:, condition:, severity: :warning, threshold: nil, message: nil, cooldown: 300)
  synchronize do
    rule = {
      id: generate_rule_id(name),
      name: name,
      condition: condition,
      severity: severity,
      threshold: threshold,
      message: message || "Alert: #{name}",
      cooldown: cooldown,
      last_triggered: nil,
      enabled: true
    }

    @rules << rule
    rule
  end
end

#all_alerts(limit: 100) ⇒ Object

Get all alerts



112
113
114
115
116
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 112

def all_alerts(limit: 100)
  synchronize do
    @alerts.last(limit)
  end
end

#check_rules ⇒ Object

Manually check all rules



93
94
95
96
97
98
99
100
101
102
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 93

def check_rules
  stats = @metrics_collector.statistics

  @rules.each do |rule|
    next unless rule[:enabled]
    next if in_cooldown?(rule)

    trigger_alert(rule, stats) if evaluate_condition(rule[:condition], stats)
  end
end

#clear_old_alerts(older_than: 86_400) ⇒ Object

Clear old alerts



143
144
145
146
147
148
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 143

def clear_old_alerts(older_than: 86_400)
  synchronize do
    cutoff = Time.now.utc - older_than
    @alerts.reject! { |a| a[:triggered_at] < cutoff && a[:status] != :active }
  end
end

#remove_rule(rule_id) ⇒ Object

Remove a rule



46
47
48
49
50
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 46

def remove_rule(rule_id)
  synchronize do
    @rules.reject! { |r| r[:id] == rule_id }
  end
end

#resolve_alert(alert_id, resolved_by: "system") ⇒ Object

Resolve alert



131
132
133
134
135
136
137
138
139
140
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 131

def resolve_alert(alert_id, resolved_by: "system")
  synchronize do
    alert = @alerts.find { |a| a[:id] == alert_id }
    if alert
      alert[:status] = :resolved
      alert[:resolved_by] = resolved_by
      alert[:resolved_at] = Time.now.utc
    end
  end
end

#start_monitoring(interval: 60) ⇒ Object

Start monitoring



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 68

def start_monitoring(interval: 60)
  synchronize do
    return if @monitoring_thread&.alive?

    @check_interval = interval
    @monitoring_thread = Thread.new do
      loop do
        check_rules
        sleep @check_interval
      rescue StandardError => e
        warn "Alert monitoring error: #{e.message}"
      end
    end
  end
end

#stop_monitoring ⇒ Object

Stop monitoring



85
86
87
88
89
90
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 85

def stop_monitoring
  synchronize do
    @monitoring_thread&.kill
    @monitoring_thread = nil
  end
end

#toggle_rule(rule_id, enabled) ⇒ Object

Enable/disable rule



53
54
55
56
57
58
 
# File 'lib/decision_agent/monitoring/alert_manager.rb', line 53

def toggle_rule(rule_id, enabled)
  synchronize do
    rule = @rules.find { |r| r[:id] == rule_id }
    rule[:enabled] = enabled if rule
  end
end