Class: DecisionAgent::Simulation::ShadowTestEngine

Inherits:
Object
  • Object
show all
Defined in:
lib/decision_agent/simulation/shadow_test_engine.rb

Overview

Engine for shadow testing - comparing new rules against production without affecting outcomes

Defined Under Namespace

Classes: ProgressTracker

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(production_agent:, version_manager: nil) ⇒ ShadowTestEngine

Returns a new instance of ShadowTestEngine.



11
12
13
14
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 11

def initialize(production_agent:, version_manager: nil)
  @production_agent = production_agent
  @version_manager = version_manager || Versioning::VersionManager.new
end

Instance Attribute Details

#production_agent ⇒ Object (readonly)

Returns the value of attribute production_agent.



9
10
11
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 9

def production_agent
  @production_agent
end

#version_manager ⇒ Object (readonly)

Returns the value of attribute version_manager.



9
10
11
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 9

def version_manager
  @version_manager
end

Instance Method Details

#add_differences(result, production_decision, shadow_decision) ⇒ Object 



70
71
72
73
74
75
76
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 70

def add_differences(result, production_decision, shadow_decision)
  result[:differences] = {
    decision_mismatch: true,
    production_explanations: production_decision&.explanations || [],
    shadow_explanations: shadow_decision&.explanations || []
  }
end

#batch_test(contexts:, shadow_version:, options: {}) ⇒ Hash

Batch shadow test multiple contexts

Parameters:

  • contexts (Array<Hash>) —

    Array of contexts to test

  • shadow_version (String, Integer, Hash) —

    Shadow rule version

  • options (Hash) (defaults to: {}) —

    Test options

    • :parallel [Boolean] Use parallel execution (default: true)
    • :thread_count [Integer] Number of threads (default: 4)
    • :progress_callback [Proc] Progress callback

Returns:

  • (Hash) —

    Batch shadow test results



86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 86

def batch_test(contexts:, shadow_version:, options: {})
  options = {
    parallel: true,
    thread_count: 4,
    progress_callback: nil,
    track_differences: true,
    record_results: false
  }.merge(options)

  shadow_agent = build_shadow_agent(shadow_version)
  results = execute_contexts_with_progress(contexts, shadow_version, shadow_agent, options)

  build_batch_report(results)
end

#build_comparison_result(context, production_decision, shadow_decision) ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 57

def build_comparison_result(context, production_decision, shadow_decision)
  {
    context: context.to_h,
    production_decision: production_decision&.decision,
    production_confidence: production_decision&.confidence || 0.0,
    shadow_decision: shadow_decision&.decision,
    shadow_confidence: shadow_decision&.confidence || 0.0,
    matches: production_decision&.decision == shadow_decision&.decision,
    confidence_delta: (shadow_decision&.confidence || 0.0) - (production_decision&.confidence || 0.0),
    timestamp: Time.now.utc.iso8601
  }
end

#execute_contexts_with_progress(contexts, shadow_version, shadow_agent, options) ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 101

def execute_contexts_with_progress(contexts, shadow_version, shadow_agent, options)
  results = []
  mutex = Mutex.new
  progress_tracker = ProgressTracker.new(contexts.size, options[:progress_callback])

  if options[:parallel] && contexts.size > 1
    execute_parallel(contexts, shadow_agent, options, mutex) do |result|
      mutex.synchronize do
        results << result
        progress_tracker.increment
      end
    end
  else
    execute_sequential_contexts(contexts, shadow_version, options, results, progress_tracker)
  end

  results
end

#execute_production_decision(context) ⇒ Object 



44
45
46
47
48
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 44

def execute_production_decision(context)
  @production_agent.decide(context: context)
rescue NoEvaluationsError
  nil
end

#execute_sequential_contexts(contexts, shadow_version, options, results, progress_tracker) ⇒ Object 



120
121
122
123
124
125
126
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 120

def execute_sequential_contexts(contexts, shadow_version, options, results, progress_tracker)
  contexts.each_with_index do |context, _index|
    result = test(context: context, shadow_version: shadow_version, options: options)
    results << result
    progress_tracker.increment
  end
end

#execute_shadow_decision(context, shadow_version) ⇒ Object 



50
51
52
53
54
55
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 50

def execute_shadow_decision(context, shadow_version)
  shadow_agent = build_shadow_agent(shadow_version)
  shadow_agent.decide(context: context)
rescue NoEvaluationsError
  nil
end

#normalize_context(context) ⇒ Object 



40
41
42
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 40

def normalize_context(context)
  context.is_a?(Context) ? context : Context.new(context)
end

#test(context:, shadow_version:, options: {}) ⇒ Hash

Execute shadow test - compare shadow version against production

Parameters:

  • context (Hash, Context) —

    Context for decision

  • shadow_version (String, Integer, Hash) —

    Shadow rule version to test

  • options (Hash) (defaults to: {}) —

    Test options

    • :track_differences [Boolean] Track and return differences (default: true)
    • :record_results [Boolean] Record results for later analysis (default: false)

Returns:

  • (Hash) —

    Shadow test result



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/decision_agent/simulation/shadow_test_engine.rb', line 23

def test(context:, shadow_version:, options: {})
  options = {
    track_differences: true,
    record_results: false
  }.merge(options)

  ctx = normalize_context(context)
  production_decision = execute_production_decision(ctx)
  shadow_decision = execute_shadow_decision(ctx, shadow_version)

  result = build_comparison_result(ctx, production_decision, shadow_decision)
  add_differences(result, production_decision, shadow_decision) if options[:track_differences] && !result[:matches]
  record_result(result, shadow_version) if options[:record_results]

  result
end