Class: DecisionAgent::Testing::BatchTestRunner

Inherits:
Object
  • Object
show all
Defined in:
lib/decision_agent/testing/batch_test_runner.rb

Overview

Executes batch tests against an agent

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(agent) ⇒ BatchTestRunner

Returns a new instance of BatchTestRunner.



43
44
45
46
47
 
# File 'lib/decision_agent/testing/batch_test_runner.rb', line 43

def initialize(agent)
  @agent = agent
  @results = []
  @checkpoint_file = nil
end

Instance Attribute Details

#agent ⇒ Object (readonly)

Returns the value of attribute agent.



41
42
43
 
# File 'lib/decision_agent/testing/batch_test_runner.rb', line 41

def agent
  @agent
end

#results ⇒ Object (readonly)

Returns the value of attribute results.



41
42
43
 
# File 'lib/decision_agent/testing/batch_test_runner.rb', line 41

def results
  @results
end

Instance Method Details

#resume(scenarios, checkpoint_file, options = {}) ⇒ Array<TestResult>

Resume batch test execution from a checkpoint

Parameters:

  • All test scenarios (including already completed ones)

  • Path to checkpoint file

  • (defaults to: {})

    Same as run method

Returns:

  • Array of test results (only newly executed ones)



118
119
120
121
 
# File 'lib/decision_agent/testing/batch_test_runner.rb', line 118

def resume(scenarios, checkpoint_file, options = {})
  options[:checkpoint_file] = checkpoint_file
  run(scenarios, options)
end

#run(scenarios, options = {}) ⇒ Array<TestResult>

Run batch tests against scenarios

Parameters:

  • Test scenarios to execute

  • (defaults to: {})

    Execution options

    • :parallel [Boolean] Use parallel execution (default: true)
    • :thread_count [Integer] Number of threads for parallel execution (default: 4)
    • :progress_callback [Proc] Callback for progress updates (called with { completed: N, total: M, percentage: X })
    • :feedback [Hash] Optional feedback to pass to agent
    • :checkpoint_file [String] Path to checkpoint file for resume capability (optional)

Returns:

  • Array of test results



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/decision_agent/testing/batch_test_runner.rb', line 58

def run(scenarios, options = {})
  @results = []
  @checkpoint_file = options[:checkpoint_file]
  options = {
    parallel: true,
    thread_count: 4,
    progress_callback: nil,
    feedback: {},
    checkpoint_file: nil
  }.merge(options)

  total = scenarios.size
  completed = 0
  mutex = Mutex.new

  # Load checkpoint if exists
  completed_scenario_ids = load_checkpoint if @checkpoint_file && File.exist?(@checkpoint_file)

  # Filter out already completed scenarios
  remaining_scenarios = if completed_scenario_ids&.any?
                          scenarios.reject { |s| completed_scenario_ids.include?(s.id) }
                        else
                          scenarios
                        end

  if options[:parallel] && remaining_scenarios.size > 1
    run_parallel(remaining_scenarios, options, mutex) do |result|
      completed += 1
      save_checkpoint(result.scenario_id) if @checkpoint_file
      options[:progress_callback]&.call(
        completed: completed + (completed_scenario_ids&.size || 0),
        total: total,
        percentage: ((completed + (completed_scenario_ids&.size || 0)).to_f / total * 100).round(2)
      )
    end
  else
    remaining_scenarios.each_with_index do |scenario, index|
      result = execute_scenario(scenario, options[:feedback])
      @results << result
      save_checkpoint(result.scenario_id) if @checkpoint_file
      completed = index + 1
      options[:progress_callback]&.call(
        completed: completed + (completed_scenario_ids&.size || 0),
        total: total,
        percentage: ((completed + (completed_scenario_ids&.size || 0)).to_f / total * 100).round(2)
      )
    end
  end

  # Clean up checkpoint file on successful completion
  delete_checkpoint if @checkpoint_file && File.exist?(@checkpoint_file)

  @results
end

#statistics ⇒ Hash

Get execution statistics

Returns:

  • Statistics about the batch test run



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/decision_agent/testing/batch_test_runner.rb', line 125

def statistics
  return {} if @results.empty?

  successful = @results.count(&:success?)
  failed = @results.size - successful
  execution_times = @results.map(&:execution_time_ms).compact

  {
    total: @results.size,
    successful: successful,
    failed: failed,
    success_rate: successful.to_f / @results.size,
    avg_execution_time_ms: execution_times.any? ? execution_times.sum / execution_times.size : 0,
    min_execution_time_ms: execution_times.min || 0,
    max_execution_time_ms: execution_times.max || 0,
    total_execution_time_ms: execution_times.sum
  }
end