Class: DecisionAgent::Testing::TestResultComparator

Inherits:
Object
  • Object
show all
Defined in:
lib/decision_agent/testing/test_result_comparator.rb

Overview

Compares test results with expected outcomes

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ TestResultComparator

Returns a new instance of TestResultComparator.



44
45
46
47
48
49
50
 
# File 'lib/decision_agent/testing/test_result_comparator.rb', line 44

def initialize(options = {})
  @options = {
    confidence_tolerance: 0.01, # 1% tolerance for confidence comparison
    fuzzy_match: false # Whether to do fuzzy matching on decisions
  }.merge(options)
  @comparison_results = []
end

Instance Attribute Details

#comparison_results ⇒ Object (readonly)

Returns the value of attribute comparison_results.



42
43
44
 
# File 'lib/decision_agent/testing/test_result_comparator.rb', line 42

def comparison_results
  @comparison_results
end

Instance Method Details

#compare(results, scenarios) ⇒ Hash

Compare test results with expected results from scenarios

Parameters:

  • results (Array<TestResult>) —

    Actual test results

  • scenarios (Array<TestScenario>) —

    Test scenarios with expected results

Returns:

  • (Hash) —

    Comparison summary with accuracy metrics



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/decision_agent/testing/test_result_comparator.rb', line 56

def compare(results, scenarios)
  @comparison_results = []

  # Create a map of scenario_id -> scenario for quick lookup
  scenarios.each_with_object({}) do |scenario, map|
    map[scenario.id] = scenario
  end

  # Create a map of scenario_id -> result for quick lookup
  result_map = results.each_with_object({}) do |result, map|
    map[result.scenario_id] = result
  end

  # Compare each scenario with its result
  scenarios.each do |scenario|
    next unless scenario.expected_result?

    result = result_map[scenario.id]
    # Only compare if we have a result (skip if result is missing)
    next unless result

    comparison = compare_single(scenario, result)
    @comparison_results << comparison
  end

  generate_summary
end

#export_csv(file_path) ⇒ Object

Export comparison results to CSV

Parameters:

  • file_path (String) —

    Path to output CSV file



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/decision_agent/testing/test_result_comparator.rb', line 106

def export_csv(file_path)
  require "csv"

  CSV.open(file_path, "w") do |csv|
    csv << %w[scenario_id match decision_match confidence_match expected_decision actual_decision expected_confidence
              actual_confidence differences]
    @comparison_results.each do |result|
      csv << [
        result.scenario_id,
        result.match,
        result.decision_match,
        result.confidence_match,
        result.expected[:decision],
        result.actual[:decision],
        result.expected[:confidence],
        result.actual[:confidence],
        result.differences.join("; ")
      ]
    end
  end
end

#export_json(file_path) ⇒ Object

Export comparison results to JSON

Parameters:

  • file_path (String) —

    Path to output JSON file



130
131
132
133
134
135
136
137
 
# File 'lib/decision_agent/testing/test_result_comparator.rb', line 130

def export_json(file_path)
  require "json"

  File.write(file_path, JSON.pretty_generate({
                                               summary: generate_summary,
                                               results: @comparison_results.map(&:to_h)
                                             }))
end

#generate_summary ⇒ Hash

Generate a summary report

Returns:

  • (Hash) —

    Summary with accuracy metrics and mismatches



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/decision_agent/testing/test_result_comparator.rb', line 86

def generate_summary
  return empty_summary if @comparison_results.empty?

  total = @comparison_results.size
  matches = @comparison_results.count(&:match)
  mismatches = total - matches

  {
    total: total,
    matches: matches,
    mismatches: mismatches,
    accuracy_rate: matches.to_f / total,
    decision_accuracy: @comparison_results.count(&:decision_match).to_f / total,
    confidence_accuracy: @comparison_results.count(&:confidence_match).to_f / total,
    mismatches_detail: @comparison_results.reject(&:match).map(&:to_h)
  }
end