Class: RubyLLM::Tribunal::TestCase

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/tribunal/test_case.rb

Overview

Represents a single evaluation test case.

Examples:

test_case = TestCase.new(
  input: "What's the return policy?",
  actual_output: "You can return items within 30 days.",
  context: ["Returns accepted within 30 days with receipt."],
  expected_output: "Items can be returned within 30 days with a receipt."
)

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(attrs = {}) ⇒ TestCase

Creates a new test case from a hash.

Options Hash (attrs):

  • :input (String) —

    The user query/prompt

  • :actual_output (String) —

    The LLM response to evaluate

  • :expected_output (String) —

    Golden/ideal answer

  • :context (Array<String>, String) —

    Ground truth context

  • :retrieval_context (Array<String>) —

    Retrieved docs from RAG

  • :metadata (Hash) —

    Additional info



33
34
35
36
37
38
39
40
41
42
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 33

def initialize(attrs = {})
  attrs = normalize_keys(attrs)

  @input = attrs[:input]
  @actual_output = attrs[:actual_output]
  @expected_output = attrs[:expected_output]
  @context = normalize_context(attrs[:context])
  @retrieval_context = normalize_context(attrs[:retrieval_context])
  @metadata = attrs[:metadata]
end

Instance Attribute Details

#actual_output ⇒ String? (readonly)

The LLM response to evaluate



21
22
23
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 21

def actual_output
  @actual_output
end

#context ⇒ Array<String>? (readonly)

Ground truth context for faithfulness checks



21
22
23
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 21

def context
  @context
end

#expected_output ⇒ String? (readonly)

Golden/ideal answer for comparison



21
22
23
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 21

def expected_output
  @expected_output
end

#input ⇒ String (readonly)

The user query/prompt (required)



21
22
23
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 21

def input
  @input
end

#metadata ⇒ Hash? (readonly)

Additional info like latency, tokens, cost



21
22
23
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 21

def metadata
  @metadata
end

#retrieval_context ⇒ Array<String>? (readonly)

Actual retrieved docs from RAG



21
22
23
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 21

def retrieval_context
  @retrieval_context
end

Instance Method Details

#to_h ⇒ Hash

Converts the test case to a hash.



94
95
96
97
98
99
100
101
102
103
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 94

def to_h
  {
    input: @input,
    actual_output: @actual_output,
    expected_output: @expected_output,
    context: @context,
    retrieval_context: @retrieval_context,
    metadata: @metadata
  }.compact
end

#with_metadata(new_metadata) ⇒ TestCase

Adds metadata (latency, tokens, cost, etc).



79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 79

def with_metadata(new_metadata)
  merged = (@metadata || {}).merge(new_metadata)
  TestCase.new(
    input: @input,
    actual_output: @actual_output,
    expected_output: @expected_output,
    context: @context,
    retrieval_context: @retrieval_context,
    metadata: merged
  )
end

#with_output(output) ⇒ TestCase

Sets the actual output on an existing test case. Useful when the dataset provides input/context but output comes from your LLM.



49
50
51
52
53
54
55
56
57
58
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 49

def with_output(output)
  TestCase.new(
    input: @input,
    actual_output: output,
    expected_output: @expected_output,
    context: @context,
    retrieval_context: @retrieval_context,
    metadata: @metadata
  )
end

#with_retrieval_context(context) ⇒ TestCase

Sets the retrieval context from your RAG pipeline.



64
65
66
67
68
69
70
71
72
73
 
# File 'lib/ruby_llm/tribunal/test_case.rb', line 64

def with_retrieval_context(context)
  TestCase.new(
    input: @input,
    actual_output: @actual_output,
    expected_output: @expected_output,
    context: @context,
    retrieval_context: normalize_context(context),
    metadata: @metadata
  )
end