Module: RubyLLM::Tribunal::EvalHelpers

Defined in:
lib/ruby_llm/tribunal/eval_helpers.rb

Overview

Helper methods for test framework integration.

Include this module in your test classes to get access to assertion methods.

Examples:

With Minitest

class MyEvalTest < Minitest::Test
  include RubyLLM::Tribunal::EvalHelpers

  def test_response_is_faithful
    response = MyApp::RAG.query("What's the return policy?")
    assert_contains response, "30 days"
    assert_faithful response, context: @docs
  end
end

With RSpec

RSpec.describe "RAG Evaluation" do
  include RubyLLM::Tribunal::EvalHelpers

  it "response is faithful" do
    response = MyApp::RAG.query("What's the return policy?")
    expect_contains response, "30 days"
    expect_faithful response, context: docs
  end
end

Defined Under Namespace

Classes: AssertionError

Instance Method Summary collapse

Instance Method Details

#assert_contains(output, value_or_opts) ⇒ Object

Assert output contains substring(s)



34
35
36
37
38
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 34

def assert_contains(output, value_or_opts)
  opts = normalize_opts(value_or_opts)
  result = Assertions::Deterministic.evaluate(:contains, output, opts)
  handle_result(result, 'contains')
end

#assert_contains_all(output, values) ⇒ Object

Assert output contains all values



54
55
56
57
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 54

def assert_contains_all(output, values)
  result = Assertions::Deterministic.evaluate(:contains_all, output, values:)
  handle_result(result, 'contains_all')
end

#assert_contains_any(output, values) ⇒ Object

Assert output contains at least one of the values



48
49
50
51
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 48

def assert_contains_any(output, values)
  result = Assertions::Deterministic.evaluate(:contains_any, output, values:)
  handle_result(result, 'contains_any')
end

#assert_correctness(output, opts = {}) ⇒ Object

Assert response is correct compared to expected



158
159
160
161
162
163
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 158

def assert_correctness(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:correctness, test_case, opts)
  print_verbose(:correctness, result, opts)
  handle_result(result, 'correctness')
end

#assert_email(output) ⇒ Object

Assert output is a valid email



120
121
122
123
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 120

def assert_email(output)
  result = Assertions::Deterministic.evaluate(:is_email, output, {})
  handle_result(result, 'is_email')
end

#assert_ends_with(output, suffix) ⇒ Object

Assert output ends with suffix



84
85
86
87
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 84

def assert_ends_with(output, suffix)
  result = Assertions::Deterministic.evaluate(:ends_with, output, value: suffix)
  handle_result(result, 'ends_with')
end

#assert_equals(output, expected) ⇒ Object

Assert output exactly equals expected



90
91
92
93
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 90

def assert_equals(output, expected)
  result = Assertions::Deterministic.evaluate(:equals, output, value: expected)
  handle_result(result, 'equals')
end

#assert_faithful(output, opts = {}) ⇒ Object

Assert response is faithful to context



134
135
136
137
138
139
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 134

def assert_faithful(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:faithful, test_case, opts)
  print_verbose(:faithful, result, opts)
  handle_result(result, 'faithful')
end

#assert_json(output) ⇒ Object

Assert output is valid JSON



66
67
68
69
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 66

def assert_json(output)
  result = Assertions::Deterministic.evaluate(:is_json, output, {})
  handle_result(result, 'is_json')
end

#assert_levenshtein(output, target, opts = {}) ⇒ Object

Assert output within Levenshtein distance of target



126
127
128
129
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 126

def assert_levenshtein(output, target, opts = {})
  result = Assertions::Deterministic.evaluate(:levenshtein, output, opts.merge(value: target))
  handle_result(result, 'levenshtein')
end

#assert_max_length(output, max) ⇒ Object

Assert output under maximum length



102
103
104
105
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 102

def assert_max_length(output, max)
  result = Assertions::Deterministic.evaluate(:max_length, output, max:)
  handle_result(result, 'max_length')
end

#assert_max_tokens(output, max) ⇒ Object

Assert output is under token limit



72
73
74
75
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 72

def assert_max_tokens(output, max)
  result = Assertions::Deterministic.evaluate(:max_tokens, output, max:)
  handle_result(result, 'max_tokens')
end

#assert_min_length(output, min) ⇒ Object

Assert output meets minimum length



96
97
98
99
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 96

def assert_min_length(output, min)
  result = Assertions::Deterministic.evaluate(:min_length, output, min:)
  handle_result(result, 'min_length')
end

#assert_refusal(output, opts = {}) ⇒ Object

Assert output appears to be a refusal



209
210
211
212
213
214
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 209

def assert_refusal(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:refusal, test_case, opts)
  print_verbose(:refusal, result, opts)
  handle_result(result, 'refusal')
end

#assert_regex(output, pattern) ⇒ Object

Assert output matches regex pattern



60
61
62
63
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 60

def assert_regex(output, pattern)
  result = Assertions::Deterministic.evaluate(:regex, output, pattern:)
  handle_result(result, 'regex')
end

#assert_relevant(output, opts = {}) ⇒ Object

Assert response is relevant to query



142
143
144
145
146
147
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 142

def assert_relevant(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:relevant, test_case, opts)
  print_verbose(:relevant, result, opts)
  handle_result(result, 'relevant')
end

#assert_similar(output, opts = {}) ⇒ Object

Assert response is semantically similar to expected



219
220
221
222
223
224
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 219

def assert_similar(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:similar, test_case, opts)
  print_verbose(:similar, result, opts)
  handle_result(result, 'similar')
end

#assert_starts_with(output, prefix) ⇒ Object

Assert output starts with prefix



78
79
80
81
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 78

def assert_starts_with(output, prefix)
  result = Assertions::Deterministic.evaluate(:starts_with, output, value: prefix)
  handle_result(result, 'starts_with')
end

#assert_url(output) ⇒ Object

Assert output is a valid URL



114
115
116
117
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 114

def assert_url(output)
  result = Assertions::Deterministic.evaluate(:is_url, output, {})
  handle_result(result, 'is_url')
end

#assert_word_count(output, opts) ⇒ Object

Assert output word count within range



108
109
110
111
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 108

def assert_word_count(output, opts)
  result = Assertions::Deterministic.evaluate(:word_count, output, opts)
  handle_result(result, 'word_count')
end

#refute_bias(output, opts = {}) ⇒ Object

Assert response has no bias



166
167
168
169
170
171
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 166

def refute_bias(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:bias, test_case, opts)
  print_verbose(:bias, result, opts)
  handle_result(result, 'bias')
end

#refute_contains(output, value_or_opts) ⇒ Object

Assert output does not contain substring(s)



41
42
43
44
45
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 41

def refute_contains(output, value_or_opts)
  opts = normalize_opts(value_or_opts)
  result = Assertions::Deterministic.evaluate(:not_contains, output, opts)
  handle_result(result, 'not_contains')
end

#refute_hallucination(output, opts = {}) ⇒ Object

Assert response has no hallucinations



150
151
152
153
154
155
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 150

def refute_hallucination(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:hallucination, test_case, opts)
  print_verbose(:hallucination, result, opts)
  handle_result(result, 'hallucination')
end

#refute_harmful(output, opts = {}) ⇒ Object

Assert response has no harmful content



185
186
187
188
189
190
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 185

def refute_harmful(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:harmful, test_case, opts)
  print_verbose(:harmful, result, opts)
  handle_result(result, 'harmful')
end

#refute_jailbreak(output, opts = {}) ⇒ Object

Assert response shows no signs of jailbreak success



193
194
195
196
197
198
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 193

def refute_jailbreak(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:jailbreak, test_case, opts)
  print_verbose(:jailbreak, result, opts)
  handle_result(result, 'jailbreak')
end

#refute_pii(output, opts = {}) ⇒ Object

Assert response contains no PII



201
202
203
204
205
206
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 201

def refute_pii(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:pii, test_case, opts)
  print_verbose(:pii, result, opts)
  handle_result(result, 'pii')
end

#refute_toxicity(output, opts = {}) ⇒ Object Also known as: refute_toxic

Assert response has no toxic content



174
175
176
177
178
179
 
# File 'lib/ruby_llm/tribunal/eval_helpers.rb', line 174

def refute_toxicity(output, opts = {})
  test_case = build_test_case(output, opts)
  result = Assertions.evaluate(:toxicity, test_case, opts)
  print_verbose(:toxicity, result, opts)
  handle_result(result, 'toxicity')
end