Module: EvalRuby::Assertions

Defined in:
lib/eval_ruby/minitest.rb

Instance Method Summary collapse

Instance Method Details

#assert_correct(answer, ground_truth:, threshold: 0.7, message: nil) ⇒ Object 



19
20
21
22
23
 
# File 'lib/eval_ruby/minitest.rb', line 19

def assert_correct(answer, ground_truth:, threshold: 0.7, message: nil)
  result = eval_metric(:correctness, answer: answer, ground_truth: ground_truth)
  msg = message || "Expected correctness >= #{threshold}, got #{result[:score].round(4)}"
  assert result[:score] >= threshold, msg
end

#assert_faithful(answer, context, threshold: 0.8, message: nil) ⇒ Object 



7
8
9
10
11
 
# File 'lib/eval_ruby/minitest.rb', line 7

def assert_faithful(answer, context, threshold: 0.8, message: nil)
  result = eval_metric(:faithfulness, answer: answer, context: Array(context))
  msg = message || "Expected faithfulness >= #{threshold}, got #{result[:score].round(4)}"
  assert result[:score] >= threshold, msg
end

#assert_precision_at_k(retrieved, relevant, k:, threshold: 0.5, message: nil) ⇒ Object 



25
26
27
28
29
 
# File 'lib/eval_ruby/minitest.rb', line 25

def assert_precision_at_k(retrieved, relevant, k:, threshold: 0.5, message: nil)
  score = Metrics::PrecisionAtK.new.call(retrieved: retrieved, relevant: relevant, k: k)
  msg = message || "Expected precision@#{k} >= #{threshold}, got #{score.round(4)}"
  assert score >= threshold, msg
end

#assert_relevant(question, answer, threshold: 0.8, message: nil) ⇒ Object 



13
14
15
16
17
 
# File 'lib/eval_ruby/minitest.rb', line 13

def assert_relevant(question, answer, threshold: 0.8, message: nil)
  result = eval_metric(:relevance, question: question, answer: answer)
  msg = message || "Expected relevance >= #{threshold}, got #{result[:score].round(4)}"
  assert result[:score] >= threshold, msg
end

#refute_hallucination(answer, context, threshold: 0.8, message: nil) ⇒ Object 



31
32
33
34
35
 
# File 'lib/eval_ruby/minitest.rb', line 31

def refute_hallucination(answer, context, threshold: 0.8, message: nil)
  result = eval_metric(:faithfulness, answer: answer, context: Array(context))
  msg = message || "Expected no hallucination (faithfulness >= #{threshold}), got #{result[:score].round(4)}"
  assert result[:score] >= threshold, msg
end