Class: EvalRuby::RSpecMatchers::BeFaithfulTo

Inherits:
Object
  • Object
show all
Defined in:
lib/eval_ruby/rspec.rb

Overview

RSpec matcher that checks if an answer is faithful to the given context.

Examples:

expect(answer).to be_faithful_to(context)
expect(answer).to be_faithful_to(context).with_threshold(0.9)

Instance Method Summary collapse

Constructor Details

#initialize(context, judge: nil) ⇒ BeFaithfulTo

Returns a new instance of BeFaithfulTo.



13
14
15
16
17
 
# File 'lib/eval_ruby/rspec.rb', line 13

def initialize(context, judge: nil)
  @context = Array(context)
  @threshold = 0.8
  @judge = judge
end

Instance Method Details

#failure_message ⇒ String

Returns:

  • (String) 


37
38
39
 
# File 'lib/eval_ruby/rspec.rb', line 37

def failure_message
  "expected answer to be faithful to context (threshold: #{@threshold}), but got score #{@score.round(4)}"
end

#failure_message_when_negated ⇒ String

Returns:

  • (String) 


42
43
44
 
# File 'lib/eval_ruby/rspec.rb', line 42

def failure_message_when_negated
  "expected answer not to be faithful to context, but got score #{@score.round(4)}"
end

#matches?(answer) ⇒ Boolean

Parameters:

  • answer (String) —

    the LLM-generated answer to evaluate

Returns:

  • (Boolean) 


28
29
30
31
32
33
34
 
# File 'lib/eval_ruby/rspec.rb', line 28

def matches?(answer)
  @answer = answer
  j = @judge || EvalRuby.send(:default_judge)
  result = Metrics::Faithfulness.new(judge: j).call(answer: answer, context: @context)
  @score = result[:score]
  @score >= @threshold
end

#with_threshold(threshold) ⇒ self

Parameters:

  • threshold (Float) —

    minimum faithfulness score (0.0 - 1.0)

Returns:

  • (self) 


21
22
23
24
 
# File 'lib/eval_ruby/rspec.rb', line 21

def with_threshold(threshold)
  @threshold = threshold
  self
end