Module: DSPy::Metrics

Extended by:
T::Sig
Defined in:
lib/dspy/evals.rb

Overview

Common metric functions for evaluation

Class Method Summary collapse

Class Method Details

.composite_and(*metrics) ⇒ Object

Composite metric - combines multiple metrics with AND logic



798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
 
# File 'lib/dspy/evals.rb', line 798

def self.composite_and(*metrics)
  proc do |example, prediction|
    results = {}
    all_passed = true
    
    metrics.each_with_index do |metric, index|
      result = metric.call(example, prediction)
      
      if result.is_a?(Hash)
        results[:"metric_#{index}"] = result
        all_passed &&= result[:passed] || result['passed'] || false
      else
        passed = !!result
        results[:"metric_#{index}"] = { passed: passed }
        all_passed &&= passed
      end
    end
    
    results[:passed] = all_passed
    results
  end
end

.contains(field: :answer, case_sensitive: false) ⇒ Object 



749
750
751
752
753
754
755
756
757
758
759
760
761
762
 
# File 'lib/dspy/evals.rb', line 749

def self.contains(field: :answer, case_sensitive: false)
  proc do |example, prediction|
    expected = extract_field(example, field)
    actual = extract_field(prediction, field)
    
    next false if expected.nil? || actual.nil?
    
    if case_sensitive
      actual.to_s.include?(expected.to_s)
    else
      actual.to_s.downcase.include?(expected.to_s.downcase)
    end
  end
end

.exact_match(field: :answer, case_sensitive: true) ⇒ Object 



727
728
729
730
731
732
733
734
735
736
737
738
739
740
 
# File 'lib/dspy/evals.rb', line 727

def self.exact_match(field: :answer, case_sensitive: true)
  proc do |example, prediction|
    expected = extract_field(example, field)
    actual = extract_field(prediction, field)
    
    next false if expected.nil? || actual.nil?
    
    if case_sensitive
      expected.to_s == actual.to_s
    else
      expected.to_s.downcase == actual.to_s.downcase
    end
  end
end

.numeric_difference(field: :answer, tolerance: 0.01) ⇒ Object 



771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
 
# File 'lib/dspy/evals.rb', line 771

def self.numeric_difference(field: :answer, tolerance: 0.01)
  proc do |example, prediction|
    expected = extract_field(example, field)
    actual = extract_field(prediction, field)
    
    next { passed: false, error: "Missing values" } if expected.nil? || actual.nil?
    
    begin
      expected_num = Float(expected)
      actual_num = Float(actual)
      difference = (expected_num - actual_num).abs
      passed = difference <= tolerance
      
      {
        passed: passed,
        difference: difference,
        expected: expected_num,
        actual: actual_num,
        tolerance: tolerance
      }
    rescue ArgumentError
      { passed: false, error: "Non-numeric values" }
    end
  end
end