Class: LangsmithrbRails::Evaluation::StringEvaluator

Inherits:
Evaluator
  • Object
show all
Defined in:
lib/langsmithrb_rails/evaluation/string_evaluator.rb

Overview

Evaluator for string comparisons

Instance Attribute Summary

Attributes inherited from Evaluator

#client, #project_name, #tags

Instance Method Summary collapse

Methods inherited from Evaluator

#evaluate_dataset, #evaluate_run, #evaluate_runs

Constructor Details

#initialize(match_type: :exact, case_sensitive: true, client: nil, project_name: nil, tags: []) ⇒ StringEvaluator

Initialize a new string evaluator

Parameters:

  • match_type (Symbol) (defaults to: :exact) —

    Type of string matching (:exact, :contains, :regex)

  • case_sensitive (Boolean) (defaults to: true) —

    Whether to perform case-sensitive matching

  • client (LangsmithrbRails::Client) (defaults to: nil) —

    LangSmith client

  • project_name (String) (defaults to: nil) —

    Optional project name for evaluations

  • tags (Array<String>) (defaults to: []) —

    Optional tags for evaluations



15
16
17
18
19
 
# File 'lib/langsmithrb_rails/evaluation/string_evaluator.rb', line 15

def initialize(match_type: :exact, case_sensitive: true, client: nil, project_name: nil, tags: [])
  super(client: client, project_name: project_name, tags: tags)
  @match_type = match_type
  @case_sensitive = case_sensitive
end

Instance Method Details

#evaluate(prediction, reference = nil, input = nil) ⇒ Hash

Evaluate a prediction against a reference

Parameters:

  • prediction (String) —

    The prediction to evaluate

  • reference (String) (defaults to: nil) —

    The reference to compare against

  • input (Hash) (defaults to: nil) —

    Optional input that generated the prediction

Returns:

  • (Hash) —

    Evaluation result with score and metadata



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/langsmithrb_rails/evaluation/string_evaluator.rb', line 26

def evaluate(prediction, reference = nil, input = nil)
  # Extract strings from prediction and reference
  prediction_str = extract_string(prediction)
  reference_str = extract_string(reference)
  
  # If reference is nil, we can't evaluate
  if reference_str.nil?
    return {
      score: nil,
      metadata: {
        error: "No reference provided for evaluation"
      }
    }
  end
  
  # Prepare strings for comparison
  unless @case_sensitive
    prediction_str = prediction_str.downcase
    reference_str = reference_str.downcase
  end
  
  # Perform comparison based on match type
  score, metadata = case @match_type
                    when :exact
                      exact_match(prediction_str, reference_str)
                    when :contains
                      contains_match(prediction_str, reference_str)
                    when :regex
                      regex_match(prediction_str, reference_str)
                    else
                      [0.0, { error: "Unknown match type: #{@match_type}" }]
                    end
  
  {
    score: score,
    metadata: metadata
  }
end