Class: Leva::BaseRun Abstract

Inherits:
Object
  • Object
show all
Defined in:
lib/leva.rb

Overview

This class is abstract.

Subclass and override #execute to implement custom run logic.

Base class for all run implementations in Leva.

Instance Method Summary collapse

Instance Method Details

#execute(record) ⇒ Object

Executes the run on a given record.

Parameters:

Returns:

  • (Object) —

    The output of the model execution.

Raises:

  • (NotImplementedError) —

    if the method is not implemented in a subclass.



54
55
56
 
# File 'lib/leva.rb', line 54

def execute(record)
  raise NotImplementedError, "#{self.class} must implement #execute"
end

#execute_and_store(experiment, dataset_record, prompt) ⇒ Leva::RunnerResult

Executes the run on a given dataset record and stores the result.

Parameters:

Returns:



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/leva.rb', line 64

def execute_and_store(experiment, dataset_record, prompt)
  # Expose these to the subclass execution
  @experiment = experiment
  @prompt = prompt
  @dataset_record = dataset_record

  result = execute(dataset_record.recordable)
  RunnerResult.create!(
    experiment: experiment,
    dataset_record: dataset_record,
    prompt: prompt,
    prediction: result,
    runner_class: self.class.name
  )
end

#extract_regex_pattern(runner_result) ⇒ Regexp?

Returns The regex pattern to use for parsing predictions.

Parameters:

Returns:

  • (Regexp, nil) —

    The regex pattern to use for parsing predictions



104
105
106
 
# File 'lib/leva.rb', line 104

def extract_regex_pattern(runner_result)
  runner_result.dataset_record.recordable.extract_regex_pattern if runner_result.dataset_record.recordable.respond_to?(:extract_regex_pattern)
end

#ground_truth(runner_result) ⇒ String

Returns The ground truth for the runner result.

Parameters:

Returns:

  • (String) —

    The ground truth for the runner result



110
111
112
 
# File 'lib/leva.rb', line 110

def ground_truth(runner_result)
  runner_result.dataset_record.ground_truth
end

#merged_llm_context ⇒ Hash

Gets the merged LLM context for the current execution. Combines the record’s context with the runner’s additional context.

Returns:

  • (Hash) —

    The merged context for LLM prompt rendering



84
85
86
87
88
89
90
 
# File 'lib/leva.rb', line 84

def merged_llm_context
  return {} unless @dataset_record

  record_context = @dataset_record.recordable.to_llm_context
  runner_context = to_llm_context(@dataset_record.recordable)
  record_context.merge(runner_context)
end

#parsed_predictions(runner_result) ⇒ Array<String>

Returns The parsed predictions.

Parameters:

Returns:

  • (Array<String>) —

    The parsed predictions



94
95
96
97
98
99
100
 
# File 'lib/leva.rb', line 94

def parsed_predictions(runner_result)
  if extract_regex_pattern(runner_result)
    runner_result.prediction.scan(extract_regex_pattern(runner_result)).map { |match| match.first&.strip }.compact
  else
    [ runner_result.prediction ]
  end
end

#to_llm_context(record) ⇒ Hash

Provides additional LLM context specific to this runner. Override in subclasses to add expensive or runner-specific context.

Parameters:

  • record (Object) —

    The recordable object to generate context for

Returns:

  • (Hash) —

    Additional context to merge with the record’s context



119
120
121
 
# File 'lib/leva.rb', line 119

def to_llm_context(record)
  {}
end