Class: RubyLLM::Evals::PromptExecution

Inherits:
ApplicationRecord show all
Includes:
JobTrackable
Defined in:
app/models/ruby_llm/evals/prompt_execution.rb

Constant Summary collapse

JUDGE_PROMPT_TEMPLATE = 
"You are an expert evaluator. Determine if the output correctly fulfills the task.\n\n## Task Given\n{{rendered_message}}\n\n## Output to Evaluate\n{{output}}\n\n## Evaluation Criteria\n{{criteria}}\n\nReturn whether the output PASSED or FAILED based on the criteria.\n".freeze

Instance Method Summary collapse

Methods included from JobTrackable

#finished?

Instance Method Details

#cost ⇒ Object 



36
37
38
39
40
41
42
43
 
# File 'app/models/ruby_llm/evals/prompt_execution.rb', line 36

def cost
  calculate_cost(
    model_name: run.model,
    provider_name: run.provider,
    input_tokens: input,
    output_tokens: output
  )
end

#execute ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'app/models/ruby_llm/evals/prompt_execution.rb', line 60

def execute
  response = sample.prompt.execute(
    variables: variables,
    files: files.map(&:blob)
  )

  message = response.content.is_a?(Hash) ? response.content.to_json : response.content.chomp

  passed = case eval_type
  when "contains" then message.include?(expected_output)
  when "exact" then message == expected_output
  when "regex" then Regexp.new(expected_output, "i").match?(message)
  when "llm_judge" then judge(message)
  end

  update(
    input: response.input_tokens,
    output: response.output_tokens,
    message:,
    passed:
  )
end

#judge_cost ⇒ Object 



45
46
47
48
49
50
51
52
53
54
 
# File 'app/models/ruby_llm/evals/prompt_execution.rb', line 45

def judge_cost
  return 0.0 if judge_model.blank?

  calculate_cost(
    model_name: judge_model,
    provider_name: judge_provider,
    input_tokens: judge_input,
    output_tokens: judge_output
  )
end

#retry_job ⇒ Object 



83
84
85
86
87
88
89
90
91
92
 
# File 'app/models/ruby_llm/evals/prompt_execution.rb', line 83

def retry_job
  queue_adapter_name = ActiveJob::Base.queue_adapter_name
  meth = :"retry_#{queue_adapter_name}_job"

  if respond_to?(meth, true)
    send(meth)
  else
    raise "Retry not supported for #{queue_adapter_name}"
  end
end

#total_cost ⇒ Object 



56
57
58
 
# File 'app/models/ruby_llm/evals/prompt_execution.rb', line 56

def total_cost
  (cost + judge_cost).round(4)
end