Class: PromptEngine::EvalSet

Inherits:
ApplicationRecord show all
Defined in:
app/models/prompt_engine/eval_set.rb

Constant Summary collapse

GRADER_TYPES = 
{
  exact_match: "Exact Match",
  regex: "Regular Expression",
  contains: "Contains Text",
  json_schema: "JSON Match (Exact)"
}.freeze

Instance Method Summary collapse

Instance Method Details

#average_success_rate ⇒ Object 



27
28
29
30
31
32
33
34
35
36
 
# File 'app/models/prompt_engine/eval_set.rb', line 27

def average_success_rate
  runs_with_results = eval_runs.completed.where("total_count > 0")
  return 0 if runs_with_results.empty?

  total_passed = runs_with_results.sum(:passed_count)
  total_count = runs_with_results.sum(:total_count)

  return 0 if total_count.zero?
  (total_passed.to_f / total_count * 100).round(1)
end

#grader_type_display ⇒ Object 



42
43
44
 
# File 'app/models/prompt_engine/eval_set.rb', line 42

def grader_type_display
  GRADER_TYPES[grader_type.to_sym] || grader_type.humanize
end

#latest_run ⇒ Object 



23
24
25
 
# File 'app/models/prompt_engine/eval_set.rb', line 23

def latest_run
  eval_runs.recent.first
end

#ready_to_run? ⇒ Boolean

Returns:

  • (Boolean) 


38
39
40
 
# File 'app/models/prompt_engine/eval_set.rb', line 38

def ready_to_run?
  test_cases.any?
end

#requires_grader_config? ⇒ Boolean

Returns:

  • (Boolean) 


46
47
48
 
# File 'app/models/prompt_engine/eval_set.rb', line 46

def requires_grader_config?
  %w[regex json_schema].include?(grader_type)
end