Class: PromptEngine::EvaluationRunner

Inherits:
Object
  • Object
show all
Defined in:
app/services/prompt_engine/evaluation_runner.rb

Instance Method Summary collapse

Constructor Details

#initialize(eval_run) ⇒ EvaluationRunner

Returns a new instance of EvaluationRunner.



5
6
7
8
9
10
11
 
# File 'app/services/prompt_engine/evaluation_runner.rb', line 5

def initialize(eval_run)
  @eval_run = eval_run
  @eval_set = eval_run.eval_set
  @prompt_version = eval_run.prompt_version
  @prompt = @prompt_version.prompt
  @client = OpenAiEvalsClient.new
end

Instance Method Details

#execute ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'app/services/prompt_engine/evaluation_runner.rb', line 13

def execute
  @eval_run.update!(status: :running, started_at: Time.current)

  # Step 1: Create or get OpenAI eval configuration
  ensure_openai_eval_exists

  # Step 2: Create test data file in JSONL format
  file_id = upload_test_data
  @eval_run.update!(openai_file_id: file_id)

  # Step 3: Create eval run on OpenAI
  openai_run = create_openai_run(file_id)
  @eval_run.update!(
    openai_run_id: openai_run["id"],
    report_url: openai_run["report_url"]
  )

  # Step 4: Poll for results
  poll_for_results

rescue => e
  @eval_run.update!(status: :failed, error_message: e.message)
  raise
end