Class: EvalRuby::Judges::OpenAI

Inherits:
Base
  • Object
show all
Defined in:
lib/eval_ruby/judges/openai.rb

Overview

OpenAI-based LLM judge using the Chat Completions API. Requires an API key set via Configuration#api_key.

Constant Summary collapse

API_URL = 
"https://api.openai.com/v1/chat/completions"

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ OpenAI

Returns a new instance of OpenAI.

Parameters:

Raises:



16
17
18
19
 
# File 'lib/eval_ruby/judges/openai.rb', line 16

def initialize(config)
  super
  raise EvalRuby::Error, "API key is required. Set via EvalRuby.configure { |c| c.api_key = '...' }" if @config.api_key.nil? || @config.api_key.empty?
end

Instance Method Details

#call(prompt) ⇒ Hash?

Returns parsed JSON response.

Parameters:

  • prompt (String) —

    the evaluation prompt

Returns:

  • (Hash, nil) —

    parsed JSON response

Raises:



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/eval_ruby/judges/openai.rb', line 25

def call(prompt)
  retries = 0
  begin
    uri = URI(API_URL)
    request = Net::HTTP::Post.new(uri)
    request["Authorization"] = "Bearer #{@config.api_key}"
    request["Content-Type"] = "application/json"
    request.body = JSON.generate({
      model: @config.judge_model,
      messages: [{role: "user", content: prompt}],
      temperature: 0.0,
      response_format: {type: "json_object"}
    })

    response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
                               read_timeout: @config.timeout) do |http|
      http.request(request)
    end

    unless response.is_a?(Net::HTTPSuccess)
      raise Error, "OpenAI API error: #{response.code} - #{response.body}"
    end

    body = JSON.parse(response.body)
    content = body.dig("choices", 0, "message", "content")
    parse_json_response(content)
  rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET => e
    retries += 1
    if retries <= @config.max_retries
      sleep(2 ** (retries - 1))
      retry
    end
    raise EvalRuby::TimeoutError, "Judge API failed after #{@config.max_retries} retries: #{e.message}"
  end
end