Class: EvalRuby::Judges::Anthropic

Inherits:
Base
  • Object
show all
Defined in:
lib/eval_ruby/judges/anthropic.rb

Overview

Anthropic-based LLM judge using the Messages API. Requires an API key set via Configuration#api_key.

Constant Summary collapse

API_URL = 
"https://api.anthropic.com/v1/messages"

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ Anthropic

Returns a new instance of Anthropic.

Parameters:

Raises:



16
17
18
19
 
# File 'lib/eval_ruby/judges/anthropic.rb', line 16

def initialize(config)
  super
  raise EvalRuby::Error, "API key is required. Set via EvalRuby.configure { |c| c.api_key = '...' }" if @config.api_key.nil? || @config.api_key.empty?
end

Instance Method Details

#call(prompt) ⇒ Hash?

Returns parsed JSON response.

Parameters:

  • prompt (String) —

    the evaluation prompt

Returns:

  • (Hash, nil) —

    parsed JSON response

Raises:



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/eval_ruby/judges/anthropic.rb', line 25

def call(prompt)
  retries = 0
  begin
    uri = URI(API_URL)
    request = Net::HTTP::Post.new(uri)
    request["x-api-key"] = @config.api_key
    request["anthropic-version"] = "2023-06-01"
    request["Content-Type"] = "application/json"
    request.body = JSON.generate({
      model: @config.judge_model,
      max_tokens: 4096,
      messages: [{role: "user", content: prompt}],
      temperature: 0.0
    })

    response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
                               read_timeout: @config.timeout) do |http|
      http.request(request)
    end

    unless response.is_a?(Net::HTTPSuccess)
      raise Error, "Anthropic API error: #{response.code} - #{response.body}"
    end

    body = JSON.parse(response.body)
    content = body.dig("content", 0, "text")
    parse_json_response(content)
  rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET => e
    retries += 1
    if retries <= @config.max_retries
      sleep(2 ** (retries - 1))
      retry
    end
    raise EvalRuby::TimeoutError, "Judge API failed after #{@config.max_retries} retries: #{e.message}"
  end
end