Class: Looped::Judge

Inherits:
Object
  • Object
show all
Extended by:
T::Sig
Defined in:
lib/looped/judge.rb

Constant Summary collapse

DEFAULT_MODEL = 
'openai/gpt-4o-mini'

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(model: nil, api_key: nil) ⇒ Judge

Returns a new instance of Judge.



14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/looped/judge.rb', line 14

def initialize(model: nil, api_key: nil)
  model_id = model || ENV.fetch('LOOPED_JUDGE_MODEL', DEFAULT_MODEL)
  resolved_api_key = api_key || resolve_api_key(model_id)

  # Configure DSPy with our LM
  DSPy.configure do |c|
    c.lm = DSPy::LM.new(model_id, api_key: resolved_api_key)
  end

  @predictor = T.let(DSPy::Predict.new(Looped::JudgeSignature), DSPy::Predict)
end

Instance Attribute Details

#predictor ⇒ Object (readonly)

Returns the value of attribute predictor.



11
12
13
 
# File 'lib/looped/judge.rb', line 11

def predictor
  @predictor
end

Instance Method Details

#evaluate(task:, solution:, expected_behavior: '') ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/looped/judge.rb', line 27

def evaluate(task:, solution:, expected_behavior: '')
  result = predictor.call(
    task: task,
    solution: solution,
    expected_behavior: expected_behavior.empty? ? infer_expected_behavior(task) : expected_behavior
  )

  Types::Judgment.new(
    score: normalize_score(result.score),
    passed: result.passed,
    critique: result.critique,
    suggestions: result.suggestions || []
  )
end

#to_feedback(judgment) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/looped/judge.rb', line 43

def to_feedback(judgment)
  parts = []
  parts << "Score: #{judgment.score}/10"
  parts << "Status: #{judgment.passed ? 'PASSED' : 'FAILED'}"
  parts << ""
  parts << "Critique:"
  parts << judgment.critique

  if judgment.suggestions.any?
    parts << ""
    parts << "Suggestions for improvement:"
    judgment.suggestions.each_with_index do |suggestion, i|
      parts << "  #{i + 1}. #{suggestion}"
    end
  end

  parts.join("\n")
end