Class: Raif::Evals::LlmJudges::Binary

Inherits:
Raif::Evals::LlmJudge show all
Defined in:
lib/raif/evals/llm_judges/binary.rb

Constant Summary

Constants included from Concerns::LlmResponseParsing

Concerns::LlmResponseParsing::ASCII_CONTROL_CHARS

Instance Attribute Summary

Attributes inherited from Task

#files, #images

Instance Method Summary collapse

Methods inherited from Raif::Evals::LlmJudge

#default_llm_model_key, #judgment_confidence, #judgment_reasoning, #low_confidence?

Methods inherited from Task

json_response_schema, prompt, #re_run, run, #run, #status, system_prompt

Methods included from Concerns::LlmResponseParsing

#parse_html_response, #parse_json_response, #parsed_response

Methods included from Concerns::HasAvailableModelTools

#available_model_tools_map

Methods included from Concerns::HasRequestedLanguage

#requested_language_name, #system_prompt_language_preference

Methods included from Concerns::HasLlm

#default_llm_model_key, #llm

Methods inherited from ApplicationRecord

table_name_prefix

Instance Method Details

#build_prompt ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/raif/evals/llm_judges/binary.rb', line 33

def build_prompt
  prompt = "    Evaluation criteria: \#{criteria}\n\n    \#{strict_mode ? \"Apply the criteria strictly without any leniency.\" : \"Apply reasonable judgment while adhering to the criteria.\"}\n  PROMPT\n\n  if examples.present?\n    prompt += \"\\nHere are examples of how to evaluate:\"\n    examples.each do |example|\n      prompt += format_example(example)\n    end\n  end\n\n  prompt += additional_context_prompt if additional_context.present?\n\n  prompt += <<~PROMPT.rstrip\n\n    Now evaluate this content:\n    \#{content_to_judge}\n\n    Does this content meet the evaluation criteria?\n  PROMPT\n\n  prompt\nend\n"

#build_system_prompt ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/raif/evals/llm_judges/binary.rb', line 17

def build_system_prompt
  "    You are an expert evaluator assessing whether content meets specific criteria.\n    Your task is to make binary pass/fail judgments with clear reasoning.\n\n    First, provide detailed reasoning/explanation of your evaluation. Then, provide a precise pass/fail judgment.\n\n    Respond with JSON matching this schema:\n    {\n      \"passes\": boolean,\n      \"reasoning\": \"detailed explanation\",\n      \"confidence\": 0.0-1.0\n    }\n  PROMPT\nend\n".strip

#passes? ⇒ Boolean

Judgment accessor methods



61
62
63
 
# File 'lib/raif/evals/llm_judges/binary.rb', line 61

def passes?
  parsed_response["passes"] if completed?
end