Class: Observ::ModerationGuardrailService

Inherits:
Object
  • Object
show all
Includes:
Concerns::ObservableService
Defined in:
app/services/observ/moderation_guardrail_service.rb

Defined Under Namespace

Classes: Result

Constant Summary collapse

THRESHOLDS =

Score thresholds for different actions

{
  critical: 0.9,  # Auto-flag as critical
  high: 0.7,      # Flag as high priority
  review: 0.5     # Flag for normal review
}.freeze
CRITICAL_CATEGORIES =

Categories that always trigger critical review

%w[
  sexual/minors
  self-harm/intent
  self-harm/instructions
  violence/graphic
].freeze

Instance Method Summary collapse

Methods included from Concerns::ObservableService

#initialize_observability, #instrument_chat, #instrument_embedding, #instrument_image_generation, #instrument_moderation, #instrument_transcription, #with_observability

Constructor Details

#initialize(observability_session: nil) ⇒ ModerationGuardrailService

Returns a new instance of ModerationGuardrailService.



37
38
39
40
41
42
43
 
# File 'app/services/observ/moderation_guardrail_service.rb', line 37

def initialize(observability_session: nil)
  initialize_observability(
    observability_session,
    service_name: "moderation_guardrail",
    metadata: {}
  )
end

Instance Method Details

#evaluate_session(session) ⇒ Array<Result>

Evaluate all traces in a session

Parameters:

Returns:

  • (Array<Result>) —

    Results for each trace



74
75
76
77
78
79
80
 
# File 'app/services/observ/moderation_guardrail_service.rb', line 74

def evaluate_session(session)
  return [] if session.traces.empty?

  session.traces.map do |trace|
    evaluate_trace(trace)
  end
end

#evaluate_session_content(session) ⇒ Result

Evaluate session-level content (aggregated input/output)

Parameters:

Returns:

  • (Result) —

    The evaluation result



86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'app/services/observ/moderation_guardrail_service.rb', line 86

def evaluate_session_content(session)
  return Result.new(action: :skipped, reason: "already_in_queue") if session.in_review_queue?

  with_observability do |_session|
    content = extract_session_content(session)
    return Result.new(action: :skipped, reason: "no_content") if content.blank?

    perform_session_moderation(session, content)
  end
rescue StandardError => e
  Rails.logger.error "[ModerationGuardrailService] Failed to evaluate session #{session.id}: #{e.message}"
  Result.new(action: :skipped, reason: "error", details: { error: e.message })
end

#evaluate_trace(trace, moderate_input: true, moderate_output: true) ⇒ Result

Evaluate a trace for moderation issues

Parameters:

  • trace (Observ::Trace) —

    The trace to evaluate

  • moderate_input (Boolean) (defaults to: true) —

    Whether to moderate input content

  • moderate_output (Boolean) (defaults to: true) —

    Whether to moderate output content

Returns:

  • (Result) —

    The evaluation result



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'app/services/observ/moderation_guardrail_service.rb', line 51

def evaluate_trace(trace, moderate_input: true, moderate_output: true)
  return Result.new(action: :skipped, reason: "already_in_queue") if trace.in_review_queue?
  return Result.new(action: :skipped, reason: "already_has_moderation") if has_existing_flags?(trace)

  with_observability do |_session|
    content = extract_trace_content(
      trace,
      moderate_input: moderate_input,
      moderate_output: moderate_output
    )
    return Result.new(action: :skipped, reason: "no_content") if content.blank?

    perform_moderation(trace, content)
  end
rescue StandardError => e
  Rails.logger.error "[ModerationGuardrailService] Failed to evaluate trace #{trace.id}: #{e.message}"
  Result.new(action: :skipped, reason: "error", details: { error: e.message })
end