Class: Observ::Trace

Inherits:
ApplicationRecord show all
Includes:
Reviewable, Scoreable
Defined in:
app/models/observ/trace.rb

Instance Method Summary collapse

Methods included from Reviewable

#enqueue_for_review!, #in_review_queue?, #pending_review?, #review_status, #reviewed?

Methods included from Scoreable

#manual_score, #score_for, #score_summary, #scored?

Instance Method Details

#create_embedding(name: "embedding", model: nil, metadata: {}, **options) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
 
# File 'app/models/observ/trace.rb', line 46

def create_embedding(name: "embedding", model: nil, metadata: {}, **options)
  observations.create!(
    observation_id: SecureRandom.uuid,
    type: "Observ::Embedding",
    name: name,
    model: model,
    metadata: metadata,
    start_time: Time.current,
    **options.slice(:model_parameters, :parent_observation_id)
  )
end

#create_generation(name: "llm_generation", model: nil, metadata: {}, **options) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
 
# File 'app/models/observ/trace.rb', line 22

def create_generation(name: "llm_generation", model: nil, metadata: {}, **options)
  observations.create!(
    observation_id: SecureRandom.uuid,
    type: "Observ::Generation",
    name: name,
    model: model,
    metadata: metadata,
    start_time: Time.current,
    **options.slice(:model_parameters, :prompt_name, :prompt_version, :parent_observation_id)
  )
end

#create_image_generation(name: "image_generation", model: nil, metadata: {}, **options) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
 
# File 'app/models/observ/trace.rb', line 58

def create_image_generation(name: "image_generation", model: nil, metadata: {}, **options)
  observations.create!(
    observation_id: SecureRandom.uuid,
    type: "Observ::ImageGeneration",
    name: name,
    model: model,
    metadata: metadata,
    start_time: Time.current,
    **options.slice(:model_parameters, :parent_observation_id)
  )
end

#create_moderation(name: "moderation", model: nil, metadata: {}, **options) ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
 
# File 'app/models/observ/trace.rb', line 82

def create_moderation(name: "moderation", model: nil, metadata: {}, **options)
  observations.create!(
    observation_id: SecureRandom.uuid,
    type: "Observ::Moderation",
    name: name,
    model: model,
    metadata: metadata,
    start_time: Time.current,
    **options.slice(:model_parameters, :parent_observation_id)
  )
end

#create_span(name:, input: nil, metadata: {}, parent_observation_id: nil) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
 
# File 'app/models/observ/trace.rb', line 34

def create_span(name:, input: nil, metadata: {}, parent_observation_id: nil)
  observations.create!(
    observation_id: SecureRandom.uuid,
    type: "Observ::Span",
    name: name,
    input: input.is_a?(String) ? input : input.to_json,
    metadata: metadata,
    parent_observation_id: parent_observation_id,
    start_time: Time.current
  )
end

#create_transcription(name: "transcription", model: nil, metadata: {}, **options) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
 
# File 'app/models/observ/trace.rb', line 70

def create_transcription(name: "transcription", model: nil, metadata: {}, **options)
  observations.create!(
    observation_id: SecureRandom.uuid,
    type: "Observ::Transcription",
    name: name,
    model: model,
    metadata: metadata,
    start_time: Time.current,
    **options.slice(:model_parameters, :parent_observation_id)
  )
end

#duration_ms ⇒ Object 



120
121
122
123
 
# File 'app/models/observ/trace.rb', line 120

def duration_ms
  return nil unless end_time
  ((end_time - start_time) * 1000).round(2)
end

#embeddings ⇒ Object 



160
161
162
 
# File 'app/models/observ/trace.rb', line 160

def embeddings
  observations.where(type: "Observ::Embedding")
end

#finalize(output: nil, metadata: {}) ⇒ Object 



94
95
96
97
98
99
100
101
102
 
# File 'app/models/observ/trace.rb', line 94

def finalize(output: nil, metadata: {})
  merged_metadata = (self.metadata || {}).merge(metadata)
  update!(
    output: output.is_a?(String) ? output : output.to_json,
    metadata: merged_metadata,
    end_time: Time.current
  )
  update_aggregated_metrics
end

#finalize_with_response(response) ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'app/models/observ/trace.rb', line 104

def finalize_with_response(response)
  if response.is_a?(String)
    finalize(output: response)
    return response
  end

  response_metadata = extract_response_metadata(response)

  finalize(
    output: response.respond_to?(:content) ? response.content : response.to_s,
    metadata: response_metadata
  )

  response.respond_to?(:content) ? response.content : response.to_s
end

#generations ⇒ Object 



152
153
154
 
# File 'app/models/observ/trace.rb', line 152

def generations
  observations.where(type: "Observ::Generation")
end

#image_generations ⇒ Object 



164
165
166
 
# File 'app/models/observ/trace.rb', line 164

def image_generations
  observations.where(type: "Observ::ImageGeneration")
end

#models_used ⇒ Object 



176
177
178
 
# File 'app/models/observ/trace.rb', line 176

def models_used
  generations.where.not(model: nil).distinct.pluck(:model)
end

#moderations ⇒ Object 



172
173
174
 
# File 'app/models/observ/trace.rb', line 172

def moderations
  observations.where(type: "Observ::Moderation")
end

#spans ⇒ Object 



156
157
158
 
# File 'app/models/observ/trace.rb', line 156

def spans
  observations.where(type: "Observ::Span")
end

#transcriptions ⇒ Object 



168
169
170
 
# File 'app/models/observ/trace.rb', line 168

def transcriptions
  observations.where(type: "Observ::Transcription")
end

#update_aggregated_metrics ⇒ Object 



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'app/models/observ/trace.rb', line 125

def update_aggregated_metrics
  # Include generations, embeddings, image generations, transcriptions, and moderations in cost calculation
  new_total_cost = (generations.sum(:cost_usd) || 0.0) +
                   (embeddings.sum(:cost_usd) || 0.0) +
                   (image_generations.sum(:cost_usd) || 0.0) +
                   (transcriptions.sum(:cost_usd) || 0.0) +
                   (moderations.sum(:cost_usd) || 0.0)

  # Database-agnostic token calculation for generations
  generation_tokens = generations.sum do |gen|
    gen.usage&.dig("total_tokens") || 0
  end

  # Embeddings only have input tokens
  embedding_tokens = embeddings.sum do |emb|
    emb.usage&.dig("input_tokens") || 0
  end

  # Image generations, transcriptions, and moderations don't use tokens
  new_total_tokens = generation_tokens + embedding_tokens

  update_columns(
    total_cost: new_total_cost,
    total_tokens: new_total_tokens
  )
end