Class: Observ::DatasetRun

Inherits:
ApplicationRecord show all
Defined in:
app/models/observ/dataset_run.rb

Instance Method Summary collapse

Instance Method Details

#average_score(name) ⇒ Object

Score aggregation



79
80
81
82
83
 
# File 'app/models/observ/dataset_run.rb', line 79

def average_score(name)
  relevant_scores = scores.where(name: name)
  return nil if relevant_scores.empty?
  relevant_scores.average(:value)&.round(4)
end

#duration_seconds ⇒ Object 



71
72
73
74
75
76
 
# File 'app/models/observ/dataset_run.rb', line 71

def duration_seconds
  return nil unless finished? && run_items.any?
  first_item = run_items.order(created_at: :asc).first
  last_item = run_items.order(updated_at: :desc).first
  (last_item.updated_at - first_item.created_at).round(1)
end

#failure_rate ⇒ Object 



62
63
64
65
 
# File 'app/models/observ/dataset_run.rb', line 62

def failure_rate
  return 0 if total_items.zero?
  (failed_items.to_f / total_items * 100).round(1)
end

#finished? ⇒ Boolean

Returns:

  • (Boolean) 


23
24
25
 
# File 'app/models/observ/dataset_run.rb', line 23

def finished?
  completed? || failed?
end

#in_progress? ⇒ Boolean

Returns:

  • (Boolean) 


27
28
29
 
# File 'app/models/observ/dataset_run.rb', line 27

def in_progress?
  pending? || running?
end

#initialize_run_items! ⇒ Object

Initialize run items for all active dataset items



49
50
51
52
53
54
 
# File 'app/models/observ/dataset_run.rb', line 49

def initialize_run_items!
  dataset.active_items.find_each do |item|
    run_items.find_or_create_by!(dataset_item: item)
  end
  update!(total_items: run_items.count)
end

#items_with_scores_count ⇒ Object 



96
97
98
 
# File 'app/models/observ/dataset_run.rb', line 96

def items_with_scores_count
  run_items.joins(:scores).distinct.count
end

#items_without_scores_count ⇒ Object 



100
101
102
 
# File 'app/models/observ/dataset_run.rb', line 100

def items_without_scores_count
  total_items - items_with_scores_count
end

#pass_rate(score_name = nil) ⇒ Object 



89
90
91
92
93
94
 
# File 'app/models/observ/dataset_run.rb', line 89

def pass_rate(score_name = nil)
  scope = scores
  scope = scope.where(name: score_name) if score_name
  return nil if scope.empty?
  (scope.where("value >= 0.5").count.to_f / scope.count * 100).round(1)
end

#pending_items_count ⇒ Object 



67
68
69
 
# File 'app/models/observ/dataset_run.rb', line 67

def pending_items_count
  total_items - completed_items - failed_items
end

#progress_percentage ⇒ Object

Progress tracking



18
19
20
21
 
# File 'app/models/observ/dataset_run.rb', line 18

def progress_percentage
  return 0 if total_items.zero?
  ((completed_items + failed_items).to_f / total_items * 100).round(1)
end

#score_summary ⇒ Object 



85
86
87
 
# File 'app/models/observ/dataset_run.rb', line 85

def score_summary
  scores.group(:name).average(:value).transform_values { |v| v.round(4) }
end

#success_rate ⇒ Object

Summary helpers for UI



57
58
59
60
 
# File 'app/models/observ/dataset_run.rb', line 57

def success_rate
  return 0 if total_items.zero?
  (completed_items.to_f / total_items * 100).round(1)
end

#update_metrics! ⇒ Object

Update aggregate metrics from run items



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'app/models/observ/dataset_run.rb', line 32

def update_metrics!
  completed = run_items.where.not(trace_id: nil).where(error: nil).count
  failed = run_items.where.not(error: nil).count

  # Calculate cost and tokens from associated traces
  trace_ids = run_items.where.not(trace_id: nil).pluck(:trace_id)
  traces = Observ::Trace.where(id: trace_ids)

  update!(
    completed_items: completed,
    failed_items: failed,
    total_cost: traces.sum(:total_cost) || 0,
    total_tokens: traces.sum(:total_tokens) || 0
  )
end