Module: Legion::Extensions::Eval::Runners::Annotation

Included in:
Client
Defined in:
lib/legion/extensions/eval/runners/annotation.rb

Instance Method Summary collapse

Instance Method Details

#assign_next(queue_name:, annotator:, count: 1) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 38

def assign_next(queue_name:, annotator:, count: 1, **)
  queue = db[:annotation_queues].where(name: queue_name).first
  return { error: 'queue_not_found' } unless queue

  pending = db[:annotation_items]
            .where(queue_id: queue[:id], status: 'pending')
            .order(:id).limit(count).all

  now = Time.now.utc
  assigned = pending.map do |item|
    db[:annotation_items].where(id: item[:id]).update(
      status: 'assigned', assigned_to: annotator, assigned_at: now
    )
    item.merge(status: 'assigned', assigned_to: annotator, assigned_at: now)
  end

  { assigned: assigned.size, items: assigned }
end

#complete_annotation(item_id:, label_score:, label_category: nil, explanation: nil) ⇒ Object 



57
58
59
60
61
62
63
64
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 57

def complete_annotation(item_id:, label_score:, label_category: nil, explanation: nil, **)
  db[:annotation_items].where(id: item_id).update(
    status: 'completed', label_score: label_score,
    label_category: label_category, explanation: explanation,
    completed_at: Time.now.utc
  )
  { completed: true, item_id: item_id }
end

#create_queue(name:, **opts) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 8

def create_queue(name:, **opts)
  db[:annotation_queues].insert(
    name:                name,
    description:         opts[:description],
    evaluator_config:    opts[:evaluator_config],
    assignment_strategy: opts.fetch(:assignment_strategy, 'round_robin'),
    items_per_annotator: opts.fetch(:items_per_annotator, 20),
    created_at:          Time.now.utc
  )
  { created: true, name: name }
rescue Sequel::UniqueConstraintViolation
  { error: 'already_exists', name: name }
end

#enqueue_items(queue_name:, items:) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 22

def enqueue_items(queue_name:, items:, **)
  queue = db[:annotation_queues].where(name: queue_name).first
  return { error: 'queue_not_found' } unless queue

  items.each do |item|
    db[:annotation_items].insert(
      queue_id: queue[:id],
      input: item[:input], output: item[:output],
      context: item[:context], span_id: item[:span_id],
      experiment_id: item[:experiment_id],
      status: 'pending', created_at: Time.now.utc
    )
  end
  { enqueued: items.size, queue: queue_name }
end

#export_to_dataset(queue_name:) ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 88

def export_to_dataset(queue_name:, **)
  queue = db[:annotation_queues].where(name: queue_name).first
  return { error: 'queue_not_found' } unless queue

  completed = db[:annotation_items]
              .where(queue_id: queue[:id], status: 'completed')
              .order(:id).all

  rows = completed.map do |item|
    { input: item[:input], output: item[:output],
      label_score: item[:label_score], label_category: item[:label_category],
      explanation: item[:explanation] }
  end

  { queue: queue_name, rows: rows, count: rows.size }
end

#queue_stats(queue_name:) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 73

def queue_stats(queue_name:, **)
  queue = db[:annotation_queues].where(name: queue_name).first
  return { error: 'queue_not_found' } unless queue

  items = db[:annotation_items].where(queue_id: queue[:id])
  {
    queue:     queue_name,
    total:     items.count,
    pending:   items.where(status: 'pending').count,
    assigned:  items.where(status: 'assigned').count,
    completed: items.where(status: 'completed').count,
    skipped:   items.where(status: 'skipped').count
  }
end

#skip_annotation(item_id:, reason: nil) ⇒ Object 



66
67
68
69
70
71
 
# File 'lib/legion/extensions/eval/runners/annotation.rb', line 66

def skip_annotation(item_id:, reason: nil, **)
  db[:annotation_items].where(id: item_id).update(
    status: 'skipped', explanation: reason, completed_at: Time.now.utc
  )
  { skipped: true, item_id: item_id }
end