Class: Evalir::Evalirator

Inherits:
Object
  • Object
show all
Defined in:
lib/evalir/evalirator.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(relevant_docids, retrieved_docids = []) ⇒ Evalirator

Instantiates a new instance of the Evalirator, using the provided judgements as a basis for later calculations.



16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/evalir/evalirator.rb', line 16

def initialize(relevant_docids, retrieved_docids = [])
  @relevant_docids = relevant_docids.to_set
  @true_positives = @false_positives = 0
  @search_hits = []
  
  retrieved_docids.each do |docid|
    if @relevant_docids.include? docid
      @true_positives = @true_positives + 1
    else
      @false_positives = @false_positives + 1
    end
    @search_hits << docid
  end
end

Instance Attribute Details

#false_positives ⇒ Object (readonly)

Gets the number of retrieved results that were in fact irrelevant.



11
12
13
 
# File 'lib/evalir/evalirator.rb', line 11

def false_positives
  @false_positives
end

#true_positives ⇒ Object (readonly)

Gets the number of retrieved results that were indeed relevant.



7
8
9
 
# File 'lib/evalir/evalirator.rb', line 7

def true_positives
  @true_positives
end

Instance Method Details

#average_precision ⇒ Object

The average precision. This is equivalent to the average of calling #precision_at_rank with 1..n, n being the number of results.



137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/evalir/evalirator.rb', line 137

def average_precision
  n = 0
  avg = 0.0
  relevant = 0

  @search_hits.each do |h|
    n = n + 1
    if @relevant_docids.include? h
      relevant = relevant + 1
      avg += (relevant.to_f / n) / @relevant_docids.size
    end
  end
  avg
end

#dcg_at(k, logbase = 2) ⇒ Object

Discounted Cumulative Gain at rank k. For a relevant search result at position x, its con- tribution to the DCG is 1.0/Math.log(x, logbase). A higher logbase means more dis- counts for results further out.



170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/evalir/evalirator.rb', line 170

def dcg_at(k, logbase=2)
  i = 1
  dcg = 0.0
  @search_hits[0, k].each do |h|
    if @relevant_docids.include? h
      dcg += i == 1 ? 1.0 : 1.0 / Math.log(i, logbase)
    end
    i += 1
  end
  dcg
end

#f1 ⇒ Object

Calculate the evenly weighted harmonic mean of #precision and #recall. This is equivalent to calling #f_measure with a parameter of 1.0.



64
65
66
 
# File 'lib/evalir/evalirator.rb', line 64

def f1
  f_measure(1.0)
end

#f_measure(beta) ⇒ Object

Calculate the weighted harmonic mean of precision and recall - β > 1 means emphasizing recall, β < 1 means emphasizing precision. β = 1 is equivalent to #f1.



73
74
75
76
77
78
 
# File 'lib/evalir/evalirator.rb', line 73

def f_measure(beta)
  betaSquared = beta ** 2
  n = (betaSquared + 1) * (precision * recall)
  d = (betaSquared * precision) + recall
  n / d
end

#false_negatives ⇒ Object

Calculate the number of false negatives. Divide by #size to get the rate, e.g: fn_rate = e.false_negatives / e.size



40
41
42
 
# File 'lib/evalir/evalirator.rb', line 40

def false_negatives
  @relevant_docids.size - @true_positives
end

#ndcg_at(k, logbase = 2) ⇒ Object

Normalized Discounted Cumulative Gain at rank k. This is the #dcg_at normalized by the optimal dcg value at rank k.



186
187
188
189
 
# File 'lib/evalir/evalirator.rb', line 186

def ndcg_at(k, logbase=2)
  dcg = dcg_at(k, logbase)
  dcg > 0 ? dcg / ideal_dcg_at(k, logbase) : 0
end

#precision ⇒ Object

Calculate the precision, e.g. the fraction of retrieved documents that were relevant.



47
48
49
 
# File 'lib/evalir/evalirator.rb', line 47

def precision
  @true_positives / size
end

#precision_at_rank(k) ⇒ Object

The precision at the rank k, meaning the precision after exactly k documents have been retrieved.



90
91
92
93
 
# File 'lib/evalir/evalirator.rb', line 90

def precision_at_rank(k)
  top_k = @search_hits[0, k].to_set
  (@relevant_docids & top_k).size.to_f / k
end

#precision_at_recall(r) ⇒ Object

Returns the precision at r percent recall. Used to plot the Precision vs. Recall curve.



98
99
100
101
102
103
 
# File 'lib/evalir/evalirator.rb', line 98

def precision_at_recall(r)
  return 1.0 if r == 0.0
  k = (size * r).ceil
  top_k = @search_hits[0, k].to_set
  (@relevant_docids & top_k).size.to_f / k
end

#precision_recall_curve(from = 0, to = 100, step = 10) ⇒ Object

Gets the data for the precision-recall curve, ranging over the interval [from, to], with a step size of step.



120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/evalir/evalirator.rb', line 120

def precision_recall_curve(from = 0, to = 100, step = 10)
  raise "From must be in the interval [0, 100)" unless (from >= 0 and from < 100)
  raise "To must be in the interval (from, 100]" unless (to > from and to <= 100)
  raise "Invalid step size - (to-from) must be divisible by step." unless ((to - from) % step) == 0
  
  data = []
  range = from..to
  range.step(step).each do |recall|
    data << self.precision_at_recall(recall/100.0)
  end
  data
end

#r_precision ⇒ Object

A single value summary which is obtained by computing the precision at the R-th position in the ranking. Here, R is the total number of relevant documents for the current query.



111
112
113
114
115
 
# File 'lib/evalir/evalirator.rb', line 111

def r_precision
  r = @relevant_docids.size
  top_r = @search_hits[0, r].to_set
  (@relevant_docids & top_r).size.to_f / r
end

#recall ⇒ Object

Calculate the recall, e.g. the fraction of relevant documents that were retrieved.



54
55
56
57
 
# File 'lib/evalir/evalirator.rb', line 54

def recall
  fn = false_negatives
  @true_positives / (@true_positives + fn + 0.0)
end

#reciprocal_rank ⇒ Object

The reciprocal rank, meaning 1 divided by the rank of the most highly ranked relevant result.



156
157
158
159
160
161
 
# File 'lib/evalir/evalirator.rb', line 156

def reciprocal_rank
  @search_hits.each_with_index do |h,i|
    return 1.0 / (i + 1) if @relevant_docids.include? h
  end
  return 0.0
end

#size ⇒ Object

Gets the size of the evaluated set, e.g. the number of search hits added.



33
34
35
 
# File 'lib/evalir/evalirator.rb', line 33

def size
  @search_hits.size.to_f
end

#top_percent(p) ⇒ Object

Returns the top p percent hits that were added to this evalirator.



82
83
84
85
 
# File 'lib/evalir/evalirator.rb', line 82

def top_percent(p)
  k = size * (p / 100.0)
  @search_hits[0,k.ceil]
end