Class: Split::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/split/experiment.rb

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  :resettable => true
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, options = {}) ⇒ Experiment

Returns a new instance of Experiment.



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/split/experiment.rb', line 14

def initialize(name, options = {})
  options = DEFAULT_OPTIONS.merge(options)

  @name = name.to_s

  alternatives = extract_alternatives_from_options(options)

  if alternatives.empty? && (exp_config = Split.configuration.experiment_for(name))
    set_alternatives_and_options(
      alternatives: load_alternatives_from_configuration,
      goals: load_goals_from_configuration,
      resettable: exp_config[:resettable],
      algorithm: exp_config[:algorithm]
    )
  else
    set_alternatives_and_options(
      alternatives: alternatives,
      goals: options[:goals],
      resettable: options[:resettable],
      algorithm: options[:algorithm]
    )
  end
end

Instance Attribute Details

#algorithm ⇒ Object 



122
123
124
 
# File 'lib/split/experiment.rb', line 122

def algorithm
  @algorithm ||= Split.configuration.algorithm
end

#alternative_probabilities ⇒ Object

Returns the value of attribute alternative_probabilities.



8
9
10
 
# File 'lib/split/experiment.rb', line 8

def alternative_probabilities
  @alternative_probabilities
end

#alternatives ⇒ Object

Returns the value of attribute alternatives.



7
8
9
 
# File 'lib/split/experiment.rb', line 7

def alternatives
  @alternatives
end

#goals ⇒ Object

Returns the value of attribute goals.



6
7
8
 
# File 'lib/split/experiment.rb', line 6

def goals
  @goals
end

#name ⇒ Object

Returns the value of attribute name.



3
4
5
 
# File 'lib/split/experiment.rb', line 3

def name
  @name
end

#resettable ⇒ Object

Returns the value of attribute resettable.



5
6
7
 
# File 'lib/split/experiment.rb', line 5

def resettable
  @resettable
end

Instance Method Details

#==(obj) ⇒ Object 



114
115
116
 
# File 'lib/split/experiment.rb', line 114

def ==(obj)
  self.name == obj.name
end

#[](name) ⇒ Object 



118
119
120
 
# File 'lib/split/experiment.rb', line 118

def [](name)
  alternatives.find{|a| a.name == name}
end

#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object 



304
305
306
307
308
309
310
 
# File 'lib/split/experiment.rb', line 304

def calc_alternative_probabilities(winning_counts, number_of_simulations)
  alternative_probabilities = {}
  winning_counts.each do |alternative, wins|
    alternative_probabilities[alternative] = wins / number_of_simulations.to_f
  end
  return alternative_probabilities
end

#calc_beta_params(goal = nil) ⇒ Object 



355
356
357
358
359
360
361
362
363
364
365
366
367
 
# File 'lib/split/experiment.rb', line 355

def calc_beta_params(goal = nil)
  beta_params = {}
  alternatives.each do |alternative|
    conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal)
    alpha = 1 + conversions
    beta = 1 + alternative.participant_count - conversions

    params = [alpha, beta]

    beta_params[alternative] = params
  end
  return beta_params
end

#calc_simulated_conversion_rates(beta_params) ⇒ Object 



337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
 
# File 'lib/split/experiment.rb', line 337

def calc_simulated_conversion_rates(beta_params)
  # initialize a random variable (from which to simulate conversion rates ~beta-distributed)
  rand = SimpleRandom.new
  rand.set_seed

  simulated_cr_hash = {}

  # create a hash which has the conversion rate pulled from each alternative's beta distribution
  beta_params.each do |alternative, params|
    alpha = params[0]
    beta = params[1]
    simulated_conversion_rate = rand.beta(alpha, beta)
    simulated_cr_hash[alternative] = simulated_conversion_rate
  end

  return simulated_cr_hash
end

#calc_time ⇒ Object 



373
374
375
 
# File 'lib/split/experiment.rb', line 373

def calc_time
  Split.redis.hget(experiment_config_key, :calc_time)
end

#calc_time=(time) ⇒ Object 



369
370
371
 
# File 'lib/split/experiment.rb', line 369

def calc_time=(time)
  Split.redis.hset(experiment_config_key, :calc_time, time)
end

#calc_winning_alternatives ⇒ Object 



257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/split/experiment.rb', line 257

def calc_winning_alternatives
  if goals.empty?
    self.estimate_winning_alternative
  else
    goals.each do |goal|
      self.estimate_winning_alternative(goal)
    end
  end

  calc_time = Time.now.day

  self.save
end

#control ⇒ Object 



164
165
166
 
# File 'lib/split/experiment.rb', line 164

def control
  alternatives.first
end

#count_simulated_wins(winning_alternatives) ⇒ Object 



312
313
314
315
316
317
318
319
320
321
322
323
 
# File 'lib/split/experiment.rb', line 312

def count_simulated_wins(winning_alternatives)
   # initialize a hash to keep track of winning alternative in simulations
  winning_counts = {}
  alternatives.each do |alternative|
    winning_counts[alternative] = 0
  end
  # count number of times each alternative won, calculate probabilities, place in hash
  winning_alternatives.each do |alternative|
    winning_counts[alternative] += 1
  end
  return winning_counts
end

#delete ⇒ Object 



235
236
237
238
239
240
241
242
243
 
# File 'lib/split/experiment.rb', line 235

def delete
  alternatives.each(&:delete)
  reset_winner
  Split.redis.srem(:experiments, name)
  Split.redis.del(name)
  delete_goals
  Split.configuration.on_experiment_delete.call(self)
  increment_version
end

#delete_goals ⇒ Object 



245
246
247
 
# File 'lib/split/experiment.rb', line 245

def delete_goals
  Split.redis.del(goals_key)
end

#estimate_winning_alternative(goal = nil) ⇒ Object 



271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/split/experiment.rb', line 271

def estimate_winning_alternative(goal = nil)
  # TODO - refactor out functionality to work with and without goals

  # initialize a hash of beta distributions based on the alternatives' conversion rates
  beta_params = calc_beta_params(goal)

  winning_alternatives = []

  Split.configuration.beta_probability_simulations.times do
    # calculate simulated conversion rates from the beta distributions
    simulated_cr_hash = calc_simulated_conversion_rates(beta_params)

    winning_alternative = find_simulated_winner(simulated_cr_hash)

    # push the winning pair to the winning_alternatives array
    winning_alternatives.push(winning_alternative)
  end

  winning_counts = count_simulated_wins(winning_alternatives)

  @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations)

  write_to_alternatives(@alternative_probabilities, goal)

  self.save
end

#extract_alternatives_from_options(options) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/split/experiment.rb', line 45

def extract_alternatives_from_options(options)
  alts = options[:alternatives] || []

  if alts.length == 1
    if alts[0].is_a? Hash
      alts = alts[0].map{|k,v| {k => v} }
    end
  end

  if alts.empty?
    exp_config = Split.configuration.experiment_for(name)
    if exp_config
      alts = load_alternatives_from_configuration
      options[:goals] = load_goals_from_configuration
      options[:resettable] = exp_config[:resettable]
      options[:algorithm] = exp_config[:algorithm]
    end
  end

  self.alternatives = alts
  self.goals = options[:goals]
  self.algorithm = options[:algorithm]
  self.resettable = options[:resettable]

  # calculate probability that each alternative is the winner
  @alternative_probabilities = {}
  alts
end

#find_simulated_winner(simulated_cr_hash) ⇒ Object 



325
326
327
328
329
330
331
332
333
334
335
 
# File 'lib/split/experiment.rb', line 325

def find_simulated_winner(simulated_cr_hash)
  # figure out which alternative had the highest simulated conversion rate
  winning_pair = ["",0.0]
  simulated_cr_hash.each do |alternative, rate|
    if rate > winning_pair[1]
      winning_pair = [alternative, rate]
    end
  end
  winner = winning_pair[0]
  return winner
end

#finished_key ⇒ Object 



220
221
222
 
# File 'lib/split/experiment.rb', line 220

def finished_key
  "#{key}:finished"
end

#goals_key ⇒ Object 



216
217
218
 
# File 'lib/split/experiment.rb', line 216

def goals_key
  "#{name}:goals"
end

#has_winner? ⇒ Boolean

Returns:

  • (Boolean) 


152
153
154
 
# File 'lib/split/experiment.rb', line 152

def has_winner?
  !winner.nil?
end

#increment_version ⇒ Object 



204
205
206
 
# File 'lib/split/experiment.rb', line 204

def increment_version
  @version = Split.redis.incr("#{name}:version")
end

#jstring(goal = nil) ⇒ Object 



377
378
379
380
381
382
383
 
# File 'lib/split/experiment.rb', line 377

def jstring(goal = nil)
  unless goal.nil?
    jstring = name + "-" + goal
  else
    jstring = name
  end
end

#key ⇒ Object 



208
209
210
211
212
213
214
 
# File 'lib/split/experiment.rb', line 208

def key
  if version.to_i > 0
    "#{name}:#{version}"
  else
    name
  end
end

#load_from_redis ⇒ Object 



249
250
251
252
253
254
255
 
# File 'lib/split/experiment.rb', line 249

def load_from_redis
  exp_config = Split.redis.hgetall(experiment_config_key)
  self.resettable = exp_config['resettable']
  self.algorithm = exp_config['algorithm']
  self.alternatives = load_alternatives_from_redis
  self.goals = load_goals_from_redis
end

#new_record? ⇒ Boolean

Returns:

  • (Boolean) 


110
111
112
 
# File 'lib/split/experiment.rb', line 110

def new_record?
  !Split.redis.exists(name)
end

#next_alternative ⇒ Object 



188
189
190
 
# File 'lib/split/experiment.rb', line 188

def next_alternative
  winner || random_alternative
end

#participant_count ⇒ Object 



160
161
162
 
# File 'lib/split/experiment.rb', line 160

def participant_count
  alternatives.inject(0){|sum,a| sum + a.participant_count}
end

#random_alternative ⇒ Object 



192
193
194
195
196
197
198
 
# File 'lib/split/experiment.rb', line 192

def random_alternative
  if alternatives.length > 1
    algorithm.choose_alternative(self)
  else
    alternatives.first
  end
end

#reset ⇒ Object 



228
229
230
231
232
233
 
# File 'lib/split/experiment.rb', line 228

def reset
  alternatives.each(&:reset)
  reset_winner
  Split.configuration.on_experiment_reset.call(self)
  increment_version
end

#reset_winner ⇒ Object 



168
169
170
 
# File 'lib/split/experiment.rb', line 168

def reset_winner
  Split.redis.hdel(:experiment_winner, name)
end

#resettable? ⇒ Boolean

Returns:

  • (Boolean) 


224
225
226
 
# File 'lib/split/experiment.rb', line 224

def resettable?
  resettable
end

#save ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/split/experiment.rb', line 74

def save
  validate!

  if new_record?
    Split.redis.sadd(:experiments, name)
    start unless Split.configuration.start_manually
    @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)}
    @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil?
  else
    existing_alternatives = load_alternatives_from_redis
    existing_goals = load_goals_from_redis
    unless existing_alternatives == @alternatives.map(&:name) && existing_goals == @goals
      reset
      @alternatives.each(&:delete)
      delete_goals
      Split.redis.del(@name)
      @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)}
      @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil?
    end
  end

  Split.redis.hset(experiment_config_key, :resettable, resettable)
  Split.redis.hset(experiment_config_key, :algorithm, algorithm.to_s)
  self
end

#set_alternatives_and_options(options) ⇒ Object 



38
39
40
41
42
43
 
# File 'lib/split/experiment.rb', line 38

def set_alternatives_and_options(options)
  self.alternatives = options[:alternatives]
  self.goals = options[:goals]
  self.resettable = options[:resettable]
  self.algorithm = options[:algorithm]
end

#start ⇒ Object 



172
173
174
 
# File 'lib/split/experiment.rb', line 172

def start
  Split.redis.hset(:experiment_start_times, @name, Time.now.to_i)
end

#start_time ⇒ Object 



176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/split/experiment.rb', line 176

def start_time
  t = Split.redis.hget(:experiment_start_times, @name)
  if t
    # Check if stored time is an integer
    if t =~ /^[-+]?[0-9]+$/
      t = Time.at(t.to_i)
    else
      t = Time.parse(t)
    end
  end
end

#validate! ⇒ Object 



100
101
102
103
104
105
106
107
108
 
# File 'lib/split/experiment.rb', line 100

def validate!
  if @alternatives.empty? && Split.configuration.experiment_for(@name).nil?
    raise ExperimentNotFound.new("Experiment #{@name} not found")
  end
  @alternatives.each {|a| a.validate! }
  unless @goals.nil? || goals.kind_of?(Array)
    raise ArgumentError, 'Goals must be an array'
  end
end

#version ⇒ Object 



200
201
202
 
# File 'lib/split/experiment.rb', line 200

def version
  @version ||= (Split.redis.get("#{name.to_s}:version").to_i || 0)
end

#winner ⇒ Object 



144
145
146
147
148
149
150
 
# File 'lib/split/experiment.rb', line 144

def winner
  if w = Split.redis.hget(:experiment_winner, name)
    Split::Alternative.new(w, name)
  else
    nil
  end
end

#winner=(winner_name) ⇒ Object 



156
157
158
 
# File 'lib/split/experiment.rb', line 156

def winner=(winner_name)
  Split.redis.hset(:experiment_winner, name, winner_name.to_s)
end

#write_to_alternatives(alternative_probabilities, goal = nil) ⇒ Object 



298
299
300
301
302
 
# File 'lib/split/experiment.rb', line 298

def write_to_alternatives(alternative_probabilities, goal = nil)
  alternatives.each do |alternative|
    alternative.set_p_winner(@alternative_probabilities[alternative], goal)
  end
end