Class: Split::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/split/experiment.rb

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  :resettable => true
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, options = {}) ⇒ Experiment

Returns a new instance of Experiment.



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/split/experiment.rb', line 16

def initialize(name, options = {})
  options = DEFAULT_OPTIONS.merge(options)

  @name = name.to_s

  alternatives = extract_alternatives_from_options(options)

  if alternatives.empty? && (exp_config = Split.configuration.experiment_for(name))
    set_alternatives_and_options(
      alternatives: load_alternatives_from_configuration,
      goals: Split::GoalsCollection.new(@name).load_from_configuration,
      metadata: load_metadata_from_configuration,
      resettable: exp_config[:resettable],
      algorithm: exp_config[:algorithm]
    )
  else
    set_alternatives_and_options(
      alternatives: alternatives,
      goals: options[:goals],
      metadata: options[:metadata],
      resettable: options[:resettable],
      algorithm: options[:algorithm]
    )
  end
end

Instance Attribute Details

#algorithm ⇒ Object 



131
132
133
 
# File 'lib/split/experiment.rb', line 131

def algorithm
  @algorithm ||= Split.configuration.algorithm
end

#alternative_probabilities ⇒ Object

Returns the value of attribute alternative_probabilities.



9
10
11
 
# File 'lib/split/experiment.rb', line 9

def alternative_probabilities
  @alternative_probabilities
end

#alternatives ⇒ Object

Returns the value of attribute alternatives.



8
9
10
 
# File 'lib/split/experiment.rb', line 8

def alternatives
  @alternatives
end

#goals ⇒ Object

Returns the value of attribute goals.



7
8
9
 
# File 'lib/split/experiment.rb', line 7

def goals
  @goals
end

#metadata ⇒ Object

Returns the value of attribute metadata.



10
11
12
 
# File 'lib/split/experiment.rb', line 10

def metadata
  @metadata
end

#name ⇒ Object

Returns the value of attribute name.



4
5
6
 
# File 'lib/split/experiment.rb', line 4

def name
  @name
end

#resettable ⇒ Object

Returns the value of attribute resettable.



6
7
8
 
# File 'lib/split/experiment.rb', line 6

def resettable
  @resettable
end

Instance Method Details

#==(obj) ⇒ Object 



123
124
125
 
# File 'lib/split/experiment.rb', line 123

def ==(obj)
  self.name == obj.name
end

#[](name) ⇒ Object 



127
128
129
 
# File 'lib/split/experiment.rb', line 127

def [](name)
  alternatives.find{|a| a.name == name}
end

#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object 



329
330
331
332
333
334
335
 
# File 'lib/split/experiment.rb', line 329

def calc_alternative_probabilities(winning_counts, number_of_simulations)
  alternative_probabilities = {}
  winning_counts.each do |alternative, wins|
    alternative_probabilities[alternative] = wins / number_of_simulations.to_f
  end
  return alternative_probabilities
end

#calc_beta_params(goal = nil) ⇒ Object 



380
381
382
383
384
385
386
387
388
389
390
391
392
 
# File 'lib/split/experiment.rb', line 380

def calc_beta_params(goal = nil)
  beta_params = {}
  alternatives.each do |alternative|
    conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal)
    alpha = 1 + conversions
    beta = 1 + alternative.participant_count - conversions

    params = [alpha, beta]

    beta_params[alternative] = params
  end
  return beta_params
end

#calc_simulated_conversion_rates(beta_params) ⇒ Object 



362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/split/experiment.rb', line 362

def calc_simulated_conversion_rates(beta_params)
  # initialize a random variable (from which to simulate conversion rates ~beta-distributed)
  rand = SimpleRandom.new
  rand.set_seed

  simulated_cr_hash = {}

  # create a hash which has the conversion rate pulled from each alternative's beta distribution
  beta_params.each do |alternative, params|
    alpha = params[0]
    beta = params[1]
    simulated_conversion_rate = rand.beta(alpha, beta)
    simulated_cr_hash[alternative] = simulated_conversion_rate
  end

  return simulated_cr_hash
end

#calc_time ⇒ Object 



398
399
400
 
# File 'lib/split/experiment.rb', line 398

def calc_time
  Split.redis.hget(experiment_config_key, :calc_time).to_i
end

#calc_time=(time) ⇒ Object 



394
395
396
 
# File 'lib/split/experiment.rb', line 394

def calc_time=(time)
  Split.redis.hset(experiment_config_key, :calc_time, time)
end

#calc_winning_alternatives ⇒ Object 



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/split/experiment.rb', line 277

def calc_winning_alternatives
  # Super simple cache so that we only recalculate winning alternatives once per day
  days_since_epoch = Time.now.utc.to_i / 86400

  if self.calc_time != days_since_epoch
    if goals.empty?
      self.estimate_winning_alternative
    else
      goals.each do |goal|
        self.estimate_winning_alternative(goal)
      end
    end

    self.calc_time = days_since_epoch

    self.save
  end
end

#control ⇒ Object 



173
174
175
 
# File 'lib/split/experiment.rb', line 173

def control
  alternatives.first
end

#count_simulated_wins(winning_alternatives) ⇒ Object 



337
338
339
340
341
342
343
344
345
346
347
348
 
# File 'lib/split/experiment.rb', line 337

def count_simulated_wins(winning_alternatives)
   # initialize a hash to keep track of winning alternative in simulations
  winning_counts = {}
  alternatives.each do |alternative|
    winning_counts[alternative] = 0
  end
  # count number of times each alternative won, calculate probabilities, place in hash
  winning_alternatives.each do |alternative|
    winning_counts[alternative] += 1
  end
  return winning_counts
end

#delete ⇒ Object 



249
250
251
252
253
254
255
256
257
258
259
260
261
262
 
# File 'lib/split/experiment.rb', line 249

def delete
  Split.configuration.on_before_experiment_delete.call(self)
  if Split.configuration.start_manually
    Split.redis.hdel(:experiment_start_times, @name)
  end
  alternatives.each(&:delete)
  reset_winner
  Split.redis.srem(:experiments, name)
  Split.redis.del(name)
  goals_collection.delete
  delete_metadata
  Split.configuration.on_experiment_delete.call(self)
  increment_version
end

#delete_metadata ⇒ Object 



264
265
266
 
# File 'lib/split/experiment.rb', line 264

def delete_metadata
  Split.redis.del(metadata_key)
end

#estimate_winning_alternative(goal = nil) ⇒ Object 



296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
 
# File 'lib/split/experiment.rb', line 296

def estimate_winning_alternative(goal = nil)
  # TODO - refactor out functionality to work with and without goals

  # initialize a hash of beta distributions based on the alternatives' conversion rates
  beta_params = calc_beta_params(goal)

  winning_alternatives = []

  Split.configuration.beta_probability_simulations.times do
    # calculate simulated conversion rates from the beta distributions
    simulated_cr_hash = calc_simulated_conversion_rates(beta_params)

    winning_alternative = find_simulated_winner(simulated_cr_hash)

    # push the winning pair to the winning_alternatives array
    winning_alternatives.push(winning_alternative)
  end

  winning_counts = count_simulated_wins(winning_alternatives)

  @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations)

  write_to_alternatives(goal)

  self.save
end

#extract_alternatives_from_options(options) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/split/experiment.rb', line 50

def extract_alternatives_from_options(options)
  alts = options[:alternatives] || []

  if alts.length == 1
    if alts[0].is_a? Hash
      alts = alts[0].map{|k,v| {k => v} }
    end
  end

  if alts.empty?
    exp_config = Split.configuration.experiment_for(name)
    if exp_config
      alts = load_alternatives_from_configuration
      options[:goals] = Split::GoalsCollection.new(@name).load_from_configuration
      options[:metadata] = load_metadata_from_configuration
      options[:resettable] = exp_config[:resettable]
      options[:algorithm] = exp_config[:algorithm]
    end
  end

  self.alternatives = alts
  self.goals = options[:goals]
  self.algorithm = options[:algorithm]
  self.resettable = options[:resettable]

  # calculate probability that each alternative is the winner
  @alternative_probabilities = {}
  alts
end

#find_simulated_winner(simulated_cr_hash) ⇒ Object 



350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/split/experiment.rb', line 350

def find_simulated_winner(simulated_cr_hash)
  # figure out which alternative had the highest simulated conversion rate
  winning_pair = ["",0.0]
  simulated_cr_hash.each do |alternative, rate|
    if rate > winning_pair[1]
      winning_pair = [alternative, rate]
    end
  end
  winner = winning_pair[0]
  return winner
end

#finished_key ⇒ Object 



229
230
231
 
# File 'lib/split/experiment.rb', line 229

def finished_key
  "#{key}:finished"
end

#goals_key ⇒ Object 



225
226
227
 
# File 'lib/split/experiment.rb', line 225

def goals_key
  "#{name}:goals"
end

#has_winner? ⇒ Boolean

Returns:

  • (Boolean) 


161
162
163
 
# File 'lib/split/experiment.rb', line 161

def has_winner?
  !winner.nil?
end

#increment_version ⇒ Object 



213
214
215
 
# File 'lib/split/experiment.rb', line 213

def increment_version
  @version = Split.redis.incr("#{name}:version")
end

#jstring(goal = nil) ⇒ Object 



402
403
404
405
406
407
408
409
 
# File 'lib/split/experiment.rb', line 402

def jstring(goal = nil)
  js_id = if goal.nil?
            name
          else
            name + "-" + goal
          end
  js_id.gsub('/', '--')
end

#key ⇒ Object 



217
218
219
220
221
222
223
 
# File 'lib/split/experiment.rb', line 217

def key
  if version.to_i > 0
    "#{name}:#{version}"
  else
    name
  end
end

#load_from_redis ⇒ Object 



268
269
270
271
272
273
274
275
 
# File 'lib/split/experiment.rb', line 268

def load_from_redis
  exp_config = Split.redis.hgetall(experiment_config_key)
  self.resettable = exp_config['resettable']
  self.algorithm = exp_config['algorithm']
  self.alternatives = load_alternatives_from_redis
  self.goals = Split::GoalsCollection.new(@name).load_from_redis
  self.metadata = load_metadata_from_redis
end

#metadata_key ⇒ Object 



233
234
235
 
# File 'lib/split/experiment.rb', line 233

def metadata_key
  "#{name}:metadata"
end

#new_record? ⇒ Boolean

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/split/experiment.rb', line 119

def new_record?
  !Split.redis.exists(name)
end

#next_alternative ⇒ Object 



197
198
199
 
# File 'lib/split/experiment.rb', line 197

def next_alternative
  winner || random_alternative
end

#participant_count ⇒ Object 



169
170
171
 
# File 'lib/split/experiment.rb', line 169

def participant_count
  alternatives.inject(0){|sum,a| sum + a.participant_count}
end

#random_alternative ⇒ Object 



201
202
203
204
205
206
207
 
# File 'lib/split/experiment.rb', line 201

def random_alternative
  if alternatives.length > 1
    algorithm.choose_alternative(self)
  else
    alternatives.first
  end
end

#reset ⇒ Object 



241
242
243
244
245
246
247
 
# File 'lib/split/experiment.rb', line 241

def reset
  Split.configuration.on_before_experiment_reset.call(self)
  alternatives.each(&:reset)
  reset_winner
  Split.configuration.on_experiment_reset.call(self)
  increment_version
end

#reset_winner ⇒ Object 



177
178
179
 
# File 'lib/split/experiment.rb', line 177

def reset_winner
  Split.redis.hdel(:experiment_winner, name)
end

#resettable? ⇒ Boolean

Returns:

  • (Boolean) 


237
238
239
 
# File 'lib/split/experiment.rb', line 237

def resettable?
  resettable
end

#save ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/split/experiment.rb', line 80

def save
  validate!

  if new_record?
    Split.redis.sadd(:experiments, name)
    start unless Split.configuration.start_manually
    @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)}
    goals_collection.save
    save_metadata
    Split.redis.set(metadata_key, @metadata.to_json) unless @metadata.nil?
  else
    existing_alternatives = load_alternatives_from_redis
    existing_goals = Split::GoalsCollection.new(@name).load_from_redis
    existing_metadata = load_metadata_from_redis
    unless existing_alternatives == @alternatives.map(&:name) && existing_goals == @goals && existing_metadata == @metadata
      reset
      @alternatives.each(&:delete)
      goals_collection.delete
      delete_metadata
      Split.redis.del(@name)
      @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)}
      goals_collection.save
      save_metadata
    end
  end

  Split.redis.hset(experiment_config_key, :resettable, resettable)
  Split.redis.hset(experiment_config_key, :algorithm, algorithm.to_s)
  self
end

#set_alternatives_and_options(options) ⇒ Object 



42
43
44
45
46
47
48
 
# File 'lib/split/experiment.rb', line 42

def set_alternatives_and_options(options)
  self.alternatives = options[:alternatives]
  self.goals = options[:goals]
  self.resettable = options[:resettable]
  self.algorithm = options[:algorithm]
  self.metadata = options[:metadata]
end

#start ⇒ Object 



181
182
183
 
# File 'lib/split/experiment.rb', line 181

def start
  Split.redis.hset(:experiment_start_times, @name, Time.now.to_i)
end

#start_time ⇒ Object 



185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/split/experiment.rb', line 185

def start_time
  t = Split.redis.hget(:experiment_start_times, @name)
  if t
    # Check if stored time is an integer
    if t =~ /^[-+]?[0-9]+$/
      t = Time.at(t.to_i)
    else
      t = Time.parse(t)
    end
  end
end

#validate! ⇒ Object 



111
112
113
114
115
116
117
 
# File 'lib/split/experiment.rb', line 111

def validate!
  if @alternatives.empty? && Split.configuration.experiment_for(@name).nil?
    raise ExperimentNotFound.new("Experiment #{@name} not found")
  end
  @alternatives.each {|a| a.validate! }
  goals_collection.validate!
end

#version ⇒ Object 



209
210
211
 
# File 'lib/split/experiment.rb', line 209

def version
  @version ||= (Split.redis.get("#{name.to_s}:version").to_i || 0)
end

#winner ⇒ Object 



153
154
155
156
157
158
159
 
# File 'lib/split/experiment.rb', line 153

def winner
  if w = Split.redis.hget(:experiment_winner, name)
    Split::Alternative.new(w, name)
  else
    nil
  end
end

#winner=(winner_name) ⇒ Object 



165
166
167
 
# File 'lib/split/experiment.rb', line 165

def winner=(winner_name)
  Split.redis.hset(:experiment_winner, name, winner_name.to_s)
end

#write_to_alternatives(goal = nil) ⇒ Object 



323
324
325
326
327
 
# File 'lib/split/experiment.rb', line 323

def write_to_alternatives(goal = nil)
  alternatives.each do |alternative|
    alternative.set_p_winner(@alternative_probabilities[alternative], goal)
  end
end