Class: Split::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/split/experiment.rb

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  resettable: true
}

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, options = {}) ⇒ Experiment

Returns a new instance of Experiment.



24
25
26
27
28
29
30
 
# File 'lib/split/experiment.rb', line 24

def initialize(name, options = {})
  options = DEFAULT_OPTIONS.merge(options)

  @name = name.to_s

  extract_alternatives_from_options(options)
end

Instance Attribute Details

#alternative_probabilities ⇒ Object

Returns the value of attribute alternative_probabilities.



7
8
9
 
# File 'lib/split/experiment.rb', line 7

def alternative_probabilities
  @alternative_probabilities
end

#alternatives ⇒ Object

Returns the value of attribute alternatives.



10
11
12
 
# File 'lib/split/experiment.rb', line 10

def alternatives
  @alternatives
end

#goals ⇒ Object

Returns the value of attribute goals.



6
7
8
 
# File 'lib/split/experiment.rb', line 6

def goals
  @goals
end

#metadata ⇒ Object

Returns the value of attribute metadata.



8
9
10
 
# File 'lib/split/experiment.rb', line 8

def metadata
  @metadata
end

#name ⇒ Object

Returns the value of attribute name.



5
6
7
 
# File 'lib/split/experiment.rb', line 5

def name
  @name
end

#resettable ⇒ Object

Returns the value of attribute resettable.



11
12
13
 
# File 'lib/split/experiment.rb', line 11

def resettable
  @resettable
end

Class Method Details

.find(name) ⇒ Object 



17
18
19
20
21
22
 
# File 'lib/split/experiment.rb', line 17

def self.find(name)
  Split.cache(:experiments, name) do
    return unless Split.redis.exists?(name)
    Experiment.new(name).tap { |exp| exp.load_from_redis }
  end
end

.finished_key(key) ⇒ Object 



32
33
34
 
# File 'lib/split/experiment.rb', line 32

def self.finished_key(key)
  "#{key}:finished"
end

Instance Method Details

#==(obj) ⇒ Object 



105
106
107
 
# File 'lib/split/experiment.rb', line 105

def ==(obj)
  self.name == obj.name
end

#[](name) ⇒ Object 



109
110
111
 
# File 'lib/split/experiment.rb', line 109

def [](name)
  alternatives.find { |a| a.name == name }
end

#algorithm ⇒ Object 



113
114
115
 
# File 'lib/split/experiment.rb', line 113

def algorithm
  @algorithm ||= Split.configuration.algorithm
end

#algorithm=(algorithm) ⇒ Object 



117
118
119
 
# File 'lib/split/experiment.rb', line 117

def algorithm=(algorithm)
  @algorithm = algorithm.is_a?(String) ? algorithm.constantize : algorithm
end

#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object 



333
334
335
336
337
338
339
 
# File 'lib/split/experiment.rb', line 333

def calc_alternative_probabilities(winning_counts, number_of_simulations)
  alternative_probabilities = {}
  winning_counts.each do |alternative, wins|
    alternative_probabilities[alternative] = wins / number_of_simulations.to_f
  end
  alternative_probabilities
end

#calc_beta_params(goal = nil) ⇒ Object 



380
381
382
383
384
385
386
387
388
389
390
391
392
 
# File 'lib/split/experiment.rb', line 380

def calc_beta_params(goal = nil)
  beta_params = {}
  alternatives.each do |alternative|
    conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal)
    alpha = 1 + conversions
    beta = 1 + alternative.participant_count - conversions

    params = [alpha, beta]

    beta_params[alternative] = params
  end
  beta_params
end

#calc_simulated_conversion_rates(beta_params) ⇒ Object 



366
367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/split/experiment.rb', line 366

def calc_simulated_conversion_rates(beta_params)
  simulated_cr_hash = {}

  # create a hash which has the conversion rate pulled from each alternative's beta distribution
  beta_params.each do |alternative, params|
    alpha = params[0]
    beta = params[1]
    simulated_conversion_rate = Split::Algorithms.beta_distribution_rng(alpha, beta)
    simulated_cr_hash[alternative] = simulated_conversion_rate
  end

  simulated_cr_hash
end

#calc_time ⇒ Object 



398
399
400
 
# File 'lib/split/experiment.rb', line 398

def calc_time
  redis.hget(experiment_config_key, :calc_time).to_i
end

#calc_time=(time) ⇒ Object 



394
395
396
 
# File 'lib/split/experiment.rb', line 394

def calc_time=(time)
  redis.hset(experiment_config_key, :calc_time, time)
end

#calc_winning_alternatives ⇒ Object 



280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
 
# File 'lib/split/experiment.rb', line 280

def calc_winning_alternatives
  return unless can_calculate_winning_alternatives?

  # Cache the winning alternatives so we recalculate them once per the specified interval.
  intervals_since_epoch =
    Time.now.utc.to_i / Split.configuration.winning_alternative_recalculation_interval

  if self.calc_time != intervals_since_epoch
    if goals.empty?
      self.estimate_winning_alternative
    else
      goals.each do |goal|
        self.estimate_winning_alternative(goal)
      end
    end

    self.calc_time = intervals_since_epoch

    self.save
  end
end

#can_calculate_winning_alternatives? ⇒ Boolean

Returns:

  • (Boolean) 


273
274
275
276
277
278
 
# File 'lib/split/experiment.rb', line 273

def can_calculate_winning_alternatives?
  self.alternatives.all? do |alternative|
    alternative.participant_count >= 0 &&
    (alternative.participant_count >= alternative.completed_count)
  end
end

#cohorting_disabled? ⇒ Boolean

Returns:

  • (Boolean) 


411
412
413
414
415
416
 
# File 'lib/split/experiment.rb', line 411

def cohorting_disabled?
  @cohorting_disabled ||= begin
    value = redis.hget(experiment_config_key, :cohorting)
    value.nil? ? false : value.downcase == "true"
  end
end

#control ⇒ Object 



161
162
163
 
# File 'lib/split/experiment.rb', line 161

def control
  alternatives.first
end

#count_simulated_wins(winning_alternatives) ⇒ Object 



341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/split/experiment.rb', line 341

def count_simulated_wins(winning_alternatives)
  # initialize a hash to keep track of winning alternative in simulations
  winning_counts = {}
  alternatives.each do |alternative|
    winning_counts[alternative] = 0
  end
  # count number of times each alternative won, calculate probabilities, place in hash
  winning_alternatives.each do |alternative|
    winning_counts[alternative] += 1
  end
  winning_counts
end

#delete ⇒ Object 



242
243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/split/experiment.rb', line 242

def delete
  Split.configuration.on_before_experiment_delete.call(self)
  if Split.configuration.start_manually
    redis.hdel(:experiment_start_times, @name)
  end
  reset_winner
  redis.srem(:experiments, name)
  remove_experiment_cohorting
  remove_experiment_configuration
  Split.configuration.on_experiment_delete.call(self)
  increment_version
end

#delete_metadata ⇒ Object 



255
256
257
 
# File 'lib/split/experiment.rb', line 255

def delete_metadata
  redis.del(metadata_key)
end

#disable_cohorting ⇒ Object 



418
419
420
421
 
# File 'lib/split/experiment.rb', line 418

def disable_cohorting
  @cohorting_disabled = true
  redis.hset(experiment_config_key, :cohorting, true.to_s)
end

#enable_cohorting ⇒ Object 



423
424
425
426
 
# File 'lib/split/experiment.rb', line 423

def enable_cohorting
  @cohorting_disabled = false
  redis.hset(experiment_config_key, :cohorting, false.to_s)
end

#estimate_winning_alternative(goal = nil) ⇒ Object 



302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
 
# File 'lib/split/experiment.rb', line 302

def estimate_winning_alternative(goal = nil)
  # initialize a hash of beta distributions based on the alternatives' conversion rates
  beta_params = calc_beta_params(goal)

  winning_alternatives = []

  Split.configuration.beta_probability_simulations.times do
    # calculate simulated conversion rates from the beta distributions
    simulated_cr_hash = calc_simulated_conversion_rates(beta_params)

    winning_alternative = find_simulated_winner(simulated_cr_hash)

    # push the winning pair to the winning_alternatives array
    winning_alternatives.push(winning_alternative)
  end

  winning_counts = count_simulated_wins(winning_alternatives)

  @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations)

  write_to_alternatives(goal)

  self.save
end

#extract_alternatives_from_options(options) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/split/experiment.rb', line 48

def extract_alternatives_from_options(options)
  alts = options[:alternatives] || []

  if alts.length == 1
    if alts[0].is_a? Hash
      alts = alts[0].map { |k, v| { k => v } }
    end
  end

  if alts.empty?
    exp_config = Split.configuration.experiment_for(name)
    if exp_config
      alts = load_alternatives_from_configuration
      options[:goals] = Split::GoalsCollection.new(@name).load_from_configuration
      options[:metadata] = load_metadata_from_configuration
      options[:resettable] = exp_config[:resettable]
      options[:algorithm] = exp_config[:algorithm]
    end
  end

  options[:alternatives] = alts

  set_alternatives_and_options(options)

  # calculate probability that each alternative is the winner
  @alternative_probabilities = {}
  alts
end

#find_simulated_winner(simulated_cr_hash) ⇒ Object 



354
355
356
357
358
359
360
361
362
363
364
 
# File 'lib/split/experiment.rb', line 354

def find_simulated_winner(simulated_cr_hash)
  # figure out which alternative had the highest simulated conversion rate
  winning_pair = ["", 0.0]
  simulated_cr_hash.each do |alternative, rate|
    if rate > winning_pair[1]
      winning_pair = [alternative, rate]
    end
  end
  winner = winning_pair[0]
  winner
end

#finished_key ⇒ Object 



221
222
223
 
# File 'lib/split/experiment.rb', line 221

def finished_key
  self.class.finished_key(key)
end

#goals_key ⇒ Object 



217
218
219
 
# File 'lib/split/experiment.rb', line 217

def goals_key
  "#{name}:goals"
end

#has_winner? ⇒ Boolean

Returns:

  • (Boolean) 


146
147
148
149
 
# File 'lib/split/experiment.rb', line 146

def has_winner?
  return @has_winner if defined? @has_winner
  @has_winner = !winner.nil?
end

#increment_version ⇒ Object 



205
206
207
 
# File 'lib/split/experiment.rb', line 205

def increment_version
  @version = redis.incr("#{name}:version")
end

#jstring(goal = nil) ⇒ Object 



402
403
404
405
406
407
408
409
 
# File 'lib/split/experiment.rb', line 402

def jstring(goal = nil)
  js_id = if goal.nil?
    name
  else
    name + "-" + goal
  end
  js_id.gsub("/", "--")
end

#key ⇒ Object 



209
210
211
212
213
214
215
 
# File 'lib/split/experiment.rb', line 209

def key
  if version.to_i > 0
    "#{name}:#{version}"
  else
    name
  end
end

#load_from_redis ⇒ Object 



259
260
261
262
263
264
265
266
267
268
269
270
271
 
# File 'lib/split/experiment.rb', line 259

def load_from_redis
  exp_config = redis.hgetall(experiment_config_key)

  options = {
    resettable: exp_config["resettable"],
    algorithm: exp_config["algorithm"],
    alternatives: load_alternatives_from_redis,
    goals: Split::GoalsCollection.new(@name).load_from_redis,
    metadata: load_metadata_from_redis
  }

  set_alternatives_and_options(options)
end

#metadata_key ⇒ Object 



225
226
227
 
# File 'lib/split/experiment.rb', line 225

def metadata_key
  "#{name}:metadata"
end

#new_record? ⇒ Boolean

Returns:

  • (Boolean) 


101
102
103
 
# File 'lib/split/experiment.rb', line 101

def new_record?
  ExperimentCatalog.find(name).nil?
end

#next_alternative ⇒ Object 



189
190
191
 
# File 'lib/split/experiment.rb', line 189

def next_alternative
  winner || random_alternative
end

#participant_count ⇒ Object 



157
158
159
 
# File 'lib/split/experiment.rb', line 157

def participant_count
  alternatives.inject(0) { |sum, a| sum + a.participant_count }
end

#random_alternative ⇒ Object 



193
194
195
196
197
198
199
 
# File 'lib/split/experiment.rb', line 193

def random_alternative
  if alternatives.length > 1
    algorithm.choose_alternative(self)
  else
    alternatives.first
  end
end

#reset ⇒ Object 



233
234
235
236
237
238
239
240
 
# File 'lib/split/experiment.rb', line 233

def reset
  Split.configuration.on_before_experiment_reset.call(self)
  Split::Cache.clear_key(@name)
  alternatives.each(&:reset)
  reset_winner
  Split.configuration.on_experiment_reset.call(self)
  increment_version
end

#reset_winner ⇒ Object 



165
166
167
168
169
 
# File 'lib/split/experiment.rb', line 165

def reset_winner
  redis.hdel(:experiment_winner, name)
  @has_winner = false
  Split::Cache.clear_key(@name)
end

#resettable? ⇒ Boolean

Returns:

  • (Boolean) 


229
230
231
 
# File 'lib/split/experiment.rb', line 229

def resettable?
  resettable
end

#save ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/split/experiment.rb', line 77

def save
  validate!

  if new_record?
    start unless Split.configuration.start_manually
    persist_experiment_configuration
  elsif experiment_configuration_has_changed?
    reset unless Split.configuration.reset_manually
    persist_experiment_configuration
  end

  redis.hmset(experiment_config_key, :resettable, resettable.to_s,
                                     :algorithm, algorithm.to_s)
  self
end

#set_alternatives_and_options(options) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/split/experiment.rb', line 36

def set_alternatives_and_options(options)
  options_with_defaults = DEFAULT_OPTIONS.merge(
    options.reject { |k, v| v.nil? }
  )

  self.alternatives = options_with_defaults[:alternatives]
  self.goals = options_with_defaults[:goals]
  self.resettable = options_with_defaults[:resettable]
  self.algorithm = options_with_defaults[:algorithm]
  self.metadata = options_with_defaults[:metadata]
end

#start ⇒ Object 



171
172
173
 
# File 'lib/split/experiment.rb', line 171

def start
  redis.hset(:experiment_start_times, @name, Time.now.to_i)
end

#start_time ⇒ Object 



175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/split/experiment.rb', line 175

def start_time
  Split.cache(:experiment_start_times, @name) do
    t = redis.hget(:experiment_start_times, @name)
    if t
      # Check if stored time is an integer
      if t =~ /^[-+]?[0-9]+$/
        Time.at(t.to_i)
      else
        Time.parse(t)
      end
    end
  end
end

#validate! ⇒ Object 



93
94
95
96
97
98
99
 
# File 'lib/split/experiment.rb', line 93

def validate!
  if @alternatives.empty? && Split.configuration.experiment_for(@name).nil?
    raise ExperimentNotFound.new("Experiment #{@name} not found")
  end
  @alternatives.each { |a| a.validate! }
  goals_collection.validate!
end

#version ⇒ Object 



201
202
203
 
# File 'lib/split/experiment.rb', line 201

def version
  @version ||= (redis.get("#{name}:version").to_i || 0)
end

#winner ⇒ Object 



135
136
137
138
139
140
141
142
143
144
 
# File 'lib/split/experiment.rb', line 135

def winner
  Split.cache(:experiment_winner, name) do
    experiment_winner = redis.hget(:experiment_winner, name)
    if experiment_winner
      Split::Alternative.new(experiment_winner, name)
    else
      nil
    end
  end
end

#winner=(winner_name) ⇒ Object 



151
152
153
154
155
 
# File 'lib/split/experiment.rb', line 151

def winner=(winner_name)
  redis.hset(:experiment_winner, name, winner_name.to_s)
  @has_winner = true
  Split.configuration.on_experiment_winner_choose.call(self)
end

#write_to_alternatives(goal = nil) ⇒ Object 



327
328
329
330
331
 
# File 'lib/split/experiment.rb', line 327

def write_to_alternatives(goal = nil)
  alternatives.each do |alternative|
    alternative.set_p_winner(@alternative_probabilities[alternative], goal)
  end
end