Class: Split::Experiment
- Inherits:
-
Object
- Object
- Split::Experiment
- Defined in:
- lib/split/experiment.rb
Constant Summary collapse
- DEFAULT_OPTIONS =
{ :resettable => true }
Instance Attribute Summary collapse
- #algorithm ⇒ Object
-
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
-
#alternatives ⇒ Object
Returns the value of attribute alternatives.
-
#goals ⇒ Object
Returns the value of attribute goals.
-
#metadata ⇒ Object
Returns the value of attribute metadata.
-
#name ⇒ Object
Returns the value of attribute name.
-
#resettable ⇒ Object
Returns the value of attribute resettable.
Instance Method Summary collapse
- #==(obj) ⇒ Object
- #[](name) ⇒ Object
- #calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
- #calc_beta_params(goal = nil) ⇒ Object
- #calc_simulated_conversion_rates(beta_params) ⇒ Object
- #calc_time ⇒ Object
- #calc_time=(time) ⇒ Object
- #calc_winning_alternatives ⇒ Object
- #control ⇒ Object
- #count_simulated_wins(winning_alternatives) ⇒ Object
- #delete ⇒ Object
- #delete_metadata ⇒ Object
- #estimate_winning_alternative(goal = nil) ⇒ Object
- #extract_alternatives_from_options(options) ⇒ Object
- #find_simulated_winner(simulated_cr_hash) ⇒ Object
- #finished_key ⇒ Object
- #goals_key ⇒ Object
- #has_winner? ⇒ Boolean
- #increment_version ⇒ Object
-
#initialize(name, options = {}) ⇒ Experiment
constructor
A new instance of Experiment.
- #jstring(goal = nil) ⇒ Object
- #key ⇒ Object
- #load_from_redis ⇒ Object
- #metadata_key ⇒ Object
- #new_record? ⇒ Boolean
- #next_alternative ⇒ Object
- #participant_count ⇒ Object
- #random_alternative ⇒ Object
- #reset ⇒ Object
- #reset_winner ⇒ Object
- #resettable? ⇒ Boolean
- #save ⇒ Object
- #set_alternatives_and_options(options) ⇒ Object
- #start ⇒ Object
- #start_time ⇒ Object
- #validate! ⇒ Object
- #version ⇒ Object
- #winner ⇒ Object
- #winner=(winner_name) ⇒ Object
- #write_to_alternatives(goal = nil) ⇒ Object
Constructor Details
#initialize(name, options = {}) ⇒ Experiment
Returns a new instance of Experiment.
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
# File 'lib/split/experiment.rb', line 16 def initialize(name, = {}) = DEFAULT_OPTIONS.merge() @name = name.to_s alternatives = () if alternatives.empty? && (exp_config = Split.configuration.experiment_for(name)) ( alternatives: load_alternatives_from_configuration, goals: Split::GoalsCollection.new(@name).load_from_configuration, metadata: , resettable: exp_config[:resettable], algorithm: exp_config[:algorithm] ) else ( alternatives: alternatives, goals: [:goals], metadata: [:metadata], resettable: [:resettable], algorithm: [:algorithm] ) end end |
Instance Attribute Details
#algorithm ⇒ Object
131 132 133 |
# File 'lib/split/experiment.rb', line 131 def algorithm @algorithm ||= Split.configuration.algorithm end |
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
9 10 11 |
# File 'lib/split/experiment.rb', line 9 def alternative_probabilities @alternative_probabilities end |
#alternatives ⇒ Object
Returns the value of attribute alternatives.
8 9 10 |
# File 'lib/split/experiment.rb', line 8 def alternatives @alternatives end |
#goals ⇒ Object
Returns the value of attribute goals.
7 8 9 |
# File 'lib/split/experiment.rb', line 7 def goals @goals end |
#metadata ⇒ Object
Returns the value of attribute metadata.
10 11 12 |
# File 'lib/split/experiment.rb', line 10 def @metadata end |
#name ⇒ Object
Returns the value of attribute name.
4 5 6 |
# File 'lib/split/experiment.rb', line 4 def name @name end |
#resettable ⇒ Object
Returns the value of attribute resettable.
6 7 8 |
# File 'lib/split/experiment.rb', line 6 def resettable @resettable end |
Instance Method Details
#==(obj) ⇒ Object
123 124 125 |
# File 'lib/split/experiment.rb', line 123 def ==(obj) self.name == obj.name end |
#[](name) ⇒ Object
127 128 129 |
# File 'lib/split/experiment.rb', line 127 def [](name) alternatives.find{|a| a.name == name} end |
#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
329 330 331 332 333 334 335 |
# File 'lib/split/experiment.rb', line 329 def calc_alternative_probabilities(winning_counts, number_of_simulations) alternative_probabilities = {} winning_counts.each do |alternative, wins| alternative_probabilities[alternative] = wins / number_of_simulations.to_f end return alternative_probabilities end |
#calc_beta_params(goal = nil) ⇒ Object
380 381 382 383 384 385 386 387 388 389 390 391 392 |
# File 'lib/split/experiment.rb', line 380 def calc_beta_params(goal = nil) beta_params = {} alternatives.each do |alternative| conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal) alpha = 1 + conversions beta = 1 + alternative.participant_count - conversions params = [alpha, beta] beta_params[alternative] = params end return beta_params end |
#calc_simulated_conversion_rates(beta_params) ⇒ Object
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 |
# File 'lib/split/experiment.rb', line 362 def calc_simulated_conversion_rates(beta_params) # initialize a random variable (from which to simulate conversion rates ~beta-distributed) rand = SimpleRandom.new rand.set_seed simulated_cr_hash = {} # create a hash which has the conversion rate pulled from each alternative's beta distribution beta_params.each do |alternative, params| alpha = params[0] beta = params[1] simulated_conversion_rate = rand.beta(alpha, beta) simulated_cr_hash[alternative] = simulated_conversion_rate end return simulated_cr_hash end |
#calc_time ⇒ Object
398 399 400 |
# File 'lib/split/experiment.rb', line 398 def calc_time Split.redis.hget(experiment_config_key, :calc_time).to_i end |
#calc_time=(time) ⇒ Object
394 395 396 |
# File 'lib/split/experiment.rb', line 394 def calc_time=(time) Split.redis.hset(experiment_config_key, :calc_time, time) end |
#calc_winning_alternatives ⇒ Object
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 |
# File 'lib/split/experiment.rb', line 277 def calc_winning_alternatives # Super simple cache so that we only recalculate winning alternatives once per day days_since_epoch = Time.now.utc.to_i / 86400 if self.calc_time != days_since_epoch if goals.empty? self.estimate_winning_alternative else goals.each do |goal| self.estimate_winning_alternative(goal) end end self.calc_time = days_since_epoch self.save end end |
#control ⇒ Object
173 174 175 |
# File 'lib/split/experiment.rb', line 173 def control alternatives.first end |
#count_simulated_wins(winning_alternatives) ⇒ Object
337 338 339 340 341 342 343 344 345 346 347 348 |
# File 'lib/split/experiment.rb', line 337 def count_simulated_wins(winning_alternatives) # initialize a hash to keep track of winning alternative in simulations winning_counts = {} alternatives.each do |alternative| winning_counts[alternative] = 0 end # count number of times each alternative won, calculate probabilities, place in hash winning_alternatives.each do |alternative| winning_counts[alternative] += 1 end return winning_counts end |
#delete ⇒ Object
249 250 251 252 253 254 255 256 257 258 259 260 261 262 |
# File 'lib/split/experiment.rb', line 249 def delete Split.configuration.on_before_experiment_delete.call(self) if Split.configuration.start_manually Split.redis.hdel(:experiment_start_times, @name) end alternatives.each(&:delete) reset_winner Split.redis.srem(:experiments, name) Split.redis.del(name) goals_collection.delete Split.configuration.on_experiment_delete.call(self) increment_version end |
#delete_metadata ⇒ Object
264 265 266 |
# File 'lib/split/experiment.rb', line 264 def Split.redis.del() end |
#estimate_winning_alternative(goal = nil) ⇒ Object
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 |
# File 'lib/split/experiment.rb', line 296 def estimate_winning_alternative(goal = nil) # TODO - refactor out functionality to work with and without goals # initialize a hash of beta distributions based on the alternatives' conversion rates beta_params = calc_beta_params(goal) winning_alternatives = [] Split.configuration.beta_probability_simulations.times do # calculate simulated conversion rates from the beta distributions simulated_cr_hash = calc_simulated_conversion_rates(beta_params) winning_alternative = find_simulated_winner(simulated_cr_hash) # push the winning pair to the winning_alternatives array winning_alternatives.push(winning_alternative) end winning_counts = count_simulated_wins(winning_alternatives) @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations) write_to_alternatives(goal) self.save end |
#extract_alternatives_from_options(options) ⇒ Object
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
# File 'lib/split/experiment.rb', line 50 def () alts = [:alternatives] || [] if alts.length == 1 if alts[0].is_a? Hash alts = alts[0].map{|k,v| {k => v} } end end if alts.empty? exp_config = Split.configuration.experiment_for(name) if exp_config alts = load_alternatives_from_configuration [:goals] = Split::GoalsCollection.new(@name).load_from_configuration [:metadata] = [:resettable] = exp_config[:resettable] [:algorithm] = exp_config[:algorithm] end end self.alternatives = alts self.goals = [:goals] self.algorithm = [:algorithm] self.resettable = [:resettable] # calculate probability that each alternative is the winner @alternative_probabilities = {} alts end |
#find_simulated_winner(simulated_cr_hash) ⇒ Object
350 351 352 353 354 355 356 357 358 359 360 |
# File 'lib/split/experiment.rb', line 350 def find_simulated_winner(simulated_cr_hash) # figure out which alternative had the highest simulated conversion rate winning_pair = ["",0.0] simulated_cr_hash.each do |alternative, rate| if rate > winning_pair[1] winning_pair = [alternative, rate] end end winner = winning_pair[0] return winner end |
#finished_key ⇒ Object
229 230 231 |
# File 'lib/split/experiment.rb', line 229 def finished_key "#{key}:finished" end |
#goals_key ⇒ Object
225 226 227 |
# File 'lib/split/experiment.rb', line 225 def goals_key "#{name}:goals" end |
#has_winner? ⇒ Boolean
161 162 163 |
# File 'lib/split/experiment.rb', line 161 def has_winner? !winner.nil? end |
#increment_version ⇒ Object
213 214 215 |
# File 'lib/split/experiment.rb', line 213 def increment_version @version = Split.redis.incr("#{name}:version") end |
#jstring(goal = nil) ⇒ Object
402 403 404 405 406 407 408 409 |
# File 'lib/split/experiment.rb', line 402 def jstring(goal = nil) js_id = if goal.nil? name else name + "-" + goal end js_id.gsub('/', '--') end |
#key ⇒ Object
217 218 219 220 221 222 223 |
# File 'lib/split/experiment.rb', line 217 def key if version.to_i > 0 "#{name}:#{version}" else name end end |
#load_from_redis ⇒ Object
268 269 270 271 272 273 274 275 |
# File 'lib/split/experiment.rb', line 268 def load_from_redis exp_config = Split.redis.hgetall(experiment_config_key) self.resettable = exp_config['resettable'] self.algorithm = exp_config['algorithm'] self.alternatives = load_alternatives_from_redis self.goals = Split::GoalsCollection.new(@name).load_from_redis self. = end |
#metadata_key ⇒ Object
233 234 235 |
# File 'lib/split/experiment.rb', line 233 def "#{name}:metadata" end |
#new_record? ⇒ Boolean
119 120 121 |
# File 'lib/split/experiment.rb', line 119 def new_record? !Split.redis.exists(name) end |
#next_alternative ⇒ Object
197 198 199 |
# File 'lib/split/experiment.rb', line 197 def next_alternative winner || random_alternative end |
#participant_count ⇒ Object
169 170 171 |
# File 'lib/split/experiment.rb', line 169 def participant_count alternatives.inject(0){|sum,a| sum + a.participant_count} end |
#random_alternative ⇒ Object
201 202 203 204 205 206 207 |
# File 'lib/split/experiment.rb', line 201 def random_alternative if alternatives.length > 1 algorithm.choose_alternative(self) else alternatives.first end end |
#reset ⇒ Object
241 242 243 244 245 246 247 |
# File 'lib/split/experiment.rb', line 241 def reset Split.configuration.on_before_experiment_reset.call(self) alternatives.each(&:reset) reset_winner Split.configuration.on_experiment_reset.call(self) increment_version end |
#reset_winner ⇒ Object
177 178 179 |
# File 'lib/split/experiment.rb', line 177 def reset_winner Split.redis.hdel(:experiment_winner, name) end |
#resettable? ⇒ Boolean
237 238 239 |
# File 'lib/split/experiment.rb', line 237 def resettable? resettable end |
#save ⇒ Object
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
# File 'lib/split/experiment.rb', line 80 def save validate! if new_record? Split.redis.sadd(:experiments, name) start unless Split.configuration.start_manually @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} goals_collection.save Split.redis.set(, @metadata.to_json) unless @metadata.nil? else existing_alternatives = load_alternatives_from_redis existing_goals = Split::GoalsCollection.new(@name).load_from_redis = unless existing_alternatives == @alternatives.map(&:name) && existing_goals == @goals && == @metadata reset @alternatives.each(&:delete) goals_collection.delete Split.redis.del(@name) @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} goals_collection.save end end Split.redis.hset(experiment_config_key, :resettable, resettable) Split.redis.hset(experiment_config_key, :algorithm, algorithm.to_s) self end |
#set_alternatives_and_options(options) ⇒ Object
42 43 44 45 46 47 48 |
# File 'lib/split/experiment.rb', line 42 def () self.alternatives = [:alternatives] self.goals = [:goals] self.resettable = [:resettable] self.algorithm = [:algorithm] self. = [:metadata] end |
#start ⇒ Object
181 182 183 |
# File 'lib/split/experiment.rb', line 181 def start Split.redis.hset(:experiment_start_times, @name, Time.now.to_i) end |
#start_time ⇒ Object
185 186 187 188 189 190 191 192 193 194 195 |
# File 'lib/split/experiment.rb', line 185 def start_time t = Split.redis.hget(:experiment_start_times, @name) if t # Check if stored time is an integer if t =~ /^[-+]?[0-9]+$/ t = Time.at(t.to_i) else t = Time.parse(t) end end end |
#validate! ⇒ Object
111 112 113 114 115 116 117 |
# File 'lib/split/experiment.rb', line 111 def validate! if @alternatives.empty? && Split.configuration.experiment_for(@name).nil? raise ExperimentNotFound.new("Experiment #{@name} not found") end @alternatives.each {|a| a.validate! } goals_collection.validate! end |
#version ⇒ Object
209 210 211 |
# File 'lib/split/experiment.rb', line 209 def version @version ||= (Split.redis.get("#{name.to_s}:version").to_i || 0) end |
#winner ⇒ Object
153 154 155 156 157 158 159 |
# File 'lib/split/experiment.rb', line 153 def winner if w = Split.redis.hget(:experiment_winner, name) Split::Alternative.new(w, name) else nil end end |
#winner=(winner_name) ⇒ Object
165 166 167 |
# File 'lib/split/experiment.rb', line 165 def winner=(winner_name) Split.redis.hset(:experiment_winner, name, winner_name.to_s) end |
#write_to_alternatives(goal = nil) ⇒ Object
323 324 325 326 327 |
# File 'lib/split/experiment.rb', line 323 def write_to_alternatives(goal = nil) alternatives.each do |alternative| alternative.set_p_winner(@alternative_probabilities[alternative], goal) end end |