Class: Split::Experiment
- Inherits:
-
Object
- Object
- Split::Experiment
- Defined in:
- lib/split/experiment.rb
Constant Summary collapse
- DEFAULT_OPTIONS =
{ :resettable => true }
Instance Attribute Summary collapse
- #algorithm ⇒ Object
-
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
-
#alternatives ⇒ Object
Returns the value of attribute alternatives.
-
#goals ⇒ Object
Returns the value of attribute goals.
-
#metadata ⇒ Object
Returns the value of attribute metadata.
-
#name ⇒ Object
Returns the value of attribute name.
-
#resettable ⇒ Object
Returns the value of attribute resettable.
Instance Method Summary collapse
- #==(obj) ⇒ Object
- #[](name) ⇒ Object
- #calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
- #calc_beta_params(goal = nil) ⇒ Object
- #calc_simulated_conversion_rates(beta_params) ⇒ Object
- #calc_time ⇒ Object
- #calc_time=(time) ⇒ Object
- #calc_winning_alternatives ⇒ Object
- #control ⇒ Object
- #count_simulated_wins(winning_alternatives) ⇒ Object
- #delete ⇒ Object
- #delete_goals ⇒ Object
- #delete_metadata ⇒ Object
- #estimate_winning_alternative(goal = nil) ⇒ Object
- #extract_alternatives_from_options(options) ⇒ Object
- #find_simulated_winner(simulated_cr_hash) ⇒ Object
- #finished_key ⇒ Object
- #goals_key ⇒ Object
- #has_winner? ⇒ Boolean
- #increment_version ⇒ Object
-
#initialize(name, options = {}) ⇒ Experiment
constructor
A new instance of Experiment.
- #jstring(goal = nil) ⇒ Object
- #key ⇒ Object
- #load_from_redis ⇒ Object
- #metadata_key ⇒ Object
- #new_record? ⇒ Boolean
- #next_alternative ⇒ Object
- #participant_count ⇒ Object
- #random_alternative ⇒ Object
- #reset ⇒ Object
- #reset_winner ⇒ Object
- #resettable? ⇒ Boolean
- #save ⇒ Object
- #set_alternatives_and_options(options) ⇒ Object
- #start ⇒ Object
- #start_time ⇒ Object
- #validate! ⇒ Object
- #version ⇒ Object
- #winner ⇒ Object
- #winner=(winner_name) ⇒ Object
- #write_to_alternatives(alternative_probabilities, goal = nil) ⇒ Object
Constructor Details
#initialize(name, options = {}) ⇒ Experiment
Returns a new instance of Experiment.
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
# File 'lib/split/experiment.rb', line 15 def initialize(name, = {}) = DEFAULT_OPTIONS.merge() @name = name.to_s alternatives = () if alternatives.empty? && (exp_config = Split.configuration.experiment_for(name)) ( alternatives: load_alternatives_from_configuration, goals: load_goals_from_configuration, metadata: , resettable: exp_config[:resettable], algorithm: exp_config[:algorithm] ) else ( alternatives: alternatives, goals: [:goals], metadata: [:metadata], resettable: [:resettable], algorithm: [:algorithm] ) end end |
Instance Attribute Details
#algorithm ⇒ Object
131 132 133 |
# File 'lib/split/experiment.rb', line 131 def algorithm @algorithm ||= Split.configuration.algorithm end |
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
8 9 10 |
# File 'lib/split/experiment.rb', line 8 def alternative_probabilities @alternative_probabilities end |
#alternatives ⇒ Object
Returns the value of attribute alternatives.
7 8 9 |
# File 'lib/split/experiment.rb', line 7 def alternatives @alternatives end |
#goals ⇒ Object
Returns the value of attribute goals.
6 7 8 |
# File 'lib/split/experiment.rb', line 6 def goals @goals end |
#metadata ⇒ Object
Returns the value of attribute metadata.
9 10 11 |
# File 'lib/split/experiment.rb', line 9 def @metadata end |
#name ⇒ Object
Returns the value of attribute name.
3 4 5 |
# File 'lib/split/experiment.rb', line 3 def name @name end |
#resettable ⇒ Object
Returns the value of attribute resettable.
5 6 7 |
# File 'lib/split/experiment.rb', line 5 def resettable @resettable end |
Instance Method Details
#==(obj) ⇒ Object
123 124 125 |
# File 'lib/split/experiment.rb', line 123 def ==(obj) self.name == obj.name end |
#[](name) ⇒ Object
127 128 129 |
# File 'lib/split/experiment.rb', line 127 def [](name) alternatives.find{|a| a.name == name} end |
#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
323 324 325 326 327 328 329 |
# File 'lib/split/experiment.rb', line 323 def calc_alternative_probabilities(winning_counts, number_of_simulations) alternative_probabilities = {} winning_counts.each do |alternative, wins| alternative_probabilities[alternative] = wins / number_of_simulations.to_f end return alternative_probabilities end |
#calc_beta_params(goal = nil) ⇒ Object
374 375 376 377 378 379 380 381 382 383 384 385 386 |
# File 'lib/split/experiment.rb', line 374 def calc_beta_params(goal = nil) beta_params = {} alternatives.each do |alternative| conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal) alpha = 1 + conversions beta = 1 + alternative.participant_count - conversions params = [alpha, beta] beta_params[alternative] = params end return beta_params end |
#calc_simulated_conversion_rates(beta_params) ⇒ Object
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 |
# File 'lib/split/experiment.rb', line 356 def calc_simulated_conversion_rates(beta_params) # initialize a random variable (from which to simulate conversion rates ~beta-distributed) rand = SimpleRandom.new rand.set_seed simulated_cr_hash = {} # create a hash which has the conversion rate pulled from each alternative's beta distribution beta_params.each do |alternative, params| alpha = params[0] beta = params[1] simulated_conversion_rate = rand.beta(alpha, beta) simulated_cr_hash[alternative] = simulated_conversion_rate end return simulated_cr_hash end |
#calc_time ⇒ Object
392 393 394 |
# File 'lib/split/experiment.rb', line 392 def calc_time Split.redis.hget(experiment_config_key, :calc_time) end |
#calc_time=(time) ⇒ Object
388 389 390 |
# File 'lib/split/experiment.rb', line 388 def calc_time=(time) Split.redis.hset(experiment_config_key, :calc_time, time) end |
#calc_winning_alternatives ⇒ Object
276 277 278 279 280 281 282 283 284 285 286 287 288 |
# File 'lib/split/experiment.rb', line 276 def calc_winning_alternatives if goals.empty? self.estimate_winning_alternative else goals.each do |goal| self.estimate_winning_alternative(goal) end end calc_time = Time.now.day self.save end |
#control ⇒ Object
173 174 175 |
# File 'lib/split/experiment.rb', line 173 def control alternatives.first end |
#count_simulated_wins(winning_alternatives) ⇒ Object
331 332 333 334 335 336 337 338 339 340 341 342 |
# File 'lib/split/experiment.rb', line 331 def count_simulated_wins(winning_alternatives) # initialize a hash to keep track of winning alternative in simulations winning_counts = {} alternatives.each do |alternative| winning_counts[alternative] = 0 end # count number of times each alternative won, calculate probabilities, place in hash winning_alternatives.each do |alternative| winning_counts[alternative] += 1 end return winning_counts end |
#delete ⇒ Object
248 249 250 251 252 253 254 255 256 257 |
# File 'lib/split/experiment.rb', line 248 def delete alternatives.each(&:delete) reset_winner Split.redis.srem(:experiments, name) Split.redis.del(name) delete_goals Split.configuration.on_experiment_delete.call(self) increment_version end |
#delete_goals ⇒ Object
259 260 261 |
# File 'lib/split/experiment.rb', line 259 def delete_goals Split.redis.del(goals_key) end |
#delete_metadata ⇒ Object
263 264 265 |
# File 'lib/split/experiment.rb', line 263 def Split.redis.del() end |
#estimate_winning_alternative(goal = nil) ⇒ Object
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 |
# File 'lib/split/experiment.rb', line 290 def estimate_winning_alternative(goal = nil) # TODO - refactor out functionality to work with and without goals # initialize a hash of beta distributions based on the alternatives' conversion rates beta_params = calc_beta_params(goal) winning_alternatives = [] Split.configuration.beta_probability_simulations.times do # calculate simulated conversion rates from the beta distributions simulated_cr_hash = calc_simulated_conversion_rates(beta_params) winning_alternative = find_simulated_winner(simulated_cr_hash) # push the winning pair to the winning_alternatives array winning_alternatives.push(winning_alternative) end winning_counts = count_simulated_wins(winning_alternatives) @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations) write_to_alternatives(@alternative_probabilities, goal) self.save end |
#extract_alternatives_from_options(options) ⇒ Object
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
# File 'lib/split/experiment.rb', line 49 def () alts = [:alternatives] || [] if alts.length == 1 if alts[0].is_a? Hash alts = alts[0].map{|k,v| {k => v} } end end if alts.empty? exp_config = Split.configuration.experiment_for(name) if exp_config alts = load_alternatives_from_configuration [:goals] = load_goals_from_configuration [:metadata] = [:resettable] = exp_config[:resettable] [:algorithm] = exp_config[:algorithm] end end self.alternatives = alts self.goals = [:goals] self.algorithm = [:algorithm] self.resettable = [:resettable] # calculate probability that each alternative is the winner @alternative_probabilities = {} alts end |
#find_simulated_winner(simulated_cr_hash) ⇒ Object
344 345 346 347 348 349 350 351 352 353 354 |
# File 'lib/split/experiment.rb', line 344 def find_simulated_winner(simulated_cr_hash) # figure out which alternative had the highest simulated conversion rate winning_pair = ["",0.0] simulated_cr_hash.each do |alternative, rate| if rate > winning_pair[1] winning_pair = [alternative, rate] end end winner = winning_pair[0] return winner end |
#finished_key ⇒ Object
229 230 231 |
# File 'lib/split/experiment.rb', line 229 def finished_key "#{key}:finished" end |
#goals_key ⇒ Object
225 226 227 |
# File 'lib/split/experiment.rb', line 225 def goals_key "#{name}:goals" end |
#has_winner? ⇒ Boolean
161 162 163 |
# File 'lib/split/experiment.rb', line 161 def has_winner? !winner.nil? end |
#increment_version ⇒ Object
213 214 215 |
# File 'lib/split/experiment.rb', line 213 def increment_version @version = Split.redis.incr("#{name}:version") end |
#jstring(goal = nil) ⇒ Object
396 397 398 399 400 401 402 |
# File 'lib/split/experiment.rb', line 396 def jstring(goal = nil) unless goal.nil? jstring = name + "-" + goal else jstring = name end end |
#key ⇒ Object
217 218 219 220 221 222 223 |
# File 'lib/split/experiment.rb', line 217 def key if version.to_i > 0 "#{name}:#{version}" else name end end |
#load_from_redis ⇒ Object
267 268 269 270 271 272 273 274 |
# File 'lib/split/experiment.rb', line 267 def load_from_redis exp_config = Split.redis.hgetall(experiment_config_key) self.resettable = exp_config['resettable'] self.algorithm = exp_config['algorithm'] self.alternatives = load_alternatives_from_redis self.goals = load_goals_from_redis self. = end |
#metadata_key ⇒ Object
233 234 235 |
# File 'lib/split/experiment.rb', line 233 def "#{name}:metadata" end |
#new_record? ⇒ Boolean
119 120 121 |
# File 'lib/split/experiment.rb', line 119 def new_record? !Split.redis.exists(name) end |
#next_alternative ⇒ Object
197 198 199 |
# File 'lib/split/experiment.rb', line 197 def next_alternative winner || random_alternative end |
#participant_count ⇒ Object
169 170 171 |
# File 'lib/split/experiment.rb', line 169 def participant_count alternatives.inject(0){|sum,a| sum + a.participant_count} end |
#random_alternative ⇒ Object
201 202 203 204 205 206 207 |
# File 'lib/split/experiment.rb', line 201 def random_alternative if alternatives.length > 1 algorithm.choose_alternative(self) else alternatives.first end end |
#reset ⇒ Object
241 242 243 244 245 246 |
# File 'lib/split/experiment.rb', line 241 def reset alternatives.each(&:reset) reset_winner Split.configuration.on_experiment_reset.call(self) increment_version end |
#reset_winner ⇒ Object
177 178 179 |
# File 'lib/split/experiment.rb', line 177 def reset_winner Split.redis.hdel(:experiment_winner, name) end |
#resettable? ⇒ Boolean
237 238 239 |
# File 'lib/split/experiment.rb', line 237 def resettable? resettable end |
#save ⇒ Object
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
# File 'lib/split/experiment.rb', line 79 def save validate! if new_record? Split.redis.sadd(:experiments, name) start unless Split.configuration.start_manually @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil? Split.redis.set(, @metadata.to_json) unless @metadata.nil? else existing_alternatives = load_alternatives_from_redis existing_goals = load_goals_from_redis = unless existing_alternatives == @alternatives.map(&:name) && existing_goals == @goals && == @metadata reset @alternatives.each(&:delete) delete_goals Split.redis.del(@name) @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil? Split.redis.set(, @metadata.to_json) unless @metadata.nil? end end Split.redis.hset(experiment_config_key, :resettable, resettable) Split.redis.hset(experiment_config_key, :algorithm, algorithm.to_s) self end |
#set_alternatives_and_options(options) ⇒ Object
41 42 43 44 45 46 47 |
# File 'lib/split/experiment.rb', line 41 def () self.alternatives = [:alternatives] self.goals = [:goals] self.resettable = [:resettable] self.algorithm = [:algorithm] self. = [:metadata] end |
#start ⇒ Object
181 182 183 |
# File 'lib/split/experiment.rb', line 181 def start Split.redis.hset(:experiment_start_times, @name, Time.now.to_i) end |
#start_time ⇒ Object
185 186 187 188 189 190 191 192 193 194 195 |
# File 'lib/split/experiment.rb', line 185 def start_time t = Split.redis.hget(:experiment_start_times, @name) if t # Check if stored time is an integer if t =~ /^[-+]?[0-9]+$/ t = Time.at(t.to_i) else t = Time.parse(t) end end end |
#validate! ⇒ Object
109 110 111 112 113 114 115 116 117 |
# File 'lib/split/experiment.rb', line 109 def validate! if @alternatives.empty? && Split.configuration.experiment_for(@name).nil? raise ExperimentNotFound.new("Experiment #{@name} not found") end @alternatives.each {|a| a.validate! } unless @goals.nil? || goals.kind_of?(Array) raise ArgumentError, 'Goals must be an array' end end |
#version ⇒ Object
209 210 211 |
# File 'lib/split/experiment.rb', line 209 def version @version ||= (Split.redis.get("#{name.to_s}:version").to_i || 0) end |
#winner ⇒ Object
153 154 155 156 157 158 159 |
# File 'lib/split/experiment.rb', line 153 def winner if w = Split.redis.hget(:experiment_winner, name) Split::Alternative.new(w, name) else nil end end |
#winner=(winner_name) ⇒ Object
165 166 167 |
# File 'lib/split/experiment.rb', line 165 def winner=(winner_name) Split.redis.hset(:experiment_winner, name, winner_name.to_s) end |
#write_to_alternatives(alternative_probabilities, goal = nil) ⇒ Object
317 318 319 320 321 |
# File 'lib/split/experiment.rb', line 317 def write_to_alternatives(alternative_probabilities, goal = nil) alternatives.each do |alternative| alternative.set_p_winner(@alternative_probabilities[alternative], goal) end end |