Class: Split::Experiment
- Inherits:
-
Object
- Object
- Split::Experiment
- Defined in:
- lib/split/experiment.rb
Constant Summary collapse
- DEFAULT_OPTIONS =
{ :resettable => true }
Instance Attribute Summary collapse
- #algorithm ⇒ Object
-
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
-
#alternatives ⇒ Object
Returns the value of attribute alternatives.
-
#goals ⇒ Object
Returns the value of attribute goals.
-
#metadata ⇒ Object
Returns the value of attribute metadata.
-
#name ⇒ Object
Returns the value of attribute name.
-
#resettable ⇒ Object
Returns the value of attribute resettable.
Instance Method Summary collapse
- #==(obj) ⇒ Object
- #[](name) ⇒ Object
- #calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
- #calc_beta_params(goal = nil) ⇒ Object
- #calc_simulated_conversion_rates(beta_params) ⇒ Object
- #calc_time ⇒ Object
- #calc_time=(time) ⇒ Object
- #calc_winning_alternatives ⇒ Object
- #control ⇒ Object
- #count_simulated_wins(winning_alternatives) ⇒ Object
- #delete ⇒ Object
- #delete_goals ⇒ Object
- #delete_metadata ⇒ Object
- #estimate_winning_alternative(goal = nil) ⇒ Object
- #extract_alternatives_from_options(options) ⇒ Object
- #find_simulated_winner(simulated_cr_hash) ⇒ Object
- #finished_key ⇒ Object
- #goals_key ⇒ Object
- #has_winner? ⇒ Boolean
- #increment_version ⇒ Object
-
#initialize(name, options = {}) ⇒ Experiment
constructor
A new instance of Experiment.
- #jstring(goal = nil) ⇒ Object
- #key ⇒ Object
- #load_from_redis ⇒ Object
- #metadata_key ⇒ Object
- #new_record? ⇒ Boolean
- #next_alternative ⇒ Object
- #participant_count ⇒ Object
- #random_alternative ⇒ Object
- #reset ⇒ Object
- #reset_winner ⇒ Object
- #resettable? ⇒ Boolean
- #save ⇒ Object
- #set_alternatives_and_options(options) ⇒ Object
- #start ⇒ Object
- #start_time ⇒ Object
- #validate! ⇒ Object
- #version ⇒ Object
- #winner ⇒ Object
- #winner=(winner_name) ⇒ Object
- #write_to_alternatives(goal = nil) ⇒ Object
Constructor Details
#initialize(name, options = {}) ⇒ Experiment
Returns a new instance of Experiment.
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
# File 'lib/split/experiment.rb', line 15 def initialize(name, = {}) = DEFAULT_OPTIONS.merge() @name = name.to_s alternatives = () if alternatives.empty? && (exp_config = Split.configuration.experiment_for(name)) ( alternatives: load_alternatives_from_configuration, goals: load_goals_from_configuration, metadata: , resettable: exp_config[:resettable], algorithm: exp_config[:algorithm] ) else ( alternatives: alternatives, goals: [:goals], metadata: [:metadata], resettable: [:resettable], algorithm: [:algorithm] ) end end |
Instance Attribute Details
#algorithm ⇒ Object
132 133 134 |
# File 'lib/split/experiment.rb', line 132 def algorithm @algorithm ||= Split.configuration.algorithm end |
#alternative_probabilities ⇒ Object
Returns the value of attribute alternative_probabilities.
8 9 10 |
# File 'lib/split/experiment.rb', line 8 def alternative_probabilities @alternative_probabilities end |
#alternatives ⇒ Object
Returns the value of attribute alternatives.
7 8 9 |
# File 'lib/split/experiment.rb', line 7 def alternatives @alternatives end |
#goals ⇒ Object
Returns the value of attribute goals.
6 7 8 |
# File 'lib/split/experiment.rb', line 6 def goals @goals end |
#metadata ⇒ Object
Returns the value of attribute metadata.
9 10 11 |
# File 'lib/split/experiment.rb', line 9 def @metadata end |
#name ⇒ Object
Returns the value of attribute name.
3 4 5 |
# File 'lib/split/experiment.rb', line 3 def name @name end |
#resettable ⇒ Object
Returns the value of attribute resettable.
5 6 7 |
# File 'lib/split/experiment.rb', line 5 def resettable @resettable end |
Instance Method Details
#==(obj) ⇒ Object
124 125 126 |
# File 'lib/split/experiment.rb', line 124 def ==(obj) self.name == obj.name end |
#[](name) ⇒ Object
128 129 130 |
# File 'lib/split/experiment.rb', line 128 def [](name) alternatives.find{|a| a.name == name} end |
#calc_alternative_probabilities(winning_counts, number_of_simulations) ⇒ Object
329 330 331 332 333 334 335 |
# File 'lib/split/experiment.rb', line 329 def calc_alternative_probabilities(winning_counts, number_of_simulations) alternative_probabilities = {} winning_counts.each do |alternative, wins| alternative_probabilities[alternative] = wins / number_of_simulations.to_f end return alternative_probabilities end |
#calc_beta_params(goal = nil) ⇒ Object
380 381 382 383 384 385 386 387 388 389 390 391 392 |
# File 'lib/split/experiment.rb', line 380 def calc_beta_params(goal = nil) beta_params = {} alternatives.each do |alternative| conversions = goal.nil? ? alternative.completed_count : alternative.completed_count(goal) alpha = 1 + conversions beta = 1 + alternative.participant_count - conversions params = [alpha, beta] beta_params[alternative] = params end return beta_params end |
#calc_simulated_conversion_rates(beta_params) ⇒ Object
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 |
# File 'lib/split/experiment.rb', line 362 def calc_simulated_conversion_rates(beta_params) # initialize a random variable (from which to simulate conversion rates ~beta-distributed) rand = SimpleRandom.new rand.set_seed simulated_cr_hash = {} # create a hash which has the conversion rate pulled from each alternative's beta distribution beta_params.each do |alternative, params| alpha = params[0] beta = params[1] simulated_conversion_rate = rand.beta(alpha, beta) simulated_cr_hash[alternative] = simulated_conversion_rate end return simulated_cr_hash end |
#calc_time ⇒ Object
398 399 400 |
# File 'lib/split/experiment.rb', line 398 def calc_time Split.redis.hget(experiment_config_key, :calc_time).to_i end |
#calc_time=(time) ⇒ Object
394 395 396 |
# File 'lib/split/experiment.rb', line 394 def calc_time=(time) Split.redis.hset(experiment_config_key, :calc_time, time) end |
#calc_winning_alternatives ⇒ Object
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 |
# File 'lib/split/experiment.rb', line 277 def calc_winning_alternatives # Super simple cache so that we only recalculate winning alternatives once per day days_since_epoch = Time.now.utc.to_i / 86400 if self.calc_time != days_since_epoch if goals.empty? self.estimate_winning_alternative else goals.each do |goal| self.estimate_winning_alternative(goal) end end self.calc_time = days_since_epoch self.save end end |
#control ⇒ Object
174 175 176 |
# File 'lib/split/experiment.rb', line 174 def control alternatives.first end |
#count_simulated_wins(winning_alternatives) ⇒ Object
337 338 339 340 341 342 343 344 345 346 347 348 |
# File 'lib/split/experiment.rb', line 337 def count_simulated_wins(winning_alternatives) # initialize a hash to keep track of winning alternative in simulations winning_counts = {} alternatives.each do |alternative| winning_counts[alternative] = 0 end # count number of times each alternative won, calculate probabilities, place in hash winning_alternatives.each do |alternative| winning_counts[alternative] += 1 end return winning_counts end |
#delete ⇒ Object
249 250 251 252 253 254 255 256 257 258 |
# File 'lib/split/experiment.rb', line 249 def delete alternatives.each(&:delete) reset_winner Split.redis.srem(:experiments, name) Split.redis.del(name) delete_goals Split.configuration.on_experiment_delete.call(self) increment_version end |
#delete_goals ⇒ Object
260 261 262 |
# File 'lib/split/experiment.rb', line 260 def delete_goals Split.redis.del(goals_key) end |
#delete_metadata ⇒ Object
264 265 266 |
# File 'lib/split/experiment.rb', line 264 def Split.redis.del() end |
#estimate_winning_alternative(goal = nil) ⇒ Object
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 |
# File 'lib/split/experiment.rb', line 296 def estimate_winning_alternative(goal = nil) # TODO - refactor out functionality to work with and without goals # initialize a hash of beta distributions based on the alternatives' conversion rates beta_params = calc_beta_params(goal) winning_alternatives = [] Split.configuration.beta_probability_simulations.times do # calculate simulated conversion rates from the beta distributions simulated_cr_hash = calc_simulated_conversion_rates(beta_params) winning_alternative = find_simulated_winner(simulated_cr_hash) # push the winning pair to the winning_alternatives array winning_alternatives.push(winning_alternative) end winning_counts = count_simulated_wins(winning_alternatives) @alternative_probabilities = calc_alternative_probabilities(winning_counts, Split.configuration.beta_probability_simulations) write_to_alternatives(goal) self.save end |
#extract_alternatives_from_options(options) ⇒ Object
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
# File 'lib/split/experiment.rb', line 49 def () alts = [:alternatives] || [] if alts.length == 1 if alts[0].is_a? Hash alts = alts[0].map{|k,v| {k => v} } end end if alts.empty? exp_config = Split.configuration.experiment_for(name) if exp_config alts = load_alternatives_from_configuration [:goals] = load_goals_from_configuration [:metadata] = [:resettable] = exp_config[:resettable] [:algorithm] = exp_config[:algorithm] end end self.alternatives = alts self.goals = [:goals] self.algorithm = [:algorithm] self.resettable = [:resettable] # calculate probability that each alternative is the winner @alternative_probabilities = {} alts end |
#find_simulated_winner(simulated_cr_hash) ⇒ Object
350 351 352 353 354 355 356 357 358 359 360 |
# File 'lib/split/experiment.rb', line 350 def find_simulated_winner(simulated_cr_hash) # figure out which alternative had the highest simulated conversion rate winning_pair = ["",0.0] simulated_cr_hash.each do |alternative, rate| if rate > winning_pair[1] winning_pair = [alternative, rate] end end winner = winning_pair[0] return winner end |
#finished_key ⇒ Object
230 231 232 |
# File 'lib/split/experiment.rb', line 230 def finished_key "#{key}:finished" end |
#goals_key ⇒ Object
226 227 228 |
# File 'lib/split/experiment.rb', line 226 def goals_key "#{name}:goals" end |
#has_winner? ⇒ Boolean
162 163 164 |
# File 'lib/split/experiment.rb', line 162 def has_winner? !winner.nil? end |
#increment_version ⇒ Object
214 215 216 |
# File 'lib/split/experiment.rb', line 214 def increment_version @version = Split.redis.incr("#{name}:version") end |
#jstring(goal = nil) ⇒ Object
402 403 404 405 406 407 408 |
# File 'lib/split/experiment.rb', line 402 def jstring(goal = nil) unless goal.nil? jstring = name + "-" + goal else jstring = name end end |
#key ⇒ Object
218 219 220 221 222 223 224 |
# File 'lib/split/experiment.rb', line 218 def key if version.to_i > 0 "#{name}:#{version}" else name end end |
#load_from_redis ⇒ Object
268 269 270 271 272 273 274 275 |
# File 'lib/split/experiment.rb', line 268 def load_from_redis exp_config = Split.redis.hgetall(experiment_config_key) self.resettable = exp_config['resettable'] self.algorithm = exp_config['algorithm'] self.alternatives = load_alternatives_from_redis self.goals = load_goals_from_redis self. = end |
#metadata_key ⇒ Object
234 235 236 |
# File 'lib/split/experiment.rb', line 234 def "#{name}:metadata" end |
#new_record? ⇒ Boolean
120 121 122 |
# File 'lib/split/experiment.rb', line 120 def new_record? !Split.redis.exists(name) end |
#next_alternative ⇒ Object
198 199 200 |
# File 'lib/split/experiment.rb', line 198 def next_alternative winner || random_alternative end |
#participant_count ⇒ Object
170 171 172 |
# File 'lib/split/experiment.rb', line 170 def participant_count alternatives.inject(0){|sum,a| sum + a.participant_count} end |
#random_alternative ⇒ Object
202 203 204 205 206 207 208 |
# File 'lib/split/experiment.rb', line 202 def random_alternative if alternatives.length > 1 algorithm.choose_alternative(self) else alternatives.first end end |
#reset ⇒ Object
242 243 244 245 246 247 |
# File 'lib/split/experiment.rb', line 242 def reset alternatives.each(&:reset) reset_winner Split.configuration.on_experiment_reset.call(self) increment_version end |
#reset_winner ⇒ Object
178 179 180 |
# File 'lib/split/experiment.rb', line 178 def reset_winner Split.redis.hdel(:experiment_winner, name) end |
#resettable? ⇒ Boolean
238 239 240 |
# File 'lib/split/experiment.rb', line 238 def resettable? resettable end |
#save ⇒ Object
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 |
# File 'lib/split/experiment.rb', line 79 def save validate! if new_record? Split.redis.sadd(:experiments, name) start unless Split.configuration.start_manually @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} save_goals Split.redis.set(, @metadata.to_json) unless @metadata.nil? else existing_alternatives = load_alternatives_from_redis existing_goals = load_goals_from_redis = unless existing_alternatives == @alternatives.map(&:name) && existing_goals == @goals && == @metadata reset @alternatives.each(&:delete) delete_goals Split.redis.del(@name) @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)} save_goals end end Split.redis.hset(experiment_config_key, :resettable, resettable) Split.redis.hset(experiment_config_key, :algorithm, algorithm.to_s) self end |
#set_alternatives_and_options(options) ⇒ Object
41 42 43 44 45 46 47 |
# File 'lib/split/experiment.rb', line 41 def () self.alternatives = [:alternatives] self.goals = [:goals] self.resettable = [:resettable] self.algorithm = [:algorithm] self. = [:metadata] end |
#start ⇒ Object
182 183 184 |
# File 'lib/split/experiment.rb', line 182 def start Split.redis.hset(:experiment_start_times, @name, Time.now.to_i) end |
#start_time ⇒ Object
186 187 188 189 190 191 192 193 194 195 196 |
# File 'lib/split/experiment.rb', line 186 def start_time t = Split.redis.hget(:experiment_start_times, @name) if t # Check if stored time is an integer if t =~ /^[-+]?[0-9]+$/ t = Time.at(t.to_i) else t = Time.parse(t) end end end |
#validate! ⇒ Object
110 111 112 113 114 115 116 117 118 |
# File 'lib/split/experiment.rb', line 110 def validate! if @alternatives.empty? && Split.configuration.experiment_for(@name).nil? raise ExperimentNotFound.new("Experiment #{@name} not found") end @alternatives.each {|a| a.validate! } unless @goals.nil? || goals.kind_of?(Array) raise ArgumentError, 'Goals must be an array' end end |
#version ⇒ Object
210 211 212 |
# File 'lib/split/experiment.rb', line 210 def version @version ||= (Split.redis.get("#{name.to_s}:version").to_i || 0) end |
#winner ⇒ Object
154 155 156 157 158 159 160 |
# File 'lib/split/experiment.rb', line 154 def winner if w = Split.redis.hget(:experiment_winner, name) Split::Alternative.new(w, name) else nil end end |
#winner=(winner_name) ⇒ Object
166 167 168 |
# File 'lib/split/experiment.rb', line 166 def winner=(winner_name) Split.redis.hset(:experiment_winner, name, winner_name.to_s) end |
#write_to_alternatives(goal = nil) ⇒ Object
323 324 325 326 327 |
# File 'lib/split/experiment.rb', line 323 def write_to_alternatives(goal = nil) alternatives.each do |alternative| alternative.set_p_winner(@alternative_probabilities[alternative], goal) end end |