lib/vanity/experiment/ab_test.rb in vanity-1.8.4 vs lib/vanity/experiment/ab_test.rb in vanity-1.9.0.beta
- old
+ new
@@ -5,27 +5,27 @@
module Vanity
module Experiment
# The meat.
class AbTest < Base
class << self
- # Convert z-score to probability.
- def probability(score)
- score = score.abs
- probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
- probability ? probability.last : 0
- end
+ # Convert z-score to probability.
+ def probability(score)
+ score = score.abs
+ probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
+ probability ? probability.last : 0
+ end
- def friendly_name
- "A/B Test"
- end
+ def friendly_name
+ "A/B Test"
+ end
end
DEFAULT_SCORE_METHOD = :z_score
def initialize(*args)
super
- @score_method = DEFAULT_SCORE_METHOD
+ @score_method = DEFAULT_SCORE_METHOD
@use_probabilities = nil
end
# -- Metric --
@@ -44,11 +44,11 @@
end
# -- Alternatives --
# Call this method once to set alternative values for this experiment
- # (requires at least two values). Call without arguments to obtain
+ # (requires at least two values). Call without arguments to obtain
# current list of alternatives.
#
# @example Define A/B test with three alternatives
# ab_test "Background color" do
# metrics :coolness
@@ -82,28 +82,28 @@
# alternative(:blue) == alternatives[2]
def alternative(value)
alternatives.find { |alt| alt.value == value }
end
- # What method to use for calculating score. Default is :ab_test, but can
+ # What method to use for calculating score. Default is :ab_test, but can
# also be set to :bayes_bandit_score to calculate probability of each
# alternative being the best.
#
# @example Define A/B test which uses bayes_bandit_score in reporting
# ab_test "noodle_test" do
# alternatives "spaghetti", "linguine"
# metrics :signup
# score_method :bayes_bandit_score
# end
def score_method(method=nil)
- if method
- @score_method = method
- end
- @score_method
+ if method
+ @score_method = method
+ end
+ @score_method
end
- # Defines an A/B test with two alternatives: false and true. This is the
+ # Defines an A/B test with two alternatives: false and true. This is the
# default pair of alternatives, so just syntactic sugar for those who love
# being explicit.
#
# @example
# ab_test "More bacon" do
@@ -114,46 +114,35 @@
def false_true
alternatives false, true
end
alias true_false false_true
- # Chooses a value for this experiment. You probably want to use the
+ # Returns fingerprint (hash) for given alternative. Can be used to lookup
+ # alternative for experiment without revealing what values are available
+ # (e.g. choosing alternative from HTTP query parameter).
+ def fingerprint(alternative)
+ Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
+ end
+
+ # Chooses a value for this experiment. You probably want to use the
# Rails helper method ab_test instead.
#
# This method picks an alternative for the current identity and returns
- # the alternative's value. It will consistently choose the same
+ # the alternative's value. It will consistently choose the same
# alternative for the same identity, and randomly split alternatives
# between different identities.
#
# @example
# color = experiment(:which_blue).choose
- def choose
+ def choose(request=nil)
if @playground.collecting?
if active?
identity = identity()
index = connection.ab_showing(@id, identity)
unless index
- index = alternative_for(identity)
- if !@playground.using_js?
- # if we have an on_assignment block, call it on new assignments
- if @on_assignment_block
- assignment = alternatives[index.to_i]
- if !connection.ab_seen @id, identity, assignment
- @on_assignment_block.call(Vanity.context, identity, assignment, self)
- end
- end
- # if we are rebalancing probabilities, keep track of how long it has been since we last rebalanced
- if @rebalance_frequency
- @assignments_since_rebalancing += 1
- if @assignments_since_rebalancing >= @rebalance_frequency
- @assignments_since_rebalancing = 0
- rebalance!
- end
- end
- connection.ab_add_participant @id, index, identity
- check_completion!
- end
+ index = alternative_for(identity).to_i
+ save_assignment_if_valid_visitor(identity, index, request) unless @playground.using_js?
end
else
index = connection.ab_get_outcome(@id) || alternative_for(identity)
end
else
@@ -163,51 +152,43 @@
index = @showing[identity]
end
alternatives[index.to_i]
end
- # Returns fingerprint (hash) for given alternative. Can be used to lookup
- # alternative for experiment without revealing what values are available
- # (e.g. choosing alternative from HTTP query parameter).
- def fingerprint(alternative)
- Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
- end
+ # -- Testing and JS Callback --
- # -- Testing --
-
- # Forces this experiment to use a particular alternative. You'll want to
- # use this from your test cases to test for the different alternatives.
+ # Forces this experiment to use a particular alternative. This may be
+ # used in test cases to force a specific alternative to obtain a
+ # deterministic test. This method also is used in the add_participant
+ # callback action when adding participants via vanity_js.
#
# @example Setup test to red button
# setup do
- # experiment(:button_color).select(:red)
+ # experiment(:button_color).chooses(:red)
# end
#
# def test_shows_red_button
# . . .
# end
#
# @example Use nil to clear selection
# teardown do
- # experiment(:green_button).select(nil)
+ # experiment(:green_button).chooses(nil)
# end
- def chooses(value)
+ def chooses(value, request=nil)
if @playground.collecting?
if value.nil?
connection.ab_not_showing @id, identity
else
index = @alternatives.index(value)
- #add them to the experiment unless they are already in it
- unless index == connection.ab_showing(@id, identity)
- connection.ab_add_participant @id, index, identity
- check_completion!
- end
+ save_assignment_if_valid_visitor(identity, index, request)
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
if (connection.ab_showing(@id, identity) && connection.ab_showing(@id, identity) != index) ||
alternative_for(identity) != index
- connection.ab_show @id, identity, index
+ connection.ab_show(@id, identity, index)
end
end
else
@showing ||= {}
@showing[identity] = value.nil? ? nil : @alternatives.index(value)
@@ -228,18 +209,18 @@
# -- Reporting --
def calculate_score
- if respond_to?(score_method)
- self.send(score_method)
- else
- score
- end
+ if respond_to?(score_method)
+ self.send(score_method)
+ else
+ score
+ end
end
- # Scores alternatives based on the current tracking data. This method
+ # Scores alternatives based on the current tracking data. This method
# returns a structure with the following attributes:
# [:alts] Ordered list of alternatives, populated with scoring info.
# [:base] Second best performing alternative.
# [:least] Least performing alternative (but more than zero conversion).
# [:choice] Choice alternative, either the outcome or best alternative.
@@ -276,11 +257,11 @@
end
# best alternative is one with highest conversion rate (best shot).
# choice alternative can only pick best if we have high probability (>90%).
best = sorted.last if sorted.last.measure > 0.0
choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
- Struct.new(:alts, :best, :base, :least, :choice, :method).new(alts, best, base, least, choice, :score)
+ Struct.new(:alts, :best, :base, :least, :choice, :method).new(alts, best, base, least, choice, :score)
end
# Scores alternatives based on the current tracking data, using Bayesian
# estimates of the best binomial bandit. Based on the R bandit package,
# http://cran.r-project.org/web/packages/bandit, which is based on
@@ -300,28 +281,28 @@
# [:difference] Difference from the least performant altenative.
#
# The choice alternative is set only if its probability is higher or
# equal to the specified probability (default is 90%).
def bayes_bandit_score(probability = 90)
- begin
- require "backports/1.9.1/kernel/define_singleton_method" if RUBY_VERSION < "1.9"
- require "integration"
- require "rubystats"
- rescue LoadError
- fail "to use bayes_bandit_score, install integration and rubystats gems"
- end
+ begin
+ require "backports/1.9.1/kernel/define_singleton_method" if RUBY_VERSION < "1.9"
+ require "integration"
+ require "rubystats"
+ rescue LoadError
+ fail "to use bayes_bandit_score, install integration and rubystats gems"
+ end
- begin
- require "gsl"
- rescue LoadError
- warn "for better integration performance, install gsl gem"
- end
+ begin
+ require "gsl"
+ rescue LoadError
+ warn "for better integration performance, install gsl gem"
+ end
- BayesianBanditScore.new(alternatives, outcome).calculate!
+ BayesianBanditScore.new(alternatives, outcome).calculate!
end
- # Use the result of #score or #bayes_bandit_score to derive a conclusion. Returns an
+ # Use the result of #score or #bayes_bandit_score to derive a conclusion. Returns an
# array of claims.
def conclusion(score = score)
claims = []
participants = score.alts.inject(0) { |t,alt| t + alt.participants }
claims << case participants
@@ -336,28 +317,28 @@
# then alternatives with no conversion.
sorted |= score.alts
# we want a result that's clearly better than 2nd best.
best, second = sorted[0], sorted[1]
if best.measure > second.measure
- diff = ((best.measure - second.measure) / second.measure * 100).round
- better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
- claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
- if score.method == :bayes_bandit_score
- if best.probability >= 90
- claims << "With %d%% probability this result is the best." % score.best.probability
- else
- claims << "This result does not have strong confidence behind it, suggest you continue this experiment."
- end
- else
- if best.probability >= 90
- claims << "With %d%% probability this result is statistically significant." % score.best.probability
- else
- claims << "This result is not statistically significant, suggest you continue this experiment."
- end
- end
- sorted.delete best
- end
+ diff = ((best.measure - second.measure) / second.measure * 100).round
+ better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
+ claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
+ if score.method == :bayes_bandit_score
+ if best.probability >= 90
+ claims << "With %d%% probability this result is the best." % score.best.probability
+ else
+ claims << "This result does not have strong confidence behind it, suggest you continue this experiment."
+ end
+ else
+ if best.probability >= 90
+ claims << "With %d%% probability this result is statistically significant." % score.best.probability
+ else
+ claims << "This result is not statistically significant, suggest you continue this experiment."
+ end
+ end
+ sorted.delete best
+ end
sorted.each do |alt|
if alt.measure > 0.0
claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
else
claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
@@ -411,11 +392,11 @@
# -- Completion --
# Defines how the experiment can choose the optimal outcome on completion.
#
# By default, Vanity will take the best alternative (highest conversion
- # rate) and use that as the outcome. You experiment may have different
+ # rate) and use that as the outcome. You experiment may have different
# needs, maybe you want the least performing alternative, or factor cost
# in the equation?
#
# The default implementation reads like this:
# outcome_is do
@@ -438,11 +419,11 @@
def complete!(outcome = nil)
return unless @playground.collecting? && active?
super
- unless outcome
+ unless outcome
if @outcome_is
begin
result = @outcome_is.call
outcome = result.id if Alternative === result && result.experiment == self
rescue
@@ -528,10 +509,48 @@
end
end
return Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
end
+ # Saves the assignment of an alternative to a person and performs the
+ # necessary housekeeping. Ignores repeat identities and filters using
+ # Playground#request_filter.
+ def save_assignment_if_valid_visitor(identity, index, request)
+ return if index == connection.ab_showing(@id, identity) || filter_visitor?(request)
+
+ call_on_assignment_if_available(identity, index)
+ rebalance_if_necessary!
+
+ connection.ab_add_participant(@id, index, identity)
+ check_completion!
+ end
+
+ def filter_visitor?(request)
+ @playground.request_filter.call(request)
+ end
+
+ def call_on_assignment_if_available(identity, index)
+ # if we have an on_assignment block, call it on new assignments
+ if @on_assignment_block
+ assignment = alternatives[index]
+ if !connection.ab_seen @id, identity, assignment
+ @on_assignment_block.call(Vanity.context, identity, assignment, self)
+ end
+ end
+ end
+
+ def rebalance_if_necessary!
+ # if we are rebalancing probabilities, keep track of how long it has been since we last rebalanced
+ if @rebalance_frequency
+ @assignments_since_rebalancing += 1
+ if @assignments_since_rebalancing >= @rebalance_frequency
+ @assignments_since_rebalancing = 0
+ rebalance!
+ end
+ end
+ end
+
begin
a = 50.0
# Returns array of [z-score, percentage]
norm_dist = []
(0.0..3.1).step(0.01) { |x| norm_dist << [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
@@ -541,10 +560,10 @@
end
module Definition
- # Define an A/B test with the given name. For example:
+ # Define an A/B test with the given name. For example:
# ab_test "New Banner" do
# alternatives :red, :green, :blue
# end
def ab_test(name, &block)
define name, :ab_test, &block