lib/vanity/experiment/ab_test.rb in vanity-0.2.2 vs lib/vanity/experiment/ab_test.rb in vanity-0.3.0
- old
+ new
@@ -1,16 +1,17 @@
module Vanity
module Experiment
- # Experiment alternative. See AbTest#alternatives.
+ # Experiment alternative. See AbTest#alternatives and AbTest#score.
class Alternative
- def initialize(experiment, id, value) #:nodoc:
+ def initialize(experiment, id, value, participants, converted, conversions) #:nodoc:
@experiment = experiment
@id = id
- @name = "option #{(@id + 1)}"
+ @name = "option #{(@id + 65).chr}"
@value = value
+ @participants, @converted, @conversions = participants, converted, conversions
end
# Alternative id, only unique for this experiment.
attr_reader :id
@@ -18,74 +19,52 @@
attr_reader :name
# Alternative value.
attr_reader :value
+ # Experiment this alternative belongs to.
+ attr_reader :experiment
+
# Number of participants who viewed this alternative.
- def participants
- redis.scard(key("participants")).to_i
- end
+ attr_reader :participants
# Number of participants who converted on this alternative.
- def converted
- redis.scard(key("converted")).to_i
- end
+ attr_reader :converted
# Number of conversions for this alternative (same participant may be counted more than once).
- def conversions
- redis[key("conversions")].to_i
- end
+ attr_reader :conversions
- # Conversion rate calculated as converted/participants.
+ # Z-score for this alternative. Populated by AbTest#score.
+ attr_accessor :z_score
+
+ # Confidence derived from z-score. Populated by AbTest#score.
+ attr_accessor :confidence
+
+ # Difference from least performing alternative. Populated by AbTest#score.
+ attr_accessor :difference
+
+ # Conversion rate calculated as converted/participants, rounded to 3 places.
def conversion_rate
- c, p = converted.to_f, participants.to_f
- p > 0 ? c/p : 0.0
+ @rate ||= (participants > 0 ? (converted.to_f/participants.to_f).round(3) : 0.0)
end
- def <=>(other)
+ def <=>(other) # sort by conversion rate
conversion_rate <=> other.conversion_rate
end
- def participating!(identity)
- redis.sadd key("participants"), identity
+ def ==(other)
+ other && id == other.id && experiment == other.experiment
end
- def conversion!(identity)
- if redis.sismember(key("participants"), identity)
- redis.sadd key("converted"), identity
- redis.incr key("conversions")
- end
- end
-
- def destroy #:nodoc:
- redis.del key("participants")
- redis.del key("converted")
- redis.del key("conversions")
- end
-
def to_s #:nodoc:
name
end
def inspect #:nodoc:
"#{name}: #{value} #{converted}/#{participants}"
end
- protected
-
- def key(name)
- @experiment.key("alts:#{id}:#{name}")
- end
-
- def redis
- @experiment.redis
- end
-
- def base
- @base ||= @experiment.alternatives.first
- end
-
end
# The meat.
class AbTest < Base
@@ -94,43 +73,64 @@
def confidence(score) #:nodoc:
score = score.abs
confidence = AbTest::Z_TO_CONFIDENCE.find { |z,p| score >= z }
confidence ? confidence.last : 0
end
+
+ def friendly_name
+ "A/B Test"
+ end
+
end
def initialize(*args) #:nodoc:
super
+ @alternatives = [false, true]
end
# -- Alternatives --
- # Call this method once to specify values for the A/B test. At least two
- # values are required.
- #
- # Call without argument to previously defined alternatives (see Alternative).
- #
- # For example:
+ # Call this method once to set alternative values for this experiment.
+ # Require at least two values. For example:
# experiment "Background color" do
# alternatives "red", "blue", "orange"
# end
- #
+ #
+ # Call without arguments to obtain current list of alternatives. For example:
# alts = experiment(:background_color).alternatives
# puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
+ #
+ # If you want to know how well each alternative is faring, use #score.
def alternatives(*args)
- args = [false, true] if args.empty?
- @alternatives = []
- args.each_with_index do |arg, i|
- @alternatives << Alternative.new(self, i, arg)
+ unless args.empty?
+ @alternatives = args.clone
end
- class << self ; self ; end.send(:define_method, :alternatives) { @alternatives }
+ class << self
+ alias :alternatives :_alternatives
+ end
alternatives
end
+ def _alternatives #:nodoc:
+ alts = []
+ @alternatives.each_with_index do |value, i|
+ participants = redis.scard(key("alts:#{i}:participants")).to_i
+ converted = redis.scard(key("alts:#{i}:converted")).to_i
+ conversions = redis[key("alts:#{i}:conversions")].to_i
+ alts << Alternative.new(self, i, value, participants, converted, conversions)
+ end
+ alts
+ end
+
# Returns an Alternative with the specified value.
def alternative(value)
- alternatives.find { |alt| alt.value == value }
+ if index = @alternatives.index(value)
+ participants = redis.scard(key("alts:#{index}:participants")).to_i
+ converted = redis.scard(key("alts:#{index}:converted")).to_i
+ conversions = redis[key("alts:#{index}:conversions")].to_i
+ Alternative.new(self, index, value, participants, converted, conversions)
+ end
end
# Sets this test to two alternatives: false and true.
def false_true
alternatives false, true
@@ -146,32 +146,36 @@
# For example:
# color = experiment(:which_blue).choose
def choose
if active?
identity = identify
- alt = alternative_for(identity)
- alt.participating! identity
- check_completion!
- alt.value
- elsif alternative = outcome
- alternative.value
+ index = redis[key("participant:#{identity}:show")]
+ unless index
+ index = alternative_for(identity)
+ redis.sadd key("alts:#{index}:participants"), identity
+ check_completion!
+ end
else
- alternatives.first.value
+ index = redis[key("outcome")] || alternative_for(identify)
end
+ @alternatives[index.to_i]
end
# Records a conversion.
#
# For example:
# experiment(:which_blue).conversion!
def conversion!
- if active?
- identity = identify
- alt = alternative_for(identity)
- alt.conversion! identity
- check_completion!
+ return unless active?
+ identity = identify
+ return if redis[key("participants:#{identity}:show")]
+ index = alternative_for(identity)
+ if redis.sismember(key("alts:#{index}:participants"), identity)
+ redis.sadd key("alts:#{index}:converted"), identity
+ redis.incr key("alts:#{index}:conversions")
end
+ check_completion!
end
# -- Testing --
@@ -189,104 +193,118 @@
# Use nil to clear out selection:
# teardown do
# experiment(:green_button).select(nil)
# end
def chooses(value)
- alternative = alternatives.find { |alt| alt.value == value }
- raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless alternative
- Vanity.context.session[:vanity] ||= {}
- Vanity.context.session[:vanity][id] = alternative.id
+ index = @alternatives.index(value)
+ raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
+ identity = identify
+ redis[key("participant:#{identity}:show")] = index
+ self
end
+ def chosen?(alternative) #:nodoc:
+ identity = identify
+ index = redis[key("participant:#{identity}:show")]
+ index && index.to_i == alternative.id
+ end
+ # Used for testing.
+ def count(identity, value, *what) #:nodoc:
+ index = @alternatives.index(value)
+ raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
+ if what.empty? || what.include?(:participant)
+ redis.sadd key("alts:#{index}:participants"), identity
+ end
+ if what.empty? || what.include?(:conversion)
+ redis.sadd key("alts:#{index}:converted"), identity
+ redis.incr key("alts:#{index}:conversions")
+ end
+ self
+ end
+
+
# -- Reporting --
- # Returns an object with the following attributes:
- # [:alts] List of alternatives as structures (see below).
- # [:best] Best alternative.
- # [:base] Second best alternative.
- # [:choice] Choice alterntive, either selected outcome or best alternative (with confidence).
+ # Returns an object with the following methods:
+ # [:alts] List of Alternative populated with interesting statistics.
+ # [:best] Best performing alternative.
+ # [:base] Second best performing alternative.
+ # [:least] Least performing alternative (but more than zero conversion).
+ # [:choice] Choice alterntive, either the outcome or best alternative (if confidence >= 90%).
#
- # Each alternative is an object with the following attributes:
- # [:id] Identifier.
- # [:conv] Conversion rate (0.0 to 1.0, rounded to 3 places).
- # [:pop] Population size (participants).
- # [:diff] Difference from least performant altenative (percentage).
- # [:z] Z-score compared to base (above).
- # [:conf] Confidence based on z-score (0, 90, 95, 99, 99.9).
+ # Alternatives returned by this method are populated with the following attributes:
+ # [:z_score] Z-score (relative to the base alternative).
+ # [:confidence] Confidence (z-score mapped to 0, 90, 95, 99 or 99.9%).
+ # [:difference] Difference from the least performant altenative.
def score
- struct = Struct.new(:id, :conv, :pop, :diff, :z, :conf)
- alts = alternatives.map { |alt| struct.new(alt.id, alt.conversion_rate.round(3), alt.participants) }
+ alts = alternatives
# sort by conversion rate to find second best and 2nd best
- sorted = alts.sort_by(&:conv)
+ sorted = alts.sort_by(&:conversion_rate)
base = sorted[-2]
# calculate z-score
- pc = base.conv
- nc = base.pop
+ pc = base.conversion_rate
+ nc = base.participants
alts.each do |alt|
- p = alt.conv
- n = alt.pop
- alt.z = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
- alt.conf = AbTest.confidence(alt.z)
+ p = alt.conversion_rate
+ n = alt.participants
+ alt.z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
+ alt.confidence = AbTest.confidence(alt.z_score)
end
# difference is measured from least performant
- if least = sorted.find { |alt| alt.conv > 0 }
+ if least = sorted.find { |alt| alt.conversion_rate > 0 }
alts.each do |alt|
- alt.diff = (alt.conv - least.conv) / least.conv * 100 if alt.conv > least.conv
+ if alt.conversion_rate > least.conversion_rate
+ alt.difference = (alt.conversion_rate - least.conversion_rate) / least.conversion_rate * 100
+ end
end
end
# best alternative is one with highest conversion rate (best shot).
# choice alternative can only pick best if we have high confidence (>90%).
- best = sorted.last if sorted.last.conv > 0
- choice = outcome ? alts[outcome.id] : (best && best.conf >= 90 ? best : nil)
- Struct.new(:alts, :best, :base, :choice).new(alts, best, base, choice)
+ best = sorted.last if sorted.last.conversion_rate > 0.0
+ choice = outcome ? alts[outcome.id] : (best && best.confidence >= 90 ? best : nil)
+ Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
end
# Use the score returned by #score to derive a conclusion. Returns an
# array of claims.
def conclusion(score = score)
claims = []
- # find name form alt structure returned from score
- name = ->(alt){ alternatives[alt.id].name }
# only interested in sorted alternatives with conversion
- sorted = score.alts.select { |alt| alt.conv > 0.0 }.sort_by(&:conv).reverse
+ sorted = score.alts.select { |alt| alt.conversion_rate > 0.0 }.sort_by(&:conversion_rate).reverse
if sorted.size > 1
# start with alternatives that have conversion, from best to worst,
# then alternatives with no conversion.
sorted |= score.alts
# we want a result that's clearly better than 2nd best.
best, second = sorted[0], sorted[1]
- if best.conv > second.conv
- diff = ((best.conv - second.conv) / second.conv * 100).round
- better = " (%d%% better than %s)" % [diff, name[second]] if diff > 0
- claims << "The best choice is %s: it converted at %.1f%%%s." % [name[best], best.conv * 100, better]
- if best.conf >= 90
- claims << "With %d%% probability this result is statistically significant." % score.best.conf
+ if best.conversion_rate > second.conversion_rate
+ diff = ((best.conversion_rate - second.conversion_rate) / second.conversion_rate * 100).round
+ better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
+ claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.conversion_rate * 100, better]
+ if best.confidence >= 90
+ claims << "With %d%% probability this result is statistically significant." % score.best.confidence
else
claims << "This result is not statistically significant, suggest you continue this experiment."
end
sorted.delete best
end
sorted.each do |alt|
- if alt.conv > 0.0
- claims << "%s converted at %.1f%%." % [name[alt].capitalize, alt.conv * 100]
+ if alt.conversion_rate > 0.0
+ claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.conversion_rate * 100]
else
- claims << "%s did not convert." % name[alt].capitalize
+ claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
end
end
else
claims << "This experiment did not run long enough to find a clear winner."
end
- claims << "#{name[score.choice].capitalize} selected as the best alternative." if score.choice
+ claims << "#{score.choice.name.gsub(/^o/, "O")} selected as the best alternative." if score.choice
claims
end
- def humanize
- "A/B Test"
- end
-
# -- Completion --
# Defines how the experiment can choose the optimal outcome on completion.
#
# The default implementation looks for the best (highest conversion rate)
@@ -309,57 +327,58 @@
def outcome
outcome = redis[key("outcome")]
outcome && alternatives[outcome.to_i]
end
- def complete! #:nodoc:
+ def complete!
+ return unless active?
super
if @outcome_is
begin
- outcome = alternatives.find_index(@outcome_is.call)
+ result = @outcome_is.call
+ outcome = result.id if result && result.experiment == self
rescue
# TODO: logging
end
- end
- unless outcome
+ else
best = score.best
outcome = best.id if best
end
# TODO: logging
- redis.setnx key("outcome"), outcome
+ redis.setnx key("outcome"), outcome || 0
end
# -- Store/validate --
- def save #:nodoc:
+ def save
fail "Experiment #{name} needs at least two alternatives" unless alternatives.count >= 2
super
end
- def reset! #:nodoc:
+ def reset!
+ @alternatives.count.times do |i|
+ redis.del key("alts:#{i}:participants")
+ redis.del key("alts:#{i}:converted")
+ redis.del key("alts:#{i}:conversions")
+ end
redis.del key(:outcome)
- alternatives.each(&:destroy)
super
end
- def destroy #:nodoc:
- redis.del key(:outcome)
- alternatives.each(&:destroy)
+ def destroy
+ reset
super
end
private
# Chooses an alternative for the identity and returns its index. This
# method always returns the same alternative for a given experiment and
# identity, and randomly distributed alternatives for each identity (in the
# same experiment).
def alternative_for(identity)
- session = Vanity.context.session[:vanity]
- index = session && session[id]
- index ||= Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % alternatives.count
- alternatives[index]
+ Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.count
end
begin
a = 0
# Returns array of [z-score, percentage]