diff --git a/lib/split.rb b/lib/split.rb index 176023fa..667ba1e1 100755 --- a/lib/split.rb +++ b/lib/split.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'redis' +require 'split/algorithms/systematic_sampling' require 'split/algorithms/block_randomization' require 'split/algorithms/weighted_sample' require 'split/algorithms/whiplash' diff --git a/lib/split/algorithms/systematic_sampling.rb b/lib/split/algorithms/systematic_sampling.rb new file mode 100644 index 00000000..46c4a843 --- /dev/null +++ b/lib/split/algorithms/systematic_sampling.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +module Split + module Algorithms + module SystematicSampling + def self.choose_alternative(experiment) + count = experiment.next_cohorting_block_count + + block_length = experiment.cohorting_block_magnitude * experiment.alternatives.length + block_num, index = count.divmod block_length + + block = generate_block(block_num, experiment) + block[index] + end + + private + + def self.generate_block(block_num, experiment) + r = Random.new(block_num + experiment.cohorting_block_seed) + block = (experiment.alternatives*experiment.cohorting_block_magnitude).shuffle(random: r) + end + end + end +end diff --git a/lib/split/configuration.rb b/lib/split/configuration.rb index f70b1a66..28e7097f 100644 --- a/lib/split/configuration.rb +++ b/lib/split/configuration.rb @@ -164,7 +164,9 @@ def normalized_experiments algorithm: value_for(settings, :algorithm), resettable: value_for(settings, :resettable), friendly_name: value_for(settings, :friendly_name), - retain_user_alternatives_after_reset: value_for(settings, :retain_user_alternatives_after_reset) + retain_user_alternatives_after_reset: value_for(settings, :retain_user_alternatives_after_reset), + cohorting_block_seed: value_for(settings, :cohorting_block_seed), + cohorting_block_magnitude: value_for(settings, :cohorting_block_magnitude) } experiment_data.each do |name, value| diff --git a/lib/split/experiment.rb b/lib/split/experiment.rb index befdf3d5..501e28e9 100644 --- a/lib/split/experiment.rb +++ b/lib/split/experiment.rb @@ -9,6 +9,8 @@ class Experiment attr_accessor :alternative_probabilities attr_accessor :metadata attr_accessor :friendly_name + attr_accessor :cohorting_block_seed + attr_accessor :cohorting_block_magnitude attr_reader :alternatives attr_reader :resettable @@ -17,6 +19,7 @@ class Experiment DEFAULT_OPTIONS = { :resettable => true, :retain_user_alternatives_after_reset => false, + :cohorting_block_magnitude => 1 } def initialize(name, options = {}) @@ -43,6 +46,11 @@ def set_alternatives_and_options(options) self.metadata = options_with_defaults[:metadata] self.friendly_name = options_with_defaults[:friendly_name] || @name self.retain_user_alternatives_after_reset = options_with_defaults[:retain_user_alternatives_after_reset] + + if self.algorithm == Split::Algorithms::SystematicSampling + self.cohorting_block_seed = options_with_defaults[:cohorting_block_seed] || self.name.sum + self.cohorting_block_magnitude = options_with_defaults[:cohorting_block_magnitude] + end end def extract_alternatives_from_options(options) @@ -64,6 +72,8 @@ def extract_alternatives_from_options(options) options[:algorithm] = exp_config[:algorithm] options[:friendly_name] = exp_config[:friendly_name] options[:retain_user_alternatives_after_reset] = exp_config[:retain_user_alternatives_after_reset] + options[:cohorting_block_seed] = exp_config[:cohorting_block_seed] + options[:cohorting_block_magnitude] = exp_config[:cohorting_block_magnitude] end end @@ -232,6 +242,14 @@ def friendly_name_key "#{name}:friendly_name" end + def cohorting_block_seed_key + "#{name}:cohorting_block_seed" + end + + def cohorting_block_magnitude_key + "#{name}:cohorting_block_magnitude" + end + def resettable? resettable end @@ -266,6 +284,8 @@ def load_from_redis options = { retain_user_alternatives_after_reset: exp_config['retain_user_alternatives_after_reset'], + cohorting_block_seed: load_cohorting_block_seed_from_redis, + cohorting_block_magnitude: load_cohorting_block_magnitude_from_redis, resettable: exp_config['resettable'], algorithm: exp_config['algorithm'], friendly_name: load_friendly_name_from_redis, @@ -423,6 +443,10 @@ def enable_cohorting redis.hset(experiment_config_key, :cohorting, false) end + def next_cohorting_block_count + Split.redis.incr("#{key}:cohorting_block_count") - 1 + end + protected def experiment_config_key @@ -446,6 +470,14 @@ def load_friendly_name_from_redis redis.get(friendly_name_key) end + def load_cohorting_block_seed_from_redis + redis.get(cohorting_block_seed_key).to_i + end + + def load_cohorting_block_magnitude_from_redis + redis.get(cohorting_block_magnitude_key).to_i + end + def load_alternatives_from_configuration alts = Split.configuration.experiment_for(@name)[:alternatives] raise ArgumentError, "Experiment configuration is missing :alternatives array" unless alts @@ -492,6 +524,8 @@ def persist_experiment_configuration goals_collection.save redis.set(metadata_key, @metadata.to_json) unless @metadata.nil? redis.set(friendly_name_key, self.friendly_name) + redis.set(cohorting_block_seed_key, self.cohorting_block_seed) + redis.set(cohorting_block_magnitude_key, self.cohorting_block_magnitude) end def remove_experiment_configuration diff --git a/spec/algorithms/systematic_sampling_spec.rb b/spec/algorithms/systematic_sampling_spec.rb new file mode 100644 index 00000000..7cb3b9da --- /dev/null +++ b/spec/algorithms/systematic_sampling_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true +require "spec_helper" + +describe Split::Algorithms::SystematicSampling do + let(:experiment) do + Split::Experiment.new( + 'link_color', + :alternatives => ['red', 'blue', 'green'], + :algorithm => Split::Algorithms::SystematicSampling, + :cohorting_block_magnitude => 2 + ) + end + + it "should return an alternative" do + expect(Split::Algorithms::SystematicSampling.choose_alternative(experiment).class).to eq(Split::Alternative) + end + + context "experiments with a random seed" do + it "cohorts the first block of users equally into each alternative" do + results = {'red' => 0, 'blue' => 0, 'green' => 0} + 6.times do + results[Split::Algorithms::SystematicSampling.choose_alternative(experiment).name] += 1 + end + + expect(results).to eq({'red' => 2, 'blue' => 2, 'green' => 2}) + end + + it "cohorts the second block of users equally into each alternative" do + 6.times do + Split::Algorithms::SystematicSampling.choose_alternative(experiment).name + end + + results = {'red' => 0, 'blue' => 0, 'green' => 0} + 6.times do + results[Split::Algorithms::SystematicSampling.choose_alternative(experiment).name] += 1 + end + + expect(results).to eq({'red' => 2, 'blue' => 2, 'green' => 2}) + end + end + + context "experiments with set seed" do + let(:seeded_experiment1) do + Split::Experiment.new( + 'link_color', + :alternatives => ['red', 'blue', 'green'], + :algorithm => Split::Algorithms::SystematicSampling, + :cohorting_block_seed => 1234 + ) + end + + let(:seeded_experiment2) do + Split::Experiment.new( + 'link_highlight', + :alternatives => ['red', 'blue', 'green'], + :algorithm => Split::Algorithms::SystematicSampling, + :cohorting_block_seed => 1234) + end + + it "cohorts users in a set order" do + results1 = [] + results2 = [] + + 12.times do + results1 << Split::Algorithms::SystematicSampling.choose_alternative(seeded_experiment1).name + end + + 12.times do + results2 << Split::Algorithms::SystematicSampling.choose_alternative(seeded_experiment2).name + end + + expect(seeded_experiment1.cohorting_block_seed).to eq(seeded_experiment2.cohorting_block_seed) + expect(results1).to eq(results2) + end + end +end diff --git a/spec/configuration_spec.rb b/spec/configuration_spec.rb index bff3002f..0b589fe6 100644 --- a/spec/configuration_spec.rb +++ b/spec/configuration_spec.rb @@ -135,6 +135,9 @@ percent: 23 resettable: false retain_user_alternatives_after_reset: true + algorithm: Split::Algorithms::SystematicSampling + cohorting_block_seed: 999 + cohorting_block_magnitude: 3 metric: my_metric another_experiment: alternatives: @@ -145,8 +148,9 @@ end it "should normalize experiments" do - expect(@config.normalized_experiments).to eq({:my_experiment=>{:resettable=>false,:retain_user_alternatives_after_reset=>true,:alternatives=>[{"Control Opt"=>0.67}, - [{"Alt One"=>0.1}, {"Alt Two"=>0.23}]]}, :another_experiment=>{:alternatives=>["a", ["b"]]}}) + expect(@config.normalized_experiments).to eq({:my_experiment=>{:resettable=>false,:retain_user_alternatives_after_reset=>true, :cohorting_block_magnitude=>3, + :algorithm=>"Split::Algorithms::SystematicSampling", :cohorting_block_seed=>999,:alternatives=>[{"Control Opt"=>0.67},[{"Alt One"=>0.1}, {"Alt Two"=>0.23}]]}, + :another_experiment=>{:alternatives=>["a", ["b"]]}}) end it "should recognize metrics" do diff --git a/spec/experiment_spec.rb b/spec/experiment_spec.rb index 80a5b42e..763d5805 100644 --- a/spec/experiment_spec.rb +++ b/spec/experiment_spec.rb @@ -128,6 +128,16 @@ def alternative(color) expect(experiment.retain_user_alternatives_after_reset).to be_truthy end + it "should be possible to make a SystematicSampling algorithm experiment with a seeded cohorting block" do + experiment = Split::Experiment.new("basket_text", :alternatives => ["Basket", "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_seed => 123) + expect(experiment.cohorting_block_seed).to eq(123) + end + + it "should be possible to make a SystematicSampling algorithm experiment with a custom cohorting block magnitude" do + experiment = Split::Experiment.new("basket_text", :alternatives => ["Basket", "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_magnitude => 4) + expect(experiment.cohorting_block_magnitude).to eq(4) + end + it "sets friendly_name" do experiment = Split::Experiment.new('basket_text', :alternatives => ['Basket', "Cart"], :friendly_name => "foo") expect(experiment.friendly_name).to eq("foo") @@ -151,6 +161,18 @@ def alternative(color) expect(Split::Experiment.new(experiment_name).retain_user_alternatives_after_reset).to eq(false) end + it 'when the experiment is using SystematicSampling algorithm, assigns cohorting_block_magnitude to a default value' do + expect(Split::Experiment.new('systematic_sampling_exp', algorithm: Split::Algorithms::SystematicSampling).cohorting_block_magnitude).to eq(1) + end + + it 'when the experiment is using SystematicSampling algorithm, assigns cohorting_block_seed to a default value' do + expect(Split::Experiment.new('systematic_sampling_exp', algorithm: Split::Algorithms::SystematicSampling).cohorting_block_seed) + .to eq(2476) + + expect(Split::Experiment.new('systematic_sampling_exp2', algorithm: Split::Algorithms::SystematicSampling).cohorting_block_seed) + .to eq(2526) + end + it "sets friendly_name" do expect(Split::Experiment.new(experiment_name).friendly_name).to eq("foo") end @@ -185,6 +207,24 @@ def alternative(color) expect(e.retain_user_alternatives_after_reset).to be_truthy end + it "should persist cohorting_block_magnitude" do + experiment = Split::Experiment.new("basket_text", :alternatives => ['Basket', "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_magnitude => 2) + experiment.save + + e = Split::ExperimentCatalog.find("basket_text") + expect(e).to eq(experiment) + expect(e.cohorting_block_magnitude).to eq(2) + end + + it "should persist cohorting_block_seed" do + experiment = Split::Experiment.new("basket_text", :alternatives => ['Basket', "Cart"], algorithm: Split::Algorithms::SystematicSampling, :cohorting_block_seed => 12345) + experiment.save + + e = Split::ExperimentCatalog.find("basket_text") + expect(e).to eq(experiment) + expect(e.cohorting_block_seed).to eq(12345) + end + describe '#metadata' do let(:experiment) { Split::Experiment.new('basket_text', :alternatives => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash, :metadata => meta) } context 'simple hash' do @@ -427,6 +467,23 @@ def alternative(color) end end + describe '#next_cohorting_block_count' do + it 'increments each call and starts at 0' do + expect(experiment.next_cohorting_block_count).to eq(0) + expect(experiment.next_cohorting_block_count).to eq(1) + expect(experiment.next_cohorting_block_count).to eq(2) + expect(experiment.next_cohorting_block_count).to eq(3) + end + + it 'persists value' do + expect(experiment.next_cohorting_block_count).to eq(0) + experiment.save + + e = Split::ExperimentCatalog.find("link_color") + expect(e.next_cohorting_block_count).to eq(1) + end + end + describe '#next_alternative' do context 'with multiple alternatives' do let(:experiment) { Split::ExperimentCatalog.find_or_create('link_color', 'blue', 'red', 'green') }