diff --git a/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go b/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go index e8ae61004cd8..9171b559884a 100644 --- a/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go +++ b/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go @@ -154,12 +154,13 @@ var ( defaultTestOptions = testOptions{ // We use fixtures more often than not as they are more likely to // detect bugs, especially in migrations. - useFixturesProbability: 0.7, - upgradeTimeout: clusterupgrade.DefaultUpgradeTimeout, - minUpgrades: 1, - maxUpgrades: 4, - minimumSupportedVersion: OldestSupportedVersion, - predecessorFunc: randomPredecessorHistory, + useFixturesProbability: 0.7, + upgradeTimeout: clusterupgrade.DefaultUpgradeTimeout, + minUpgrades: 1, + maxUpgrades: 4, + minimumSupportedVersion: OldestSupportedVersion, + predecessorFunc: randomPredecessorHistory, + overriddenMutatorProbabilities: make(map[string]float64), } // OldestSupportedVersion is the oldest cockroachdb version @@ -249,13 +250,14 @@ type ( // testOptions contains some options that can be changed by the user // that expose some control over the generated test plan and behaviour. testOptions struct { - useFixturesProbability float64 - upgradeTimeout time.Duration - minUpgrades int - maxUpgrades int - minimumSupportedVersion *clusterupgrade.Version - predecessorFunc predecessorFunc - settings []install.ClusterSettingOption + useFixturesProbability float64 + upgradeTimeout time.Duration + minUpgrades int + maxUpgrades int + minimumSupportedVersion *clusterupgrade.Version + predecessorFunc predecessorFunc + settings []install.ClusterSettingOption + overriddenMutatorProbabilities map[string]float64 } CustomOption func(*testOptions) @@ -392,6 +394,23 @@ func AlwaysUseLatestPredecessors(opts *testOptions) { opts.predecessorFunc = latestPredecessorHistory } +// WithMutatorProbability allows tests to override the default +// probability that a mutator will be applied to a test plan. +func WithMutatorProbability(name string, probability float64) CustomOption { + return func(opts *testOptions) { + opts.overriddenMutatorProbabilities[name] = probability + } +} + +// DisableMutators disables all mutators with the names passed. +func DisableMutators(names ...string) CustomOption { + return func(opts *testOptions) { + for _, name := range names { + WithMutatorProbability(name, 0)(opts) + } + } +} + // NewTest creates a Test struct that users can use to create and run // a mixed-version roachtest. func NewTest( diff --git a/pkg/cmd/roachtest/roachtestutil/mixedversion/planner.go b/pkg/cmd/roachtest/roachtestutil/mixedversion/planner.go index 04e02720b393..6aa43b5f08a0 100644 --- a/pkg/cmd/roachtest/roachtestutil/mixedversion/planner.go +++ b/pkg/cmd/roachtest/roachtestutil/mixedversion/planner.go @@ -98,7 +98,9 @@ type ( // Probability returns the probability that this mutator will run // in any given test run. Every mutator should run under some // probability. Making this an explicit part of the interface - // makes that more prominent. + // makes that more prominent. Note that tests are able to override + // this probability for specific mutator implementations as + // needed. Probability() float64 // Generate takes a test plan and a RNG and returns the list of // mutations that should be applied to the plan. @@ -242,7 +244,7 @@ func (p *testPlanner) Plan() *TestPlan { // Probabilistically enable some of of the mutators on the base test // plan generated above. for _, mut := range planMutators { - if p.prng.Float64() < mut.Probability() { + if p.mutatorEnabled(mut) { mutations := mut.Generate(p.prng, testPlan) testPlan.applyMutations(p.prng, mutations) testPlan.enabledMutators = append(testPlan.enabledMutators, mut) @@ -460,6 +462,15 @@ func (p *testPlanner) newRNG() *rand.Rand { return rngFromRNG(p.prng) } +func (p *testPlanner) mutatorEnabled(mut mutator) bool { + probability := mut.Probability() + if p, ok := p.options.overriddenMutatorProbabilities[mut.Name()]; ok { + probability = p + } + + return p.prng.Float64() < probability +} + func newUpgradePlan(from, to *clusterupgrade.Version) *upgradePlan { return &upgradePlan{ from: from, diff --git a/pkg/cmd/roachtest/roachtestutil/mixedversion/planner_test.go b/pkg/cmd/roachtest/roachtestutil/mixedversion/planner_test.go index d29ed0e8bf97..a2021e27c81e 100644 --- a/pkg/cmd/roachtest/roachtestutil/mixedversion/planner_test.go +++ b/pkg/cmd/roachtest/roachtestutil/mixedversion/planner_test.go @@ -57,11 +57,22 @@ var ( const seed = 12345 // expectations are based on this seed func TestTestPlanner(t *testing.T) { - reset := setBuildVersion() - defer reset() + // Tests run from an empty list of mutators; the only way to add + // mutators is by using the `add-mutators` directive in the + // test. This allows the output to remain stable when new mutators + // are added, and also allows us to test mutators explicitly and in + // isolation. + resetMutators := func() { planMutators = nil } + resetBuildVersion := setBuildVersion() + defer func() { + resetBuildVersion() + resetMutators() + }() datadriven.Walk(t, datapathutils.TestDataPath(t, "planner"), func(t *testing.T, path string) { + resetMutators() mvt := newTest() + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { if d.Cmd == "plan" { plan, err := mvt.plan() @@ -76,6 +87,20 @@ func TestTestPlanner(t *testing.T) { } switch d.Cmd { + case "add-mutators": + for _, arg := range d.CmdArgs { + var mut mutator + switch mutatorName := arg.Key; mutatorName { + case "concurrent_user_hooks_mutator": + mut = concurrentUserHooksMutator{} + case "remove_user_hooks_mutator": + mut = removeUserHooksMutator{} + default: + t.Fatalf("unknown mutator: %s", mutatorName) + } + + planMutators = append(planMutators, mut) + } case "mixed-version-test": mvt = createDataDrivenMixedVersionTest(t, d.CmdArgs) case "on-startup": @@ -353,6 +378,24 @@ func createDataDrivenMixedVersionTest(t *testing.T, args []datadriven.CmdArg) *T require.NoError(t, err) isLocal = boolP(b) + case "mutator_probabilities": + if len(arg.Vals)%2 != 0 { + t.Fatalf("even number of values required for %s directive", arg.Key) + } + + var j int + for j < len(arg.Vals) { + name, probStr := arg.Vals[j], arg.Vals[j+1] + prob, err := strconv.ParseFloat(probStr, 64) + require.NoError(t, err) + opts = append(opts, WithMutatorProbability(name, prob)) + + j += 2 + } + + case "disable_mutator": + opts = append(opts, DisableMutators(arg.Vals[0])) + default: t.Errorf("unknown mixed-version-test option: %s", arg.Key) } @@ -684,6 +727,42 @@ NEXT_STEP: return fmt.Errorf("no concurrent step that includes: %#v", names) } +// concurrentUserHooksMutator is a test mutator that inserts a step +// concurrently with every user-provided hook. +type concurrentUserHooksMutator struct{} + +func (concurrentUserHooksMutator) Name() string { return "concurrent_user_hooks_mutator" } +func (concurrentUserHooksMutator) Probability() float64 { return 0.5 } + +func (concurrentUserHooksMutator) Generate(rng *rand.Rand, plan *TestPlan) []mutation { + // Insert our `testSingleStep` implementation concurrently with every + // user-provided function. + return plan. + newStepSelector(). + Filter(func(s *singleStep) bool { + _, ok := s.impl.(runHookStep) + return ok + }). + InsertConcurrent(&testSingleStep{}) +} + +// removeUserHooksMutator is a test mutator that removes every +// user-provided hook from the plan. +type removeUserHooksMutator struct{} + +func (removeUserHooksMutator) Name() string { return "remove_user_hooks_mutator" } +func (removeUserHooksMutator) Probability() float64 { return 0.5 } + +func (removeUserHooksMutator) Generate(rng *rand.Rand, plan *TestPlan) []mutation { + return plan. + newStepSelector(). + Filter(func(s *singleStep) bool { + _, ok := s.impl.(runHookStep) + return ok + }). + Remove() +} + func dummyHook(context.Context, *logger.Logger, *rand.Rand, *Helper) error { return nil } diff --git a/pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/conflicting_mutators b/pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/conflicting_mutators new file mode 100644 index 000000000000..e4ac2ce90b44 --- /dev/null +++ b/pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/conflicting_mutators @@ -0,0 +1,47 @@ +# Tests that we are able to deal with mutators that insert relative to +# steps that are later removed by subsequent mutations. The initial +# insertion should create a concurrent step with user-hooks (see +# `mutator_probabilities` test) and the second mutator removes user +# hooks, flattening that concurrent run. + +add-mutators concurrent_user_hooks_mutator remove_user_hooks_mutator +---- +ok + +# ensure both mutators are always applied +mixed-version-test num_upgrades=1 mutator_probabilities=(concurrent_user_hooks_mutator, 1, remove_user_hooks_mutator, 1) +---- +ok + +in-mixed-version name=(my mixed version feature) +---- +ok + +plan debug=true +---- +mixed-version test plan for upgrading from "v22.2.8" to "" with mutators {concurrent_user_hooks_mutator, remove_user_hooks_mutator}: +├── install fixtures for version "v22.2.8" (1) [stage=cluster-setup] +├── start cluster at version "v22.2.8" (2) [stage=cluster-setup] +├── wait for nodes :1-4 to reach cluster version '22.2' (3) [stage=cluster-setup] +└── upgrade cluster from "v22.2.8" to "" + ├── prevent auto-upgrades by setting `preserve_downgrade_option` (4) [stage=init] + ├── upgrade nodes :1-4 from "v22.2.8" to "" + │ ├── restart node 4 with binary version (5) [stage=temporary-upgrade] + │ ├── restart node 2 with binary version (6) [stage=temporary-upgrade] + │ ├── restart node 3 with binary version (7) [stage=temporary-upgrade] + │ ├── testSingleStep (8) [stage=temporary-upgrade] + │ └── restart node 1 with binary version (9) [stage=temporary-upgrade] + ├── downgrade nodes :1-4 from "" to "v22.2.8" + │ ├── restart node 2 with binary version v22.2.8 (10) [stage=rollback-upgrade] + │ ├── restart node 4 with binary version v22.2.8 (11) [stage=rollback-upgrade] + │ ├── testSingleStep (12) [stage=rollback-upgrade] + │ ├── restart node 1 with binary version v22.2.8 (13) [stage=rollback-upgrade] + │ └── restart node 3 with binary version v22.2.8 (14) [stage=rollback-upgrade] + ├── upgrade nodes :1-4 from "v22.2.8" to "" + │ ├── restart node 2 with binary version (15) [stage=last-upgrade] + │ ├── restart node 4 with binary version (16) [stage=last-upgrade] + │ ├── testSingleStep (17) [stage=last-upgrade] + │ ├── restart node 1 with binary version (18) [stage=last-upgrade] + │ └── restart node 3 with binary version (19) [stage=last-upgrade] + ├── finalize upgrade by resetting `preserve_downgrade_option` (20) [stage=running-upgrade-migrations] + └── wait for nodes :1-4 to reach cluster version (21) [stage=running-upgrade-migrations] diff --git a/pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/mutator_probabilities b/pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/mutator_probabilities new file mode 100644 index 000000000000..8406139e2439 --- /dev/null +++ b/pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/mutator_probabilities @@ -0,0 +1,49 @@ +# Tests that we are able to override probabilities for specific mutators. + +add-mutators concurrent_user_hooks_mutator +---- +ok + +# ensure our `test_mutator` is always applied +mixed-version-test num_upgrades=1 mutator_probabilities=(concurrent_user_hooks_mutator, 1) +---- +ok + +in-mixed-version name=(my mixed version feature) +---- +ok + +plan debug=true +---- +mixed-version test plan for upgrading from "v22.2.8" to "" with mutators {concurrent_user_hooks_mutator}: +├── install fixtures for version "v22.2.8" (1) [stage=cluster-setup] +├── start cluster at version "v22.2.8" (2) [stage=cluster-setup] +├── wait for nodes :1-4 to reach cluster version '22.2' (3) [stage=cluster-setup] +└── upgrade cluster from "v22.2.8" to "" + ├── prevent auto-upgrades by setting `preserve_downgrade_option` (4) [stage=init] + ├── upgrade nodes :1-4 from "v22.2.8" to "" + │ ├── restart node 4 with binary version (5) [stage=temporary-upgrade] + │ ├── restart node 2 with binary version (6) [stage=temporary-upgrade] + │ ├── restart node 3 with binary version (7) [stage=temporary-upgrade] + │ ├── run following steps concurrently + │ │ ├── run "my mixed version feature", after 500ms delay (8) [stage=temporary-upgrade] + │ │ └── testSingleStep, after 5s delay (9) [stage=temporary-upgrade] + │ └── restart node 1 with binary version (10) [stage=temporary-upgrade] + ├── downgrade nodes :1-4 from "" to "v22.2.8" + │ ├── restart node 2 with binary version v22.2.8 (11) [stage=rollback-upgrade] + │ ├── restart node 4 with binary version v22.2.8 (12) [stage=rollback-upgrade] + │ ├── run following steps concurrently + │ │ ├── run "my mixed version feature", after 0s delay (13) [stage=rollback-upgrade] + │ │ └── testSingleStep, after 30s delay (14) [stage=rollback-upgrade] + │ ├── restart node 1 with binary version v22.2.8 (15) [stage=rollback-upgrade] + │ └── restart node 3 with binary version v22.2.8 (16) [stage=rollback-upgrade] + ├── upgrade nodes :1-4 from "v22.2.8" to "" + │ ├── restart node 2 with binary version (17) [stage=last-upgrade] + │ ├── restart node 4 with binary version (18) [stage=last-upgrade] + │ ├── run following steps concurrently + │ │ ├── run "my mixed version feature", after 500ms delay (19) [stage=last-upgrade] + │ │ └── testSingleStep, after 30s delay (20) [stage=last-upgrade] + │ ├── restart node 1 with binary version (21) [stage=last-upgrade] + │ └── restart node 3 with binary version (22) [stage=last-upgrade] + ├── finalize upgrade by resetting `preserve_downgrade_option` (23) [stage=running-upgrade-migrations] + └── wait for nodes :1-4 to reach cluster version (24) [stage=running-upgrade-migrations]