-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
123806: roachtest: add cluster settings operations r=itsbilal a=herkolategan This change adds operations to mutate cluster settings. The values are mutated based on a preset frequency and RNG. For example, if the frequency of a cluster setting operation is set to "30 minutes" it will only change to a new value every 30 minutes, even if the operation has been run multiple times within the same 30-minute window. Running in the same window will just set the same value again. Epic: None Release Note: None Co-authored-by: Herko Lategan <[email protected]>
- Loading branch information
Showing
6 changed files
with
214 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| // Copyright 2024 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
| // | ||
|
|
||
| package operations | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "math/rand" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" | ||
| "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/operation" | ||
| "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" | ||
| "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" | ||
| "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestflags" | ||
| "github.com/cockroachdb/cockroach/pkg/util/randutil" | ||
| "github.com/cockroachdb/cockroach/pkg/util/timeutil" | ||
| ) | ||
|
|
||
| type clusterSettingOp struct { | ||
| Name string | ||
| Generator func() string | ||
| Owner registry.Owner | ||
| } | ||
|
|
||
| func timeBasedValue( | ||
| timeSupplier func() time.Time, frequency time.Duration, valueForSegment func(int64) string, | ||
| ) func() string { | ||
| return func() string { | ||
| segment := timeSupplier().Unix() / int64(frequency.Seconds()) | ||
| return valueForSegment(segment) | ||
| } | ||
| } | ||
|
|
||
| // timeBasedValues returns a function that returns a value from the given list | ||
| // of values based on the current time and the given frequency. | ||
| func timeBasedValues( | ||
| timeSupplier func() time.Time, values []string, frequency time.Duration, | ||
| ) func() string { | ||
| return timeBasedValue(timeSupplier, frequency, func(segment int64) string { | ||
| idx := int(segment) % len(values) | ||
| return values[idx] | ||
| }) | ||
| } | ||
|
|
||
| // timeBasedRandomValue returns a function that returns a random value based on | ||
| // the current time and the given frequency. | ||
| func timeBasedRandomValue( | ||
| timeSupplier func() time.Time, frequency time.Duration, valueFromRNG func(*rand.Rand) string, | ||
| ) func() string { | ||
| return func() string { | ||
| segment := timeSupplier().Unix() / int64(frequency.Seconds()) | ||
| return valueFromRNG(randutil.NewTestRandWithSeed(segment)) | ||
| } | ||
| } | ||
|
|
||
| func setClusterSetting( | ||
| ctx context.Context, o operation.Operation, c cluster.Cluster, op clusterSettingOp, | ||
| ) registry.OperationCleanup { | ||
| value := op.Generator() | ||
| conn := c.Conn(ctx, o.L(), 1, option.VirtualClusterName(roachtestflags.VirtualCluster)) | ||
| defer conn.Close() | ||
|
|
||
| o.Status(fmt.Sprintf("setting cluster setting %s to %s", op.Name, value)) | ||
| _, err := conn.ExecContext(ctx, fmt.Sprintf("SET CLUSTER SETTING %s = '%s'", op.Name, value)) | ||
| if err != nil { | ||
| o.Fatal(err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func registerClusterSettings(r registry.Registry) { | ||
| timeSupplier := func() time.Time { | ||
| return timeutil.Now() | ||
| } | ||
| ops := []clusterSettingOp{ | ||
| // Converts all leases to expiration. Tradeoff between lower throughput and higher availability. | ||
| // Weekly cycle. | ||
| { | ||
| Name: "kv.expiration_leases_only.enabled", | ||
| Generator: timeBasedValues(timeSupplier, []string{"true", "false"}, 24*7*time.Minute), | ||
| Owner: registry.OwnerKV, | ||
| }, | ||
| // When running multi-store with `--wal-failover=among-stores`, this configures | ||
| // the threshold to trigger a fail-over to a secondary store’s WAL. | ||
| // 20-minute cycle. | ||
| { | ||
| Name: "storage.wal_failover.unhealthy_op_threshold", | ||
| Generator: timeBasedRandomValue(timeSupplier, 20*time.Minute, func(rng *rand.Rand) string { | ||
| return fmt.Sprintf("%d", rng.Intn(246)+5) | ||
| }), | ||
| Owner: registry.OwnerStorage, | ||
| }, | ||
| } | ||
| sanitizeOpName := func(name string) string { | ||
| return strings.ReplaceAll(name, ".", "_") | ||
| } | ||
| for _, op := range ops { | ||
| r.AddOperation(registry.OperationSpec{ | ||
| Name: "cluster-settings/scheduled/" + sanitizeOpName(op.Name), | ||
| Owner: op.Owner, | ||
| Timeout: 5 * time.Minute, | ||
| CompatibleClouds: registry.AllClouds, | ||
| Dependencies: []registry.OperationDependency{registry.OperationRequiresNodes}, | ||
| Run: func(ctx context.Context, o operation.Operation, c cluster.Cluster) registry.OperationCleanup { | ||
| return setClusterSetting(ctx, o, c, op) | ||
| }, | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| // Copyright 2024 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
| // | ||
|
|
||
| package operations | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "math/rand" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestTimeBasedValueGenerator(t *testing.T) { | ||
| // Define a time supplier that steps by 10 minutes | ||
| curTime := time.Date(2022, 1, 1, 0, 20, 0, 0, time.UTC) | ||
| timeSupplier := func() time.Time { | ||
| defer func() { | ||
| curTime = curTime.Add(10 * time.Minute) | ||
| }() | ||
| return curTime | ||
| } | ||
|
|
||
| // Define the values and frequency for the test | ||
| values := []string{"value1", "value2", "value3"} | ||
| frequency := 15 * time.Minute | ||
|
|
||
| // Create the value generator | ||
| valueGenerator := timeBasedValues(timeSupplier, values, frequency) | ||
| // Call the value generator and check the result, one value should be generated | ||
| // twice and the other once, since the frequency is 15 minutes and the time | ||
| // supplier steps by 10 minutes. | ||
| counts := map[string]int{} | ||
| for i := 0; i < 3; i++ { | ||
| value := valueGenerator() | ||
| counts[value]++ | ||
| } | ||
| expectedCounts := map[string]int{ | ||
| "value2": 1, | ||
| "value3": 2, | ||
| } | ||
| require.Equal(t, expectedCounts, counts) | ||
| } | ||
|
|
||
| func TestTimeBasedRandomValueGenerator(t *testing.T) { | ||
| // Define a time supplier that steps by 10 minutes | ||
| curTime := time.Date(2022, 1, 1, 0, 20, 0, 0, time.UTC) | ||
| timeSupplier := func() time.Time { | ||
| defer func() { | ||
| curTime = curTime.Add(10 * time.Minute) | ||
| }() | ||
| return curTime | ||
| } | ||
|
|
||
| // Create the value generator | ||
| valueGenerator := timeBasedRandomValue(timeSupplier, 30*time.Minute, func(rng *rand.Rand) string { | ||
| return fmt.Sprintf("%d", rng.Intn(246)+5) | ||
| }) | ||
| // Call the value generator and check the result, we expect the same value to | ||
| // be generated a few times since the call frequency is higher than the random | ||
| // value generation frequency. | ||
| counts := map[string]int{} | ||
| for i := 0; i < 10; i++ { | ||
| value := valueGenerator() | ||
| counts[value]++ | ||
| } | ||
| expectedCounts := map[string]int{ | ||
| "108": 3, "178": 3, "43": 1, "58": 3, | ||
| } | ||
| require.Equal(t, expectedCounts, counts) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters