roachtest: add cluster settings operations

pkg/BUILD.bazel

@@ -149,6 +149,7 @@ ALL_TESTS = [
     "//pkg/cmd/roachprod-microbench/cluster:cluster_test",
     "//pkg/cmd/roachprod-microbench:roachprod-microbench_test",
     "//pkg/cmd/roachtest/clusterstats:clusterstats_test",
+    "//pkg/cmd/roachtest/operations:operations_test",
     "//pkg/cmd/roachtest/option:option_test",
     "//pkg/cmd/roachtest/registry:registry_test",
     "//pkg/cmd/roachtest/roachtestflags:roachtestflags_test",
@@ -1164,6 +1165,7 @@ GO_TARGETS = [
     "//pkg/cmd/roachtest/grafana:grafana",
     "//pkg/cmd/roachtest/operation:operation",
     "//pkg/cmd/roachtest/operations:operations",
+    "//pkg/cmd/roachtest/operations:operations_test",
     "//pkg/cmd/roachtest/option:option",
     "//pkg/cmd/roachtest/option:option_test",
     "//pkg/cmd/roachtest/registry:registry",

pkg/cmd/roachtest/operations/BUILD.bazel

@@ -1,10 +1,11 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "operations",
     srcs = [
         "add_column.go",
         "add_index.go",
+        "cluster_settings.go",
         "disk_stall.go",
         "network_partition.go",
         "node_kill.go",
@@ -22,5 +23,13 @@ go_library(
         "//pkg/cmd/roachtest/roachtestutil",
         "//pkg/roachprod",
         "//pkg/util/randutil",
+        "//pkg/util/timeutil",
     ],
 )
+
+go_test(
+    name = "operations_test",
+    srcs = ["cluster_settings_test.go"],
+    embed = [":operations"],
+    deps = ["@com_github_stretchr_testify//require"],
+)

pkg/cmd/roachtest/operations/cluster_settings.go

@@ -0,0 +1,120 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+//
+
+package operations
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/operation"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestflags"
+	"github.com/cockroachdb/cockroach/pkg/util/randutil"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
+)
+
+type clusterSettingOp struct {
+	Name      string
+	Generator func() string
+	Owner     registry.Owner
+}
+
+func timeBasedValue(
+	timeSupplier func() time.Time, frequency time.Duration, valueForSegment func(int64) string,
+) func() string {
+	return func() string {
+		segment := timeSupplier().Unix() / int64(frequency.Seconds())
+		return valueForSegment(segment)
+	}
+}
+
+// timeBasedValues returns a function that returns a value from the given list
+// of values based on the current time and the given frequency.
+func timeBasedValues(
+	timeSupplier func() time.Time, values []string, frequency time.Duration,
+) func() string {
+	return timeBasedValue(timeSupplier, frequency, func(segment int64) string {
+		idx := int(segment) % len(values)
+		return values[idx]
+	})
+}
+
+// timeBasedRandomValue returns a function that returns a random value based on
+// the current time and the given frequency.
+func timeBasedRandomValue(
+	timeSupplier func() time.Time, frequency time.Duration, valueFromRNG func(*rand.Rand) string,
+) func() string {
+	return func() string {
+		segment := timeSupplier().Unix() / int64(frequency.Seconds())
+		return valueFromRNG(randutil.NewTestRandWithSeed(segment))
+	}
+}
+
+func setClusterSetting(
+	ctx context.Context, o operation.Operation, c cluster.Cluster, op clusterSettingOp,
+) registry.OperationCleanup {
+	value := op.Generator()
+	conn := c.Conn(ctx, o.L(), 1, option.VirtualClusterName(roachtestflags.VirtualCluster))
+	defer conn.Close()
+
+	o.Status(fmt.Sprintf("setting cluster setting %s to %s", op.Name, value))
+	_, err := conn.ExecContext(ctx, fmt.Sprintf("SET CLUSTER SETTING %s = '%s'", op.Name, value))
+	if err != nil {
+		o.Fatal(err)
+	}
+	return nil
+}
+
+func registerClusterSettings(r registry.Registry) {
+	timeSupplier := func() time.Time {
+		return timeutil.Now()
+	}
+	ops := []clusterSettingOp{
+		// Converts all leases to expiration. Tradeoff between lower throughput and higher availability.
+		// Weekly cycle.
+		{
+			Name:      "kv.expiration_leases_only.enabled",
+			Generator: timeBasedValues(timeSupplier, []string{"true", "false"}, 24*7*time.Minute),
+			Owner:     registry.OwnerKV,
+		},
+		// When running multi-store with `--wal-failover=among-stores`, this configures
+		// the threshold to trigger a fail-over to a secondary store’s WAL.
+		// 20-minute cycle.
+		{
+			Name: "storage.wal_failover.unhealthy_op_threshold",
+			Generator: timeBasedRandomValue(timeSupplier, 20*time.Minute, func(rng *rand.Rand) string {
+				return fmt.Sprintf("%d", rng.Intn(246)+5)
+			}),
+			Owner: registry.OwnerStorage,
+		},
+	}
+	sanitizeOpName := func(name string) string {
+		return strings.ReplaceAll(name, ".", "_")
+	}
+	for _, op := range ops {
+		r.AddOperation(registry.OperationSpec{
+			Name:             "cluster-settings/scheduled/" + sanitizeOpName(op.Name),
+			Owner:            op.Owner,
+			Timeout:          5 * time.Minute,
+			CompatibleClouds: registry.AllClouds,
+			Dependencies:     []registry.OperationDependency{registry.OperationRequiresNodes},
+			Run: func(ctx context.Context, o operation.Operation, c cluster.Cluster) registry.OperationCleanup {
+				return setClusterSetting(ctx, o, c, op)
+			},
+		})
+	}
+}

pkg/cmd/roachtest/operations/cluster_settings_test.go

@@ -0,0 +1,80 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+//
+
+package operations
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTimeBasedValueGenerator(t *testing.T) {
+	// Define a time supplier that steps by 10 minutes
+	curTime := time.Date(2022, 1, 1, 0, 20, 0, 0, time.UTC)
+	timeSupplier := func() time.Time {
+		defer func() {
+			curTime = curTime.Add(10 * time.Minute)
+		}()
+		return curTime
+	}
+
+	// Define the values and frequency for the test
+	values := []string{"value1", "value2", "value3"}
+	frequency := 15 * time.Minute
+
+	// Create the value generator
+	valueGenerator := timeBasedValues(timeSupplier, values, frequency)
+	// Call the value generator and check the result, one value should be generated
+	// twice and the other once, since the frequency is 15 minutes and the time
+	// supplier steps by 10 minutes.
+	counts := map[string]int{}
+	for i := 0; i < 3; i++ {
+		value := valueGenerator()
+		counts[value]++
+	}
+	expectedCounts := map[string]int{
+		"value2": 1,
+		"value3": 2,
+	}
+	require.Equal(t, expectedCounts, counts)
+}
+
+func TestTimeBasedRandomValueGenerator(t *testing.T) {
+	// Define a time supplier that steps by 10 minutes
+	curTime := time.Date(2022, 1, 1, 0, 20, 0, 0, time.UTC)
+	timeSupplier := func() time.Time {
+		defer func() {
+			curTime = curTime.Add(10 * time.Minute)
+		}()
+		return curTime
+	}
+
+	// Create the value generator
+	valueGenerator := timeBasedRandomValue(timeSupplier, 30*time.Minute, func(rng *rand.Rand) string {
+		return fmt.Sprintf("%d", rng.Intn(246)+5)
+	})
+	// Call the value generator and check the result, we expect the same value to
+	// be generated a few times since the call frequency is higher than the random
+	// value generation frequency.
+	counts := map[string]int{}
+	for i := 0; i < 10; i++ {
+		value := valueGenerator()
+		counts[value]++
+	}
+	expectedCounts := map[string]int{
+		"108": 3, "178": 3, "43": 1, "58": 3,
+	}
+	require.Equal(t, expectedCounts, counts)
+}

pkg/cmd/roachtest/operations/register.go

@@ -19,4 +19,5 @@ func RegisterOperations(r registry.Registry) {
 	registerNetworkPartition(r)
 	registerDiskStall(r)
 	registerNodeKill(r)
+	registerClusterSettings(r)
 }

pkg/cmd/roachtest/roachtestutil/operations/dependency.go

@@ -29,7 +29,7 @@ func CheckDependencies(
 	for _, dep := range spec.Dependencies {
 		switch dep {
 		case registry.OperationRequiresNodes:
-			if len(c.Nodes()) == 0 {
+			if len(c.All()) == 0 {
 				return false, nil
 			}
 		case registry.OperationRequiresPopulatedDatabase: