Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kvserver: use leader leases in various flow control tests #136182

Merged
merged 6 commits into from
Nov 26, 2024
Merged

kvserver: use leader leases in various flow control tests #136182

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2c89915
kvserver: use leader leases in TestFlowControlCrashedNode
arulajmani Nov 26, 2024
37956d5
kvserver: use leader leases in TestFlowControlCrashedNode2
arulajmani Nov 26, 2024
0f1d0a0
kvserver: use leader leases for TestFlowControlRaftSnapshotV2
arulajmani Nov 26, 2024
9decbb6
kvserver: use leader leases for TestFlowControlRaftMembershipV2
arulajmani Nov 26, 2024
3013817
kvserver: use epoch based leases in tests that want quiesced ranges
arulajmani Nov 26, 2024
4d9dbf3
kvserver: use leader leases in some flow control tests
arulajmani Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 100 additions & 57 deletions pkg/kv/kvserver/flow_control_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
"github.com/cockroachdb/cockroach/pkg/util/admission"
"github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/randutil"
Expand Down Expand Up @@ -592,18 +593,16 @@ func TestFlowControlCrashedNode(t *testing.T) {
// mechanism below, and for quiesced ranges, that can effectively disable
// the last-updated mechanism since quiesced ranges aren't being ticked, and
// we only check the last-updated state when ticked. So we disable range
// quiescence.
kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, true)
kvflowcontrol.Enabled.Override(ctx, &st.SV, true)

// quiescence by turning on leader leases, regardless of any metamorphism.
kvserver.OverrideDefaultLeaseType(ctx, &st.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay sticky.
manualClock := hlc.NewHybridManualClock()
tc := testcluster.StartTestCluster(t, numNodes, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Settings: st,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
// Reduce the RangeLeaseDuration to speeds up failure detection
// below.
RangeLeaseDuration: time.Second,
Expand All @@ -628,6 +627,9 @@ func TestFlowControlCrashedNode(t *testing.T) {
return true
},
},
Server: &server.TestingKnobs{
WallClock: manualClock,
},
},
},
})
Expand Down Expand Up @@ -712,7 +714,13 @@ func TestFlowControlRaftSnapshot(t *testing.T) {
st := cluster.MakeTestingClusterSettings()
kvflowcontrol.Enabled.Override(ctx, &st.SV, true)
kvflowcontrol.Mode.Override(ctx, &st.SV, kvflowcontrol.ApplyToAll)

// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &st.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay sticky.
manualClock := hlc.NewHybridManualClock()
for i := 0; i < numServers; i++ {
stickyServerArgs[i] = base.TestServerArgs{
Settings: st,
Expand All @@ -722,14 +730,10 @@ func TestFlowControlRaftSnapshot(t *testing.T) {
StickyVFSID: strconv.FormatInt(int64(i), 10),
},
},
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Server: &server.TestingKnobs{
StickyVFSRegistry: fs.NewStickyRegistry(),
WallClock: manualClock,
},
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
Expand Down Expand Up @@ -1003,17 +1007,21 @@ func TestFlowControlRaftTransportBreak(t *testing.T) {

st := cluster.MakeTestingClusterSettings()
kvflowcontrol.Enabled.Override(ctx, &st.SV, true)

// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &st.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay sticky.
manualClock := hlc.NewHybridManualClock()
tc := testcluster.StartTestCluster(t, numNodes, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Settings: st,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Server: &server.TestingKnobs{
WallClock: manualClock,
},
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
UseOnlyForScratchRanges: true,
Expand Down Expand Up @@ -1594,7 +1602,7 @@ func TestFlowControlQuiescedRange(t *testing.T) {
disableFallbackTokenDispatch.Store(true)

st := cluster.MakeTestingClusterSettings()
kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, false) // override metamorphism
kvserver.OverrideDefaultLeaseType(ctx, &st.SV, roachpb.LeaseEpoch) // override metamorphism
kvflowcontrol.Enabled.Override(ctx, &st.SV, true)

tc := testcluster.StartTestCluster(t, numNodes, base.TestClusterArgs{
Expand Down Expand Up @@ -1734,7 +1742,7 @@ func TestFlowControlUnquiescedRange(t *testing.T) {
disablePiggybackTokenDispatch.Store(true)

st := cluster.MakeTestingClusterSettings()
kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, false) // override metamorphism
kvserver.OverrideDefaultLeaseType(ctx, &st.SV, roachpb.LeaseEpoch) // override metamorphism
kvflowcontrol.Enabled.Override(ctx, &st.SV, true)

tc := testcluster.StartTestCluster(t, numNodes, base.TestClusterArgs{
Expand Down Expand Up @@ -2541,16 +2549,21 @@ func TestFlowControlAdmissionPostSplitMergeV2(t *testing.T) {
var disableWorkQueueGranting atomic.Bool
disableWorkQueueGranting.Store(true)
settings := cluster.MakeTestingClusterSettings()
// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay
// sticky.
manualClock := hlc.NewHybridManualClock()
tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Settings: settings,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Server: &server.TestingKnobs{
WallClock: manualClock,
},
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
UseOnlyForScratchRanges: true,
Expand Down Expand Up @@ -2693,15 +2706,19 @@ func TestFlowControlCrashedNodeV2(t *testing.T) {
}, func(t *testing.T, mode kvflowcontrol.ModeT) {
ctx := context.Background()
settings := cluster.MakeTestingClusterSettings()
kvserver.ExpirationLeasesOnly.Override(ctx, &settings.SV, true)
// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay
// sticky.
manualClock := hlc.NewHybridManualClock()
tc := testcluster.StartTestCluster(t, 2, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Settings: settings,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
// Reduce the RangeLeaseDuration to speeds up failure detection
// below.
RangeLeaseDuration: time.Second,
Expand All @@ -2721,6 +2738,9 @@ func TestFlowControlCrashedNodeV2(t *testing.T) {
return true
},
},
Server: &server.TestingKnobs{
WallClock: manualClock,
},
},
},
})
Expand Down Expand Up @@ -2808,6 +2828,14 @@ func TestFlowControlRaftSnapshotV2(t *testing.T) {
bypassReplicaUnreachable.Store(false)
ctx := context.Background()
settings := cluster.MakeTestingClusterSettings()
// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay
// sticky.
manualClock := hlc.NewHybridManualClock()
for i := 0; i < numServers; i++ {
stickyServerArgs[i] = base.TestServerArgs{
Settings: settings,
Expand All @@ -2817,14 +2845,10 @@ func TestFlowControlRaftSnapshotV2(t *testing.T) {
StickyVFSID: strconv.FormatInt(int64(i), 10),
},
},
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Server: &server.TestingKnobs{
StickyVFSRegistry: fs.NewStickyRegistry(),
WallClock: manualClock,
},
Store: &kvserver.StoreTestingKnobs{
RaftReportUnreachableBypass: func(_ roachpb.ReplicaID) bool {
Expand Down Expand Up @@ -3074,15 +3098,18 @@ func TestFlowControlRaftMembershipV2(t *testing.T) {
settings := cluster.MakeTestingClusterSettings()
var disableWorkQueueGranting atomic.Bool
disableWorkQueueGranting.Store(true)
// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay
// sticky.
manualClock := hlc.NewHybridManualClock()
tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Settings: settings,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to deal
// with leadership changing hands unless intentional.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
Expand All @@ -3098,6 +3125,9 @@ func TestFlowControlRaftMembershipV2(t *testing.T) {
return disableWorkQueueGranting.Load()
},
},
Server: &server.TestingKnobs{
WallClock: manualClock,
},
},
},
})
Expand Down Expand Up @@ -3453,7 +3483,7 @@ func TestFlowControlUnquiescedRangeV2(t *testing.T) {

settings := cluster.MakeTestingClusterSettings()
// Override metamorphism to allow range quiescence.
kvserver.ExpirationLeasesOnly.Override(ctx, &settings.SV, false)
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseEpoch)
tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Expand Down Expand Up @@ -3911,18 +3941,23 @@ func TestFlowControlV1ToV2Transition(t *testing.T) {
settings := cluster.MakeTestingClusterSettings()

argsPerServer := make(map[int]base.TestServerArgs)
// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay sticky.
manualClock := hlc.NewHybridManualClock()
for i := range serverLevels {
// Every node starts off using the v1 protocol but we will ratchet up the
// levels on servers at different times as we go to test the transition.
serverLevels[i].Store(kvflowcontrol.V2NotEnabledWhenLeader)
argsPerServer[i] = base.TestServerArgs{
Settings: settings,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to deal
// with leadership changing hands unintentionally.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Server: &server.TestingKnobs{
WallClock: manualClock,
},
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
UseOnlyForScratchRanges: true,
Expand Down Expand Up @@ -4550,6 +4585,13 @@ func TestFlowControlSendQueue(t *testing.T) {
kvflowcontrol.ElasticTokensPerStream.Override(ctx, &settings.SV, 2<<20)
kvflowcontrol.RegularTokensPerStream.Override(ctx, &settings.SV, 4<<20)

// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay sticky.
manualClock := hlc.NewHybridManualClock()
stickyArgsPerServer := make(map[int]base.TestServerArgs)
for i := range disableWorkQueueGrantingServers {
// Start with admission (logical token return) disabled across all nodes.
Expand All @@ -4562,14 +4604,10 @@ func TestFlowControlSendQueue(t *testing.T) {
StickyVFSID: strconv.FormatInt(int64(i), 10),
},
},
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to deal
// with leadership changing hands unintentionally.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Server: &server.TestingKnobs{
StickyVFSRegistry: fs.NewStickyRegistry(),
WallClock: manualClock,
},
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
Expand Down Expand Up @@ -4923,15 +4961,17 @@ func TestFlowControlRepeatedlySwitchMode(t *testing.T) {

ctx := context.Background()
settings := cluster.MakeTestingClusterSettings()
// This test doesn't want leadership changing hands, and leader leases (by
// virtue of raft fortification) help ensure this. Override to disable any
// metamorphosis.
kvserver.OverrideDefaultLeaseType(ctx, &settings.SV, roachpb.LeaseLeader)
// Using a manual clock here ensures that StoreLiveness support, once
// established, never expires. By extension, leadership should stay sticky.
manualClock := hlc.NewHybridManualClock()
tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
Settings: settings,
RaftConfig: base.RaftConfig{
// Suppress timeout-based elections. This test doesn't want to
// deal with leadership changing hands.
RaftElectionTimeoutTicks: 1000000,
},
Knobs: base.TestingKnobs{
Store: &kvserver.StoreTestingKnobs{
FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{
Expand All @@ -4943,6 +4983,9 @@ func TestFlowControlRepeatedlySwitchMode(t *testing.T) {
},
},
},
Server: &server.TestingKnobs{
WallClock: manualClock,
},
},
},
})
Expand Down
Loading