From 4fdc2033384b09f570555027564271804ed19386 Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Mon, 9 Dec 2024 20:56:09 -0500 Subject: [PATCH] rac2: improve TestRangeController wrt match and inflight-bytes This is in preparation for RangeController needing the per-replica inflight bytes. The existing test was unnecessarily and implicitly changing match. This happened in SendMsgAppRaftMuLocked, handling of set_replicas and raft_event. This is now changed to only adjust match explicitly via admit. And set_replicas will set match to 0, but there is now a provision to explicitly provide a match value, which is used to preserve an existing match value. This permits maintaining ReplicaStateInfo.InflightBytes (which is currently only relevant to the test) in the test code, by adjusting it when match and next are changed. This is done via looking at entries in MemoryStorage. The wait_for_eval_send_q is slightly modified, since it needs to actually send the entry at index 1, so that it can be retrieved from MemoryStorage. Informs #135814 Epic: CRDB-37515 Release note: None --- .../kvserver/kvflowcontrol/rac2/BUILD.bazel | 1 + .../kvflowcontrol/rac2/range_controller.go | 3 + .../rac2/range_controller_test.go | 118 ++++++++++++++---- .../rac2/testdata/range_controller/close | 8 +- .../testdata/range_controller/do_force_flush | 8 +- .../range_controller/force_flush_index | 20 +-- .../force_flush_index_push_pull | 12 +- .../force_flush_no_send_q_joint_config | 12 +- .../range_controller/handle_raft_event | 31 +++-- .../handle_raft_event_partial_msg_app | 6 +- .../leaseholder_force_flush_no_send_q | 12 +- .../range_controller/no_send_q_choice | 12 +- .../testdata/range_controller/probe_state | 2 +- .../range_controller/quorum_without_send_q | 6 +- .../send_q_entries_without_ac | 26 ++-- .../testdata/range_controller/send_q_watcher | 10 +- .../testdata/range_controller/sendq_push_pull | 8 +- .../range_controller/wait_for_eval_send_q | 72 +++++++---- 18 files changed, 239 insertions(+), 128 deletions(-) diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/BUILD.bazel b/pkg/kv/kvserver/kvflowcontrol/rac2/BUILD.bazel index 2c78422bb6ec..33e2e5899050 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/BUILD.bazel +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/BUILD.bazel @@ -82,6 +82,7 @@ go_test( "//pkg/util/syncutil", "//pkg/util/timeutil", "@com_github_cockroachdb_datadriven//:datadriven", + "@com_github_cockroachdb_errors//:errors", "@com_github_dustin_go_humanize//:go-humanize", "@com_github_gogo_protobuf//jsonpb", "@com_github_guptarohit_asciigraph//:asciigraph", diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller.go b/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller.go index 2e0ff31ebe28..0a328bc0b5d5 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller.go +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller.go @@ -189,6 +189,9 @@ type ReplicaStateInfo struct { // (Match, Next) is in-flight. Match uint64 Next uint64 + // InflightBytes are the bytes that have been sent but not yet persisted. It + // corresponds to tracker.Inflights.bytes. + InflightBytes uint64 } // sendQueueStatRefreshInterval is the interval at which the send queue stats diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller_test.go b/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller_test.go index e483c53cc29a..0ccf958ad541 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller_test.go +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/range_controller_test.go @@ -9,6 +9,7 @@ import ( "cmp" "context" "fmt" + "math" "slices" "sort" "strconv" @@ -39,6 +40,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/datadriven" + "github.com/cockroachdb/errors" "github.com/gogo/protobuf/jsonpb" "github.com/stretchr/testify/require" ) @@ -370,15 +372,20 @@ func (s *testingRCState) getOrInitRange( s.ranges[r.rangeID] = testRC } s.maybeSetInitialTokens(r) - // Send through an empty raft event to trigger creating necessary replica - // send streams for the range. - event := testRC.makeRaftEventWithReplicasState() - event.MsgAppMode = mode - func() { - testRC.rc.opts.ReplicaMutexAsserter.RaftMu.Lock() - defer testRC.rc.opts.ReplicaMutexAsserter.RaftMu.Unlock() - require.NoError(t, testRC.rc.HandleRaftEventRaftMuLocked(s.testCtx, event)) - }() + if !ok { + // Send through an empty raft event to trigger creating necessary replica + // send streams for the range. + event := testRC.makeRaftEventWithReplicasState() + event.MsgAppMode = mode + func() { + testRC.rc.opts.ReplicaMutexAsserter.RaftMu.Lock() + defer testRC.rc.opts.ReplicaMutexAsserter.RaftMu.Unlock() + require.NoError(t, testRC.rc.HandleRaftEventRaftMuLocked(s.testCtx, event)) + }() + } + // Else, this is an existing testingRCRange. The caller may want to send an + // empty raft event too, to enact some changes. + return testRC } @@ -484,8 +491,7 @@ func (r *testingRCRange) SendMsgAppRaftMuLocked( if !ok { panic("unknown replica") } - testR.info.Match = max(msg.Entries[0].Index-1, testR.info.Match) - testR.info.Next = msg.Entries[len(msg.Entries)-1].Index + 1 + testR = updateNext(r, testR, msg.Entries[len(msg.Entries)-1].Index+1) r.mu.r.replicaSet[replicaID] = testR return msg, true } @@ -529,7 +535,7 @@ func (r *testingRCRange) admit(ctx context.Context, storeID roachpb.StoreID, av for _, v := range av.Admitted { // Ensure that Match doesn't lag behind the highest index in the // AdmittedVector. - replica.info.Match = max(replica.info.Match, v) + replica = tryUpdateMatch(r, replica, v) } replicaID = replica.desc.ReplicaID r.mu.r.replicaSet[replicaID] = replica @@ -548,6 +554,23 @@ func (r *testingRCRange) admit(ctx context.Context, storeID roachpb.StoreID, av r.rc.AdmitRaftMuLocked(ctx, replicaID, av) } +func (r *testingRCRange) updateReplicas(t *testing.T, tr testingRange) { + r.mu.Lock() + defer r.mu.Unlock() + for replicaID, replica := range tr.replicaSet { + require.Equal(t, uint64(0), replica.info.InflightBytes) + if replica.info.Match+1 < replica.info.Next { + entries, err := r.raftLog.Entries(replica.info.Match+1, replica.info.Next, math.MaxUint64) + require.NoError(t, err) + for i := range entries { + replica.info.InflightBytes += uint64(len(entries[i].Data)) + } + tr.replicaSet[replicaID] = replica + } + } + r.mu.r = tr +} + type testingRange struct { rangeID roachpb.RangeID tenantID roachpb.TenantID @@ -598,6 +621,45 @@ type testingReplica struct { info ReplicaStateInfo } +func tryUpdateMatch(r *testingRCRange, replica testingReplica, match uint64) testingReplica { + if match <= replica.info.Match { + return replica + } + entries, err := r.raftLog.Entries(replica.info.Match+1, match+1, math.MaxUint64) + if err != nil { + panic(err) + } + for i := range entries { + entrySize := uint64(len(entries[i].Data)) + if replica.info.InflightBytes < entrySize { + panic(errors.Errorf( + "inflight-bytes %d < entrySize %d at index %d", replica.info.InflightBytes, entrySize, i)) + } + replica.info.InflightBytes -= entrySize + } + replica.info.Match = match + return replica +} + +func updateNext(r *testingRCRange, replica testingReplica, next uint64) testingReplica { + if next < replica.info.Next { + panic(errors.Errorf("next %d < replica.info.Next %d", next, replica.info.Next)) + } + if next == replica.info.Next { + return replica + } + entries, err := r.raftLog.Entries(replica.info.Next, next, math.MaxUint64) + if err != nil { + panic(err) + } + for i := range entries { + entrySize := uint64(len(entries[i].Data)) + replica.info.InflightBytes += entrySize + } + replica.info.Next = next + return replica +} + func scanRanges(t *testing.T, input string) []testingRange { replicas := []testingRange{} @@ -694,11 +756,12 @@ func scanReplica(t *testing.T, line string) testingReplica { } next := uint64(0) + match := uint64(0) // The fourth field is optional, if set it contains the tracker state of the // replica on the leader replica (localReplicaID). The valid states are // Probe, Replicate, and Snapshot. if len(parts) > 3 { - require.Equal(t, 5, len(parts)) + require.LessOrEqual(t, 5, len(parts)) parts[3] = strings.TrimSpace(parts[3]) require.True(t, strings.HasPrefix(parts[3], "state=")) parts[3] = strings.TrimPrefix(strings.TrimSpace(parts[3]), "state=") @@ -718,6 +781,15 @@ func scanReplica(t *testing.T, line string) testingReplica { nextInt, err := strconv.Atoi(parts[4]) require.NoError(t, err) next = uint64(nextInt) + if len(parts) > 5 { + require.Equal(t, 6, len(parts)) + parts[5] = strings.TrimSpace(parts[5]) + require.True(t, strings.HasPrefix(parts[5], "match=")) + parts[5] = strings.TrimPrefix(strings.TrimSpace(parts[5]), "match=") + matchInt, err := strconv.Atoi(parts[5]) + require.NoError(t, err) + match = uint64(matchInt) + } } return testingReplica{ @@ -727,7 +799,7 @@ func scanReplica(t *testing.T, line string) testingReplica { ReplicaID: roachpb.ReplicaID(replicaID), Type: replicaType, }, - info: ReplicaStateInfo{State: state, Next: next}, + info: ReplicaStateInfo{State: state, Match: match, Next: next}, } } @@ -1165,11 +1237,7 @@ func TestRangeController(t *testing.T) { } for _, r := range scanRanges(t, d.Input) { testRC := state.getOrInitRange(t, r, mode) - func() { - testRC.mu.Lock() - defer testRC.mu.Unlock() - testRC.mu.r = r - }() + testRC.updateReplicas(t, r) func() { testRC.rc.opts.ReplicaMutexAsserter.RaftMu.Lock() defer testRC.rc.opts.ReplicaMutexAsserter.RaftMu.Unlock() @@ -1356,11 +1424,9 @@ func TestRangeController(t *testing.T) { // Else MsgAppPull mode, so raftEvent.MsgApps is unpopulated. if len(msgApp.Entries) > 0 { - // Bump the Next and Index fields for replicas that have - // MsgApps being sent to them. The Match index is only updated - // if it increases. - testR.info.Match = max(msgApp.Entries[0].Index-1, testR.info.Match) - testR.info.Next = msgApp.Entries[len(msgApp.Entries)-1].Index + 1 + // Bump the Next field for replicas that have MsgApps being + // sent to them. + testR = updateNext(testRC, testR, msgApp.Entries[len(msgApp.Entries)-1].Index+1) testRC.mu.r.replicaSet[replicaID] = testR } else if testR.desc.ReplicaID == testRC.mu.r.localReplicaID && len(raftEvent.Entries) > 0 { @@ -1368,8 +1434,8 @@ func TestRangeController(t *testing.T) { // // TODO(sumeer): many of the test cases are sending MsgApps to // the leader. Stop doing it. - testR.info.Match = max(raftEvent.Entries[0].Index-1, testR.info.Match) - testR.info.Next = raftEvent.Entries[len(raftEvent.Entries)-1].Index + 1 + testR = updateNext( + testRC, testR, raftEvent.Entries[len(raftEvent.Entries)-1].Index+1) testRC.mu.r.replicaSet[replicaID] = testR } } diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/close b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/close index 23acb9031b93..c98916e49cf2 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/close +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/close @@ -114,21 +114,21 @@ NormalPri: # tracked deductions for entries in [1,3]. set_replicas range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=4 - store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=4 - store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=4 + store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=4 match=2 + store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=4 match=1 store_id=3 replica_id=3 type=VOTER_FULL state=StateSnapshot next=4 ---- r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3] stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=3 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/do_force_flush b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/do_force_flush index 04c717c1579e..9b4a9d609d5f 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/do_force_flush +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/do_force_flush @@ -164,7 +164,7 @@ t1/s3: eval reg=-4.5 MiB/+16 MiB ela=-4.5 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[4,7) send_queue=[7,7) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,7) send_queue=[7,7) precise_q_size=+0 B eval deducted: reg=+7.5 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -183,7 +183,7 @@ LowPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[4,7) send_queue=[7,7) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,7) send_queue=[7,7) precise_q_size=+0 B eval deducted: reg=+4.5 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -247,7 +247,7 @@ NormalPri: term=1 index=5 tokens=1572864 term=1 index=6 tokens=1572864 ++++ -(n2,s2):2: state=replicate closed=false inflight=[4,6) send_queue=[6,7) precise_q_size=+1.5 MiB force-flushing +(n2,s2):2: state=replicate closed=false inflight=[1,6) send_queue=[6,7) precise_q_size=+1.5 MiB force-flushing eval deducted: reg=+0 B ela=+7.5 MiB eval original in send-q: reg=+1.5 MiB ela=+0 B LowPri: @@ -279,7 +279,7 @@ NormalPri: term=1 index=5 tokens=1572864 term=1 index=6 tokens=1572864 ++++ -(n2,s2):2: state=replicate closed=false inflight=[6,7) send_queue=[7,7) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,7) send_queue=[7,7) precise_q_size=+0 B eval deducted: reg=+0 B ela=+7.5 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index index 2eeb748a43f9..42a37562a621 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index @@ -141,7 +141,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,6) precise_q_size=+18 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,6) precise_q_size=+18 MiB watching-for-tokens eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+18 MiB ela=+0 B LowPri: @@ -176,7 +176,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,6) precise_q_size=+18 MiB force-flushing (stop=3) +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,6) precise_q_size=+18 MiB force-flushing (stop=3) eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+18 MiB ela=+0 B LowPri: @@ -210,7 +210,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,4) send_queue=[4,6) precise_q_size=+12 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,6) precise_q_size=+12 MiB watching-for-tokens eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+12 MiB ela=+0 B LowPri: @@ -246,7 +246,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,4) send_queue=[4,6) precise_q_size=+12 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,6) precise_q_size=+12 MiB watching-for-tokens eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+12 MiB ela=+0 B LowPri: @@ -279,7 +279,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,4) send_queue=[4,6) precise_q_size=+12 MiB force-flushing (stop=4) +(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,6) precise_q_size=+12 MiB force-flushing (stop=4) eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+12 MiB ela=+0 B LowPri: @@ -349,7 +349,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[4,5) send_queue=[5,6) precise_q_size=+6.0 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,5) send_queue=[5,6) precise_q_size=+6.0 MiB watching-for-tokens eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: @@ -388,7 +388,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[4,5) send_queue=[5,6) precise_q_size=+6.0 MiB force-flushing +(n2,s2):2: state=replicate closed=false inflight=[1,5) send_queue=[5,6) precise_q_size=+6.0 MiB force-flushing eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: @@ -415,7 +415,7 @@ NormalPri: term=1 index=4 tokens=6291456 term=1 index=5 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[5,6) send_queue=[6,6) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,6) send_queue=[6,6) precise_q_size=+0 B eval deducted: reg=+0 B ela=+30 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -447,7 +447,7 @@ t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[6,7) send_queue=[7,7) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,7) send_queue=[7,7) precise_q_size=+0 B eval deducted: reg=+36 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -458,7 +458,7 @@ NormalPri: term=1 index=5 tokens=6291456 term=1 index=6 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[6,7) send_queue=[7,7) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,7) send_queue=[7,7) precise_q_size=+0 B eval deducted: reg=+6.0 MiB ela=+30 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index_push_pull b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index_push_pull index 785633082977..9b630daebe95 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index_push_pull +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_index_push_pull @@ -247,14 +247,14 @@ NormalPri: term=1 index=2 tokens=6291456 term=1 index=3 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+6.0 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+6.0 MiB watching-for-tokens eval deducted: reg=+0 B ela=+18 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: term=1 index=1 tokens=6291456 term=1 index=2 tokens=6291456 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+6.0 MiB force-flushing +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+6.0 MiB force-flushing eval deducted: reg=+0 B ela=+18 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: @@ -300,14 +300,14 @@ NormalPri: term=1 index=2 tokens=6291456 term=1 index=3 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+6.0 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+6.0 MiB watching-for-tokens eval deducted: reg=+0 B ela=+18 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: term=1 index=1 tokens=6291456 term=1 index=2 tokens=6291456 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+6.0 MiB force-flushing +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+6.0 MiB force-flushing eval deducted: reg=+0 B ela=+18 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: @@ -328,14 +328,14 @@ NormalPri: term=1 index=2 tokens=6291456 term=1 index=3 tokens=6291456 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+6.0 MiB watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+6.0 MiB watching-for-tokens eval deducted: reg=+0 B ela=+18 MiB eval original in send-q: reg=+6.0 MiB ela=+0 B LowPri: term=1 index=1 tokens=6291456 term=1 index=2 tokens=6291456 ++++ -(n3,s3):3: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+18 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_no_send_q_joint_config b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_no_send_q_joint_config index a97fe33185c5..3de5a33f1741 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_no_send_q_joint_config +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/force_flush_no_send_q_joint_config @@ -157,7 +157,7 @@ t1/s4: eval reg=-1.0 MiB/+16 MiB ela=-2.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -166,13 +166,13 @@ NormalPri: ++++ (n2,s2):2: closed ++++ -(n3,s3):3VOTER_DEMOTING_LEARNER: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n3,s3):3VOTER_DEMOTING_LEARNER: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=2 tokens=1048576 ++++ -(n4,s4):4VOTER_INCOMING: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n4,s4):4VOTER_INCOMING: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -254,7 +254,7 @@ t1/s4: eval reg=-2.0 MiB/+16 MiB ela=-3.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+3.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -266,14 +266,14 @@ NormalPri: eval deducted: reg=+0 B ela=+1.0 MiB eval original in send-q: reg=+1.0 MiB ela=+0 B ++++ -(n3,s3):3VOTER_DEMOTING_LEARNER: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3VOTER_DEMOTING_LEARNER: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n4,s4):4VOTER_INCOMING: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n4,s4):4VOTER_INCOMING: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event index 078bc218152f..224a0ae88747 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event @@ -115,8 +115,8 @@ NormalPri: # delay. set_replicas range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=4 - store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=4 - store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=4 + store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=4 match=3 + store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=4 match=3 store_id=3 replica_id=3 type=VOTER_FULL state=StateProbe next=4 ---- r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3] @@ -130,11 +130,11 @@ now=200ms stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[4,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B ++++ -(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[4,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B ++++ @@ -151,11 +151,11 @@ now=400ms stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[4,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B ++++ -(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[4,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B ++++ @@ -199,14 +199,29 @@ check_state range_id=1 tenant_id={1} local_replica_id=1 name=a pri=low-pri done=false waited=false err= +stream_state range_id=1 +---- +(n1,s1):1: state=replicate closed=false inflight=[5,5) send_queue=[5,5) precise_q_size=+0 B +eval deducted: reg=+0 B ela=+0 B +eval original in send-q: reg=+0 B ela=+0 B +++++ +(n2,s2):2: state=replicate closed=false inflight=[4,5) send_queue=[5,5) precise_q_size=+0 B +eval deducted: reg=+16 MiB ela=+0 B +eval original in send-q: reg=+0 B ela=+0 B +NormalPri: + term=1 index=4 tokens=16777216 +++++ +(n3,s3):3: closed +++++ + # Change the state of s3 to replicate and s2 to StateSnapshot, this should # trigger the operation to refresh, ignore s2 now that it is in StateProbe and # check s3 for available tokens as it is now in StateReplicate. set_replicas range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=5 - store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=5 + store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=5 match=4 store_id=2 replica_id=2 type=VOTER_FULL state=StateSnapshot next=4 - store_id=3 replica_id=3 type=VOTER_FULL state=StateReplicate next=5 + store_id=3 replica_id=3 type=VOTER_FULL state=StateReplicate next=5 match=4 ---- r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3] diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event_partial_msg_app b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event_partial_msg_app index 75a2291bbe94..a2f425944c80 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event_partial_msg_app +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/handle_raft_event_partial_msg_app @@ -86,7 +86,7 @@ t1/s3: eval reg=+14 MiB/+16 MiB ela=+6.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[4,6) send_queue=[6,6) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,6) send_queue=[6,6) precise_q_size=+0 B eval deducted: reg=+5.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -96,7 +96,7 @@ NormalPri: term=1 index=4 tokens=1048576 term=1 index=5 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,5) send_queue=[5,6) precise_q_size=+1.0 MiB +(n2,s2):2: state=replicate closed=false inflight=[1,5) send_queue=[5,6) precise_q_size=+1.0 MiB eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: @@ -160,7 +160,7 @@ t1/s3: eval reg=+13 MiB/+16 MiB ela=+5.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[6,7) send_queue=[7,7) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[5,7) send_queue=[7,7) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/leaseholder_force_flush_no_send_q b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/leaseholder_force_flush_no_send_q index b11167ea7a2e..1c60218f5781 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/leaseholder_force_flush_no_send_q +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/leaseholder_force_flush_no_send_q @@ -101,7 +101,7 @@ t1/s3: eval reg=-2.0 MiB/+16 MiB ela=-2.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -114,7 +114,7 @@ eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: term=1 index=1 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -155,7 +155,7 @@ kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue : 0 stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -168,7 +168,7 @@ eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: term=1 index=1 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -207,7 +207,7 @@ t1/s3: eval reg=-2.0 MiB/+16 MiB ela=-2.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -220,7 +220,7 @@ eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: term=1 index=1 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/no_send_q_choice b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/no_send_q_choice index edaa91a79054..0246effd727c 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/no_send_q_choice +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/no_send_q_choice @@ -110,7 +110,7 @@ t1/s5: eval reg=-2.0 MiB/+16 MiB ela=-2.5 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -125,14 +125,14 @@ eval original in send-q: reg=+2.0 MiB ela=+0 B eval deducted: reg=+0 B ela=+2.0 MiB eval original in send-q: reg=+2.0 MiB ela=+0 B ++++ -(n4,s4):4: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n4,s4):4: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=1 tokens=1048576 term=1 index=2 tokens=1048576 ++++ -(n5,s5):5: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n5,s5):5: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -208,7 +208,7 @@ t1/s5: eval reg=-2.0 MiB/+16 MiB ela=-3.5 MiB/+8.0 MiB # Replica 5 now has a send-queue. stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+3.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -216,7 +216,7 @@ NormalPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -224,7 +224,7 @@ NormalPri: ++++ (n3,s3):3: closed ++++ -(n4,s4):4: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n4,s4):4: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+3.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/probe_state b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/probe_state index 26a3750ef39c..aa9878a6b50e 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/probe_state +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/probe_state @@ -115,7 +115,7 @@ t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/quorum_without_send_q b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/quorum_without_send_q index 0c7b8579f0bd..b54baae65606 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/quorum_without_send_q +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/quorum_without_send_q @@ -80,7 +80,7 @@ t1/s3: eval reg=-2.0 MiB/+16 MiB ela=-2.0 MiB/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -93,7 +93,7 @@ eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: term=1 index=1 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,3) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,3) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -155,7 +155,7 @@ t1/s3: eval reg=+0 B/+16 MiB ela=+0 B/+8.0 MiB stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+3.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_entries_without_ac b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_entries_without_ac index d2fea71bf30a..c71d23f9cccf 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_entries_without_ac +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_entries_without_ac @@ -150,7 +150,7 @@ t1/s3: eval reg=-1.0 MiB/+16 MiB ela=-1.0 MiB/+8.0 MiB # the send-queue. stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -160,7 +160,7 @@ NormalPri: eval deducted: reg=+0 B ela=+1.0 MiB eval original in send-q: reg=+1.0 MiB ela=+0 B ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -198,7 +198,7 @@ t1/s3: eval reg=-1.0 MiB/+16 MiB ela=-1.0 MiB/+8.0 MiB # Replica 2 has 10KiB of send tokens and is waiting for a scheduler event. stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -208,7 +208,7 @@ NormalPri: eval deducted: reg=+0 B ela=+1.0 MiB eval original in send-q: reg=+1.0 MiB ela=+0 B ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -223,19 +223,19 @@ scheduled-replicas: 2 # only composed of entries that need 0 tokens. handle_scheduler_event range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=2 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+0 B watching-for-tokens +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+0 B watching-for-tokens eval deducted: reg=+0 B ela=+1.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: term=1 index=2 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -261,19 +261,19 @@ t1/s3: eval reg=-1.0 MiB/+16 MiB ela=-1.0 MiB/+8.0 MiB # Replica 2 is waiting for a scheduler event. stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=2 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+0 B deducted=+4.0 KiB +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+0 B deducted=+4.0 KiB eval deducted: reg=+0 B ela=+1.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: term=1 index=2 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -286,19 +286,19 @@ scheduled-replicas: 2 # returns the 4KiB of tokens it deducted. handle_scheduler_event range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: term=1 index=2 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+1.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: term=1 index=2 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_watcher b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_watcher index dcf91e2717fc..7f2c6777786b 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_watcher +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/send_q_watcher @@ -191,7 +191,7 @@ NormalPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+1.0 MiB deducted=+512 KiB +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+1.0 MiB deducted=+512 KiB eval deducted: reg=+0 B ela=+3.0 MiB eval original in send-q: reg=+1.0 MiB ela=+0 B LowPri: @@ -224,7 +224,7 @@ NormalPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+3.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -359,7 +359,7 @@ LowPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+0 B watching-for-tokens +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+0 B watching-for-tokens eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -403,7 +403,7 @@ LowPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,3) send_queue=[3,4) precise_q_size=+0 B deducted=+1.1 MiB +(n3,s3):3: state=replicate closed=false inflight=[1,3) send_queue=[3,4) precise_q_size=+0 B deducted=+1.1 MiB eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -432,7 +432,7 @@ LowPri: term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[3,4) send_queue=[4,4) precise_q_size=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,4) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B eval original in send-q: reg=+0 B ela=+0 B LowPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/sendq_push_pull b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/sendq_push_pull index 0999437b5eb1..f428fbf9317f 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/sendq_push_pull +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/sendq_push_pull @@ -76,7 +76,7 @@ t1/s3: eval reg=+16 MiB/+16 MiB ela=+1.0 MiB/+8.0 MiB # send from the send-queue. stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[3,5) send_queue=[5,5) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,5) send_queue=[5,5) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+5.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -86,7 +86,7 @@ NormalPri: term=1 index=1 tokens=1048576 term=1 index=3 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,5) send_queue=[5,5) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,5) send_queue=[5,5) precise_q_size=+0 B eval deducted: reg=+2.0 MiB ela=+5.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -129,7 +129,7 @@ t1/s3: eval reg=+13 MiB/+16 MiB ela=+0 B/+8.0 MiB # mode. stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[5,6) send_queue=[6,6) precise_q_size=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,6) send_queue=[6,6) precise_q_size=+0 B eval deducted: reg=+3.0 MiB ela=+5.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: @@ -140,7 +140,7 @@ NormalPri: term=1 index=3 tokens=1048576 term=1 index=5 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[5,6) send_queue=[6,6) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,6) send_queue=[6,6) precise_q_size=+0 B eval deducted: reg=+3.0 MiB ela=+5.0 MiB eval original in send-q: reg=+0 B ela=+0 B LowPri: diff --git a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/wait_for_eval_send_q b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/wait_for_eval_send_q index d2161265dcdb..c6647a5f3fe4 100644 --- a/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/wait_for_eval_send_q +++ b/pkg/kv/kvserver/kvflowcontrol/rac2/testdata/range_controller/wait_for_eval_send_q @@ -3,8 +3,8 @@ # set regular streams to initially have 8MiB tokens and a limit of 8MiB # tokens. Replicas 2 and 3 have a send-queue. init regular_limit=8MiB regular_init=8MiB elastic_limit=1 elastic_init=1 -range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=2 - store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=2 +range_id=1 tenant_id=1 local_replica_id=1 next_raft_index=1 + store_id=1 replica_id=1 type=VOTER_FULL state=StateReplicate next=1 store_id=2 replica_id=2 type=VOTER_FULL state=StateReplicate next=1 store_id=3 replica_id=3 type=VOTER_FULL state=StateReplicate next=1 ---- @@ -16,19 +16,39 @@ t1/s2: eval reg=+8.0 MiB/+8.0 MiB ela=+1 B/+1 B t1/s3: eval reg=+8.0 MiB/+8.0 MiB ela=+1 B/+1 B send reg=+8.0 MiB/+8.0 MiB ela=+1 B/+1 B +# Create MsgApps with different entries for each replica, so replicas 2 and 3 +# have a send-queue. +raft_event +range_id=1 + entries + term=1 index=1 pri=NormalPri size=1MiB + sending + replica_id=1 [1,2) + replica_id=2 [1,1) + replica_id=3 [1,1) +---- +t1/s1: eval reg=+7.0 MiB/+8.0 MiB ela=-1024 KiB/+1 B + send reg=+7.0 MiB/+8.0 MiB ela=-1024 KiB/+1 B +t1/s2: eval reg=+7.0 MiB/+8.0 MiB ela=-1024 KiB/+1 B + send reg=+8.0 MiB/+8.0 MiB ela=+1 B/+1 B +t1/s3: eval reg=+7.0 MiB/+8.0 MiB ela=-1024 KiB/+1 B + send reg=+8.0 MiB/+8.0 MiB ela=+1 B/+1 B + stream_state range_id=1 ---- (n1,s1):1: state=replicate closed=false inflight=[1,2) send_queue=[2,2) precise_q_size=+0 B -eval deducted: reg=+0 B ela=+0 B +eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B +NormalPri: + term=1 index=1 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[1,1) send_queue=[1,2) precise_q_size=+0 B -eval deducted: reg=+0 B ela=+0 B -eval original in send-q: reg=+0 B ela=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,1) send_queue=[1,2) precise_q_size=+1.0 MiB +eval deducted: reg=+1.0 MiB ela=+0 B +eval original in send-q: reg=+1.0 MiB ela=+0 B ++++ -(n3,s3):3: state=replicate closed=false inflight=[1,1) send_queue=[1,2) precise_q_size=+0 B -eval deducted: reg=+0 B ela=+0 B -eval original in send-q: reg=+0 B ela=+0 B +(n3,s3):3: state=replicate closed=false inflight=[1,1) send_queue=[1,2) precise_q_size=+1.0 MiB +eval deducted: reg=+1.0 MiB ela=+0 B +eval original in send-q: reg=+1.0 MiB ela=+0 B ++++ # Start a high priority evaluation. It should not complete due to lack of @@ -50,8 +70,10 @@ r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3] stream_state range_id=1 ---- (n1,s1):1: state=replicate closed=false inflight=[1,2) send_queue=[2,2) precise_q_size=+0 B -eval deducted: reg=+0 B ela=+0 B +eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B +NormalPri: + term=1 index=1 tokens=1048576 ++++ (n2,s2):2: state=replicate closed=false inflight=[1,2) send_queue=[2,2) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B @@ -78,8 +100,10 @@ r1: [(n1,s1):1*,(n2,s2):2,(n3,s3):3] stream_state range_id=1 ---- (n1,s1):1: state=replicate closed=false inflight=[1,2) send_queue=[2,2) precise_q_size=+0 B -eval deducted: reg=+0 B ela=+0 B +eval deducted: reg=+1.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B +NormalPri: + term=1 index=1 tokens=1048576 ++++ (n2,s2):2: state=replicate closed=false inflight=[1,2) send_queue=[2,2) precise_q_size=+0 B eval deducted: reg=+0 B ela=+0 B @@ -109,8 +133,8 @@ range_id=1 replica_id=2 [2,3) replica_id=3 [2,4) ---- -t1/s1: eval reg=+5.0 MiB/+8.0 MiB ela=-3.0 MiB/+1 B - send reg=+5.0 MiB/+8.0 MiB ela=-3.0 MiB/+1 B +t1/s1: eval reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B + send reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B t1/s2: eval reg=+5.0 MiB/+8.0 MiB ela=-3.0 MiB/+1 B send reg=+7.0 MiB/+8.0 MiB ela=-1024 KiB/+1 B t1/s3: eval reg=+5.0 MiB/+8.0 MiB ela=-3.0 MiB/+1 B @@ -118,21 +142,22 @@ t1/s3: eval reg=+5.0 MiB/+8.0 MiB ela=-3.0 MiB/+1 B stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[2,5) send_queue=[5,5) precise_q_size=+0 B -eval deducted: reg=+3.0 MiB ela=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,5) send_queue=[5,5) precise_q_size=+0 B +eval deducted: reg=+4.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: + term=1 index=1 tokens=1048576 term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 term=1 index=4 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[2,3) send_queue=[3,5) precise_q_size=+2.0 MiB +(n2,s2):2: state=replicate closed=false inflight=[1,3) send_queue=[3,5) precise_q_size=+2.0 MiB eval deducted: reg=+3.0 MiB ela=+0 B eval original in send-q: reg=+2.0 MiB ela=+0 B NormalPri: term=1 index=2 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[2,4) send_queue=[4,5) precise_q_size=+1.0 MiB +(n3,s3):3: state=replicate closed=false inflight=[1,4) send_queue=[4,5) precise_q_size=+1.0 MiB eval deducted: reg=+3.0 MiB ela=+0 B eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: @@ -158,8 +183,8 @@ range_id=1 replica_id=2 [3,6) replica_id=3 [4,5) ---- -t1/s1: eval reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B - send reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B +t1/s1: eval reg=+3.0 MiB/+8.0 MiB ela=-5.0 MiB/+1 B + send reg=+3.0 MiB/+8.0 MiB ela=-5.0 MiB/+1 B t1/s2: eval reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B send reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B t1/s3: eval reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B @@ -167,16 +192,17 @@ t1/s3: eval reg=+4.0 MiB/+8.0 MiB ela=-4.0 MiB/+1 B stream_state range_id=1 ---- -(n1,s1):1: state=replicate closed=false inflight=[5,6) send_queue=[6,6) precise_q_size=+0 B -eval deducted: reg=+4.0 MiB ela=+0 B +(n1,s1):1: state=replicate closed=false inflight=[1,6) send_queue=[6,6) precise_q_size=+0 B +eval deducted: reg=+5.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: + term=1 index=1 tokens=1048576 term=1 index=2 tokens=1048576 term=1 index=3 tokens=1048576 term=1 index=4 tokens=1048576 term=1 index=5 tokens=1048576 ++++ -(n2,s2):2: state=replicate closed=false inflight=[3,6) send_queue=[6,6) precise_q_size=+0 B +(n2,s2):2: state=replicate closed=false inflight=[1,6) send_queue=[6,6) precise_q_size=+0 B eval deducted: reg=+4.0 MiB ela=+0 B eval original in send-q: reg=+0 B ela=+0 B NormalPri: @@ -185,7 +211,7 @@ NormalPri: term=1 index=4 tokens=1048576 term=1 index=5 tokens=1048576 ++++ -(n3,s3):3: state=replicate closed=false inflight=[4,5) send_queue=[5,6) precise_q_size=+1.0 MiB +(n3,s3):3: state=replicate closed=false inflight=[1,5) send_queue=[5,6) precise_q_size=+1.0 MiB eval deducted: reg=+4.0 MiB ela=+0 B eval original in send-q: reg=+1.0 MiB ela=+0 B NormalPri: