diff --git a/pkg/storage/client_raft_test.go b/pkg/storage/client_raft_test.go index de2c6002e036..fd0512adc7a3 100644 --- a/pkg/storage/client_raft_test.go +++ b/pkg/storage/client_raft_test.go @@ -3325,124 +3325,6 @@ func TestRemoveRangeWithoutGC(t *testing.T) { }) } -// TestCheckConsistencyMultiStore creates a Db with three stores ] -// with three way replication. A value is added to the Db, and a -// consistency check is run. -func TestCheckConsistencyMultiStore(t *testing.T) { - defer leaktest.AfterTest(t)() - - const numStores = 3 - mtc := &multiTestContext{} - defer mtc.Stop() - mtc.Start(t, numStores) - // Setup replication of range 1 on store 0 to stores 1 and 2. - mtc.replicateRange(1, 1, 2) - - // Write something to the DB. - putArgs := putArgs([]byte("a"), []byte("b")) - if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), putArgs); err != nil { - t.Fatal(err) - } - - // Run consistency check. - checkArgs := roachpb.CheckConsistencyRequest{ - Span: roachpb.Span{ - // span of keys that include "a". - Key: []byte("a"), - EndKey: []byte("aa"), - }, - } - if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{ - Timestamp: mtc.stores[0].Clock().Now(), - }, &checkArgs); err != nil { - t.Fatal(err) - } - -} - -func TestCheckConsistencyInconsistent(t *testing.T) { - defer leaktest.AfterTest(t)() - - sc := storage.TestStoreConfig(nil) - mtc := &multiTestContext{storeConfig: &sc} - // Store 0 will report a diff with inconsistent key "e". - diffKey := []byte("e") - var diffTimestamp hlc.Timestamp - notifyReportDiff := make(chan struct{}, 1) - sc.TestingKnobs.BadChecksumReportDiff = - func(s roachpb.StoreIdent, diff []storage.ReplicaSnapshotDiff) { - if s != mtc.Store(0).Ident { - t.Errorf("BadChecksumReportDiff called from follower (StoreIdent = %s)", s) - return - } - if len(diff) != 1 { - t.Errorf("diff length = %d, diff = %v", len(diff), diff) - } - d := diff[0] - if d.LeaseHolder || !bytes.Equal(diffKey, d.Key) || diffTimestamp != d.Timestamp { - t.Errorf("diff = %v", d) - } - notifyReportDiff <- struct{}{} - } - // Store 0 will panic. - notifyPanic := make(chan struct{}, 1) - sc.TestingKnobs.BadChecksumPanic = func(s roachpb.StoreIdent) { - if s != mtc.Store(0).Ident { - t.Errorf("BadChecksumPanic called from follower (StoreIdent = %s)", s) - return - } - notifyPanic <- struct{}{} - } - - const numStores = 3 - defer mtc.Stop() - mtc.Start(t, numStores) - // Setup replication of range 1 on store 0 to stores 1 and 2. - mtc.replicateRange(1, 1, 2) - - // Write something to the DB. - pArgs := putArgs([]byte("a"), []byte("b")) - if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), pArgs); err != nil { - t.Fatal(err) - } - pArgs = putArgs([]byte("c"), []byte("d")) - if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), pArgs); err != nil { - t.Fatal(err) - } - - // Write some arbitrary data only to store 1. Inconsistent key "e"! - var val roachpb.Value - val.SetInt(42) - diffTimestamp = mtc.stores[1].Clock().Now() - if err := engine.MVCCPut( - context.Background(), mtc.stores[1].Engine(), nil, diffKey, diffTimestamp, val, nil, - ); err != nil { - t.Fatal(err) - } - - // Run consistency check. - checkArgs := roachpb.CheckConsistencyRequest{ - Span: roachpb.Span{ - // span of keys that include "a" & "c". - Key: []byte("a"), - EndKey: []byte("z"), - }, - } - if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), &checkArgs); err != nil { - t.Fatal(err) - } - select { - case <-notifyReportDiff: - case <-time.After(5 * time.Second): - t.Fatal("CheckConsistency() failed to report a diff as expected") - } - select { - case <-notifyPanic: - case <-time.After(5 * time.Second): - t.Fatal("CheckConsistency() failed to panic as expected") - } -} - func TestTransferRaftLeadership(t *testing.T) { defer leaktest.AfterTest(t)() diff --git a/pkg/storage/consistency_queue_test.go b/pkg/storage/consistency_queue_test.go index cf7ff2447af6..af3a4ff473a0 100644 --- a/pkg/storage/consistency_queue_test.go +++ b/pkg/storage/consistency_queue_test.go @@ -15,12 +15,17 @@ package storage_test import ( + "bytes" "context" "testing" + "time" "github.com/cockroachdb/cockroach/pkg/config" + "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/engine" + "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" ) @@ -53,3 +58,121 @@ func TestConsistencyQueueRequiresLive(t *testing.T) { t.Fatalf("expected shouldQ false; got %t, %f", shouldQ, priority) } } + +// TestCheckConsistencyMultiStore creates a node with three stores +// with three way replication. A value is added to the node, and a +// consistency check is run. +func TestCheckConsistencyMultiStore(t *testing.T) { + defer leaktest.AfterTest(t)() + + const numStores = 3 + mtc := &multiTestContext{} + defer mtc.Stop() + mtc.Start(t, numStores) + // Setup replication of range 1 on store 0 to stores 1 and 2. + mtc.replicateRange(1, 1, 2) + + // Write something to the DB. + putArgs := putArgs([]byte("a"), []byte("b")) + if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), putArgs); err != nil { + t.Fatal(err) + } + + // Run consistency check. + checkArgs := roachpb.CheckConsistencyRequest{ + Span: roachpb.Span{ + // span of keys that include "a". + Key: []byte("a"), + EndKey: []byte("aa"), + }, + } + if _, err := client.SendWrappedWith(context.Background(), rg1(mtc.stores[0]), roachpb.Header{ + Timestamp: mtc.stores[0].Clock().Now(), + }, &checkArgs); err != nil { + t.Fatal(err) + } + +} + +func TestCheckConsistencyInconsistent(t *testing.T) { + defer leaktest.AfterTest(t)() + + sc := storage.TestStoreConfig(nil) + mtc := &multiTestContext{storeConfig: &sc} + // Store 0 will report a diff with inconsistent key "e". + diffKey := []byte("e") + var diffTimestamp hlc.Timestamp + notifyReportDiff := make(chan struct{}, 1) + sc.TestingKnobs.BadChecksumReportDiff = + func(s roachpb.StoreIdent, diff []storage.ReplicaSnapshotDiff) { + if s != mtc.Store(0).Ident { + t.Errorf("BadChecksumReportDiff called from follower (StoreIdent = %s)", s) + return + } + if len(diff) != 1 { + t.Errorf("diff length = %d, diff = %v", len(diff), diff) + } + d := diff[0] + if d.LeaseHolder || !bytes.Equal(diffKey, d.Key) || diffTimestamp != d.Timestamp { + t.Errorf("diff = %v", d) + } + notifyReportDiff <- struct{}{} + } + // Store 0 will panic. + notifyPanic := make(chan struct{}, 1) + sc.TestingKnobs.BadChecksumPanic = func(s roachpb.StoreIdent) { + if s != mtc.Store(0).Ident { + t.Errorf("BadChecksumPanic called from follower (StoreIdent = %s)", s) + return + } + notifyPanic <- struct{}{} + } + + const numStores = 3 + defer mtc.Stop() + mtc.Start(t, numStores) + // Setup replication of range 1 on store 0 to stores 1 and 2. + mtc.replicateRange(1, 1, 2) + + // Write something to the DB. + pArgs := putArgs([]byte("a"), []byte("b")) + if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), pArgs); err != nil { + t.Fatal(err) + } + pArgs = putArgs([]byte("c"), []byte("d")) + if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), pArgs); err != nil { + t.Fatal(err) + } + + // Write some arbitrary data only to store 1. Inconsistent key "e"! + var val roachpb.Value + val.SetInt(42) + diffTimestamp = mtc.stores[1].Clock().Now() + if err := engine.MVCCPut( + context.Background(), mtc.stores[1].Engine(), nil, diffKey, diffTimestamp, val, nil, + ); err != nil { + t.Fatal(err) + } + + // Run consistency check. + checkArgs := roachpb.CheckConsistencyRequest{ + Span: roachpb.Span{ + // span of keys that include "a" & "c". + Key: []byte("a"), + EndKey: []byte("z"), + }, + } + if _, err := client.SendWrapped(context.Background(), rg1(mtc.stores[0]), &checkArgs); err != nil { + t.Fatal(err) + } + select { + case <-notifyReportDiff: + case <-time.After(5 * time.Second): + t.Fatal("CheckConsistency() failed to report a diff as expected") + } + select { + case <-notifyPanic: + case <-time.After(5 * time.Second): + t.Fatal("CheckConsistency() failed to panic as expected") + } +}