From 8f7f7bf2bcdb60f92c6aaac8f34a9fdd18d60c76 Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Tue, 28 Sep 2021 15:29:11 -0400 Subject: [PATCH] roachtest: Add sstable-corruption roachtest Adds a new roachtest, sstable-corruption/table, that imports TPCC, then finds an sstable that contains table keys (which are replicated) on one node, then calls `dd` to write random bytes into the middle of them. It then checks that the corrupt node either crashes on restart, or crashes after the tpcc workload is run on it. Informs #67568. Release note: None. --- pkg/cmd/roachtest/tests/BUILD.bazel | 1 + pkg/cmd/roachtest/tests/registry.go | 1 + pkg/cmd/roachtest/tests/sstable_corruption.go | 115 ++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 pkg/cmd/roachtest/tests/sstable_corruption.go diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel index e36e8667ff6e..33133abcc4c7 100644 --- a/pkg/cmd/roachtest/tests/BUILD.bazel +++ b/pkg/cmd/roachtest/tests/BUILD.bazel @@ -106,6 +106,7 @@ go_library( "sqlalchemy.go", "sqlalchemy_blocklist.go", "sqlsmith.go", + "sstable_corruption.go", "status_server.go", "synctest.go", "sysbench.go", diff --git a/pkg/cmd/roachtest/tests/registry.go b/pkg/cmd/roachtest/tests/registry.go index 6a360326a1f2..69e03f9fc9e4 100644 --- a/pkg/cmd/roachtest/tests/registry.go +++ b/pkg/cmd/roachtest/tests/registry.go @@ -99,6 +99,7 @@ func RegisterTests(r registry.Registry) { registerSequelize(r) registerSQLAlchemy(r) registerSQLSmith(r) + registerSSTableCorruption(r) registerSyncTest(r) registerSysbench(r) registerTLP(r) diff --git a/pkg/cmd/roachtest/tests/sstable_corruption.go b/pkg/cmd/roachtest/tests/sstable_corruption.go new file mode 100644 index 000000000000..075b02d0e9df --- /dev/null +++ b/pkg/cmd/roachtest/tests/sstable_corruption.go @@ -0,0 +1,115 @@ +// Copyright 2021 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package tests + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/stretchr/testify/require" +) + +func runSSTableCorruption(ctx context.Context, t test.Test, c cluster.Cluster) { + crdbNodes := c.Range(1, c.Spec().NodeCount) + workloadNode := c.Node(1) + // Corrupt all nodes. + corruptNodes := crdbNodes + + t.Status("installing cockroach") + c.Put(ctx, t.Cockroach(), "./cockroach", crdbNodes) + c.Start(ctx, crdbNodes) + + { + m := c.NewMonitor(ctx, crdbNodes) + + m.Go(func(ctx context.Context) error { + // We don't really need tpcc, we just need a good amount of data. Enough + // to have multiple ranges, and some sstables with only table keys. + t.Status("importing tpcc fixture") + c.Run(ctx, workloadNode, + "./cockroach workload fixtures import tpcc --warehouses=100 --fks=false --checks=false") + return nil + }) + m.Wait() + } + + c.Stop(ctx, crdbNodes) + + for _, node := range corruptNodes { + tableSSTs, err := c.RunWithBuffer(ctx, t.L(), c.Node(node), + "./cockroach debug pebble manifest dump {store-dir}/MANIFEST-* | grep -v added | grep -v deleted | grep \"\\[/Table\"") + if err != nil { + t.Fatal(err) + } + strTableSSTs := strings.Split(string(tableSSTs), "\n") + if len(strTableSSTs) == 0 { + t.Fatal("expected at least one sst containing table keys only, got none") + } + // Corrupt up to 6 SSTs containing table keys. + corruptedFiles := 0 + for _, sstLine := range strTableSSTs { + sstLine = strings.TrimSpace(sstLine) + firstFileIdx := strings.Index(sstLine, ":") + _, err = strconv.Atoi(sstLine[:firstFileIdx]) + if err != nil { + t.Fatalf("error when converting %s to int: %s", sstLine[:firstFileIdx], err.Error()) + } + + t.Status(fmt.Sprintf("corrupting sstable %s on node %d", sstLine[:firstFileIdx], node)) + c.Run(ctx, c.Node(node), fmt.Sprintf("dd if=/dev/urandom of={store-dir}/%s.sst seek=256 count=128 bs=1 conv=notrunc", sstLine[:firstFileIdx])) + corruptedFiles++ + if corruptedFiles >= 6 { + break + } + } + } + + if err := c.StartE(ctx, crdbNodes); err != nil { + // Node detected corruption on start and crashed. This is good. No need + // to run workload; the test is complete. + _ = c.WipeE(ctx, t.L(), corruptNodes) + return + } + + { + m := c.NewMonitor(ctx) + // Run a workload to try to get the node to notice corruption and crash. + m.Go(func(ctx context.Context) error { + _ = c.RunE(ctx, workloadNode, + fmt.Sprintf("./cockroach workload run tpcc --warehouses=100 --tolerate-errors --duration=%s", 2*time.Minute)) + // Don't return an error from the workload. We want outcome of WaitE to be + // determined by the monitor noticing that a node died. The workload may + // also fail, despite --tolerate-errors, if a node crashes too early. + return nil + }) + require.Error(t, m.WaitE()) + } + + // Exempt corrupted nodes from roachtest harness' post-test liveness checks. + _ = c.WipeE(ctx, t.L(), corruptNodes) +} + +func registerSSTableCorruption(r registry.Registry) { + r.Add(registry.TestSpec{ + Name: "sstable-corruption/table", + Owner: registry.OwnerStorage, + Cluster: r.MakeClusterSpec(3), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + runSSTableCorruption(ctx, t, c) + }, + }) +}