Merge #86823 #86866 #86991 #87039

86823: ccl: drop privileges when dropping external connection r=adityamaru a=RichardJCai Release justification: Bug fix to newly added feature Release note: None 86866: sql/logictest: add metamorphic param for global MVCC range tombstone r=tbg,yuzefovich a=erikgrinaker **kvserver: add `GlobalMVCCRangeTombstone` testing knob** This patch adds a set of testing knobs to enable writing an MVCC range tombstone at the bottom of the keyspace during cluster bootstrapping. This also subsumes the previous mechanism to trigger this from the envvar `COCKROACH_GLOBAL_MVCC_RANGE_TOMBSTONE`, for e.g. roachtests and roachperf benchmarks. Release justification: non-production code changes Release note: None **sql/logictest: add metamorphic param for global MVCC range tombstone** This patch adds a metamorphic parameter for the SQL logic tests that will randomly write a global MVCC range tombstone across the entire user keyspace during cluster bootstrapping. This should not semantically affect the test data written above it, but will activate MVCC range tombstone code paths in the storage layer for testing. Touches #84042. Release justification: non-production code changes Release note: None 86991: sql/pgwire: fix service latency for COPY r=otan a=rafiss fixes #86741 Release note (bug fix): Fixed the latency that is reported for COPY comands in the CLI and stats reporting. Release justification: low risk bug fix 87039: vendor: bump Pebble to c2dc3de22e91 r=jbowens a=itsbilal c2dc3de2 *: don't close pointIter twice if bypassing merging iter b38417b0 db: wait for table cache releasing goroutines to exit c751cef4 *: optimize external iters for single-file, no-rangekey cases e729e744 metamorphic: test Clone with Options Release note: None. Release justification: bug fixes and low-risk updates to new functionality. Co-authored-by: richardjcai <[email protected]> Co-authored-by: Erik Grinaker <[email protected]> Co-authored-by: Rafi Shamim <[email protected]> Co-authored-by: Bilal Akhtar <[email protected]>
cockroachdb · Aug 29, 2022 · 9da058b · 9da058b
5 parents de5ff28 + ce92aac + f6953e0 + b6f4a3b + 7ad18be
commit 9da058b
Show file tree

Hide file tree

Showing 23 changed files with 185 additions and 27 deletions.
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -1421,10 +1421,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "4dc4a2931b55e1b4431aa9d10736dc1f4f9058af248235654833efd67c3d18c6",
-        strip_prefix = "github.com/cockroachdb/[email protected]20220823190348-4cc0974fdade",
+        sha256 = "b024c32024f91e1b10ed56448ae313e0458562116dac1aa6ef4efc080964431d",
+        strip_prefix = "github.com/cockroachdb/[email protected]20220829152249-c2dc3de22e91",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220823190348-4cc0974fdade.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220829152249-c2dc3de22e91.zip",
         ],
     )
     go_repository(

diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl
@@ -186,7 +186,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.13.0.zip": "b3d43d8f95edf65f73a5348f29e1159823cac64b148f8d3bb48340bf55d70872",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20211118104740-dabe8e521a4f.zip": "1972c3f171f118add3fd9e64bcea6cbb9959a3b7fa0ada308e8a7310813fea74",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220823190348-4cc0974fdade.zip": "4dc4a2931b55e1b4431aa9d10736dc1f4f9058af248235654833efd67c3d18c6",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220829152249-c2dc3de22e91.zip": "b024c32024f91e1b10ed56448ae313e0458562116dac1aa6ef4efc080964431d",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.3.zip": "7778b1e4485e4f17f35e5e592d87eb99c29e173ac9507801d000ad76dd0c261e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/sentry-go/com_github_cockroachdb_sentry_go-v0.6.1-cockroachdb.2.zip": "fbb2207d02aecfdd411b1357efe1192dbb827959e36b7cab7491731ac55935c9",

diff --git a/go.mod b/go.mod
@@ -47,7 +47,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.13.0
 	github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f
-	github.com/cockroachdb/pebble v0.0.0-20220823190348-4cc0974fdade
+	github.com/cockroachdb/pebble v0.0.0-20220829152249-c2dc3de22e91
 	github.com/cockroachdb/redact v1.1.3
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

diff --git a/go.sum b/go.sum
@@ -470,8 +470,8 @@ github.com/cockroachdb/gostdlib v1.13.0/go.mod h1:eXX95p9QDrYwJfJ6AgeN9QnRa/lqqi
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74=
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20220823190348-4cc0974fdade h1:lp4fMGMYvLNCKSaHfXMIamULFwO7FtLpM08Z8iQe9a8=
-github.com/cockroachdb/pebble v0.0.0-20220823190348-4cc0974fdade/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0=
+github.com/cockroachdb/pebble v0.0.0-20220829152249-c2dc3de22e91 h1:sskFpsa91pwZ7SHAO0oqu1uEpI5flpEb4G0Sqt9qiLI=
+github.com/cockroachdb/pebble v0.0.0-20220829152249-c2dc3de22e91/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0=
 github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

diff --git a/pkg/ccl/cloudccl/externalconn/testdata/privileges_external_connection b/pkg/ccl/cloudccl/externalconn/testdata/privileges_external_connection
@@ -61,6 +61,14 @@ exec-sql
 GRANT DROP ON EXTERNAL CONNECTION "drop-privileged" TO testuser;
 ----
 
+# Verify that the privileges exist.
+query-sql
+SELECT * FROM system.privileges
+----
+root /externalconn/drop-privileged {ALL} {}
+root /externalconn/drop-privileged-dup {ALL} {}
+testuser /externalconn/drop-privileged {DROP} {}
+
 exec-sql user=testuser
 DROP EXTERNAL CONNECTION "drop-privileged"
 ----
@@ -79,6 +87,11 @@ exec-sql
 DROP EXTERNAL CONNECTION 'drop-privileged-dup'
 ----
 
+# Verify that the privileges are dropped.
+query-sql
+SELECT * FROM system.privileges
+----
+
 subtest end
 
 subtest create-grants-all
@@ -116,6 +129,14 @@ exec-sql user=testuser
 CREATE EXTERNAL CONNECTION 'not-root' AS 'userfile:///bar'
 ----
 
+# Verify that the privileges exist.
+query-sql
+SELECT * FROM system.privileges
+----
+root /externalconn/root {ALL} {}
+testuser /externalconn/not-root {ALL} {}
+testuser /global/ {EXTERNALCONNECTION} {}
+
 exec-sql user=testuser
 BACKUP TABLE foo INTO 'external://not-root'
 ----

diff --git a/pkg/kv/kvclient/rangefeed/BUILD.bazel b/pkg/kv/kvclient/rangefeed/BUILD.bazel
@@ -18,7 +18,6 @@ go_library(
         "//pkg/keys",
         "//pkg/kv",
         "//pkg/kv/kvclient/kvcoord",
-        "//pkg/kv/kvserver/kvserverbase",
         "//pkg/roachpb",
         "//pkg/settings",
         "//pkg/settings/cluster",

diff --git a/pkg/kv/kvclient/rangefeed/rangefeed.go b/pkg/kv/kvclient/rangefeed/rangefeed.go
@@ -21,7 +21,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord"
-	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
@@ -83,6 +82,12 @@ type TestingKnobs struct {
 
 	// OnRangefeedRestart is called when a rangefeed restarts.
 	OnRangefeedRestart func()
+
+	// IgnoreOnDeleteRangeError will ignore any errors where a DeleteRange event
+	// is emitted without an OnDeleteRange handler. This can be used e.g. with
+	// StoreTestingKnobs.GlobalMVCCRangeTombstone, to prevent the global tombstone
+	// causing rangefeed errors for consumers who don't expect it.
+	IgnoreOnDeleteRangeError bool
 }
 
 // ModuleTestingKnobs is part of the base.ModuleTestingKnobs interface.
@@ -361,7 +366,7 @@ func (f *RangeFeed) processEvents(
 				f.onSSTable(ctx, ev.SST, ev.RegisteredSpan)
 			case ev.DeleteRange != nil:
 				if f.onDeleteRange == nil {
-					if kvserverbase.GlobalMVCCRangeTombstoneForTesting {
+					if f.knobs.IgnoreOnDeleteRangeError {
 						continue
 					}
 					return errors.AssertionFailedf(

diff --git a/pkg/kv/kvserver/batcheval/cmd_init_put.go b/pkg/kv/kvserver/batcheval/cmd_init_put.go
@@ -14,7 +14,6 @@ import (
 	"context"
 
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result"
-	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 )
@@ -33,7 +32,7 @@ func InitPut(
 	args := cArgs.Args.(*roachpb.InitPutRequest)
 	h := cArgs.Header
 
-	if args.FailOnTombstones && kvserverbase.GlobalMVCCRangeTombstoneForTesting {
+	if args.FailOnTombstones && cArgs.EvalCtx.EvalKnobs().DisableInitPutFailOnTombstones {
 		args.FailOnTombstones = false
 	}
 

diff --git a/pkg/kv/kvserver/kvserverbase/BUILD.bazel b/pkg/kv/kvserver/kvserverbase/BUILD.bazel
@@ -18,7 +18,6 @@ go_library(
         "//pkg/kv/kvserver/kvserverpb",
         "//pkg/roachpb",
         "//pkg/settings",
-        "//pkg/util/envutil",
         "//pkg/util/errorutil",
         "//pkg/util/hlc",
         "//pkg/util/quotapool",

diff --git a/pkg/kv/kvserver/kvserverbase/base.go b/pkg/kv/kvserver/kvserverbase/base.go
@@ -20,7 +20,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
-	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/quotapool"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/redact"
@@ -219,10 +218,3 @@ var SplitByLoadMergeDelay = settings.RegisterDurationSetting(
 // MaxCommandSizeDefault is the default for the kv.raft.command.max_size
 // cluster setting.
 const MaxCommandSizeDefault = 64 << 20
-
-// GlobalMVCCRangeTombstoneForTesting will write an MVCC range tombstone at the
-// bottom of the SQL table data keyspace during cluster bootstrapping, for
-// performance and correctness testing. This shouldn't affect data written above
-// it, but activates range key-specific code paths in the storage layer.
-var GlobalMVCCRangeTombstoneForTesting = envutil.EnvOrDefaultBool(
-	"COCKROACH_GLOBAL_MVCC_RANGE_TOMBSTONE", false)
diff --git a/pkg/kv/kvserver/kvserverbase/knobs.go b/pkg/kv/kvserver/kvserverbase/knobs.go
@@ -38,6 +38,11 @@ type BatchEvalTestingKnobs struct {
 	// default, this is not allowed because it is unsafe. See cmd_gc.go for an
 	// explanation of why.
 	AllowGCWithNewThresholdAndKeys bool
+
+	// DisableInitPutFailOnTombstones disables FailOnTombstones for InitPut. This
+	// is useful together with e.g. StoreTestingKnobs.GlobalMVCCRangeTombstone,
+	// where we still want InitPut to succeed on top of the range tombstone.
+	DisableInitPutFailOnTombstones bool
 }
 
 // IntentResolverTestingKnobs contains testing helpers that are used during

diff --git a/pkg/kv/kvserver/store_init.go b/pkg/kv/kvserver/store_init.go
@@ -14,7 +14,6 @@ import (
 	"context"
 
 	"github.com/cockroachdb/cockroach/pkg/keys"
-	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader"
@@ -204,7 +203,7 @@ func WriteInitialClusterData(
 
 		// If requested, write an MVCC range tombstone at the bottom of the
 		// keyspace, for performance and correctness testing.
-		if kvserverbase.GlobalMVCCRangeTombstoneForTesting {
+		if knobs.GlobalMVCCRangeTombstone {
 			if err := writeGlobalMVCCRangeTombstone(ctx, batch, desc, now.Prev()); err != nil {
 				return err
 			}

diff --git a/pkg/kv/kvserver/testing_knobs.go b/pkg/kv/kvserver/testing_knobs.go
@@ -422,6 +422,18 @@ type StoreTestingKnobs struct {
 
 	// EnqueueReplicaInterceptor intercepts calls to `store.Enqueue()`.
 	EnqueueReplicaInterceptor func(queueName string, replica *Replica)
+
+	// GlobalMVCCRangeTombstone will write a global MVCC range tombstone across
+	// the entire user keyspace during cluster bootstrapping. This will be written
+	// below all other data, and thus won't affect query results, but it does
+	// activate MVCC range tombstone code paths in the storage layer for testing.
+	//
+	// This must typically be combined with the following knobs to prevent
+	// various components choking on the range tombstone:
+	//
+	// - rangefeed.TestingKnobs.IgnoreOnDeleteRangeError
+	// - kvserverbase.BatchEvalTestingKnobs.DisableInitPutFailOnTombstones
+	GlobalMVCCRangeTombstone bool
 }
 
 // ModuleTestingKnobs is part of the base.ModuleTestingKnobs interface.

diff --git a/pkg/server/config.go b/pkg/server/config.go
@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/config/zonepb"
 	"github.com/cockroachdb/cockroach/pkg/docs"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/server/status"
@@ -224,9 +225,32 @@ func MakeBaseConfig(st *cluster.Settings, tr *tracing.Tracer) BaseConfig {
 	// in a tag that is prefixed with "nsql".
 	baseCfg.AmbientCtx.AddLogTag("n", baseCfg.IDContainer)
 	baseCfg.InitDefaults()
+	baseCfg.InitTestingKnobs()
+
 	return baseCfg
 }
 
+// InitTestingKnobs sets up any testing knobs based on e.g. envvars.
+func (cfg *BaseConfig) InitTestingKnobs() {
+	// If requested, write an MVCC range tombstone at the bottom of the SQL table
+	// data keyspace during cluster bootstrapping, for performance and correctness
+	// testing. This shouldn't affect data written above it, but activates range
+	// key-specific code paths in the storage layer. We'll also have to tweak
+	// rangefeeds and batcheval to not choke on it.
+	if envutil.EnvOrDefaultBool("COCKROACH_GLOBAL_MVCC_RANGE_TOMBSTONE", false) {
+		if cfg.TestingKnobs.Store == nil {
+			cfg.TestingKnobs.Store = &kvserver.StoreTestingKnobs{}
+		}
+		if cfg.TestingKnobs.RangeFeed == nil {
+			cfg.TestingKnobs.RangeFeed = &rangefeed.TestingKnobs{}
+		}
+		storeKnobs := cfg.TestingKnobs.Store.(*kvserver.StoreTestingKnobs)
+		storeKnobs.GlobalMVCCRangeTombstone = true
+		storeKnobs.EvalKnobs.DisableInitPutFailOnTombstones = true
+		cfg.TestingKnobs.RangeFeed.(*rangefeed.TestingKnobs).IgnoreOnDeleteRangeError = true
+	}
+}
+
 // Config holds the parameters needed to set up a combined KV and SQL server.
 type Config struct {
 	BaseConfig

diff --git a/pkg/sql/conn_executor.go b/pkg/sql/conn_executor.go
@@ -2047,9 +2047,19 @@ func (ex *connExecutor) execCmd() error {
 			}
 		}
 	case CopyIn:
+		ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionQueryReceived, tcmd.TimeReceived)
+		ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionStartParse, tcmd.ParseStart)
+		ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionEndParse, tcmd.ParseEnd)
 		res = ex.clientComm.CreateCopyInResult(pos)
 		var err error
 		ev, payload, err = ex.execCopyIn(ctx, tcmd)
+		// Note: we write to ex.statsCollector.phaseTimes, instead of ex.phaseTimes,
+		// because:
+		// - stats use ex.statsCollector, not ex.phasetimes.
+		// - ex.statsCollector merely contains a copy of the times, that
+		//   was created when the statement started executing (via the
+		//   reset() method).
+		ex.statsCollector.PhaseTimes().SetSessionPhaseTime(sessionphase.SessionQueryServiced, timeutil.Now())
 		if err != nil {
 			return err
 		}

diff --git a/pkg/sql/conn_io.go b/pkg/sql/conn_io.go
@@ -328,6 +328,13 @@ type CopyIn struct {
 	// CopyDone is decremented once execution finishes, signaling that control of
 	// the connection is being handed back to the network routine.
 	CopyDone *sync.WaitGroup
+	// TimeReceived is the time at which the message was received
+	// from the client. Used to compute the service latency.
+	TimeReceived time.Time
+	// ParseStart/ParseEnd are the timing info for parsing of the query. Used for
+	// stats reporting.
+	ParseStart time.Time
+	ParseEnd   time.Time
 }
 
 // command implements the Command interface.

diff --git a/pkg/sql/drop_external_connection.go b/pkg/sql/drop_external_connection.go
@@ -98,6 +98,18 @@ func (p *planner) dropExternalConnection(params runParams, n *tree.DropExternalC
 		return errors.Wrapf(err, "failed to delete external connection")
 	}
 
+	// We must also DELETE all rows from system.privileges that refer to
+	// external connection.
+	if _, err = params.extendedEvalCtx.ExecCfg.InternalExecutor.ExecEx(
+		params.ctx,
+		dropExternalConnectionOp,
+		params.p.Txn(),
+		sessiondata.InternalExecutorOverride{User: username.NodeUserName()},
+		`DELETE FROM system.privileges WHERE path = $1`, ecPrivilege.GetPath(),
+	); err != nil {
+		return errors.Wrapf(err, "failed to delete external connection")
+	}
+
 	return nil
 }
 

diff --git a/pkg/sql/logictest/BUILD.bazel b/pkg/sql/logictest/BUILD.bazel
@@ -29,7 +29,9 @@ go_library(
         "//pkg/base",
         "//pkg/cloud/externalconn/providers",
         "//pkg/clusterversion",
+        "//pkg/kv/kvclient/rangefeed",
         "//pkg/kv/kvserver",
+        "//pkg/kv/kvserver/kvserverbase",
         "//pkg/roachpb",
         "//pkg/security/username",
         "//pkg/server",
@@ -56,6 +58,7 @@ go_library(
         "//pkg/testutils/skip",
         "//pkg/testutils/sqlutils",
         "//pkg/upgrade",
+        "//pkg/util",
         "//pkg/util/envutil",
         "//pkg/util/log",
         "//pkg/util/randutil",

diff --git a/pkg/sql/logictest/logic.go b/pkg/sql/logictest/logic.go
@@ -40,7 +40,9 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	_ "github.com/cockroachdb/cockroach/pkg/cloud/externalconn/providers" // imported to register ExternalConnection providers
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server"
@@ -67,6 +69,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/upgrade"
+	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
@@ -530,6 +533,13 @@ var (
 		"declarative-corpus", "",
 		"enables generation and storage of a declarative schema changer	corpus",
 	)
+
+	// globalMVCCRangeTombstone will write a global MVCC range tombstone across
+	// the entire user keyspace during cluster bootstrapping. This should not
+	// semantically affect the test data written above it, but will activate MVCC
+	// range tombstone code paths in the storage layer for testing.
+	globalMVCCRangeTombstone = util.ConstantWithMetamorphicTestBool(
+		"logictest-global-mvcc-range-tombstone", false)
 )
 
 const queryRewritePlaceholderPrefix = "__async_query_rewrite_placeholder"
@@ -1186,6 +1196,12 @@ func (t *logicTest) newCluster(
 	// when run with fakedist-disk config, so we'll use a larger limit here.
 	// There isn't really a downside to doing so.
 	tempStorageDiskLimit := int64(512 << 20) /* 512 MiB */
+	// MVCC range tombstones are only available in 22.2 or newer.
+	enableGlobalMVCCRangeTombstone := globalMVCCRangeTombstone &&
+		(t.cfg.BootstrapVersion.Equal(roachpb.Version{}) ||
+			!t.cfg.BootstrapVersion.Less(roachpb.Version{Major: 22, Minor: 2})) &&
+		(t.cfg.BinaryVersion.Equal(roachpb.Version{}) ||
+			!t.cfg.BinaryVersion.Less(roachpb.Version{Major: 22, Minor: 2}))
 
 	params := base.TestClusterArgs{
 		ServerArgs: base.TestServerArgs{
@@ -1197,7 +1213,11 @@ func (t *logicTest) newCluster(
 			Knobs: base.TestingKnobs{
 				Store: &kvserver.StoreTestingKnobs{
 					// The consistency queue makes a lot of noisy logs during logic tests.
-					DisableConsistencyQueue: true,
+					DisableConsistencyQueue:  true,
+					GlobalMVCCRangeTombstone: enableGlobalMVCCRangeTombstone,
+					EvalKnobs: kvserverbase.BatchEvalTestingKnobs{
+						DisableInitPutFailOnTombstones: enableGlobalMVCCRangeTombstone,
+					},
 				},
 				SQLEvalContext: &eval.TestingKnobs{
 					AssertBinaryExprReturnTypes:     true,
@@ -1217,6 +1237,9 @@ func (t *logicTest) newCluster(
 				SQLDeclarativeSchemaChanger: &scexec.TestingKnobs{
 					BeforeStage: corpusCollectionCallback,
 				},
+				RangeFeed: rangefeed.TestingKnobs{
+					IgnoreOnDeleteRangeError: enableGlobalMVCCRangeTombstone,
+				},
 			},
 			ClusterName:   "testclustername",
 			ExternalIODir: t.sharedIODir,