diff --git a/DEPS.bzl b/DEPS.bzl index d9b2cb31bd5a..6521addc1b9d 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1421,10 +1421,10 @@ def go_deps(): patches = [ "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch", ], - sha256 = "4dc4a2931b55e1b4431aa9d10736dc1f4f9058af248235654833efd67c3d18c6", - strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20220823190348-4cc0974fdade", + sha256 = "b024c32024f91e1b10ed56448ae313e0458562116dac1aa6ef4efc080964431d", + strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20220829152249-c2dc3de22e91", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220823190348-4cc0974fdade.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220829152249-c2dc3de22e91.zip", ], ) go_repository( diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index a4173cded00e..d584d1ac152a 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -186,7 +186,7 @@ DISTDIR_FILES = { "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.13.0.zip": "b3d43d8f95edf65f73a5348f29e1159823cac64b148f8d3bb48340bf55d70872", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20211118104740-dabe8e521a4f.zip": "1972c3f171f118add3fd9e64bcea6cbb9959a3b7fa0ada308e8a7310813fea74", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220823190348-4cc0974fdade.zip": "4dc4a2931b55e1b4431aa9d10736dc1f4f9058af248235654833efd67c3d18c6", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220829152249-c2dc3de22e91.zip": "b024c32024f91e1b10ed56448ae313e0458562116dac1aa6ef4efc080964431d", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.3.zip": "7778b1e4485e4f17f35e5e592d87eb99c29e173ac9507801d000ad76dd0c261e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/sentry-go/com_github_cockroachdb_sentry_go-v0.6.1-cockroachdb.2.zip": "fbb2207d02aecfdd411b1357efe1192dbb827959e36b7cab7491731ac55935c9", diff --git a/go.mod b/go.mod index 4e59392db834..8caf0b67b883 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 github.com/cockroachdb/gostdlib v1.13.0 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f - github.com/cockroachdb/pebble v0.0.0-20220823190348-4cc0974fdade + github.com/cockroachdb/pebble v0.0.0-20220829152249-c2dc3de22e91 github.com/cockroachdb/redact v1.1.3 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b diff --git a/go.sum b/go.sum index 973c1c7ba756..bb0b8967b08c 100644 --- a/go.sum +++ b/go.sum @@ -470,8 +470,8 @@ github.com/cockroachdb/gostdlib v1.13.0/go.mod h1:eXX95p9QDrYwJfJ6AgeN9QnRa/lqqi github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20220823190348-4cc0974fdade h1:lp4fMGMYvLNCKSaHfXMIamULFwO7FtLpM08Z8iQe9a8= -github.com/cockroachdb/pebble v0.0.0-20220823190348-4cc0974fdade/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0= +github.com/cockroachdb/pebble v0.0.0-20220829152249-c2dc3de22e91 h1:sskFpsa91pwZ7SHAO0oqu1uEpI5flpEb4G0Sqt9qiLI= +github.com/cockroachdb/pebble v0.0.0-20220829152249-c2dc3de22e91/go.mod h1:890yq1fUb9b6dGNwssgeUO5vQV9qfXnCPxAJhBQfXw0= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= diff --git a/pkg/ccl/cloudccl/externalconn/testdata/privileges_external_connection b/pkg/ccl/cloudccl/externalconn/testdata/privileges_external_connection index f694fadf9de5..651e411513a8 100644 --- a/pkg/ccl/cloudccl/externalconn/testdata/privileges_external_connection +++ b/pkg/ccl/cloudccl/externalconn/testdata/privileges_external_connection @@ -61,6 +61,14 @@ exec-sql GRANT DROP ON EXTERNAL CONNECTION "drop-privileged" TO testuser; ---- +# Verify that the privileges exist. +query-sql +SELECT * FROM system.privileges +---- +root /externalconn/drop-privileged {ALL} {} +root /externalconn/drop-privileged-dup {ALL} {} +testuser /externalconn/drop-privileged {DROP} {} + exec-sql user=testuser DROP EXTERNAL CONNECTION "drop-privileged" ---- @@ -79,6 +87,11 @@ exec-sql DROP EXTERNAL CONNECTION 'drop-privileged-dup' ---- +# Verify that the privileges are dropped. +query-sql +SELECT * FROM system.privileges +---- + subtest end subtest create-grants-all @@ -116,6 +129,14 @@ exec-sql user=testuser CREATE EXTERNAL CONNECTION 'not-root' AS 'userfile:///bar' ---- +# Verify that the privileges exist. +query-sql +SELECT * FROM system.privileges +---- +root /externalconn/root {ALL} {} +testuser /externalconn/not-root {ALL} {} +testuser /global/ {EXTERNALCONNECTION} {} + exec-sql user=testuser BACKUP TABLE foo INTO 'external://not-root' ---- diff --git a/pkg/kv/kvclient/rangefeed/BUILD.bazel b/pkg/kv/kvclient/rangefeed/BUILD.bazel index c2206f4d9f69..635b1bd3112b 100644 --- a/pkg/kv/kvclient/rangefeed/BUILD.bazel +++ b/pkg/kv/kvclient/rangefeed/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//pkg/keys", "//pkg/kv", "//pkg/kv/kvclient/kvcoord", - "//pkg/kv/kvserver/kvserverbase", "//pkg/roachpb", "//pkg/settings", "//pkg/settings/cluster", diff --git a/pkg/kv/kvclient/rangefeed/rangefeed.go b/pkg/kv/kvclient/rangefeed/rangefeed.go index eda29adfdca1..22e90735353f 100644 --- a/pkg/kv/kvclient/rangefeed/rangefeed.go +++ b/pkg/kv/kvclient/rangefeed/rangefeed.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" @@ -83,6 +82,12 @@ type TestingKnobs struct { // OnRangefeedRestart is called when a rangefeed restarts. OnRangefeedRestart func() + + // IgnoreOnDeleteRangeError will ignore any errors where a DeleteRange event + // is emitted without an OnDeleteRange handler. This can be used e.g. with + // StoreTestingKnobs.GlobalMVCCRangeTombstone, to prevent the global tombstone + // causing rangefeed errors for consumers who don't expect it. + IgnoreOnDeleteRangeError bool } // ModuleTestingKnobs is part of the base.ModuleTestingKnobs interface. @@ -361,7 +366,7 @@ func (f *RangeFeed) processEvents( f.onSSTable(ctx, ev.SST, ev.RegisteredSpan) case ev.DeleteRange != nil: if f.onDeleteRange == nil { - if kvserverbase.GlobalMVCCRangeTombstoneForTesting { + if f.knobs.IgnoreOnDeleteRangeError { continue } return errors.AssertionFailedf( diff --git a/pkg/kv/kvserver/batcheval/cmd_init_put.go b/pkg/kv/kvserver/batcheval/cmd_init_put.go index 6c2c432cc1c6..30ebb1e2451a 100644 --- a/pkg/kv/kvserver/batcheval/cmd_init_put.go +++ b/pkg/kv/kvserver/batcheval/cmd_init_put.go @@ -14,7 +14,6 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/result" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/storage" ) @@ -33,7 +32,7 @@ func InitPut( args := cArgs.Args.(*roachpb.InitPutRequest) h := cArgs.Header - if args.FailOnTombstones && kvserverbase.GlobalMVCCRangeTombstoneForTesting { + if args.FailOnTombstones && cArgs.EvalCtx.EvalKnobs().DisableInitPutFailOnTombstones { args.FailOnTombstones = false } diff --git a/pkg/kv/kvserver/kvserverbase/BUILD.bazel b/pkg/kv/kvserver/kvserverbase/BUILD.bazel index a424377a5adb..09d46d29899d 100644 --- a/pkg/kv/kvserver/kvserverbase/BUILD.bazel +++ b/pkg/kv/kvserver/kvserverbase/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//pkg/kv/kvserver/kvserverpb", "//pkg/roachpb", "//pkg/settings", - "//pkg/util/envutil", "//pkg/util/errorutil", "//pkg/util/hlc", "//pkg/util/quotapool", diff --git a/pkg/kv/kvserver/kvserverbase/base.go b/pkg/kv/kvserver/kvserverbase/base.go index 35c55dca2cfe..b2f8df0c683b 100644 --- a/pkg/kv/kvserver/kvserverbase/base.go +++ b/pkg/kv/kvserver/kvserverbase/base.go @@ -20,7 +20,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" - "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/quotapool" "github.com/cockroachdb/errors" "github.com/cockroachdb/redact" @@ -219,10 +218,3 @@ var SplitByLoadMergeDelay = settings.RegisterDurationSetting( // MaxCommandSizeDefault is the default for the kv.raft.command.max_size // cluster setting. const MaxCommandSizeDefault = 64 << 20 - -// GlobalMVCCRangeTombstoneForTesting will write an MVCC range tombstone at the -// bottom of the SQL table data keyspace during cluster bootstrapping, for -// performance and correctness testing. This shouldn't affect data written above -// it, but activates range key-specific code paths in the storage layer. -var GlobalMVCCRangeTombstoneForTesting = envutil.EnvOrDefaultBool( - "COCKROACH_GLOBAL_MVCC_RANGE_TOMBSTONE", false) diff --git a/pkg/kv/kvserver/kvserverbase/knobs.go b/pkg/kv/kvserver/kvserverbase/knobs.go index 95d50a800d43..7e420b9d167f 100644 --- a/pkg/kv/kvserver/kvserverbase/knobs.go +++ b/pkg/kv/kvserver/kvserverbase/knobs.go @@ -38,6 +38,11 @@ type BatchEvalTestingKnobs struct { // default, this is not allowed because it is unsafe. See cmd_gc.go for an // explanation of why. AllowGCWithNewThresholdAndKeys bool + + // DisableInitPutFailOnTombstones disables FailOnTombstones for InitPut. This + // is useful together with e.g. StoreTestingKnobs.GlobalMVCCRangeTombstone, + // where we still want InitPut to succeed on top of the range tombstone. + DisableInitPutFailOnTombstones bool } // IntentResolverTestingKnobs contains testing helpers that are used during diff --git a/pkg/kv/kvserver/store_init.go b/pkg/kv/kvserver/store_init.go index 7d594c6944b3..854daae6c465 100644 --- a/pkg/kv/kvserver/store_init.go +++ b/pkg/kv/kvserver/store_init.go @@ -14,7 +14,6 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/keys" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/rditer" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" @@ -204,7 +203,7 @@ func WriteInitialClusterData( // If requested, write an MVCC range tombstone at the bottom of the // keyspace, for performance and correctness testing. - if kvserverbase.GlobalMVCCRangeTombstoneForTesting { + if knobs.GlobalMVCCRangeTombstone { if err := writeGlobalMVCCRangeTombstone(ctx, batch, desc, now.Prev()); err != nil { return err } diff --git a/pkg/kv/kvserver/testing_knobs.go b/pkg/kv/kvserver/testing_knobs.go index ce473fc30341..2763515b0774 100644 --- a/pkg/kv/kvserver/testing_knobs.go +++ b/pkg/kv/kvserver/testing_knobs.go @@ -422,6 +422,18 @@ type StoreTestingKnobs struct { // EnqueueReplicaInterceptor intercepts calls to `store.Enqueue()`. EnqueueReplicaInterceptor func(queueName string, replica *Replica) + + // GlobalMVCCRangeTombstone will write a global MVCC range tombstone across + // the entire user keyspace during cluster bootstrapping. This will be written + // below all other data, and thus won't affect query results, but it does + // activate MVCC range tombstone code paths in the storage layer for testing. + // + // This must typically be combined with the following knobs to prevent + // various components choking on the range tombstone: + // + // - rangefeed.TestingKnobs.IgnoreOnDeleteRangeError + // - kvserverbase.BatchEvalTestingKnobs.DisableInitPutFailOnTombstones + GlobalMVCCRangeTombstone bool } // ModuleTestingKnobs is part of the base.ModuleTestingKnobs interface. diff --git a/pkg/server/config.go b/pkg/server/config.go index ba4a4d0573f8..f8ff1467366a 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -25,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/docs" + "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server/status" @@ -224,9 +225,32 @@ func MakeBaseConfig(st *cluster.Settings, tr *tracing.Tracer) BaseConfig { // in a tag that is prefixed with "nsql". baseCfg.AmbientCtx.AddLogTag("n", baseCfg.IDContainer) baseCfg.InitDefaults() + baseCfg.InitTestingKnobs() + return baseCfg } +// InitTestingKnobs sets up any testing knobs based on e.g. envvars. +func (cfg *BaseConfig) InitTestingKnobs() { + // If requested, write an MVCC range tombstone at the bottom of the SQL table + // data keyspace during cluster bootstrapping, for performance and correctness + // testing. This shouldn't affect data written above it, but activates range + // key-specific code paths in the storage layer. We'll also have to tweak + // rangefeeds and batcheval to not choke on it. + if envutil.EnvOrDefaultBool("COCKROACH_GLOBAL_MVCC_RANGE_TOMBSTONE", false) { + if cfg.TestingKnobs.Store == nil { + cfg.TestingKnobs.Store = &kvserver.StoreTestingKnobs{} + } + if cfg.TestingKnobs.RangeFeed == nil { + cfg.TestingKnobs.RangeFeed = &rangefeed.TestingKnobs{} + } + storeKnobs := cfg.TestingKnobs.Store.(*kvserver.StoreTestingKnobs) + storeKnobs.GlobalMVCCRangeTombstone = true + storeKnobs.EvalKnobs.DisableInitPutFailOnTombstones = true + cfg.TestingKnobs.RangeFeed.(*rangefeed.TestingKnobs).IgnoreOnDeleteRangeError = true + } +} + // Config holds the parameters needed to set up a combined KV and SQL server. type Config struct { BaseConfig diff --git a/pkg/sql/conn_executor.go b/pkg/sql/conn_executor.go index b2cdd6e09864..5b7edf367853 100644 --- a/pkg/sql/conn_executor.go +++ b/pkg/sql/conn_executor.go @@ -2047,9 +2047,19 @@ func (ex *connExecutor) execCmd() error { } } case CopyIn: + ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionQueryReceived, tcmd.TimeReceived) + ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionStartParse, tcmd.ParseStart) + ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionEndParse, tcmd.ParseEnd) res = ex.clientComm.CreateCopyInResult(pos) var err error ev, payload, err = ex.execCopyIn(ctx, tcmd) + // Note: we write to ex.statsCollector.phaseTimes, instead of ex.phaseTimes, + // because: + // - stats use ex.statsCollector, not ex.phasetimes. + // - ex.statsCollector merely contains a copy of the times, that + // was created when the statement started executing (via the + // reset() method). + ex.statsCollector.PhaseTimes().SetSessionPhaseTime(sessionphase.SessionQueryServiced, timeutil.Now()) if err != nil { return err } diff --git a/pkg/sql/conn_io.go b/pkg/sql/conn_io.go index e6290165e9f5..40db4caf38ba 100644 --- a/pkg/sql/conn_io.go +++ b/pkg/sql/conn_io.go @@ -328,6 +328,13 @@ type CopyIn struct { // CopyDone is decremented once execution finishes, signaling that control of // the connection is being handed back to the network routine. CopyDone *sync.WaitGroup + // TimeReceived is the time at which the message was received + // from the client. Used to compute the service latency. + TimeReceived time.Time + // ParseStart/ParseEnd are the timing info for parsing of the query. Used for + // stats reporting. + ParseStart time.Time + ParseEnd time.Time } // command implements the Command interface. diff --git a/pkg/sql/drop_external_connection.go b/pkg/sql/drop_external_connection.go index e72dc911f4f9..576b955892b7 100644 --- a/pkg/sql/drop_external_connection.go +++ b/pkg/sql/drop_external_connection.go @@ -98,6 +98,18 @@ func (p *planner) dropExternalConnection(params runParams, n *tree.DropExternalC return errors.Wrapf(err, "failed to delete external connection") } + // We must also DELETE all rows from system.privileges that refer to + // external connection. + if _, err = params.extendedEvalCtx.ExecCfg.InternalExecutor.ExecEx( + params.ctx, + dropExternalConnectionOp, + params.p.Txn(), + sessiondata.InternalExecutorOverride{User: username.NodeUserName()}, + `DELETE FROM system.privileges WHERE path = $1`, ecPrivilege.GetPath(), + ); err != nil { + return errors.Wrapf(err, "failed to delete external connection") + } + return nil } diff --git a/pkg/sql/logictest/BUILD.bazel b/pkg/sql/logictest/BUILD.bazel index aac8d57a4a43..55b4a85c5fac 100644 --- a/pkg/sql/logictest/BUILD.bazel +++ b/pkg/sql/logictest/BUILD.bazel @@ -29,7 +29,9 @@ go_library( "//pkg/base", "//pkg/cloud/externalconn/providers", "//pkg/clusterversion", + "//pkg/kv/kvclient/rangefeed", "//pkg/kv/kvserver", + "//pkg/kv/kvserver/kvserverbase", "//pkg/roachpb", "//pkg/security/username", "//pkg/server", @@ -56,6 +58,7 @@ go_library( "//pkg/testutils/skip", "//pkg/testutils/sqlutils", "//pkg/upgrade", + "//pkg/util", "//pkg/util/envutil", "//pkg/util/log", "//pkg/util/randutil", diff --git a/pkg/sql/logictest/logic.go b/pkg/sql/logictest/logic.go index e274ac7bfe4e..5712d50595d7 100644 --- a/pkg/sql/logictest/logic.go +++ b/pkg/sql/logictest/logic.go @@ -40,7 +40,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" _ "github.com/cockroachdb/cockroach/pkg/cloud/externalconn/providers" // imported to register ExternalConnection providers "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" + "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security/username" "github.com/cockroachdb/cockroach/pkg/server" @@ -67,6 +69,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/upgrade" + "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/envutil" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/randutil" @@ -530,6 +533,13 @@ var ( "declarative-corpus", "", "enables generation and storage of a declarative schema changer corpus", ) + + // globalMVCCRangeTombstone will write a global MVCC range tombstone across + // the entire user keyspace during cluster bootstrapping. This should not + // semantically affect the test data written above it, but will activate MVCC + // range tombstone code paths in the storage layer for testing. + globalMVCCRangeTombstone = util.ConstantWithMetamorphicTestBool( + "logictest-global-mvcc-range-tombstone", false) ) const queryRewritePlaceholderPrefix = "__async_query_rewrite_placeholder" @@ -1186,6 +1196,12 @@ func (t *logicTest) newCluster( // when run with fakedist-disk config, so we'll use a larger limit here. // There isn't really a downside to doing so. tempStorageDiskLimit := int64(512 << 20) /* 512 MiB */ + // MVCC range tombstones are only available in 22.2 or newer. + enableGlobalMVCCRangeTombstone := globalMVCCRangeTombstone && + (t.cfg.BootstrapVersion.Equal(roachpb.Version{}) || + !t.cfg.BootstrapVersion.Less(roachpb.Version{Major: 22, Minor: 2})) && + (t.cfg.BinaryVersion.Equal(roachpb.Version{}) || + !t.cfg.BinaryVersion.Less(roachpb.Version{Major: 22, Minor: 2})) params := base.TestClusterArgs{ ServerArgs: base.TestServerArgs{ @@ -1197,7 +1213,11 @@ func (t *logicTest) newCluster( Knobs: base.TestingKnobs{ Store: &kvserver.StoreTestingKnobs{ // The consistency queue makes a lot of noisy logs during logic tests. - DisableConsistencyQueue: true, + DisableConsistencyQueue: true, + GlobalMVCCRangeTombstone: enableGlobalMVCCRangeTombstone, + EvalKnobs: kvserverbase.BatchEvalTestingKnobs{ + DisableInitPutFailOnTombstones: enableGlobalMVCCRangeTombstone, + }, }, SQLEvalContext: &eval.TestingKnobs{ AssertBinaryExprReturnTypes: true, @@ -1217,6 +1237,9 @@ func (t *logicTest) newCluster( SQLDeclarativeSchemaChanger: &scexec.TestingKnobs{ BeforeStage: corpusCollectionCallback, }, + RangeFeed: rangefeed.TestingKnobs{ + IgnoreOnDeleteRangeError: enableGlobalMVCCRangeTombstone, + }, }, ClusterName: "testclustername", ExternalIODir: t.sharedIODir, diff --git a/pkg/sql/pgwire/conn.go b/pkg/sql/pgwire/conn.go index 9b721fa10454..b6ff8d18f85d 100644 --- a/pkg/sql/pgwire/conn.go +++ b/pkg/sql/pgwire/conn.go @@ -860,7 +860,17 @@ func (c *conn) handleSimpleQuery( } copyDone := sync.WaitGroup{} copyDone.Add(1) - if err := c.stmtBuf.Push(ctx, sql.CopyIn{Conn: c, Stmt: cp, CopyDone: ©Done}); err != nil { + if err := c.stmtBuf.Push( + ctx, + sql.CopyIn{ + Conn: c, + Stmt: cp, + CopyDone: ©Done, + TimeReceived: timeReceived, + ParseStart: startParse, + ParseEnd: endParse, + }, + ); err != nil { return err } copyDone.Wait() diff --git a/pkg/sql/pgwire/testdata/pgtest/copy b/pkg/sql/pgwire/testdata/pgtest/copy index 51e2ee03784a..9d19c069bd44 100644 --- a/pkg/sql/pgwire/testdata/pgtest/copy +++ b/pkg/sql/pgwire/testdata/pgtest/copy @@ -927,3 +927,37 @@ ReadyForQuery {"Type":"DataRow","Values":[{"text":"{2,3,4,5,6}"}]} {"Type":"CommandComplete","CommandTag":"SELECT 2"} {"Type":"ReadyForQuery","TxStatus":"I"} + + +send crdb_only +Query {"String": "COPY c(d) FROM STDIN WITH CSV"} +CopyData {"Data": "\"{0,1}\"\n"} +CopyData {"Data": "\"{2,3,4,5,6}\"\n"} +CopyData {"Data": "\\.\n"} +CopyDone +---- + + +until crdb_only ignore=RowDescription +ReadyForQuery +---- +{"Type":"CopyInResponse","ColumnFormatCodes":[0]} +{"Type":"CommandComplete","CommandTag":"COPY 2"} +{"Type":"ReadyForQuery","TxStatus":"I"} + +# SHOW LAST QUERY STATISTICS cannot be used as a statement source, so we +# store the result as a variable in the test. +let $copy_service_latency crdb_only +Query {"String": "SHOW LAST QUERY STATISTICS RETURNING service_latency"} +---- + +send crdb_only +Query {"String": "SELECT '$copy_service_latency'::INTERVAL > '0 seconds'::INTERVAL"} +---- + +until crdb_only ignore=RowDescription +ReadyForQuery +---- +{"Type":"DataRow","Values":[{"text":"t"}]} +{"Type":"CommandComplete","CommandTag":"SELECT 1"} +{"Type":"ReadyForQuery","TxStatus":"I"} diff --git a/pkg/testutils/pgtest/datadriven.go b/pkg/testutils/pgtest/datadriven.go index 212769b7838c..73767b64d3c7 100644 --- a/pkg/testutils/pgtest/datadriven.go +++ b/pkg/testutils/pgtest/datadriven.go @@ -92,7 +92,11 @@ func RunTest(t *testing.T, path, addr, user string) { return d.Expected case "let": - require.Len(t, d.CmdArgs, 1, "only one argument permitted for let") + if (d.HasArg("crdb_only") && !p.isCockroachDB) || + (d.HasArg("noncrdb_only") && p.isCockroachDB) { + return d.Expected + } + require.GreaterOrEqual(t, len(d.CmdArgs), 1, "at least one argument required for let") require.Truef(t, strings.HasPrefix(d.CmdArgs[0].Key, "$"), "let argument must begin with '$'") lines := strings.Split(d.Input, "\n") require.Len(t, lines, 1, "only one input command permitted for let") diff --git a/vendor b/vendor index de32bf616c1b..ae570073390b 160000 --- a/vendor +++ b/vendor @@ -1 +1 @@ -Subproject commit de32bf616c1ba93bbcbecf6691698ed081d0c72b +Subproject commit ae570073390b63c28096ed52f90642f5087cbed2