From b732f2d70c456926c6e9cfbd6a35811619d53e32 Mon Sep 17 00:00:00 2001 From: Andrew Werner Date: Mon, 31 Jan 2022 14:36:04 -0500 Subject: [PATCH] sql,server: provide an approximate creation time for indexes This change plumbs a timestamp through to the index descriptor when it is created. The timestamp used is based on the transaction start time. The timestamp is then plumbed into the `crdb_internal.table_indexes` table and exposed as a new NULL-able TIMESTAMP column, `created_at`. Then, lastly, the timestamp is plumbed through to the status server via the `TableIndexStatsRequest`. Release note (sql change): The database now records the approximate time when an index was created it. This information is exposed via a new NULL-able TIMESTAMP column, `created_at`, on `crdb_internal.table_indexes`. --- docs/generated/http/full.md | 1 + pkg/ccl/importccl/import_table_creation.go | 2 + .../testdata/logic_test/crdb_internal_tenant | 4 +- .../serverccl/statusccl/tenant_status_test.go | 74 +++++++-- pkg/cli/testdata/doctor/test_recreate_zipdir | 12 +- pkg/server/index_usage_stats.go | 11 +- pkg/server/serverpb/status.proto | 4 + pkg/sql/alter_primary_key.go | 2 + pkg/sql/alter_table.go | 2 + pkg/sql/catalog/descpb/structured.proto | 4 + pkg/sql/catalog/table_elements.go | 7 + pkg/sql/catalog/tabledesc/BUILD.bazel | 1 + pkg/sql/catalog/tabledesc/index.go | 9 ++ pkg/sql/catalog/tabledesc/validate_test.go | 1 + pkg/sql/crdb_internal.go | 15 +- pkg/sql/create_index.go | 2 + pkg/sql/create_sequence.go | 1 + pkg/sql/create_table.go | 12 ++ .../testdata/logic_test/crdb_internal | 4 +- .../testdata/logic_test/create_statements | 6 +- .../testdata/logic_test/dependencies | 5 +- .../testdata/logic_test/hash_sharded_index | 4 +- .../testdata/show_trace_nonmetamorphic | 147 +++++++----------- pkg/sql/schema_change_plan_node.go | 1 + .../scbuild/testdata/alter_table | 24 +-- .../scbuild/testdata/create_index | 12 +- .../scbuild/testdata/drop_database | 4 +- .../scbuild/testdata/drop_schema | 2 +- .../schemachanger/scbuild/testdata/drop_table | 2 +- pkg/sql/schemachanger/scdeps/exec_deps.go | 20 +++ pkg/sql/schemachanger/scdeps/run_deps.go | 4 +- .../schemachanger/scdeps/sctestdeps/config.go | 25 +++ .../scdeps/sctestdeps/test_deps.go | 11 ++ .../scdeps/sctestdeps/test_state.go | 5 + pkg/sql/schemachanger/scexec/BUILD.bazel | 1 + pkg/sql/schemachanger/scexec/dependencies.go | 1 + pkg/sql/schemachanger/scexec/exec_mutation.go | 2 +- .../scexec/executor_external_test.go | 6 +- .../scexec/mocks_generated_test.go | 14 ++ .../scexec/scmutationexec/scmutationexec.go | 26 +++- pkg/sql/schemachanger/scpb/elements.proto | 8 +- .../testdata/alter_table_add_column | 3 + 42 files changed, 336 insertions(+), 165 deletions(-) diff --git a/docs/generated/http/full.md b/docs/generated/http/full.md index 0058540ac85a..ee226f4f2aad 100644 --- a/docs/generated/http/full.md +++ b/docs/generated/http/full.md @@ -4181,6 +4181,7 @@ Response object returned by TableIndexStatsResponse. | index_name | [string](#cockroach.server.serverpb.TableIndexStatsResponse-string) | | index_name is the name of the index. | [reserved](#support-status) | | index_type | [string](#cockroach.server.serverpb.TableIndexStatsResponse-string) | | index_type is the type of the index i.e. primary, secondary. | [reserved](#support-status) | | create_statement | [string](#cockroach.server.serverpb.TableIndexStatsResponse-string) | | create_statement is the SQL statement that would re-create the current index if executed. | [reserved](#support-status) | +| created_at | [google.protobuf.Timestamp](#cockroach.server.serverpb.TableIndexStatsResponse-google.protobuf.Timestamp) | | CreatedAt is an approximate timestamp at which the index was created. Note that it may not always be populated. | [reserved](#support-status) | diff --git a/pkg/ccl/importccl/import_table_creation.go b/pkg/ccl/importccl/import_table_creation.go index 2dfec9aca408..fc6a240c23a3 100644 --- a/pkg/ccl/importccl/import_table_creation.go +++ b/pkg/ccl/importccl/import_table_creation.go @@ -34,6 +34,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" "github.com/lib/pq/oid" ) @@ -168,6 +169,7 @@ func MakeSimpleTableDescriptor( Regions: makeImportRegionOperator(""), SessionDataStack: sessiondata.NewStack(&sessiondata.SessionData{}), ClientNoticeSender: &faketreeeval.DummyClientNoticeSender{}, + TxnTimestamp: timeutil.Unix(0, walltime), Settings: st, } affected := make(map[descpb.ID]*tabledesc.Mutable) diff --git a/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant b/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant index 670e099bbeb7..c4477aa8a6fd 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant +++ b/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant @@ -275,10 +275,10 @@ SELECT * FROM crdb_internal.table_columns WHERE descriptor_name = '' ---- descriptor_id descriptor_name column_id column_name column_type nullable default_expr hidden -query ITITTBBB colnames +query ITITTBBBT colnames SELECT * FROM crdb_internal.table_indexes WHERE descriptor_name = '' ---- -descriptor_id descriptor_name index_id index_name index_type is_unique is_inverted is_sharded +descriptor_id descriptor_name index_id index_name index_type is_unique is_inverted is_sharded created_at query ITITTITTB colnames SELECT * FROM crdb_internal.index_columns WHERE descriptor_name = '' diff --git a/pkg/ccl/serverccl/statusccl/tenant_status_test.go b/pkg/ccl/serverccl/statusccl/tenant_status_test.go index ef0a87c06a17..1bea8f4d9fa1 100644 --- a/pkg/ccl/serverccl/statusccl/tenant_status_test.go +++ b/pkg/ccl/serverccl/statusccl/tenant_status_test.go @@ -508,7 +508,7 @@ WHERE } func testTableIndexStats(ctx context.Context, t *testing.T, testHelper *tenantTestHelper) { - getTableIndexStats := func(helper *tenantTestHelper, db string) *serverpb.TableIndexStatsResponse { + getTableIndexStats := func(t *testing.T, helper *tenantTestHelper, db string) *serverpb.TableIndexStatsResponse { // Get index usage stats using function call. cluster := helper.testCluster() status := cluster.tenantStatusSrv(randomServer) @@ -520,6 +520,8 @@ func testTableIndexStats(ctx context.Context, t *testing.T, testHelper *tenantTe cluster := testHelper.testCluster() + timePreCreate := timeutil.Now() + // Create table on a database. cluster.tenantConn(0).Exec(t, ` CREATE DATABASE test_db1; @@ -555,21 +557,61 @@ FROM pg_catalog.pg_indexes WHERE tablename = 'test' AND indexname = $1` // Get index usage stats and assert expected results. - resp := getTableIndexStats(testHelper, "test_db1") - require.Equal(t, uint64(1), resp.Statistics[0].Statistics.Stats.TotalReadCount) - require.True(t, resp.Statistics[0].Statistics.Stats.LastRead.After(timePreRead)) - indexName := resp.Statistics[0].IndexName - createStmt := cluster.tenantConn(0).QueryStr(t, getCreateStmtQuery, indexName)[0][0] - print(createStmt) - require.Equal(t, resp.Statistics[0].CreateStatement, createStmt) - - resp = getTableIndexStats(testHelper, "test_db2") - require.Equal(t, uint64(0), resp.Statistics[0].Statistics.Stats.TotalReadCount) - require.Equal(t, resp.Statistics[0].Statistics.Stats.LastRead, time.Time{}) - indexName = resp.Statistics[0].IndexName - cluster.tenantConn(0).Exec(t, `SET DATABASE=test_db2`) - createStmt = cluster.tenantConn(0).QueryStr(t, getCreateStmtQuery, indexName)[0][0] - require.Equal(t, resp.Statistics[0].CreateStatement, createStmt) + requireAfter := func(t *testing.T, a, b *time.Time) { + t.Helper() + require.NotNil(t, a) + require.NotNil(t, b) + require.Truef(t, a.After(*b), "%v is not after %v", a, b) + } + requireBetween := func(t *testing.T, before time.Time, ts *time.Time, after time.Time) { + t.Helper() + requireAfter(t, ts, &before) + requireAfter(t, &after, ts) + } + + t.Run("validate read index", func(t *testing.T) { + resp := getTableIndexStats(t, testHelper, "test_db1") + require.Equal(t, uint64(1), resp.Statistics[0].Statistics.Stats.TotalReadCount) + requireAfter(t, &resp.Statistics[0].Statistics.Stats.LastRead, &timePreRead) + indexName := resp.Statistics[0].IndexName + createStmt := cluster.tenantConn(0).QueryStr(t, getCreateStmtQuery, indexName)[0][0] + print(createStmt) + require.Equal(t, resp.Statistics[0].CreateStatement, createStmt) + requireBetween(t, timePreCreate, resp.Statistics[0].CreatedAt, timePreRead) + }) + + t.Run("validate unread index", func(t *testing.T) { + resp := getTableIndexStats(t, testHelper, "test_db2") + require.Equal(t, uint64(0), resp.Statistics[0].Statistics.Stats.TotalReadCount) + require.Equal(t, resp.Statistics[0].Statistics.Stats.LastRead, time.Time{}) + indexName := resp.Statistics[0].IndexName + cluster.tenantConn(0).Exec(t, `SET DATABASE=test_db2`) + createStmt := cluster.tenantConn(0).QueryStr(t, getCreateStmtQuery, indexName)[0][0] + require.Equal(t, resp.Statistics[0].CreateStatement, createStmt) + requireBetween(t, timePreCreate, resp.Statistics[0].CreatedAt, timePreRead) + }) + + // Test that a subsequent index creation has an appropriate timestamp. + t.Run("validate CreatedAt for new index", func(t *testing.T) { + timeBeforeCreateNewIndex := timeutil.Now() + cluster.tenantConn(0).Exec(t, ` +SET DATABASE=test_db2; +CREATE INDEX idx2 ON test (b, a)`) + timeAfterCreateNewIndex := timeutil.Now() + + resp := getTableIndexStats(t, testHelper, "test_db2") + var stat serverpb.TableIndexStatsResponse_ExtendedCollectedIndexUsageStatistics + var found bool + for _, idx := range resp.Statistics { + if found = idx.IndexName == "idx2"; found { + stat = *idx + break + } + } + require.True(t, found) + requireBetween(t, + timeBeforeCreateNewIndex, stat.CreatedAt, timeAfterCreateNewIndex) + }) } func ensureExpectedStmtFingerprintExistsInRPCResponse( diff --git a/pkg/cli/testdata/doctor/test_recreate_zipdir b/pkg/cli/testdata/doctor/test_recreate_zipdir index 2c0249ed2b11..63931053831a 100644 --- a/pkg/cli/testdata/doctor/test_recreate_zipdir +++ b/pkg/cli/testdata/doctor/test_recreate_zipdir @@ -12,16 +12,16 @@ SELECT crdb_internal.unsafe_upsert_descriptor(50, decode('12380a0964656661756c74 SELECT crdb_internal.unsafe_upsert_namespace_entry(0, 0, 'defaultdb', 50, true); SELECT crdb_internal.unsafe_upsert_descriptor(51, decode('12370a08706f73746772657310331a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f7418012200280140004a00', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(0, 0, 'postgres', 51, true); -SELECT crdb_internal.unsafe_upsert_descriptor(53, decode('0afa040a0575736572731835203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a046e616d6510031a0d0807100018003007509308600020013000680070007800800100880100980100422d0a076164647265737310041a0d080710001800300750930860002001300068007000780080010088010098010042310a0b6372656469745f6361726410051a0d08071000180030075093086000200130006800700078008001008801009801004806525d0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b2013d0a077072696d61727910001a0269641a04636974791a046e616d651a07616464726573731a0b6372656469745f63617264200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200aa02290836100210041802180120352a11666b5f636974795f7265665f757365727330023800400048007000aa02290837100210041802180120352a11666b5f636974795f7265665f757365727330023800400048007000aa0229083a100110021802180120352a11666b5f636974795f7265665f757365727330023800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(53, decode('0afd040a0575736572731835203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a046e616d6510031a0d0807100018003007509308600020013000680070007800800100880100980100422d0a076164647265737310041a0d080710001800300750930860002001300068007000780080010088010098010042310a0b6372656469745f6361726410051a0d0807100018003007509308600020013000680070007800800100880100980100480652600a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b2013d0a077072696d61727910001a0269641a04636974791a046e616d651a07616464726573731a0b6372656469745f63617264200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200aa02290836100210041802180120352a11666b5f636974795f7265665f757365727330023800400048007000aa02290837100210041802180120352a11666b5f636974795f7265665f757365727330023800400048007000aa0229083a100110021802180120352a11666b5f636974795f7265665f757365727330023800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'users', 53, true); -SELECT crdb_internal.unsafe_upsert_descriptor(54, decode('0a9e070a0876656869636c65731836203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a047479706510031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a086f776e65725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510051a0d080510001800300050da08600020013000680070007800800100880100980100422c0a0673746174757310061a0d080710001800300750930860002001300068007000780080010088010098010042360a1063757272656e745f6c6f636174696f6e10071a0d080710001800300750930860002001300068007000780080010088010098010042290a0365787410081a0d081210001800300050da1d6000200130006800700078008001008801009801004809525d0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c001008802005a83010a2576656869636c65735f6175746f5f696e6465785f666b5f636974795f7265665f75736572731002180022046369747922086f776e65725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010088020060036a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201650a077072696d61727910001a0269641a04636974791a04747970651a086f776e65725f69641a0d6372656174696f6e5f74696d651a067374617475731a1063757272656e745f6c6f636174696f6e1a03657874200120022003200420052006200720082800b80101c20100e80100f2010408001200f801008002009202009a0200a202290836100210041802180120352a11666b5f636974795f7265665f757365727330003800400048007000aa02340837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c657330023800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(54, decode('0aa4070a0876656869636c65731836203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a047479706510031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a086f776e65725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510051a0d080510001800300050da08600020013000680070007800800100880100980100422c0a0673746174757310061a0d080710001800300750930860002001300068007000780080010088010098010042360a1063757272656e745f6c6f636174696f6e10071a0d080710001800300750930860002001300068007000780080010088010098010042290a0365787410081a0d081210001800300050da1d600020013000680070007800800100880100980100480952600a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c801008802005a86010a2576656869636c65735f6175746f5f696e6465785f666b5f636974795f7265665f75736572731002180022046369747922086f776e65725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010088020060036a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201650a077072696d61727910001a0269641a04636974791a04747970651a086f776e65725f69641a0d6372656174696f6e5f74696d651a067374617475731a1063757272656e745f6c6f636174696f6e1a03657874200120022003200420052006200720082800b80101c20100e80100f2010408001200f801008002009202009a0200a202290836100210041802180120352a11666b5f636974795f7265665f757365727330003800400048007000aa02340837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c657330023800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'vehicles', 54, true); -SELECT crdb_internal.unsafe_upsert_descriptor(55, decode('0aae0a0a0572696465731837203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d080710001800300750930860002000300068007000780080010088010098010042320a0c76656869636c655f6369747910031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a0872696465725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042300a0a76656869636c655f696410051a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d73746172745f6164647265737310061a0d080710001800300750930860002001300068007000780080010088010098010042310a0b656e645f6164647265737310071a0d080710001800300750930860002001300068007000780080010088010098010042300a0a73746172745f74696d6510081a0d080510001800300050da08600020013000680070007800800100880100980100422e0a08656e645f74696d6510091a0d080510001800300050da08600020013000680070007800800100880100980100422d0a07726576656e7565100a1a0d08031002180a300050a40d600020013000680070007800800100880100980100480b525d0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c001008802005a80010a2272696465735f6175746f5f696e6465785f666b5f636974795f7265665f757365727310021800220463697479220872696465725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c001008802005a97010a2d72696465735f6175746f5f696e6465785f666b5f76656869636c655f636974795f7265665f76656869636c657310031800220c76656869636c655f63697479220a76656869636c655f69643003300538023801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010088020060046a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800103880103980100a2013a0a1376656869636c655f63697479203d20636974791217636865636b5f76656869636c655f636974795f63697479180028022803300038004000b2018a010a077072696d61727910001a0269641a04636974791a0c76656869636c655f636974791a0872696465725f69641a0a76656869636c655f69641a0d73746172745f616464726573731a0b656e645f616464726573731a0a73746172745f74696d651a08656e645f74696d651a07726576656e7565200120022003200420052006200720082009200a2800b80101c20100e80100f2010408001200f801008002009202009a0200a202290837100210041802180120352a11666b5f636974795f7265665f757365727330003800400048007000a202340837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c657330003800400048007000aa02290838100110021802180120372a11666b5f636974795f7265665f726964657330023800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(55, decode('0ab70a0a0572696465731837203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d080710001800300750930860002000300068007000780080010088010098010042320a0c76656869636c655f6369747910031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a0872696465725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042300a0a76656869636c655f696410051a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d73746172745f6164647265737310061a0d080710001800300750930860002001300068007000780080010088010098010042310a0b656e645f6164647265737310071a0d080710001800300750930860002001300068007000780080010088010098010042300a0a73746172745f74696d6510081a0d080510001800300050da08600020013000680070007800800100880100980100422e0a08656e645f74696d6510091a0d080510001800300050da08600020013000680070007800800100880100980100422d0a07726576656e7565100a1a0d08031002180a300050a40d600020013000680070007800800100880100980100480b52600a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c801008802005a83010a2272696465735f6175746f5f696e6465785f666b5f636974795f7265665f757365727310021800220463697479220872696465725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c801008802005a9a010a2d72696465735f6175746f5f696e6465785f666b5f76656869636c655f636974795f7265665f76656869636c657310031800220c76656869636c655f63697479220a76656869636c655f69643003300538023801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010088020060046a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800103880103980100a2013a0a1376656869636c655f63697479203d20636974791217636865636b5f76656869636c655f636974795f63697479180028022803300038004000b2018a010a077072696d61727910001a0269641a04636974791a0c76656869636c655f636974791a0872696465725f69641a0a76656869636c655f69641a0d73746172745f616464726573731a0b656e645f616464726573731a0a73746172745f74696d651a08656e645f74696d651a07726576656e7565200120022003200420052006200720082009200a2800b80101c20100e80100f2010408001200f801008002009202009a0200a202290837100210041802180120352a11666b5f636974795f7265665f757365727330003800400048007000a202340837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c657330003800400048007000aa02290838100110021802180120372a11666b5f636974795f7265665f726964657330023800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'rides', 55, true); -SELECT crdb_internal.unsafe_upsert_descriptor(56, decode('0ac9040a1a76656869636c655f6c6f636174696f6e5f686973746f726965731838203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07726964655f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422f0a0974696d657374616d7010031a0d080510001800300050da0860002000300068007000780080010088010098010042290a036c617410041a0d080210401800300050bd05600020013000680070007800800100880100980100422a0a046c6f6e6710051a0d080210401800300050bd05600020013000680070007800800100880100980100480652710a077072696d617279100118012204636974792207726964655f6964220974696d657374616d703001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b2013c0a077072696d61727910001a04636974791a07726964655f69641a0974696d657374616d701a036c61741a046c6f6e67200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a202290838100110021802180120372a11666b5f636974795f7265665f726964657330003800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(56, decode('0acc040a1a76656869636c655f6c6f636174696f6e5f686973746f726965731838203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07726964655f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422f0a0974696d657374616d7010031a0d080510001800300050da0860002000300068007000780080010088010098010042290a036c617410041a0d080210401800300050bd05600020013000680070007800800100880100980100422a0a046c6f6e6710051a0d080210401800300050bd05600020013000680070007800800100880100980100480652740a077072696d617279100118012204636974792207726964655f6964220974696d657374616d703001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b2013c0a077072696d61727910001a04636974791a07726964655f69641a0974696d657374616d701a036c61741a046c6f6e67200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a202290838100110021802180120372a11666b5f636974795f7265665f726964657330003800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'vehicle_location_histories', 56, true); -SELECT crdb_internal.unsafe_upsert_descriptor(57, decode('0a9c040a0b70726f6d6f5f636f6465731839203428013a00422a0a04636f646510011a0d080710001800300750930860002000300068007000780080010088010098010042310a0b6465736372697074696f6e10021a0d080710001800300750930860002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510031a0d080510001800300050da0860002001300068007000780080010088010098010042350a0f65787069726174696f6e5f74696d6510041a0d080510001800300050da08600020013000680070007800800100880100980100422b0a0572756c657310051a0d081210001800300050da1d600020013000680070007800800100880100980100480652550a077072696d617279100118012204636f6465300140004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b201510a077072696d61727910001a04636f64651a0b6465736372697074696f6e1a0d6372656174696f6e5f74696d651a0f65787069726174696f6e5f74696d651a0572756c6573200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(57, decode('0a9f040a0b70726f6d6f5f636f6465731839203428013a00422a0a04636f646510011a0d080710001800300750930860002000300068007000780080010088010098010042310a0b6465736372697074696f6e10021a0d080710001800300750930860002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510031a0d080510001800300050da0860002001300068007000780080010088010098010042350a0f65787069726174696f6e5f74696d6510041a0d080510001800300050da08600020013000680070007800800100880100980100422b0a0572756c657310051a0d081210001800300050da1d600020013000680070007800800100880100980100480652580a077072696d617279100118012204636f6465300140004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b201510a077072696d61727910001a04636f64651a0b6465736372697074696f6e1a0d6372656174696f6e5f74696d651a0f65787069726174696f6e5f74696d651a0572756c6573200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'promo_codes', 57, true); -SELECT crdb_internal.unsafe_upsert_descriptor(58, decode('0ac9040a10757365725f70726f6d6f5f636f646573183a203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07757365725f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422a0a04636f646510031a0d0807100018003007509308600020003000680070007800800100880100980100422f0a0974696d657374616d7010041a0d080510001800300050da0860002001300068007000780080010088010098010042300a0b75736167655f636f756e7410051a0c080110401800300050146000200130006800700078008001008801009801004806526c0a077072696d617279100118012204636974792207757365725f69642204636f64653001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201440a077072696d61727910001a04636974791a07757365725f69641a04636f64651a0974696d657374616d701a0b75736167655f636f756e74200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a20229083a100110021802180120352a11666b5f636974795f7265665f757365727330003800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(58, decode('0acc040a10757365725f70726f6d6f5f636f646573183a203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07757365725f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422a0a04636f646510031a0d0807100018003007509308600020003000680070007800800100880100980100422f0a0974696d657374616d7010041a0d080510001800300050da0860002001300068007000780080010088010098010042300a0b75736167655f636f756e7410051a0c080110401800300050146000200130006800700078008001008801009801004806526f0a077072696d617279100118012204636974792207757365725f69642204636f64653001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010088020060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201440a077072696d61727910001a04636974791a07757365725f69641a04636f64651a0974696d657374616d701a0b75736167655f636f756e74200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a20229083a100110021802180120352a11666b5f636974795f7265665f757365727330003800400048007000b20200b80200c0021dc80200e00200f00200800300880300', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'user_promo_codes', 58, true); COMMIT; diff --git a/pkg/server/index_usage_stats.go b/pkg/server/index_usage_stats.go index 102d864469c3..204a4ba7cbc7 100644 --- a/pkg/server/index_usage_stats.go +++ b/pkg/server/index_usage_stats.go @@ -230,7 +230,8 @@ func getTableIndexUsageStats( ti.index_type, total_reads, last_read, - indexdef + indexdef, + ti.created_at FROM crdb_internal.index_usage_statistics AS us JOIN crdb_internal.table_indexes AS ti ON us.index_id = ti.index_id AND us.table_id = ti.descriptor_id @@ -241,7 +242,7 @@ func getTableIndexUsageStats( tableID, ) - const expectedNumDatums = 6 + const expectedNumDatums = 7 it, err := ie.QueryIteratorEx(ctx, "index-usage-stats", nil, sessiondata.InternalExecutorOverride{ @@ -279,6 +280,11 @@ func getTableIndexUsageStats( lastRead = tree.MustBeDTimestampTZ(row[4]).Time } createStmt := tree.MustBeDString(row[5]) + var createdAt *time.Time + if row[6] != tree.DNull { + ts := tree.MustBeDTimestamp(row[6]) + createdAt = &ts.Time + } if err != nil { return nil, err @@ -298,6 +304,7 @@ func getTableIndexUsageStats( IndexName: string(indexName), IndexType: string(indexType), CreateStatement: string(createStmt), + CreatedAt: createdAt, } idxUsageStats = append(idxUsageStats, idxStatsRow) diff --git a/pkg/server/serverpb/status.proto b/pkg/server/serverpb/status.proto index 68ebbbe70ba0..87747bda1697 100644 --- a/pkg/server/serverpb/status.proto +++ b/pkg/server/serverpb/status.proto @@ -1426,6 +1426,10 @@ message TableIndexStatsResponse { // create_statement is the SQL statement that would re-create the // current index if executed. string create_statement = 4; + // CreatedAt is an approximate timestamp at which the index was created. + // Note that it may not always be populated. + google.protobuf.Timestamp created_at = 5 [(gogoproto.stdtime) = true]; + } repeated ExtendedCollectedIndexUsageStatistics statistics = 1; diff --git a/pkg/sql/alter_primary_key.go b/pkg/sql/alter_primary_key.go index 351bbd37964d..2ea5d128bb16 100644 --- a/pkg/sql/alter_primary_key.go +++ b/pkg/sql/alter_primary_key.go @@ -12,6 +12,7 @@ package sql import ( "context" + "time" "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/catalog" @@ -180,6 +181,7 @@ func (p *planner) AlterPrimaryKey( Type: descpb.IndexDescriptor_FORWARD, Version: descpb.LatestNonPrimaryIndexDescriptorVersion, ConstraintID: tableDesc.GetNextConstraintID(), + CreatedAtNanos: p.EvalContext().GetTxnTimestamp(time.Microsecond).UnixNano(), } tableDesc.NextConstraintID++ diff --git a/pkg/sql/alter_table.go b/pkg/sql/alter_table.go index 591124956e6b..a53a5916c649 100644 --- a/pkg/sql/alter_table.go +++ b/pkg/sql/alter_table.go @@ -16,6 +16,7 @@ import ( gojson "encoding/json" "fmt" "sort" + "time" "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" @@ -270,6 +271,7 @@ func (n *alterTableNode) startExec(params runParams) error { Name: string(d.Name), Unique: true, StoreColumnNames: d.Storing.ToStrings(), + CreatedAtNanos: params.EvalContext().GetTxnTimestamp(time.Microsecond).UnixNano(), } if err := idx.FillColumns(d.Columns); err != nil { return err diff --git a/pkg/sql/catalog/descpb/structured.proto b/pkg/sql/catalog/descpb/structured.proto index ae60b9f7e5be..554f1775c0ba 100644 --- a/pkg/sql/catalog/descpb/structured.proto +++ b/pkg/sql/catalog/descpb/structured.proto @@ -538,6 +538,10 @@ message IndexDescriptor { // indexes. optional uint32 constraint_id = 33 [(gogoproto.customname) = "ConstraintID", (gogoproto.casttype) = "ConstraintID", (gogoproto.nullable) = false]; + + // CreatedAtNanos, if non-zero, represents an approximate time at which the + // index was created. + optional int64 created_at_nanos = 25 [(gogoproto.nullable) = false]; } // ConstraintToUpdate represents a constraint to be added to the table and diff --git a/pkg/sql/catalog/table_elements.go b/pkg/sql/catalog/table_elements.go index b9f4bbda105d..cfaf24b62d59 100644 --- a/pkg/sql/catalog/table_elements.go +++ b/pkg/sql/catalog/table_elements.go @@ -11,6 +11,8 @@ package catalog import ( + "time" + "github.com/cockroachdb/cockroach/pkg/geo/geoindex" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" @@ -209,6 +211,11 @@ type Index interface { // checked by the schema changer before they are brought back // online. ForcePut() bool + + // CreatedAt is an approximate timestamp at which the index was created. + // It is derived from the statement time at which the relevant statement + // was issued. + CreatedAt() time.Time } // Column is an interface around the column descriptor types. diff --git a/pkg/sql/catalog/tabledesc/BUILD.bazel b/pkg/sql/catalog/tabledesc/BUILD.bazel index ed810895514c..bcbc9b271937 100644 --- a/pkg/sql/catalog/tabledesc/BUILD.bazel +++ b/pkg/sql/catalog/tabledesc/BUILD.bazel @@ -47,6 +47,7 @@ go_library( "//pkg/util/iterutil", "//pkg/util/log", "//pkg/util/protoutil", + "//pkg/util/timeutil", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_redact//:redact", "@com_github_lib_pq//oid", diff --git a/pkg/sql/catalog/tabledesc/index.go b/pkg/sql/catalog/tabledesc/index.go index 35bef93eb28b..6bc028c6b293 100644 --- a/pkg/sql/catalog/tabledesc/index.go +++ b/pkg/sql/catalog/tabledesc/index.go @@ -13,6 +13,7 @@ package tabledesc import ( "fmt" "strings" + "time" "github.com/cockroachdb/cockroach/pkg/geo/geoindex" "github.com/cockroachdb/cockroach/pkg/sql/catalog" @@ -21,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/cockroach/pkg/util/iterutil" "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" ) var _ catalog.Index = (*index)(nil) @@ -351,6 +353,13 @@ func (w index) ForcePut() bool { return w.Merging() || w.desc.UseDeletePreservingEncoding } +func (w index) CreatedAt() time.Time { + if w.desc.CreatedAtNanos == 0 { + return time.Time{} + } + return timeutil.Unix(0, w.desc.CreatedAtNanos) +} + // partitioning is the backing struct for a catalog.Partitioning interface. type partitioning struct { desc *descpb.PartitioningDescriptor diff --git a/pkg/sql/catalog/tabledesc/validate_test.go b/pkg/sql/catalog/tabledesc/validate_test.go index 9ffcd7d29441..f83eca7b0f3a 100644 --- a/pkg/sql/catalog/tabledesc/validate_test.go +++ b/pkg/sql/catalog/tabledesc/validate_test.go @@ -170,6 +170,7 @@ var validationMap = []struct { "Predicate": {status: iSolemnlySwearThisFieldIsValidated}, "UseDeletePreservingEncoding": {status: thisFieldReferencesNoObjects}, "ConstraintID": {status: iSolemnlySwearThisFieldIsValidated}, + "CreatedAtNanos": {status: thisFieldReferencesNoObjects}, }, }, { diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index ba379ae3b4d5..462996a28a82 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -2623,8 +2623,6 @@ CREATE TABLE crdb_internal.table_columns ( } // crdbInternalTableIndexesTable exposes the index descriptors. -// -// TODO(tbg): prefix with kv_. var crdbInternalTableIndexesTable = virtualSchemaTable{ comment: "indexes accessible by current user in current database (KV scan)", schema: ` @@ -2636,7 +2634,8 @@ CREATE TABLE crdb_internal.table_indexes ( index_type STRING NOT NULL, is_unique BOOL NOT NULL, is_inverted BOOL NOT NULL, - is_sharded BOOL NOT NULL + is_sharded BOOL NOT NULL, + created_at TIMESTAMP ) `, generator: func(ctx context.Context, p *planner, dbContext catalog.DatabaseDescriptor, stopper *stop.Stopper) (virtualTableGenerator, cleanupFunc, error) { @@ -2658,6 +2657,15 @@ CREATE TABLE crdb_internal.table_indexes ( if idx.Primary() { idxType = primary } + createdAt := tree.DNull + if ts := idx.CreatedAt(); !ts.IsZero() { + tsDatum, err := tree.MakeDTimestamp(ts, time.Nanosecond) + if err != nil { + log.Warningf(ctx, "failed to construct timestamp for index: %v", err) + } else { + createdAt = tsDatum + } + } row = append(row, tableID, tableName, @@ -2667,6 +2675,7 @@ CREATE TABLE crdb_internal.table_indexes ( tree.MakeDBool(tree.DBool(idx.IsUnique())), tree.MakeDBool(idx.GetType() == descpb.IndexDescriptor_INVERTED), tree.MakeDBool(tree.DBool(idx.IsSharded())), + createdAt, ) return pusher.pushRow(row...) }) diff --git a/pkg/sql/create_index.go b/pkg/sql/create_index.go index e7cad505a074..cfb0d30617e6 100644 --- a/pkg/sql/create_index.go +++ b/pkg/sql/create_index.go @@ -12,6 +12,7 @@ package sql import ( "context" + "time" "github.com/cockroachdb/cockroach/pkg/docs" "github.com/cockroachdb/cockroach/pkg/geo/geoindex" @@ -195,6 +196,7 @@ func makeIndexDescriptor( Unique: n.Unique, StoreColumnNames: n.Storing.ToStrings(), CreatedExplicitly: true, + CreatedAtNanos: params.EvalContext().GetTxnTimestamp(time.Microsecond).UnixNano(), } if n.Inverted { diff --git a/pkg/sql/create_sequence.go b/pkg/sql/create_sequence.go index 0a854910dffa..83e23144f035 100644 --- a/pkg/sql/create_sequence.go +++ b/pkg/sql/create_sequence.go @@ -278,6 +278,7 @@ func NewSequenceTableDesc( KeyColumnDirections: []descpb.IndexDescriptor_Direction{descpb.IndexDescriptor_ASC}, EncodingType: descpb.PrimaryIndexEncoding, Version: descpb.LatestPrimaryIndexDescriptorVersion, + CreatedAtNanos: creationTime.WallTime, }) desc.Families = []descpb.ColumnFamilyDescriptor{ { diff --git a/pkg/sql/create_table.go b/pkg/sql/create_table.go index 04eca6a63b84..abc4cb467bf5 100644 --- a/pkg/sql/create_table.go +++ b/pkg/sql/create_table.go @@ -16,6 +16,7 @@ import ( "go/constant" "strconv" "strings" + "time" "github.com/cockroachdb/cockroach/pkg/build" "github.com/cockroachdb/cockroach/pkg/docs" @@ -2172,6 +2173,17 @@ func NewTableDesc( return nil, err } + // Note that due to historical reasons, the automatic creation of the primary + // index occurs in AllocateIDs. That call does not have access to the current + // timestamp to set the created_at timestamp. + if desc.IsPhysicalTable() && !catalog.IsSystemDescriptor(&desc) { + ts := evalCtx.GetTxnTimestamp(time.Microsecond).UnixNano() + _ = catalog.ForEachNonDropIndex(&desc, func(idx catalog.Index) error { + idx.IndexDesc().CreatedAtNanos = ts + return nil + }) + } + // Record the types of indexes that the table has. if err := catalog.ForEachNonDropIndex(&desc, func(idx catalog.Index) error { if idx.IsSharded() { diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal b/pkg/sql/logictest/testdata/logic_test/crdb_internal index 72c43296b0d2..0f75accd8cf9 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal @@ -396,10 +396,10 @@ SELECT * FROM crdb_internal.table_columns WHERE descriptor_name = '' ---- descriptor_id descriptor_name column_id column_name column_type nullable default_expr hidden -query ITITTBBB colnames +query ITITTBBBT colnames SELECT * FROM crdb_internal.table_indexes WHERE descriptor_name = '' ---- -descriptor_id descriptor_name index_id index_name index_type is_unique is_inverted is_sharded +descriptor_id descriptor_name index_id index_name index_type is_unique is_inverted is_sharded created_at query ITITTITTB colnames SELECT * FROM crdb_internal.index_columns WHERE descriptor_name = '' diff --git a/pkg/sql/logictest/testdata/logic_test/create_statements b/pkg/sql/logictest/testdata/logic_test/create_statements index 17fdb42c0429..7985370836fc 100644 --- a/pkg/sql/logictest/testdata/logic_test/create_statements +++ b/pkg/sql/logictest/testdata/logic_test/create_statements @@ -1444,7 +1444,8 @@ CREATE TABLE crdb_internal.table_indexes ( index_type STRING NOT NULL, is_unique BOOL NOT NULL, is_inverted BOOL NOT NULL, - is_sharded BOOL NOT NULL + is_sharded BOOL NOT NULL, + created_at TIMESTAMP NULL ) CREATE TABLE crdb_internal.table_indexes ( descriptor_id INT8 NULL, descriptor_name STRING NOT NULL, @@ -1453,7 +1454,8 @@ CREATE TABLE crdb_internal.table_indexes ( index_type STRING NOT NULL, is_unique BOOL NOT NULL, is_inverted BOOL NOT NULL, - is_sharded BOOL NOT NULL + is_sharded BOOL NOT NULL, + created_at TIMESTAMP NULL ) {} {} CREATE TABLE crdb_internal.table_row_statistics ( table_id INT8 NOT NULL, diff --git a/pkg/sql/logictest/testdata/logic_test/dependencies b/pkg/sql/logictest/testdata/logic_test/dependencies index 41dcfcfa5492..0d18cbab6770 100644 --- a/pkg/sql/logictest/testdata/logic_test/dependencies +++ b/pkg/sql/logictest/testdata/logic_test/dependencies @@ -42,7 +42,10 @@ descriptor_id descriptor_name column_id column_name column_type 115 test_uwi_child 2 rowid family:IntFamily width:64 precision:0 locale:"" visible_type:0 oid:20 time_precision_is_set:false false unique_rowid() true query ITITTBBB colnames -SELECT * FROM crdb_internal.table_indexes WHERE descriptor_name LIKE 'test_%' ORDER BY descriptor_id, index_id + SELECT descriptor_id, descriptor_name, index_id, index_name, index_type, is_unique, is_inverted, is_sharded + FROM crdb_internal.table_indexes + WHERE descriptor_name LIKE 'test_%' +ORDER BY descriptor_id, index_id ---- descriptor_id descriptor_name index_id index_name index_type is_unique is_inverted is_sharded 106 test_kv 1 test_kv_pkey primary true false false diff --git a/pkg/sql/logictest/testdata/logic_test/hash_sharded_index b/pkg/sql/logictest/testdata/logic_test/hash_sharded_index index 8c5ce19dffe5..6f33e5232d81 100644 --- a/pkg/sql/logictest/testdata/logic_test/hash_sharded_index +++ b/pkg/sql/logictest/testdata/logic_test/hash_sharded_index @@ -648,7 +648,9 @@ statement ok CREATE TABLE poor_t (a INT PRIMARY KEY, b INT, INDEX t_idx_b (b) USING HASH WITH (bucket_count=8)) query ITITTBBB colnames -SELECT * FROM crdb_internal.table_indexes WHERE descriptor_name = 'poor_t' +SELECT descriptor_id, descriptor_name, index_id, index_name, index_type, is_unique, is_inverted, is_sharded + FROM crdb_internal.table_indexes + WHERE descriptor_name = 'poor_t' ---- descriptor_id descriptor_name index_id index_name index_type is_unique is_inverted is_sharded 130 poor_t 1 poor_t_pkey primary true false false diff --git a/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic b/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic index 985016559a29..b4ee814df380 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic +++ b/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic @@ -2,20 +2,48 @@ # Check SHOW KV TRACE FOR SESSION. +let $trace_query +SELECT $$ + SELECT + operation, + regexp_replace( + regexp_replace( + regexp_replace( + regexp_replace( + message, + 'create_as_of_time:<[^>]+>', + 'create_as_of_time:<...>' + ), + 'created_at_nanos:[1-9]\d*', + 'created_at_nanos:...', + 'g' + ), + 'mutationJobs:<[^>]*>', + 'mutationJobs:<...>' + ), + 'drop_time:[1-9]\d*', + 'drop_time:...' + ) + AS message + FROM + [SHOW KV TRACE FOR SESSION] + WHERE + message NOT LIKE '%Z/%' + AND operation NOT LIKE 'kv.DistSender: sending partial batch%' + AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' + AND tag NOT LIKE '%intExec=%' + AND tag NOT LIKE '%scExec%' + AND tag NOT LIKE '%IndexBackfiller%' + AND operation != 'dist sender send' +$$ + statement ok SET tracing = on,kv,results; CREATE DATABASE t; SET tracing = off # Check the KV trace; we need to remove the eventlog entry and # internal queries since the timestamp is non-deterministic. query TT -SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- batch flow coordinator CPut /NamespaceTable/30/1/106/0/"public"/4/1 -> 107 batch flow coordinator CPut /Table/3/1/107/2/1 -> schema: version:1 parent_id:106 privileges: users: users: owner_proto:"admin" version:2 > > @@ -29,18 +57,10 @@ statement ok SET tracing = on,kv,results; CREATE TABLE t.kv(k INT PRIMARY KEY, v INT, FAMILY "primary" (k, v)); SET tracing = off query TT -SELECT operation, regexp_replace(message, 'create_as_of_time:<[^>]+>', 'create_as_of_time:<...>') as message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- batch flow coordinator CPut /NamespaceTable/30/1/106/107/"kv"/4/1 -> 108 -batch flow coordinator CPut /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:2 > +batch flow coordinator CPut /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 created_at_nanos:... > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:2 > sql query rows affected: 0 # We avoid using the full trace output, because that would make the @@ -53,28 +73,16 @@ statement ok SET tracing = on,kv,results; CREATE UNIQUE INDEX woo ON t.kv(v); SET tracing = off query TT -SELECT operation, - regexp_replace(regexp_replace(message, 'create_as_of_time:<[^>]+>', 'create_as_of_time:<...>'), 'mutationJobs:<[^>]*>', 'mutationJobs:<...>') as message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - /* since the transactions involved are multi-range, intent resolving is async and - sometimes there's still intents around */ - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- -batch flow coordinator Put /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:2 > state:DELETE_ONLY direction:ADD mutation_id:1 rollback:false > next_mutation_id:2 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:3 > +batch flow coordinator Put /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 created_at_nanos:... > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:2 created_at_nanos:... > state:DELETE_ONLY direction:ADD mutation_id:1 rollback:false > next_mutation_id:2 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:3 > sql query rows affected: 0 statement ok SET tracing = on,kv,results; INSERT INTO t.kv(k, v) VALUES (1,2); SET tracing = off query TT -SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] - WHERE operation != 'dist sender send' AND operation != 'kv.DistSender: sending partial batch' +$trace_query ---- batch flow coordinator CPut /Table/108/1/1/0 -> /TUPLE/2:2:Int/2 batch flow coordinator InitPut /Table/108/2/2/0 -> /BYTES/0x89 @@ -87,8 +95,7 @@ SET tracing = on,kv,results; INSERT INTO t.kv(k, v) VALUES (1,2); SET tracing = query TT set tracing=off; -SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] - WHERE operation != 'dist sender send' AND operation != 'kv.DistSender: sending partial batch' +$trace_query ---- batch flow coordinator CPut /Table/108/1/1/0 -> /TUPLE/2:2:Int/2 batch flow coordinator InitPut /Table/108/2/2/0 -> /BYTES/0x89 @@ -99,8 +106,7 @@ SET tracing = on,kv,results; INSERT INTO t.kv(k, v) VALUES (2,2); SET tracing = query TT set tracing=off; -SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] - WHERE operation != 'dist sender send' AND operation != 'kv.DistSender: sending partial batch' +$trace_query ---- batch flow coordinator CPut /Table/108/1/2/0 -> /TUPLE/2:2:Int/2 batch flow coordinator InitPut /Table/108/2/2/0 -> /BYTES/0x8a @@ -110,32 +116,17 @@ statement ok SET tracing = on,kv,results; CREATE TABLE t.kv2 AS TABLE t.kv; SET tracing = off query TT -SELECT operation, regexp_replace(regexp_replace(message, 'create_as_of_time:<[^>]+>', 'create_as_of_time:<...>'), '\d\d\d\d\d+', '...PK...') as message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- batch flow coordinator CPut /NamespaceTable/30/1/106/107/"kv2"/4/1 -> 109 -batch flow coordinator CPut /Table/3/1/109/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:ADD offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:2 > +batch flow coordinator CPut /Table/3/1/109/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 created_at_nanos:... > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:ADD offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:2 > sql query rows affected: 0 statement ok SET tracing = on,kv,results; UPDATE t.kv2 SET v = v + 2; SET tracing = off query TT -SELECT operation, message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- colbatchscan Scan /Table/109/{1-2} colbatchscan fetched: /kv2/kv2_pkey/-9222809086901354496/k/v -> /1/2 @@ -147,8 +138,7 @@ statement ok SET tracing = on,kv,results; DELETE FROM t.kv2; SET tracing = off query TT -SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] - WHERE operation != 'dist sender send' +$trace_query ---- batch flow coordinator DelRange /Table/109/1 - /Table/109/2 batch flow coordinator fast path completed @@ -158,27 +148,17 @@ statement ok SET tracing = on,kv,results; DROP TABLE t.kv2; SET tracing = off query TT -SELECT operation, - regexp_replace(regexp_replace(regexp_replace(message, 'create_as_of_time:<[^>]+>', 'create_as_of_time:<...>'), 'drop_job_id:[1-9]\d*', 'drop_job_id:...'), 'drop_time:\d+', 'drop_time:...') as message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- batch flow coordinator Del /NamespaceTable/30/1/106/107/"kv2"/4/1 -batch flow coordinator Put /Table/3/1/109/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:2 > +batch flow coordinator Put /Table/3/1/109/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 created_at_nanos:... > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:2 > sql query rows affected: 0 statement ok SET tracing = on,kv,results; DELETE FROM t.kv; SET tracing = off query TT -SELECT operation, message FROM [SHOW KV TRACE FOR SESSION] - WHERE operation != 'dist sender send' AND operation != 'kv.DistSender: sending partial batch' +$trace_query ---- colbatchscan Scan /Table/108/{1-2} colbatchscan fetched: /kv/kv_pkey/1/v -> /2 @@ -191,38 +171,19 @@ statement ok SET tracing = on,kv,results; DROP INDEX t.kv@woo CASCADE; SET tracing = off query TT -SELECT operation, - regexp_replace(regexp_replace(message, 'create_as_of_time:<[^>]+>', 'create_as_of_time:<...>'), 'mutationJobs:<[^>]*>', 'mutationJobs:<...>') as message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - /* since the transactions involved are multi-range, intent resolving is async and - sometimes there's still intents around */ - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- -batch flow coordinator Put /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:4 > state:DELETE_AND_WRITE_ONLY direction:DROP mutation_id:2 rollback:false > next_mutation_id:3 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:5 > +batch flow coordinator Put /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 created_at_nanos:... > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:4 created_at_nanos:... > state:DELETE_AND_WRITE_ONLY direction:DROP mutation_id:2 rollback:false > next_mutation_id:3 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:5 > sql query rows affected: 0 statement ok SET tracing = on,kv,results; DROP TABLE t.kv; SET tracing = off query TT -SELECT operation, regexp_replace(regexp_replace(regexp_replace(message, 'create_as_of_time:<[^>]+>', 'create_as_of_time:<...>'), 'job_id:[1-9]\d*', 'job_id:...', 'g'), 'drop_time:\d+', 'drop_time:...', 'g') as message - FROM [SHOW KV TRACE FOR SESSION] -WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' - AND operation NOT LIKE 'kv.DistSender: sending partial batch%' -- order of partial batches is not deterministic - AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%' - AND tag NOT LIKE '%intExec=%' - AND tag NOT LIKE '%scExec%' - AND tag NOT LIKE '%IndexBackfiller%' - AND operation != 'dist sender send' +$trace_query ---- batch flow coordinator Del /NamespaceTable/30/1/106/107/"kv"/4/1 -batch flow coordinator Put /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:3 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:5 > +batch flow coordinator Put /Table/3/1/108/2/1 -> table: parent_id:106 unexposed_parent_schema_id:107 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false constraint_id:1 created_at_nanos:... > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:3 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false exclude_data_from_backup:false next_constraint_id:5 > sql query rows affected: 0 # Check that session tracing does not inhibit the fast path for inserts & diff --git a/pkg/sql/schema_change_plan_node.go b/pkg/sql/schema_change_plan_node.go index a1013369113c..828a2956d801 100644 --- a/pkg/sql/schema_change_plan_node.go +++ b/pkg/sql/schema_change_plan_node.go @@ -185,6 +185,7 @@ func newSchemaChangerTxnRunDependencies( scdeps.NewNoopPeriodicProgressFlusher(), execCfg.IndexValidator, scdeps.NewPartitioner(execCfg.Settings, evalContext), + scdeps.NewConstantClock(evalContext.GetTxnTimestamp(time.Microsecond).Time), execCfg.DescMetadaUpdaterFactory, NewSchemaChangerEventLogger(txn, execCfg, 1), schemaChangerJobID, diff --git a/pkg/sql/schemachanger/scbuild/testdata/alter_table b/pkg/sql/schemachanger/scbuild/testdata/alter_table index 3866d4da2c2b..69ea0b4a36fb 100644 --- a/pkg/sql/schemachanger/scbuild/testdata/alter_table +++ b/pkg/sql/schemachanger/scbuild/testdata/alter_table @@ -34,7 +34,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN j INT - [[PrimaryIndex:{DescID: 104, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -45,7 +45,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN j INT - [[PrimaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -89,7 +89,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN j INT DEFAULT 123 - [[PrimaryIndex:{DescID: 104, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -100,7 +100,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN j INT DEFAULT 123 - [[PrimaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -162,7 +162,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN k INT DEFAULT 456; - [[PrimaryIndex:{DescID: 104, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -173,7 +173,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN k INT DEFAULT 456; - [[PrimaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -218,7 +218,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN a INT AS (i+1) STORED - [[PrimaryIndex:{DescID: 104, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -229,7 +229,7 @@ ALTER TABLE defaultdb.foo ADD COLUMN a INT AS (i+1) STORED - [[PrimaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -303,7 +303,7 @@ ALTER TABLE defaultdb.bar ADD COLUMN b INT; - [[PrimaryIndex:{DescID: 104, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -314,7 +314,7 @@ ALTER TABLE defaultdb.bar ADD COLUMN b INT; - [[PrimaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -327,7 +327,7 @@ ALTER TABLE defaultdb.bar ADD COLUMN b INT; - [[PrimaryIndex:{DescID: 105, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 2 @@ -340,7 +340,7 @@ ALTER TABLE defaultdb.bar ADD COLUMN b INT; - [[PrimaryIndex:{DescID: 105, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 2 diff --git a/pkg/sql/schemachanger/scbuild/testdata/create_index b/pkg/sql/schemachanger/scbuild/testdata/create_index index 239c377d2e57..c948593be9e3 100644 --- a/pkg/sql/schemachanger/scbuild/testdata/create_index +++ b/pkg/sql/schemachanger/scbuild/testdata/create_index @@ -13,7 +13,7 @@ CREATE INDEX id1 ON defaultdb.t1(id, name) STORING (money) - [[SecondaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC - ASC keyColumnIds: @@ -38,7 +38,7 @@ CREATE INVERTED INDEX CONCURRENTLY id2 concurrently: true indexId: 2 inverted: true - keyColumnDirection: + keyColumnDirections: - ASC - ASC keyColumnIds: @@ -74,7 +74,7 @@ CREATE INDEX id3 - [[SecondaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC - ASC keyColumnIds: @@ -119,7 +119,7 @@ CREATE INDEX id4 - [[PrimaryIndex:{DescID: 104, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -133,7 +133,7 @@ CREATE INDEX id4 - [[PrimaryIndex:{DescID: 104, IndexID: 2}, PUBLIC], ABSENT] details: indexId: 2 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -148,7 +148,7 @@ CREATE INDEX id4 - [[SecondaryIndex:{DescID: 104, IndexID: 3}, PUBLIC], ABSENT] details: indexId: 3 - keyColumnDirection: + keyColumnDirections: - ASC - ASC keyColumnIds: diff --git a/pkg/sql/schemachanger/scbuild/testdata/drop_database b/pkg/sql/schemachanger/scbuild/testdata/drop_database index c1c3dfada44a..b0a1331d0f9e 100644 --- a/pkg/sql/schemachanger/scbuild/testdata/drop_database +++ b/pkg/sql/schemachanger/scbuild/testdata/drop_database @@ -391,7 +391,7 @@ DROP DATABASE db1 CASCADE - [[PrimaryIndex:{DescID: 109, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 @@ -405,7 +405,7 @@ DROP DATABASE db1 CASCADE - [[PrimaryIndex:{DescID: 110, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 diff --git a/pkg/sql/schemachanger/scbuild/testdata/drop_schema b/pkg/sql/schemachanger/scbuild/testdata/drop_schema index c38ca7bb8fe8..ea121ef7b0b2 100644 --- a/pkg/sql/schemachanger/scbuild/testdata/drop_schema +++ b/pkg/sql/schemachanger/scbuild/testdata/drop_schema @@ -261,7 +261,7 @@ DROP SCHEMA defaultdb.SC1 CASCADE - [[PrimaryIndex:{DescID: 107, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 diff --git a/pkg/sql/schemachanger/scbuild/testdata/drop_table b/pkg/sql/schemachanger/scbuild/testdata/drop_table index cfccdc13173f..3fd61d716ea4 100644 --- a/pkg/sql/schemachanger/scbuild/testdata/drop_table +++ b/pkg/sql/schemachanger/scbuild/testdata/drop_table @@ -317,7 +317,7 @@ DROP TABLE defaultdb.shipments CASCADE; - [[PrimaryIndex:{DescID: 109, IndexID: 1}, ABSENT], PUBLIC] details: indexId: 1 - keyColumnDirection: + keyColumnDirections: - ASC keyColumnIds: - 1 diff --git a/pkg/sql/schemachanger/scdeps/exec_deps.go b/pkg/sql/schemachanger/scdeps/exec_deps.go index d5a9118bc768..067404b9f4b9 100644 --- a/pkg/sql/schemachanger/scdeps/exec_deps.go +++ b/pkg/sql/schemachanger/scdeps/exec_deps.go @@ -56,6 +56,7 @@ func NewExecutorDependencies( backfillFlusher scexec.PeriodicProgressFlusher, indexValidator scexec.IndexValidator, partitioner scmutationexec.Partitioner, + clock scmutationexec.Clock, commentUpdaterFactory scexec.DescriptorMetadataUpdaterFactory, eventLogger scexec.EventLogger, schemaChangerJobID jobspb.JobID, @@ -79,6 +80,7 @@ func NewExecutorDependencies( partitioner: partitioner, user: user, sessionData: sessionData, + clock: clock, } } @@ -314,6 +316,7 @@ func (d *txnDeps) SetResumeSpans( type execDeps struct { txnDeps partitioner scmutationexec.Partitioner + clock scmutationexec.Clock commentUpdaterFactory scexec.DescriptorMetadataUpdaterFactory backfiller scexec.Backfiller backfillTracker scexec.BackfillTracker @@ -323,6 +326,10 @@ type execDeps struct { sessionData *sessiondata.SessionData } +func (d *execDeps) Clock() scmutationexec.Clock { + return d.clock +} + var _ scexec.Dependencies = (*execDeps)(nil) // Catalog implements the scexec.Dependencies interface. @@ -439,3 +446,16 @@ func (n noopPeriodicProgressFlusher) StartPeriodicUpdates( ) (stop func() error) { return func() error { return nil } } + +type constantClock struct { + ts time.Time +} + +// NewConstantClock constructs a new clock for use in execution. +func NewConstantClock(ts time.Time) scmutationexec.Clock { + return constantClock{ts: ts} +} + +func (c constantClock) ApproximateTime() time.Time { + return c.ts +} diff --git a/pkg/sql/schemachanger/scdeps/run_deps.go b/pkg/sql/schemachanger/scdeps/run_deps.go index 47b8eed2fda3..3deb9e297d45 100644 --- a/pkg/sql/schemachanger/scdeps/run_deps.go +++ b/pkg/sql/schemachanger/scdeps/run_deps.go @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" ) // NewJobRunDependencies returns an scrun.JobRunDependencies implementation built from the @@ -119,7 +120,8 @@ func (d *jobExecutionDeps) WithTxnInJob(ctx context.Context, fn scrun.JobTxnFunc periodicProgressFlusher: newPeriodicProgressFlusher(d.settings), statements: d.statements, partitioner: d.partitioner, - user: d.job.Payload().UsernameProto.Decode(), + user: pl.UsernameProto.Decode(), + clock: NewConstantClock(timeutil.FromUnixMicros(pl.StartedMicros)), commentUpdaterFactory: d.commentUpdaterFactory, sessionData: d.sessionData, }) diff --git a/pkg/sql/schemachanger/scdeps/sctestdeps/config.go b/pkg/sql/schemachanger/scdeps/sctestdeps/config.go index 3a03820c052e..9f7e0048a1d1 100644 --- a/pkg/sql/schemachanger/scdeps/sctestdeps/config.go +++ b/pkg/sql/schemachanger/scdeps/sctestdeps/config.go @@ -11,8 +11,11 @@ package sctestdeps import ( + "time" + "github.com/cockroachdb/cockroach/pkg/sql/catalog" "github.com/cockroachdb/cockroach/pkg/sql/catalog/nstree" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" @@ -40,9 +43,19 @@ func WithNamespace(c nstree.Catalog) Option { } // WithDescriptors sets the TestState descriptors to the provided value. +// This function also scrubs any volatile timestamps from the descriptor. func WithDescriptors(c nstree.Catalog) Option { return optionFunc(func(state *TestState) { _ = c.ForEachDescriptorEntry(func(desc catalog.Descriptor) error { + if table, isTable := desc.(catalog.TableDescriptor); isTable { + mut := table.NewBuilder().BuildExistingMutable().(*tabledesc.Mutable) + for _, idx := range mut.AllIndexes() { + if !idx.CreatedAt().IsZero() { + idx.IndexDesc().CreatedAtNanos = defaultOverriddenCreatedAt.UnixNano() + } + } + desc = mut.ImmutableCopy() + } state.catalog.UpsertDescriptorEntry(desc) return nil }) @@ -95,10 +108,22 @@ func WithBackfiller(backfiller scexec.Backfiller) Option { }) } +var ( + // defaultOverriddenCreatedAt is used to populate the CreatedAt timestamp for + // all descriptors injected into the catalog. We inject this to make the + // tests deterministic. + defaultOverriddenCreatedAt = time.Date(2022, time.January, 1, 0, 0, 0, 0, time.UTC) + + // defaultCreatedAt is used to populated the CreatedAt timestamp for all newly + // created indexes. + defaultCreatedAt = defaultOverriddenCreatedAt.Add(time.Hour) +) + var defaultOptions = []Option{ optionFunc(func(state *TestState) { state.backfillTracker = &testBackfillTracker{deps: state} state.backfiller = &testBackfiller{s: state} state.indexSpanSplitter = &indexSpanSplitter{} + state.approximateTimestamp = defaultCreatedAt }), } diff --git a/pkg/sql/schemachanger/scdeps/sctestdeps/test_deps.go b/pkg/sql/schemachanger/scdeps/sctestdeps/test_deps.go index cd9fc7fcfd06..ac904a9552e9 100644 --- a/pkg/sql/schemachanger/scdeps/sctestdeps/test_deps.go +++ b/pkg/sql/schemachanger/scdeps/sctestdeps/test_deps.go @@ -13,6 +13,7 @@ package sctestdeps import ( "context" "sort" + "time" "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/jobs" @@ -455,6 +456,16 @@ func (s *TestState) descBuilderWithSynthetic(id descpb.ID) catalog.DescriptorBui var _ scexec.Dependencies = (*TestState)(nil) +// Clock is part of the scexec.Dependencies interface. +func (s *TestState) Clock() scmutationexec.Clock { + return s +} + +// ApproximateTime is part of the scmutationexec.Clock interface. +func (s *TestState) ApproximateTime() time.Time { + return s.approximateTimestamp +} + // Catalog implements the scexec.Dependencies interface. func (s *TestState) Catalog() scexec.Catalog { return s diff --git a/pkg/sql/schemachanger/scdeps/sctestdeps/test_state.go b/pkg/sql/schemachanger/scdeps/sctestdeps/test_state.go index 58645b5eb93a..66d15258a845 100644 --- a/pkg/sql/schemachanger/scdeps/sctestdeps/test_state.go +++ b/pkg/sql/schemachanger/scdeps/sctestdeps/test_state.go @@ -14,6 +14,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" @@ -50,6 +51,10 @@ type TestState struct { backfiller scexec.Backfiller indexSpanSplitter scexec.IndexSpanSplitter backfillTracker scexec.BackfillTracker + + // approximateTimestamp is used to populate approximate timestamps in + // descriptors. + approximateTimestamp time.Time } // NewTestDependencies returns a TestState populated with the provided options. diff --git a/pkg/sql/schemachanger/scexec/BUILD.bazel b/pkg/sql/schemachanger/scexec/BUILD.bazel index e0daff70f0d1..5d511fc1c1f5 100644 --- a/pkg/sql/schemachanger/scexec/BUILD.bazel +++ b/pkg/sql/schemachanger/scexec/BUILD.bazel @@ -85,6 +85,7 @@ go_test( "//pkg/util/log", "//pkg/util/log/eventpb", "//pkg/util/randutil", + "//pkg/util/timeutil", "@com_github_golang_mock//gomock", "@com_github_stretchr_testify//require", ], diff --git a/pkg/sql/schemachanger/scexec/dependencies.go b/pkg/sql/schemachanger/scexec/dependencies.go index 9dccfc3c1a49..df50c7e0c305 100644 --- a/pkg/sql/schemachanger/scexec/dependencies.go +++ b/pkg/sql/schemachanger/scexec/dependencies.go @@ -31,6 +31,7 @@ import ( type Dependencies interface { Catalog() Catalog Partitioner() scmutationexec.Partitioner + Clock() scmutationexec.Clock TransactionalJobRegistry() TransactionalJobRegistry IndexBackfiller() Backfiller BackfillProgressTracker() BackfillTracker diff --git a/pkg/sql/schemachanger/scexec/exec_mutation.go b/pkg/sql/schemachanger/scexec/exec_mutation.go index bc6ec158f357..8381de829972 100644 --- a/pkg/sql/schemachanger/scexec/exec_mutation.go +++ b/pkg/sql/schemachanger/scexec/exec_mutation.go @@ -34,7 +34,7 @@ import ( func executeDescriptorMutationOps(ctx context.Context, deps Dependencies, ops []scop.Op) error { mvs := newMutationVisitorState(deps.Catalog()) - v := scmutationexec.NewMutationVisitor(deps.Catalog(), mvs, deps.Partitioner()) + v := scmutationexec.NewMutationVisitor(deps.Catalog(), mvs, deps.Partitioner(), deps.Clock()) for _, op := range ops { if err := op.(scop.MutationOp).Visit(ctx, v); err != nil { return err diff --git a/pkg/sql/schemachanger/scexec/executor_external_test.go b/pkg/sql/schemachanger/scexec/executor_external_test.go index 7c879859cc42..cb231d679f45 100644 --- a/pkg/sql/schemachanger/scexec/executor_external_test.go +++ b/pkg/sql/schemachanger/scexec/executor_external_test.go @@ -43,6 +43,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log/eventpb" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/stretchr/testify/require" ) @@ -69,8 +70,9 @@ func (ti testInfra) newExecDeps( noopBackfiller{}, /* backfiller */ scdeps.NewNoOpBackfillTracker(ti.lm.Codec()), scdeps.NewNoopPeriodicProgressFlusher(), - noopIndexValidator{}, /* indexValidator */ - noopPartitioner{}, /* partitioner */ + noopIndexValidator{}, /* indexValidator */ + noopPartitioner{}, /* partitioner */ + scdeps.NewConstantClock(timeutil.Now()), noopMetadataUpdaterFactory{}, /* commentUpdaterFactory*/ noopEventLogger{}, /* eventLogger */ 1, /* schemaChangerJobID */ diff --git a/pkg/sql/schemachanger/scexec/mocks_generated_test.go b/pkg/sql/schemachanger/scexec/mocks_generated_test.go index 0a7e9f0acc46..da5e699d16e0 100644 --- a/pkg/sql/schemachanger/scexec/mocks_generated_test.go +++ b/pkg/sql/schemachanger/scexec/mocks_generated_test.go @@ -173,6 +173,20 @@ func (mr *MockDependenciesMockRecorder) Catalog() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Catalog", reflect.TypeOf((*MockDependencies)(nil).Catalog)) } +// Clock mocks base method. +func (m *MockDependencies) Clock() scmutationexec.Clock { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Clock") + ret0, _ := ret[0].(scmutationexec.Clock) + return ret0 +} + +// Clock indicates an expected call of Clock. +func (mr *MockDependenciesMockRecorder) Clock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clock", reflect.TypeOf((*MockDependencies)(nil).Clock)) +} + // DescriptorMetadataUpdater mocks base method. func (m *MockDependencies) DescriptorMetadataUpdater(arg0 context.Context) scexec.DescriptorMetadataUpdater { m.ctrl.T.Helper() diff --git a/pkg/sql/schemachanger/scexec/scmutationexec/scmutationexec.go b/pkg/sql/schemachanger/scexec/scmutationexec/scmutationexec.go index acaccacb76f1..147600460cec 100644 --- a/pkg/sql/schemachanger/scexec/scmutationexec/scmutationexec.go +++ b/pkg/sql/schemachanger/scexec/scmutationexec/scmutationexec.go @@ -13,6 +13,7 @@ package scmutationexec import ( "context" "sort" + "time" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/keys" @@ -32,6 +33,15 @@ import ( "github.com/cockroachdb/redact" ) +// Clock is used to provide a timestamp to track loosely when something +// happened. It can be used for things like observability and telemetry and +// not for anything involving correctness. +type Clock interface { + + // ApproximateTime provides a present timestamp. + ApproximateTime() time.Time +} + // CatalogReader describes catalog read operations as required by the mutation // visitor. type CatalogReader interface { @@ -115,19 +125,21 @@ type MutationVisitorStateUpdater interface { // NewMutationVisitor creates a new scop.MutationVisitor. func NewMutationVisitor( - cr CatalogReader, s MutationVisitorStateUpdater, p Partitioner, + cr CatalogReader, s MutationVisitorStateUpdater, p Partitioner, clock Clock, ) scop.MutationVisitor { return &visitor{ - cr: cr, - s: s, - p: p, + cr: cr, + s: s, + p: p, + clock: clock, } } type visitor struct { - cr CatalogReader - s MutationVisitorStateUpdater - p Partitioner + clock Clock + cr CatalogReader + s MutationVisitorStateUpdater + p Partitioner } func (m *visitor) RemoveJobReference(ctx context.Context, reference scop.RemoveJobReference) error { diff --git a/pkg/sql/schemachanger/scpb/elements.proto b/pkg/sql/schemachanger/scpb/elements.proto index e51a4a5779b4..7edab78774be 100644 --- a/pkg/sql/schemachanger/scpb/elements.proto +++ b/pkg/sql/schemachanger/scpb/elements.proto @@ -92,9 +92,9 @@ message PrimaryIndex { uint32 index_id = 2 [(gogoproto.customname) = "IndexID", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.IndexID"]; bool unique = 3; repeated uint32 key_column_ids = 4 [(gogoproto.customname) = "KeyColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; - repeated Direction key_column_direction = 5 [(gogoproto.customname) = "KeyColumnDirections"]; + repeated Direction key_column_directions = 5; repeated uint32 key_suffix_column_ids = 6 [(gogoproto.customname) = "KeySuffixColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; - cockroach.sql.catalog.catpb.ShardedDescriptor sharded_descriptor = 7 [(gogoproto.customname) = "ShardedDescriptor"]; + cockroach.sql.catalog.catpb.ShardedDescriptor sharded_descriptor = 7; repeated uint32 storing_column_ids = 8 [(gogoproto.customname) = "StoringColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; repeated uint32 composite_column_ids = 9 [(gogoproto.customname) = "CompositeColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; bool inverted = 10; @@ -116,9 +116,9 @@ message SecondaryIndex { uint32 index_id = 2 [(gogoproto.customname) = "IndexID", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.IndexID"]; bool unique = 3; repeated uint32 key_column_ids = 4 [(gogoproto.customname) = "KeyColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; - repeated Direction key_column_direction = 5 [(gogoproto.customname) = "KeyColumnDirections"]; + repeated Direction key_column_directions = 5; repeated uint32 key_suffix_column_ids = 6 [(gogoproto.customname) = "KeySuffixColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; - cockroach.sql.catalog.catpb.ShardedDescriptor sharded_descriptor = 7 [(gogoproto.customname) = "ShardedDescriptor"]; + cockroach.sql.catalog.catpb.ShardedDescriptor sharded_descriptor = 7; repeated uint32 storing_column_ids = 8 [(gogoproto.customname) = "StoringColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; repeated uint32 composite_column_ids = 9 [(gogoproto.customname) = "CompositeColumnIDs", (gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/sql/sem/catid.ColumnID"]; bool inverted = 10; diff --git a/pkg/sql/schemachanger/testdata/alter_table_add_column b/pkg/sql/schemachanger/testdata/alter_table_add_column index 6ec1ff32ad35..5481e66e5420 100644 --- a/pkg/sql/schemachanger/testdata/alter_table_add_column +++ b/pkg/sql/schemachanger/testdata/alter_table_add_column @@ -166,6 +166,7 @@ upsert descriptor #106 - constraintId: 2 - createdExplicitly: true + constraintId: 1 + + createdAtNanos: "1640995200000000000" encodingType: 1 foreignKey: {} geoConfig: {} @@ -190,6 +191,7 @@ upsert descriptor #106 parentId: 104 primaryIndex: - constraintId: 1 + - createdAtNanos: "1640995200000000000" + constraintId: 2 + createdExplicitly: true encodingType: 1 @@ -245,6 +247,7 @@ upsert descriptor #106 - - direction: DROP - index: - constraintId: 1 + - createdAtNanos: "1640995200000000000" - encodingType: 1 - foreignKey: {} - geoConfig: {}