diff --git a/docs/generated/logformats.md b/docs/generated/logformats.md index 1f07098d5a4a..eb197a4e3087 100644 --- a/docs/generated/logformats.md +++ b/docs/generated/logformats.md @@ -254,24 +254,34 @@ reliably that no counter was present. Example single-line unstructured entry: - I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 started with engine type ‹2› +~~~ +I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 started with engine type ‹2› +~~~ Example multi-line unstructured entry: - I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 node startup completed: - I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 +CockroachDB node starting at 2021-01-16 21:49 (took 0.0s) +~~~ +I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 node startup completed: +I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 +CockroachDB node starting at 2021-01-16 21:49 (took 0.0s) +~~~ Example structured entry: - I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventType":"node_restart"} +~~~ +I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventType":"node_restart"} +~~~ Example long entries broken up into multiple lines: - I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.... - I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 |aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +~~~ +I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.... +I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 |aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +~~~ - I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventTy... - I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 |pe":"node_restart"} +~~~ +I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventTy... +I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 |pe":"node_restart"} +~~~ ### Backward-compatibility notes diff --git a/pkg/ccl/backupccl/backup_planning.go b/pkg/ccl/backupccl/backup_planning.go index a56c99b5c428..233c4a5c2384 100644 --- a/pkg/ccl/backupccl/backup_planning.go +++ b/pkg/ccl/backupccl/backup_planning.go @@ -61,6 +61,7 @@ const ( backupOptEncKMS = "kms" backupOptWithPrivileges = "privileges" backupOptAsJSON = "as_json" + backupOptWithDebugIDs = "debug_ids" localityURLParam = "COCKROACH_LOCALITY" defaultLocalityValue = "default" ) diff --git a/pkg/ccl/backupccl/show.go b/pkg/ccl/backupccl/show.go index eb94e3b3b40e..ef88d1d8788c 100644 --- a/pkg/ccl/backupccl/show.go +++ b/pkg/ccl/backupccl/show.go @@ -92,6 +92,7 @@ func showBackupPlanHook( backupOptEncKMS: sql.KVStringOptRequireValue, backupOptWithPrivileges: sql.KVStringOptRequireNoValue, backupOptAsJSON: sql.KVStringOptRequireNoValue, + backupOptWithDebugIDs: sql.KVStringOptRequireNoValue, } optsFn, err := p.TypeAsStringOpts(ctx, backup.Options, expected) if err != nil { @@ -260,6 +261,20 @@ func backupShowerHeaders(showSchemas bool, opts map[string]string) colinfo.Resul baseHeaders = append(baseHeaders, colinfo.ResultColumn{Name: "privileges", Typ: types.String}) baseHeaders = append(baseHeaders, colinfo.ResultColumn{Name: "owner", Typ: types.String}) } + + if _, shouldShowIDs := opts[backupOptWithDebugIDs]; shouldShowIDs { + baseHeaders = append( + colinfo.ResultColumns{ + baseHeaders[0], + {Name: "database_id", Typ: types.Int}, + baseHeaders[1], + {Name: "parent_schema_id", Typ: types.Int}, + baseHeaders[2], + {Name: "object_id", Typ: types.Int}, + }, + baseHeaders[3:]..., + ) + } return baseHeaders } @@ -325,6 +340,9 @@ func backupShowerDefault( var parentSchemaName string var descriptorType string + var dbID descpb.ID + var parentSchemaID descpb.ID + createStmtDatum := tree.DNull dataSizeDatum := tree.DNull rowCountDatum := tree.DNull @@ -338,14 +356,19 @@ func backupShowerDefault( case catalog.SchemaDescriptor: descriptorType = "schema" dbName = dbIDToName[desc.GetParentID()] + dbID = desc.GetParentID() case catalog.TypeDescriptor: descriptorType = "type" dbName = dbIDToName[desc.GetParentID()] + dbID = desc.GetParentID() parentSchemaName = schemaIDToName[desc.GetParentSchemaID()] + parentSchemaID = desc.GetParentSchemaID() case catalog.TableDescriptor: descriptorType = "table" dbName = dbIDToName[desc.GetParentID()] + dbID = desc.GetParentID() parentSchemaName = schemaIDToName[desc.GetParentSchemaID()] + parentSchemaID = desc.GetParentSchemaID() descSize := descSizes[desc.GetID()] dataSizeDatum = tree.NewDInt(tree.DInt(descSize.DataSize)) rowCountDatum = tree.NewDInt(tree.DInt(descSize.Rows)) @@ -386,6 +409,20 @@ func backupShowerDefault( owner := desc.GetPrivileges().Owner().SQLIdentifier() row = append(row, tree.NewDString(owner)) } + if _, shouldShowIDs := opts[backupOptWithDebugIDs]; shouldShowIDs { + // If showing debug IDs, interleave the IDs with the corresponding object names. + row = append( + tree.Datums{ + row[0], + nullIfZero(dbID), + row[1], + nullIfZero(parentSchemaID), + row[2], + nullIfZero(desc.GetID()), + }, + row[3:]..., + ) + } rows = append(rows, row) } for _, t := range manifest.Tenants { @@ -407,6 +444,20 @@ func backupShowerDefault( if _, shouldShowPrivileges := opts[backupOptWithPrivileges]; shouldShowPrivileges { row = append(row, tree.DNull) } + if _, shouldShowIDs := opts[backupOptWithDebugIDs]; shouldShowIDs { + // If showing debug IDs, interleave the IDs with the corresponding object names. + row = append( + tree.Datums{ + row[0], + tree.DNull, // Database ID + row[1], + tree.DNull, // Parent Schema ID + row[2], + tree.NewDInt(tree.DInt(t.ID)), // Object ID + }, + row[3:]..., + ) + } rows = append(rows, row) } } @@ -422,6 +473,13 @@ func nullIfEmpty(s string) tree.Datum { return tree.NewDString(s) } +func nullIfZero(i descpb.ID) tree.Datum { + if i == 0 { + return tree.DNull + } + return tree.NewDInt(tree.DInt(i)) +} + func showPrivileges(descriptor *descpb.Descriptor) string { var privStringBuilder strings.Builder diff --git a/pkg/ccl/backupccl/show_test.go b/pkg/ccl/backupccl/show_test.go index 1d35d31ecb81..c867482adac9 100644 --- a/pkg/ccl/backupccl/show_test.go +++ b/pkg/ccl/backupccl/show_test.go @@ -479,6 +479,11 @@ func TestShowBackupTenants(t *testing.T) { require.Equal(t, [][]string{ {"/Tenant/10", "/Tenant/11"}, }, res) + + res = systemDB.QueryStr(t, `SELECT database_id, parent_schema_id, object_id FROM [SHOW BACKUP 'nodelocal://1/t10' WITH debug_ids]`) + require.Equal(t, [][]string{ + {"NULL", "NULL", "10"}, + }, res) } func TestShowBackupPrivileges(t *testing.T) { @@ -586,3 +591,63 @@ func showUpgradedForeignKeysTest(exportDir string) func(t *testing.T) { } } } + +func TestShowBackupWithDebugIDs(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + const numAccounts = 11 + // Create test database with bank table + _, _, sqlDB, _, cleanupFn := BackupRestoreTestSetup(t, singleNode, numAccounts, InitManualReplication) + defer cleanupFn() + + // add 1 type, 1 schema, and 2 tables to the database + sqlDB.Exec(t, ` + SET CLUSTER SETTING sql.cross_db_fks.enabled = TRUE; + CREATE TYPE data.welcome AS ENUM ('hello', 'hi'); + USE data; CREATE SCHEMA sc; + CREATE TABLE data.sc.t1 (a INT); + CREATE TABLE data.sc.t2 (a data.welcome); + `) + + const full = LocalFoo + "/full" + + beforeTS := sqlDB.QueryStr(t, `SELECT now()::timestamp::string`)[0][0] + sqlDB.Exec(t, fmt.Sprintf(`BACKUP DATABASE data TO $1 AS OF SYSTEM TIME '%s'`, beforeTS), full) + + // extract the object IDs for the database and public schema + databaseRow := sqlDB.QueryStr(t, ` + SELECT database_name, database_id, parent_schema_name, parent_schema_id, object_name, object_id, object_type + FROM [SHOW BACKUP $1 WITH debug_ids] + WHERE object_name = 'bank'`, full) + require.NotEmpty(t, databaseRow) + dbID, err := strconv.Atoi(databaseRow[0][1]) + require.NoError(t, err) + publicID, err := strconv.Atoi(databaseRow[0][3]) + require.NoError(t, err) + + require.Greater(t, dbID, 0) + require.Greater(t, publicID, 0) + + res := sqlDB.QueryStr(t, ` + SELECT database_name, database_id, parent_schema_name, parent_schema_id, object_name, object_id, object_type + FROM [SHOW BACKUP $1 WITH debug_ids] + ORDER BY object_id`, full) + + dbIDStr := strconv.Itoa(dbID) + publicIDStr := strconv.Itoa(publicID) + schemaIDStr := strconv.Itoa(dbID + 4) + + expectedObjects := [][]string{ + {"NULL", "NULL", "NULL", "NULL", "data", dbIDStr, "database"}, + {"data", dbIDStr, "public", publicIDStr, "bank", strconv.Itoa(dbID + 1), "table"}, + {"data", dbIDStr, "public", publicIDStr, "welcome", strconv.Itoa(dbID + 2), "type"}, + {"data", dbIDStr, "public", publicIDStr, "_welcome", strconv.Itoa(dbID + 3), "type"}, + {"data", dbIDStr, "NULL", "NULL", "sc", schemaIDStr, "schema"}, + {"data", dbIDStr, "sc", schemaIDStr, "t1", strconv.Itoa(dbID + 5), "table"}, + {"data", dbIDStr, "sc", schemaIDStr, "t2", strconv.Itoa(dbID + 6), "table"}, + } + + require.Equal(t, expectedObjects, res) + +} diff --git a/pkg/util/log/format_crdb_v2.go b/pkg/util/log/format_crdb_v2.go index acc39bee9b5b..db49cd0a5ac8 100644 --- a/pkg/util/log/format_crdb_v2.go +++ b/pkg/util/log/format_crdb_v2.go @@ -114,24 +114,34 @@ reliably that no counter was present. Example single-line unstructured entry: - I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 started with engine type ‹2› +~~~ +I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 started with engine type ‹2› +~~~ Example multi-line unstructured entry: - I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 node startup completed: - I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 +CockroachDB node starting at 2021-01-16 21:49 (took 0.0s) +~~~ +I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 node startup completed: +I210116 21:49:17.083093 14 1@cli/start.go:690 ⋮ [-] 40 +CockroachDB node starting at 2021-01-16 21:49 (took 0.0s) +~~~ Example structured entry: - I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventType":"node_restart"} +~~~ +I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventType":"node_restart"} +~~~ Example long entries broken up into multiple lines: - I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.... - I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 |aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +~~~ +I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.... +I210116 21:49:17.073282 14 server/node.go:464 ⋮ [] 23 |aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +~~~ - I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventTy... - I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 |pe":"node_restart"} +~~~ +I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 ={"Timestamp":1610833757080706620,"EventTy... +I210116 21:49:17.080713 14 1@util/log/event_log.go:32 ⋮ [] 32 |pe":"node_restart"} +~~~ ### Backward-compatibility notes