diff --git a/pkg/ccl/backupccl/backup_test.go b/pkg/ccl/backupccl/backup_test.go index d04d156ff763..1c54d3ee8813 100644 --- a/pkg/ccl/backupccl/backup_test.go +++ b/pkg/ccl/backupccl/backup_test.go @@ -6873,9 +6873,6 @@ func TestBackupRestoreInsideTenant(t *testing.T) { defer log.Scope(t).Close(t) makeTenant := func(srv serverutils.TestServerInterface, tenant uint64) (*sqlutils.SQLRunner, func()) { - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() - _, conn := serverutils.StartTenant(t, srv, base.TestTenantArgs{TenantID: roachpb.MakeTenantID(tenant)}) cleanup := func() { conn.Close() } return sqlutils.MakeSQLRunner(conn), cleanup @@ -7031,16 +7028,11 @@ func TestBackupRestoreTenant(t *testing.T) { systemDB.Exec(t, `BACKUP system.users TO 'nodelocal://1/users'`) systemDB.CheckQueryResults(t, `SELECT manifest->>'tenants' FROM [SHOW BACKUP 'nodelocal://1/users' WITH as_json]`, [][]string{{"[]"}}) - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() - _, conn11 := serverutils.StartTenant(t, srv, base.TestTenantArgs{TenantID: roachpb.MakeTenantID(11)}) defer conn11.Close() tenant11 := sqlutils.MakeSQLRunner(conn11) tenant11.Exec(t, `CREATE DATABASE foo; CREATE TABLE foo.baz(i int primary key); INSERT INTO foo.baz VALUES (111), (211)`) - log.TestingClearServerIdentifiers() - _, conn20 := serverutils.StartTenant(t, srv, base.TestTenantArgs{TenantID: roachpb.MakeTenantID(20)}) defer conn20.Close() tenant20 := sqlutils.MakeSQLRunner(conn20) @@ -7100,8 +7092,6 @@ func TestBackupRestoreTenant(t *testing.T) { [][]string{{`100`, `0`, `0`, `0`}}, ) - log.TestingClearServerIdentifiers() - ten10Stopper := stop.NewStopper() _, restoreConn10 := serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{ @@ -7146,8 +7136,6 @@ func TestBackupRestoreTenant(t *testing.T) { [][]string{{`10`, `true`, `{"id": "10", "state": "ACTIVE"}`}}, ) - log.TestingClearServerIdentifiers() - _, restoreConn10 = serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(10), Existing: true}, ) @@ -7187,8 +7175,6 @@ func TestBackupRestoreTenant(t *testing.T) { [][]string{{`10`, `true`, `{"id": "10", "state": "ACTIVE"}`}}, ) - log.TestingClearServerIdentifiers() - _, restoreConn10 := serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(10), Existing: true}, ) @@ -7218,8 +7204,6 @@ func TestBackupRestoreTenant(t *testing.T) { }, ) - log.TestingClearServerIdentifiers() - _, restoreConn10 := serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(10), Existing: true}, ) @@ -7229,8 +7213,6 @@ func TestBackupRestoreTenant(t *testing.T) { restoreTenant10.CheckQueryResults(t, `select * from foo.bar`, tenant10.QueryStr(t, `select * from foo.bar`)) restoreTenant10.CheckQueryResults(t, `select * from foo.bar2`, tenant10.QueryStr(t, `select * from foo.bar2`)) - log.TestingClearServerIdentifiers() - _, restoreConn11 := serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(11), Existing: true}, ) @@ -7249,8 +7231,6 @@ func TestBackupRestoreTenant(t *testing.T) { restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/t10' AS OF SYSTEM TIME `+ts1) - log.TestingClearServerIdentifiers() - _, restoreConn10 := serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(10), Existing: true}, ) @@ -7269,8 +7249,6 @@ func TestBackupRestoreTenant(t *testing.T) { restoreDB.Exec(t, `RESTORE TENANT 20 FROM 'nodelocal://1/t20'`) - log.TestingClearServerIdentifiers() - _, restoreConn20 := serverutils.StartTenant( t, restoreTC.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(20), Existing: true}, ) diff --git a/pkg/ccl/changefeedccl/helpers_test.go b/pkg/ccl/changefeedccl/helpers_test.go index f6bce4d1ee5b..a1c0a911f0f3 100644 --- a/pkg/ccl/changefeedccl/helpers_test.go +++ b/pkg/ccl/changefeedccl/helpers_test.go @@ -410,7 +410,6 @@ func startTestCluster(t testing.TB) (serverutils.TestClusterInterface, *gosql.DB func startTestTenant( t testing.TB, options feedTestOptions, ) (serverutils.TestServerInterface, *gosql.DB, func()) { - log.TestingClearServerIdentifiers() ctx := context.Background() kvServer, _, cleanup := startTestFullServer(t, options) diff --git a/pkg/ccl/importccl/import_stmt_test.go b/pkg/ccl/importccl/import_stmt_test.go index 5a411925e707..fb432171cde8 100644 --- a/pkg/ccl/importccl/import_stmt_test.go +++ b/pkg/ccl/importccl/import_stmt_test.go @@ -6875,9 +6875,6 @@ func TestImportInTenant(t *testing.T) { defer conn10.Close() t10 := sqlutils.MakeSQLRunner(conn10) - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() - // Setup a few tenants, each with a different table. _, conn11 := serverutils.StartTenant(t, tc.Server(0), base.TestTenantArgs{TenantID: roachpb.MakeTenantID(11)}) defer conn11.Close() diff --git a/pkg/ccl/kvccl/kvtenantccl/tenant_upgrade_test.go b/pkg/ccl/kvccl/kvtenantccl/tenant_upgrade_test.go index 7e333084cc7d..ce7e9094a7dd 100644 --- a/pkg/ccl/kvccl/kvtenantccl/tenant_upgrade_test.go +++ b/pkg/ccl/kvccl/kvtenantccl/tenant_upgrade_test.go @@ -26,7 +26,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/leaktest" - "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/stretchr/testify/require" ) @@ -90,8 +89,6 @@ func TestTenantUpgrade(t *testing.T) { TestingKnobs: base.TestingKnobs{}, Settings: settings, } - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() tenant, err := tc.Server(0).StartTenant(ctx, tenantArgs) require.NoError(t, err) return connectToTenant(t, tenant.SQLAddr()) @@ -130,7 +127,6 @@ func TestTenantUpgrade(t *testing.T) { // Restart the tenant and ensure that the version is correct. cleanup() { - log.TestingClearServerIdentifiers() tenantServer, err := tc.Server(0).StartTenant(ctx, base.TestTenantArgs{ TenantID: roachpb.MakeTenantID(initialTenantID), Existing: true, @@ -156,7 +152,6 @@ func TestTenantUpgrade(t *testing.T) { // Restart the new tenant and ensure it has the right version. cleanup() { - log.TestingClearServerIdentifiers() tenantServer, err := tc.Server(0).StartTenant(ctx, base.TestTenantArgs{ TenantID: roachpb.MakeTenantID(postUpgradeTenantID), Existing: true, @@ -286,8 +281,6 @@ func TestTenantUpgradeFailure(t *testing.T) { }, Settings: settings, } - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() return &tenantInfo{tenantArgs: &tenantArgs, v2onMigrationStopper: v2onMigrationStopper} } @@ -334,7 +327,6 @@ func TestTenantUpgradeFailure(t *testing.T) { // Restart the tenant and ensure that the version is correct. cleanup() { - log.TestingClearServerIdentifiers() tca, cleanup := startAndConnectToTenant(t, tenantInfo) defer cleanup() initialTenantRunner = sqlutils.MakeSQLRunner(tca) diff --git a/pkg/ccl/serverccl/statusccl/BUILD.bazel b/pkg/ccl/serverccl/statusccl/BUILD.bazel index ac07d654f687..6dc238131364 100644 --- a/pkg/ccl/serverccl/statusccl/BUILD.bazel +++ b/pkg/ccl/serverccl/statusccl/BUILD.bazel @@ -16,7 +16,6 @@ go_library( "//pkg/testutils/serverutils", "//pkg/testutils/sqlutils", "//pkg/util/httputil", - "//pkg/util/log", "//pkg/util/protoutil", "@com_github_stretchr_testify//require", ], diff --git a/pkg/ccl/serverccl/statusccl/tenant_grpc_test.go b/pkg/ccl/serverccl/statusccl/tenant_grpc_test.go index 8868fdf69648..524ff64c9598 100644 --- a/pkg/ccl/serverccl/statusccl/tenant_grpc_test.go +++ b/pkg/ccl/serverccl/statusccl/tenant_grpc_test.go @@ -91,7 +91,6 @@ func TestTenantGRPCServices(t *testing.T) { sqlRunner.Exec(t, "CREATE TABLE test (id int)") sqlRunner.Exec(t, "INSERT INTO test VALUES (1)") - log.TestingClearServerIdentifiers() tenant2, connTenant2 := serverutils.StartTenant(t, server, base.TestTenantArgs{ TenantID: tenantID, Existing: true, @@ -110,7 +109,6 @@ func TestTenantGRPCServices(t *testing.T) { require.Contains(t, string(body), "INSERT INTO test VALUES") }) - log.TestingClearServerIdentifiers() tenant3, connTenant3 := serverutils.StartTenant(t, server, base.TestTenantArgs{ TenantID: roachpb.MakeTenantID(11), TestingKnobs: testingKnobs, diff --git a/pkg/ccl/serverccl/statusccl/tenant_test_utils.go b/pkg/ccl/serverccl/statusccl/tenant_test_utils.go index 871bde2c215f..71463b44cec8 100644 --- a/pkg/ccl/serverccl/statusccl/tenant_test_utils.go +++ b/pkg/ccl/serverccl/statusccl/tenant_test_utils.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/util/httputil" - "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/stretchr/testify/require" ) @@ -58,7 +57,6 @@ func newTestTenant( tenantParams.Existing = existing tenantParams.TestingKnobs = knobs - log.TestingClearServerIdentifiers() tenant, tenantConn := serverutils.StartTenant(t, server, tenantParams) sqlDB := sqlutils.MakeSQLRunner(tenantConn) status := tenant.StatusServer().(serverpb.SQLStatusServer) diff --git a/pkg/ccl/sqlproxyccl/proxy_handler_test.go b/pkg/ccl/sqlproxyccl/proxy_handler_test.go index e980e16d6ae7..f3af06fa83be 100644 --- a/pkg/ccl/sqlproxyccl/proxy_handler_test.go +++ b/pkg/ccl/sqlproxyccl/proxy_handler_test.go @@ -1160,7 +1160,6 @@ func newDirectoryServer( return nil, status.Error(codes.NotFound, "tenant not found") } - log.TestingClearServerIdentifiers() tenantStopper := tenantdirsvr.NewSubStopper(tdsStopper) ten, err := srv.StartTenant(ctx, base.TestTenantArgs{ Existing: true, diff --git a/pkg/ccl/sqlproxyccl/tenant/directory_test.go b/pkg/ccl/sqlproxyccl/tenant/directory_test.go index 62c49b45e9e9..d480f809e3ea 100644 --- a/pkg/ccl/sqlproxyccl/tenant/directory_test.go +++ b/pkg/ccl/sqlproxyccl/tenant/directory_test.go @@ -486,7 +486,6 @@ func destroyTenant(tc serverutils.TestClusterInterface, id roachpb.TenantID) err func startTenant( ctx context.Context, srv serverutils.TestServerInterface, id uint64, ) (*tenantdirsvr.Process, error) { - log.TestingClearServerIdentifiers() tenantStopper := tenantdirsvr.NewSubStopper(srv.Stopper()) t, err := srv.StartTenant( ctx, diff --git a/pkg/ccl/streamingccl/streamingest/stream_ingestion_job_test.go b/pkg/ccl/streamingccl/streamingest/stream_ingestion_job_test.go index 80c290898aa3..81df7e9710d2 100644 --- a/pkg/ccl/streamingccl/streamingest/stream_ingestion_job_test.go +++ b/pkg/ccl/streamingccl/streamingest/stream_ingestion_job_test.go @@ -69,9 +69,6 @@ SET CLUSTER SETTING changefeed.experimental_poll_interval = '10ms' `) require.NoError(t, err) - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() - // Start the destination server. hDest, cleanupDest := streamingtest.NewReplicationHelper(t, base.TestServerArgs{}) defer cleanupDest() diff --git a/pkg/ccl/testccl/sqlccl/temp_table_clean_test.go b/pkg/ccl/testccl/sqlccl/temp_table_clean_test.go index ca6bcaa777dd..4d1848ff9e7b 100644 --- a/pkg/ccl/testccl/sqlccl/temp_table_clean_test.go +++ b/pkg/ccl/testccl/sqlccl/temp_table_clean_test.go @@ -20,7 +20,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" - "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/syncutil" ) @@ -100,7 +99,6 @@ func TestTenantTempTableCleanup(t *testing.T) { }, }, ) - log.TestingClearServerIdentifiers() tenantStoppers := []*stop.Stopper{stop.NewStopper(), stop.NewStopper()} _, tenantPrimaryDB := serverutils.StartTenant(t, tc.Server(0), base.TestTenantArgs{ @@ -118,8 +116,7 @@ func TestTenantTempTableCleanup(t *testing.T) { tenantSQL.Exec(t, "SET experimental_enable_temp_tables = 'on'") tenantSQL.Exec(t, "set cluster setting sql.temp_object_cleaner.cleanup_interval='1 seconds'") tenantSQL.Exec(t, "CREATE TEMP TABLE temp_table (x INT PRIMARY KEY, y INT);") - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() + _, tenantSecondDB := serverutils.StartTenant(t, tc.Server(1), base.TestTenantArgs{ Existing: true, @@ -127,7 +124,6 @@ func TestTenantTempTableCleanup(t *testing.T) { Settings: settings, Stopper: tenantStoppers[1], }) - log.TestingClearServerIdentifiers() tenantSecondSQL := sqlutils.MakeSQLRunner(tenantSecondDB) tenantSecondSQL.CheckQueryResults(t, "SELECT table_name FROM [SHOW TABLES]", [][]string{ @@ -160,8 +156,6 @@ func TestTenantTempTableCleanup(t *testing.T) { // Enable our hook to allow the database to be // brought up. pause() - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() // Once we restart the tenant, no sessions should exist // so all temporary tables should be cleaned up. tenantStoppers[0] = stop.NewStopper() diff --git a/pkg/cli/democluster/BUILD.bazel b/pkg/cli/democluster/BUILD.bazel index b9decb4d733e..495285b8bd70 100644 --- a/pkg/cli/democluster/BUILD.bazel +++ b/pkg/cli/democluster/BUILD.bazel @@ -30,7 +30,6 @@ go_library( "//pkg/server/serverpb", "//pkg/server/status", "//pkg/settings", - "//pkg/sql", "//pkg/sql/catalog/catalogkeys", "//pkg/sql/catalog/catconstants", "//pkg/sql/distsql", diff --git a/pkg/cli/democluster/demo_cluster.go b/pkg/cli/democluster/demo_cluster.go index 3cc726548f78..a897d0229e32 100644 --- a/pkg/cli/democluster/demo_cluster.go +++ b/pkg/cli/democluster/demo_cluster.go @@ -37,7 +37,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/server/status" "github.com/cockroachdb/cockroach/pkg/settings" - "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catconstants" "github.com/cockroachdb/cockroach/pkg/sql/distsql" @@ -407,7 +406,6 @@ func (c *transientCluster) Start( StartingHTTPPort: c.demoCtx.HTTPPort - 2, Locality: c.demoCtx.Localities[i], TestingKnobs: base.TestingKnobs{ - TenantTestingKnobs: &sql.TenantTestingKnobs{DisableLogTags: true}, Server: &server.TestingKnobs{ ContextTestingKnobs: rpc.ContextTestingKnobs{ ArtificialLatencyMap: latencyMap, diff --git a/pkg/cli/interactive_tests/test_log_config_msg.tcl b/pkg/cli/interactive_tests/test_log_config_msg.tcl index 8749f6da974c..dd81593f8734 100644 --- a/pkg/cli/interactive_tests/test_log_config_msg.tcl +++ b/pkg/cli/interactive_tests/test_log_config_msg.tcl @@ -9,30 +9,23 @@ start_server $argv start_test "Check that the cluster and node ID is reported at the start of the first log file." spawn tail -n 1000 -F logs/db/logs/cockroach.log -eexpect "\\\[config\\\] * clusterID:" -eexpect "\\\[config\\\] * nodeID:" eexpect "node startup completed" +eexpect "start.go*clusterID:" +eexpect "start.go*nodeID:" end_test stop_server $argv -# Make a server with a tiny log buffer so as to force frequent log rotation. -system "$argv start-single-node --insecure --pid-file=server_pid --background -s=path=logs/db --log-file-max-size=2k >>logs/expect-cmd.log 2>&1; - $argv sql --insecure -e 'select 1'" -# Stop the server, which also flushes and closes the log files. +# Restart the server, to check that the server identifiers are also reported after restarts. +start_server $argv stop_server $argv start_test "Check that the cluster and node ID is reported at the start of new log files." # Verify that the string "restarted pre-existing node" can be found -# somewhere. This ensures that if this string ever changes, the test -# below won't report a false negative. +# somewhere. system "grep -q 'restarted pre-existing node' logs/db/logs/*.log" -# Verify that "cockroach.log" is not the file where the server reports -# it just started. -system "if grep -q 'restarted pre-existing node' logs/db/logs/cockroach.log; then false; fi" # Verify that the last log file does contain the cluster ID. -system "grep -qF '\[config\] clusterID:' logs/db/logs/cockroach.log" -system "grep -qF '\[config\] nodeID:' logs/db/logs/cockroach.log" +system "grep -q 'start\.go.*clusterID:' logs/db/logs/cockroach.log" +system "grep -q 'start\.go.*nodeID:' logs/db/logs/cockroach.log" end_test - diff --git a/pkg/cli/start.go b/pkg/cli/start.go index 231d7b28a35a..beb4103bebf4 100644 --- a/pkg/cli/start.go +++ b/pkg/cli/start.go @@ -682,12 +682,6 @@ If problems persist, please see %s.` return err } - // Remember the server identifiers for logging. - // TODO(knz): Remove this. - nodeID := s.NodeID() - clusterID := s.ClusterID().String() - log.SetNodeIDs(clusterID, int32(nodeID)) - // Now inform the user that the server is running and tell the // user about its run-time derived parameters. return reportServerInfo(ctx, tBegin, &serverCfg, s.ClusterSettings(), true /* isHostNode */, initialStart) diff --git a/pkg/cmd/roachtest/tests/multitenant.go b/pkg/cmd/roachtest/tests/multitenant.go index b14c1abae5a8..3b06b7f8701f 100644 --- a/pkg/cmd/roachtest/tests/multitenant.go +++ b/pkg/cmd/roachtest/tests/multitenant.go @@ -58,17 +58,17 @@ func runAcceptanceMultitenant(ctx context.Context, t test.Test, c cluster.Cluste t.Status("checking log file contents") // Check that the server identifiers are present in the tenant log file. - logFile := filepath.Join(tenant.logDir(), "cockroach.log") + logFile := filepath.Join(tenant.logDir(), "*.log") if err := c.RunE(ctx, c.Node(1), - "grep", "-q", "'\\[config\\] .* clusterID:'", logFile); err != nil { + "grep", "-q", "'start\\.go.*clusterID:'", logFile); err != nil { t.Fatal(errors.Wrap(err, "cluster ID not found in log file")) } if err := c.RunE(ctx, c.Node(1), - "grep", "-q", "'\\[config\\] .* tenantID:'", logFile); err != nil { + "grep", "-q", "'start\\.go.*tenantID:'", logFile); err != nil { t.Fatal(errors.Wrap(err, "tenant ID not found in log file")) } if err := c.RunE(ctx, c.Node(1), - "grep", "-q", "'\\[config\\] .* instanceID:'", logFile); err != nil { + "grep", "-q", "'start\\.go.*instanceID:'", logFile); err != nil { t.Fatal(errors.Wrap(err, "SQL instance ID not found in log file")) } } diff --git a/pkg/server/config.go b/pkg/server/config.go index b16ccb983756..df43f258cae8 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -16,7 +16,9 @@ import ( "fmt" "net" "runtime" + "strconv" "strings" + "sync/atomic" "text/tabwriter" "time" @@ -38,6 +40,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/netutil" "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/cockroachdb/cockroach/pkg/util/tracing" + "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble" "github.com/cockroachdb/pebble/vfs" @@ -117,6 +120,10 @@ type BaseConfig struct { Tracer *tracing.Tracer + // idProvider is an interface that makes the logging package + // able to peek into the server IDs defined by this configuration. + idProvider *idProvider + // IDContainer is the Node ID / SQL Instance ID container // that will contain the ID for the server to instantiate. IDContainer *base.NodeIDContainer @@ -177,11 +184,17 @@ func MakeBaseConfig(st *cluster.Settings, tr *tracing.Tracer) BaseConfig { if tr == nil { panic("nil Tracer") } + idsProvider := &idProvider{ + clusterID: &base.ClusterIDContainer{}, + serverID: &base.NodeIDContainer{}, + } + baseCfg := BaseConfig{ Tracer: tr, - IDContainer: &base.NodeIDContainer{}, - ClusterIDContainer: &base.ClusterIDContainer{}, - AmbientCtx: log.AmbientContext{Tracer: tr}, + idProvider: idsProvider, + IDContainer: idsProvider.serverID, + ClusterIDContainer: idsProvider.clusterID, + AmbientCtx: log.AmbientContext{Tracer: tr, ServerIDs: idsProvider}, Config: new(base.Config), Settings: st, MaxOffset: MaxOffsetType(base.DefaultMaxClockOffset), @@ -760,3 +773,109 @@ func parseAttributes(attrsStr string) roachpb.Attributes { } return roachpb.Attributes{Attrs: filtered} } + +// idProvider connects the server ID containers in this +// package to the logging package. +// +// For each of the "main" data items, it also memoizes its +// representation as a string (the one needed by the +// log.ServerIdentificationPayload interface) as soon as the value is +// initialized. This saves on conversion costs. +type idProvider struct { + // clusterID contains the cluster ID (initialized late). + clusterID *base.ClusterIDContainer + // clusterStr is the memoized representation of clusterID, once known. + clusterStr atomic.Value + + // tenantID is the tenant ID for this server. + tenantID roachpb.TenantID + // tenantStr is the memoized representation of tenantID. + tenantStr atomic.Value + + // serverID contains the node ID for KV nodes (when tenantID.IsSet() == + // false), or the SQL instance ID for SQL-only servers (when + // tenantID.IsSet() == true). + serverID *base.NodeIDContainer + // serverStr is the memoized representation of serverID. + serverStr atomic.Value +} + +var _ log.ServerIdentificationPayload = (*idProvider)(nil) + +// ServerIdentityString implements the log.ServerIdentificationPayload interface. +func (s *idProvider) ServerIdentityString(key log.ServerIdentificationKey) string { + switch key { + case log.IdentifyClusterID: + c := s.clusterStr.Load() + cs, ok := c.(string) + if !ok { + cid := s.clusterID.Get() + if cid != uuid.Nil { + cs = cid.String() + s.clusterStr.Store(cs) + } + } + return cs + + case log.IdentifyTenantID: + t := s.tenantStr.Load() + ts, ok := t.(string) + if !ok { + tid := s.tenantID + if tid.IsSet() { + ts = strconv.FormatUint(tid.ToUint64(), 10) + s.tenantStr.Store(ts) + } + } + return ts + + case log.IdentifyInstanceID: + // If tenantID is not set, this is a KV node and it has no SQL + // instance ID. + if !s.tenantID.IsSet() { + return "" + } + return s.maybeMemoizeServerID() + + case log.IdentifyKVNodeID: + // If tenantID is set, this is a SQL-only server and it has no + // node ID. + if s.tenantID.IsSet() { + return "" + } + return s.maybeMemoizeServerID() + } + + return "" +} + +// SetTenant informs the provider that it provides data for +// a SQL server. +// +// Note: this should not be called concurrently with logging which may +// invoke the method from the log.ServerIdentificationPayload +// interface. +func (s *idProvider) SetTenant(tenantID roachpb.TenantID) { + if !tenantID.IsSet() { + panic("programming error: invalid tenant ID") + } + if s.tenantID.IsSet() { + panic("programming error: provider already set for tenant server") + } + s.tenantID = tenantID +} + +// maybeMemoizeServerID saves the representation of serverID to +// serverStr if the former is initialized. +func (s *idProvider) maybeMemoizeServerID() string { + si := s.serverStr.Load() + sis, ok := si.(string) + if !ok { + sid := s.serverID.Get() + if sid != 0 { + sis = strconv.FormatUint(uint64(sid), 10) + s.serverStr.Store(sis) + } + } + return sis +} diff --git a/pkg/server/tenant.go b/pkg/server/tenant.go index bcac5943ac39..e234c8902830 100644 --- a/pkg/server/tenant.go +++ b/pkg/server/tenant.go @@ -65,6 +65,10 @@ func StartTenant( return nil, "", "", err } + // Inform the server identity provider that we're operating + // for a tenant server. + baseCfg.idProvider.SetTenant(sqlCfg.TenantID) + args, err := makeTenantSQLServerArgs(ctx, stopper, kvClusterName, baseCfg, sqlCfg) if err != nil { return nil, "", "", err @@ -262,17 +266,6 @@ func StartTenant( return nil, "", "", err } - if knobs, ok := baseCfg.TestingKnobs.TenantTestingKnobs.(*sql.TenantTestingKnobs); !ok || !knobs.DisableLogTags { - // Register the server's identifiers so that log events are - // decorated with the server's identity. This helps when gathering - // log events from multiple servers into the same log collector. - // - // We do this only here, as the identifiers may not be known before this point. - clusterID := args.rpcContext.ClusterID.Get().String() - log.SetNodeIDs(clusterID, 0 /* nodeID is not known for a SQL-only server. */) - log.SetTenantIDs(args.TenantID.String(), int32(s.SQLInstanceID())) - } - externalUsageFn := func(ctx context.Context) multitenant.ExternalUsage { userTimeMillis, _, err := status.GetCPUTime(ctx) if err != nil { diff --git a/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/BUILD.bazel b/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/BUILD.bazel index f0ee625179be..cdfacdf2d062 100644 --- a/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/BUILD.bazel +++ b/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/BUILD.bazel @@ -21,7 +21,6 @@ go_library( "//pkg/testutils/serverutils", "//pkg/testutils/sqlutils", "//pkg/testutils/testcluster", - "//pkg/util/log", "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/cluster.go b/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/cluster.go index 6e06f97b885b..478fb77f9de4 100644 --- a/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/cluster.go +++ b/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster/cluster.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/security" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" - "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/stretchr/testify/require" ) @@ -46,10 +45,6 @@ func NewHandle(t *testing.T, tc *testcluster.TestCluster) *Handle { // InitializeTenant initializes a tenant with the given ID, returning the // relevant tenant state. func (h *Handle) InitializeTenant(ctx context.Context, tenID roachpb.TenantID) *Tenant { - // Prevent a logging assertion that the server ID is initialized multiple - // times. - log.TestingClearServerIdentifiers() - testServer := h.tc.Server(0) tenantState := &Tenant{t: h.t} if tenID == roachpb.SystemTenantID { diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go index b7bbd8843c46..cee543922d41 100644 --- a/pkg/sql/exec_util.go +++ b/pkg/sql/exec_util.go @@ -1347,10 +1347,6 @@ type TenantTestingKnobs struct { // OverrideTokenBucketProvider allows a test-only TokenBucketProvider (which // can optionally forward requests to the real provider). OverrideTokenBucketProvider func(origProvider kvtenant.TokenBucketProvider) kvtenant.TokenBucketProvider - - // DisableLogTags can be set to true to cause the tenant server to avoid - // setting any global log tags for cluster id or node id. - DisableLogTags bool } var _ base.ModuleTestingKnobs = &TenantTestingKnobs{} diff --git a/pkg/sql/logictest/logic.go b/pkg/sql/logictest/logic.go index fe2403516394..11ae086c8434 100644 --- a/pkg/sql/logictest/logic.go +++ b/pkg/sql/logictest/logic.go @@ -1547,9 +1547,6 @@ func (t *logicTest) newCluster(serverArgs TestServerArgs, opts []clusterOpt) { TracingDefault: params.ServerArgs.TracingDefault, } - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() - tenant, err := t.cluster.Server(i).StartTenant(context.Background(), tenantArgs) if err != nil { t.rootT.Fatalf("%+v", err) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain b/pkg/sql/opt/exec/execbuilder/testdata/explain index 5b9945cfccbb..7c6213e3467c 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain @@ -1618,7 +1618,7 @@ regions: query T EXPLAIN (OPT, MEMO) SELECT * FROM tc JOIN t ON k=a ---- -memo (optimized, ~10KB, required=[presentation: info:10]) +memo (optimized, ~11KB, required=[presentation: info:10]) ├── G1: (explain G2 [presentation: a:1,b:2,k:6,v:7]) │ └── [presentation: info:10] │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7]" [presentation: a:1,b:2,k:6,v:7]) @@ -1677,7 +1677,7 @@ TABLE t ├── tableoid oid [hidden] [system] └── PRIMARY INDEX t_pkey └── k int not null -memo (optimized, ~10KB, required=[presentation: info:10]) +memo (optimized, ~11KB, required=[presentation: info:10]) ├── G1: (explain G2 [presentation: a:1,b:2,k:6,v:7]) │ └── [presentation: info:10] │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7]" [presentation: a:1,b:2,k:6,v:7]) diff --git a/pkg/sql/opt/memo/testdata/memo b/pkg/sql/opt/memo/testdata/memo index 017b9250f0df..9e0110bbe3a9 100644 --- a/pkg/sql/opt/memo/testdata/memo +++ b/pkg/sql/opt/memo/testdata/memo @@ -128,7 +128,7 @@ WHERE a.y>1 AND a.x::string=b.x ORDER BY y LIMIT 10 ---- -memo (optimized, ~20KB, required=[presentation: y:2,x:5,c:10] [ordering: +2]) +memo (optimized, ~23KB, required=[presentation: y:2,x:5,c:10] [ordering: +2]) ├── G1: (project G2 G3 y x) │ ├── [presentation: y:2,x:5,c:10] [ordering: +2] │ │ ├── best: (project G2="[ordering: +2]" G3 y x) @@ -203,7 +203,7 @@ SELECT 1 AS a, 1+z AS b, left(x, 10)::TIMESTAMP AS c, left(x, 10)::TIMESTAMPTZ A FROM b WHERE z=1 AND concat(x, 'foo', x)=concat(x, 'foo', x) ---- -memo (optimized, ~6KB, required=[presentation: a:5,b:6,c:7,d:8]) +memo (optimized, ~7KB, required=[presentation: a:5,b:6,c:7,d:8]) ├── G1: (project G2 G3) │ └── [presentation: a:5,b:6,c:7,d:8] │ ├── best: (project G2 G3) @@ -238,7 +238,7 @@ memo (optimized, ~6KB, required=[presentation: a:5,b:6,c:7,d:8]) memo SELECT x FROM a WHERE x = 1 AND x+y = 1 ---- -memo (optimized, ~7KB, required=[presentation: x:1]) +memo (optimized, ~8KB, required=[presentation: x:1]) ├── G1: (project G2 G3 x) │ └── [presentation: x:1] │ ├── best: (project G2 G3 x) @@ -268,7 +268,7 @@ memo (optimized, ~7KB, required=[presentation: x:1]) memo SELECT x, y FROM a UNION SELECT x+1, y+1 FROM a ---- -memo (optimized, ~7KB, required=[presentation: x:11,y:12]) +memo (optimized, ~8KB, required=[presentation: x:11,y:12]) ├── G1: (union G2 G3) (union G2 G3 ordering=+11,+12) │ └── [presentation: x:11,y:12] │ ├── best: (union G2 G3) @@ -301,7 +301,7 @@ memo (optimized, ~7KB, required=[presentation: x:11,y:12]) memo SELECT array_agg(x) FROM (SELECT * FROM a) ---- -memo (optimized, ~4KB, required=[presentation: array_agg:5]) +memo (optimized, ~5KB, required=[presentation: array_agg:5]) ├── G1: (scalar-group-by G2 G3 cols=()) │ └── [presentation: array_agg:5] │ ├── best: (scalar-group-by G2 G3 cols=()) @@ -317,7 +317,7 @@ memo (optimized, ~4KB, required=[presentation: array_agg:5]) memo SELECT array_agg(x) FROM (SELECT * FROM a) GROUP BY y ---- -memo (optimized, ~4KB, required=[presentation: array_agg:5]) +memo (optimized, ~5KB, required=[presentation: array_agg:5]) ├── G1: (project G2 G3 array_agg) │ └── [presentation: array_agg:5] │ ├── best: (project G2 G3 array_agg) @@ -338,7 +338,7 @@ memo (optimized, ~4KB, required=[presentation: array_agg:5]) memo SELECT array_agg(x) FROM (SELECT * FROM a ORDER BY y) ---- -memo (optimized, ~3KB, required=[presentation: array_agg:5]) +memo (optimized, ~4KB, required=[presentation: array_agg:5]) ├── G1: (scalar-group-by G2 G3 cols=(),ordering=+2) │ └── [presentation: array_agg:5] │ ├── best: (scalar-group-by G2="[ordering: +2]" G3 cols=(),ordering=+2) @@ -357,7 +357,7 @@ memo (optimized, ~3KB, required=[presentation: array_agg:5]) memo SELECT DISTINCT info FROM [EXPLAIN SELECT 123 AS k] ---- -memo (optimized, ~7KB, required=[presentation: info:3]) +memo (optimized, ~9KB, required=[presentation: info:3]) ├── G1: (distinct-on G2 G3 cols=(3)) │ └── [presentation: info:3] │ ├── best: (distinct-on G2 G3 cols=(3)) @@ -385,7 +385,7 @@ memo (optimized, ~7KB, required=[presentation: info:3]) memo SELECT DISTINCT tag FROM [SHOW TRACE FOR SESSION] ---- -memo (optimized, ~6KB, required=[presentation: tag:11]) +memo (optimized, ~7KB, required=[presentation: tag:11]) ├── G1: (distinct-on G2 G3 cols=(11)) │ └── [presentation: tag:11] │ ├── best: (distinct-on G2 G3 cols=(11)) diff --git a/pkg/sql/opt/memo/testdata/stats/inverted-geo b/pkg/sql/opt/memo/testdata/stats/inverted-geo index a7aeeb0a3142..07b70b0d7ebc 100644 --- a/pkg/sql/opt/memo/testdata/stats/inverted-geo +++ b/pkg/sql/opt/memo/testdata/stats/inverted-geo @@ -91,7 +91,7 @@ project memo SELECT i FROM t WHERE st_intersects('LINESTRING(0.5 0.5, 0.7 0.7)', g) ORDER BY i LIMIT 1 ---- -memo (optimized, ~11KB, required=[presentation: i:1]) +memo (optimized, ~13KB, required=[presentation: i:1]) ├── G1: (project G2 G3 i) │ └── [presentation: i:1] │ ├── best: (project G2 G3 i) @@ -195,7 +195,7 @@ project memo SELECT i FROM t WHERE st_intersects('LINESTRING(100 100, 150 150)', g) ORDER BY i LIMIT 1 ---- -memo (optimized, ~11KB, required=[presentation: i:1]) +memo (optimized, ~13KB, required=[presentation: i:1]) ├── G1: (project G2 G3 i) │ └── [presentation: i:1] │ ├── best: (project G2 G3 i) @@ -303,7 +303,7 @@ select memo SELECT * FROM t WHERE g IS NULL OR st_intersects('LINESTRING(100 100, 150 150)', g) ---- -memo (optimized, ~4KB, required=[presentation: i:1,g:2]) +memo (optimized, ~5KB, required=[presentation: i:1,g:2]) ├── G1: (select G2 G3) │ └── [presentation: i:1,g:2] │ ├── best: (select G2 G3) diff --git a/pkg/sql/opt/xform/testdata/physprops/ordering b/pkg/sql/opt/xform/testdata/physprops/ordering index 0bafb91b8f83..3bd75ecf1e64 100644 --- a/pkg/sql/opt/xform/testdata/physprops/ordering +++ b/pkg/sql/opt/xform/testdata/physprops/ordering @@ -302,7 +302,7 @@ project memo SELECT y, x-1 AS z FROM a WHERE x>y ORDER BY x, y DESC ---- -memo (optimized, ~5KB, required=[presentation: y:2,z:7] [ordering: +1,-2]) +memo (optimized, ~6KB, required=[presentation: y:2,z:7] [ordering: +1,-2]) ├── G1: (project G2 G3 x y) │ ├── [presentation: y:2,z:7] [ordering: +1,-2] │ │ ├── best: (project G2="[ordering: +1,-2]" G3 x y) @@ -355,7 +355,7 @@ sort memo SELECT y, z FROM a WHERE x>y ORDER BY y ---- -memo (optimized, ~5KB, required=[presentation: y:2,z:3] [ordering: +2]) +memo (optimized, ~6KB, required=[presentation: y:2,z:3] [ordering: +2]) ├── G1: (project G2 G3 y z) │ ├── [presentation: y:2,z:3] [ordering: +2] │ │ ├── best: (sort G1) @@ -686,7 +686,7 @@ memo (optimized, ~3KB, required=[presentation: info:7]) memo SELECT y FROM a WITH ORDINALITY ORDER BY ordinality ---- -memo (optimized, ~4KB, required=[presentation: y:2] [ordering: +7]) +memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +7]) ├── G1: (ordinality G2) │ ├── [presentation: y:2] [ordering: +7] │ │ ├── best: (ordinality G2) @@ -702,7 +702,7 @@ memo (optimized, ~4KB, required=[presentation: y:2] [ordering: +7]) memo SELECT y FROM a WITH ORDINALITY ORDER BY -ordinality ---- -memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +8]) +memo (optimized, ~6KB, required=[presentation: y:2] [ordering: +8]) ├── G1: (project G2 G3 y) │ ├── [presentation: y:2] [ordering: +8] │ │ ├── best: (sort G1) @@ -725,7 +725,7 @@ memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +8]) memo SELECT y FROM a WITH ORDINALITY ORDER BY ordinality, x ---- -memo (optimized, ~6KB, required=[presentation: y:2] [ordering: +7]) +memo (optimized, ~7KB, required=[presentation: y:2] [ordering: +7]) ├── G1: (ordinality G2) │ ├── [presentation: y:2] [ordering: +7] │ │ ├── best: (ordinality G2) @@ -741,7 +741,7 @@ memo (optimized, ~6KB, required=[presentation: y:2] [ordering: +7]) memo SELECT y FROM (SELECT * FROM a ORDER BY y) WITH ORDINALITY ORDER BY y, ordinality ---- -memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +2,+7]) +memo (optimized, ~6KB, required=[presentation: y:2] [ordering: +2,+7]) ├── G1: (ordinality G2 ordering=+2) │ ├── [presentation: y:2] [ordering: +2,+7] │ │ ├── best: (ordinality G2="[ordering: +2]" ordering=+2) @@ -760,7 +760,7 @@ memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +2,+7]) memo SELECT y FROM (SELECT * FROM a ORDER BY y) WITH ORDINALITY ORDER BY ordinality, y ---- -memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +7]) +memo (optimized, ~6KB, required=[presentation: y:2] [ordering: +7]) ├── G1: (ordinality G2 ordering=+2) │ ├── [presentation: y:2] [ordering: +7] │ │ ├── best: (ordinality G2="[ordering: +2]" ordering=+2) @@ -779,7 +779,7 @@ memo (optimized, ~5KB, required=[presentation: y:2] [ordering: +7]) memo SELECT y FROM a WITH ORDINALITY ORDER BY ordinality DESC ---- -memo (optimized, ~4KB, required=[presentation: y:2] [ordering: -7]) +memo (optimized, ~5KB, required=[presentation: y:2] [ordering: -7]) ├── G1: (ordinality G2) │ ├── [presentation: y:2] [ordering: -7] │ │ ├── best: (sort G1) diff --git a/pkg/sql/opt/xform/testdata/rules/groupby b/pkg/sql/opt/xform/testdata/rules/groupby index 37b92ce5b521..a3b59a4c4088 100644 --- a/pkg/sql/opt/xform/testdata/rules/groupby +++ b/pkg/sql/opt/xform/testdata/rules/groupby @@ -449,7 +449,7 @@ project memo SELECT min(a) FROM abc ---- -memo (optimized, ~5KB, required=[presentation: min:7]) +memo (optimized, ~6KB, required=[presentation: min:7]) ├── G1: (scalar-group-by G2 G3 cols=()) (scalar-group-by G4 G5 cols=()) │ └── [presentation: min:7] │ ├── best: (scalar-group-by G4 G5 cols=()) @@ -475,7 +475,7 @@ memo (optimized, ~5KB, required=[presentation: min:7]) memo SELECT min(b) FROM abc ---- -memo (optimized, ~6KB, required=[presentation: min:7]) +memo (optimized, ~7KB, required=[presentation: min:7]) ├── G1: (scalar-group-by G2 G3 cols=()) (scalar-group-by G4 G5 cols=()) │ └── [presentation: min:7] │ ├── best: (scalar-group-by G2 G3 cols=()) @@ -511,7 +511,7 @@ memo (optimized, ~6KB, required=[presentation: min:7]) memo SELECT max(a) FROM abc ---- -memo (optimized, ~5KB, required=[presentation: max:7]) +memo (optimized, ~6KB, required=[presentation: max:7]) ├── G1: (scalar-group-by G2 G3 cols=()) (scalar-group-by G4 G5 cols=()) │ └── [presentation: max:7] │ ├── best: (scalar-group-by G4 G5 cols=()) @@ -537,7 +537,7 @@ memo (optimized, ~5KB, required=[presentation: max:7]) memo SELECT max(b) FROM abc ---- -memo (optimized, ~6KB, required=[presentation: max:7]) +memo (optimized, ~7KB, required=[presentation: max:7]) ├── G1: (scalar-group-by G2 G3 cols=()) (scalar-group-by G4 G5 cols=()) │ └── [presentation: max:7] │ ├── best: (scalar-group-by G2 G3 cols=()) @@ -650,7 +650,7 @@ scalar-group-by memo SELECT max(b) FROM abc ---- -memo (optimized, ~6KB, required=[presentation: max:7]) +memo (optimized, ~7KB, required=[presentation: max:7]) ├── G1: (scalar-group-by G2 G3 cols=()) (scalar-group-by G4 G5 cols=()) │ └── [presentation: max:7] │ ├── best: (scalar-group-by G2 G3 cols=()) @@ -1575,7 +1575,7 @@ scalar-group-by memo SELECT array_agg(w) FROM (SELECT * FROM kuvw ORDER BY w) GROUP BY u,v ---- -memo (optimized, ~8KB, required=[presentation: array_agg:7]) +memo (optimized, ~9KB, required=[presentation: array_agg:7]) ├── G1: (project G2 G3 array_agg) │ └── [presentation: array_agg:7] │ ├── best: (project G2 G3 array_agg) @@ -1609,7 +1609,7 @@ memo (optimized, ~8KB, required=[presentation: array_agg:7]) memo SELECT sum(w) FROM kuvw GROUP BY u,v,w ---- -memo (optimized, ~6KB, required=[presentation: sum:7]) +memo (optimized, ~7KB, required=[presentation: sum:7]) ├── G1: (project G2 G3 sum) │ └── [presentation: sum:7] │ ├── best: (project G2 G3 sum) @@ -1640,7 +1640,7 @@ memo (optimized, ~6KB, required=[presentation: sum:7]) memo SELECT sum(w) FROM kuvw GROUP BY v ---- -memo (optimized, ~6KB, required=[presentation: sum:7]) +memo (optimized, ~7KB, required=[presentation: sum:7]) ├── G1: (project G2 G3 sum) │ └── [presentation: sum:7] │ ├── best: (project G2 G3 sum) @@ -1665,7 +1665,7 @@ memo (optimized, ~6KB, required=[presentation: sum:7]) memo SELECT array_agg(w) FROM (SELECT * FROM kuvw ORDER BY u,w) GROUP BY v ---- -memo (optimized, ~7KB, required=[presentation: array_agg:7]) +memo (optimized, ~8KB, required=[presentation: array_agg:7]) ├── G1: (project G2 G3 array_agg) │ └── [presentation: array_agg:7] │ ├── best: (project G2 G3 array_agg) @@ -1693,7 +1693,7 @@ memo (optimized, ~7KB, required=[presentation: array_agg:7]) memo SELECT array_agg(k) FROM (SELECT * FROM kuvw WHERE u=v ORDER BY u) GROUP BY w ---- -memo (optimized, ~12KB, required=[presentation: array_agg:7]) +memo (optimized, ~14KB, required=[presentation: array_agg:7]) ├── G1: (project G2 G3 array_agg) │ └── [presentation: array_agg:7] │ ├── best: (project G2 G3 array_agg) @@ -1766,7 +1766,7 @@ memo (optimized, ~12KB, required=[presentation: array_agg:7]) memo SELECT sum(k) FROM (SELECT * FROM kuvw WHERE u=v) GROUP BY u,w ---- -memo (optimized, ~12KB, required=[presentation: sum:7]) +memo (optimized, ~14KB, required=[presentation: sum:7]) ├── G1: (project G2 G3 sum) │ └── [presentation: sum:7] │ ├── best: (project G2 G3 sum) @@ -1840,7 +1840,7 @@ memo (optimized, ~12KB, required=[presentation: sum:7]) memo SELECT array_agg(w) FROM (SELECT * FROM kuvw ORDER BY w DESC) GROUP BY u,v ---- -memo (optimized, ~7KB, required=[presentation: array_agg:7]) +memo (optimized, ~8KB, required=[presentation: array_agg:7]) ├── G1: (project G2 G3 array_agg) │ └── [presentation: array_agg:7] │ ├── best: (project G2 G3 array_agg) @@ -1869,7 +1869,7 @@ memo (optimized, ~7KB, required=[presentation: array_agg:7]) memo SELECT DISTINCT u, v, w FROM kuvw ---- -memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4]) +memo (optimized, ~6KB, required=[presentation: u:2,v:3,w:4]) ├── G1: (distinct-on G2 G3 cols=(2-4)) (distinct-on G2 G3 cols=(2-4),ordering=+2,+3,+4) (distinct-on G2 G3 cols=(2-4),ordering=+4,+3,+2) (distinct-on G2 G3 cols=(2-4),ordering=+3,+4) │ └── [presentation: u:2,v:3,w:4] │ ├── best: (distinct-on G2="[ordering: +2,+3,+4]" G3 cols=(2-4),ordering=+2,+3,+4) @@ -1893,7 +1893,7 @@ memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4]) memo SELECT DISTINCT ON (u, v) u, v, w FROM kuvw ---- -memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4]) +memo (optimized, ~6KB, required=[presentation: u:2,v:3,w:4]) ├── G1: (distinct-on G2 G3 cols=(2,3)) (distinct-on G2 G3 cols=(2,3),ordering=+2,+3) (distinct-on G2 G3 cols=(2,3),ordering=+3) │ └── [presentation: u:2,v:3,w:4] │ ├── best: (distinct-on G2="[ordering: +2,+3]" G3 cols=(2,3),ordering=+2,+3) @@ -2016,7 +2016,7 @@ memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: +2]) memo SELECT DISTINCT ON (u) u, v, w FROM kuvw ORDER BY u, v, w ---- -memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: +2]) +memo (optimized, ~6KB, required=[presentation: u:2,v:3,w:4] [ordering: +2]) ├── G1: (distinct-on G2 G3 cols=(2),ordering=+3,+4 opt(2)) (distinct-on G2 G3 cols=(2),ordering=+2,+3,+4) (distinct-on G2 G3 cols=(2),ordering=+3,+4) │ ├── [presentation: u:2,v:3,w:4] [ordering: +2] │ │ ├── best: (distinct-on G2="[ordering: +2,+3,+4]" G3 cols=(2),ordering=+3,+4 opt(2)) @@ -2047,7 +2047,7 @@ memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: +2]) memo SELECT DISTINCT ON (w, u) u, v, w FROM kuvw ORDER BY w, u, v DESC ---- -memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4,+2]) +memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: +4,+2]) ├── G1: (distinct-on G2 G3 cols=(2,4),ordering=-3 opt(2,4)) │ ├── [presentation: u:2,v:3,w:4] [ordering: +4,+2] │ │ ├── best: (distinct-on G2="[ordering: +4,+2,-3]" G3 cols=(2,4),ordering=-3 opt(2,4)) @@ -2078,7 +2078,7 @@ memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4,+2]) memo SELECT DISTINCT ON (w) u, v, w FROM kuvw ORDER BY w, u DESC, v ---- -memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) +memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) ├── G1: (distinct-on G2 G3 cols=(4),ordering=-2,+3 opt(4)) │ ├── [presentation: u:2,v:3,w:4] [ordering: +4] │ │ ├── best: (distinct-on G2="[ordering: +4,-2,+3]" G3 cols=(4),ordering=-2,+3 opt(4)) @@ -2108,7 +2108,7 @@ memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) memo SELECT DISTINCT ON (w) u, v, w FROM kuvw ORDER BY w DESC, u DESC, v ---- -memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: -4]) +memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: -4]) ├── G1: (distinct-on G2 G3 cols=(4),ordering=-2,+3 opt(4)) │ ├── [presentation: u:2,v:3,w:4] [ordering: -4] │ │ ├── best: (sort G1) @@ -2138,7 +2138,7 @@ memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: -4]) memo SELECT DISTINCT ON (w) u, v, w FROM kuvw ORDER BY w, u, v DESC ---- -memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) +memo (optimized, ~5KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) ├── G1: (distinct-on G2 G3 cols=(4),ordering=+2,-3 opt(4)) │ ├── [presentation: u:2,v:3,w:4] [ordering: +4] │ │ ├── best: (distinct-on G2="[ordering: +4,+2,-3]" G3 cols=(4),ordering=+2,-3 opt(4)) @@ -2172,7 +2172,7 @@ memo (optimized, ~4KB, required=[presentation: u:2,v:3,w:4] [ordering: +4]) memo SELECT (SELECT w FROM kuvw WHERE v=1 AND x=u) FROM xyz ORDER BY x+1, x ---- -memo (optimized, ~27KB, required=[presentation: w:12] [ordering: +13,+1]) +memo (optimized, ~32KB, required=[presentation: w:12] [ordering: +13,+1]) ├── G1: (project G2 G3 x) │ ├── [presentation: w:12] [ordering: +13,+1] │ │ ├── best: (sort G1) @@ -2246,7 +2246,7 @@ memo (optimized, ~27KB, required=[presentation: w:12] [ordering: +13,+1]) memo INSERT INTO xyz SELECT v, w, 1.0 FROM kuvw ON CONFLICT (x) DO NOTHING ---- -memo (optimized, ~21KB, required=[]) +memo (optimized, ~24KB, required=[]) ├── G1: (insert G2 G3 G4 xyz) │ └── [] │ ├── best: (insert G2 G3 G4 xyz) @@ -2294,7 +2294,7 @@ memo (optimized, ~21KB, required=[]) memo INSERT INTO xyz SELECT v, w, 1.0 FROM kuvw ON CONFLICT (x) DO UPDATE SET z=2.0 ---- -memo (optimized, ~22KB, required=[]) +memo (optimized, ~25KB, required=[]) ├── G1: (upsert G2 G3 G4 xyz) │ └── [] │ ├── best: (upsert G2 G3 G4 xyz) @@ -3280,7 +3280,7 @@ INDEX gg (g) memo SELECT d, e, count(*) FROM defg GROUP BY d, e LIMIT 10 ---- -memo (optimized, ~16KB, required=[presentation: d:1,e:2,count:8]) +memo (optimized, ~18KB, required=[presentation: d:1,e:2,count:8]) ├── G1: (limit G2 G3) (limit G4 G3) (limit G5 G3) (limit G6 G3) │ └── [presentation: d:1,e:2,count:8] │ ├── best: (limit G4="[limit hint: 10.00]" G3) diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 2e5a56aaef0a..4fe874d3fedb 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -182,7 +182,7 @@ inner-join (merge) memo expect=ReorderJoins SELECT * FROM abc, stu, xyz WHERE abc.a=stu.s AND stu.s=xyz.x ---- -memo (optimized, ~37KB, required=[presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:13,z:14]) +memo (optimized, ~42KB, required=[presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:13,z:14]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (inner-join G5 G6 G7) (inner-join G6 G5 G7) (inner-join G8 G9 G7) (inner-join G9 G8 G7) (merge-join G2 G3 G10 inner-join,+1,+7) (merge-join G3 G2 G10 inner-join,+7,+1) (lookup-join G3 G10 abc@ab,keyCols=[7],outCols=(1-3,7-9,12-14)) (merge-join G5 G6 G10 inner-join,+7,+12) (merge-join G6 G5 G10 inner-join,+12,+7) (lookup-join G6 G10 stu,keyCols=[12],outCols=(1-3,7-9,12-14)) (merge-join G8 G9 G10 inner-join,+7,+12) (lookup-join G8 G10 xyz@xy,keyCols=[7],outCols=(1-3,7-9,12-14)) (merge-join G9 G8 G10 inner-join,+12,+7) │ └── [presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:13,z:14] │ ├── best: (merge-join G5="[ordering: +7]" G6="[ordering: +(1|12)]" G10 inner-join,+7,+12) @@ -244,7 +244,7 @@ memo (optimized, ~37KB, required=[presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:1 memo SELECT * FROM abc, stu, xyz, pqr WHERE a = 1 ---- -memo (optimized, ~24KB, required=[presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:13,z:14,p:18,q:19,r:20,s:21,t:22]) +memo (optimized, ~28KB, required=[presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:13,z:14,p:18,q:19,r:20,s:21,t:22]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) │ └── [presentation: a:1,b:2,c:3,s:7,t:8,u:9,x:12,y:13,z:14,p:18,q:19,r:20,s:21,t:22] │ ├── best: (inner-join G3 G2 G4) @@ -290,7 +290,7 @@ SELECT * FROM stu, abc, xyz, pqr WHERE u = a AND a = x AND x = p ---- -memo (optimized, ~32KB, required=[presentation: s:1,t:2,u:3,a:6,b:7,c:8,x:12,y:13,z:14,p:18,q:19,r:20,s:21,t:22]) +memo (optimized, ~36KB, required=[presentation: s:1,t:2,u:3,a:6,b:7,c:8,x:12,y:13,z:14,p:18,q:19,r:20,s:21,t:22]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (merge-join G2 G3 G5 inner-join,+3,+6) (merge-join G3 G2 G5 inner-join,+6,+3) (lookup-join G3 G5 stu@uts,keyCols=[6],outCols=(1-3,6-8,12-14,18-22)) │ └── [presentation: s:1,t:2,u:3,a:6,b:7,c:8,x:12,y:13,z:14,p:18,q:19,r:20,s:21,t:22] │ ├── best: (merge-join G2="[ordering: +3]" G3="[ordering: +(6|12|18)]" G5 inner-join,+3,+6) @@ -384,7 +384,7 @@ INNER JOIN LATERAL ( ) ON a = v ---- -memo (optimized, ~13KB, required=[presentation: a:1,b:2,c:3,v:7,s:8,t:9,u:10]) +memo (optimized, ~15KB, required=[presentation: a:1,b:2,c:3,v:7,s:8,t:9,u:10]) ├── G1: (inner-join-apply G2 G3 G4) │ └── [presentation: a:1,b:2,c:3,v:7,s:8,t:9,u:10] │ ├── best: (inner-join-apply G2 G3 G4) @@ -1196,7 +1196,7 @@ New expression 2 of 2: memo expect=ReorderJoins SELECT * FROM abc JOIN xyz ON a=z ---- -memo (optimized, ~11KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~12KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (merge-join G2 G3 G5 inner-join,+1,+9) (lookup-join G3 G5 abc@ab,keyCols=[9],outCols=(1-3,7-9)) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (inner-join G2 G3 G4) @@ -1224,7 +1224,7 @@ memo (optimized, ~11KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) memo expect=ReorderJoins SELECT * FROM abc FULL OUTER JOIN xyz ON a=z ---- -memo (optimized, ~9KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~11KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (full-join G2 G3 G4) (full-join G3 G2 G4) (merge-join G2 G3 G5 full-join,+1,+9) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (full-join G2 G3 G4) @@ -1310,7 +1310,7 @@ full-join (hash) memo expect-not=ReorderJoins SELECT * FROM abc INNER LOOKUP JOIN xyz ON a=x ---- -memo (optimized, ~10KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~11KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (inner-join G2 G3 G4) (lookup-join G2 G5 xyz@xy,keyCols=[1],outCols=(1-3,7-9)) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (lookup-join G2 G5 xyz@xy,keyCols=[1],outCols=(1-3,7-9)) @@ -1393,7 +1393,7 @@ New expression 1 of 1: memo SELECT * FROM abc LEFT OUTER JOIN xyz ON a=z ---- -memo (optimized, ~9KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~11KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (left-join G2 G3 G4) (right-join G3 G2 G4) (merge-join G2 G3 G5 left-join,+1,+9) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (left-join G2 G3 G4) @@ -1440,7 +1440,7 @@ right-join (hash) memo SELECT * FROM abc RIGHT OUTER JOIN xyz ON a=z ---- -memo (optimized, ~11KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~12KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (left-join G2 G3 G4) (right-join G3 G2 G4) (lookup-join G2 G5 abc@ab,keyCols=[9],outCols=(1-3,7-9)) (merge-join G3 G2 G5 right-join,+1,+9) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (left-join G2 G3 G4) @@ -1724,7 +1724,7 @@ inner-join (merge) memo SELECT * FROM abc JOIN xyz ON a=x ---- -memo (optimized, ~12KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~14KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (merge-join G2 G3 G5 inner-join,+1,+7) (lookup-join G2 G5 xyz@xy,keyCols=[1],outCols=(1-3,7-9)) (merge-join G3 G2 G5 inner-join,+7,+1) (lookup-join G3 G5 abc@ab,keyCols=[7],outCols=(1-3,7-9)) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (merge-join G3="[ordering: +7]" G2="[ordering: +1]" G5 inner-join,+7,+1) @@ -1753,7 +1753,7 @@ memo (optimized, ~12KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) memo SELECT * FROM abc INNER HASH JOIN xyz ON a=x ---- -memo (optimized, ~9KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~10KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (inner-join G2 G3 G4) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (inner-join G2 G3 G4) @@ -1823,7 +1823,7 @@ inner-join (lookup xyz@xy) memo SELECT * FROM stu AS l JOIN stu AS r ON (l.s, l.t, l.u) = (r.s, r.t, r.u) ---- -memo (optimized, ~16KB, required=[presentation: s:1,t:2,u:3,s:6,t:7,u:8]) +memo (optimized, ~18KB, required=[presentation: s:1,t:2,u:3,s:6,t:7,u:8]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (merge-join G2 G3 G5 inner-join,+1,+2,+3,+6,+7,+8) (merge-join G2 G3 G5 inner-join,+3,+2,+1,+8,+7,+6) (lookup-join G2 G5 stu [as=r],keyCols=[1 2 3],outCols=(1-3,6-8)) (lookup-join G2 G5 stu@uts [as=r],keyCols=[3 2 1],outCols=(1-3,6-8)) (merge-join G3 G2 G5 inner-join,+6,+7,+8,+1,+2,+3) (merge-join G3 G2 G5 inner-join,+8,+7,+6,+3,+2,+1) (lookup-join G3 G5 stu [as=l],keyCols=[6 7 8],outCols=(1-3,6-8)) (lookup-join G3 G5 stu@uts [as=l],keyCols=[8 7 6],outCols=(1-3,6-8)) │ └── [presentation: s:1,t:2,u:3,s:6,t:7,u:8] │ ├── best: (merge-join G2="[ordering: +1,+2,+3]" G3="[ordering: +6,+7,+8]" G5 inner-join,+1,+2,+3,+6,+7,+8) @@ -2030,7 +2030,7 @@ left-join (merge) memo SELECT * FROM abc JOIN xyz ON a=b ---- -memo (optimized, ~14KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) +memo (optimized, ~16KB, required=[presentation: a:1,b:2,c:3,x:7,y:8,z:9]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) │ └── [presentation: a:1,b:2,c:3,x:7,y:8,z:9] │ ├── best: (inner-join G3 G2 G4) @@ -2068,7 +2068,7 @@ CREATE TABLE kfloat (k FLOAT PRIMARY KEY) memo SELECT * FROM abc JOIN kfloat ON a=k ---- -memo (optimized, ~10KB, required=[presentation: a:1,b:2,c:3,k:7]) +memo (optimized, ~12KB, required=[presentation: a:1,b:2,c:3,k:7]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) │ └── [presentation: a:1,b:2,c:3,k:7] │ ├── best: (inner-join G3 G2 G4) @@ -2554,7 +2554,7 @@ No new expressions. memo expect-not=GenerateLookupJoins SELECT a,b,n,m FROM small INNER HASH JOIN abcd ON a=m ---- -memo (optimized, ~8KB, required=[presentation: a:6,b:7,n:2,m:1]) +memo (optimized, ~9KB, required=[presentation: a:6,b:7,n:2,m:1]) ├── G1: (inner-join G2 G3 G4) │ └── [presentation: a:6,b:7,n:2,m:1] │ ├── best: (inner-join G2 G3 G4) @@ -5346,7 +5346,7 @@ WHERE n.name = 'Upper West Side' OR n.name = 'Upper East Side' GROUP BY n.name, n.geom ---- -memo (optimized, ~29KB, required=[presentation: name:16,popn_per_sqkm:22]) +memo (optimized, ~33KB, required=[presentation: name:16,popn_per_sqkm:22]) ├── G1: (project G2 G3 name) │ └── [presentation: name:16,popn_per_sqkm:22] │ ├── best: (project G2 G3 name) @@ -8205,7 +8205,7 @@ SELECT 1 FROM (VALUES (1), (1)) JOIN (VALUES (1), (1), (1)) ON true UNION ALL SELECT 1 FROM (VALUES (1), (1), (1)) JOIN (VALUES (1), (1)) ON true ---- -memo (optimized, ~18KB, required=[presentation: ?column?:7]) +memo (optimized, ~21KB, required=[presentation: ?column?:7]) ├── G1: (union-all G2 G3) │ └── [presentation: ?column?:7] │ ├── best: (union-all G2 G3) diff --git a/pkg/sql/opt/xform/testdata/rules/join_order b/pkg/sql/opt/xform/testdata/rules/join_order index a4d9f258eefc..f219f6ae03ea 100644 --- a/pkg/sql/opt/xform/testdata/rules/join_order +++ b/pkg/sql/opt/xform/testdata/rules/join_order @@ -312,7 +312,7 @@ New expression 3 of 3: memo join-limit=0 expect-not=ReorderJoins SELECT * FROM bx, cy, abc WHERE a = 1 AND abc.b = bx.b AND abc.c = cy.c ---- -memo (optimized, ~24KB, required=[presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d:12]) +memo (optimized, ~28KB, required=[presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d:12]) ├── G1: (inner-join G2 G3 G4) (merge-join G2 G3 G5 inner-join,+1,+10) │ └── [presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d:12] │ ├── best: (merge-join G2="[ordering: +1]" G3 G5 inner-join,+1,+10) @@ -365,7 +365,7 @@ memo (optimized, ~24KB, required=[presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d: memo join-limit=2 SELECT * FROM bx, cy, abc WHERE a = 1 AND abc.b = bx.b AND abc.c = cy.c ---- -memo (optimized, ~39KB, required=[presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d:12]) +memo (optimized, ~45KB, required=[presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d:12]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (inner-join G5 G6 G7) (inner-join G6 G5 G7) (merge-join G2 G3 G8 inner-join,+1,+10) (merge-join G3 G2 G8 inner-join,+10,+1) (lookup-join G3 G8 bx,keyCols=[10],outCols=(1,2,5,6,9-12)) (merge-join G5 G6 G8 inner-join,+5,+11) (merge-join G6 G5 G8 inner-join,+11,+5) (lookup-join G6 G8 cy,keyCols=[11],outCols=(1,2,5,6,9-12)) │ └── [presentation: b:1,x:2,c:5,y:6,a:9,b:10,c:11,d:12] │ ├── best: (lookup-join G3 G8 bx,keyCols=[10],outCols=(1,2,5,6,9-12)) @@ -535,7 +535,7 @@ inner-join (cross) memo join-limit=0 SELECT * FROM bx, cy, dz, abc WHERE x = y AND y = z AND z = a ---- -memo (optimized, ~25KB, required=[presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b:14,c:15,d:16]) +memo (optimized, ~28KB, required=[presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b:14,c:15,d:16]) ├── G1: (inner-join G2 G3 G4) (merge-join G2 G3 G5 inner-join,+2,+6) │ └── [presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b:14,c:15,d:16] │ ├── best: (inner-join G2 G3 G4) @@ -601,7 +601,7 @@ memo (optimized, ~25KB, required=[presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b: memo join-limit=3 SELECT * FROM bx, cy, dz, abc WHERE x = y AND y = z AND z = a ---- -memo (optimized, ~55KB, required=[presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b:14,c:15,d:16]) +memo (optimized, ~61KB, required=[presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b:14,c:15,d:16]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) (inner-join G5 G6 G7) (inner-join G6 G5 G7) (inner-join G8 G9 G7) (inner-join G9 G8 G7) (inner-join G10 G11 G12) (inner-join G11 G10 G12) (inner-join G13 G14 G12) (inner-join G14 G13 G12) (inner-join G15 G16 G12) (inner-join G16 G15 G12) (inner-join G17 G18 G12) (inner-join G18 G17 G12) (merge-join G3 G2 G19 inner-join,+6,+2) (merge-join G6 G5 G19 inner-join,+10,+6) (merge-join G9 G8 G19 inner-join,+10,+6) (merge-join G11 G10 G19 inner-join,+13,+10) (merge-join G14 G13 G19 inner-join,+13,+10) (merge-join G16 G15 G19 inner-join,+13,+10) (lookup-join G17 G19 abc,keyCols=[10],outCols=(1,2,5,6,9,10,13-16)) (merge-join G18 G17 G19 inner-join,+13,+10) │ └── [presentation: b:1,x:2,c:5,y:6,d:9,z:10,a:13,b:14,c:15,d:16] │ ├── best: (inner-join G3 G2 G4) diff --git a/pkg/sql/opt/xform/testdata/rules/limit b/pkg/sql/opt/xform/testdata/rules/limit index 2c31f58200eb..f02b8e4c39cd 100644 --- a/pkg/sql/opt/xform/testdata/rules/limit +++ b/pkg/sql/opt/xform/testdata/rules/limit @@ -229,7 +229,7 @@ limit memo SELECT s FROM a WHERE s='foo' LIMIT 1 ---- -memo (optimized, ~7KB, required=[presentation: s:4]) +memo (optimized, ~8KB, required=[presentation: s:4]) ├── G1: (limit G2 G3) (scan a@s_idx,cols=(4),constrained,lim=1) (scan a@si_idx,cols=(4),constrained,lim=1) │ └── [presentation: s:4] │ ├── best: (scan a@s_idx,cols=(4),constrained,lim=1) @@ -1657,7 +1657,7 @@ top-k memo expect=GenerateTopK SELECT * FROM a ORDER BY k LIMIT 1 ---- -memo (optimized, ~3KB, required=[presentation: k:1,i:2,f:3,s:4,j:5]) +memo (optimized, ~4KB, required=[presentation: k:1,i:2,f:3,s:4,j:5]) ├── G1: (limit G2 G3 ordering=+1) (scan a,cols=(1-5),lim=1) (top-k G2 &{1 +1}) │ └── [presentation: k:1,i:2,f:3,s:4,j:5] │ ├── best: (scan a,cols=(1-5),lim=1) diff --git a/pkg/sql/opt/xform/testdata/rules/scan b/pkg/sql/opt/xform/testdata/rules/scan index 7b8a1a12d7eb..e44de0e9721b 100644 --- a/pkg/sql/opt/xform/testdata/rules/scan +++ b/pkg/sql/opt/xform/testdata/rules/scan @@ -143,7 +143,7 @@ scan a@si_idx,rev memo SELECT k FROM a ORDER BY k ASC ---- -memo (optimized, ~2KB, required=[presentation: k:1] [ordering: +1]) +memo (optimized, ~3KB, required=[presentation: k:1] [ordering: +1]) └── G1: (scan a,cols=(1)) (scan a@s_idx,cols=(1)) (scan a@si_idx,cols=(1)) ├── [presentation: k:1] [ordering: +1] │ ├── best: (scan a,cols=(1)) @@ -165,7 +165,7 @@ scan a@s_idx memo SELECT s, i, f FROM a ORDER BY s, k, i ---- -memo (optimized, ~2KB, required=[presentation: s:4,i:2,f:3] [ordering: +4,+1]) +memo (optimized, ~3KB, required=[presentation: s:4,i:2,f:3] [ordering: +4,+1]) └── G1: (scan a,cols=(1-4)) (scan a@s_idx,cols=(1-4)) ├── [presentation: s:4,i:2,f:3] [ordering: +4,+1] │ ├── best: (scan a@s_idx,cols=(1-4)) @@ -219,7 +219,7 @@ scan p@p_i_idx,partial memo SELECT s, i, f FROM a ORDER BY f ---- -memo (optimized, ~2KB, required=[presentation: s:4,i:2,f:3] [ordering: +3]) +memo (optimized, ~3KB, required=[presentation: s:4,i:2,f:3] [ordering: +3]) └── G1: (scan a,cols=(2-4)) (scan a@s_idx,cols=(2-4)) ├── [presentation: s:4,i:2,f:3] [ordering: +3] │ ├── best: (sort G1) @@ -231,7 +231,7 @@ memo (optimized, ~2KB, required=[presentation: s:4,i:2,f:3] [ordering: +3]) memo SELECT s, i, f FROM a ORDER BY s DESC, i ---- -memo (optimized, ~2KB, required=[presentation: s:4,i:2,f:3] [ordering: -4,+2]) +memo (optimized, ~3KB, required=[presentation: s:4,i:2,f:3] [ordering: -4,+2]) └── G1: (scan a,cols=(2-4)) (scan a@s_idx,cols=(2-4)) ├── [presentation: s:4,i:2,f:3] [ordering: -4,+2] │ ├── best: (sort G1="[ordering: -4]") @@ -274,7 +274,7 @@ CREATE TABLE abc ( memo SELECT d FROM abc ORDER BY lower(d) ---- -memo (optimized, ~3KB, required=[presentation: d:4] [ordering: +7]) +memo (optimized, ~4KB, required=[presentation: d:4] [ordering: +7]) ├── G1: (project G2 G3 d) │ ├── [presentation: d:4] [ordering: +7] │ │ ├── best: (sort G1) @@ -304,7 +304,7 @@ scan a memo SELECT s, i, f FROM a ORDER BY k, i, s ---- -memo (optimized, ~2KB, required=[presentation: s:4,i:2,f:3] [ordering: +1]) +memo (optimized, ~3KB, required=[presentation: s:4,i:2,f:3] [ordering: +1]) └── G1: (scan a,cols=(1-4)) (scan a@s_idx,cols=(1-4)) ├── [presentation: s:4,i:2,f:3] [ordering: +1] │ ├── best: (scan a,cols=(1-4)) @@ -324,7 +324,7 @@ scan a@si_idx,rev memo SELECT s, j FROM a ORDER BY s ---- -memo (optimized, ~2KB, required=[presentation: s:4,j:5] [ordering: +4]) +memo (optimized, ~3KB, required=[presentation: s:4,j:5] [ordering: +4]) └── G1: (scan a,cols=(4,5)) (scan a@si_idx,cols=(4,5)) ├── [presentation: s:4,j:5] [ordering: +4] │ ├── best: (scan a@si_idx,rev,cols=(4,5)) @@ -351,7 +351,7 @@ sort (segmented) memo SELECT i, k FROM a ORDER BY s DESC, i, k ---- -memo (optimized, ~2KB, required=[presentation: i:2,k:1] [ordering: -4,+2,+1]) +memo (optimized, ~3KB, required=[presentation: i:2,k:1] [ordering: -4,+2,+1]) └── G1: (scan a,cols=(1,2,4)) (scan a@s_idx,cols=(1,2,4)) (scan a@si_idx,cols=(1,2,4)) ├── [presentation: i:2,k:1] [ordering: -4,+2,+1] │ ├── best: (sort G1="[ordering: -4]") diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 8cf695c27d4b..a648d3633fc3 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -263,7 +263,7 @@ CREATE INDEX idx2 ON p (s) WHERE i > 0 memo expect=GeneratePartialIndexScans SELECT * FROM p WHERE i > 0 AND s = 'foo' ---- -memo (optimized, ~14KB, required=[presentation: k:1,i:2,f:3,s:4,b:5]) +memo (optimized, ~17KB, required=[presentation: k:1,i:2,f:3,s:4,b:5]) ├── G1: (select G2 G3) (index-join G4 p,cols=(1-5)) (index-join G5 p,cols=(1-5)) (index-join G6 p,cols=(1-5)) (index-join G7 p,cols=(1-5)) │ └── [presentation: k:1,i:2,f:3,s:4,b:5] │ ├── best: (index-join G4 p,cols=(1-5)) @@ -311,7 +311,7 @@ memo (optimized, ~14KB, required=[presentation: k:1,i:2,f:3,s:4,b:5]) memo expect-not=GeneratePartialIndexScans SELECT i FROM p WHERE s = 'bar' ---- -memo (optimized, ~8KB, required=[presentation: i:2]) +memo (optimized, ~10KB, required=[presentation: i:2]) ├── G1: (project G2 G3 i) │ └── [presentation: i:2] │ ├── best: (project G2 G3 i) @@ -531,7 +531,7 @@ scan a memo SELECT k FROM a WHERE k = 1 ---- -memo (optimized, ~5KB, required=[presentation: k:1]) +memo (optimized, ~6KB, required=[presentation: k:1]) ├── G1: (select G2 G3) (scan a,cols=(1),constrained) │ └── [presentation: k:1] │ ├── best: (scan a,cols=(1),constrained) @@ -560,7 +560,7 @@ project memo SELECT k FROM a WHERE v > 1 ---- -memo (optimized, ~6KB, required=[presentation: k:1]) +memo (optimized, ~7KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) │ └── [presentation: k:1] │ ├── best: (project G2 G3 k) @@ -597,7 +597,7 @@ project memo SELECT k FROM a WHERE u = 1 AND k = 5 ---- -memo (optimized, ~8KB, required=[presentation: k:1]) +memo (optimized, ~9KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) │ └── [presentation: k:1] │ ├── best: (project G2 G3 k) @@ -643,7 +643,7 @@ project memo SELECT k FROM a WHERE u = 1 AND k+u = 1 ---- -memo (optimized, ~8KB, required=[presentation: k:1]) +memo (optimized, ~9KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) │ └── [presentation: k:1] │ ├── best: (project G2 G3 k) @@ -695,7 +695,7 @@ project memo SELECT k FROM a WHERE u = 1 AND v = 5 ---- -memo (optimized, ~9KB, required=[presentation: k:1]) +memo (optimized, ~10KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) │ └── [presentation: k:1] │ ├── best: (project G2 G3 k) @@ -787,7 +787,7 @@ index-join b memo SELECT * FROM b WHERE v >= 1 AND v <= 10 ---- -memo (optimized, ~6KB, required=[presentation: k:1,u:2,v:3,j:4]) +memo (optimized, ~7KB, required=[presentation: k:1,u:2,v:3,j:4]) ├── G1: (select G2 G3) (index-join G4 b,cols=(1-4)) │ └── [presentation: k:1,u:2,v:3,j:4] │ ├── best: (index-join G4 b,cols=(1-4)) @@ -849,7 +849,7 @@ index-join b memo SELECT * FROM b WHERE v >= 1 AND v <= 10 AND k > 5 ---- -memo (optimized, ~8KB, required=[presentation: k:1,u:2,v:3,j:4]) +memo (optimized, ~9KB, required=[presentation: k:1,u:2,v:3,j:4]) ├── G1: (select G2 G3) (select G4 G5) (index-join G6 b,cols=(1-4)) │ └── [presentation: k:1,u:2,v:3,j:4] │ ├── best: (index-join G6 b,cols=(1-4)) @@ -947,7 +947,7 @@ select memo SELECT * FROM b WHERE v >= 1 AND v <= 10 AND k+u = 1 ---- -memo (optimized, ~7KB, required=[presentation: k:1,u:2,v:3,j:4]) +memo (optimized, ~8KB, required=[presentation: k:1,u:2,v:3,j:4]) ├── G1: (select G2 G3) (select G4 G5) │ └── [presentation: k:1,u:2,v:3,j:4] │ ├── best: (select G4 G5) @@ -1011,7 +1011,7 @@ select memo SELECT * FROM b WHERE v >= 1 AND v <= 10 AND k+u = 1 AND k > 5 ---- -memo (optimized, ~9KB, required=[presentation: k:1,u:2,v:3,j:4]) +memo (optimized, ~11KB, required=[presentation: k:1,u:2,v:3,j:4]) ├── G1: (select G2 G3) (select G4 G5) (select G6 G7) │ └── [presentation: k:1,u:2,v:3,j:4] │ ├── best: (select G6 G7) @@ -1079,7 +1079,7 @@ select memo SELECT * FROM b WHERE (u, k, v) > (1, 2, 3) AND (u, k, v) < (8, 9, 10) ---- -memo (optimized, ~7KB, required=[presentation: k:1,u:2,v:3,j:4]) +memo (optimized, ~8KB, required=[presentation: k:1,u:2,v:3,j:4]) ├── G1: (select G2 G3) (select G4 G3) │ └── [presentation: k:1,u:2,v:3,j:4] │ ├── best: (select G4 G3) @@ -1240,7 +1240,7 @@ New expression 2 of 2: memo SELECT s, i, f FROM kifs WHERE s='foo' ORDER BY s DESC, i ---- -memo (optimized, ~6KB, required=[presentation: s:4,i:2,f:3] [ordering: +2 opt(4)]) +memo (optimized, ~7KB, required=[presentation: s:4,i:2,f:3] [ordering: +2 opt(4)]) ├── G1: (select G2 G3) (scan kifs@s_idx,cols=(2-4),constrained) (index-join G4 kifs,cols=(2-4)) │ ├── [presentation: s:4,i:2,f:3] [ordering: +2 opt(4)] │ │ ├── best: (sort G1) @@ -1273,7 +1273,7 @@ memo (optimized, ~6KB, required=[presentation: s:4,i:2,f:3] [ordering: +2 opt(4) memo SELECT j FROM kifs WHERE s = 'foo' ---- -memo (optimized, ~7KB, required=[presentation: j:5]) +memo (optimized, ~8KB, required=[presentation: j:5]) ├── G1: (project G2 G3 j) │ └── [presentation: j:5] │ ├── best: (project G2 G3 j) @@ -1299,7 +1299,7 @@ memo (optimized, ~7KB, required=[presentation: j:5]) memo SELECT i, k FROM kifs WHERE s >= 'foo' ---- -memo (optimized, ~6KB, required=[presentation: i:2,k:1]) +memo (optimized, ~7KB, required=[presentation: i:2,k:1]) ├── G1: (project G2 G3 k i) │ └── [presentation: i:2,k:1] │ ├── best: (project G2 G3 k i) @@ -1710,7 +1710,7 @@ CREATE INDEX idx2 ON p (i) WHERE s = 'foo' memo SELECT i FROM p WHERE i = 3 AND s = 'foo' ---- -memo (optimized, ~18KB, required=[presentation: i:2]) +memo (optimized, ~21KB, required=[presentation: i:2]) ├── G1: (project G2 G3 i) │ └── [presentation: i:2] │ ├── best: (project G2 G3 i) @@ -1784,7 +1784,7 @@ CREATE INDEX idx ON p (i) WHERE s = 'foo' memo expect-not=GenerateConstrainedScans SELECT i FROM p WHERE s = 'bar' AND i = 5 ---- -memo (optimized, ~8KB, required=[presentation: i:2]) +memo (optimized, ~9KB, required=[presentation: i:2]) ├── G1: (project G2 G3 i) │ └── [presentation: i:2] │ ├── best: (project G2 G3 i) @@ -2138,7 +2138,7 @@ project memo expect=GenerateInvertedIndexScans SELECT k FROM b WHERE j @> '{"a": "b"}' ---- -memo (optimized, ~7KB, required=[presentation: k:1]) +memo (optimized, ~8KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) (project G4 G3 k) │ └── [presentation: k:1] │ ├── best: (project G4 G3 k) @@ -4428,7 +4428,7 @@ CREATE INVERTED INDEX idx2 ON pi (j) WHERE s = 'bar' memo expect=GenerateInvertedIndexScans SELECT * FROM pi WHERE j @> '{"a": "b"}' AND s = 'bar' ---- -memo (optimized, ~11KB, required=[presentation: k:1,s:2,j:3]) +memo (optimized, ~13KB, required=[presentation: k:1,s:2,j:3]) ├── G1: (select G2 G3) (select G4 G5) (index-join G6 pi,cols=(1-3)) │ └── [presentation: k:1,s:2,j:3] │ ├── best: (index-join G6 pi,cols=(1-3)) @@ -4476,7 +4476,7 @@ CREATE INVERTED INDEX idx ON pi (j) WHERE s IN ('foo', 'bar') memo expect-not=GenerateInvertedIndexScans SELECT * FROM pi WHERE j @> '{"a": "b"}' AND s = 'baz' ---- -memo (optimized, ~7KB, required=[presentation: k:1,s:2,j:3]) +memo (optimized, ~8KB, required=[presentation: k:1,s:2,j:3]) ├── G1: (select G2 G3) │ └── [presentation: k:1,s:2,j:3] │ ├── best: (select G2 G3) @@ -5263,7 +5263,7 @@ inner-join (zigzag pqr@q pqr@r) memo expect=GenerateZigzagJoins SELECT q,r FROM pqr WHERE q = 1 AND r = 2 ---- -memo (optimized, ~13KB, required=[presentation: q:2,r:3]) +memo (optimized, ~15KB, required=[presentation: q:2,r:3]) ├── G1: (select G2 G3) (select G4 G5) (select G6 G7) (select G8 G7) (zigzag-join G3 pqr@q pqr@r) │ └── [presentation: q:2,r:3] │ ├── best: (zigzag-join G3 pqr@q pqr@r) @@ -5344,7 +5344,7 @@ inner-join (lookup pqr) memo expect=GenerateZigzagJoins SELECT q,r,s FROM pqr WHERE q = 1 AND r = 2 ---- -memo (optimized, ~15KB, required=[presentation: q:2,r:3,s:4]) +memo (optimized, ~18KB, required=[presentation: q:2,r:3,s:4]) ├── G1: (select G2 G3) (select G4 G5) (select G6 G7) (select G8 G7) (lookup-join G9 G10 pqr,keyCols=[1],outCols=(2-4)) │ └── [presentation: q:2,r:3,s:4] │ ├── best: (lookup-join G9 G10 pqr,keyCols=[1],outCols=(2-4)) @@ -5409,7 +5409,7 @@ inner-join (zigzag pqr@q pqr@s) memo expect=GenerateZigzagJoins SELECT q,s FROM pqr WHERE q = 1 AND s = 'foo' ---- -memo (optimized, ~11KB, required=[presentation: q:2,s:4]) +memo (optimized, ~13KB, required=[presentation: q:2,s:4]) ├── G1: (select G2 G3) (select G4 G5) (select G6 G7) (zigzag-join G3 pqr@q pqr@s) │ └── [presentation: q:2,s:4] │ ├── best: (zigzag-join G3 pqr@q pqr@s) @@ -5912,7 +5912,7 @@ select memo SELECT p,q,r,s FROM pqr WHERE q = 1 AND r = 1 AND s = 'foo' ---- -memo (optimized, ~31KB, required=[presentation: p:1,q:2,r:3,s:4]) +memo (optimized, ~36KB, required=[presentation: p:1,q:2,r:3,s:4]) ├── G1: (select G2 G3) (select G4 G5) (select G6 G7) (select G8 G9) (select G10 G9) (lookup-join G11 G12 pqr,keyCols=[1],outCols=(1-4)) (zigzag-join G3 pqr@q pqr@s) (zigzag-join G3 pqr@q pqr@rs) (lookup-join G13 G9 pqr,keyCols=[1],outCols=(1-4)) │ └── [presentation: p:1,q:2,r:3,s:4] │ ├── best: (zigzag-join G3 pqr@q pqr@s) @@ -6004,7 +6004,7 @@ select memo SELECT q,t FROM pqr WHERE q = 1 AND t = 'foo' ---- -memo (optimized, ~9KB, required=[presentation: q:2,t:5]) +memo (optimized, ~10KB, required=[presentation: q:2,t:5]) ├── G1: (select G2 G3) (select G4 G5) (select G6 G7) │ └── [presentation: q:2,t:5] │ ├── best: (select G4 G5) @@ -6043,7 +6043,7 @@ memo (optimized, ~9KB, required=[presentation: q:2,t:5]) memo SELECT c FROM zz_redundant WHERE b = 1 ---- -memo (optimized, ~6KB, required=[presentation: c:3]) +memo (optimized, ~7KB, required=[presentation: c:3]) ├── G1: (project G2 G3 c) │ └── [presentation: c:3] │ ├── best: (project G2 G3 c) @@ -7806,7 +7806,7 @@ CREATE TABLE t58390 ( memo SELECT * FROM t58390 WHERE a > 1 OR b > 1 ---- -memo (optimized, ~19KB, required=[presentation: k:1,a:2,b:3,c:4]) +memo (optimized, ~22KB, required=[presentation: k:1,a:2,b:3,c:4]) ├── G1: (select G2 G3) (index-join G4 t58390,cols=(1-4)) (distinct-on G5 G6 cols=(1)) (distinct-on G5 G6 cols=(1),ordering=+1) │ └── [presentation: k:1,a:2,b:3,c:4] │ ├── best: (select G2 G3) @@ -7921,7 +7921,7 @@ JOIN t61795 AS t2 ON t1.c = t1.b AND t1.b = t2.b WHERE t1.a = 10 OR t2.b != abs(t2.b) ORDER BY t1.b ASC ---- -memo (optimized, ~29KB, required=[presentation: a:1] [ordering: +2]) +memo (optimized, ~33KB, required=[presentation: a:1] [ordering: +2]) ├── G1: (project G2 G3 a b) │ ├── [presentation: a:1] [ordering: +2] │ │ ├── best: (sort G1) diff --git a/pkg/sql/opt/xform/testdata/rules/set b/pkg/sql/opt/xform/testdata/rules/set index adf8bedb2d4e..02ed119b4576 100644 --- a/pkg/sql/opt/xform/testdata/rules/set +++ b/pkg/sql/opt/xform/testdata/rules/set @@ -21,7 +21,7 @@ CREATE TABLE kuvw ( memo expect=GenerateStreamingSetOp SELECT u,v,w FROM kuvw UNION SELECT w,v,u FROM kuvw ---- -memo (optimized, ~9KB, required=[presentation: u:13,v:14,w:15]) +memo (optimized, ~11KB, required=[presentation: u:13,v:14,w:15]) ├── G1: (union G2 G3) (union G2 G3 ordering=+13,+14,+15) (union G2 G3 ordering=+15,+14,+13) (union G2 G3 ordering=+14,+15,+13) (union G2 G3 ordering=+14,+13,+15) │ └── [presentation: u:13,v:14,w:15] │ ├── best: (union G2="[ordering: +2,+3,+4]" G3="[ordering: +10,+9,+8]" ordering=+13,+14,+15) @@ -74,7 +74,7 @@ memo (optimized, ~9KB, required=[presentation: u:13,v:14,w:15]) memo expect=GenerateStreamingSetOp SELECT * FROM kuvw INTERSECT SELECT * FROM kuvw ---- -memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) +memo (optimized, ~11KB, required=[presentation: k:1,u:2,v:3,w:4]) ├── G1: (intersect-all G2 G3) (intersect-all G2 G3 ordering=+1,+2,+3,+4) (intersect-all G2 G3 ordering=+2,+3,+4,+1) (intersect-all G2 G3 ordering=+4,+3,+2,+1) (intersect-all G2 G3 ordering=+3,+4,+1,+2) (intersect-all G2 G3 ordering=+4,+1,+2,+3) │ └── [presentation: k:1,u:2,v:3,w:4] │ ├── best: (intersect-all G2="[ordering: +1]" G3="[ordering: +7]" ordering=+1,+2,+3,+4) @@ -121,7 +121,7 @@ memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) memo expect=GenerateStreamingSetOp SELECT * FROM kuvw INTERSECT ALL SELECT * FROM kuvw ---- -memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) +memo (optimized, ~11KB, required=[presentation: k:1,u:2,v:3,w:4]) ├── G1: (intersect-all G2 G3) (intersect-all G2 G3 ordering=+1,+2,+3,+4) (intersect-all G2 G3 ordering=+2,+3,+4,+1) (intersect-all G2 G3 ordering=+4,+3,+2,+1) (intersect-all G2 G3 ordering=+3,+4,+1,+2) (intersect-all G2 G3 ordering=+4,+1,+2,+3) │ └── [presentation: k:1,u:2,v:3,w:4] │ ├── best: (intersect-all G2="[ordering: +1]" G3="[ordering: +7]" ordering=+1,+2,+3,+4) @@ -168,7 +168,7 @@ memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) memo expect=GenerateStreamingSetOp SELECT * FROM kuvw EXCEPT SELECT * FROM kuvw ---- -memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) +memo (optimized, ~11KB, required=[presentation: k:1,u:2,v:3,w:4]) ├── G1: (except-all G2 G3) (except-all G2 G3 ordering=+1,+2,+3,+4) (except-all G2 G3 ordering=+2,+3,+4,+1) (except-all G2 G3 ordering=+4,+3,+2,+1) (except-all G2 G3 ordering=+3,+4,+1,+2) (except-all G2 G3 ordering=+4,+1,+2,+3) │ └── [presentation: k:1,u:2,v:3,w:4] │ ├── best: (except-all G2="[ordering: +1]" G3="[ordering: +7]" ordering=+1,+2,+3,+4) @@ -215,7 +215,7 @@ memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) memo expect=GenerateStreamingSetOp SELECT * FROM kuvw EXCEPT ALL SELECT * FROM kuvw ---- -memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) +memo (optimized, ~11KB, required=[presentation: k:1,u:2,v:3,w:4]) ├── G1: (except-all G2 G3) (except-all G2 G3 ordering=+1,+2,+3,+4) (except-all G2 G3 ordering=+2,+3,+4,+1) (except-all G2 G3 ordering=+4,+3,+2,+1) (except-all G2 G3 ordering=+3,+4,+1,+2) (except-all G2 G3 ordering=+4,+1,+2,+3) │ └── [presentation: k:1,u:2,v:3,w:4] │ ├── best: (except-all G2="[ordering: +1]" G3="[ordering: +7]" ordering=+1,+2,+3,+4) @@ -263,7 +263,7 @@ memo (optimized, ~10KB, required=[presentation: k:1,u:2,v:3,w:4]) memo expect-not=GenerateStreamingSetOp SELECT * FROM kuvw UNION ALL SELECT * FROM kuvw ---- -memo (optimized, ~8KB, required=[presentation: k:13,u:14,v:15,w:16]) +memo (optimized, ~9KB, required=[presentation: k:13,u:14,v:15,w:16]) ├── G1: (union-all G2 G3) │ └── [presentation: k:13,u:14,v:15,w:16] │ ├── best: (union-all G2 G3) diff --git a/pkg/sql/sqltestutils/BUILD.bazel b/pkg/sql/sqltestutils/BUILD.bazel index c2667814b97e..10cb2718c01f 100644 --- a/pkg/sql/sqltestutils/BUILD.bazel +++ b/pkg/sql/sqltestutils/BUILD.bazel @@ -30,7 +30,6 @@ go_library( "//pkg/testutils/skip", "//pkg/testutils/sqlutils", "//pkg/util/cloudinfo", - "//pkg/util/log", "//pkg/util/protoutil", "//pkg/util/treeprinter", "@com_github_cockroachdb_datadriven//:datadriven", diff --git a/pkg/sql/sqltestutils/telemetry.go b/pkg/sql/sqltestutils/telemetry.go index da354fc6b7ba..3403b29aba82 100644 --- a/pkg/sql/sqltestutils/telemetry.go +++ b/pkg/sql/sqltestutils/telemetry.go @@ -34,7 +34,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" "github.com/cockroachdb/cockroach/pkg/util/cloudinfo" - "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/treeprinter" "github.com/cockroachdb/datadriven" "github.com/cockroachdb/errors" @@ -156,9 +155,6 @@ func (tt *telemetryTest) Start(t *testing.T, serverArgs []base.TestServerArgs) { tt.serverDB = tt.cluster.ServerConn(0) tt.prepareCluster(tt.serverDB) - // Prevent a logging assertion that the server ID is initialized multiple times. - log.TestingClearServerIdentifiers() - tt.tenant, tt.tenantDB = serverutils.StartTenant(tt.t, tt.server, base.TestTenantArgs{ TenantID: serverutils.TestTenantID(), AllowSettingClusterSettings: true, diff --git a/pkg/util/fast_int_set.go b/pkg/util/fast_int_set.go index f0d9272c5ba3..84e16aa6c742 100644 --- a/pkg/util/fast_int_set.go +++ b/pkg/util/fast_int_set.go @@ -26,15 +26,24 @@ import ( // FastIntSet keeps track of a set of integers. It does not perform any // allocations when the values are small. It is not thread-safe. type FastIntSet struct { - // We use a uint64 as long as all elements are between 0 and 63. If we add an - // element outside of this range, we use both the uint64 and the Sparse; the - // Sparse then holds all elements of the set and the uint64 provides a fast - // path in certain cases. We don't just use the latter directly because it's - // larger and can't be passed around by value. - small uint64 + small bitmap large *intsets.Sparse } +// We maintain a bitmap for small element values (specifically 0 to +// smallCutoff-1). When this bitmap is sufficient, we avoid allocating the +// `Sparse` set. Even when we have to allocate the `Sparse` set, we still +// maintain the bitmap as it can provide a fast path for certain operations. +// Note: this can be set to a smaller value, e.g. for testing. +const smallCutoff = 128 + +// bitmap implements a bitmap of size smallCutoff. +type bitmap struct { + // We don't use an array because that makes Go always keep the struct on the + // stack (see https://github.com/golang/go/issues/24416). + lo, hi uint64 +} + // MakeFastIntSet returns a set initialized with the given values. func MakeFastIntSet(vals ...int) FastIntSet { var res FastIntSet @@ -44,10 +53,6 @@ func MakeFastIntSet(vals ...int) FastIntSet { return res } -// We store bits for values smaller than this cutoff. -// Note: this can be set to a smaller value, e.g. for testing. -const smallCutoff = 64 - func (s *FastIntSet) toLarge() *intsets.Sparse { if s.large != nil { return s.large @@ -76,7 +81,7 @@ func (s *FastIntSet) fitsInSmall() bool { func (s *FastIntSet) Add(i int) { withinSmallBounds := i >= 0 && i < smallCutoff if withinSmallBounds { - s.small |= (1 << uint64(i)) + s.small.Set(i) } if !withinSmallBounds && s.large == nil { s.large = s.toLarge() @@ -95,10 +100,8 @@ func (s *FastIntSet) AddRange(from, to int) { panic("invalid range when adding range to FastIntSet") } - withinSmallBounds := from >= 0 && to < smallCutoff - if withinSmallBounds && s.large == nil { - nValues := to - from + 1 - s.small |= (1<= 0 && to < smallCutoff { + s.small.SetRange(from, to) } else { for i := from; i <= to; i++ { s.Add(i) @@ -109,7 +112,7 @@ func (s *FastIntSet) AddRange(from, to int) { // Remove removes a value from the set. No-op if the value is not in the set. func (s *FastIntSet) Remove(i int) { if i >= 0 && i < smallCutoff { - s.small &= ^(1 << uint64(i)) + s.small.Unset(i) } if s.large != nil { s.large.Remove(i) @@ -119,7 +122,7 @@ func (s *FastIntSet) Remove(i int) { // Contains returns true if the set contains the value. func (s FastIntSet) Contains(i int) bool { if i >= 0 && i < smallCutoff { - return (s.small & (1 << uint64(i))) != 0 + return s.small.IsSet(i) } if s.large != nil { return s.large.Has(i) @@ -129,13 +132,13 @@ func (s FastIntSet) Contains(i int) bool { // Empty returns true if the set is empty. func (s FastIntSet) Empty() bool { - return s.small == 0 && (s.large == nil || s.large.IsEmpty()) + return s.small == bitmap{} && (s.large == nil || s.large.IsEmpty()) } // Len returns the number of the elements in the set. func (s FastIntSet) Len() int { if s.large == nil { - return bits.OnesCount64(s.small) + return s.small.OnesCount() } return s.large.Len() } @@ -147,8 +150,8 @@ func (s FastIntSet) Next(startVal int) (int, bool) { startVal = 0 } if startVal >= 0 && startVal < smallCutoff { - if ntz := bits.TrailingZeros64(s.small >> uint64(startVal)); ntz < 64 { - return startVal + ntz, true + if nextVal, ok := s.small.Next(startVal); ok { + return nextVal, true } } if s.large != nil { @@ -160,17 +163,22 @@ func (s FastIntSet) Next(startVal int) (int, bool) { // ForEach calls a function for each value in the set (in increasing order). func (s FastIntSet) ForEach(f func(i int)) { - if s.large != nil { + if !s.fitsInSmall() { for x := s.large.Min(); x != intsets.MaxInt; x = s.large.LowerBound(x + 1) { f(x) } return } - for v := s.small; v != 0; { + for v := s.small.lo; v != 0; { i := bits.TrailingZeros64(v) f(i) v &^= 1 << uint(i) } + for v := s.small.hi; v != 0; { + i := bits.TrailingZeros64(v) + f(64 + i) + v &^= 1 << uint(i) + } } // Ordered returns a slice with all the integers in the set, in increasing order. @@ -217,7 +225,7 @@ func (s *FastIntSet) CopyFrom(other FastIntSet) { // UnionWith adds all the elements from rhs to this set. func (s *FastIntSet) UnionWith(rhs FastIntSet) { - s.small |= rhs.small + s.small.UnionWith(rhs.small) if s.large == nil && rhs.large == nil { // Fast path. return @@ -244,7 +252,7 @@ func (s FastIntSet) Union(rhs FastIntSet) FastIntSet { // IntersectionWith removes any elements not in rhs from this set. func (s *FastIntSet) IntersectionWith(rhs FastIntSet) { - s.small &= rhs.small + s.small.IntersectionWith(rhs.small) if rhs.large == nil { s.large = nil } @@ -265,7 +273,7 @@ func (s FastIntSet) Intersection(rhs FastIntSet) FastIntSet { // Intersects returns true if s has any elements in common with rhs. func (s FastIntSet) Intersects(rhs FastIntSet) bool { - if (s.small & rhs.small) != 0 { + if s.small.Intersects(rhs.small) { return true } if s.large == nil || rhs.large == nil { @@ -276,7 +284,7 @@ func (s FastIntSet) Intersects(rhs FastIntSet) bool { // DifferenceWith removes any elements in rhs from this set. func (s *FastIntSet) DifferenceWith(rhs FastIntSet) { - s.small &^= rhs.small + s.small.DifferenceWith(rhs.small) if s.large == nil { // Fast path return @@ -308,13 +316,13 @@ func (s FastIntSet) Equals(rhs FastIntSet) bool { // SubsetOf returns true if rhs contains all the elements in s. func (s FastIntSet) SubsetOf(rhs FastIntSet) bool { if s.fitsInSmall() { - return (s.small & rhs.small) == s.small + return s.small.SubsetOf(rhs.small) } - if rhs.large != nil { - return s.large.SubsetOf(rhs.large) + if rhs.fitsInSmall() { + // s doesn't fit in small and rhs does. + return false } - // s doesn't fit in small and rhs is "small". - return false + return s.large.SubsetOf(rhs.large) } // Encode the set and write it to a bytes.Buffer using binary.varint byte @@ -337,9 +345,9 @@ func (s *FastIntSet) Encode(buf *bytes.Buffer) error { //gcassert:noescape tmp := make([]byte, binary.MaxVarintLen64+1) - if s.fitsInSmall() { + if s.small.hi == 0 && s.fitsInSmall() { n := binary.PutUvarint(tmp, 0) - n += binary.PutUvarint(tmp[n:], s.small) + n += binary.PutUvarint(tmp[n:], s.small.lo) buf.Write(tmp[:n]) } else { n := binary.PutUvarint(tmp, uint64(s.Len())) @@ -359,18 +367,15 @@ func (s *FastIntSet) Decode(br io.ByteReader) error { if err != nil { return err } - s.small = 0 - if s.large != nil { - s.large.Clear() - } + *s = FastIntSet{} if length == 0 { - // Special case: the bitmap is encoded directly. + // Special case: a 64-bit bitmap is encoded directly. val, err := binary.ReadUvarint(br) if err != nil { return err } - s.small = val + s.small.lo = val } else { for i := 0; i < int(length); i++ { elem, err := binary.ReadUvarint(br) @@ -383,3 +388,84 @@ func (s *FastIntSet) Decode(br io.ByteReader) error { } return nil } + +func (v bitmap) IsSet(i int) bool { + w := v.lo + if i >= 64 { + w = v.hi + } + return w&(1<= 64: + v.hi |= mask(from&63, to&63) + default: + v.lo |= mask(from, 63) + v.hi |= mask(0, to&63) + } +} + +func (v *bitmap) UnionWith(other bitmap) { + v.lo |= other.lo + v.hi |= other.hi +} + +func (v *bitmap) IntersectionWith(other bitmap) { + v.lo &= other.lo + v.hi &= other.hi +} + +func (v bitmap) Intersects(other bitmap) bool { + return ((v.lo & other.lo) | (v.hi & other.hi)) != 0 +} + +func (v *bitmap) DifferenceWith(other bitmap) { + v.lo &^= other.lo + v.hi &^= other.hi +} + +func (v bitmap) SubsetOf(other bitmap) bool { + return (v.lo&other.lo == v.lo) && (v.hi&other.hi == v.hi) +} + +func (v bitmap) OnesCount() int { + return bits.OnesCount64(v.lo) + bits.OnesCount64(v.hi) +} + +func (v bitmap) Next(startVal int) (nextVal int, ok bool) { + if startVal < 64 { + if ntz := bits.TrailingZeros64(v.lo >> uint64(startVal)); ntz < 64 { + // Found next element in the low word. + return startVal + ntz, true + } + startVal = 64 + } + // Check high word. + if ntz := bits.TrailingZeros64(v.hi >> uint64(startVal&63)); ntz < 64 { + return startVal + ntz, true + } + return -1, false +} diff --git a/pkg/util/fast_int_set_testonly.go b/pkg/util/fast_int_set_testonly.go index 7116503451a0..da522a754364 100644 --- a/pkg/util/fast_int_set_testonly.go +++ b/pkg/util/fast_int_set_testonly.go @@ -31,7 +31,7 @@ import ( // allocations when the values are small. It is not thread-safe. type FastIntSet struct { // Used to keep the size of the struct the same. - _ uint64 + _ struct{ lo, hi uint64 } s *intsets.Sparse } diff --git a/pkg/util/log/BUILD.bazel b/pkg/util/log/BUILD.bazel index 4541c4cc94c7..64eee95868fa 100644 --- a/pkg/util/log/BUILD.bazel +++ b/pkg/util/log/BUILD.bazel @@ -37,6 +37,7 @@ go_library( "log_flush.go", "redact.go", "registry.go", + "server_ident.go", "sinks.go", "stderr_redirect.go", "stderr_redirect_unix.go", diff --git a/pkg/util/log/ambient_context.go b/pkg/util/log/ambient_context.go index c90393a4b6c3..e92c19dfc7ae 100644 --- a/pkg/util/log/ambient_context.go +++ b/pkg/util/log/ambient_context.go @@ -54,6 +54,10 @@ type AmbientContext struct { // Tracer is used to open spans (see AnnotateCtxWithSpan). Tracer *tracing.Tracer + // ServerIDs will be embedded into contexts that don't already have + // one. + ServerIDs ServerIdentificationPayload + // eventLog will be embedded into contexts that don't already have an event // log or an open span (if not nil). eventLog *ctxEventLog @@ -136,6 +140,9 @@ func (ac *AmbientContext) ResetAndAnnotateCtx(ctx context.Context) context.Conte if ac.tags != nil { ctx = logtags.WithTags(ctx, ac.tags) } + if ac.ServerIDs != nil { + ctx = context.WithValue(ctx, ServerIdentificationContextKey{}, ac.ServerIDs) + } return ctx } } @@ -147,6 +154,9 @@ func (ac *AmbientContext) annotateCtxInternal(ctx context.Context) context.Conte if ac.tags != nil { ctx = logtags.AddTags(ctx, ac.tags) } + if ac.ServerIDs != nil && ctx.Value(ServerIdentificationContextKey{}) == nil { + ctx = context.WithValue(ctx, ServerIdentificationContextKey{}, ac.ServerIDs) + } return ctx } @@ -171,6 +181,9 @@ func (ac *AmbientContext) AnnotateCtxWithSpan( if ac.tags != nil { ctx = logtags.AddTags(ctx, ac.tags) } + if ac.ServerIDs != nil && ctx.Value(ServerIdentificationContextKey{}) == nil { + ctx = context.WithValue(ctx, ServerIdentificationContextKey{}, ac.ServerIDs) + } } return tracing.EnsureChildSpan(ctx, ac.Tracer, opName) diff --git a/pkg/util/log/clog.go b/pkg/util/log/clog.go index 909153e78ba4..6542840cb425 100644 --- a/pkg/util/log/clog.go +++ b/pkg/util/log/clog.go @@ -21,12 +21,10 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/cli/exit" - "github.com/cockroachdb/cockroach/pkg/util/log/channel" "github.com/cockroachdb/cockroach/pkg/util/log/logpb" "github.com/cockroachdb/cockroach/pkg/util/log/severity" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/errors" - "github.com/cockroachdb/logtags" ) // logging is the global state of the logging setup. @@ -102,28 +100,10 @@ type loggingT struct { firstUseStack string } - idMu struct { - syncutil.RWMutex - idPayload - } - allSinkInfos sinkInfoRegistry allLoggers loggerRegistry } -type idPayload struct { - // the Cluster ID is reported on every new log file so as to ease - // the correlation of panic reports with self-reported log files. - clusterID string - // the node ID is reported like the cluster ID, for the same reasons. - // We avoid using roahcpb.NodeID to avoid a circular reference. - nodeID int32 - // ditto for the tenant ID. - tenantID string - // ditto for the SQL instance ID. - sqlInstanceID int32 -} - func init() { logging.bufPool.New = newBuffer logging.bufSlicePool.New = newBufferSlice @@ -221,12 +201,6 @@ func FatalChan() <-chan struct{} { return logging.mu.fatalCh } -func (l *loggingT) idPayload() idPayload { - l.idMu.RLock() - defer l.idMu.RUnlock() - return l.idMu.idPayload -} - // s ignalFatalCh signals the listeners of l.mu.fatalCh by closing the // channel. // l.mu is not held. @@ -242,50 +216,6 @@ func (l *loggingT) signalFatalCh() { } } -// SetNodeIDs stores the Node and Cluster ID for further reference. -func SetNodeIDs(clusterID string, nodeID int32) { - // Ensure that the IDs are logged with the same format as for - // new log files, even on the first log file. This ensures that grep - // will always find it. - ctx := logtags.AddTag(context.Background(), "config", nil) - logfDepth(ctx, 1, severity.INFO, channel.OPS, "clusterID: %s", clusterID) - if nodeID != 0 { - logfDepth(ctx, 1, severity.INFO, channel.OPS, "nodeID: n%d", nodeID) - } - - // Perform the change proper. - logging.idMu.Lock() - defer logging.idMu.Unlock() - - if logging.idMu.clusterID != "" { - panic("clusterID already set") - } - - logging.idMu.clusterID = clusterID - logging.idMu.nodeID = nodeID -} - -// SetTenantIDs stores the tenant ID and instance ID for further reference. -func SetTenantIDs(tenantID string, sqlInstanceID int32) { - // Ensure that the IDs are logged with the same format as for - // new log files, even on the first log file. This ensures that grep - // will always find it. - ctx := logtags.AddTag(context.Background(), "config", nil) - logfDepth(ctx, 1, severity.INFO, channel.OPS, "tenantID: %s", tenantID) - logfDepth(ctx, 1, severity.INFO, channel.OPS, "instanceID: %d", sqlInstanceID) - - // Perform the change proper. - logging.idMu.Lock() - defer logging.idMu.Unlock() - - if logging.idMu.tenantID != "" { - panic("tenantID already set") - } - - logging.idMu.tenantID = tenantID - logging.idMu.sqlInstanceID = sqlInstanceID -} - // outputLogEntry marshals a log entry proto into bytes, and writes // the data to the log files. If a trace location is set, stack traces // are added to the entry before marshaling. diff --git a/pkg/util/log/clog_test.go b/pkg/util/log/clog_test.go index b15a1df847f7..dfe4b47ec942 100644 --- a/pkg/util/log/clog_test.go +++ b/pkg/util/log/clog_test.go @@ -742,9 +742,9 @@ func BenchmarkLogEntry_String(b *testing.B) { entry := &logEntry{ idPayload: idPayload{ clusterID: "fooo", - nodeID: 10, + nodeID: "10", tenantID: "12", - sqlInstanceID: 9, + sqlInstanceID: "9", }, ts: timeutil.Now().UnixNano(), header: false, diff --git a/pkg/util/log/flags.go b/pkg/util/log/flags.go index 34b476bbc697..22c76439445c 100644 --- a/pkg/util/log/flags.go +++ b/pkg/util/log/flags.go @@ -461,16 +461,6 @@ func (l *sinkInfo) describeAppliedConfig() (c logconfig.CommonSinkConfig) { return c } -// TestingClearServerIdentifiers clears the server identity from the -// logging system. This is for use in tests that start multiple -// servers with conflicting identities subsequently. -// See discussion here: https://github.com/cockroachdb/cockroach/issues/58938 -func TestingClearServerIdentifiers() { - logging.idMu.Lock() - logging.idMu.idPayload = idPayload{} - logging.idMu.Unlock() -} - // TestingResetActive clears the active bit. This is for use in tests // that use stderr redirection alongside other tests that use // logging. diff --git a/pkg/util/log/format_json.go b/pkg/util/log/format_json.go index dcbf4d82fbdf..a324b9ae8fe9 100644 --- a/pkg/util/log/format_json.go +++ b/pkg/util/log/format_json.go @@ -274,26 +274,23 @@ func formatJSON(entry logEntry, forFluent bool, tags tagChoice) *buffer { escapeString(buf, entry.clusterID) buf.WriteByte('"') } - if entry.nodeID != 0 { + if entry.nodeID != "" { buf.WriteString(`,"`) buf.WriteString(jtags['N'].tags[tags]) buf.WriteString(`":`) - n = buf.someDigits(0, int(entry.nodeID)) - buf.Write(buf.tmp[:n]) + buf.WriteString(entry.nodeID) } if entry.tenantID != "" { buf.WriteString(`,"`) buf.WriteString(jtags['T'].tags[tags]) - buf.WriteString(`":"`) - escapeString(buf, entry.tenantID) - buf.WriteByte('"') + buf.WriteString(`":`) + buf.WriteString(entry.tenantID) } - if entry.sqlInstanceID != 0 { + if entry.sqlInstanceID != "" { buf.WriteString(`,"`) buf.WriteString(jtags['q'].tags[tags]) buf.WriteString(`":`) - n = buf.someDigits(0, int(entry.sqlInstanceID)) - buf.Write(buf.tmp[:n]) + buf.WriteString(entry.sqlInstanceID) } // The binary version. diff --git a/pkg/util/log/format_json_test.go b/pkg/util/log/format_json_test.go index b847748f0f98..51dc50ab12f0 100644 --- a/pkg/util/log/format_json_test.go +++ b/pkg/util/log/format_json_test.go @@ -49,8 +49,8 @@ func TestJSONFormats(t *testing.T) { }(), // Normal (non-header) entries. {}, - {idPayload: idPayload{clusterID: "abc", nodeID: 123}}, - {idPayload: idPayload{tenantID: "abc", sqlInstanceID: 123}}, + {idPayload: idPayload{clusterID: "abc", nodeID: "123"}}, + {idPayload: idPayload{tenantID: "456", sqlInstanceID: "123"}}, makeStructuredEntry(ctx, severity.INFO, channel.DEV, 0, &eventpb.RenameDatabase{ CommonEventDetails: eventpb.CommonEventDetails{ Timestamp: 123, diff --git a/pkg/util/log/log_entry.go b/pkg/util/log/log_entry.go index 3bc3c7282d0f..a8d411fee5c3 100644 --- a/pkg/util/log/log_entry.go +++ b/pkg/util/log/log_entry.go @@ -191,7 +191,7 @@ func makeUnsafePayload(m string) entryPayload { // makeEntry creates a logEntry. func makeEntry(ctx context.Context, s Severity, c Channel, depth int) (res logEntry) { - ids := logging.idPayload() + ids := getIdentificationPayload(ctx) res = logEntry{ idPayload: ids, @@ -285,20 +285,6 @@ func (l *sinkInfo) getStartLines(now time.Time) []*buffer { makeStartLine(f, "arguments: %s", os.Args), ) - ids := logging.idPayload() - if ids.clusterID != "" { - messages = append(messages, makeStartLine(f, "clusterID: %s", logging.idMu.clusterID)) - } - if ids.nodeID != 0 { - messages = append(messages, makeStartLine(f, "nodeID: n%d", logging.idMu.nodeID)) - } - if ids.tenantID != "" { - messages = append(messages, makeStartLine(f, "tenantID: %s", logging.idMu.tenantID)) - } - if ids.sqlInstanceID != 0 { - messages = append(messages, makeStartLine(f, "instanceID: %d", logging.idMu.sqlInstanceID)) - } - // Including a non-ascii character in the first 1024 bytes of the log helps // viewers that attempt to guess the character encoding. messages = append(messages, makeStartLine(f, "log format (utf8=\u2713): %s", Safe(f.formatterName()))) diff --git a/pkg/util/log/server_ident.go b/pkg/util/log/server_ident.go new file mode 100644 index 000000000000..6767925f142a --- /dev/null +++ b/pkg/util/log/server_ident.go @@ -0,0 +1,70 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package log + +import "context" + +// ServerIdentificationContextKey is the type of a context.Value key +// used to carry ServerIdentificationPayload values. +type ServerIdentificationContextKey struct{} + +// ServerIdentificationPayload is the type of a context.Value payload +// associated with a ServerIdentificationContextKey. +type ServerIdentificationPayload interface { + // ServerIdentityString retrieves an identifier corresponding to the + // given retrieval key. If there is no value known for a given key, + // the method can return the empty string. + ServerIdentityString(key ServerIdentificationKey) string +} + +// ServerIdentificationKey represents a possible parameter to the +// ServerIdentityString() method in ServerIdentificationPayload. +type ServerIdentificationKey int + +const ( + // IdentifyClusterID retrieves the cluster ID of the server. + IdentifyClusterID ServerIdentificationKey = iota + // IdentifyKVNodeID retrieves the KV node ID of the server. + IdentifyKVNodeID + // IdentifyInstanceID retrieves the SQL instance ID of the server. + IdentifyInstanceID + // IdentifyTenantID retrieves the tenant ID of the server. + IdentifyTenantID +) + +type idPayload struct { + // the Cluster ID is reported on every new log file so as to ease + // the correlation of panic reports with self-reported log files. + clusterID string + // the node ID is reported like the cluster ID, for the same reasons. + // We avoid using roahcpb.NodeID to avoid a circular reference. + nodeID string + // ditto for the tenant ID. + tenantID string + // ditto for the SQL instance ID. + sqlInstanceID string +} + +func getIdentificationPayload(ctx context.Context) (res idPayload) { + r := ctx.Value(ServerIdentificationContextKey{}) + if r == nil { + return res + } + si, ok := r.(ServerIdentificationPayload) + if !ok { + return res + } + res.clusterID = si.ServerIdentityString(IdentifyClusterID) + res.nodeID = si.ServerIdentityString(IdentifyKVNodeID) + res.sqlInstanceID = si.ServerIdentityString(IdentifyInstanceID) + res.tenantID = si.ServerIdentityString(IdentifyTenantID) + return res +} diff --git a/pkg/util/log/test_log_scope.go b/pkg/util/log/test_log_scope.go index d2164b97db17..c5e0d1d0ffc5 100644 --- a/pkg/util/log/test_log_scope.go +++ b/pkg/util/log/test_log_scope.go @@ -160,10 +160,6 @@ func newLogScope(t tShim, mostlyInline bool) (sc *TestLogScope) { // destination directory. cfg := getTestConfig(&sc.logDir, mostlyInline) - // Reset the server identifiers, so that new servers - // can report their IDs through logging. - TestingClearServerIdentifiers() - // Switch to the new configuration. TestingResetActive() sc.cleanupFn, err = ApplyConfig(cfg) diff --git a/pkg/util/log/testdata/json b/pkg/util/log/testdata/json index 51ab443e24e9..e3c388daba3d 100644 --- a/pkg/util/log/testdata/json +++ b/pkg/util/log/testdata/json @@ -16,10 +16,10 @@ json-fluent-compact: {"tag":"logtest.dev","c":0,"t":"1136214245.654321000","x":" json-compact: {"c":0,"t":"1136214245.654321000","x":"abc","N":123,"s":0,"g":11,"f":"","l":123,"n":0,"r":0,"message":""} json: {"channel_numeric":0,"channel":"DEV","timestamp":"1136214245.654321000","cluster_id":"abc","node_id":123,"severity_numeric":0,"severity":"UNKNOWN","goroutine":11,"file":"","line":123,"entry_counter":0,"redactable":0,"message":""} # -json-fluent-compact: {"tag":"logtest.dev","c":0,"t":"1136214245.654321000","T":"abc","q":123,"s":0,"g":11,"f":"","l":123,"n":0,"r":0,"message":""} - json-fluent: {"tag":"logtest.dev","channel_numeric":0,"channel":"DEV","timestamp":"1136214245.654321000","tenant_id":"abc","instance_id":123,"severity_numeric":0,"severity":"UNKNOWN","goroutine":11,"file":"","line":123,"entry_counter":0,"redactable":0,"message":""} - json-compact: {"c":0,"t":"1136214245.654321000","T":"abc","q":123,"s":0,"g":11,"f":"","l":123,"n":0,"r":0,"message":""} - json: {"channel_numeric":0,"channel":"DEV","timestamp":"1136214245.654321000","tenant_id":"abc","instance_id":123,"severity_numeric":0,"severity":"UNKNOWN","goroutine":11,"file":"","line":123,"entry_counter":0,"redactable":0,"message":""} +json-fluent-compact: {"tag":"logtest.dev","c":0,"t":"1136214245.654321000","T":456,"q":123,"s":0,"g":11,"f":"","l":123,"n":0,"r":0,"message":""} + json-fluent: {"tag":"logtest.dev","channel_numeric":0,"channel":"DEV","timestamp":"1136214245.654321000","tenant_id":456,"instance_id":123,"severity_numeric":0,"severity":"UNKNOWN","goroutine":11,"file":"","line":123,"entry_counter":0,"redactable":0,"message":""} + json-compact: {"c":0,"t":"1136214245.654321000","T":456,"q":123,"s":0,"g":11,"f":"","l":123,"n":0,"r":0,"message":""} + json: {"channel_numeric":0,"channel":"DEV","timestamp":"1136214245.654321000","tenant_id":456,"instance_id":123,"severity_numeric":0,"severity":"UNKNOWN","goroutine":11,"file":"","line":123,"entry_counter":0,"redactable":0,"message":""} # json-fluent-compact: {"tag":"logtest.dev","c":0,"t":"1136214245.654321000","v":"v999.0.0","s":1,"sev":"I","g":11,"f":"util/log/format_json_test.go","l":123,"n":0,"r":1,"tags":{"noval":"","s":"‹1›","long":"‹2›"},"event":{"Timestamp":123,"EventType":"rename_database","DatabaseName":"‹hello›","NewDatabaseName":"‹world›"}} json-fluent: {"tag":"logtest.dev","channel_numeric":0,"channel":"DEV","timestamp":"1136214245.654321000","version":"v999.0.0","severity_numeric":1,"severity":"INFO","goroutine":11,"file":"util/log/format_json_test.go","line":123,"entry_counter":0,"redactable":1,"tags":{"noval":"","s":"‹1›","long":"‹2›"},"event":{"Timestamp":123,"EventType":"rename_database","DatabaseName":"‹hello›","NewDatabaseName":"‹world›"}}