From e96602b14103d8fda00a826ee6dac41546350a33 Mon Sep 17 00:00:00 2001 From: Rebecca Taft Date: Wed, 3 Aug 2022 08:25:47 -0500 Subject: [PATCH] sql: add join types and algorithms to telemetry logging This commit adds several new fields to the SampledQuery structure used for telemetry logging. Closes #85425 Release note (sql change): The structured payloads used for telemetry logs now include the following new fields: `InnerJoinCount`: The number of inner joins in the query plan. `LeftOuterJoinCount`: The number of left (or right) outer joins in the query plan. `FullOuterJoinCount`: The number of full outer joins in the query plan. `SemiJoinCount`: The number of semi joins in the query plan. `AntiJoinCount`: The number of anti joins in the query plan. `IntersectAllJoinCount`: The number of intersect all joins in the query plan. `ExceptAllJoinCount`: The number of except all joins in the query plan. `HashJoinCount`: The number of hash joins in the query plan. `CrossJoinCount`: The number of cross joins in the query plan. `IndexJoinCount`: The number of index joins in the query plan. `LookupJoinCount`: The number of lookup joins in the query plan. `MergeJoinCount`: The number of merge joins in the query plan. `InvertedJoinCount`: The number of inverted joins in the query plan. `ApplyJoinCount`: The number of apply joins in the query plan. `ZigZagJoinCount`: The number of zig zag joins in the query plan. --- docs/generated/eventlog.md | 15 ++ pkg/sql/exec_log.go | 16 ++ pkg/sql/instrumentation.go | 9 + pkg/sql/opt/exec/execbuilder/builder.go | 9 + pkg/sql/opt/exec/execbuilder/relational.go | 70 +++++- pkg/sql/opt/exec/factory.go | 16 ++ pkg/sql/plan_opt.go | 4 + pkg/sql/telemetry_logging_test.go | 236 ++++++++++++++++++ pkg/util/log/eventpb/json_encode_generated.go | 135 ++++++++++ pkg/util/log/eventpb/telemetry.proto | 45 ++++ 10 files changed, 550 insertions(+), 5 deletions(-) diff --git a/docs/generated/eventlog.md b/docs/generated/eventlog.md index 5b307f2661ff..8d1c08c58b12 100644 --- a/docs/generated/eventlog.md +++ b/docs/generated/eventlog.md @@ -2515,6 +2515,21 @@ contains common SQL event/execution details. | `BytesRead` | The number of bytes read from disk. | no | | `RowsRead` | The number of rows read from disk. | no | | `RowsWritten` | The number of rows written. | no | +| `InnerJoinCount` | The number of inner joins in the query plan. | no | +| `LeftOuterJoinCount` | The number of left (or right) outer joins in the query plan. | no | +| `FullOuterJoinCount` | The number of full outer joins in the query plan. | no | +| `SemiJoinCount` | The number of semi joins in the query plan. | no | +| `AntiJoinCount` | The number of anti joins in the query plan. | no | +| `IntersectAllJoinCount` | The number of intersect all joins in the query plan. | no | +| `ExceptAllJoinCount` | The number of except all joins in the query plan. | no | +| `HashJoinCount` | The number of hash joins in the query plan. | no | +| `CrossJoinCount` | The number of cross joins in the query plan. | no | +| `IndexJoinCount` | The number of index joins in the query plan. | no | +| `LookupJoinCount` | The number of lookup joins in the query plan. | no | +| `MergeJoinCount` | The number of merge joins in the query plan. | no | +| `InvertedJoinCount` | The number of inverted joins in the query plan. | no | +| `ApplyJoinCount` | The number of apply joins in the query plan. | no | +| `ZigZagJoinCount` | The number of zig zag joins in the query plan. | no | #### Common fields diff --git a/pkg/sql/exec_log.go b/pkg/sql/exec_log.go index f789c6c86e89..576ceaeaf3e0 100644 --- a/pkg/sql/exec_log.go +++ b/pkg/sql/exec_log.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/sql/catalog" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/opt/exec" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/privilege" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -409,6 +410,21 @@ func (p *planner) maybeLogStatementInternal( BytesRead: queryStats.bytesRead, RowsRead: queryStats.rowsRead, RowsWritten: queryStats.rowsWritten, + InnerJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.InnerJoin]), + LeftOuterJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.LeftOuterJoin]), + FullOuterJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.FullOuterJoin]), + SemiJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.LeftSemiJoin]), + AntiJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.LeftAntiJoin]), + IntersectAllJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.IntersectAllJoin]), + ExceptAllJoinCount: int64(p.curPlan.instrumentation.joinTypeCounts[descpb.ExceptAllJoin]), + HashJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.HashJoin]), + CrossJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.CrossJoin]), + IndexJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.IndexJoin]), + LookupJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.LookupJoin]), + MergeJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.MergeJoin]), + InvertedJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.InvertedJoin]), + ApplyJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.ApplyJoin]), + ZigZagJoinCount: int64(p.curPlan.instrumentation.joinAlgorithmCounts[exec.ZigZagJoin]), } p.logOperationalEventsOnlyExternally(ctx, eventLogEntry{event: &sampledQuery}) } else { diff --git a/pkg/sql/instrumentation.go b/pkg/sql/instrumentation.go index c5f0b4c1dfe9..d698bbf8c738 100644 --- a/pkg/sql/instrumentation.go +++ b/pkg/sql/instrumentation.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/execstats" "github.com/cockroachdb/cockroach/pkg/sql/opt/exec" @@ -150,6 +151,14 @@ type instrumentationHelper struct { // nanosSinceStatsCollected is the maximum number of nanoseconds that have // passed since stats were collected on any table scanned by this query. nanosSinceStatsCollected time.Duration + + // joinTypeCounts records the number of times each type of logical join was + // used in the query. + joinTypeCounts map[descpb.JoinType]int + + // joinAlgorithmCounts records the number of times each type of join algorithm + // was used in the query. + joinAlgorithmCounts map[exec.JoinAlgorithm]int } // outputMode indicates how the statement output needs to be populated (for diff --git a/pkg/sql/opt/exec/execbuilder/builder.go b/pkg/sql/opt/exec/execbuilder/builder.go index 8b853943402e..f891472aeafa 100644 --- a/pkg/sql/opt/exec/execbuilder/builder.go +++ b/pkg/sql/opt/exec/execbuilder/builder.go @@ -13,6 +13,7 @@ package execbuilder import ( "time" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/exec" @@ -143,6 +144,14 @@ type Builder struct { // passed since stats were collected on any table scanned by this query. NanosSinceStatsCollected time.Duration + // JoinTypeCounts records the number of times each type of logical join was + // used in the query. + JoinTypeCounts map[descpb.JoinType]int + + // JoinAlgorithmCounts records the number of times each type of join algorithm + // was used in the query. + JoinAlgorithmCounts map[exec.JoinAlgorithm]int + // wrapFunctionOverride overrides default implementation to return resolvable // function reference for function with specified function name. // The default can be overridden by calling SetSearchPath method to provide diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index 22bd447b4b62..cd46aea2ed47 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -1044,6 +1044,8 @@ func (b *Builder) buildApplyJoin(join memo.RelExpr) (execPlan, error) { ep := execPlan{outputCols: outputCols} + b.recordJoinType(joinType) + b.recordJoinAlgorithm(exec.ApplyJoin) ep.root, err = b.factory.ConstructApplyJoin( joinType, leftPlan.root, @@ -1128,11 +1130,12 @@ func (b *Builder) buildHashJoin(join memo.RelExpr) (execPlan, error) { rightExpr.Relational().OutputCols, *filters, ) + isCrossJoin := len(leftEq) == 0 if !b.disableTelemetry { - if len(leftEq) > 0 { - telemetry.Inc(sqltelemetry.JoinAlgoHashUseCounter) - } else { + if isCrossJoin { telemetry.Inc(sqltelemetry.JoinAlgoCrossUseCounter) + } else { + telemetry.Inc(sqltelemetry.JoinAlgoHashUseCounter) } telemetry.Inc(opt.JoinTypeToUseCounter(join.Op())) } @@ -1160,6 +1163,12 @@ func (b *Builder) buildHashJoin(join memo.RelExpr) (execPlan, error) { leftEqColsAreKey := leftExpr.Relational().FuncDeps.ColsAreStrictKey(leftEq.ToSet()) rightEqColsAreKey := rightExpr.Relational().FuncDeps.ColsAreStrictKey(rightEq.ToSet()) + b.recordJoinType(joinType) + if isCrossJoin { + b.recordJoinAlgorithm(exec.CrossJoin) + } else { + b.recordJoinAlgorithm(exec.HashJoin) + } ep.root, err = b.factory.ConstructHashJoin( joinType, left.root, right.root, @@ -1214,6 +1223,8 @@ func (b *Builder) buildMergeJoin(join *memo.MergeJoinExpr) (execPlan, error) { reqOrd := ep.reqOrdering(join) leftEqColsAreKey := leftExpr.Relational().FuncDeps.ColsAreStrictKey(leftEq.ColSet()) rightEqColsAreKey := rightExpr.Relational().FuncDeps.ColsAreStrictKey(rightEq.ColSet()) + b.recordJoinType(joinType) + b.recordJoinAlgorithm(exec.MergeJoin) ep.root, err = b.factory.ConstructMergeJoin( joinType, left.root, right.root, @@ -1542,6 +1553,13 @@ func (b *Builder) buildSetOp(set memo.RelExpr) (execPlan, error) { panic(errors.AssertionFailedf("invalid operator %s", redact.Safe(set.Op()))) } + switch typ { + case tree.IntersectOp: + b.recordJoinType(descpb.IntersectAllJoin) + case tree.ExceptOp: + b.recordJoinType(descpb.ExceptAllJoin) + } + hardLimit := uint64(0) if set.Op() == opt.LocalityOptimizedSearchOp { if !b.disableTelemetry { @@ -1568,11 +1586,17 @@ func (b *Builder) buildSetOp(set memo.RelExpr) (execPlan, error) { if typ == tree.UnionOp && all { ep.root, err = b.factory.ConstructUnionAll(left.root, right.root, reqOrdering, hardLimit) } else if len(streamingOrdering) > 0 { + if typ != tree.UnionOp { + b.recordJoinAlgorithm(exec.MergeJoin) + } ep.root, err = b.factory.ConstructStreamingSetOp(typ, all, left.root, right.root, streamingOrdering, reqOrdering) } else { if len(reqOrdering) > 0 { return execPlan{}, errors.AssertionFailedf("hash set op is not supported with a required ordering") } + if typ != tree.UnionOp { + b.recordJoinAlgorithm(exec.HashJoin) + } ep.root, err = b.factory.ConstructHashSetOp(typ, all, left.root, right.root) } if err != nil { @@ -1732,6 +1756,7 @@ func (b *Builder) buildIndexJoin(join *memo.IndexJoinExpr) (execPlan, error) { } res := execPlan{outputCols: output} + b.recordJoinAlgorithm(exec.IndexJoin) res.root, err = b.factory.ConstructIndexJoin( input.root, tab, keyCols, needed, res.reqOrdering(join), locking, join.RequiredPhysical().LimitHintInt64(), ) @@ -1827,8 +1852,11 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { locking = forUpdateLocking } + joinType := joinOpToJoinType(join.JoinType) + b.recordJoinType(joinType) + b.recordJoinAlgorithm(exec.LookupJoin) res.root, err = b.factory.ConstructLookupJoin( - joinOpToJoinType(join.JoinType), + joinType, input.root, tab, idx, @@ -1942,8 +1970,11 @@ func (b *Builder) buildInvertedJoin(join *memo.InvertedJoinExpr) (execPlan, erro locking = forUpdateLocking } + joinType := joinOpToJoinType(join.JoinType) + b.recordJoinType(joinType) + b.recordJoinAlgorithm(exec.InvertedJoin) res.root, err = b.factory.ConstructInvertedJoin( - joinOpToJoinType(join.JoinType), + joinType, invertedExpr, input.root, tab, @@ -2043,6 +2074,7 @@ func (b *Builder) buildZigzagJoin(join *memo.ZigzagJoinExpr) (execPlan, error) { return execPlan{}, err } + b.recordJoinAlgorithm(exec.ZigZagJoin) res.root, err = b.factory.ConstructZigzagJoin( leftTable, leftIndex, @@ -2662,6 +2694,34 @@ func (b *Builder) statementTag(expr memo.RelExpr) string { } } +// recordJoinType increments the counter for the given join type for telemetry +// reporting. +func (b *Builder) recordJoinType(joinType descpb.JoinType) { + if b.JoinTypeCounts == nil { + const numJoinTypes = 7 + b.JoinTypeCounts = make(map[descpb.JoinType]int, numJoinTypes) + } + // Don't bother distinguishing between left and right. + switch joinType { + case descpb.RightOuterJoin: + joinType = descpb.LeftOuterJoin + case descpb.RightSemiJoin: + joinType = descpb.LeftSemiJoin + case descpb.RightAntiJoin: + joinType = descpb.LeftAntiJoin + } + b.JoinTypeCounts[joinType]++ +} + +// recordJoinAlgorithm increments the counter for the given join algorithm for +// telemetry reporting. +func (b *Builder) recordJoinAlgorithm(joinAlgorithm exec.JoinAlgorithm) { + if b.JoinAlgorithmCounts == nil { + b.JoinAlgorithmCounts = make(map[exec.JoinAlgorithm]int, exec.NumJoinAlgorithms) + } + b.JoinAlgorithmCounts[joinAlgorithm]++ +} + // boundedStalenessAllowList contains the operators that may be used with // bounded staleness queries. var boundedStalenessAllowList = map[opt.Operator]struct{}{ diff --git a/pkg/sql/opt/exec/factory.go b/pkg/sql/opt/exec/factory.go index a5794c94fddc..b3acbaa4abad 100644 --- a/pkg/sql/opt/exec/factory.go +++ b/pkg/sql/opt/exec/factory.go @@ -364,3 +364,19 @@ const ( // Streaming means that the grouping columns are fully ordered. Streaming ) + +// JoinAlgorithm is the type of join algorithm used. +type JoinAlgorithm int8 + +// The following are all the supported join algorithms. +const ( + HashJoin JoinAlgorithm = iota + CrossJoin + IndexJoin + LookupJoin + MergeJoin + InvertedJoin + ApplyJoin + ZigZagJoin + NumJoinAlgorithms +) diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index 56e63e75dd24..c738c8918408 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -631,6 +631,8 @@ func (opc *optPlanningCtx) runExecBuilder( planTop.instrumentation.maxFullScanRows = bld.MaxFullScanRows planTop.instrumentation.totalScanRows = bld.TotalScanRows planTop.instrumentation.nanosSinceStatsCollected = bld.NanosSinceStatsCollected + planTop.instrumentation.joinTypeCounts = bld.JoinTypeCounts + planTop.instrumentation.joinAlgorithmCounts = bld.JoinAlgorithmCounts } else { // Create an explain factory and record the explain.Plan. explainFactory := explain.NewFactory(f) @@ -652,6 +654,8 @@ func (opc *optPlanningCtx) runExecBuilder( planTop.instrumentation.maxFullScanRows = bld.MaxFullScanRows planTop.instrumentation.totalScanRows = bld.TotalScanRows planTop.instrumentation.nanosSinceStatsCollected = bld.NanosSinceStatsCollected + planTop.instrumentation.joinTypeCounts = bld.JoinTypeCounts + planTop.instrumentation.joinAlgorithmCounts = bld.JoinAlgorithmCounts planTop.instrumentation.RecordExplainPlan(explainPlan) } diff --git a/pkg/sql/telemetry_logging_test.go b/pkg/sql/telemetry_logging_test.go index df20044a295e..41d90427f2b7 100644 --- a/pkg/sql/telemetry_logging_test.go +++ b/pkg/sql/telemetry_logging_test.go @@ -555,3 +555,239 @@ func TestNoTelemetryLogOnTroubleshootMode(t *testing.T) { } } } + +func TestTelemetryLogJoinTypesAndAlgorithms(t *testing.T) { + defer leaktest.AfterTest(t)() + sc := log.ScopeWithoutShowLogs(t) + defer sc.Close(t) + + cleanup := installTelemetryLogFileSink(sc, t) + defer cleanup() + + s, sqlDB, _ := serverutils.StartServer(t, base.TestServerArgs{}) + db := sqlutils.MakeSQLRunner(sqlDB) + defer s.Stopper().Stop(context.Background()) + + db.Exec(t, `SET CLUSTER SETTING sql.telemetry.query_sampling.enabled = true;`) + db.Exec(t, "CREATE TABLE t ("+ + "pk INT PRIMARY KEY,"+ + "col1 INT,"+ + "col2 INT,"+ + "other STRING,"+ + "j JSON,"+ + "INDEX other_index (other),"+ + "INVERTED INDEX j_index (j)"+ + ");") + db.Exec(t, "CREATE TABLE u ("+ + "pk INT PRIMARY KEY,"+ + "fk INT REFERENCES t (pk),"+ + "j JSON,"+ + "INDEX fk_index (fk),"+ + "INVERTED INDEX j_index (j)"+ + ");") + + stubMaxEventFrequency := int64(1000000) + telemetryMaxEventFrequency.Override(context.Background(), &s.ClusterSettings().SV, stubMaxEventFrequency) + + testData := []struct { + name string + query string + expectedLogStatement string + expectedNumLogs int + expectedJoinTypes map[string]int + expectedJoinAlgorithms map[string]int + }{ + { + "no-index-join", + "SELECT * FROM t LIMIT 1;", + `SELECT * FROM \"\".\"\".t LIMIT ‹1›`, + 1, + map[string]int{}, + map[string]int{}, + }, + { + "index-join", + "SELECT * FROM t WHERE other='other';", + `"SELECT * FROM \"\".\"\".t WHERE other = ‹'other'›"`, + 1, + map[string]int{}, + map[string]int{"IndexJoin": 1}, + }, + { + "inner-hash-join", + "SELECT * FROM t INNER HASH JOIN u ON t.pk = u.fk;", + `"SELECT * FROM \"\".\"\".t INNER HASH JOIN \"\".\"\".u ON t.pk = u.fk"`, + 1, + map[string]int{"InnerJoin": 1}, + map[string]int{"HashJoin": 1}, + }, + { + "cross-join", + "SELECT * FROM t CROSS JOIN u", + `"SELECT * FROM \"\".\"\".t CROSS JOIN \"\".\"\".u"`, + 1, + map[string]int{"InnerJoin": 1}, + map[string]int{"CrossJoin": 1}, + }, + { + "left-hash-join", + "SELECT * FROM t LEFT OUTER HASH JOIN u ON t.pk = u.fk;", + `"SELECT * FROM \"\".\"\".t LEFT HASH JOIN \"\".\"\".u ON t.pk = u.fk"`, + 1, + map[string]int{"LeftOuterJoin": 1}, + map[string]int{"HashJoin": 1}, + }, + { + "full-hash-join", + "SELECT * FROM t FULL OUTER HASH JOIN u ON t.pk = u.fk;", + `"SELECT * FROM \"\".\"\".t FULL HASH JOIN \"\".\"\".u ON t.pk = u.fk"`, + 1, + map[string]int{"FullOuterJoin": 1}, + map[string]int{"HashJoin": 1}, + }, + { + "anti-merge-join", + "SELECT * FROM t@t_pkey WHERE NOT EXISTS (SELECT * FROM u@fk_index WHERE t.pk = u.fk);", + `"SELECT * FROM \"\".\"\".t@t_pkey WHERE NOT EXISTS (SELECT * FROM \"\".\"\".u@fk_index WHERE t.pk = u.fk)"`, + 1, + map[string]int{"AntiJoin": 1}, + map[string]int{"MergeJoin": 1}, + }, + { + "inner-lookup-join", + "SELECT * FROM t INNER LOOKUP JOIN u ON t.pk = u.fk;", + `"SELECT * FROM \"\".\"\".t INNER LOOKUP JOIN \"\".\"\".u ON t.pk = u.fk"`, + 1, + map[string]int{"InnerJoin": 2}, + map[string]int{"LookupJoin": 2}, + }, + { + "inner-merge-join", + "SELECT * FROM t INNER MERGE JOIN u ON t.pk = u.fk;", + `"SELECT * FROM \"\".\"\".t INNER MERGE JOIN \"\".\"\".u ON t.pk = u.fk"`, + 1, + map[string]int{"InnerJoin": 1}, + map[string]int{"MergeJoin": 1}, + }, + { + "inner-inverted-join", + "SELECT * FROM t INNER INVERTED JOIN u ON t.j @> u.j;", + `"SELECT * FROM \"\".\"\".t INNER INVERTED JOIN \"\".\"\".u ON t.j @> u.j"`, + 1, + map[string]int{"InnerJoin": 2}, + map[string]int{"InvertedJoin": 1, "LookupJoin": 1}, + }, + { + "semi-apply-join", + "SELECT * FROM t WHERE col1 IN (SELECT generate_series(col1, col2) FROM u);", + `"SELECT * FROM \"\".\"\".t WHERE col1 IN (SELECT generate_series(col1, col2) FROM \"\".\"\".u)"`, + 1, + map[string]int{"SemiJoin": 1}, + map[string]int{"ApplyJoin": 1}, + }, + { + "zig-zag-join", + "SELECT * FROM t@{FORCE_ZIGZAG} WHERE t.j @> '{\"a\":\"b\"}' AND t.j @> '{\"c\":\"d\"}';", + `"SELECT * FROM \"\".\"\".t@{FORCE_ZIGZAG} WHERE (t.j @> ‹'{\"a\":\"b\"}'›) AND (t.j @> ‹'{\"c\":\"d\"}'›)"`, + 1, + map[string]int{"InnerJoin": 1}, + map[string]int{"ZigZagJoin": 1, "LookupJoin": 1}, + }, + { + "intersect-all-merge-join", + "SELECT * FROM (SELECT t.pk FROM t INTERSECT ALL SELECT u.pk FROM u) ORDER BY pk;", + `"SELECT * FROM (SELECT t.pk FROM \"\".\"\".t INTERSECT ALL SELECT u.pk FROM \"\".\"\".u) ORDER BY pk"`, + 1, + map[string]int{"IntersectAllJoin": 1}, + map[string]int{"MergeJoin": 1}, + }, + { + "except-all-hash-join", + "SELECT t.col1 FROM t EXCEPT SELECT u.fk FROM u;", + `"SELECT t.col1 FROM \"\".\"\".t EXCEPT SELECT u.fk FROM \"\".\"\".u"`, + 1, + map[string]int{"ExceptAllJoin": 1}, + map[string]int{"HashJoin": 1}, + }, + { + // UNION is not implemented with a join. + "union", + "SELECT t.col1 FROM t UNION SELECT u.fk FROM u;", + `"SELECT t.col1 FROM \"\".\"\".t UNION SELECT u.fk FROM \"\".\"\".u"`, + 1, + map[string]int{}, + map[string]int{}, + }, + } + + for _, tc := range testData { + db.Exec(t, tc.query) + } + + log.Flush() + + entries, err := log.FetchEntriesFromFiles( + 0, + math.MaxInt64, + 10000, + regexp.MustCompile(`"EventType":"sampled_query"`), + log.WithMarkedSensitiveData, + ) + + if err != nil { + t.Fatal(err) + } + + if len(entries) == 0 { + t.Fatal(errors.Newf("no entries found")) + } + + for _, tc := range testData { + numLogsFound := 0 + for i := len(entries) - 1; i >= 0; i-- { + e := entries[i] + if strings.Contains(e.Message, tc.expectedLogStatement) { + numLogsFound++ + for joinType, count := range tc.expectedJoinTypes { + msg := fmt.Sprintf("\"%sCount\":%d", joinType, count) + containsJoinType := strings.Contains(e.Message, msg) + if !containsJoinType { + t.Errorf("%s: expected %s to be found, but found none in: %s", tc.name, msg, e.Message) + } + } + for _, joinType := range []string{ + "InnerJoin", "LeftOuterJoin", "FullOuterJoin", "SemiJoin", "AntiJoin", "IntersectAllJoin", + "ExceptAllJoin", + } { + if _, ok := tc.expectedJoinTypes[joinType]; !ok { + containsJoinType := strings.Contains(e.Message, joinType) + if containsJoinType { + t.Errorf("%s: unexpected \"%s\" found in: %s", tc.name, joinType, e.Message) + } + } + } + for joinAlg, count := range tc.expectedJoinAlgorithms { + msg := fmt.Sprintf("\"%sCount\":%d", joinAlg, count) + containsJoinAlg := strings.Contains(e.Message, msg) + if !containsJoinAlg { + t.Errorf("%s: expected %s to be found, but found none in: %s", tc.name, msg, e.Message) + } + } + for _, joinAlg := range []string{ + "HashJoin", "CrossJoin", "IndexJoin", "LookupJoin", "MergeJoin", "InvertedJoin", + "ApplyJoin", "ZigZagJoin", + } { + if _, ok := tc.expectedJoinAlgorithms[joinAlg]; !ok { + containsJoinAlg := strings.Contains(e.Message, joinAlg) + if containsJoinAlg { + t.Errorf("%s: unexpected \"%s\" found in: %s", tc.name, joinAlg, e.Message) + } + } + } + } + } + if numLogsFound != tc.expectedNumLogs { + t.Errorf("%s: expected %d log entries, found %d", tc.name, tc.expectedNumLogs, numLogsFound) + } + } +} diff --git a/pkg/util/log/eventpb/json_encode_generated.go b/pkg/util/log/eventpb/json_encode_generated.go index 84a73ff8aef2..1272d0ae8a16 100644 --- a/pkg/util/log/eventpb/json_encode_generated.go +++ b/pkg/util/log/eventpb/json_encode_generated.go @@ -3667,6 +3667,141 @@ func (m *SampledQuery) AppendJSONFields(printComma bool, b redact.RedactableByte b = strconv.AppendInt(b, int64(m.RowsWritten), 10) } + if m.InnerJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"InnerJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.InnerJoinCount), 10) + } + + if m.LeftOuterJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"LeftOuterJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.LeftOuterJoinCount), 10) + } + + if m.FullOuterJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"FullOuterJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.FullOuterJoinCount), 10) + } + + if m.SemiJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"SemiJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.SemiJoinCount), 10) + } + + if m.AntiJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"AntiJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.AntiJoinCount), 10) + } + + if m.IntersectAllJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"IntersectAllJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.IntersectAllJoinCount), 10) + } + + if m.ExceptAllJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"ExceptAllJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.ExceptAllJoinCount), 10) + } + + if m.HashJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"HashJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.HashJoinCount), 10) + } + + if m.CrossJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"CrossJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.CrossJoinCount), 10) + } + + if m.IndexJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"IndexJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.IndexJoinCount), 10) + } + + if m.LookupJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"LookupJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.LookupJoinCount), 10) + } + + if m.MergeJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"MergeJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.MergeJoinCount), 10) + } + + if m.InvertedJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"InvertedJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.InvertedJoinCount), 10) + } + + if m.ApplyJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"ApplyJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.ApplyJoinCount), 10) + } + + if m.ZigZagJoinCount != 0 { + if printComma { + b = append(b, ',') + } + printComma = true + b = append(b, "\"ZigZagJoinCount\":"...) + b = strconv.AppendInt(b, int64(m.ZigZagJoinCount), 10) + } + return printComma, b } diff --git a/pkg/util/log/eventpb/telemetry.proto b/pkg/util/log/eventpb/telemetry.proto index 380e904ea510..e82b74355e76 100644 --- a/pkg/util/log/eventpb/telemetry.proto +++ b/pkg/util/log/eventpb/telemetry.proto @@ -90,6 +90,51 @@ message SampledQuery { // The number of rows written. int64 rows_written = 21 [(gogoproto.jsontag) = ",omitempty"]; + // The number of inner joins in the query plan. + int64 inner_join_count = 22 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of left (or right) outer joins in the query plan. + int64 left_outer_join_count = 23 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of full outer joins in the query plan. + int64 full_outer_join_count = 24 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of semi joins in the query plan. + int64 semi_join_count = 25 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of anti joins in the query plan. + int64 anti_join_count = 26 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of intersect all joins in the query plan. + int64 intersect_all_join_count = 27 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of except all joins in the query plan. + int64 except_all_join_count = 28 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of hash joins in the query plan. + int64 hash_join_count = 29 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of cross joins in the query plan. + int64 cross_join_count = 30 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of index joins in the query plan. + int64 index_join_count = 31 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of lookup joins in the query plan. + int64 lookup_join_count = 32 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of merge joins in the query plan. + int64 merge_join_count = 33 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of inverted joins in the query plan. + int64 inverted_join_count = 34 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of apply joins in the query plan. + int64 apply_join_count = 35 [(gogoproto.jsontag) = ",omitempty"]; + + // The number of zig zag joins in the query plan. + int64 zig_zag_join_count = 36 [(gogoproto.jsontag) = ",omitempty"]; + reserved 12; }