From 64298e410b743ed027e3ed6a13ba71e50e1458c9 Mon Sep 17 00:00:00 2001 From: Rohan Yadav Date: Wed, 18 Sep 2019 15:58:02 -0400 Subject: [PATCH] opt: Add usage statistics to partition constrained scans This PR adds telemetry information to the optimizer where partition information is used to constrain table scans. Work for https://github.com/cockroachlabs/registration/issues/228. Release justification: Low risk monitoring improvement. Release note: None --- .../logic_test/partitioning_constrained_scans | 6 ++++++ pkg/sql/opt/exec/execbuilder/relational.go | 5 +++++ pkg/sql/opt/ops/relational.opt | 5 +++++ pkg/sql/opt/xform/custom_funcs.go | 2 ++ pkg/sql/opt/xform/testdata/rules/groupby | 4 ++-- pkg/sql/opt/xform/testdata/rules/join | 2 +- pkg/sql/opt/xform/testdata/rules/scan | 4 ++-- pkg/sql/opt/xform/testdata/rules/select | 2 +- pkg/sql/sqltelemetry/partitioning.go | 16 +++++++++++----- 9 files changed, 35 insertions(+), 11 deletions(-) diff --git a/pkg/ccl/logictestccl/testdata/logic_test/partitioning_constrained_scans b/pkg/ccl/logictestccl/testdata/logic_test/partitioning_constrained_scans index a9a9eb6afa82..a31b301729aa 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/partitioning_constrained_scans +++ b/pkg/ccl/logictestccl/testdata/logic_test/partitioning_constrained_scans @@ -186,3 +186,9 @@ select │ └── constraint: /1/2/3/4: [ - /'earth'/'us'/'cali') [/'earth'/'us'/'cali'/7 - /'earth'/'us'/'cali'/8] [/'earth'/'us'/e'cali\x00'/7 - /'earth'/'us'/'seatle') [/'earth'/'us'/'seatle'/7 - /'earth'/'us'/'seatle'/8] [/'earth'/'us'/e'seatle\x00'/7 - /'jupiter'/'titan'/'red spot') [/'jupiter'/'titan'/'red spot'/7 - /'jupiter'/'titan'/'red spot'/8] [/'jupiter'/'titan'/e'red spot\x00'/7 - ] └── filters └── (val > 6) AND (val < 9) + +# Test telemetry about this optimization works. +query T +SELECT feature_name FROM crdb_internal.feature_usage WHERE feature_name='sql.partitioning.partition-constrained-scan' AND usage_count > 0 +---- +sql.partitioning.partition-constrained-scan diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index 08e43cae33fd..cfb5ae852069 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/builtins" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" + "github.com/cockroachdb/cockroach/pkg/sql/sqltelemetry" "github.com/cockroachdb/cockroach/pkg/util" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/errorutil/unimplemented" @@ -455,6 +456,10 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) { rowCount = 0 } + if scan.PartitionConstrainedScan { + sqltelemetry.IncrementPartitioningCounter(sqltelemetry.PartitionConstrainedScan) + } + root, err := b.factory.ConstructScan( tab, tab.Index(scan.Index), diff --git a/pkg/sql/opt/ops/relational.opt b/pkg/sql/opt/ops/relational.opt index 8d409ae5f6c4..6d74f1e352fe 100644 --- a/pkg/sql/opt/ops/relational.opt +++ b/pkg/sql/opt/ops/relational.opt @@ -64,6 +64,11 @@ define ScanPrivate { # Flags modify how the table is scanned, such as which index is used to scan. Flags ScanFlags + + # PartitionConstrainedScan records whether or not we were able to use partitions + # to constrain the lookup spans further. This flag is used to record telemetry + # about how often this optimization is getting applied. + PartitionConstrainedScan bool } # VirtualScan returns a result set containing every row in a virtual table. diff --git a/pkg/sql/opt/xform/custom_funcs.go b/pkg/sql/opt/xform/custom_funcs.go index 249713b90532..bf048cf0e38e 100644 --- a/pkg/sql/opt/xform/custom_funcs.go +++ b/pkg/sql/opt/xform/custom_funcs.go @@ -612,6 +612,8 @@ func (c *CustomFuncs) GenerateConstrainedScans( newScanPrivate := *scanPrivate newScanPrivate.Index = iter.indexOrdinal newScanPrivate.Constraint = constraint + // Record whether we were able to use partitions to constrain the scan. + newScanPrivate.PartitionConstrainedScan = isIndexPartitioned // If the alternate index includes the set of needed columns, then construct // a new Scan operator using that index. diff --git a/pkg/sql/opt/xform/testdata/rules/groupby b/pkg/sql/opt/xform/testdata/rules/groupby index 5a9ded2968db..073cb9ed800f 100644 --- a/pkg/sql/opt/xform/testdata/rules/groupby +++ b/pkg/sql/opt/xform/testdata/rules/groupby @@ -681,7 +681,7 @@ memo (optimized, ~5KB, required=[presentation: max:5]) memo SELECT array_agg(w) FROM (SELECT * FROM kuvw ORDER BY w) GROUP BY u,v ---- -memo (optimized, ~5KB, required=[presentation: array_agg:5]) +memo (optimized, ~6KB, required=[presentation: array_agg:5]) ├── G1: (project G2 G3 array_agg) │ └── [presentation: array_agg:5] │ ├── best: (project G2 G3 array_agg) @@ -715,7 +715,7 @@ memo (optimized, ~5KB, required=[presentation: array_agg:5]) memo SELECT sum(w) FROM kuvw GROUP BY u,v,w ---- -memo (optimized, ~5KB, required=[presentation: sum:5]) +memo (optimized, ~6KB, required=[presentation: sum:5]) ├── G1: (project G2 G3 sum) │ └── [presentation: sum:5] │ ├── best: (project G2 G3 sum) diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 13d9bf00e378..d77fe01de0ba 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -653,7 +653,7 @@ left-join (merge) memo SELECT * FROM abc JOIN xyz ON a=b ---- -memo (optimized, ~12KB, required=[presentation: a:1,b:2,c:3,x:5,y:6,z:7]) +memo (optimized, ~13KB, required=[presentation: a:1,b:2,c:3,x:5,y:6,z:7]) ├── G1: (inner-join G2 G3 G4) (inner-join G3 G2 G4) │ └── [presentation: a:1,b:2,c:3,x:5,y:6,z:7] │ ├── best: (inner-join G3 G2 G4) diff --git a/pkg/sql/opt/xform/testdata/rules/scan b/pkg/sql/opt/xform/testdata/rules/scan index 553e8af5332a..a6016c3dc419 100644 --- a/pkg/sql/opt/xform/testdata/rules/scan +++ b/pkg/sql/opt/xform/testdata/rules/scan @@ -289,7 +289,7 @@ memo (optimized, ~2KB, required=[presentation: s:4,i:2,f:3] [ordering: -4,+2]) memo SELECT s, i, f FROM a WHERE s='foo' ORDER BY s DESC, i ---- -memo (optimized, ~5KB, required=[presentation: s:4,i:2,f:3] [ordering: +2 opt(4)]) +memo (optimized, ~6KB, required=[presentation: s:4,i:2,f:3] [ordering: +2 opt(4)]) ├── G1: (select G2 G3) (scan a@s_idx,cols=(2-4),constrained) (index-join G4 a,cols=(2-4)) │ ├── [presentation: s:4,i:2,f:3] [ordering: +2 opt(4)] │ │ ├── best: (sort G1) @@ -367,7 +367,7 @@ memo (optimized, ~3KB, required=[presentation: d:4] [ordering: +5]) memo SELECT j FROM a WHERE s = 'foo' ---- -memo (optimized, ~6KB, required=[presentation: j:5]) +memo (optimized, ~7KB, required=[presentation: j:5]) ├── G1: (project G2 G3 j) │ └── [presentation: j:5] │ ├── best: (project G2 G3 j) diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 5903e2acd8a1..497c7d3f0642 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -68,7 +68,7 @@ project memo SELECT k FROM a WHERE v > 1 ---- -memo (optimized, ~5KB, required=[presentation: k:1]) +memo (optimized, ~6KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) │ └── [presentation: k:1] │ ├── best: (project G2 G3 k) diff --git a/pkg/sql/sqltelemetry/partitioning.go b/pkg/sql/sqltelemetry/partitioning.go index b8f88a353a96..c2870575d354 100644 --- a/pkg/sql/sqltelemetry/partitioning.go +++ b/pkg/sql/sqltelemetry/partitioning.go @@ -16,18 +16,23 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/telemetry" ) -// PartitioningTelemetryType is an enum used to represent the different partitioning related operations -// that we are recording telemetry for. +// PartitioningTelemetryType is an enum used to represent the different +// partitioning related operations that we are recording telemetry for. type PartitioningTelemetryType int const ( _ PartitioningTelemetryType = iota - // AlterAllPartitions represents an ALTER ALL PARTITIONS statement (ALTER PARTITION OF INDEX t@*) + // AlterAllPartitions represents an ALTER ALL PARTITIONS + // statement (ALTER PARTITION OF INDEX t@*) AlterAllPartitions + // PartitionConstrainedScan represents when the optimizer was + // able to use partitioning to constrain a scan. + PartitionConstrainedScan ) var partitioningTelemetryMap = map[PartitioningTelemetryType]string{ - AlterAllPartitions: "alter-all-partitions", + AlterAllPartitions: "alter-all-partitions", + PartitionConstrainedScan: "partition-constrained-scan", } func (p PartitioningTelemetryType) String() string { @@ -43,7 +48,8 @@ func init() { } } -// IncrementPartitioningCounter is used to increment the telemetry counter for a particular partitioning operation. +// IncrementPartitioningCounter is used to increment the telemetry +// counter for a particular partitioning operation. func IncrementPartitioningCounter(partitioningType PartitioningTelemetryType) { telemetry.Inc(partitioningTelemetryCounters[partitioningType]) }