diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 82628f60edc3..b88510812749 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -375,6 +375,7 @@ #!/pkg/ccl/cliccl/ @cockroachdb/unowned /pkg/ccl/cmdccl/stub-schema-registry/ @cockroachdb/cdc-prs /pkg/ccl/cmdccl/clusterrepl/ @cockroachdb/disaster-recovery +/pkg/ccl/gqpccl/ @cockroachdb/sql-queries-prs #!/pkg/ccl/gssapiccl/ @cockroachdb/unowned /pkg/ccl/jwtauthccl/ @cockroachdb/cloud-identity #!/pkg/ccl/kvccl/ @cockroachdb/kv-noreview diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 41f7ac5229ec..f8a15a24ddc4 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -866,6 +866,7 @@ GO_TARGETS = [ "//pkg/ccl/cmdccl/clusterrepl:clusterrepl_lib", "//pkg/ccl/cmdccl/stub-schema-registry:stub-schema-registry", "//pkg/ccl/cmdccl/stub-schema-registry:stub-schema-registry_lib", + "//pkg/ccl/gqpccl:gqpccl", "//pkg/ccl/gssapiccl:gssapiccl", "//pkg/ccl/importerccl:importerccl_test", "//pkg/ccl/jobsccl/jobsprotectedtsccl:jobsprotectedtsccl_test", @@ -1880,6 +1881,7 @@ GO_TARGETS = [ "//pkg/sql/gcjob:gcjob", "//pkg/sql/gcjob:gcjob_test", "//pkg/sql/gcjob_test:gcjob_test_test", + "//pkg/sql/gpq:gpq", "//pkg/sql/idxrecommendations:idxrecommendations", "//pkg/sql/idxrecommendations:idxrecommendations_test", "//pkg/sql/idxusage:idxusage", diff --git a/pkg/ccl/BUILD.bazel b/pkg/ccl/BUILD.bazel index 494a6fd5eef6..180b79a553db 100644 --- a/pkg/ccl/BUILD.bazel +++ b/pkg/ccl/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//pkg/ccl/buildccl", "//pkg/ccl/changefeedccl", "//pkg/ccl/cliccl", + "//pkg/ccl/gqpccl", "//pkg/ccl/gssapiccl", "//pkg/ccl/jwtauthccl", "//pkg/ccl/kvccl", diff --git a/pkg/ccl/ccl_init.go b/pkg/ccl/ccl_init.go index fc315a0a087f..d14239571ddc 100644 --- a/pkg/ccl/ccl_init.go +++ b/pkg/ccl/ccl_init.go @@ -18,6 +18,7 @@ import ( _ "github.com/cockroachdb/cockroach/pkg/ccl/buildccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/cliccl" + _ "github.com/cockroachdb/cockroach/pkg/ccl/gqpccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/gssapiccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/kvccl" diff --git a/pkg/ccl/gqpccl/BUILD.bazel b/pkg/ccl/gqpccl/BUILD.bazel new file mode 100644 index 000000000000..03a600bd3f3f --- /dev/null +++ b/pkg/ccl/gqpccl/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "gqpccl", + srcs = ["gpq.go"], + importpath = "github.com/cockroachdb/cockroach/pkg/ccl/gqpccl", + visibility = ["//visibility:public"], + deps = [ + "//pkg/ccl/utilccl", + "//pkg/settings/cluster", + "//pkg/sql/gpq", + ], +) diff --git a/pkg/ccl/gqpccl/gpq.go b/pkg/ccl/gqpccl/gpq.go new file mode 100644 index 000000000000..d841f4691e7c --- /dev/null +++ b/pkg/ccl/gqpccl/gpq.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package gpqccl + +import ( + "github.com/cockroachdb/cockroach/pkg/ccl/utilccl" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/gpq" +) + +func init() { + gpq.CheckClusterSupportsGenericQueryPlans = checkClusterSupportsGenericQueryPlans +} + +func checkClusterSupportsGenericQueryPlans(settings *cluster.Settings) error { + return utilccl.CheckEnterpriseEnabled(settings, "generic query plans") +} diff --git a/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql b/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql index eb56dff76138..42c2a64e95ef 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql +++ b/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql @@ -143,6 +143,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -164,6 +165,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: diff --git a/pkg/ccl/logictestccl/testdata/logic_test/generic b/pkg/ccl/logictestccl/testdata/logic_test/generic new file mode 100644 index 000000000000..239f19396a55 --- /dev/null +++ b/pkg/ccl/logictestccl/testdata/logic_test/generic @@ -0,0 +1,1174 @@ +# LogicTest: local + +# Disable stats collection to prevent automatic stats collection from +# invalidating plans. +statement ok +SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; + +statement ok +CREATE TABLE t ( + k INT PRIMARY KEY, + a INT, + b INT, + c INT, + s STRING, + t TIMESTAMPTZ, + INDEX (a), + INDEX (s), + INDEX (t) +) + +statement ok +CREATE TABLE c ( + k INT PRIMARY KEY, + a INT, + INDEX (a) +) + +statement ok +CREATE TABLE g ( + k INT PRIMARY KEY, + a INT, + b INT, + INDEX (a, b) +) + +statement ok +SET plan_cache_mode = force_generic_plan + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 + +# An ideal generic plan can be built during PREPARE for queries with no +# placeholders nor stable expressions. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +SET plan_cache_mode = force_custom_plan + +# The ideal generic plan is reused even when forcing a custom plan because it +# will always be optimal. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +DEALLOCATE p + +# Prepare the same query with plan_cache_mode set to force_custom_plan. +statement ok +PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 + +# Execute it once with plan_cache_mode set to force_custom_plan. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +SET plan_cache_mode = force_generic_plan + +# The plan is an ideal generic plan (it has no placeholders), so it can be +# reused with plan_cache_mode set to force_generic_plan. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE k = $1 + +# An ideal generic plan can be built during PREPARE when the placeholder +# fast-path can be used. +query T +EXPLAIN ANALYZE EXECUTE p(33) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: [/33 - /33] + +statement ok +SET plan_cache_mode = force_custom_plan + +# The ideal generic plan is reused even when forcing a custom plan because it +# will always be optimal. +query T +EXPLAIN ANALYZE EXECUTE p(33) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: [/33 - /33] + +statement ok +SET plan_cache_mode = force_generic_plan + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = $1 AND c = $2 + +# A simple generic plan can be built during EXECUTE. +query T +EXPLAIN ANALYZE EXECUTE p(1, 2) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key +│ pred: c = "$2" +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +# The generic plan can be reused with different placeholder values. +query T +EXPLAIN ANALYZE EXECUTE p(11, 22) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key +│ pred: c = "$2" +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +statement ok +SET plan_cache_mode = force_custom_plan + +# The generic plan is not reused when forcing a custom plan. +query T +EXPLAIN ANALYZE EXECUTE p(1, 2) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 2 +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +SET plan_cache_mode = force_generic_plan + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE t = now() + +# A generic plan with a stable expression. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_t_idx + │ equality: (column9) = (t) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 1 column, 1 row + +# The generic plan can be reused. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_t_idx + │ equality: (column9) = (t) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 1 column, 1 row + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT k FROM t WHERE s LIKE $1 + +# A suboptimal generic query plan is chosen if it is forced. +query T +EXPLAIN ANALYZE EXECUTE p('foo%') +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: s LIKE 'foo%' +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_s_idx + spans: (/NULL - ] + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT k FROM t WHERE c = $1 + +# A simple generic plan. +query T +EXPLAIN ANALYZE EXECUTE p(1) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 1 +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: FULL SCAN + +statement ok +ALTER TABLE t ADD COLUMN z INT + +# A schema change invalidates the generic plan, and it must be re-optimized. +query T +EXPLAIN ANALYZE EXECUTE p(1) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 1 +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: FULL SCAN + +statement ok +DEALLOCATE p + +statement ok +ALTER TABLE t DROP COLUMN z + +statement ok +SET plan_cache_mode = auto + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 + +# An ideal generic plan is used immediately with plan_cache_mode=auto. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = $1 AND c = $2 + +statement ok +EXECUTE p(1, 2); +EXECUTE p(10, 20); +EXECUTE p(100, 200); +EXECUTE p(1000, 2000); +EXECUTE p(10000, 20000); + +# On the sixth execution a generic query plan will be generated. The cost of the +# generic plan is more than the average cost of the five custom plans (plus some +# overhead cost of optimization), so the custom plan is chosen. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 20000 +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/10000 - /10000] + +statement ok +DEALLOCATE p + +# Now use a more complex query that could have multiple join orderings. +statement ok +PREPARE p AS +SELECT * FROM t +JOIN c ON t.k = c.a +JOIN g ON c.k = g.a +WHERE t.a = $1 AND t.c = $2 + +statement ok +EXECUTE p(1, 2); +EXECUTE p(10, 20); +EXECUTE p(100, 200); +EXECUTE p(1000, 2000); + +# The first five executions will use a custom plan. This is the fifth. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join (streamer) +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: g@g_a_b_idx +│ equality: (k) = (a) +│ +└── • lookup join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: c@c_a_idx + │ equality: (k) = (a) + │ + └── • filter + │ nodes: + │ regions: + │ actual row count: 0 + │ filter: c = 20000 + │ + └── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/10000 - /10000] + +# On the sixth execution a generic query plan will be generated. The cost of the +# generic plan is less than the average cost of the five custom plans (plus some +# overhead cost of optimization), so the generic plan is chosen. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: g@g_a_b_idx +│ equality: (k) = (a) +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: c@c_a_idx + │ equality: (k) = (a) + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_pkey + │ equality: (k) = (k) + │ equality cols are key + │ pred: c = "$2" + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +# On the seventh execution the generic plan is reused. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: g@g_a_b_idx +│ equality: (k) = (a) +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: c@c_a_idx + │ equality: (k) = (a) + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_pkey + │ equality: (k) = (k) + │ equality cols are key + │ pred: c = "$2" + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +statement ok +DEALLOCATE p + +# Now use a query with a bad generic query plan. +statement ok +PREPARE p AS +SELECT * FROM g WHERE a = $1 ORDER BY b LIMIT 10 + +statement ok +EXECUTE p(1); +EXECUTE p(10); +EXECUTE p(100); +EXECUTE p(1000); +EXECUTE p(10000); + +# The generic plan is generated on the sixth execution, but is more expensive +# than the average custom plan. +query T +EXPLAIN ANALYZE EXECUTE p(10) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: g@g_a_b_idx + spans: [/10 - /10] + limit: 10 + +statement ok +SET plan_cache_mode = force_generic_plan + +# The generic plan previously generated is reused when forcing a generic plan. +query T +EXPLAIN ANALYZE EXECUTE p(10) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• limit +│ count: 10 +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: g@g_a_b_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 1 column, 1 row diff --git a/pkg/ccl/logictestccl/tests/local/BUILD.bazel b/pkg/ccl/logictestccl/tests/local/BUILD.bazel index ef7ea4aaef3b..096337816aaf 100644 --- a/pkg/ccl/logictestccl/tests/local/BUILD.bazel +++ b/pkg/ccl/logictestccl/tests/local/BUILD.bazel @@ -9,7 +9,7 @@ go_test( "//pkg/ccl/logictestccl:testdata", # keep ], exec_properties = {"test.Pool": "large"}, - shard_count = 44, + shard_count = 45, tags = ["cpu:1"], deps = [ "//pkg/base", diff --git a/pkg/ccl/logictestccl/tests/local/generated_test.go b/pkg/ccl/logictestccl/tests/local/generated_test.go index c67aea9b5aaf..d7be759cfae6 100644 --- a/pkg/ccl/logictestccl/tests/local/generated_test.go +++ b/pkg/ccl/logictestccl/tests/local/generated_test.go @@ -148,6 +148,13 @@ func TestCCLLogic_fk_read_committed( runCCLLogicTest(t, "fk_read_committed") } +func TestCCLLogic_generic( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runCCLLogicTest(t, "generic") +} + func TestCCLLogic_hash_sharded_index_read_committed( t *testing.T, ) { diff --git a/pkg/ccl/telemetryccl/testdata/telemetry/generic b/pkg/ccl/telemetryccl/testdata/telemetry/generic new file mode 100644 index 000000000000..cdf25c994b70 --- /dev/null +++ b/pkg/ccl/telemetryccl/testdata/telemetry/generic @@ -0,0 +1,134 @@ +# This file contains telemetry tests for CCL-related generic query plan +# counters. + +exec +CREATE TABLE kv ( + k INT PRIMARY KEY, + v INT, + INDEX (v) +) +---- + +exec +CREATE TABLE ab ( + a INT PRIMARY KEY, + b INT +) +---- + +exec +PREPARE p_pk AS +SELECT * FROM kv WHERE k = $1 +---- + +exec +PREPARE p_join AS +SELECT * FROM kv +JOIN ab ON a = k +WHERE v = $1 +---- + +feature-list +sql.plan.type.* +---- + +exec +SET plan_cache_mode = force_custom_plan +---- + +exec +SET index_recommendations_enabled = false +---- + +exec +SET application_name = 'testuser' +---- + +feature-usage +EXECUTE p_pk(100) +---- +sql.plan.type.force-custom + +feature-usage +EXECUTE p_join(1) +---- +sql.plan.type.force-custom + +# Non-prepared statements do not increment plan type counters. +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +exec +SET plan_cache_mode = force_generic_plan +---- + +feature-usage +EXECUTE p_pk(100) +---- +sql.plan.type.force-generic + +feature-usage +EXECUTE p_join(0) +---- +sql.plan.type.force-generic + +# Non-prepared statements do not increment plan type counters. +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +exec +SET plan_cache_mode = auto +---- + +# If the placeholder fast-path is used, the plan is always generic. +feature-usage +EXECUTE p_pk(100) +---- +sql.plan.type.auto-generic + +# The first five executions of p_join have custom plans while establishing an +# average cost. One of the custom executions occurred above. +feature-usage +EXECUTE p_join(2) +---- +sql.plan.type.auto-custom + +feature-usage +EXECUTE p_join(3) +---- +sql.plan.type.auto-custom + +feature-usage +EXECUTE p_join(4) +---- +sql.plan.type.auto-custom + +feature-usage +EXECUTE p_join(5) +---- +sql.plan.type.auto-custom + +# The sixth execution uses a generic plan. +feature-usage +EXECUTE p_join(6) +---- +sql.plan.type.auto-generic + +# Non-prepared statements do not increment plan type counters. +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +feature-usage +SELECT * FROM kv WHERE v = 100 +---- diff --git a/pkg/sql/BUILD.bazel b/pkg/sql/BUILD.bazel index cb06ef2ea2b0..a15520017fe7 100644 --- a/pkg/sql/BUILD.bazel +++ b/pkg/sql/BUILD.bazel @@ -424,6 +424,7 @@ go_library( "//pkg/sql/faketreeeval", "//pkg/sql/flowinfra", "//pkg/sql/gcjob/gcjobnotifier", + "//pkg/sql/gpq", "//pkg/sql/idxrecommendations", "//pkg/sql/idxusage", "//pkg/sql/inverted", @@ -684,6 +685,7 @@ go_test( "pg_oid_test.go", "pgwire_internal_test.go", "plan_opt_test.go", + "prepared_stmt_test.go", "privileged_accessor_test.go", "rand_test.go", "region_util_test.go", diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go index 32661b16bf37..865779f9d8d6 100644 --- a/pkg/sql/exec_util.go +++ b/pkg/sql/exec_util.go @@ -3826,6 +3826,10 @@ func (m *sessionDataMutator) SetOptimizerPushOffsetIntoIndexJoin(val bool) { m.data.OptimizerPushOffsetIntoIndexJoin = val } +func (m *sessionDataMutator) SetPlanCacheMode(val sessiondatapb.PlanCacheMode) { + m.data.PlanCacheMode = val +} + // Utility functions related to scrubbing sensitive information on SQL Stats. // quantizeCounts ensures that the Count field in the diff --git a/pkg/sql/gpq/BUILD.bazel b/pkg/sql/gpq/BUILD.bazel new file mode 100644 index 000000000000..44dcd993facb --- /dev/null +++ b/pkg/sql/gpq/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "gpq", + srcs = ["gpq.go"], + importpath = "github.com/cockroachdb/cockroach/pkg/sql/gpq", + visibility = ["//visibility:public"], + deps = [ + "//pkg/settings/cluster", + "//pkg/sql/sqlerrors", + "@com_github_cockroachdb_errors//:errors", + ], +) diff --git a/pkg/sql/gpq/gpq.go b/pkg/sql/gpq/gpq.go new file mode 100644 index 000000000000..3c3e77da85c2 --- /dev/null +++ b/pkg/sql/gpq/gpq.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package gpq + +import ( + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/sqlerrors" + "github.com/cockroachdb/errors" +) + +var CheckClusterSupportsGenericQueryPlans = func(settings *cluster.Settings) error { + return sqlerrors.NewCCLRequiredError( + errors.New("plan_cache_mode=force_generic_plan and plan_cache_mode=auto require a CCL binary"), + ) +} diff --git a/pkg/sql/instrumentation.go b/pkg/sql/instrumentation.go index dec7738d90d1..c762f65eb0ec 100644 --- a/pkg/sql/instrumentation.go +++ b/pkg/sql/instrumentation.go @@ -156,6 +156,16 @@ type instrumentationHelper struct { distribution physicalplan.PlanDistribution vectorized bool containsMutation bool + // generic is true if the plan can be fully-optimized once and re-used + // without re-optimization. Plans without placeholders and fold-able stable + // expressions, and plans utilizing the placeholder fast-path are always + // considered "generic". Plans that are fully optimized with placeholders + // present via the force_generic_plan or auto settings for plan_cache_mode + // are also considered "generic". + generic bool + // optimized is true if the plan was optimized or re-optimized during the + // current execution. + optimized bool traceMetadata execNodeTraceMetadata @@ -759,11 +769,13 @@ func (ih *instrumentationHelper) RecordExplainPlan(explainPlan *explain.Plan) { // RecordPlanInfo records top-level information about the plan. func (ih *instrumentationHelper) RecordPlanInfo( - distribution physicalplan.PlanDistribution, vectorized, containsMutation bool, + distribution physicalplan.PlanDistribution, vectorized, containsMutation, generic, optimized bool, ) { ih.distribution = distribution ih.vectorized = vectorized ih.containsMutation = containsMutation + ih.generic = generic + ih.optimized = optimized } // PlanForStats returns the plan as an ExplainTreePlanNode tree, if it was @@ -779,6 +791,7 @@ func (ih *instrumentationHelper) PlanForStats(ctx context.Context) *appstatspb.E }) ob.AddDistribution(ih.distribution.String()) ob.AddVectorized(ih.vectorized) + ob.AddPlanType(ih.generic, ih.optimized) if err := emitExplain(ctx, ob, ih.evalCtx, ih.codec, ih.explainPlan); err != nil { log.Warningf(ctx, "unable to emit explain plan tree: %v", err) return nil @@ -804,6 +817,7 @@ func (ih *instrumentationHelper) emitExplainAnalyzePlanToOutputBuilder( ob.AddExecutionTime(phaseTimes.GetRunLatency()) ob.AddDistribution(ih.distribution.String()) ob.AddVectorized(ih.vectorized) + ob.AddPlanType(ih.generic, ih.optimized) if queryStats != nil { if queryStats.KVRowsRead != 0 { diff --git a/pkg/sql/logictest/testdata/logic_test/generic_license b/pkg/sql/logictest/testdata/logic_test/generic_license new file mode 100644 index 000000000000..d86c4f353038 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/generic_license @@ -0,0 +1,7 @@ +# LogicTest: local + +statement error pgcode XXC01 pq: plan_cache_mode=force_generic_plan and plan_cache_mode=auto require a CCL binary +SET plan_cache_mode = force_generic_plan + +statement error pgcode XXC01 pq: plan_cache_mode=force_generic_plan and plan_cache_mode=auto require a CCL binary +SET plan_cache_mode = auto diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index 6ffe2d7a0f53..b0ba4395afcf 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -6193,6 +6193,7 @@ override_multi_region_zone_config off parallelize_multi_key_lookup_joins_enabled off password_encryption scram-sha-256 pg_trgm.similarity_threshold 0.3 +plan_cache_mode force_custom_plan plpgsql_use_strict_into off prefer_lookup_joins_for_fks off prepared_statements_cache_size 0 B diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog index 658383ddffce..14e4511c3c0f 100644 --- a/pkg/sql/logictest/testdata/logic_test/pg_catalog +++ b/pkg/sql/logictest/testdata/logic_test/pg_catalog @@ -2912,6 +2912,7 @@ override_multi_region_zone_config off N parallelize_multi_key_lookup_joins_enabled off NULL NULL NULL string password_encryption scram-sha-256 NULL NULL NULL string pg_trgm.similarity_threshold 0.3 NULL NULL NULL string +plan_cache_mode force_custom_plan NULL NULL NULL string plpgsql_use_strict_into off NULL NULL NULL string prefer_lookup_joins_for_fks off NULL NULL NULL string prepared_statements_cache_size 0 B NULL NULL NULL string @@ -3098,6 +3099,7 @@ override_multi_region_zone_config off N parallelize_multi_key_lookup_joins_enabled off NULL user NULL off off password_encryption scram-sha-256 NULL user NULL scram-sha-256 scram-sha-256 pg_trgm.similarity_threshold 0.3 NULL user NULL 0.3 0.3 +plan_cache_mode force_custom_plan NULL user NULL force_custom_plan force_custom_plan plpgsql_use_strict_into off NULL user NULL off off prefer_lookup_joins_for_fks off NULL user NULL off off prepared_statements_cache_size 0 B NULL user NULL 0 B 0 B @@ -3283,6 +3285,7 @@ override_multi_region_zone_config NULL NULL NULL parallelize_multi_key_lookup_joins_enabled NULL NULL NULL NULL NULL password_encryption NULL NULL NULL NULL NULL pg_trgm.similarity_threshold NULL NULL NULL NULL NULL +plan_cache_mode NULL NULL NULL NULL NULL plpgsql_use_strict_into NULL NULL NULL NULL NULL prefer_lookup_joins_for_fks NULL NULL NULL NULL NULL prepared_statements_cache_size NULL NULL NULL NULL NULL diff --git a/pkg/sql/logictest/testdata/logic_test/show_source b/pkg/sql/logictest/testdata/logic_test/show_source index 79138132399a..e9b95438c1f6 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_source +++ b/pkg/sql/logictest/testdata/logic_test/show_source @@ -149,6 +149,7 @@ override_multi_region_zone_config off parallelize_multi_key_lookup_joins_enabled off password_encryption scram-sha-256 pg_trgm.similarity_threshold 0.3 +plan_cache_mode force_custom_plan plpgsql_use_strict_into off prefer_lookup_joins_for_fks off prepared_statements_cache_size 0 B diff --git a/pkg/sql/logictest/tests/local/generated_test.go b/pkg/sql/logictest/tests/local/generated_test.go index 709e50072c6c..05fe0939e0e0 100644 --- a/pkg/sql/logictest/tests/local/generated_test.go +++ b/pkg/sql/logictest/tests/local/generated_test.go @@ -890,6 +890,13 @@ func TestLogic_generator_probe_ranges( runLogicTest(t, "generator_probe_ranges") } +func TestLogic_generic_license( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runLogicTest(t, "generic_license") +} + func TestLogic_geospatial( t *testing.T, ) { diff --git a/pkg/sql/opt/exec/execbuilder/scalar.go b/pkg/sql/opt/exec/execbuilder/scalar.go index 7b0ec2be500d..98464809d5c0 100644 --- a/pkg/sql/opt/exec/execbuilder/scalar.go +++ b/pkg/sql/opt/exec/execbuilder/scalar.go @@ -57,7 +57,7 @@ func init() { opt.VariableOp: (*Builder).buildVariable, opt.ConstOp: (*Builder).buildTypedExpr, opt.NullOp: (*Builder).buildNull, - opt.PlaceholderOp: (*Builder).buildTypedExpr, + opt.PlaceholderOp: (*Builder).buildPlaceholder, opt.TupleOp: (*Builder).buildTuple, opt.FunctionOp: (*Builder).buildFunction, opt.CaseOp: (*Builder).buildCase, @@ -131,6 +131,15 @@ func (b *Builder) buildTypedExpr( return scalar.Private().(tree.TypedExpr), nil } +func (b *Builder) buildPlaceholder( + ctx *buildScalarCtx, scalar opt.ScalarExpr, +) (tree.TypedExpr, error) { + if b.evalCtx != nil && b.evalCtx.Placeholders != nil { + return eval.Expr(b.ctx, b.evalCtx, scalar.Private().(*tree.Placeholder)) + } + return b.buildTypedExpr(ctx, scalar) +} + func (b *Builder) buildNull(ctx *buildScalarCtx, scalar opt.ScalarExpr) (tree.TypedExpr, error) { retypedNull, ok := eval.ReType(tree.DNull, scalar.DataType()) if !ok { diff --git a/pkg/sql/opt/exec/execbuilder/testdata/call b/pkg/sql/opt/exec/execbuilder/testdata/call index b318dec3fbb1..b78fe3fc6bd0 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/call +++ b/pkg/sql/opt/exec/execbuilder/testdata/call @@ -137,6 +137,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -158,6 +159,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/cascade b/pkg/sql/opt/exec/execbuilder/testdata/cascade index 01fe66c547d4..8742dc9ca85e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/cascade +++ b/pkg/sql/opt/exec/execbuilder/testdata/cascade @@ -1047,6 +1047,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 9 (72 B, 18 KVs, 9 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize b/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize index 59731bda3eed..f699769d19d1 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize +++ b/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize @@ -56,6 +56,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 5 (40 B, 10 KVs, 5 gRPC calls) maximum memory usage: network usage: @@ -93,6 +94,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc b/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc index 0384a71c0e15..03634cb83372 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc +++ b/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc @@ -105,6 +105,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -145,6 +146,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -164,6 +166,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -183,6 +186,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -205,6 +209,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -236,6 +241,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -259,6 +265,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -285,6 +292,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -311,6 +319,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain b/pkg/sql/opt/exec/execbuilder/testdata/explain index 62138d8c6fb4..bda5def571c8 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain @@ -2230,6 +2230,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze index a7d4bf5243d2..895b7f33e4d1 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze @@ -10,6 +10,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -41,6 +42,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans index 9a45bd00102c..d52c5ea589e1 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans @@ -65,6 +65,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: @@ -130,6 +131,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: @@ -203,6 +205,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: @@ -289,6 +292,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 5 (40 B, 10 KVs, 5 gRPC calls) maximum memory usage: network usage: @@ -330,6 +334,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -366,6 +371,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_read_committed b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_read_committed index 564e9d10bcd5..9ca9b9f0d331 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_read_committed +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_read_committed @@ -24,6 +24,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -64,6 +65,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 7 (56 B, 14 KVs, 7 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial index c0de285c231b..083ccf3051d9 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial +++ b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial @@ -26,6 +26,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 6 (48 B, 12 KVs, 6 gRPC calls) maximum memory usage: network usage: @@ -118,6 +119,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, re-optimized rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: @@ -194,6 +196,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, re-optimized rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit index 28637346a383..9f31c8481599 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit +++ b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit @@ -80,6 +80,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: @@ -164,6 +165,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, reused rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: @@ -321,6 +323,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: @@ -416,6 +419,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/prepare b/pkg/sql/opt/exec/execbuilder/testdata/prepare index 2c12f2f84978..784433d906f8 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/prepare +++ b/pkg/sql/opt/exec/execbuilder/testdata/prepare @@ -115,6 +115,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, reused rows decoded from KV: 1 (8 B, 2 KVs, 1 gRPC calls) maximum memory usage: network usage: @@ -134,7 +135,7 @@ quality of service: regular KV bytes read: 8 B KV gRPC calls: 1 estimated max memory allocated: 0 B - estimated row count: 0 + estimated row count: 1 table: ab@ab_pkey spans: [/1 - /1] @@ -145,6 +146,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, reused maximum memory usage: network usage: regions: @@ -162,6 +164,6 @@ quality of service: regular KV bytes read: 0 B KV gRPC calls: 0 estimated max memory allocated: 0 B - estimated row count: 0 + estimated row count: 1 table: ab@ab_pkey spans: [/2 - /2] diff --git a/pkg/sql/opt/exec/execbuilder/testdata/unique b/pkg/sql/opt/exec/execbuilder/testdata/unique index d29040c4fe01..fa4f136b8d2e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/unique +++ b/pkg/sql/opt/exec/execbuilder/testdata/unique @@ -605,6 +605,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: @@ -720,6 +721,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 6 (48 B, 12 KVs, 6 gRPC calls) maximum memory usage: network usage: @@ -2602,6 +2604,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local b/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local index a8076c8be0d9..9299a6206d4b 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local +++ b/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local @@ -42,6 +42,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2,001 (16 KiB, 4,002 KVs, 2,001 gRPC calls) maximum memory usage: network usage: @@ -74,6 +75,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -123,6 +125,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -209,6 +212,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/explain/output.go b/pkg/sql/opt/exec/explain/output.go index 5505050d477c..cbea9ba14caf 100644 --- a/pkg/sql/opt/exec/explain/output.go +++ b/pkg/sql/opt/exec/explain/output.go @@ -307,6 +307,19 @@ func (ob *OutputBuilder) AddVectorized(value bool) { ob.AddFlakyTopLevelField(DeflakeVectorized, "vectorized", fmt.Sprintf("%t", value)) } +// AddGeneric adds a top-level generic field, if value is true. Cannot be called +// while inside a node. +func (ob *OutputBuilder) AddPlanType(generic, optimized bool) { + switch { + case generic && optimized: + ob.AddTopLevelField("plan type", "generic, re-optimized") + case generic && !optimized: + ob.AddTopLevelField("plan type", "generic, reused") + default: + ob.AddTopLevelField("plan type", "custom") + } +} + // AddPlanningTime adds a top-level planning time field. Cannot be called // while inside a node. func (ob *OutputBuilder) AddPlanningTime(delta time.Duration) { diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go index 3a52ae49169e..88d3850a2713 100644 --- a/pkg/sql/opt/memo/memo.go +++ b/pkg/sql/opt/memo/memo.go @@ -517,6 +517,17 @@ func (m *Memo) IsOptimized() bool { return ok && rel.RequiredPhysical() != nil } +// OptimizationCost returns a rough estimate of the cost of optimization of the +// memo. It is dependent on the number of tables in the metadata, based on the +// reasoning that queries with more tables likely have more joins, which tend +// to be the biggest contributors to optimization overhead. +func (m *Memo) OptimizationCost() Cost { + // This cpuCostFactor is the same as cpuCostFactor in the coster. + // TODO(mgartner): Package these constants up in a shared location. + const cpuCostFactor = 0.01 + return Cost(m.Metadata().NumTables()) * 1000 * cpuCostFactor +} + // NextRank returns a new rank that can be assigned to a scalar expression. func (m *Memo) NextRank() opt.ScalarRank { m.curRank++ diff --git a/pkg/sql/opt/memo/statistics_builder.go b/pkg/sql/opt/memo/statistics_builder.go index 58b3525087c0..31d5b42edea1 100644 --- a/pkg/sql/opt/memo/statistics_builder.go +++ b/pkg/sql/opt/memo/statistics_builder.go @@ -3370,6 +3370,8 @@ func (sb *statisticsBuilder) applyFilters( func (sb *statisticsBuilder) applyFiltersItem( filter *FiltersItem, e RelExpr, relProps *props.Relational, unapplied *filterCount, ) (constrainedCols, histCols opt.ColSet) { + s := relProps.Statistics() + // Before checking anything, try to replace any virtual computed column // expressions with the corresponding virtual column. if sb.evalCtx.SessionData().OptimizerUseVirtualComputedColumnStats && @@ -3395,6 +3397,13 @@ func (sb *statisticsBuilder) applyFiltersItem( return opt.ColSet{}, opt.ColSet{} } + // Special case: a placeholder equality filter. + if col, ok := isPlaceholderEqualityFilter(filter.Condition); ok { + cols := opt.MakeColSet(col) + sb.ensureColStat(cols, 1 /* maxDistinctCount */, e, s) + return cols, opt.ColSet{} + } + // Special case: The current conjunct is an inverted join condition which is // handled by selectivityFromInvertedJoinCondition. if isInvertedJoinCond(filter.Condition) { @@ -3434,7 +3443,6 @@ func (sb *statisticsBuilder) applyFiltersItem( // want to make sure that we don't include columns that were only present in // equality conjuncts such as var1=var2. The selectivity of these conjuncts // will be accounted for in selectivityFromEquivalencies. - s := relProps.Statistics() scalarProps := filter.ScalarProps() constrainedCols.UnionWith(scalarProps.OuterCols) if scalarProps.Constraints != nil { @@ -4854,6 +4862,17 @@ func isSimilarityFilter(e opt.ScalarExpr) bool { return false } +// isPlaceholderEqualityFilter returns a column ID and true if the given +// condition is an equality between a column and a placeholder. +func isPlaceholderEqualityFilter(e opt.ScalarExpr) (opt.ColumnID, bool) { + if e.Op() == opt.EqOp && e.Child(1).Op() == opt.PlaceholderOp { + if v, ok := e.Child(0).(*VariableExpr); ok { + return v.Col, true + } + } + return 0, false +} + // isInvertedJoinCond returns true if the given condition is either an index- // accelerated geospatial function, a bounding box operation, or a contains // operation with two variable arguments. diff --git a/pkg/sql/opt/memo/testdata/stats/generic b/pkg/sql/opt/memo/testdata/stats/generic new file mode 100644 index 000000000000..ba395c1fa05b --- /dev/null +++ b/pkg/sql/opt/memo/testdata/stats/generic @@ -0,0 +1,168 @@ +exec-ddl +CREATE TABLE t ( + k INT PRIMARY KEY, + i INT, + s STRING +) +---- + +exec-ddl +ALTER TABLE t INJECT STATISTICS '[ + { + "columns": ["k"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 10000 + }, + { + "columns": ["i"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 10000, + "distinct_count": 1000 + }, + { + "columns": ["s"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 10000, + "distinct_count": 100 + } +]' +---- + +norm +SELECT * FROM t WHERE k = $1 +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string) + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── stats: [rows=1, distinct(1)=1, null(1)=0] + ├── key: () + ├── fd: ()-->(1-3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── k:1 = $1 [type=bool, outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] + +norm +SELECT * FROM t WHERE i = $1 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string) + ├── has-placeholder + ├── stats: [rows=10, distinct(2)=1, null(2)=0] + ├── key: (1) + ├── fd: ()-->(2), (1)-->(3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── i:2 = $1 [type=bool, outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + +norm +SELECT * FROM t WHERE $1 = s +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string!null) + ├── has-placeholder + ├── stats: [rows=100, distinct(3)=1, null(3)=0] + ├── key: (1) + ├── fd: ()-->(3), (1)-->(2) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(3)=100, null(3)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── s:3 = $1 [type=bool, outer=(3), constraints=(/3: (/NULL - ]), fd=()-->(3)] + +norm +SELECT * FROM t WHERE i = $1 AND s = $2 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string!null) + ├── has-placeholder + ├── stats: [rows=0.91, distinct(2)=0.91, null(2)=0, distinct(3)=0.91, null(3)=0, distinct(2,3)=0.91, null(2,3)=0] + ├── key: (1) + ├── fd: ()-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0, distinct(3)=100, null(3)=0, distinct(2,3)=10000, null(2,3)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + ├── i:2 = $1 [type=bool, outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + └── s:3 = $2 [type=bool, outer=(3), constraints=(/3: (/NULL - ]), fd=()-->(3)] + +norm +SELECT * FROM t WHERE i > $1 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333, distinct(2)=1000, null(2)=0] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── i:2 > $1 [type=bool, outer=(2), constraints=(/2: (/NULL - ])] + +norm +SELECT * FROM t WHERE i = $1 OR i = $2 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333, distinct(2)=1000, null(2)=0] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── (i:2 = $1) OR (i:2 = $2) [type=bool, outer=(2), constraints=(/2: (/NULL - ])] + +norm +SELECT * FROM t WHERE i IN ($1, $2, $3) +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── i:2 IN ($1, $2, $3) [type=bool, outer=(2)] + +norm +SELECT * FROM t WHERE i = $1 OR s = $2 +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── (i:2 = $1) OR (s:3 = $2) [type=bool, outer=(2,3)] diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index e555f579b950..6adae781c987 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -797,6 +797,11 @@ func (md *Metadata) AllTables() []TableMeta { return md.tables } +// NumTables returns the number of tables in the metadata. +func (md *Metadata) NumTables() int { + return len(md.tables) +} + // AddColumn assigns a new unique id to a column within the query and records // its alias and type. If the alias is empty, a "column" alias is created. func (md *Metadata) AddColumn(alias string, typ *types.T) ColumnID { diff --git a/pkg/sql/opt/norm/BUILD.bazel b/pkg/sql/opt/norm/BUILD.bazel index a167e1cb4429..1196d1ab227d 100644 --- a/pkg/sql/opt/norm/BUILD.bazel +++ b/pkg/sql/opt/norm/BUILD.bazel @@ -90,6 +90,7 @@ go_test( "//pkg/sql/sem/catconstants", "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", + "//pkg/sql/sessiondatapb", "//pkg/sql/types", "//pkg/testutils/datapathutils", "//pkg/util/leaktest", diff --git a/pkg/sql/opt/norm/factory_test.go b/pkg/sql/opt/norm/factory_test.go index ca1bae9a3435..d8adff216cda 100644 --- a/pkg/sql/opt/norm/factory_test.go +++ b/pkg/sql/opt/norm/factory_test.go @@ -24,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/catconstants" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb" "github.com/cockroachdb/cockroach/pkg/sql/types" ) @@ -69,33 +70,31 @@ func TestSimplifyFilters(t *testing.T) { } } -// Test CopyAndReplace on an already optimized join. Before CopyAndReplace is -// called, the join has a placeholder that causes the optimizer to use a merge -// join. After CopyAndReplace substitutes a constant for the placeholder, the -// optimizer switches to a lookup join. A similar pattern is used by the -// ApplyJoin execution operator which replaces variables with constants in an -// already optimized tree. The CopyAndReplace code must take care to copy over -// the normalized tree rather than the optimized tree by using the FirstExpr -// method. +// Test CopyAndReplace on an already optimized memo. Before CopyAndReplace is +// called, the query has a placeholder that causes the optimizer to use a +// parameterized lookup-join. After CopyAndReplace substitutes a constant for +// the placeholder, the optimizer switches to a constrained scan. A similar +// pattern is used by the ApplyJoin execution operator which replaces variables +// with constants in an already optimized tree. The CopyAndReplace code must +// take care to copy over the normalized tree rather than the optimized tree by +// using the FirstExpr method. func TestCopyAndReplace(t *testing.T) { cat := testcat.New() if _, err := cat.ExecuteDDL("CREATE TABLE ab (a INT PRIMARY KEY, b INT)"); err != nil { t.Fatal(err) } - if _, err := cat.ExecuteDDL("CREATE TABLE cde (c INT PRIMARY KEY, d INT, e INT, INDEX(d))"); err != nil { - t.Fatal(err) - } evalCtx := eval.MakeTestingEvalContext(cluster.MakeTestingClusterSettings()) - evalCtx.SessionData().OptimizerMergeJoinsEnabled = true + evalCtx.SessionData().PlanCacheMode = sessiondatapb.PlanCacheModeAuto var o xform.Optimizer - testutils.BuildQuery(t, &o, cat, &evalCtx, "SELECT * FROM cde INNER JOIN ab ON a=c AND d=$1") + testutils.BuildQuery(t, &o, cat, &evalCtx, "SELECT * FROM ab WHERE a = $1") if e, err := o.Optimize(); err != nil { t.Fatal(err) - } else if e.Op() != opt.MergeJoinOp { - t.Errorf("expected optimizer to choose merge-join, not %v", e.Op()) + } else if e.Op() != opt.ProjectOp || e.Child(0).Op() != opt.LookupJoinOp { + t.Errorf("expected optimizer to choose a (project (lookup-join)), not (%v (%v))", + e.Op(), e.Child(0).Op()) } m := o.Factory().DetachMemo() @@ -112,8 +111,10 @@ func TestCopyAndReplace(t *testing.T) { if e, err := o.Optimize(); err != nil { t.Fatal(err) - } else if e.Op() != opt.LookupJoinOp { - t.Errorf("expected optimizer to choose lookup-join, not %v", e.Op()) + } else if e.Op() != opt.ScanOp { + t.Errorf("expected optimizer to choose a constrained scan, not %v", e.Op()) + } else if e.(*memo.ScanExpr).Constraint == nil { + t.Errorf("expected optimizer to choose a constrained scan") } } diff --git a/pkg/sql/opt/ops/scalar.opt b/pkg/sql/opt/ops/scalar.opt index 4417fadf1e72..7572e006e9dd 100644 --- a/pkg/sql/opt/ops/scalar.opt +++ b/pkg/sql/opt/ops/scalar.opt @@ -154,6 +154,7 @@ define False { [Scalar] define Placeholder { + # Value is always a *tree.Placeholder. Value TypedExpr } diff --git a/pkg/sql/opt/testutils/opttester/BUILD.bazel b/pkg/sql/opt/testutils/opttester/BUILD.bazel index 0ae32ad6f3b2..17368a6684c0 100644 --- a/pkg/sql/opt/testutils/opttester/BUILD.bazel +++ b/pkg/sql/opt/testutils/opttester/BUILD.bazel @@ -45,6 +45,7 @@ go_library( "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", "//pkg/sql/sem/volatility", + "//pkg/sql/sessiondatapb", "//pkg/sql/stats", "//pkg/testutils/datapathutils", "//pkg/testutils/floatcmp", diff --git a/pkg/sql/opt/testutils/opttester/opt_tester.go b/pkg/sql/opt/testutils/opttester/opt_tester.go index 9cb01f9c0bfe..4a3d88b7281b 100644 --- a/pkg/sql/opt/testutils/opttester/opt_tester.go +++ b/pkg/sql/opt/testutils/opttester/opt_tester.go @@ -60,6 +60,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb" "github.com/cockroachdb/cockroach/pkg/sql/stats" "github.com/cockroachdb/cockroach/pkg/testutils/datapathutils" "github.com/cockroachdb/cockroach/pkg/testutils/floatcmp" @@ -168,6 +169,9 @@ type Flags struct { // memo/check_expr.go. DisableCheckExpr bool + // Generic enables optimizations for generic query plans. + Generic bool + // ExploreTraceRule restricts the ExploreTrace output to only show the effects // of a specific rule. ExploreTraceRule opt.RuleName @@ -460,7 +464,7 @@ func New(catalog cat.Catalog, sqlStr string) *OptTester { // modifies the existing set of the flags. // // - no-stable-folds: disallows constant folding for stable operators; only -// used with "norm". +// used with "norm", "opt", "exprnorm", and "expropt". // // - fully-qualify-names: fully qualify all column names in the test output. // @@ -478,6 +482,11 @@ func New(catalog cat.Catalog, sqlStr string) *OptTester { // // - disable-check-expr: skips the assertions in memo/check_expr.go. // +// - generic: enables optimizations for generic query plans. +// NOTE: This flag sets the plan_cache_mode session setting to "auto", which +// cannot be done via the "set" flag because it requires a CCL license, +// which optimizer tests are not set up to utilize. +// // - rule: used with exploretrace; the value is the name of a rule. When // specified, the exploretrace output is filtered to only show expression // changes due to that specific rule. @@ -1004,6 +1013,9 @@ func (f *Flags) Set(arg datadriven.CmdArg) error { case "disable-check-expr": f.DisableCheckExpr = true + case "generic": + f.evalCtx.SessionData().PlanCacheMode = sessiondatapb.PlanCacheModeAuto + case "rule": if len(arg.Vals) != 1 { return fmt.Errorf("rule requires one argument") @@ -1225,7 +1237,9 @@ func (ot *OptTester) OptimizeWithTables(tables map[cat.StableID]cat.Table) (opt. o.NotifyOnMatchedRule(func(ruleName opt.RuleName) bool { return !ot.Flags.DisableRules.Contains(int(ruleName)) }) - o.Factory().FoldingControl().AllowStableFolds() + if !ot.Flags.NoStableFolds { + o.Factory().FoldingControl().AllowStableFolds() + } return ot.optimizeExpr(o, tables) } diff --git a/pkg/sql/opt/xform/BUILD.bazel b/pkg/sql/opt/xform/BUILD.bazel index c45d407c200a..987994e8ef6a 100644 --- a/pkg/sql/opt/xform/BUILD.bazel +++ b/pkg/sql/opt/xform/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "cycle_funcs.go", "explorer.go", "general_funcs.go", + "generic_funcs.go", "groupby_funcs.go", "index_scan_builder.go", "insert_funcs.go", @@ -48,6 +49,8 @@ go_library( "//pkg/sql/rowinfra", "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", + "//pkg/sql/sem/volatility", + "//pkg/sql/sessiondatapb", "//pkg/sql/types", "//pkg/util/buildutil", "//pkg/util/cancelchecker", diff --git a/pkg/sql/opt/xform/generic_funcs.go b/pkg/sql/opt/xform/generic_funcs.go new file mode 100644 index 000000000000..805d431f728f --- /dev/null +++ b/pkg/sql/opt/xform/generic_funcs.go @@ -0,0 +1,131 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package xform + +import ( + "fmt" + + "github.com/cockroachdb/cockroach/pkg/sql/opt" + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb" + "github.com/cockroachdb/cockroach/pkg/sql/types" +) + +// GenericRulesEnabled returns true if rules for optimizing generic query plans +// are enabled, based on the plan_cache_mode session setting. +func (c *CustomFuncs) GenericRulesEnabled() bool { + return c.e.evalCtx.SessionData().PlanCacheMode != sessiondatapb.PlanCacheModeForceCustom +} + +// HasPlaceholdersOrStableExprs returns true if the given relational expression's subtree has +// at least one placeholder. +func (c *CustomFuncs) HasPlaceholdersOrStableExprs(e memo.RelExpr) bool { + return e.Relational().HasPlaceholder || e.Relational().VolatilitySet.HasStable() +} + +// GenerateParameterizedJoinValuesAndFilters returns a single-row Values +// expression containing placeholders and stable expressions in the given +// filters. It also returns a new set of filters where the placeholders and +// stable expressions have been replaced with variables referencing the columns +// produced by the returned Values expression. If the given filters have no +// placeholders or stable expressions, ok=false is returned. +func (c *CustomFuncs) GenerateParameterizedJoinValuesAndFilters( + filters memo.FiltersExpr, +) (values memo.RelExpr, newFilters memo.FiltersExpr, ok bool) { + var exprs memo.ScalarListExpr + var cols opt.ColList + placeholderCols := make(map[tree.PlaceholderIdx]opt.ColumnID) + + // replace recursively walks the expression tree and replaces placeholders + // and stable expressions. It collects the replaced expressions and creates + // columns representing those expressions. Those expressions and columns + // will be used in the Values expression created below. + var replace func(e opt.Expr) opt.Expr + replace = func(e opt.Expr) opt.Expr { + switch t := e.(type) { + case *memo.PlaceholderExpr: + idx := t.Value.(*tree.Placeholder).Idx + // Reuse the same column for duplicate placeholder references. + if col, ok := placeholderCols[idx]; ok { + return c.e.f.ConstructVariable(col) + } + col := c.e.f.Metadata().AddColumn(fmt.Sprintf("$%d", idx+1), t.DataType()) + placeholderCols[idx] = col + exprs = append(exprs, t) + cols = append(cols, col) + return c.e.f.ConstructVariable(col) + + case *memo.FunctionExpr: + // TODO(mgartner): Consider including other expressions that could + // be stable: casts, assignment casts, UDFCallExprs, unary ops, + // comparisons, binary ops. + // TODO(mgartner): Include functions with arguments if they are all + // constants or placeholders. + if t.Overload.Volatility == volatility.Stable && len(t.Args) == 0 { + col := c.e.f.Metadata().AddColumn("", t.DataType()) + exprs = append(exprs, t) + cols = append(cols, col) + return c.e.f.ConstructVariable(col) + } + } + + return c.e.f.Replace(e, replace) + } + + // Replace placeholders and stable expressions in each filter. + for i := range filters { + cond := filters[i].Condition + if newCond := replace(cond).(opt.ScalarExpr); newCond != cond { + if newFilters == nil { + // Lazily allocate newFilters. + newFilters = make(memo.FiltersExpr, len(filters)) + copy(newFilters, filters[:i]) + } + // Construct a new filter if placeholders were replaced. + newFilters[i] = c.e.f.ConstructFiltersItem(newCond) + } else if newFilters != nil { + // Otherwise copy the filter if newFilters has been allocated. + newFilters[i] = filters[i] + } + } + + // If no placeholders or stable expressions were replaced, there is nothing + // to do. + if len(exprs) == 0 { + return nil, nil, false + } + + // Create the Values expression with one row and one column for each + // replaced expression. + typs := make([]*types.T, len(exprs)) + for i, e := range exprs { + typs[i] = e.DataType() + } + tupleTyp := types.MakeTuple(typs) + rows := memo.ScalarListExpr{c.e.f.ConstructTuple(exprs, tupleTyp)} + values = c.e.f.ConstructValues(rows, &memo.ValuesPrivate{ + Cols: cols, + ID: c.e.f.Metadata().NextUniqueID(), + }) + + return values, newFilters, true +} + +// ParameterizedJoinPrivate returns JoinPrivate that disabled join reordering and +// merge join exploration. +func (c *CustomFuncs) ParameterizedJoinPrivate() *memo.JoinPrivate { + return &memo.JoinPrivate{ + Flags: memo.DisallowMergeJoin, + SkipReorderJoins: true, + } +} diff --git a/pkg/sql/opt/xform/rules/generic.opt b/pkg/sql/opt/xform/rules/generic.opt new file mode 100644 index 000000000000..4b89e4ab00c9 --- /dev/null +++ b/pkg/sql/opt/xform/rules/generic.opt @@ -0,0 +1,58 @@ +# ============================================================================= +# generic.opt contains exploration rules for optimizing generic query plans. +# ============================================================================= + +# GenerateParameterizedJoin is an exploration rule that converts a Select +# expression with placeholders and stable expression in the filters into an +# InnerJoin that joins the Select's input with a Values expression that produces +# the placeholder values and stable expressions. +# +# This rule allows generic query plans, in which placeholder values are not +# known and stable expressions are not folded, to be optimized. By converting +# the Select into an InnerJoin, the optimizer can, in many cases, plan a lookup +# join which has similar performance characteristics to the constrained Scan +# that would be planned if the placeholder values were known. +# +# For example, consider a schema and query like: +# +# CREATE TABLE t (i INT PRIMARY KEY) +# SELECT * FROM t WHERE i = $1 +# +# GenerateParameterizedJoin will perform the first transformation below, from a +# Select into a Join. GenerateLookupJoins will perform the second transformation +# from a (hash) Join into a LookupJoin. +# +# Select (i=$1) Join (i=col_$1) LookupJoin (t@t_pkey) +# | -> / \ -> | +# | / \ | +# Scan t Values ($1) Scan t Values ($1) +# +[GenerateParameterizedJoin, Explore] +(Select + $scan:(Scan $scanPrivate:*) & + (GenericRulesEnabled) & + (IsCanonicalScan $scanPrivate) + $filters:* & + (HasPlaceholdersOrStableExprs (Root)) & + (Let + ( + $values + $newFilters + $ok + ):(GenerateParameterizedJoinValuesAndFilters + $filters + ) + $ok + ) +) +=> +(Project + (InnerJoin + $values + $scan + $newFilters + (ParameterizedJoinPrivate) + ) + [] + (OutputCols (Root)) +) diff --git a/pkg/sql/opt/xform/testdata/external/hibernate b/pkg/sql/opt/xform/testdata/external/hibernate index d32dc8e91361..37ec682218cb 100644 --- a/pkg/sql/opt/xform/testdata/external/hibernate +++ b/pkg/sql/opt/xform/testdata/external/hibernate @@ -886,7 +886,7 @@ project └── filters └── min:14 = $1 [outer=(14), constraints=(/14: (/NULL - ]), fd=()-->(14)] -opt +opt generic select person0_.id as id1_2_, person0_.address as address2_2_, @@ -951,7 +951,7 @@ project └── filters └── max:16 = 0 [outer=(16), constraints=(/16: [/0 - /0]; tight), fd=()-->(16)] -opt +opt generic select person0_.id as id1_2_, person0_.address as address2_2_, @@ -991,21 +991,32 @@ project ├── has-placeholder ├── key: () ├── fd: ()-->(1-6,9,12), (12)==(1), (1)==(12) - ├── select + ├── project │ ├── columns: phones1_.id:9!null person_id:12 │ ├── cardinality: [0 - 1] │ ├── has-placeholder │ ├── key: () │ ├── fd: ()-->(9,12) - │ ├── scan phone [as=phones1_] - │ │ ├── columns: phones1_.id:9!null person_id:12 - │ │ ├── key: (9) - │ │ └── fd: (9)-->(12) - │ └── filters - │ └── phones1_.id:9 = $1 [outer=(9), constraints=(/9: (/NULL - ]), fd=()-->(9)] + │ └── inner-join (lookup phone [as=phones1_]) + │ ├── columns: phones1_.id:9!null person_id:12 "$1":17!null + │ ├── flags: disallow merge join + │ ├── key columns: [17] = [9] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(9,12,17), (17)==(9), (9)==(17) + │ ├── values + │ │ ├── columns: "$1":17 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(17) + │ │ └── ($1,) + │ └── filters (true) └── filters (true) -opt +opt generic select person0_.id as id1_2_, person0_.address as address2_2_, @@ -1045,18 +1056,29 @@ project ├── has-placeholder ├── key: () ├── fd: ()-->(1-6,9,12), (12)==(1), (1)==(12) - ├── select + ├── project │ ├── columns: phones1_.id:9!null person_id:12 │ ├── cardinality: [0 - 1] │ ├── has-placeholder │ ├── key: () │ ├── fd: ()-->(9,12) - │ ├── scan phone [as=phones1_] - │ │ ├── columns: phones1_.id:9!null person_id:12 - │ │ ├── key: (9) - │ │ └── fd: (9)-->(12) - │ └── filters - │ └── phones1_.id:9 = $1 [outer=(9), constraints=(/9: (/NULL - ]), fd=()-->(9)] + │ └── inner-join (lookup phone [as=phones1_]) + │ ├── columns: phones1_.id:9!null person_id:12 "$1":17!null + │ ├── flags: disallow merge join + │ ├── key columns: [17] = [9] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(9,12,17), (17)==(9), (9)==(17) + │ ├── values + │ │ ├── columns: "$1":17 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(17) + │ │ └── ($1,) + │ └── filters (true) └── filters (true) opt @@ -1863,30 +1885,31 @@ project │ ├── has-placeholder │ ├── key: (1) │ ├── fd: ()-->(4), (1)-->(2-4,16) - │ ├── right-join (hash) - │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null successfulbid:10 true:15 + │ ├── project + │ │ ├── columns: true:15 bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null successfulbid:10 │ │ ├── has-placeholder │ │ ├── fd: ()-->(4), (1)-->(2,3) - │ │ ├── project - │ │ │ ├── columns: true:15!null successfulbid:10 - │ │ │ ├── fd: ()-->(15) - │ │ │ ├── scan tauction2 [as=a] - │ │ │ │ └── columns: successfulbid:10 - │ │ │ └── projections - │ │ │ └── true [as=true:15] - │ │ ├── select - │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null + │ │ ├── right-join (hash) + │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null successfulbid:10 │ │ │ ├── has-placeholder - │ │ │ ├── key: (1) │ │ │ ├── fd: ()-->(4), (1)-->(2,3) - │ │ │ ├── scan tbid2 [as=bids0_] - │ │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4 + │ │ │ ├── scan tauction2 [as=a] + │ │ │ │ └── columns: successfulbid:10 + │ │ │ ├── select + │ │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null + │ │ │ │ ├── has-placeholder │ │ │ │ ├── key: (1) - │ │ │ │ └── fd: (1)-->(2-4) + │ │ │ │ ├── fd: ()-->(4), (1)-->(2,3) + │ │ │ │ ├── scan tbid2 [as=bids0_] + │ │ │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4 + │ │ │ │ │ ├── key: (1) + │ │ │ │ │ └── fd: (1)-->(2-4) + │ │ │ │ └── filters + │ │ │ │ └── auctionid:4 = $1 [outer=(4), constraints=(/4: (/NULL - ]), fd=()-->(4)] │ │ │ └── filters - │ │ │ └── auctionid:4 = $1 [outer=(4), constraints=(/4: (/NULL - ]), fd=()-->(4)] - │ │ └── filters - │ │ └── successfulbid:10 = bids0_.id:1 [outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ │ └── successfulbid:10 = bids0_.id:1 [outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── projections + │ │ └── CASE successfulbid:10 IS NULL WHEN true THEN CAST(NULL AS BOOL) ELSE true END [as=true:15, outer=(10)] │ └── aggregations │ ├── const-not-null-agg [as=true_agg:16, outer=(15)] │ │ └── true:15 diff --git a/pkg/sql/opt/xform/testdata/external/nova b/pkg/sql/opt/xform/testdata/external/nova index 5cf93d63c7db..a0d420805bcd 100644 --- a/pkg/sql/opt/xform/testdata/external/nova +++ b/pkg/sql/opt/xform/testdata/external/nova @@ -107,7 +107,7 @@ create table instance_type_extra_specs ) ---- -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -206,18 +206,38 @@ project │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ ├── select + │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── flavorid:7 = $2 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":38!null + │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,38), (38)==(7), (7)==(38) + │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_flavorid_key) + │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null flavorid:7!null "$2":38!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ │ ├── key columns: [38] = [7] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,38), (38)==(7), (7)==(38) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$2":38 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(38) + │ │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters │ │ │ │ │ │ └── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] │ │ │ │ │ └── projections @@ -690,7 +710,7 @@ sort └── filters └── instance_type_extra_specs_1.instance_type_id:22 = instance_types.id:1 [outer=(1,22), constraints=(/1: (/NULL - ]; /22: (/NULL - ]), fd=(1)==(22), (22)==(1)] -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -796,26 +816,38 @@ project │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ ├── index-join instance_types + │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ └── select - │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13!null + │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$1":43!null "$4":44!null + │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1,2,13) - │ │ │ │ │ │ │ ├── scan instance_types@instance_types_name_deleted_key - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13 - │ │ │ │ │ │ │ │ ├── constraint: /2/13: (/NULL - ] - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(2,13), (2,13)~~>(1) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ ├── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ └── name:2 = $4 [outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + │ │ │ │ │ │ │ ├── fd: ()-->(1-16,43,44), (43)==(13), (2)==(44), (44)==(2), (13)==(43) + │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_name_deleted_key) + │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13!null "$1":43!null "$4":44!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ │ ├── key columns: [44 43] = [2 13] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1,2,13,43,44), (44)==(2), (13)==(43), (43)==(13), (2)==(44) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$1":43 "$4":44 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(43,44) + │ │ │ │ │ │ │ │ │ └── ($1, $4) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] │ │ │ │ │ │ └── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] @@ -829,7 +861,7 @@ project │ └── instance_type_extra_specs_1.deleted:37 = $7 [outer=(37), constraints=(/37: (/NULL - ]), fd=()-->(37)] └── filters (true) -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -922,38 +954,70 @@ project │ │ │ │ ├── has-placeholder │ │ │ │ ├── key: () │ │ │ │ ├── fd: ()-->(1-16,20,30) - │ │ │ │ ├── project - │ │ │ │ │ ├── columns: true:30 instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 instance_type_projects.instance_type_id:20 + │ │ │ │ ├── left-join (merge) + │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 instance_type_projects.instance_type_id:20 true:30 + │ │ │ │ │ ├── left ordering: +1 + │ │ │ │ │ ├── right ordering: +20 │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ ├── has-placeholder │ │ │ │ │ ├── key: () │ │ │ │ │ ├── fd: ()-->(1-16,20,30) - │ │ │ │ │ ├── left-join (lookup instance_type_projects@instance_type_projects_instance_type_id_project_id_deleted_key) - │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 instance_type_projects.instance_type_id:20 project_id:21 instance_type_projects.deleted:22 - │ │ │ │ │ │ ├── key columns: [1] = [20] + │ │ │ │ │ ├── project + │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 + │ │ │ │ │ │ ├── fd: ()-->(1-16) + │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$4":43!null "$1":44!null + │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ ├── key columns: [43] = [1] + │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ ├── fd: ()-->(1-16,43,44), (43)==(1), (13)==(44), (44)==(13), (1)==(43) + │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ ├── columns: "$4":43 "$1":44 + │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(43,44) + │ │ │ │ │ │ │ └── ($4, $1) + │ │ │ │ │ │ └── filters + │ │ │ │ │ │ └── instance_types.deleted:13 = "$1":44 [outer=(13,44), constraints=(/13: (/NULL - ]; /44: (/NULL - ]), fd=(13)==(44), (44)==(13)] + │ │ │ │ │ ├── project + │ │ │ │ │ │ ├── columns: true:30!null instance_type_projects.instance_type_id:20!null + │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ ├── fd: ()-->(20,30) + │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ ├── columns: instance_type_projects.instance_type_id:20!null project_id:21!null instance_type_projects.deleted:22!null │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ ├── scan instance_types - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13 instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(2-16), (7,13)~~>(1-6,8-12,14-16), (2,13)~~>(1,3-12,14-16) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ ├── instance_types.id:1 = $4 [outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] - │ │ │ │ │ │ │ └── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ └── filters - │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] - │ │ │ │ │ │ ├── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] - │ │ │ │ │ │ └── instance_type_projects.instance_type_id:20 = $4 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] - │ │ │ │ │ └── projections - │ │ │ │ │ └── CASE instance_type_projects.instance_type_id:20 IS NULL WHEN true THEN CAST(NULL AS BOOL) ELSE true END [as=true:30, outer=(20)] + │ │ │ │ │ │ │ ├── fd: ()-->(20-22) + │ │ │ │ │ │ │ └── inner-join (lookup instance_type_projects@instance_type_projects_instance_type_id_project_id_deleted_key) + │ │ │ │ │ │ │ ├── columns: instance_type_projects.instance_type_id:20!null project_id:21!null instance_type_projects.deleted:22!null "$2":45!null "$3":46!null "$4":47!null + │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ ├── key columns: [47 46 45] = [20 21 22] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(20-22,45-47), (45)==(22), (21)==(46), (46)==(21), (20)==(47), (47)==(20), (22)==(45) + │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ ├── columns: "$2":45 "$3":46 "$4":47 + │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(45-47) + │ │ │ │ │ │ │ │ └── ($2, $3, $4) + │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ └── projections + │ │ │ │ │ │ └── true [as=true:30] + │ │ │ │ │ └── filters (true) │ │ │ │ └── filters │ │ │ │ └── is_public:12 OR (true:30 IS NOT NULL) [outer=(12,30)] │ │ │ └── $5 @@ -962,7 +1026,7 @@ project │ └── instance_type_extra_specs_1.deleted:37 = $7 [outer=(37), constraints=(/37: (/NULL - ]), fd=()-->(37)] └── filters (true) -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -1059,18 +1123,38 @@ project │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ ├── select + │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── name:2 = $2 [outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":38!null + │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,38), (38)==(2), (2)==(38) + │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_name_key) + │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null "$2":38!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ │ ├── key columns: [38] = [2] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1,2,38), (38)==(2), (2)==(38) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$2":38 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(38) + │ │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters │ │ │ │ │ │ └── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] │ │ │ │ │ └── projections @@ -1082,7 +1166,7 @@ project │ └── filters (true) └── filters (true) -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -1179,18 +1263,38 @@ project │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ ├── select + │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── flavorid:7 = $2 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":38!null + │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,38), (38)==(7), (7)==(38) + │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_flavorid_key) + │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null flavorid:7!null "$2":38!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ │ ├── key columns: [38] = [7] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,38), (38)==(7), (7)==(38) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$2":38 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(38) + │ │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters │ │ │ │ │ │ └── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] │ │ │ │ │ └── projections @@ -1505,7 +1609,7 @@ sort └── filters └── instance_type_extra_specs_1.instance_type_id:36 = instance_types.id:1 [outer=(1,36), constraints=(/1: (/NULL - ]; /36: (/NULL - ]), fd=(1)==(36), (36)==(1)] -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -1611,26 +1715,38 @@ project │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ ├── index-join instance_types + │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ └── select - │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null + │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$1":43!null "$4":44!null + │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13) - │ │ │ │ │ │ │ ├── scan instance_types@instance_types_flavorid_deleted_key - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13 - │ │ │ │ │ │ │ │ ├── constraint: /7/13: (/NULL - ] - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(7,13), (7,13)~~>(1) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ ├── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ └── flavorid:7 = $4 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ ├── fd: ()-->(1-16,43,44), (43)==(13), (7)==(44), (44)==(7), (13)==(43) + │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_flavorid_deleted_key) + │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null "$1":43!null "$4":44!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ │ ├── key columns: [44 43] = [7 13] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13,43,44), (44)==(7), (13)==(43), (43)==(13), (7)==(44) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$1":43 "$4":44 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(43,44) + │ │ │ │ │ │ │ │ │ └── ($1, $4) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] │ │ │ │ │ │ └── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] @@ -2241,7 +2357,7 @@ sort └── filters └── instance_type_extra_specs_1.instance_type_id:50 = instance_types.id:1 [outer=(1,50), constraints=(/1: (/NULL - ]; /50: (/NULL - ]), fd=(1)==(50), (50)==(1)] -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -2350,26 +2466,38 @@ project │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ ├── index-join instance_types + │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ └── select - │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null + │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$1":43!null "$4":44!null + │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13) - │ │ │ │ │ │ │ ├── scan instance_types@instance_types_flavorid_deleted_key - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13 - │ │ │ │ │ │ │ │ ├── constraint: /7/13: (/NULL - ] - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(7,13), (7,13)~~>(1) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ ├── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ └── flavorid:7 = $4 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ ├── fd: ()-->(1-16,43,44), (43)==(13), (7)==(44), (44)==(7), (13)==(43) + │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_flavorid_deleted_key) + │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null "$1":43!null "$4":44!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ │ ├── key columns: [44 43] = [7 13] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13,43,44), (44)==(7), (13)==(43), (43)==(13), (7)==(44) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$1":43 "$4":44 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(43,44) + │ │ │ │ │ │ │ │ │ └── ($1, $4) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ └── filters │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] │ │ │ │ │ │ └── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] @@ -2383,7 +2511,7 @@ project │ └── instance_type_extra_specs_1.deleted:37 = $7 [outer=(37), constraints=(/37: (/NULL - ]), fd=()-->(37)] └── filters (true) -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -2467,36 +2595,69 @@ project │ │ │ │ ├── has-placeholder │ │ │ │ ├── key: () │ │ │ │ ├── fd: ()-->(1-12,14,15,19,27) - │ │ │ │ ├── project - │ │ │ │ │ ├── columns: true:27 flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 flavor_projects.flavor_id:19 + │ │ │ │ ├── left-join (merge) + │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 flavor_projects.flavor_id:19 true:27 + │ │ │ │ │ ├── left ordering: +1 + │ │ │ │ │ ├── right ordering: +19 │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ ├── has-placeholder │ │ │ │ │ ├── key: () │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,27) - │ │ │ │ │ ├── left-join (lookup flavor_projects@flavor_projects_flavor_id_project_id_key) - │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 flavor_projects.flavor_id:19 project_id:20 - │ │ │ │ │ │ ├── key columns: [1] = [19] + │ │ │ │ │ ├── project + │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 + │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) + │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":38!null + │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ ├── key columns: [38] = [1] + │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,38), (38)==(1), (1)==(38) + │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ ├── columns: "$2":38 + │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(38) + │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ ├── project + │ │ │ │ │ │ ├── columns: true:27!null flavor_projects.flavor_id:19!null + │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ ├── fd: ()-->(19,27) + │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ ├── columns: flavor_projects.flavor_id:19!null project_id:20!null │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ └── flavors.id:1 = $2 [outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] - │ │ │ │ │ │ └── filters - │ │ │ │ │ │ ├── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] - │ │ │ │ │ │ └── flavor_projects.flavor_id:19 = $2 [outer=(19), constraints=(/19: (/NULL - ]), fd=()-->(19)] - │ │ │ │ │ └── projections - │ │ │ │ │ └── CASE flavor_projects.flavor_id:19 IS NULL WHEN true THEN CAST(NULL AS BOOL) ELSE true END [as=true:27, outer=(19)] + │ │ │ │ │ │ │ ├── fd: ()-->(19,20) + │ │ │ │ │ │ │ └── inner-join (lookup flavor_projects@flavor_projects_flavor_id_project_id_key) + │ │ │ │ │ │ │ ├── columns: flavor_projects.flavor_id:19!null project_id:20!null "$1":39!null "$2":40!null + │ │ │ │ │ │ │ ├── flags: disallow merge join + │ │ │ │ │ │ │ ├── key columns: [40 39] = [19 20] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(19,20,39,40), (39)==(20), (19)==(40), (40)==(19), (20)==(39) + │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ ├── columns: "$1":39 "$2":40 + │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(39,40) + │ │ │ │ │ │ │ │ └── ($1, $2) + │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ └── projections + │ │ │ │ │ │ └── true [as=true:27] + │ │ │ │ │ └── filters (true) │ │ │ │ └── filters │ │ │ │ └── is_public:12 OR (true:27 IS NOT NULL) [outer=(12,27)] │ │ │ └── $3 diff --git a/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan b/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan index f45c1337140b..65c50c3bac28 100644 --- a/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan +++ b/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan @@ -95,6 +95,25 @@ SELECT v+1 FROM kv WHERE k = $1 ---- no fast path + +# Inject statistics so that the estimated row count is high. +exec-ddl +ALTER TABLE abcd INJECT STATISTICS '[ + { + "columns": ["a"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 5 + }, + { + "columns": ["b"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 5 + } +]' +---- + # The fast path should not kick in because the estimated row count is too high. placeholder-fast-path SELECT a, b, c FROM abcd WHERE a=$1 AND b=$2 @@ -120,7 +139,7 @@ SELECT a, b, c FROM abcd WHERE a=$1 AND b=$2 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=1.1, distinct(1)=1.1, null(1)=0, distinct(2)=1, null(2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── $1 @@ -132,7 +151,7 @@ SELECT a, b, c FROM abcd WHERE b=$1 AND a=$2 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=1.1, distinct(1)=1.1, null(1)=0, distinct(2)=1, null(2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── $2 @@ -145,7 +164,7 @@ SELECT a, b, c FROM abcd WHERE a=0 AND b=$1 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=0.666, distinct(1)=0.666, null(1)=0, distinct(2)=0.666, null(2)=0, distinct(1,2)=0.666, null(1,2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── 0 @@ -157,7 +176,7 @@ SELECT a, b, c FROM abcd WHERE a=$1 AND b=0 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=3.3, distinct(1)=3.3, null(1)=0, distinct(2)=1, null(2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── $1 @@ -170,7 +189,7 @@ SELECT a, b, c FROM abcd WHERE a=1+2 AND b=$1 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=0.666, distinct(1)=0.666, null(1)=0, distinct(2)=0.666, null(2)=0, distinct(1,2)=0.666, null(1,2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── 3 @@ -182,7 +201,7 @@ SELECT a, b, c FROM abcd WHERE a=fnv32a('foo') AND b=$1 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=0.666, distinct(1)=0.666, null(1)=0, distinct(2)=0.666, null(2)=0, distinct(1,2)=0.666, null(1,2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── 2851307223 @@ -230,7 +249,7 @@ SELECT a, d FROM abcd WHERE d=$1 AND c=$2 placeholder-scan abcd@abcd_d_c_a_idx ├── columns: a:1 d:4!null ├── has-placeholder - ├── stats: [rows=1.0989] + ├── stats: [rows=9.8901] ├── fd: ()-->(4) └── span ├── $1 @@ -290,7 +309,7 @@ SELECT a, b FROM partial1 WHERE a = $1 placeholder-scan partial1@pseudo_ab,partial ├── columns: a:2!null b:3 ├── has-placeholder - ├── stats: [rows=3.3, distinct(2)=1, null(2)=0] + ├── stats: [rows=9.9, distinct(2)=1, null(2)=0] ├── fd: ()-->(2) └── span └── $1 diff --git a/pkg/sql/opt/xform/testdata/rules/generic b/pkg/sql/opt/xform/testdata/rules/generic new file mode 100644 index 000000000000..fed232cac6d0 --- /dev/null +++ b/pkg/sql/opt/xform/testdata/rules/generic @@ -0,0 +1,674 @@ +exec-ddl +CREATE TABLE t ( + k INT PRIMARY KEY, + i INT, + s STRING, + b BOOL, + t TIMESTAMPTZ, + INDEX (i, s, b), + INDEX (i, t), + INDEX (t) +) +---- + +# -------------------------------------------------- +# GenerateParameterizedJoin +# -------------------------------------------------- + +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE k = $1 +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5 "$1":8!null + ├── flags: disallow merge join + ├── key columns: [8] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (8)==(1), (1)==(8) + ├── values + │ ├── columns: "$1":8 + │ ├── cardinality: [1 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8) + │ └── ($1,) + └── filters (true) + +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE k = $1::INT +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5 "$1":8!null + ├── flags: disallow merge join + ├── key columns: [8] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (8)==(1), (1)==(8) + ├── values + │ ├── columns: "$1":8 + │ ├── cardinality: [1 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8) + │ └── ($1,) + └── filters (true) + +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND s = $2 AND b = $3 +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4!null t:5 + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2-4), (1)-->(5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4!null t:5 "$1":8!null "$2":9!null "$3":10!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2-4,8-10), (1)-->(5), (2)==(8), (8)==(2), (3)==(9), (9)==(3), (4)==(10), (10)==(4) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3!null b:4!null "$1":8!null "$2":9!null "$3":10!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9 10] = [2 3 4] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2-4,8-10), (2)==(8), (8)==(2), (3)==(9), (9)==(3), (4)==(10), (10)==(4) + │ ├── values + │ │ ├── columns: "$1":8 "$2":9 "$3":10 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8-10) + │ │ └── ($1, $2, $3) + │ └── filters (true) + └── filters (true) + +# A placeholder referenced multiple times in the filters should only appear once +# in the Values expression. +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE k = $1 AND i = $1 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5 "$1":8!null + ├── flags: disallow merge join + ├── key columns: [8] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (2)==(1,8), (8)==(1,2), (1)==(2,8) + ├── values + │ ├── columns: "$1":8 + │ ├── cardinality: [1 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8) + │ └── ($1,) + └── filters + └── k:1 = i:2 [outer=(1,2), constraints=(/1: (/NULL - ]; /2: (/NULL - ]), fd=(1)==(2), (2)==(1)] + +# The generated join should not be reordered and merge joins should not be +# explored on it. +opt generic expect=GenerateParameterizedJoin expect-not=(ReorderJoins,GenerateMergeJoins) +SELECT * FROM t WHERE i = $1 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5 + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2), (1)-->(3-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5 "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2,8), (1)-->(3-5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5 "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [2] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,8), (1)-->(5), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters (true) + └── filters (true) + +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE k = (SELECT i FROM t WHERE k = $1) +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5 k:8!null i:9!null + ├── key columns: [9] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8,9), (9)==(1), (1)==(9) + ├── project + │ ├── columns: k:8!null i:9 + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8,9) + │ └── inner-join (lookup t) + │ ├── columns: k:8!null i:9 "$1":15!null + │ ├── flags: disallow merge join + │ ├── key columns: [15] = [8] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8,9,15), (15)==(8), (8)==(15) + │ ├── values + │ │ ├── columns: "$1":15 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(15) + │ │ └── ($1,) + │ └── filters (true) + └── filters (true) + +# TODO(mgartner): The rule doesn't apply because the filters do not reference +# the placeholder directly. Consider ways to handle cases like this. +opt generic +SELECT * FROM t WHERE k = (SELECT $1::INT) +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── select + ├── columns: k:1!null i:2 s:3 b:4 t:5 int8:8!null + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (8)==(1), (1)==(8) + ├── project + │ ├── columns: int8:8 k:1!null i:2 s:3 b:4 t:5 + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(8), (1)-->(2-5) + │ ├── scan t + │ │ ├── columns: k:1!null i:2 s:3 b:4 t:5 + │ │ ├── key: (1) + │ │ └── fd: (1)-->(2-5) + │ └── projections + │ └── $1 [as=int8:8] + └── filters + └── k:1 = int8:8 [outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(1)==(8), (8)==(1)] + +exec-ddl +CREATE INDEX partial_idx ON t(t) WHERE t IS NOT NULL +---- + +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE t = $1 +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5!null + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(5), (1)-->(2-4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5!null "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(5,8), (1)-->(2-4), (5)==(8), (8)==(5) + ├── inner-join (lookup t@partial_idx,partial) + │ ├── columns: k:1!null t:5!null "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [5] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(5,8), (5)==(8), (8)==(5) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- + +exec-ddl +CREATE INDEX partial_idx ON t(i, t) WHERE i IS NOT NULL AND t IS NOT NULL +---- + +opt generic expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = $2 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null "$2":9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8,9), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + ├── inner-join (lookup t@partial_idx,partial) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null "$2":9!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9] = [2 5] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8,9), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + │ ├── values + │ │ ├── columns: "$1":8 "$2":9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, $2) + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- + +exec-ddl +CREATE INDEX partial_idx ON t(s) WHERE k = i +---- + +opt generic +SELECT * FROM t@partial_idx WHERE s = $1 AND k = $2 AND i = $2 +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 "$1":8!null "$2":9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8,9), (2)==(1,9), (3)==(8), (8)==(3), (9)==(1,2), (1)==(2,9) + ├── inner-join (lookup t@partial_idx,partial) + │ ├── columns: k:1!null s:3!null "$1":8!null "$2":9!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9] = [3 1] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(1,3,8,9), (8)==(3), (1)==(9), (9)==(1), (3)==(8) + │ ├── values + │ │ ├── columns: "$1":8 "$2":9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, $2) + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- + +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE t = now() +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5!null + ├── stable + ├── key: (1) + ├── fd: ()-->(5), (1)-->(2-4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5!null column8:8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable + ├── key: (1) + ├── fd: ()-->(5,8), (1)-->(2-4), (5)==(8), (8)==(5) + ├── inner-join (lookup t@t_t_idx) + │ ├── columns: k:1!null t:5!null column8:8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [5] + │ ├── stable + │ ├── key: (1) + │ ├── fd: ()-->(5,8), (5)==(8), (8)==(5) + │ ├── values + │ │ ├── columns: column8:8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── (now(),) + │ └── filters (true) + └── filters (true) + +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = now() +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8,9), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9] = [2 5] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8,9), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + │ ├── values + │ │ ├── columns: "$1":8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, now()) + │ └── filters (true) + └── filters (true) + +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t > now() +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2), (1)-->(3-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,8,9), (1)-->(3-5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9!null + │ ├── flags: disallow merge join + │ ├── lookup expression + │ │ └── filters + │ │ ├── "$1":8 = i:2 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] + │ │ └── t:5 > column9:9 [outer=(5,9), constraints=(/5: (/NULL - ]; /9: (/NULL - ])] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,8,9), (1)-->(5), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, now()) + │ └── filters (true) + └── filters (true) + +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = now() + $2 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 "$2":10 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8-10), (1)-->(3,4), (2)==(8), (8)==(2) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 "$2":10 column11:11!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8-11), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(11), (11)==(5) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9 "$2":10 column11:11!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 11] = [2 5] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8-11), (2)==(8), (8)==(2), (5)==(11), (11)==(5) + │ ├── project + │ │ ├── columns: column11:11 "$1":8 column9:9 "$2":10 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8-11) + │ │ ├── values + │ │ │ ├── columns: "$1":8 column9:9 "$2":10 + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── stable, has-placeholder + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(8-10) + │ │ │ └── ($1, now(), $2) + │ │ └── projections + │ │ └── column9:9 + "$2":10 [as=column11:11, outer=(9,10), stable] + │ └── filters (true) + └── filters (true) + +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = now() + '1 hr'::INTERVAL +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8,9), (1)-->(3,4), (2)==(8), (8)==(2) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 column10:10!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8-10), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(10), (10)==(5) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9 column10:10!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 10] = [2 5] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8-10), (2)==(8), (8)==(2), (5)==(10), (10)==(5) + │ ├── project + │ │ ├── columns: column10:10 "$1":8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8-10) + │ │ ├── values + │ │ │ ├── columns: "$1":8 column9:9 + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── stable, has-placeholder + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(8,9) + │ │ │ └── ($1, now()) + │ │ └── projections + │ │ └── column9:9 + '01:00:00' [as=column10:10, outer=(9), stable] + │ └── filters (true) + └── filters (true) + +# TODO(mgartner): Apply the rule to stable, non-leaf expressions. +opt generic no-stable-folds +SELECT * FROM t WHERE t = '2024-01-01 12:00:00'::TIMESTAMP::TIMESTAMPTZ +---- +select + ├── columns: k:1!null i:2 s:3 b:4 t:5!null + ├── stable + ├── key: (1) + ├── fd: ()-->(5), (1)-->(2-4) + ├── scan t + │ ├── columns: k:1!null i:2 s:3 b:4 t:5 + │ ├── key: (1) + │ └── fd: (1)-->(2-5) + └── filters + └── t:5 = '2024-01-01 12:00:00'::TIMESTAMPTZ [outer=(5), stable, constraints=(/5: (/NULL - ]), fd=()-->(5)] + +# A stable function is not included in the Values expression if it has +# arguments. +# TODO(mgartner): We should be able to relax this restriction as long as all the +# arguments are constants or placeholders. +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND s = quote_literal(1::INT) +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3), (1)-->(4,5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3,8), (1)-->(4,5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3!null b:4 "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [2] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,3,8), (1)-->(4), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters + │ └── s:3 = quote_literal(1) [outer=(3), stable, constraints=(/3: (/NULL - ]), fd=()-->(3)] + └── filters (true) + +# A stable function is not included in the Values expression if its arguments +# reference a column from the table. This would create an illegal outer column +# reference in a non-apply-join. +opt generic no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND s = quote_literal(i) +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3), (1)-->(4,5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3,8), (1)-->(4,5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3!null b:4 "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [2] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,3,8), (1)-->(4), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters + │ └── s:3 = quote_literal(i:2) [outer=(2,3), stable, constraints=(/3: (/NULL - ]), fd=(2)-->(3)] + └── filters (true) + +# The rule does not match if there are no placeholders or stable expressions in +# the filters. +opt generic expect-not=GenerateParameterizedJoin +SELECT * FROM t WHERE i = 1 AND s = 'foo' +---- +index-join t + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── key: (1) + ├── fd: ()-->(2,3), (1)-->(4,5) + └── scan t@t_i_s_b_idx + ├── columns: k:1!null i:2!null s:3!null b:4 + ├── constraint: /2/3/4/1: [/1/'foo' - /1/'foo'] + ├── key: (1) + └── fd: ()-->(2,3), (1)-->(4) + +# The rule does not match if generic optimizations are disabled. +opt expect-not=GenerateParameterizedJoin +SELECT * FROM t WHERE k = $1 AND s = quote_literal(1::INT) +---- +select + ├── columns: k:1!null i:2 s:3!null b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + ├── scan t + │ ├── columns: k:1!null i:2 s:3 b:4 t:5 + │ ├── key: (1) + │ └── fd: (1)-->(2-5) + └── filters + ├── k:1 = $1 [outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] + └── s:3 = e'\'1\'' [outer=(3), constraints=(/3: [/e'\'1\'' - /e'\'1\'']; tight), fd=()-->(3)] diff --git a/pkg/sql/plan.go b/pkg/sql/plan.go index 156602f1f2cc..128a50010b9b 100644 --- a/pkg/sql/plan.go +++ b/pkg/sql/plan.go @@ -486,7 +486,11 @@ func (p *planTop) savePlanInfo() { distribution = physicalplan.PartiallyDistributedPlan } containsMutation := p.flags.IsSet(planFlagContainsMutation) - p.instrumentation.RecordPlanInfo(distribution, vectorized, containsMutation) + generic := p.flags.IsSet(planFlagGeneric) + optimized := p.flags.IsSet(planFlagOptimized) + p.instrumentation.RecordPlanInfo( + distribution, vectorized, containsMutation, generic, optimized, + ) } // startExec calls startExec() on each planNode using a depth-first, post-order @@ -631,6 +635,15 @@ const ( // planFlagSessionMigration is set if the plan is being created during // a session migration. planFlagSessionMigration + + // planFlagGeneric is set if a generic query plan was used. A generic query + // plan is a plan that is fully-optimized once and can be reused without + // being re-optimized. + planFlagGeneric + + // planFlagOptimized is set if optimization was performed during the + // current execution of the query. + planFlagOptimized ) func (pf planFlags) IsSet(flag planFlags) bool { diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index 7737f274926a..4014c119228e 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -55,7 +55,7 @@ var queryCacheEnabled = settings.RegisterBoolSetting( // - Columns // - Types // - AnonymizedStr -// - Memo (for reuse during exec, if appropriate). +// - BaseMemo (for reuse during exec, if appropriate). func (p *planner) prepareUsingOptimizer( ctx context.Context, origin PreparedStatementOrigin, ) (planFlags, error) { @@ -154,7 +154,7 @@ func (p *planner) prepareUsingOptimizer( stmt.Prepared.StatementNoConstants = pm.StatementNoConstants stmt.Prepared.Columns = pm.Columns stmt.Prepared.Types = pm.Types - stmt.Prepared.Memo = cachedData.Memo + stmt.Prepared.BaseMemo = cachedData.Memo return opc.flags, nil } opc.log(ctx, "query cache hit but memo is stale (prepare)") @@ -167,7 +167,8 @@ func (p *planner) prepareUsingOptimizer( opc.flags.Set(planFlagOptCacheMiss) } - memo, err := opc.buildReusableMemo(ctx) + // Build the memo. Do not attempt to build a generic plan at PREPARE-time. + memo, _, err := opc.buildReusableMemo(ctx, false /* buildGeneric */) if err != nil { return 0, err } @@ -217,7 +218,7 @@ func (p *planner) prepareUsingOptimizer( stmt.Prepared.Columns = resultCols stmt.Prepared.Types = p.semaCtx.Placeholders.Types if opc.allowMemoReuse { - stmt.Prepared.Memo = memo + stmt.Prepared.BaseMemo = memo if opc.useCache { // execPrepare sets the PrepareMetadata.InferredTypes field after this // point. However, once the PrepareMetadata goes into the cache, it @@ -407,30 +408,49 @@ func (opc *optPlanningCtx) log(ctx context.Context, msg redact.SafeString) { } } +type memoType int + +const ( + memoTypeUnknown memoType = iota + memoTypeCustom + memoTypeGeneric + memoTypeIdealGeneric +) + // buildReusableMemo builds the statement into a memo that can be stored for // prepared statements and can later be used as a starting point for -// optimization. The returned memo is fully detached from the planner and can be -// used with reuseMemo independently and concurrently by multiple threads. -func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, _ error) { +// optimization. The returned memo is fully optimized if: +// +// 1. The statement does not contain placeholders nor fold-able stable +// operators. +// 2. Or, the placeholder fast path is used. +// 3. Or, buildGeneric is true and the plan is fully optimized as best as +// possible in the presence of placeholders. +// +// The returned memo is fully detached from the planner and can be used with +// reuseMemo independently and concurrently by multiple threads. +func (opc *optPlanningCtx) buildReusableMemo( + ctx context.Context, buildGeneric bool, +) (*memo.Memo, memoType, error) { p := opc.p _, isCanned := opc.p.stmt.AST.(*tree.CannedOptPlan) if isCanned { if !p.EvalContext().SessionData().AllowPrepareAsOptPlan { - return nil, pgerror.New(pgcode.InsufficientPrivilege, + return nil, memoTypeUnknown, pgerror.New(pgcode.InsufficientPrivilege, "PREPARE AS OPT PLAN is a testing facility that should not be used directly", ) } if !p.SessionData().User().IsRootUser() { - return nil, pgerror.New(pgcode.InsufficientPrivilege, + return nil, memoTypeUnknown, pgerror.New(pgcode.InsufficientPrivilege, "PREPARE AS OPT PLAN may only be used by root", ) } } if p.SessionData().SaveTablesPrefix != "" && !p.SessionData().User().IsRootUser() { - return nil, pgerror.New(pgcode.InsufficientPrivilege, + return nil, memoTypeUnknown, pgerror.New(pgcode.InsufficientPrivilege, "sub-expression tables creation may only be used by root", ) } @@ -448,7 +468,7 @@ func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, bld.SkipAOST = true } if err := bld.Build(); err != nil { - return nil, err + return nil, memoTypeUnknown, err } if bld.DisableMemoReuse { @@ -462,38 +482,50 @@ func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, // We don't support placeholders inside the canned plan. The main reason // is that they would be invisible to the parser (which reports the number // of placeholders, used to initialize the relevant structures). - return nil, pgerror.Newf(pgcode.Syntax, + return nil, memoTypeUnknown, pgerror.Newf(pgcode.Syntax, "placeholders are not supported with PREPARE AS OPT PLAN") } - // With a canned plan, we don't want to optimize the memo. - return opc.optimizer.DetachMemo(ctx), nil + // With a canned plan, we don't want to optimize the memo. Since we + // won't optimize it, we consider it an ideal generic plan. + return opc.optimizer.DetachMemo(ctx), memoTypeIdealGeneric, nil } - if f.Memo().HasPlaceholders() { - // Try the placeholder fast path. - _, ok, err := opc.optimizer.TryPlaceholderFastPath() - if err != nil { - return nil, err - } - if ok { - opc.log(ctx, "placeholder fast path") + // If the memo doesn't have placeholders and did not encounter any stable + // operators that can be constant-folded, then fully optimize it now - it + // can be reused without further changes to build the execution tree. + if !f.Memo().HasPlaceholders() && !f.FoldingControl().PreventedStableFold() { + opc.log(ctx, "optimizing (no placeholders)") + if _, err := opc.optimizer.Optimize(); err != nil { + return nil, memoTypeUnknown, err } - } else { - // If the memo doesn't have placeholders and did not encounter any stable - // operators that can be constant folded, then fully optimize it now - it - // can be reused without further changes to build the execution tree. - if !f.FoldingControl().PreventedStableFold() { - opc.log(ctx, "optimizing (no placeholders)") - if _, err := opc.optimizer.Optimize(); err != nil { - return nil, err - } + opc.flags.Set(planFlagOptimized) + return opc.optimizer.DetachMemo(ctx), memoTypeIdealGeneric, nil + } + + // If the memo has placeholders, first try the placeholder fast path. + _, ok, err := opc.optimizer.TryPlaceholderFastPath() + if err != nil { + return nil, memoTypeUnknown, err + } + if ok { + opc.log(ctx, "placeholder fast path") + opc.flags.Set(planFlagOptimized) + return opc.optimizer.DetachMemo(ctx), memoTypeIdealGeneric, nil + } else if buildGeneric { + // Build a generic query plan if the placeholder fast path failed and a + // generic plan was requested. + opc.log(ctx, "optimizing (generic)") + if _, err := opc.optimizer.Optimize(); err != nil { + return nil, memoTypeUnknown, err } + opc.flags.Set(planFlagOptimized) + return opc.optimizer.DetachMemo(ctx), memoTypeGeneric, nil } // Detach the prepared memo from the factory and transfer its ownership // to the prepared statement. DetachMemo will re-initialize the optimizer // to an empty memo. - return opc.optimizer.DetachMemo(ctx), nil + return opc.optimizer.DetachMemo(ctx), memoTypeCustom, nil } // reuseMemo returns an optimized memo using a cached memo as a starting point. @@ -506,10 +538,11 @@ func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, func (opc *optPlanningCtx) reuseMemo( ctx context.Context, cachedMemo *memo.Memo, ) (*memo.Memo, error) { + opc.incPlanTypeTelemetry(cachedMemo) if cachedMemo.IsOptimized() { - // The query could have been already fully optimized if there were no - // placeholders or the placeholder fast path succeeded (see - // buildReusableMemo). + // The query could have been already fully optimized in + // buildReusableMemo, in which case it is considered a "generic" plan. + opc.flags.Set(planFlagGeneric) return cachedMemo, nil } f := opc.optimizer.Factory() @@ -524,42 +557,272 @@ func (opc *optPlanningCtx) reuseMemo( if _, err := opc.optimizer.Optimize(); err != nil { return nil, err } - return f.Memo(), nil + opc.flags.Set(planFlagOptimized) + mem := f.Memo() + if prep := opc.p.stmt.Prepared; opc.allowMemoReuse && prep != nil { + prep.Costs.AddCustom(mem.RootExpr().(memo.RelExpr).Cost() + mem.OptimizationCost()) + } + return mem, nil } -// buildExecMemo creates a fully optimized memo, possibly reusing a previously -// cached memo as a starting point. +// incPlanTypeTelemetry increments the telemetry counters for the type of the +// plan: generic or custom. +func (opc *optPlanningCtx) incPlanTypeTelemetry(cachedMemo *memo.Memo) { + switch opc.p.SessionData().PlanCacheMode { + case sessiondatapb.PlanCacheModeForceCustom: + telemetry.Inc(sqltelemetry.PlanTypeForceCustomCounter) + case sessiondatapb.PlanCacheModeForceGeneric: + telemetry.Inc(sqltelemetry.PlanTypeForceGenericCounter) + case sessiondatapb.PlanCacheModeAuto: + if cachedMemo.IsOptimized() { + // A fully optimized memo is generic. + telemetry.Inc(sqltelemetry.PlanTypeAutoGenericCounter) + } else { + telemetry.Inc(sqltelemetry.PlanTypeAutoCustomCounter) + } + } +} + +// useGenericPlan returns true if a generic query plan should be used instead of +// a custom plan. +func (opc *optPlanningCtx) useGenericPlan() bool { + switch opc.p.SessionData().PlanCacheMode { + case sessiondatapb.PlanCacheModeForceGeneric: + return true + case sessiondatapb.PlanCacheModeAuto: + prep := opc.p.stmt.Prepared + // We need to build CustomPlanThreshold custom plans before considering + // a generic plan. + if prep.Costs.NumCustom() < CustomPlanThreshold { + return false + } + // A generic plan should be used if we have CustomPlanThreshold custom + // plan costs and: + // + // 1. The generic cost is unknown because a generic plan has not been + // built. + // 2. Or, the cost of the generic plan is less than or equal to the + // average cost of the custom plans. + // + return prep.Costs.Generic() == 0 || prep.Costs.Generic() < prep.Costs.AvgCustom() + default: + return false + } +} + +// chooseValidPreparedMemo returns an optimized memo that is equal to, or built +// from, baseMemo or genericMemo. It returns nil if both memos are stale. It +// selects baseMemo or genericMemo based on the following rules, in order: // -// The returned memo is only safe to use in one thread, during execution of the -// current statement. -func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ error) { - if resumeProc := opc.p.storedProcTxnState.getResumeProc(); resumeProc != nil { - // We are executing a stored procedure which has paused to commit or - // rollback its transaction. Use resumeProc to resume execution in a new - // transaction where the control statement left off. - opc.log(ctx, "resuming stored procedure execution in a new transaction") - return opc.reuseMemo(ctx, resumeProc) +// 1. If baseMemo is fully optimized and not stale, it is returned as-is. +// 2. If plan_cache_mode=force_generic_plan is true then genericMemo is +// returned as-is if it is not stale. +// 3. If plan_cache_mode=auto, there have been at least 5 custom plans +// generated, and the cost of the generic memo is less than the average cost +// of the custom plans, then the generic memo is returned as-is if it is not +// stale. If the cost of the generic memo is greater than or equal to the +// average cost of the custom plans, then the baseMemo is returned if it is +// not stale. +// 4. If plan_cache_mode=force_custom_plan, baseMemo is returned if it is not +// stale. +// 5. Otherwise, nil is returned and the caller is responsible for building a +// new memo. +// +// The logic is structured to avoid unnecessary (*memo.Memo).IsStale calls, +// since they can be expensive. +func (opc *optPlanningCtx) chooseValidPreparedMemo( + ctx context.Context, baseMemo *memo.Memo, genericMemo *memo.Memo, +) (*memo.Memo, error) { + // First check for a fully optimized, non-stale, base memo. + if baseMemo != nil && baseMemo.IsOptimized() { + isStale, err := baseMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) + if err != nil { + return nil, err + } else if !isStale { + return baseMemo, nil + } + } + + prep := opc.p.stmt.Prepared + reuseGeneric := opc.useGenericPlan() + + // Next check for a non-stale, generic memo. + if reuseGeneric && genericMemo != nil { + isStale, err := genericMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) + if err != nil { + return nil, err + } else if !isStale { + return genericMemo, nil + } else { + // Clear the generic cost if the memo is stale. DDL or new stats + // could drastically change the cost of generic and custom plans, so + // we should re-consider which to use. + prep.Costs.ClearGeneric() + } + } + + // Next, check for a non-stale, normalized memo, if a generic memo should + // not be reused. + if !reuseGeneric && baseMemo != nil && !baseMemo.IsOptimized() { + isStale, err := baseMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) + if err != nil { + return nil, err + } else if !isStale { + return baseMemo, nil + } else { + // Clear the custom costs if the memo is stale. DDL or new stats + // could drastically change the cost of generic and custom plans, so + // we should re-consider which to use. + prep.Costs.ClearCustom() + } } + // A valid memo was not found. + return nil, nil +} + +// fetchPreparedMemo attempts to fetch a memo from the prepared statement +// struct. If a valid (i.e., non-stale) memo is found, it is used. Otherwise, a +// new statement will be built. +// +// The plan_cache_mode session setting controls how this function decides +// between what type of memo to use or reuse: +// +// - force_custom_plan: A fully optimized generic memo will be used if it +// either has no placeholders nor fold-able stable expressions, or it +// utilizes the placeholder fast-path. Otherwise, a normalized memo will be +// fetched or rebuilt, copied into a new memo with placeholders replaced +// with values, and re-optimized. +// +// - force_generic_plan: A fully optimized generic memo will always be used. +// The BaseMemo will be used if it is fully optimized. Otherwise, the +// GenericMemo will be used. +// +// - auto: A "custom plan" will be optimized for first five executions of the +// prepared statement. On the sixth execution, a "generic plan" will be +// generated. If its cost is less than the average cost of the custom plans +// (plus some optimization overhead cost), then the generic plan will be +// used. Otherwise, a custom plan will be used. +func (opc *optPlanningCtx) fetchPreparedMemo(ctx context.Context) (_ *memo.Memo, err error) { + p := opc.p + prep := p.stmt.Prepared + if !opc.allowMemoReuse || prep == nil { + return nil, nil + } + + // If the statement was previously prepared, check for a reusable memo. + // First check for a valid (non-stale) memo. + validMemo, err := opc.chooseValidPreparedMemo(ctx, prep.BaseMemo, prep.GenericMemo) + if err != nil { + return nil, err + } + if validMemo != nil { + opc.log(ctx, "reusing cached memo") + return opc.reuseMemo(ctx, validMemo) + } + + // Otherwise, we need to rebuild the memo. + // + // TODO(mgartner): If we have a non-stale, normalized base memo, we can + // build a generic memo from it instead of building the memo from + // scratch. + opc.log(ctx, "rebuilding cached memo") + buildGeneric := opc.useGenericPlan() + newMemo, typ, err := opc.buildReusableMemo(ctx, buildGeneric) + if err != nil { + return nil, err + } + switch typ { + case memoTypeIdealGeneric: + // If we have an "ideal" generic memo, store it as a base memo. It will + // always be used regardless of plan_cache_mode, so there is no need to + // set GenericCost. + prep.BaseMemo = newMemo + case memoTypeGeneric: + prep.GenericMemo = newMemo + prep.Costs.SetGeneric(newMemo.RootExpr().(memo.RelExpr).Cost()) + // Now that the cost of the generic plan is known, we need to + // re-evaluate the decision to use a generic or custom plan. + if !opc.useGenericPlan() { + // The generic plan that we just built is too expensive, so we need + // to build a custom plan. We recursively call fetchPreparedMemo in + // case we have a custom plan that can be reused as a starting point + // for optimization. The function should not recurse more than once. + return opc.fetchPreparedMemo(ctx) + } + case memoTypeCustom: + prep.BaseMemo = newMemo + default: + return nil, errors.AssertionFailedf("unexpected memo type %v", typ) + } + + // Re-optimize the memo, if necessary. + return opc.reuseMemo(ctx, newMemo) +} + +// fetchPreparedMemoLegacy attempts to fetch a prepared memo. If a valid (i.e., +// non-stale) memo is found, it is used. Otherwise, a new statement will be +// built. If memo reuse is not allowed, nil is returned. +func (opc *optPlanningCtx) fetchPreparedMemoLegacy(ctx context.Context) (_ *memo.Memo, err error) { prepared := opc.p.stmt.Prepared p := opc.p - if opc.allowMemoReuse && prepared != nil && prepared.Memo != nil { + if opc.allowMemoReuse && prepared != nil && prepared.BaseMemo != nil { // We are executing a previously prepared statement and a reusable memo is // available. // If the prepared memo has been invalidated by schema or other changes, // re-prepare it. - if isStale, err := prepared.Memo.IsStale(ctx, p.EvalContext(), opc.catalog); err != nil { + if isStale, err := prepared.BaseMemo.IsStale(ctx, p.EvalContext(), opc.catalog); err != nil { return nil, err } else if isStale { opc.log(ctx, "rebuilding cached memo") - prepared.Memo, err = opc.buildReusableMemo(ctx) + prepared.BaseMemo, _, err = opc.buildReusableMemo(ctx, false /* buildGeneric */) if err != nil { return nil, err } } opc.log(ctx, "reusing cached memo") - return opc.reuseMemo(ctx, prepared.Memo) + return opc.reuseMemo(ctx, prepared.BaseMemo) + } + + return nil, nil +} + +// buildExecMemo creates a fully optimized memo, possibly reusing a previously +// cached memo as a starting point. +// +// The returned memo is only safe to use in one thread, during execution of the +// current statement. +func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ error) { + if resumeProc := opc.p.storedProcTxnState.getResumeProc(); resumeProc != nil { + // We are executing a stored procedure which has paused to commit or + // rollback its transaction. Use resumeProc to resume execution in a new + // transaction where the control statement left off. + opc.log(ctx, "resuming stored procedure execution in a new transaction") + return opc.reuseMemo(ctx, resumeProc) + } + + p := opc.p + if p.SessionData().PlanCacheMode == sessiondatapb.PlanCacheModeForceCustom { + // Fallback to the legacy logic for reusing memos if plan_cache_mode is + // set to force_custom_plan. + m, err := opc.fetchPreparedMemoLegacy(ctx) + if err != nil { + return nil, err + } + if m != nil { + return m, nil + } + } else { + // Use new logic for reusing memos if plan_cache_mode is set to + // force_generic_plan or auto. + m, err := opc.fetchPreparedMemo(ctx) + if err != nil { + return nil, err + } + if m != nil { + return m, nil + } } if opc.useCache { @@ -570,7 +833,7 @@ func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ e return nil, err } else if isStale { opc.log(ctx, "query cache hit but needed update") - cachedData.Memo, err = opc.buildReusableMemo(ctx) + cachedData.Memo, _, err = opc.buildReusableMemo(ctx, false /* buildGeneric */) if err != nil { return nil, err } diff --git a/pkg/sql/prepared_stmt.go b/pkg/sql/prepared_stmt.go index c1a0736f611f..edc1123bf75e 100644 --- a/pkg/sql/prepared_stmt.go +++ b/pkg/sql/prepared_stmt.go @@ -55,10 +55,27 @@ const ( type PreparedStatement struct { querycache.PrepareMetadata - // Memo is the memoized data structure constructed by the cost-based optimizer - // during prepare of a SQL statement. It can significantly speed up execution - // if it is used by the optimizer as a starting point. - Memo *memo.Memo + // BaseMemo is the memoized data structure constructed by the cost-based + // optimizer during prepare of a SQL statement. + // + // It may be a fully-optimized memo if it contains an "ideal generic plan" + // that is guaranteed to be optimal across all executions of the prepared + // statement. Ideal generic plans are generated when the statement has no + // placeholders nor fold-able stable expressions, or when the placeholder + // fast-path is utilized. + // + // If it is not an ideal generic plan, it is an unoptimized, normalized + // memo that is used as a starting point for optimization of custom plans. + BaseMemo *memo.Memo + + // GenericMemo, if present, is a fully-optimized memo that can be executed + // as-is. + // TODO(mgartner): Put all fully-optimized plans in the GenericMemo field to + // reduce confusion. + GenericMemo *memo.Memo + + // Costs tracks the costs of previously optimized custom and generic plans. + Costs planCosts // refCount keeps track of the number of references to this PreparedStatement. // New references are registered through incRef(). @@ -84,8 +101,11 @@ func (p *PreparedStatement) MemoryEstimate() int64 { // 1. Size of the prepare metadata. // 2. Size of the prepared memo, if using the cost-based optimizer. size := p.PrepareMetadata.MemoryEstimate() - if p.Memo != nil { - size += p.Memo.MemoryEstimate() + if p.BaseMemo != nil { + size += p.BaseMemo.MemoryEstimate() + } + if p.GenericMemo != nil { + size += p.GenericMemo.MemoryEstimate() } return size } @@ -107,6 +127,75 @@ func (p *PreparedStatement) incRef(ctx context.Context) { p.refCount++ } +const ( + // CustomPlanThreshold is the maximum number of custom plan costs tracked by + // planCosts. It is also the number of custom plans executed when + // plan_cache_mode=auto before attempting to generate a generic plan. + CustomPlanThreshold = 5 +) + +// planCosts tracks costs of generic and custom plans. +type planCosts struct { + generic memo.Cost + custom struct { + nextIdx int + length int + costs [CustomPlanThreshold]memo.Cost + } +} + +// Generic returns the cost of the generic plan. +func (p *planCosts) Generic() memo.Cost { + return p.generic +} + +// SetGeneric sets the cost of the generic plan. +func (p *planCosts) SetGeneric(cost memo.Cost) { + p.generic = cost +} + +// AddCustom adds a custom plan cost to the planCosts, evicting the oldest cost +// if necessary. +func (p *planCosts) AddCustom(cost memo.Cost) { + p.custom.costs[p.custom.nextIdx] = cost + p.custom.nextIdx++ + if p.custom.nextIdx >= CustomPlanThreshold { + p.custom.nextIdx = 0 + } + if p.custom.length < CustomPlanThreshold { + p.custom.length++ + } +} + +// NumCustom returns the number of custom plan costs in the planCosts. +func (p *planCosts) NumCustom() int { + return p.custom.length +} + +// AvgCustom returns the average cost of all the custom plan costs in planCosts. +// If there are no custom plan costs, it returns 0. +func (p *planCosts) AvgCustom() memo.Cost { + if p.custom.length == 0 { + return 0 + } + var sum memo.Cost + for i := 0; i < p.custom.length; i++ { + sum += p.custom.costs[i] + } + return sum / memo.Cost(p.custom.length) +} + +// ClearGeneric clears the generic cost. +func (p *planCosts) ClearGeneric() { + p.generic = 0 +} + +// ClearCustom clears any previously added custom costs. +func (p *planCosts) ClearCustom() { + p.custom.nextIdx = 0 + p.custom.length = 0 +} + // preparedStatementsAccessor gives a planner access to a session's collection // of prepared statements. type preparedStatementsAccessor interface { diff --git a/pkg/sql/prepared_stmt_test.go b/pkg/sql/prepared_stmt_test.go new file mode 100644 index 000000000000..f0293e1f29d3 --- /dev/null +++ b/pkg/sql/prepared_stmt_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package sql + +import ( + "testing" + + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" +) + +func TestPlanCosts(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + type testCase struct { + input []memo.Cost + expectedNum int + expectedAvg memo.Cost + } + testCases := []testCase{ + {input: []memo.Cost{}, expectedNum: 0, expectedAvg: 0}, + {input: []memo.Cost{0, 0}, expectedNum: 2, expectedAvg: 0}, + {input: []memo.Cost{1, 1}, expectedNum: 2, expectedAvg: 1}, + {input: []memo.Cost{1, 2, 3, 4, 5}, expectedNum: 5, expectedAvg: 3}, + {input: []memo.Cost{1, 2, 3, 4, 5, 6}, expectedNum: 5, expectedAvg: 4}, + {input: []memo.Cost{9, 9, 9, 9, 9, 1, 2, 3, 4, 5}, expectedNum: 5, expectedAvg: 3}, + } + var pc planCosts + for _, tc := range testCases { + pc.ClearCustom() + for _, cost := range tc.input { + pc.AddCustom(cost) + } + if pc.NumCustom() != tc.expectedNum { + t.Errorf("expected Len() to be %d, got %d", tc.expectedNum, pc.NumCustom()) + } + if pc.AvgCustom() != tc.expectedAvg { + t.Errorf("expected Avg() to be %f, got %f", tc.expectedAvg, pc.AvgCustom()) + } + } +} diff --git a/pkg/sql/schemachanger/comparator_generated_test.go b/pkg/sql/schemachanger/comparator_generated_test.go index 859a6604cc3a..8023f3ecda1a 100644 --- a/pkg/sql/schemachanger/comparator_generated_test.go +++ b/pkg/sql/schemachanger/comparator_generated_test.go @@ -688,6 +688,11 @@ func TestSchemaChangeComparator_generator_probe_ranges(t *testing.T) { var logicTestFile = "pkg/sql/logictest/testdata/logic_test/generator_probe_ranges" runSchemaChangeComparatorTest(t, logicTestFile) } +func TestSchemaChangeComparator_generic_license(t *testing.T) { + defer leaktest.AfterTest(t)() + var logicTestFile = "pkg/sql/logictest/testdata/logic_test/generic_license" + runSchemaChangeComparatorTest(t, logicTestFile) +} func TestSchemaChangeComparator_geospatial(t *testing.T) { defer leaktest.AfterTest(t)() var logicTestFile = "pkg/sql/logictest/testdata/logic_test/geospatial" diff --git a/pkg/sql/sessiondatapb/local_only_session_data.proto b/pkg/sql/sessiondatapb/local_only_session_data.proto index f9cf95db0a90..598c9c1e0614 100644 --- a/pkg/sql/sessiondatapb/local_only_session_data.proto +++ b/pkg/sql/sessiondatapb/local_only_session_data.proto @@ -523,6 +523,9 @@ message LocalOnlySessionData { // OptimizerPushOffsetIntoIndexJoin, when true, indicates that the optimizer // should push offset expressions into index joins. bool optimizer_push_offset_into_index_join = 132; + // PlanCacheMode indicates the method that the optimizer should use to choose + // between a custom and generic query plan. + PlanCacheMode plan_cache_mode = 133; /////////////////////////////////////////////////////////////////////////// // WARNING: consider whether a session parameter you're adding needs to // @@ -539,6 +542,23 @@ enum ReplicationMode { REPLICATION_MODE_DATABASE = 2; } +// PlanCacheMode controls the optimizer's decision to use a custom or generic +// query plan. +enum PlanCacheMode { + option (gogoproto.goproto_enum_prefix) = false; + option (gogoproto.goproto_enum_stringer) = false; + + // PlanCacheModeForceCustom forces the optimizer to use a custom query plan. + force_custom_plan = 0 [(gogoproto.enumvalue_customname) = "PlanCacheModeForceCustom"]; + + // PlanCacheModeForceCustom forces the optimizer to use a generic query plan. + force_generic_plan = 1 [(gogoproto.enumvalue_customname) = "PlanCacheModeForceGeneric"]; + + // PlanCacheModeAuto allows the optimizer to automatically choose between a + // custom and generic query plan. + auto = 2 [(gogoproto.enumvalue_customname) = "PlanCacheModeAuto"]; +} + // SequenceCacheEntry is an entry in a SequenceCache. message SequenceCacheEntry { // CachedVersion stores the descpb.DescriptorVersion that cached values are associated with. diff --git a/pkg/sql/sessiondatapb/session_data.go b/pkg/sql/sessiondatapb/session_data.go index d0db8c1c640e..cf37f2546220 100644 --- a/pkg/sql/sessiondatapb/session_data.go +++ b/pkg/sql/sessiondatapb/session_data.go @@ -80,6 +80,25 @@ func VectorizeExecModeFromString(val string) (VectorizeExecMode, bool) { return m, true } +func (m PlanCacheMode) String() string { + name, ok := PlanCacheMode_name[int32(m)] + if !ok { + return fmt.Sprintf("invalid (%d)", m) + } + return name +} + +// PlanCacheModeFromString converts a string into a PlanCacheMode. False is +// returned if the conversion was unsuccessful. +func PlanCacheModeFromString(val string) (PlanCacheMode, bool) { + lowerVal := strings.ToLower(val) + m, ok := PlanCacheMode_value[lowerVal] + if !ok { + return 0, false + } + return PlanCacheMode(m), true +} + // User retrieves the current user. func (s *SessionData) User() username.SQLUsername { return s.UserProto.Decode() diff --git a/pkg/sql/sessiondatapb/session_data.proto b/pkg/sql/sessiondatapb/session_data.proto index c069a7ae5ba5..03896dd215fb 100644 --- a/pkg/sql/sessiondatapb/session_data.proto +++ b/pkg/sql/sessiondatapb/session_data.proto @@ -162,7 +162,7 @@ message DataConversionConfig { util.timeutil.pgdate.DateStyle date_style = 4 [(gogoproto.nullable) = false]; } -// VectorizeExecMode controls if an when the Executor executes queries using +// VectorizeExecMode controls if and when the Executor executes queries using // the columnar execution engine. enum VectorizeExecMode { option (gogoproto.goproto_enum_prefix) = false; diff --git a/pkg/sql/sqltelemetry/planning.go b/pkg/sql/sqltelemetry/planning.go index 4e62fb641bd0..7f1789ed35e0 100644 --- a/pkg/sql/sqltelemetry/planning.go +++ b/pkg/sql/sqltelemetry/planning.go @@ -209,6 +209,22 @@ var CancelQueriesUseCounter = telemetry.GetCounterOnce("sql.session.cancel-queri // CANCEL SESSIONS is run. var CancelSessionsUseCounter = telemetry.GetCounterOnce("sql.session.cancel-sessions") +// PlanTypeForceCustomCounter is to be incremented whenever a custom plan is +// used when plan_cache_mode=force_custom_plan. +var PlanTypeForceCustomCounter = telemetry.GetCounterOnce("sql.plan.type.force-custom") + +// PlanTypeForceGenericCounter is to be incremented whenever a generic plan is used +// when plan_cache_mode=force_generic_plan. +var PlanTypeForceGenericCounter = telemetry.GetCounterOnce("sql.plan.type.force-generic") + +// PlanTypeAutoCustomCounter is to be incremented whenever a generic plan is +// used when plan_cache_mode=auto. +var PlanTypeAutoCustomCounter = telemetry.GetCounterOnce("sql.plan.type.auto-custom") + +// PlanTypeAutoGenericCounter is to be incremented whenever a custom plan is +// used when plan_cache_mode=auto. +var PlanTypeAutoGenericCounter = telemetry.GetCounterOnce("sql.plan.type.auto-generic") + // We can't parameterize these telemetry counters, so just make a bunch of // buckets for setting the join reorder limit since the range of reasonable // values for the join reorder limit is quite small. diff --git a/pkg/sql/testdata/explain_tree b/pkg/sql/testdata/explain_tree index 5803523fc74b..4fbef824d764 100644 --- a/pkg/sql/testdata/explain_tree +++ b/pkg/sql/testdata/explain_tree @@ -11,6 +11,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -46,6 +47,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -81,6 +83,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -162,6 +165,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -197,6 +201,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -289,6 +294,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable diff --git a/pkg/sql/vars.go b/pkg/sql/vars.go index 8d48ea6a705c..01b0e8fb7e1a 100644 --- a/pkg/sql/vars.go +++ b/pkg/sql/vars.go @@ -32,6 +32,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/colfetcher" "github.com/cockroachdb/cockroach/pkg/sql/delegate" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" + "github.com/cockroachdb/cockroach/pkg/sql/gpq" "github.com/cockroachdb/cockroach/pkg/sql/lex" "github.com/cockroachdb/cockroach/pkg/sql/paramparse" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" @@ -3396,6 +3397,35 @@ var varGen = map[string]sessionVar{ }, GlobalDefault: globalTrue, }, + + `plan_cache_mode`: { + Set: func(_ context.Context, m sessionDataMutator, s string) error { + mode, ok := sessiondatapb.PlanCacheModeFromString(s) + if !ok { + return newVarValueError( + `plan_cache_mode`, + s, + sessiondatapb.PlanCacheModeForceCustom.String(), + sessiondatapb.PlanCacheModeForceGeneric.String(), + sessiondatapb.PlanCacheModeAuto.String(), + ) + } + if mode == sessiondatapb.PlanCacheModeForceGeneric || + mode == sessiondatapb.PlanCacheModeAuto { + if err := gpq.CheckClusterSupportsGenericQueryPlans(m.settings); err != nil { + return err + } + } + m.SetPlanCacheMode(mode) + return nil + }, + Get: func(evalCtx *extendedEvalContext, _ *kv.Txn) (string, error) { + return evalCtx.SessionData().PlanCacheMode.String(), nil + }, + GlobalDefault: func(sv *settings.Values) string { + return sessiondatapb.PlanCacheModeForceCustom.String() + }, + }, } func ReplicationModeFromString(s string) (sessiondatapb.ReplicationMode, error) {