From ab44516e937215ec1c44fb4b6e5c1682278b724a Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Fri, 14 Jun 2024 14:59:27 -0400 Subject: [PATCH 01/15] sql: add plan_cache_mode session setting The `plan_cache_mode` session setting has been added. It allows for three options: `force_custom_plan`, `force_generic_plan`, and `auto`. The session currently has no effect. In future commits it will control how the system chooses between using a custom or generic query plan. Release note: None --- pkg/sql/exec_util.go | 4 ++++ .../testdata/logic_test/information_schema | 7 +++--- .../logictest/testdata/logic_test/pg_catalog | 3 +++ .../logictest/testdata/logic_test/show_source | 1 + .../local_only_session_data.proto | 20 ++++++++++++++++ pkg/sql/sessiondatapb/session_data.go | 19 +++++++++++++++ pkg/sql/sessiondatapb/session_data.proto | 2 +- pkg/sql/vars.go | 23 +++++++++++++++++++ 8 files changed, 75 insertions(+), 4 deletions(-) diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go index c402af753b90..c5349b70525f 100644 --- a/pkg/sql/exec_util.go +++ b/pkg/sql/exec_util.go @@ -3828,6 +3828,10 @@ func (m *sessionDataMutator) SetOptimizerPushOffsetIntoIndexJoin(val bool) { m.data.OptimizerPushOffsetIntoIndexJoin = val } +func (m *sessionDataMutator) SetPlanCacheMode(val sessiondatapb.PlanCacheMode) { + m.data.PlanCacheMode = val +} + // Utility functions related to scrubbing sensitive information on SQL Stats. // quantizeCounts ensures that the Count field in the diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index cc2b8d091d18..e5cbcc55654f 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -4836,19 +4836,19 @@ statement ok CREATE SEQUENCE test_seq_3 AS smallint statement ok -CREATE SEQUENCE test_seq_4 AS integer +CREATE SEQUENCE test_seq_4 AS integer statement ok CREATE SEQUENCE test_seq_5 AS bigint statement ok -CREATE SEQUENCE test_seq_6 AS INT2 +CREATE SEQUENCE test_seq_6 AS INT2 statement ok CREATE SEQUENCE test_seq_7 AS INT4 statement ok -CREATE SEQUENCE test_seq_8 AS INT8 +CREATE SEQUENCE test_seq_8 AS INT8 query TTTTIIITTTTT colnames,rowsort SELECT * FROM information_schema.sequences @@ -5591,6 +5591,7 @@ override_multi_region_zone_config off parallelize_multi_key_lookup_joins_enabled off password_encryption scram-sha-256 pg_trgm.similarity_threshold 0.3 +plan_cache_mode force_custom_plan prefer_lookup_joins_for_fks off prepared_statements_cache_size 0 B propagate_input_ordering off diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog index 5cc039c36a73..8c0ac82e17d9 100644 --- a/pkg/sql/logictest/testdata/logic_test/pg_catalog +++ b/pkg/sql/logictest/testdata/logic_test/pg_catalog @@ -2908,6 +2908,7 @@ override_multi_region_zone_config off N parallelize_multi_key_lookup_joins_enabled off NULL NULL NULL string password_encryption scram-sha-256 NULL NULL NULL string pg_trgm.similarity_threshold 0.3 NULL NULL NULL string +plan_cache_mode force_custom_plan NULL NULL NULL string prefer_lookup_joins_for_fks off NULL NULL NULL string prepared_statements_cache_size 0 B NULL NULL NULL string propagate_input_ordering off NULL NULL NULL string @@ -3090,6 +3091,7 @@ override_multi_region_zone_config off N parallelize_multi_key_lookup_joins_enabled off NULL user NULL off off password_encryption scram-sha-256 NULL user NULL scram-sha-256 scram-sha-256 pg_trgm.similarity_threshold 0.3 NULL user NULL 0.3 0.3 +plan_cache_mode force_custom_plan NULL user NULL force_custom_plan force_custom_plan prefer_lookup_joins_for_fks off NULL user NULL off off prepared_statements_cache_size 0 B NULL user NULL 0 B 0 B propagate_input_ordering off NULL user NULL off off @@ -3271,6 +3273,7 @@ override_multi_region_zone_config NULL NULL NULL parallelize_multi_key_lookup_joins_enabled NULL NULL NULL NULL NULL password_encryption NULL NULL NULL NULL NULL pg_trgm.similarity_threshold NULL NULL NULL NULL NULL +plan_cache_mode NULL NULL NULL NULL NULL prefer_lookup_joins_for_fks NULL NULL NULL NULL NULL prepared_statements_cache_size NULL NULL NULL NULL NULL propagate_input_ordering NULL NULL NULL NULL NULL diff --git a/pkg/sql/logictest/testdata/logic_test/show_source b/pkg/sql/logictest/testdata/logic_test/show_source index b8ad70b01bca..eab7b43689bf 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_source +++ b/pkg/sql/logictest/testdata/logic_test/show_source @@ -146,6 +146,7 @@ override_multi_region_zone_config off parallelize_multi_key_lookup_joins_enabled off password_encryption scram-sha-256 pg_trgm.similarity_threshold 0.3 +plan_cache_mode force_custom_plan prefer_lookup_joins_for_fks off prepared_statements_cache_size 0 B propagate_input_ordering off diff --git a/pkg/sql/sessiondatapb/local_only_session_data.proto b/pkg/sql/sessiondatapb/local_only_session_data.proto index 9b4020357247..d4dd87d3718e 100644 --- a/pkg/sql/sessiondatapb/local_only_session_data.proto +++ b/pkg/sql/sessiondatapb/local_only_session_data.proto @@ -507,6 +507,9 @@ message LocalOnlySessionData { // OptimizerPushOffsetIntoIndexJoin, when true, indicates that the optimizer // should push offset expressions into index joins. bool optimizer_push_offset_into_index_join = 132; + // PlanCacheMode indicates the method that the optimizer should use to choose + // between a custom and generic query plan. + PlanCacheMode plan_cache_mode = 133; /////////////////////////////////////////////////////////////////////////// // WARNING: consider whether a session parameter you're adding needs to // @@ -523,6 +526,23 @@ enum ReplicationMode { REPLICATION_MODE_DATABASE = 2; } +// PlanCacheMode controls the optimizer's decision to use a custom or generic +// query plan. +enum PlanCacheMode { + option (gogoproto.goproto_enum_prefix) = false; + option (gogoproto.goproto_enum_stringer) = false; + + // PlanCacheModeForceCustom forces the optimizer to use a custom query plan. + force_custom_plan = 0 [(gogoproto.enumvalue_customname) = "PlanCacheModeForceCustom"]; + + // PlanCacheModeForceCustom forces the optimizer to use a generic query plan. + force_generic_plan = 1 [(gogoproto.enumvalue_customname) = "PlanCacheModeForceGeneric"]; + + // PlanCacheModeAuto allows the optimizer to automatically choose between a + // custom and generic query plan. + auto = 2 [(gogoproto.enumvalue_customname) = "PlanCacheModeAuto"]; +} + // SequenceCacheEntry is an entry in a SequenceCache. message SequenceCacheEntry { // CachedVersion stores the descpb.DescriptorVersion that cached values are associated with. diff --git a/pkg/sql/sessiondatapb/session_data.go b/pkg/sql/sessiondatapb/session_data.go index d0db8c1c640e..cf37f2546220 100644 --- a/pkg/sql/sessiondatapb/session_data.go +++ b/pkg/sql/sessiondatapb/session_data.go @@ -80,6 +80,25 @@ func VectorizeExecModeFromString(val string) (VectorizeExecMode, bool) { return m, true } +func (m PlanCacheMode) String() string { + name, ok := PlanCacheMode_name[int32(m)] + if !ok { + return fmt.Sprintf("invalid (%d)", m) + } + return name +} + +// PlanCacheModeFromString converts a string into a PlanCacheMode. False is +// returned if the conversion was unsuccessful. +func PlanCacheModeFromString(val string) (PlanCacheMode, bool) { + lowerVal := strings.ToLower(val) + m, ok := PlanCacheMode_value[lowerVal] + if !ok { + return 0, false + } + return PlanCacheMode(m), true +} + // User retrieves the current user. func (s *SessionData) User() username.SQLUsername { return s.UserProto.Decode() diff --git a/pkg/sql/sessiondatapb/session_data.proto b/pkg/sql/sessiondatapb/session_data.proto index c069a7ae5ba5..03896dd215fb 100644 --- a/pkg/sql/sessiondatapb/session_data.proto +++ b/pkg/sql/sessiondatapb/session_data.proto @@ -162,7 +162,7 @@ message DataConversionConfig { util.timeutil.pgdate.DateStyle date_style = 4 [(gogoproto.nullable) = false]; } -// VectorizeExecMode controls if an when the Executor executes queries using +// VectorizeExecMode controls if and when the Executor executes queries using // the columnar execution engine. enum VectorizeExecMode { option (gogoproto.goproto_enum_prefix) = false; diff --git a/pkg/sql/vars.go b/pkg/sql/vars.go index fff8a6ca3b72..f63806d0094c 100644 --- a/pkg/sql/vars.go +++ b/pkg/sql/vars.go @@ -3350,6 +3350,29 @@ var varGen = map[string]sessionVar{ }, GlobalDefault: globalFalse, }, + + `plan_cache_mode`: { + Set: func(_ context.Context, m sessionDataMutator, s string) error { + mode, ok := sessiondatapb.PlanCacheModeFromString(s) + if !ok { + return newVarValueError( + `plan_cache_mode`, + s, + sessiondatapb.PlanCacheModeForceCustom.String(), + sessiondatapb.PlanCacheModeForceGeneric.String(), + sessiondatapb.PlanCacheModeAuto.String(), + ) + } + m.SetPlanCacheMode(mode) + return nil + }, + Get: func(evalCtx *extendedEvalContext, _ *kv.Txn) (string, error) { + return evalCtx.SessionData().PlanCacheMode.String(), nil + }, + GlobalDefault: func(sv *settings.Values) string { + return sessiondatapb.PlanCacheModeForceCustom.String() + }, + }, } func ReplicationModeFromString(s string) (sessiondatapb.ReplicationMode, error) { From 63cb99c74ab762de315f60a4d690d3ff2ba25320 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 17 Jun 2024 13:34:33 -0400 Subject: [PATCH 02/15] opt: add ConvertSelectWithPlaceholdersToJoin exploration rule The `ConvertSelectWithPlaceholdersToJoin` exploration rule has been added which enables optimizations to apply to query plans where placeholder values are not known. See the rule's comments for more details. The rule does not currently apply to any query plans because it only matches Select expressions with filters that have placeholders, and we currently replace all placeholders with constant values before exploration. It will be used in the future when we enable exploration of generic query plans. Release note: None --- pkg/sql/opt/ops/scalar.opt | 1 + pkg/sql/opt/xform/BUILD.bazel | 1 + pkg/sql/opt/xform/generic_funcs.go | 122 ++++++ pkg/sql/opt/xform/rules/generic.opt | 48 +++ pkg/sql/opt/xform/testdata/external/hibernate | 48 ++- pkg/sql/opt/xform/testdata/external/nova | 349 +++++++++++++----- pkg/sql/opt/xform/testdata/rules/generic | 213 +++++++++++ 7 files changed, 669 insertions(+), 113 deletions(-) create mode 100644 pkg/sql/opt/xform/generic_funcs.go create mode 100644 pkg/sql/opt/xform/rules/generic.opt create mode 100644 pkg/sql/opt/xform/testdata/rules/generic diff --git a/pkg/sql/opt/ops/scalar.opt b/pkg/sql/opt/ops/scalar.opt index 38cf51f909a8..15d02b010c0a 100644 --- a/pkg/sql/opt/ops/scalar.opt +++ b/pkg/sql/opt/ops/scalar.opt @@ -147,6 +147,7 @@ define False { [Scalar] define Placeholder { + # Value is always a *tree.Placeholder. Value TypedExpr } diff --git a/pkg/sql/opt/xform/BUILD.bazel b/pkg/sql/opt/xform/BUILD.bazel index c384ff01da99..c1a8d266152a 100644 --- a/pkg/sql/opt/xform/BUILD.bazel +++ b/pkg/sql/opt/xform/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "cycle_funcs.go", "explorer.go", "general_funcs.go", + "generic_funcs.go", "groupby_funcs.go", "index_scan_builder.go", "insert_funcs.go", diff --git a/pkg/sql/opt/xform/generic_funcs.go b/pkg/sql/opt/xform/generic_funcs.go new file mode 100644 index 000000000000..542bbe01f52f --- /dev/null +++ b/pkg/sql/opt/xform/generic_funcs.go @@ -0,0 +1,122 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package xform + +import ( + "fmt" + + "github.com/cockroachdb/cockroach/pkg/sql/opt" + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/types" + "github.com/cockroachdb/cockroach/pkg/util/intsets" + "github.com/cockroachdb/errors" +) + +// HasPlaceholders returns true if the given relational expression's subtree has +// at least one placeholder. +func (c *CustomFuncs) HasPlaceholders(e memo.RelExpr) bool { + return e.Relational().HasPlaceholder +} + +// GeneratePlaceholderValuesAndJoinFilters returns a single-row Values +// expression containing placeholders in the given filters. It also returns a +// new set of filters where the placeholders have been replaced with variables +// referencing the columns produced by the returned Values expression. If the +// given filters have no placeholders, ok=false is returned. +func (c *CustomFuncs) GeneratePlaceholderValuesAndJoinFilters( + filters memo.FiltersExpr, +) (values memo.RelExpr, newFilters memo.FiltersExpr, ok bool) { + // Collect all the placeholders in the filters. + // + // collectPlaceholders recursively walks the scalar expression and collects + // placeholder expressions into the placeholders slice. + var placeholders []*memo.PlaceholderExpr + var seenIndexes intsets.Fast + var collectPlaceholders func(e opt.Expr) + collectPlaceholders = func(e opt.Expr) { + if p, ok := e.(*memo.PlaceholderExpr); ok { + idx := int(p.Value.(*tree.Placeholder).Idx) + // Don't include the same placeholder multiple times. + if !seenIndexes.Contains(idx) { + seenIndexes.Add(idx) + placeholders = append(placeholders, p) + } + return + } + for i, n := 0, e.ChildCount(); i < n; i++ { + collectPlaceholders(e.Child(i)) + } + } + + for i := range filters { + // Only traverse the scalar expression if it contains a placeholder. + if filters[i].ScalarProps().HasPlaceholder { + collectPlaceholders(filters[i].Condition) + } + } + + // If there are no placeholders in the filters, there is nothing to do. + if len(placeholders) == 0 { + return nil, nil, false + } + + // Create the Values expression with one row and one column for each + // placeholder. + cols := make(opt.ColList, len(placeholders)) + colIDs := make(map[tree.PlaceholderIdx]opt.ColumnID, len(placeholders)) + typs := make([]*types.T, len(placeholders)) + exprs := make(memo.ScalarListExpr, len(placeholders)) + for i, p := range placeholders { + idx := p.Value.(*tree.Placeholder).Idx + col := c.e.f.Metadata().AddColumn(fmt.Sprintf("$%d", idx+1), p.DataType()) + cols[i] = col + colIDs[idx] = col + exprs[i] = p + typs[i] = p.DataType() + } + + tupleTyp := types.MakeTuple(typs) + rows := memo.ScalarListExpr{c.e.f.ConstructTuple(exprs, tupleTyp)} + values = c.e.f.ConstructValues(rows, &memo.ValuesPrivate{ + Cols: cols, + ID: c.e.f.Metadata().NextUniqueID(), + }) + + // Create new filters by replacing the placeholders in the filters with + // variables. + var replace func(e opt.Expr) opt.Expr + replace = func(e opt.Expr) opt.Expr { + if p, ok := e.(*memo.PlaceholderExpr); ok { + idx := p.Value.(*tree.Placeholder).Idx + col, ok := colIDs[idx] + if !ok { + panic(errors.AssertionFailedf("unknown placeholder %d", idx)) + } + return c.e.f.ConstructVariable(col) + } + return c.e.f.Replace(e, replace) + } + + newFilters = make(memo.FiltersExpr, len(filters)) + for i := range newFilters { + cond := filters[i].Condition + if newCond := replace(cond).(opt.ScalarExpr); newCond != cond { + // Construct a new filter if placeholders were replaced. + newFilters[i] = c.e.f.ConstructFiltersItem(newCond) + } else { + // Otherwise copy the filter. + newFilters[i] = filters[i] + } + } + + return values, newFilters, true +} diff --git a/pkg/sql/opt/xform/rules/generic.opt b/pkg/sql/opt/xform/rules/generic.opt new file mode 100644 index 000000000000..466eda932a65 --- /dev/null +++ b/pkg/sql/opt/xform/rules/generic.opt @@ -0,0 +1,48 @@ +# ============================================================================= +# generic.opt contains exploration rules for optimizing generic query plans. +# ============================================================================= + +# ConvertSelectWithPlaceholdersToJoin is an exploration rule that converts a +# Select expression with placeholders in the filters into an InnerJoin that +# joins the Select's input with a Values expression that produces the +# placeholder values. +# +# This rule allows generic query plans, in which placeholder values are not +# known, to be optimized. By converting the Select into an InnerJoin, the +# optimizer can plan a lookup join, in many cases, which has similar performance +# characteristics to the constrained Scan that would be planned if the +# placeholder values were known. For example, consider a schema and query like: +# +# CREATE TABLE t (i INT PRIMARY KEY) +# SELECT * FROM t WHERE i = $1 +# +# ConvertSelectWithPlaceholdersToJoin will perform the first conversion below, +# from a Select into a Join. GenerateLookupJoins will perform the second +# conversion from a (hash) Join into a LookupJoin. +# +# +# Select (i=$1) Join (i=col_$1) LookupJoin (t@t_pkey) +# | -> / \ -> | +# | / \ | +# Scan t Values ($1) Scan t Values ($1) +# +[ConvertSelectWithPlaceholdersToJoin, Explore] +(Select + $scan:(Scan $scanPrivate:*) & (IsCanonicalScan $scanPrivate) + $filters:* & + (HasPlaceholders (Root)) & + (Let + ( + $values + $newFilters + $ok + ):(GeneratePlaceholderValuesAndJoinFilters $filters) + $ok + ) +) +=> +(Project + (InnerJoin $values $scan $newFilters (EmptyJoinPrivate)) + [] + (OutputCols (Root)) +) diff --git a/pkg/sql/opt/xform/testdata/external/hibernate b/pkg/sql/opt/xform/testdata/external/hibernate index e8cd7980eff5..527deac7f5aa 100644 --- a/pkg/sql/opt/xform/testdata/external/hibernate +++ b/pkg/sql/opt/xform/testdata/external/hibernate @@ -991,18 +991,28 @@ project ├── has-placeholder ├── key: () ├── fd: ()-->(1-6,9,12), (12)==(1), (1)==(12) - ├── select + ├── project │ ├── columns: phones1_.id:9!null person_id:12 │ ├── cardinality: [0 - 1] │ ├── has-placeholder │ ├── key: () │ ├── fd: ()-->(9,12) - │ ├── scan phone [as=phones1_] - │ │ ├── columns: phones1_.id:9!null person_id:12 - │ │ ├── key: (9) - │ │ └── fd: (9)-->(12) - │ └── filters - │ └── phones1_.id:9 = $1 [outer=(9), constraints=(/9: (/NULL - ]), fd=()-->(9)] + │ └── inner-join (lookup phone [as=phones1_]) + │ ├── columns: phones1_.id:9!null person_id:12 "$1":16!null + │ ├── key columns: [16] = [9] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(9,12,16), (16)==(9), (9)==(16) + │ ├── values + │ │ ├── columns: "$1":16 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(16) + │ │ └── ($1,) + │ └── filters (true) └── filters (true) opt @@ -1045,18 +1055,28 @@ project ├── has-placeholder ├── key: () ├── fd: ()-->(1-6,9,12), (12)==(1), (1)==(12) - ├── select + ├── project │ ├── columns: phones1_.id:9!null person_id:12 │ ├── cardinality: [0 - 1] │ ├── has-placeholder │ ├── key: () │ ├── fd: ()-->(9,12) - │ ├── scan phone [as=phones1_] - │ │ ├── columns: phones1_.id:9!null person_id:12 - │ │ ├── key: (9) - │ │ └── fd: (9)-->(12) - │ └── filters - │ └── phones1_.id:9 = $1 [outer=(9), constraints=(/9: (/NULL - ]), fd=()-->(9)] + │ └── inner-join (lookup phone [as=phones1_]) + │ ├── columns: phones1_.id:9!null person_id:12 "$1":16!null + │ ├── key columns: [16] = [9] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(9,12,16), (16)==(9), (9)==(16) + │ ├── values + │ │ ├── columns: "$1":16 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(16) + │ │ └── ($1,) + │ └── filters (true) └── filters (true) opt diff --git a/pkg/sql/opt/xform/testdata/external/nova b/pkg/sql/opt/xform/testdata/external/nova index af01738cca81..168c05632af8 100644 --- a/pkg/sql/opt/xform/testdata/external/nova +++ b/pkg/sql/opt/xform/testdata/external/nova @@ -212,18 +212,37 @@ project │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ └── flavorid:7 = $2 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":37!null + │ │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(7), (7)==(37) + │ │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_flavorid_key) + │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null flavorid:7!null "$2":37!null + │ │ │ │ │ │ │ │ │ ├── key columns: [37] = [7] + │ │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,37), (37)==(7), (7)==(37) + │ │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ │ ├── columns: "$2":37 + │ │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(37) + │ │ │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ └── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] │ │ │ │ │ │ └── projections @@ -996,26 +1015,37 @@ project │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ │ ├── index-join instance_types + │ │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ │ └── select - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13!null + │ │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ ├── fd: ()-->(1,2,13) - │ │ │ │ │ │ │ │ ├── scan instance_types@instance_types_name_deleted_key - │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13 - │ │ │ │ │ │ │ │ │ ├── constraint: /2/13: (/NULL - ] - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2,13), (2,13)~~>(1) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ ├── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ │ └── name:2 = $4 [outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(13), (2)==(43), (43)==(2), (13)==(42) + │ │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_name_deleted_key) + │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13!null "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ │ ├── key columns: [43 42] = [2 13] + │ │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(1,2,13,42,43), (43)==(2), (13)==(42), (42)==(13), (2)==(43) + │ │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ │ ├── columns: "$1":42 "$4":43 + │ │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(42,43) + │ │ │ │ │ │ │ │ │ │ └── ($1, $4) + │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] │ │ │ │ │ │ │ └── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] @@ -1163,38 +1193,68 @@ project │ │ │ │ │ ├── has-placeholder │ │ │ │ │ ├── key: () │ │ │ │ │ ├── fd: ()-->(1-16,30) - │ │ │ │ │ ├── project - │ │ │ │ │ │ ├── columns: true:29 instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 instance_type_projects.instance_type_id:20 + │ │ │ │ │ ├── left-join (merge) + │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 instance_type_projects.instance_type_id:20 true:29 + │ │ │ │ │ │ ├── left ordering: +1 + │ │ │ │ │ │ ├── right ordering: +20 │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-16,20,29) - │ │ │ │ │ │ ├── left-join (lookup instance_type_projects@instance_type_projects_instance_type_id_project_id_deleted_key) - │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 instance_type_projects.instance_type_id:20 project_id:21 instance_type_projects.deleted:22 - │ │ │ │ │ │ │ ├── key columns: [1] = [20] + │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 + │ │ │ │ │ │ │ ├── fd: ()-->(1-16) + │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$4":42!null "$1":43!null + │ │ │ │ │ │ │ ├── key columns: [42] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(1), (13)==(43), (43)==(13), (1)==(42) + │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ ├── columns: "$4":42 "$1":43 + │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(42,43) + │ │ │ │ │ │ │ │ └── ($4, $1) + │ │ │ │ │ │ │ └── filters + │ │ │ │ │ │ │ └── instance_types.deleted:13 = "$1":43 [outer=(13,43), constraints=(/13: (/NULL - ]; /43: (/NULL - ]), fd=(13)==(43), (43)==(13)] + │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ ├── columns: true:29!null instance_type_projects.instance_type_id:20!null + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(20,29) + │ │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ │ ├── columns: instance_type_projects.instance_type_id:20!null project_id:21!null instance_type_projects.deleted:22!null │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ │ ├── scan instance_types - │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13 instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2-16), (7,13)~~>(1-6,8-12,14-16), (2,13)~~>(1,3-12,14-16) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ ├── instance_types.id:1 = $4 [outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] - │ │ │ │ │ │ │ │ └── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] - │ │ │ │ │ │ │ ├── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] - │ │ │ │ │ │ │ └── instance_type_projects.instance_type_id:20 = $4 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] - │ │ │ │ │ │ └── projections - │ │ │ │ │ │ └── CASE instance_type_projects.instance_type_id:20 IS NULL WHEN true THEN CAST(NULL AS BOOL) ELSE true END [as=true:29, outer=(20)] + │ │ │ │ │ │ │ │ ├── fd: ()-->(20-22) + │ │ │ │ │ │ │ │ └── inner-join (lookup instance_type_projects@instance_type_projects_instance_type_id_project_id_deleted_key) + │ │ │ │ │ │ │ │ ├── columns: instance_type_projects.instance_type_id:20!null project_id:21!null instance_type_projects.deleted:22!null "$2":44!null "$3":45!null "$4":46!null + │ │ │ │ │ │ │ │ ├── key columns: [46 45 44] = [20 21 22] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(20-22,44-46), (44)==(22), (21)==(45), (45)==(21), (20)==(46), (46)==(20), (22)==(44) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$2":44 "$3":45 "$4":46 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(44-46) + │ │ │ │ │ │ │ │ │ └── ($2, $3, $4) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── projections + │ │ │ │ │ │ │ └── true [as=true:29] + │ │ │ │ │ │ └── filters (true) │ │ │ │ │ └── aggregations │ │ │ │ │ ├── const-not-null-agg [as=true_agg:30, outer=(29)] │ │ │ │ │ │ └── true:29 @@ -1341,18 +1401,37 @@ project │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ └── name:2 = $2 [outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + │ │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":37!null + │ │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(2), (2)==(37) + │ │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_name_key) + │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null "$2":37!null + │ │ │ │ │ │ │ │ │ ├── key columns: [37] = [2] + │ │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(1,2,37), (37)==(2), (2)==(37) + │ │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ │ ├── columns: "$2":37 + │ │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(37) + │ │ │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ └── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] │ │ │ │ │ │ └── projections @@ -1498,18 +1577,37 @@ project │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ │ ├── select + │ │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ └── flavorid:7 = $2 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":37!null + │ │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(7), (7)==(37) + │ │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_flavorid_key) + │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null flavorid:7!null "$2":37!null + │ │ │ │ │ │ │ │ │ ├── key columns: [37] = [7] + │ │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,37), (37)==(7), (7)==(37) + │ │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ │ ├── columns: "$2":37 + │ │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(37) + │ │ │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ └── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] │ │ │ │ │ │ └── projections @@ -2046,26 +2144,37 @@ project │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ │ ├── index-join instance_types + │ │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ │ └── select - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null + │ │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13) - │ │ │ │ │ │ │ │ ├── scan instance_types@instance_types_flavorid_deleted_key - │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13 - │ │ │ │ │ │ │ │ │ ├── constraint: /7/13: (/NULL - ] - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(7,13), (7,13)~~>(1) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ ├── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ │ └── flavorid:7 = $4 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(13), (7)==(43), (43)==(7), (13)==(42) + │ │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_flavorid_deleted_key) + │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ │ ├── key columns: [43 42] = [7 13] + │ │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13,42,43), (43)==(7), (13)==(42), (42)==(13), (7)==(43) + │ │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ │ ├── columns: "$1":42 "$4":43 + │ │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(42,43) + │ │ │ │ │ │ │ │ │ │ └── ($1, $4) + │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] │ │ │ │ │ │ │ └── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] @@ -3024,26 +3133,37 @@ project │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ ├── fd: ()-->(1-16,20-22) - │ │ │ │ │ │ │ ├── index-join instance_types + │ │ │ │ │ │ │ ├── project │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16) - │ │ │ │ │ │ │ │ └── select - │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null + │ │ │ │ │ │ │ │ └── inner-join (lookup instance_types) + │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ ├── key columns: [1] = [1] + │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13) - │ │ │ │ │ │ │ │ ├── scan instance_types@instance_types_flavorid_deleted_key - │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13 - │ │ │ │ │ │ │ │ │ ├── constraint: /7/13: (/NULL - ] - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(7,13), (7,13)~~>(1) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ ├── instance_types.deleted:13 = $1 [outer=(13), constraints=(/13: (/NULL - ]), fd=()-->(13)] - │ │ │ │ │ │ │ │ └── flavorid:7 = $4 [outer=(7), constraints=(/7: (/NULL - ]), fd=()-->(7)] + │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(13), (7)==(43), (43)==(7), (13)==(42) + │ │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_flavorid_deleted_key) + │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ │ ├── key columns: [43 42] = [7 13] + │ │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(1,7,13,42,43), (43)==(7), (13)==(42), (42)==(13), (7)==(43) + │ │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ │ ├── columns: "$1":42 "$4":43 + │ │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ │ ├── fd: ()-->(42,43) + │ │ │ │ │ │ │ │ │ │ └── ($1, $4) + │ │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ │ └── filters (true) │ │ │ │ │ │ │ └── filters │ │ │ │ │ │ │ ├── instance_type_projects.deleted:22 = $2 [outer=(22), constraints=(/22: (/NULL - ]), fd=()-->(22)] │ │ │ │ │ │ │ └── project_id:21 = $3 [outer=(21), constraints=(/21: (/NULL - ]), fd=()-->(21)] @@ -3182,36 +3302,67 @@ project │ │ │ │ │ ├── has-placeholder │ │ │ │ │ ├── key: () │ │ │ │ │ ├── fd: ()-->(1-12,14,15,27) - │ │ │ │ │ ├── project - │ │ │ │ │ │ ├── columns: true:26 flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 flavor_projects.flavor_id:19 + │ │ │ │ │ ├── left-join (merge) + │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 flavor_projects.flavor_id:19 true:26 + │ │ │ │ │ │ ├── left ordering: +1 + │ │ │ │ │ │ ├── right ordering: +19 │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ ├── key: () │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,26) - │ │ │ │ │ │ ├── left-join (lookup flavor_projects@flavor_projects_flavor_id_project_id_key) - │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 flavor_projects.flavor_id:19 project_id:20 - │ │ │ │ │ │ │ ├── key columns: [1] = [19] + │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,19,20) - │ │ │ │ │ │ │ ├── select - │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 + │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) + │ │ │ │ │ │ │ └── inner-join (lookup flavors) + │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":37!null + │ │ │ │ │ │ │ ├── key columns: [37] = [1] + │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(1), (1)==(37) + │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ ├── columns: "$2":37 + │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(37) + │ │ │ │ │ │ │ │ └── ($2,) + │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ ├── columns: true:26!null flavor_projects.flavor_id:19!null + │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ ├── fd: ()-->(19,26) + │ │ │ │ │ │ │ ├── project + │ │ │ │ │ │ │ │ ├── columns: flavor_projects.flavor_id:19!null project_id:20!null │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] │ │ │ │ │ │ │ │ ├── has-placeholder │ │ │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) - │ │ │ │ │ │ │ │ ├── scan flavors - │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 - │ │ │ │ │ │ │ │ │ ├── key: (1) - │ │ │ │ │ │ │ │ │ └── fd: (1)-->(2-12,14,15), (7)-->(1-6,8-12,14,15), (2)-->(1,3-12,14,15) - │ │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ │ └── flavors.id:1 = $2 [outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] - │ │ │ │ │ │ │ └── filters - │ │ │ │ │ │ │ ├── project_id:20 = $1 [outer=(20), constraints=(/20: (/NULL - ]), fd=()-->(20)] - │ │ │ │ │ │ │ └── flavor_projects.flavor_id:19 = $2 [outer=(19), constraints=(/19: (/NULL - ]), fd=()-->(19)] - │ │ │ │ │ │ └── projections - │ │ │ │ │ │ └── CASE flavor_projects.flavor_id:19 IS NULL WHEN true THEN CAST(NULL AS BOOL) ELSE true END [as=true:26, outer=(19)] + │ │ │ │ │ │ │ │ ├── fd: ()-->(19,20) + │ │ │ │ │ │ │ │ └── inner-join (lookup flavor_projects@flavor_projects_flavor_id_project_id_key) + │ │ │ │ │ │ │ │ ├── columns: flavor_projects.flavor_id:19!null project_id:20!null "$1":38!null "$2":39!null + │ │ │ │ │ │ │ │ ├── key columns: [39 38] = [19 20] + │ │ │ │ │ │ │ │ ├── lookup columns are key + │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] + │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ ├── fd: ()-->(19,20,38,39), (38)==(20), (19)==(39), (39)==(19), (20)==(38) + │ │ │ │ │ │ │ │ ├── values + │ │ │ │ │ │ │ │ │ ├── columns: "$1":38 "$2":39 + │ │ │ │ │ │ │ │ │ ├── cardinality: [1 - 1] + │ │ │ │ │ │ │ │ │ ├── has-placeholder + │ │ │ │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ │ │ │ ├── fd: ()-->(38,39) + │ │ │ │ │ │ │ │ │ └── ($1, $2) + │ │ │ │ │ │ │ │ └── filters (true) + │ │ │ │ │ │ │ └── projections + │ │ │ │ │ │ │ └── true [as=true:26] + │ │ │ │ │ │ └── filters (true) │ │ │ │ │ └── aggregations │ │ │ │ │ ├── const-not-null-agg [as=true_agg:27, outer=(26)] │ │ │ │ │ │ └── true:26 diff --git a/pkg/sql/opt/xform/testdata/rules/generic b/pkg/sql/opt/xform/testdata/rules/generic new file mode 100644 index 000000000000..523b0d2e3b44 --- /dev/null +++ b/pkg/sql/opt/xform/testdata/rules/generic @@ -0,0 +1,213 @@ +exec-ddl +CREATE TABLE t ( + k INT PRIMARY KEY, + i INT, + s STRING, + b BOOL, + t TIMESTAMP, + INDEX (i, s, b) +) +---- + +# -------------------------------------------------- +# ConvertSelectWithPlaceholdersToJoin +# -------------------------------------------------- + +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE k = $1 +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5 "$1":8!null + ├── key columns: [8] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (8)==(1), (1)==(8) + ├── values + │ ├── columns: "$1":8 + │ ├── cardinality: [1 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8) + │ └── ($1,) + └── filters (true) + +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE k = $1::INT +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5 "$1":8!null + ├── key columns: [8] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (8)==(1), (1)==(8) + ├── values + │ ├── columns: "$1":8 + │ ├── cardinality: [1 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8) + │ └── ($1,) + └── filters (true) + +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE i = $1 AND s = $2 AND b = $3 +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4!null t:5 + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2-4), (1)-->(5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4!null t:5 "$1":8!null "$2":9!null "$3":10!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2-4,8-10), (1)-->(5), (2)==(8), (8)==(2), (3)==(9), (9)==(3), (4)==(10), (10)==(4) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3!null b:4!null "$1":8!null "$2":9!null "$3":10!null + │ ├── key columns: [8 9 10] = [2 3 4] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2-4,8-10), (2)==(8), (8)==(2), (3)==(9), (9)==(3), (4)==(10), (10)==(4) + │ ├── values + │ │ ├── columns: "$1":8 "$2":9 "$3":10 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8-10) + │ │ └── ($1, $2, $3) + │ └── filters (true) + └── filters (true) + +# A placeholder referenced multiple times in the filters should only appear once +# in the Values expression. +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE k = $1 AND i = $1 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5 "$1":8!null + ├── key columns: [8] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (2)==(1,8), (8)==(1,2), (1)==(2,8) + ├── values + │ ├── columns: "$1":8 + │ ├── cardinality: [1 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8) + │ └── ($1,) + └── filters + └── k:1 = i:2 [outer=(1,2), constraints=(/1: (/NULL - ]; /2: (/NULL - ]), fd=(1)==(2), (2)==(1)] + +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE k = (SELECT i FROM t WHERE k = $1) +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5 k:8!null i:9!null + ├── key columns: [9] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8,9), (9)==(1), (1)==(9) + ├── project + │ ├── columns: k:8!null i:9 + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8,9) + │ └── inner-join (lookup t) + │ ├── columns: k:8!null i:9 "$1":15!null + │ ├── key columns: [15] = [8] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(8,9,15), (15)==(8), (8)==(15) + │ ├── values + │ │ ├── columns: "$1":15 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(15) + │ │ └── ($1,) + │ └── filters (true) + └── filters (true) + +# TODO(mgartner): The rule doesn't apply because the filters do not reference +# the placeholder directly. Consider ways to handle cases like this. +opt +SELECT * FROM t WHERE k = (SELECT $1::INT) +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── select + ├── columns: k:1!null i:2 s:3 b:4 t:5 int8:8!null + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8), (8)==(1), (1)==(8) + ├── project + │ ├── columns: int8:8 k:1!null i:2 s:3 b:4 t:5 + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(8), (1)-->(2-5) + │ ├── scan t + │ │ ├── columns: k:1!null i:2 s:3 b:4 t:5 + │ │ ├── key: (1) + │ │ └── fd: (1)-->(2-5) + │ └── projections + │ └── $1 [as=int8:8] + └── filters + └── k:1 = int8:8 [outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(1)==(8), (8)==(1)] + + +# The rule does not match if there are no placeholders in the filters. +opt expect-not=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE i = 1 AND s = 'foo' +---- +index-join t + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── key: (1) + ├── fd: ()-->(2,3), (1)-->(4,5) + └── scan t@t_i_s_b_idx + ├── columns: k:1!null i:2!null s:3!null b:4 + ├── constraint: /2/3/4/1: [/1/'foo' - /1/'foo'] + ├── key: (1) + └── fd: ()-->(2,3), (1)-->(4) From 3c11b62996db7449e9dcd0532e61257b3bc907e8 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 17 Jun 2024 14:01:29 -0400 Subject: [PATCH 03/15] opt: reduce exploration for joins created by ConvertSelectWithPlaceholdersToJoin Join reordering and merge join exploration is now disabled for joins created by the `ConvertSelectWithPlaceholdersToJoin` rule. The rule's main goal is to assist in generating lookup joins. This changes reduces unnecessary exploration. Release note: None --- pkg/sql/opt/xform/generic_funcs.go | 9 +++++ pkg/sql/opt/xform/rules/generic.opt | 2 +- pkg/sql/opt/xform/testdata/external/hibernate | 2 + pkg/sql/opt/xform/testdata/external/nova | 10 +++++ pkg/sql/opt/xform/testdata/rules/generic | 39 +++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/pkg/sql/opt/xform/generic_funcs.go b/pkg/sql/opt/xform/generic_funcs.go index 542bbe01f52f..a6e2795029a5 100644 --- a/pkg/sql/opt/xform/generic_funcs.go +++ b/pkg/sql/opt/xform/generic_funcs.go @@ -120,3 +120,12 @@ func (c *CustomFuncs) GeneratePlaceholderValuesAndJoinFilters( return values, newFilters, true } + +// GenericJoinPrivate returns JoinPrivate that disabled join reordering and +// merge join exploration. +func (c *CustomFuncs) GenericJoinPrivate() *memo.JoinPrivate { + return &memo.JoinPrivate{ + Flags: memo.DisallowMergeJoin, + SkipReorderJoins: true, + } +} diff --git a/pkg/sql/opt/xform/rules/generic.opt b/pkg/sql/opt/xform/rules/generic.opt index 466eda932a65..3816a14be24b 100644 --- a/pkg/sql/opt/xform/rules/generic.opt +++ b/pkg/sql/opt/xform/rules/generic.opt @@ -42,7 +42,7 @@ ) => (Project - (InnerJoin $values $scan $newFilters (EmptyJoinPrivate)) + (InnerJoin $values $scan $newFilters (GenericJoinPrivate)) [] (OutputCols (Root)) ) diff --git a/pkg/sql/opt/xform/testdata/external/hibernate b/pkg/sql/opt/xform/testdata/external/hibernate index 527deac7f5aa..1523df6ebca8 100644 --- a/pkg/sql/opt/xform/testdata/external/hibernate +++ b/pkg/sql/opt/xform/testdata/external/hibernate @@ -999,6 +999,7 @@ project │ ├── fd: ()-->(9,12) │ └── inner-join (lookup phone [as=phones1_]) │ ├── columns: phones1_.id:9!null person_id:12 "$1":16!null + │ ├── flags: disallow merge join │ ├── key columns: [16] = [9] │ ├── lookup columns are key │ ├── cardinality: [0 - 1] @@ -1063,6 +1064,7 @@ project │ ├── fd: ()-->(9,12) │ └── inner-join (lookup phone [as=phones1_]) │ ├── columns: phones1_.id:9!null person_id:12 "$1":16!null + │ ├── flags: disallow merge join │ ├── key columns: [16] = [9] │ ├── lookup columns are key │ ├── cardinality: [0 - 1] diff --git a/pkg/sql/opt/xform/testdata/external/nova b/pkg/sql/opt/xform/testdata/external/nova index 168c05632af8..a36bb9fd3dae 100644 --- a/pkg/sql/opt/xform/testdata/external/nova +++ b/pkg/sql/opt/xform/testdata/external/nova @@ -228,6 +228,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(7), (7)==(37) │ │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_flavorid_key) │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null flavorid:7!null "$2":37!null + │ │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ │ ├── key columns: [37] = [7] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -1031,6 +1032,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(13), (2)==(43), (43)==(2), (13)==(42) │ │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_name_deleted_key) │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2!null instance_types.deleted:13!null "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ │ ├── key columns: [43 42] = [2 13] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -1209,6 +1211,7 @@ project │ │ │ │ │ │ │ ├── fd: ()-->(1-16) │ │ │ │ │ │ │ └── inner-join (lookup instance_types) │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null name:2 memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7 swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 instance_types.deleted:13!null instance_types.deleted_at:14 instance_types.created_at:15 instance_types.updated_at:16 "$4":42!null "$1":43!null + │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ ├── key columns: [42] = [1] │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -1238,6 +1241,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(20-22) │ │ │ │ │ │ │ │ └── inner-join (lookup instance_type_projects@instance_type_projects_instance_type_id_project_id_deleted_key) │ │ │ │ │ │ │ │ ├── columns: instance_type_projects.instance_type_id:20!null project_id:21!null instance_type_projects.deleted:22!null "$2":44!null "$3":45!null "$4":46!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ ├── key columns: [46 45 44] = [20 21 22] │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -1417,6 +1421,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(2), (2)==(37) │ │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_name_key) │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null "$2":37!null + │ │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ │ ├── key columns: [37] = [2] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -1593,6 +1598,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15,37), (37)==(7), (7)==(37) │ │ │ │ │ │ │ │ ├── inner-join (lookup flavors@flavors_flavorid_key) │ │ │ │ │ │ │ │ │ ├── columns: flavors.id:1!null flavorid:7!null "$2":37!null + │ │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ │ ├── key columns: [37] = [7] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -2160,6 +2166,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(13), (7)==(43), (43)==(7), (13)==(42) │ │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_flavorid_deleted_key) │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ │ ├── key columns: [43 42] = [7 13] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -3149,6 +3156,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(1-16,42,43), (42)==(13), (7)==(43), (43)==(7), (13)==(42) │ │ │ │ │ │ │ │ ├── inner-join (lookup instance_types@instance_types_flavorid_deleted_key) │ │ │ │ │ │ │ │ │ ├── columns: instance_types.id:1!null flavorid:7!null instance_types.deleted:13!null "$1":42!null "$4":43!null + │ │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ │ ├── key columns: [43 42] = [7 13] │ │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -3318,6 +3326,7 @@ project │ │ │ │ │ │ │ ├── fd: ()-->(1-12,14,15) │ │ │ │ │ │ │ └── inner-join (lookup flavors) │ │ │ │ │ │ │ ├── columns: flavors.id:1!null name:2!null memory_mb:3!null vcpus:4!null root_gb:5 ephemeral_gb:6 flavorid:7!null swap:8!null rxtx_factor:9 vcpu_weight:10 disabled:11 is_public:12 flavors.created_at:14 flavors.updated_at:15 "$2":37!null + │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ ├── key columns: [37] = [1] │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ ├── cardinality: [0 - 1] @@ -3346,6 +3355,7 @@ project │ │ │ │ │ │ │ │ ├── fd: ()-->(19,20) │ │ │ │ │ │ │ │ └── inner-join (lookup flavor_projects@flavor_projects_flavor_id_project_id_key) │ │ │ │ │ │ │ │ ├── columns: flavor_projects.flavor_id:19!null project_id:20!null "$1":38!null "$2":39!null + │ │ │ │ │ │ │ │ ├── flags: disallow merge join │ │ │ │ │ │ │ │ ├── key columns: [39 38] = [19 20] │ │ │ │ │ │ │ │ ├── lookup columns are key │ │ │ │ │ │ │ │ ├── cardinality: [0 - 1] diff --git a/pkg/sql/opt/xform/testdata/rules/generic b/pkg/sql/opt/xform/testdata/rules/generic index 523b0d2e3b44..13794eefd39e 100644 --- a/pkg/sql/opt/xform/testdata/rules/generic +++ b/pkg/sql/opt/xform/testdata/rules/generic @@ -24,6 +24,7 @@ project ├── fd: ()-->(1-5) └── inner-join (lookup t) ├── columns: k:1!null i:2 s:3 b:4 t:5 "$1":8!null + ├── flags: disallow merge join ├── key columns: [8] = [1] ├── lookup columns are key ├── cardinality: [0 - 1] @@ -50,6 +51,7 @@ project ├── fd: ()-->(1-5) └── inner-join (lookup t) ├── columns: k:1!null i:2 s:3 b:4 t:5 "$1":8!null + ├── flags: disallow merge join ├── key columns: [8] = [1] ├── lookup columns are key ├── cardinality: [0 - 1] @@ -82,6 +84,7 @@ project ├── fd: ()-->(2-4,8-10), (1)-->(5), (2)==(8), (8)==(2), (3)==(9), (9)==(3), (4)==(10), (10)==(4) ├── inner-join (lookup t@t_i_s_b_idx) │ ├── columns: k:1!null i:2!null s:3!null b:4!null "$1":8!null "$2":9!null "$3":10!null + │ ├── flags: disallow merge join │ ├── key columns: [8 9 10] = [2 3 4] │ ├── has-placeholder │ ├── key: (1) @@ -109,6 +112,7 @@ project ├── fd: ()-->(1-5) └── inner-join (lookup t) ├── columns: k:1!null i:2!null s:3 b:4 t:5 "$1":8!null + ├── flags: disallow merge join ├── key columns: [8] = [1] ├── lookup columns are key ├── cardinality: [0 - 1] @@ -125,6 +129,40 @@ project └── filters └── k:1 = i:2 [outer=(1,2), constraints=(/1: (/NULL - ]; /2: (/NULL - ]), fd=(1)==(2), (2)==(1)] +# The generated join should not be reordered and merge joins should not be +# explored on it. +opt expect=ConvertSelectWithPlaceholdersToJoin expect-not=(ReorderJoins,GenerateMergeJoins) +SELECT * FROM t WHERE i = $1 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5 + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2), (1)-->(3-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5 "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2,8), (1)-->(3-5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3 b:4 "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [2] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,8), (1)-->(3,4), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters (true) + └── filters (true) + opt expect=ConvertSelectWithPlaceholdersToJoin SELECT * FROM t WHERE k = (SELECT i FROM t WHERE k = $1) ---- @@ -150,6 +188,7 @@ project │ ├── fd: ()-->(8,9) │ └── inner-join (lookup t) │ ├── columns: k:8!null i:9 "$1":15!null + │ ├── flags: disallow merge join │ ├── key columns: [15] = [8] │ ├── lookup columns are key │ ├── cardinality: [0 - 1] From e6240cc0d7f017ef5c8fff13963d679a74daf610 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 17 Jun 2024 14:48:56 -0400 Subject: [PATCH 04/15] opt: add stats tests for generic query plans This commit adds some stats tests with queries containing placeholders. Release note: None --- pkg/sql/opt/memo/testdata/stats/generic | 168 ++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 pkg/sql/opt/memo/testdata/stats/generic diff --git a/pkg/sql/opt/memo/testdata/stats/generic b/pkg/sql/opt/memo/testdata/stats/generic new file mode 100644 index 000000000000..e2097b2db932 --- /dev/null +++ b/pkg/sql/opt/memo/testdata/stats/generic @@ -0,0 +1,168 @@ +exec-ddl +CREATE TABLE t ( + k INT PRIMARY KEY, + i INT, + s STRING +) +---- + +exec-ddl +ALTER TABLE t INJECT STATISTICS '[ + { + "columns": ["k"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 10000 + }, + { + "columns": ["i"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 10000, + "distinct_count": 1000 + }, + { + "columns": ["s"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 10000, + "distinct_count": 100 + } +]' +---- + +norm +SELECT * FROM t WHERE k = $1 +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string) + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── stats: [rows=1, distinct(1)=1, null(1)=0] + ├── key: () + ├── fd: ()-->(1-3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── k:1 = $1 [type=bool, outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] + +norm +SELECT * FROM t WHERE i = $1 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333, distinct(2)=1000, null(2)=0] + ├── key: (1) + ├── fd: ()-->(2), (1)-->(3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── i:2 = $1 [type=bool, outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + +norm +SELECT * FROM t WHERE $1 = s +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string!null) + ├── has-placeholder + ├── stats: [rows=3333.333, distinct(3)=100, null(3)=0] + ├── key: (1) + ├── fd: ()-->(3), (1)-->(2) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(3)=100, null(3)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── s:3 = $1 [type=bool, outer=(3), constraints=(/3: (/NULL - ]), fd=()-->(3)] + +norm +SELECT * FROM t WHERE i = $1 AND s = $2 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string!null) + ├── has-placeholder + ├── stats: [rows=1111.111, distinct(2)=1000, null(2)=0, distinct(3)=100, null(3)=0] + ├── key: (1) + ├── fd: ()-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0, distinct(3)=100, null(3)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + ├── i:2 = $1 [type=bool, outer=(2), constraints=(/2: (/NULL - ]), fd=()-->(2)] + └── s:3 = $2 [type=bool, outer=(3), constraints=(/3: (/NULL - ]), fd=()-->(3)] + +norm +SELECT * FROM t WHERE i > $1 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333, distinct(2)=1000, null(2)=0] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── i:2 > $1 [type=bool, outer=(2), constraints=(/2: (/NULL - ])] + +norm +SELECT * FROM t WHERE i = $1 OR i = $2 +---- +select + ├── columns: k:1(int!null) i:2(int!null) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333, distinct(2)=1000, null(2)=0] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── (i:2 = $1) OR (i:2 = $2) [type=bool, outer=(2), constraints=(/2: (/NULL - ])] + +norm +SELECT * FROM t WHERE i IN ($1, $2, $3) +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── i:2 IN ($1, $2, $3) [type=bool, outer=(2)] + +norm +SELECT * FROM t WHERE i = $1 OR s = $2 +---- +select + ├── columns: k:1(int!null) i:2(int) s:3(string) + ├── has-placeholder + ├── stats: [rows=3333.333] + ├── key: (1) + ├── fd: (1)-->(2,3) + ├── scan t + │ ├── columns: k:1(int!null) i:2(int) s:3(string) + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── filters + └── (i:2 = $1) OR (s:3 = $2) [type=bool, outer=(2,3)] From e6ece7ce20c2fd8c620f1861e7c8d216cbff9e0c Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 17 Jun 2024 15:04:54 -0400 Subject: [PATCH 05/15] opt: improve stats for filters with placeholder equalities Stats estimates for filters of the form `col = $1` have been improved. While histograms cannot be filtered without knowing the placeholder value, we do know that `col` is limited to a one distinct value. This improves the estimate over the simple 1/3 "unknown selectivity" that was applied for these filters before this commit. This changed required a modification to `TestCopyAndReplace` because it made it such that a merge-join was no longer selected when optimizing the query with placeholders. Now the test checks that `CopyAndReplace` works correctly when copying a fully optimized memo with a parameterized lookup-join. I've confirmed that the test still catches the bug fixed in the original PR that added the test, #35020. Release note: None --- pkg/sql/opt/exec/execbuilder/testdata/prepare | 4 +- pkg/sql/opt/memo/statistics_builder.go | 21 ++++++++++- pkg/sql/opt/memo/testdata/stats/generic | 8 ++-- pkg/sql/opt/norm/factory_test.go | 32 ++++++++-------- pkg/sql/opt/xform/testdata/external/hibernate | 37 ++++++++++--------- .../xform/testdata/placeholder-fast-path/scan | 35 ++++++++++++++---- 6 files changed, 88 insertions(+), 49 deletions(-) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/prepare b/pkg/sql/opt/exec/execbuilder/testdata/prepare index 2c12f2f84978..d79ef3825daf 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/prepare +++ b/pkg/sql/opt/exec/execbuilder/testdata/prepare @@ -134,7 +134,7 @@ quality of service: regular KV bytes read: 8 B KV gRPC calls: 1 estimated max memory allocated: 0 B - estimated row count: 0 + estimated row count: 1 table: ab@ab_pkey spans: [/1 - /1] @@ -162,6 +162,6 @@ quality of service: regular KV bytes read: 0 B KV gRPC calls: 0 estimated max memory allocated: 0 B - estimated row count: 0 + estimated row count: 1 table: ab@ab_pkey spans: [/2 - /2] diff --git a/pkg/sql/opt/memo/statistics_builder.go b/pkg/sql/opt/memo/statistics_builder.go index 5ec5651dc935..9f119baef31f 100644 --- a/pkg/sql/opt/memo/statistics_builder.go +++ b/pkg/sql/opt/memo/statistics_builder.go @@ -3321,6 +3321,8 @@ func (sb *statisticsBuilder) applyFilters( func (sb *statisticsBuilder) applyFiltersItem( filter *FiltersItem, e RelExpr, relProps *props.Relational, unapplied *filterCount, ) (constrainedCols, histCols opt.ColSet) { + s := relProps.Statistics() + // Before checking anything, try to replace any virtual computed column // expressions with the corresponding virtual column. if sb.evalCtx.SessionData().OptimizerUseVirtualComputedColumnStats && @@ -3346,6 +3348,13 @@ func (sb *statisticsBuilder) applyFiltersItem( return opt.ColSet{}, opt.ColSet{} } + // Special case: a placeholder equality filter. + if col, ok := isPlaceholderEqualityFilter(filter.Condition); ok { + cols := opt.MakeColSet(col) + sb.ensureColStat(cols, 1 /* maxDistinctCount */, e, s) + return cols, opt.ColSet{} + } + // Special case: The current conjunct is an inverted join condition which is // handled by selectivityFromInvertedJoinCondition. if isInvertedJoinCond(filter.Condition) { @@ -3385,7 +3394,6 @@ func (sb *statisticsBuilder) applyFiltersItem( // want to make sure that we don't include columns that were only present in // equality conjuncts such as var1=var2. The selectivity of these conjuncts // will be accounted for in selectivityFromEquivalencies. - s := relProps.Statistics() scalarProps := filter.ScalarProps() constrainedCols.UnionWith(scalarProps.OuterCols) if scalarProps.Constraints != nil { @@ -4808,6 +4816,17 @@ func isSimilarityFilter(e opt.ScalarExpr) bool { return false } +// isPlaceholderEqualityFilter returns a column ID and true if the given +// condition is an equality between a column and a placeholder. +func isPlaceholderEqualityFilter(e opt.ScalarExpr) (opt.ColumnID, bool) { + if e.Op() == opt.EqOp && e.Child(1).Op() == opt.PlaceholderOp { + if v, ok := e.Child(0).(*VariableExpr); ok { + return v.Col, true + } + } + return 0, false +} + // isInvertedJoinCond returns true if the given condition is either an index- // accelerated geospatial function, a bounding box operation, or a contains // operation with two variable arguments. diff --git a/pkg/sql/opt/memo/testdata/stats/generic b/pkg/sql/opt/memo/testdata/stats/generic index e2097b2db932..ba395c1fa05b 100644 --- a/pkg/sql/opt/memo/testdata/stats/generic +++ b/pkg/sql/opt/memo/testdata/stats/generic @@ -53,7 +53,7 @@ SELECT * FROM t WHERE i = $1 select ├── columns: k:1(int!null) i:2(int!null) s:3(string) ├── has-placeholder - ├── stats: [rows=3333.333, distinct(2)=1000, null(2)=0] + ├── stats: [rows=10, distinct(2)=1, null(2)=0] ├── key: (1) ├── fd: ()-->(2), (1)-->(3) ├── scan t @@ -70,7 +70,7 @@ SELECT * FROM t WHERE $1 = s select ├── columns: k:1(int!null) i:2(int) s:3(string!null) ├── has-placeholder - ├── stats: [rows=3333.333, distinct(3)=100, null(3)=0] + ├── stats: [rows=100, distinct(3)=1, null(3)=0] ├── key: (1) ├── fd: ()-->(3), (1)-->(2) ├── scan t @@ -87,12 +87,12 @@ SELECT * FROM t WHERE i = $1 AND s = $2 select ├── columns: k:1(int!null) i:2(int!null) s:3(string!null) ├── has-placeholder - ├── stats: [rows=1111.111, distinct(2)=1000, null(2)=0, distinct(3)=100, null(3)=0] + ├── stats: [rows=0.91, distinct(2)=0.91, null(2)=0, distinct(3)=0.91, null(3)=0, distinct(2,3)=0.91, null(2,3)=0] ├── key: (1) ├── fd: ()-->(2,3) ├── scan t │ ├── columns: k:1(int!null) i:2(int) s:3(string) - │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0, distinct(3)=100, null(3)=0] + │ ├── stats: [rows=10000, distinct(1)=10000, null(1)=0, distinct(2)=1000, null(2)=0, distinct(3)=100, null(3)=0, distinct(2,3)=10000, null(2,3)=0] │ ├── key: (1) │ └── fd: (1)-->(2,3) └── filters diff --git a/pkg/sql/opt/norm/factory_test.go b/pkg/sql/opt/norm/factory_test.go index 2a0b3af1cc0b..756f45e8a7b6 100644 --- a/pkg/sql/opt/norm/factory_test.go +++ b/pkg/sql/opt/norm/factory_test.go @@ -69,32 +69,30 @@ func TestSimplifyFilters(t *testing.T) { } } -// Test CopyAndReplace on an already optimized join. Before CopyAndReplace is -// called, the join has a placeholder that causes the optimizer to use a merge -// join. After CopyAndReplace substitutes a constant for the placeholder, the -// optimizer switches to a lookup join. A similar pattern is used by the -// ApplyJoin execution operator which replaces variables with constants in an -// already optimized tree. The CopyAndReplace code must take care to copy over -// the normalized tree rather than the optimized tree by using the FirstExpr -// method. +// Test CopyAndReplace on an already optimized memo. Before CopyAndReplace is +// called, the query has a placeholder that causes the optimizer to use a +// parameterized lookup-join. After CopyAndReplace substitutes a constant for +// the placeholder, the optimizer switches to a constrained scan. A similar +// pattern is used by the ApplyJoin execution operator which replaces variables +// with constants in an already optimized tree. The CopyAndReplace code must +// take care to copy over the normalized tree rather than the optimized tree by +// using the FirstExpr method. func TestCopyAndReplace(t *testing.T) { cat := testcat.New() if _, err := cat.ExecuteDDL("CREATE TABLE ab (a INT PRIMARY KEY, b INT)"); err != nil { t.Fatal(err) } - if _, err := cat.ExecuteDDL("CREATE TABLE cde (c INT PRIMARY KEY, d INT, e INT, INDEX(d))"); err != nil { - t.Fatal(err) - } evalCtx := eval.MakeTestingEvalContext(cluster.MakeTestingClusterSettings()) var o xform.Optimizer - testutils.BuildQuery(t, &o, cat, &evalCtx, "SELECT * FROM cde INNER JOIN ab ON a=c AND d=$1") + testutils.BuildQuery(t, &o, cat, &evalCtx, "SELECT * FROM ab WHERE a = $1") if e, err := o.Optimize(); err != nil { t.Fatal(err) - } else if e.Op() != opt.MergeJoinOp { - t.Errorf("expected optimizer to choose merge-join, not %v", e.Op()) + } else if e.Op() != opt.ProjectOp || e.Child(0).Op() != opt.LookupJoinOp { + t.Errorf("expected optimizer to choose a (project (lookup-join)), not (%v (%v))", + e.Op(), e.Child(0).Op()) } m := o.Factory().DetachMemo() @@ -111,8 +109,10 @@ func TestCopyAndReplace(t *testing.T) { if e, err := o.Optimize(); err != nil { t.Fatal(err) - } else if e.Op() != opt.LookupJoinOp { - t.Errorf("expected optimizer to choose lookup-join, not %v", e.Op()) + } else if e.Op() != opt.ScanOp { + t.Errorf("expected optimizer to choose a constrained scan, not %v", e.Op()) + } else if e.(*memo.ScanExpr).Constraint == nil { + t.Errorf("expected optimizer to choose a constrained scan") } } diff --git a/pkg/sql/opt/xform/testdata/external/hibernate b/pkg/sql/opt/xform/testdata/external/hibernate index 1523df6ebca8..5a8fad72719c 100644 --- a/pkg/sql/opt/xform/testdata/external/hibernate +++ b/pkg/sql/opt/xform/testdata/external/hibernate @@ -1885,30 +1885,31 @@ project │ ├── has-placeholder │ ├── key: (1) │ ├── fd: ()-->(4), (1)-->(2-4,15) - │ ├── right-join (hash) - │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null successfulbid:10 true:14 + │ ├── project + │ │ ├── columns: true:14 bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null successfulbid:10 │ │ ├── has-placeholder │ │ ├── fd: ()-->(4), (1)-->(2,3) - │ │ ├── project - │ │ │ ├── columns: true:14!null successfulbid:10 - │ │ │ ├── fd: ()-->(14) - │ │ │ ├── scan tauction2 [as=a] - │ │ │ │ └── columns: successfulbid:10 - │ │ │ └── projections - │ │ │ └── true [as=true:14] - │ │ ├── select - │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null + │ │ ├── right-join (hash) + │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null successfulbid:10 │ │ │ ├── has-placeholder - │ │ │ ├── key: (1) │ │ │ ├── fd: ()-->(4), (1)-->(2,3) - │ │ │ ├── scan tbid2 [as=bids0_] - │ │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4 + │ │ │ ├── scan tauction2 [as=a] + │ │ │ │ └── columns: successfulbid:10 + │ │ │ ├── select + │ │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4!null + │ │ │ │ ├── has-placeholder │ │ │ │ ├── key: (1) - │ │ │ │ └── fd: (1)-->(2-4) + │ │ │ │ ├── fd: ()-->(4), (1)-->(2,3) + │ │ │ │ ├── scan tbid2 [as=bids0_] + │ │ │ │ │ ├── columns: bids0_.id:1!null amount:2 createddatetime:3 auctionid:4 + │ │ │ │ │ ├── key: (1) + │ │ │ │ │ └── fd: (1)-->(2-4) + │ │ │ │ └── filters + │ │ │ │ └── auctionid:4 = $1 [outer=(4), constraints=(/4: (/NULL - ]), fd=()-->(4)] │ │ │ └── filters - │ │ │ └── auctionid:4 = $1 [outer=(4), constraints=(/4: (/NULL - ]), fd=()-->(4)] - │ │ └── filters - │ │ └── successfulbid:10 = bids0_.id:1 [outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ │ └── successfulbid:10 = bids0_.id:1 [outer=(1,10), constraints=(/1: (/NULL - ]; /10: (/NULL - ]), fd=(1)==(10), (10)==(1)] + │ │ └── projections + │ │ └── CASE successfulbid:10 IS NULL WHEN true THEN CAST(NULL AS BOOL) ELSE true END [as=true:14, outer=(10)] │ └── aggregations │ ├── const-not-null-agg [as=true_agg:15, outer=(14)] │ │ └── true:14 diff --git a/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan b/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan index f45c1337140b..65c50c3bac28 100644 --- a/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan +++ b/pkg/sql/opt/xform/testdata/placeholder-fast-path/scan @@ -95,6 +95,25 @@ SELECT v+1 FROM kv WHERE k = $1 ---- no fast path + +# Inject statistics so that the estimated row count is high. +exec-ddl +ALTER TABLE abcd INJECT STATISTICS '[ + { + "columns": ["a"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 5 + }, + { + "columns": ["b"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 10000, + "distinct_count": 5 + } +]' +---- + # The fast path should not kick in because the estimated row count is too high. placeholder-fast-path SELECT a, b, c FROM abcd WHERE a=$1 AND b=$2 @@ -120,7 +139,7 @@ SELECT a, b, c FROM abcd WHERE a=$1 AND b=$2 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=1.1, distinct(1)=1.1, null(1)=0, distinct(2)=1, null(2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── $1 @@ -132,7 +151,7 @@ SELECT a, b, c FROM abcd WHERE b=$1 AND a=$2 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=1.1, distinct(1)=1.1, null(1)=0, distinct(2)=1, null(2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── $2 @@ -145,7 +164,7 @@ SELECT a, b, c FROM abcd WHERE a=0 AND b=$1 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=0.666, distinct(1)=0.666, null(1)=0, distinct(2)=0.666, null(2)=0, distinct(1,2)=0.666, null(1,2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── 0 @@ -157,7 +176,7 @@ SELECT a, b, c FROM abcd WHERE a=$1 AND b=0 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=3.3, distinct(1)=3.3, null(1)=0, distinct(2)=1, null(2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── $1 @@ -170,7 +189,7 @@ SELECT a, b, c FROM abcd WHERE a=1+2 AND b=$1 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=0.666, distinct(1)=0.666, null(1)=0, distinct(2)=0.666, null(2)=0, distinct(1,2)=0.666, null(1,2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── 3 @@ -182,7 +201,7 @@ SELECT a, b, c FROM abcd WHERE a=fnv32a('foo') AND b=$1 placeholder-scan abcd@abcd_a_b_idx ├── columns: a:1!null b:2!null c:3 ├── has-placeholder - ├── stats: [rows=0.666, distinct(1)=0.666, null(1)=0, distinct(2)=0.666, null(2)=0, distinct(1,2)=0.666, null(1,2)=0] + ├── stats: [rows=1.998, distinct(1)=1, null(1)=0, distinct(2)=1, null(2)=0, distinct(1,2)=1, null(1,2)=0] ├── fd: ()-->(1,2) └── span ├── 2851307223 @@ -230,7 +249,7 @@ SELECT a, d FROM abcd WHERE d=$1 AND c=$2 placeholder-scan abcd@abcd_d_c_a_idx ├── columns: a:1 d:4!null ├── has-placeholder - ├── stats: [rows=1.0989] + ├── stats: [rows=9.8901] ├── fd: ()-->(4) └── span ├── $1 @@ -290,7 +309,7 @@ SELECT a, b FROM partial1 WHERE a = $1 placeholder-scan partial1@pseudo_ab,partial ├── columns: a:2!null b:3 ├── has-placeholder - ├── stats: [rows=3.3, distinct(2)=1, null(2)=0] + ├── stats: [rows=9.9, distinct(2)=1, null(2)=0] ├── fd: ()-->(2) └── span └── $1 From c22f2713e6ca002b9ce0e6b7280b95b0b6d1c2ea Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 18 Jun 2024 15:05:43 -0400 Subject: [PATCH 06/15] opt: add generic query plan tests with partial indexes Tests for generic query plans with partial indexes have been added. No code changes are needed in order for a query with a filter like `col = $1` to utilize an index on `(col) WHERE col IS NOT NULL`. Release note: None --- pkg/sql/opt/xform/testdata/rules/generic | 123 +++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/pkg/sql/opt/xform/testdata/rules/generic b/pkg/sql/opt/xform/testdata/rules/generic index 13794eefd39e..dbd694762060 100644 --- a/pkg/sql/opt/xform/testdata/rules/generic +++ b/pkg/sql/opt/xform/testdata/rules/generic @@ -236,6 +236,129 @@ project └── filters └── k:1 = int8:8 [outer=(1,8), constraints=(/1: (/NULL - ]; /8: (/NULL - ]), fd=(1)==(8), (8)==(1)] +exec-ddl +CREATE INDEX partial_idx ON t(t) WHERE t IS NOT NULL +---- + +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE t = $1 +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5!null + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(5), (1)-->(2-4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5!null "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(5,8), (1)-->(2-4), (5)==(8), (8)==(5) + ├── inner-join (lookup t@partial_idx,partial) + │ ├── columns: k:1!null t:5!null "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [5] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(5,8), (5)==(8), (8)==(5) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- + +exec-ddl +CREATE INDEX partial_idx ON t(i, t) WHERE i IS NOT NULL AND t IS NOT NULL +---- + +opt expect=ConvertSelectWithPlaceholdersToJoin +SELECT * FROM t WHERE i = $1 AND t = $2 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null "$2":9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8,9), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + ├── inner-join (lookup t@partial_idx,partial) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null "$2":9!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9] = [2 5] + │ ├── has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8,9), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + │ ├── values + │ │ ├── columns: "$1":8 "$2":9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, $2) + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- + +exec-ddl +CREATE INDEX partial_idx ON t(s) WHERE k = i +---- + +opt +SELECT * FROM t@partial_idx WHERE s = $1 AND k = $2 AND i = $2 +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 "$1":8!null "$2":9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5,8,9), (2)==(1,9), (3)==(8), (8)==(3), (9)==(1,2), (1)==(2,9) + ├── inner-join (lookup t@partial_idx,partial) + │ ├── columns: k:1!null s:3!null "$1":8!null "$2":9!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9] = [3 1] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── has-placeholder + │ ├── key: () + │ ├── fd: ()-->(1,3,8,9), (8)==(3), (1)==(9), (9)==(1), (3)==(8) + │ ├── values + │ │ ├── columns: "$1":8 "$2":9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, $2) + │ └── filters (true) + └── filters (true) + +exec-ddl +DROP INDEX partial_idx +---- # The rule does not match if there are no placeholders in the filters. opt expect-not=ConvertSelectWithPlaceholdersToJoin From f4395b0923dd185344c135b03b50a14bd472efdd Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 1 Jul 2024 17:20:07 -0400 Subject: [PATCH 07/15] sql: rename PreparedStatement.Memo to BaseMemo Release note: None --- pkg/sql/plan_opt.go | 15 +++++++-------- pkg/sql/prepared_stmt.go | 13 +++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index 2a6acd20356e..a544329ad4b2 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -54,7 +54,7 @@ var queryCacheEnabled = settings.RegisterBoolSetting( // - Columns // - Types // - AnonymizedStr -// - Memo (for reuse during exec, if appropriate). +// - BaseMemo (for reuse during exec, if appropriate). func (p *planner) prepareUsingOptimizer( ctx context.Context, origin PreparedStatementOrigin, ) (planFlags, error) { @@ -153,7 +153,7 @@ func (p *planner) prepareUsingOptimizer( stmt.Prepared.StatementNoConstants = pm.StatementNoConstants stmt.Prepared.Columns = pm.Columns stmt.Prepared.Types = pm.Types - stmt.Prepared.Memo = cachedData.Memo + stmt.Prepared.BaseMemo = cachedData.Memo return opc.flags, nil } opc.log(ctx, "query cache hit but memo is stale (prepare)") @@ -212,7 +212,7 @@ func (p *planner) prepareUsingOptimizer( stmt.Prepared.Columns = resultCols stmt.Prepared.Types = p.semaCtx.Placeholders.Types if opc.allowMemoReuse { - stmt.Prepared.Memo = memo + stmt.Prepared.BaseMemo = memo if opc.useCache { // execPrepare sets the PrepareMetadata.InferredTypes field after this // point. However, once the PrepareMetadata goes into the cache, it @@ -530,24 +530,23 @@ func (opc *optPlanningCtx) reuseMemo( func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ error) { prepared := opc.p.stmt.Prepared p := opc.p - if opc.allowMemoReuse && prepared != nil && prepared.Memo != nil { + if opc.allowMemoReuse && prepared != nil && prepared.BaseMemo != nil { // We are executing a previously prepared statement and a reusable memo is // available. // If the prepared memo has been invalidated by schema or other changes, // re-prepare it. - if isStale, err := prepared.Memo.IsStale(ctx, p.EvalContext(), opc.catalog); err != nil { + if isStale, err := prepared.BaseMemo.IsStale(ctx, p.EvalContext(), opc.catalog); err != nil { return nil, err } else if isStale { opc.log(ctx, "rebuilding cached memo") - prepared.Memo, err = opc.buildReusableMemo(ctx) + prepared.BaseMemo, err = opc.buildReusableMemo(ctx) if err != nil { return nil, err } } opc.log(ctx, "reusing cached memo") - memo, err := opc.reuseMemo(ctx, prepared.Memo) - return memo, err + return opc.reuseMemo(ctx, prepared.BaseMemo) } if opc.useCache { diff --git a/pkg/sql/prepared_stmt.go b/pkg/sql/prepared_stmt.go index c1a0736f611f..815a9ce74b5f 100644 --- a/pkg/sql/prepared_stmt.go +++ b/pkg/sql/prepared_stmt.go @@ -55,10 +55,11 @@ const ( type PreparedStatement struct { querycache.PrepareMetadata - // Memo is the memoized data structure constructed by the cost-based optimizer - // during prepare of a SQL statement. It can significantly speed up execution - // if it is used by the optimizer as a starting point. - Memo *memo.Memo + // BaseMemo is the memoized data structure constructed by the cost-based + // optimizer during prepare of a SQL statement. It may be a fully-optimized + // memo if the prepared statement has no placeholders and no fold-able + // stable expressions. Otherwise, it is an unoptimized, normalized memo. + BaseMemo *memo.Memo // refCount keeps track of the number of references to this PreparedStatement. // New references are registered through incRef(). @@ -84,8 +85,8 @@ func (p *PreparedStatement) MemoryEstimate() int64 { // 1. Size of the prepare metadata. // 2. Size of the prepared memo, if using the cost-based optimizer. size := p.PrepareMetadata.MemoryEstimate() - if p.Memo != nil { - size += p.Memo.MemoryEstimate() + if p.BaseMemo != nil { + size += p.BaseMemo.MemoryEstimate() } return size } From e3a6f17ef55c511adc2f72e4d295936d56fe73de Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 1 Jul 2024 17:30:22 -0400 Subject: [PATCH 08/15] sql: refactor plan_opt.go The logic for picking a prepared memo has been moved to a function, `fetchPreparedMemoLegacy`. This makes it easy in the following commit to fallback to this logic based on the `plan_cache_mode` setting. Release note: None --- pkg/sql/plan_opt.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index a544329ad4b2..b7e9a3c84bbd 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -522,12 +522,10 @@ func (opc *optPlanningCtx) reuseMemo( return f.Memo(), nil } -// buildExecMemo creates a fully optimized memo, possibly reusing a previously -// cached memo as a starting point. -// -// The returned memo is only safe to use in one thread, during execution of the -// current statement. -func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ error) { +// fetchPreparedMemoLegacy attempts to fetch a prepared memo. If a valid (i.e., +// non-stale) memo is found, it is used. Otherwise, a new statement will be +// built. If memo reuse is not allowed, nil is returned. +func (opc *optPlanningCtx) fetchPreparedMemoLegacy(ctx context.Context) (_ *memo.Memo, err error) { prepared := opc.p.stmt.Prepared p := opc.p if opc.allowMemoReuse && prepared != nil && prepared.BaseMemo != nil { @@ -549,6 +547,24 @@ func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ e return opc.reuseMemo(ctx, prepared.BaseMemo) } + return nil, nil +} + +// buildExecMemo creates a fully optimized memo, possibly reusing a previously +// cached memo as a starting point. +// +// The returned memo is only safe to use in one thread, during execution of the +// current statement. +func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ error) { + m, err := opc.fetchPreparedMemoLegacy(ctx) + if err != nil { + return nil, err + } + if m != nil { + return m, nil + } + + p := opc.p if opc.useCache { // Consult the query cache. cachedData, ok := p.execCfg.QueryCache.Find(&p.queryCacheSession, opc.p.stmt.SQL) From 74d690a7341641f8a0da4e955e8fc4b793cca3ec Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 1 Jul 2024 17:35:36 -0400 Subject: [PATCH 09/15] sql: implement plan_cache_mode=force_generic_plan The `force_generic_plan` option for the session setting `plan_cache_mode` has been implemented. Under this setting, prepared statements will reuse a fully optimized, generic query plan, if a non-stale one exists. If such a plan does not exist or is stale, a new generic query plan will be built. The term "generic query plan" is used in user-facing contexts to represent any plan that is fully optimized without knowledge of specific placeholder values nor folding stable expressions. Therefore, all of the following are considered "generic": 1. A plan for a query with no placeholders nor fold-able stable expressions. 2. A plan utilizing the placeholder fast-path. 3. A plan fully optimized without replacing placeholders with values. However, only (3) is currently stored in the `PreparedStatement.GenericMemo` field. This was a deliberate decision that allows falling back to the original prepared statement logic when switching between `force_custom_plan` and `force_generic_plan`. This will make backports with this commit less risky. My goal, in a future commit, is to put all generic query plans in the `GenericMemo` field to reduce the confusion between the user-facing definition of "generic" and the internal logic. This commit does not change the behavior of the query cache. Fully optimized plans for statements without placeholders or placeholder fast-path plans are still cached in the query cache. Generic query plans optimized in the presence of placeholders are not cached. Release note (sql change): The session setting `plan_cache_mode=force_generic_plan` can now be used to force prepared statements to use query plans that are optimized once and reused in future executions without re-optimization, as long as it does not become stale due to schema changes or a collection of new table statistics. The setting takes effect during `EXECUTE` commands. `EXPLAIN ANALYZE` includes a "plan type" field. If a generic query plan is optimized for the current execution, the "plan type" will be "generic, re-optimized". If a generic query plan is reused for the current execution without performing optimization, the "plan type" will be "generic, reused". Otherwise, the "plan type" will be "custom". --- .../testdata/logic_test/explain_call_plpgsql | 2 + pkg/sql/instrumentation.go | 16 +- pkg/sql/opt/exec/execbuilder/scalar.go | 11 +- pkg/sql/opt/exec/execbuilder/testdata/call | 2 + pkg/sql/opt/exec/execbuilder/testdata/cascade | 1 + .../exec/execbuilder/testdata/dist_vectorize | 2 + .../exec/execbuilder/testdata/distsql_misc | 9 + pkg/sql/opt/exec/execbuilder/testdata/explain | 1 + .../exec/execbuilder/testdata/explain_analyze | 4 + .../testdata/explain_analyze_plans | 6 + pkg/sql/opt/exec/execbuilder/testdata/generic | 635 ++++++++++++++++++ .../testdata/inverted_index_geospatial | 3 + .../execbuilder/testdata/lookup_join_limit | 4 + pkg/sql/opt/exec/execbuilder/testdata/prepare | 2 + pkg/sql/opt/exec/execbuilder/testdata/unique | 3 + .../exec/execbuilder/testdata/vectorize_local | 4 + .../execbuilder/tests/local/generated_test.go | 7 + pkg/sql/opt/exec/explain/output.go | 13 + pkg/sql/plan.go | 15 +- pkg/sql/plan_opt.go | 230 +++++-- pkg/sql/prepared_stmt.go | 9 + pkg/sql/testdata/explain_tree | 6 + 22 files changed, 942 insertions(+), 43 deletions(-) create mode 100644 pkg/sql/opt/exec/execbuilder/testdata/generic diff --git a/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql b/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql index d5e99c8327d8..1fbf19e05032 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql +++ b/pkg/ccl/logictestccl/testdata/logic_test/explain_call_plpgsql @@ -138,6 +138,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -158,6 +159,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: diff --git a/pkg/sql/instrumentation.go b/pkg/sql/instrumentation.go index c219c6e94c5a..f31469a0fab5 100644 --- a/pkg/sql/instrumentation.go +++ b/pkg/sql/instrumentation.go @@ -156,6 +156,16 @@ type instrumentationHelper struct { distribution physicalplan.PlanDistribution vectorized bool containsMutation bool + // generic is true if the plan can be fully-optimized once and re-used + // without re-optimization. Plans without placeholders and fold-able stable + // expressions, and plans utilizing the placeholder fast-path are always + // considered "generic". Plans that are fully optimized with placeholders + // present via the force_generic_plan or auto settings for plan_cache_mode + // are also considered "generic". + generic bool + // optimized is true if the plan was optimized or re-optimized during the + // current execution. + optimized bool traceMetadata execNodeTraceMetadata @@ -759,11 +769,13 @@ func (ih *instrumentationHelper) RecordExplainPlan(explainPlan *explain.Plan) { // RecordPlanInfo records top-level information about the plan. func (ih *instrumentationHelper) RecordPlanInfo( - distribution physicalplan.PlanDistribution, vectorized, containsMutation bool, + distribution physicalplan.PlanDistribution, vectorized, containsMutation, generic, optimized bool, ) { ih.distribution = distribution ih.vectorized = vectorized ih.containsMutation = containsMutation + ih.generic = generic + ih.optimized = optimized } // PlanForStats returns the plan as an ExplainTreePlanNode tree, if it was @@ -779,6 +791,7 @@ func (ih *instrumentationHelper) PlanForStats(ctx context.Context) *appstatspb.E }) ob.AddDistribution(ih.distribution.String()) ob.AddVectorized(ih.vectorized) + ob.AddPlanType(ih.generic, ih.optimized) if err := emitExplain(ctx, ob, ih.evalCtx, ih.codec, ih.explainPlan); err != nil { log.Warningf(ctx, "unable to emit explain plan tree: %v", err) return nil @@ -804,6 +817,7 @@ func (ih *instrumentationHelper) emitExplainAnalyzePlanToOutputBuilder( ob.AddExecutionTime(phaseTimes.GetRunLatency()) ob.AddDistribution(ih.distribution.String()) ob.AddVectorized(ih.vectorized) + ob.AddPlanType(ih.generic, ih.optimized) if queryStats != nil { if queryStats.KVRowsRead != 0 { diff --git a/pkg/sql/opt/exec/execbuilder/scalar.go b/pkg/sql/opt/exec/execbuilder/scalar.go index 1231841171b1..56aa16b6282e 100644 --- a/pkg/sql/opt/exec/execbuilder/scalar.go +++ b/pkg/sql/opt/exec/execbuilder/scalar.go @@ -57,7 +57,7 @@ func init() { opt.VariableOp: (*Builder).buildVariable, opt.ConstOp: (*Builder).buildTypedExpr, opt.NullOp: (*Builder).buildNull, - opt.PlaceholderOp: (*Builder).buildTypedExpr, + opt.PlaceholderOp: (*Builder).buildPlaceholder, opt.TupleOp: (*Builder).buildTuple, opt.FunctionOp: (*Builder).buildFunction, opt.CaseOp: (*Builder).buildCase, @@ -130,6 +130,15 @@ func (b *Builder) buildTypedExpr( return scalar.Private().(tree.TypedExpr), nil } +func (b *Builder) buildPlaceholder( + ctx *buildScalarCtx, scalar opt.ScalarExpr, +) (tree.TypedExpr, error) { + if b.evalCtx != nil && b.evalCtx.Placeholders != nil { + return eval.Expr(b.ctx, b.evalCtx, scalar.Private().(*tree.Placeholder)) + } + return b.buildTypedExpr(ctx, scalar) +} + func (b *Builder) buildNull(ctx *buildScalarCtx, scalar opt.ScalarExpr) (tree.TypedExpr, error) { retypedNull, ok := eval.ReType(tree.DNull, scalar.DataType()) if !ok { diff --git a/pkg/sql/opt/exec/execbuilder/testdata/call b/pkg/sql/opt/exec/execbuilder/testdata/call index cc3560e7abf6..4ae14cbaf1af 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/call +++ b/pkg/sql/opt/exec/execbuilder/testdata/call @@ -133,6 +133,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -153,6 +154,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/cascade b/pkg/sql/opt/exec/execbuilder/testdata/cascade index 01fe66c547d4..8742dc9ca85e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/cascade +++ b/pkg/sql/opt/exec/execbuilder/testdata/cascade @@ -1047,6 +1047,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 9 (72 B, 18 KVs, 9 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize b/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize index 75c44c3355af..7b0ca039bc6e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize +++ b/pkg/sql/opt/exec/execbuilder/testdata/dist_vectorize @@ -56,6 +56,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 5 (40 B, 10 KVs, 5 gRPC calls) maximum memory usage: network usage: @@ -93,6 +94,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc b/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc index 8af3271082a8..4933c1e8a471 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc +++ b/pkg/sql/opt/exec/execbuilder/testdata/distsql_misc @@ -105,6 +105,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -148,6 +149,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -167,6 +169,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -186,6 +189,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -208,6 +212,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -239,6 +244,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -262,6 +268,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -288,6 +295,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: @@ -314,6 +322,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 1,000 (7.8 KiB, 2,000 KVs, 1,000 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain b/pkg/sql/opt/exec/execbuilder/testdata/explain index 70b900d790cc..2b4d3766c470 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain @@ -2230,6 +2230,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze index 60740d987605..595172bcb374 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze @@ -13,6 +13,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -44,6 +45,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -82,6 +84,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, re-optimized rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -122,6 +125,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 7 (56 B, 14 KVs, 7 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans index 3a67ade395ff..cb4dda7edacd 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain_analyze_plans @@ -65,6 +65,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: @@ -130,6 +131,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: @@ -203,6 +205,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: @@ -289,6 +292,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 5 (40 B, 10 KVs, 5 gRPC calls) maximum memory usage: network usage: @@ -330,6 +334,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom maximum memory usage: network usage: regions: @@ -366,6 +371,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/generic b/pkg/sql/opt/exec/execbuilder/testdata/generic new file mode 100644 index 000000000000..a1c4824add6c --- /dev/null +++ b/pkg/sql/opt/exec/execbuilder/testdata/generic @@ -0,0 +1,635 @@ +# LogicTest: local + +# Disable stats collection to prevent automatic stats collection from +# invalidating plans. +statement ok +SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; + +statement ok +CREATE TABLE t ( + k INT PRIMARY KEY, + a INT, + b INT, + c INT, + t TIMESTAMPTZ, + INDEX (a), + INDEX (t) +) + +statement ok +SET plan_cache_mode = force_generic_plan + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 + +# A generic, reusable plan can be built during PREPARE for queries with no +# placeholders nor stable expressions. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +SET plan_cache_mode = force_custom_plan + +# The generic plan is reused even when forcing a custom plan because it will +# always be optimal. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +DEALLOCATE p + +# Prepare the same query with plan_cache_mode set to force_custom_plan. +statement ok +PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 + +# Execute it once with plan_cache_mode set to force_custom_plan. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +SET plan_cache_mode = force_generic_plan + +# The plan is generic (it has no placeholders), so it can be reused with +# plan_cache_mode set to force_generic_plan. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE k = $1 + +# A generic, reusable plan can be built during PREPARE when the placeholder +# fast-path can be used. +query T +EXPLAIN ANALYZE EXECUTE p(33) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: [/33 - /33] + +statement ok +SET plan_cache_mode = force_custom_plan + +# The generic plan is reused even when forcing a custom plan because it will +# always be optimal. +query T +EXPLAIN ANALYZE EXECUTE p(33) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: [/33 - /33] + +statement ok +SET plan_cache_mode = force_generic_plan + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = $1 AND c = $2 + +# A simple generic plan can be built during EXECUTE. +query T +EXPLAIN ANALYZE EXECUTE p(1, 2) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key +│ pred: c = "$2" +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +# The generic plan can be reused with different placeholder values. +query T +EXPLAIN ANALYZE EXECUTE p(11, 22) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key +│ pred: c = "$2" +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +statement ok +SET plan_cache_mode = force_custom_plan + +# The generic plan is not reused when forcing a custom plan. +query T +EXPLAIN ANALYZE EXECUTE p(1, 2) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 2 +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +SET plan_cache_mode = force_generic_plan + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE t = now() + +# A generic plan with a stable expression. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: t = now() +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: FULL SCAN + +# The generic plan can be reused. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: t = now() +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: FULL SCAN + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT k FROM t WHERE c = $1 + +# A simple generic plan. +query T +EXPLAIN ANALYZE EXECUTE p(1) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 1 +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: FULL SCAN + +statement ok +ALTER TABLE t ADD COLUMN z INT + +# A schema change invalidates the generic plan, and it must be re-optimized. +query T +EXPLAIN ANALYZE EXECUTE p(1) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 1 +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_pkey + spans: FULL SCAN + +statement ok +DEALLOCATE p + +statement ok +ALTER TABLE t DROP COLUMN z diff --git a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial index c0de285c231b..083ccf3051d9 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial +++ b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index_geospatial @@ -26,6 +26,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 6 (48 B, 12 KVs, 6 gRPC calls) maximum memory usage: network usage: @@ -118,6 +119,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, re-optimized rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: @@ -194,6 +196,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, re-optimized rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit index 28637346a383..9f31c8481599 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit +++ b/pkg/sql/opt/exec/execbuilder/testdata/lookup_join_limit @@ -80,6 +80,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: @@ -164,6 +165,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, reused rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: @@ -321,6 +323,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2 (16 B, 4 KVs, 2 gRPC calls) maximum memory usage: network usage: @@ -416,6 +419,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/prepare b/pkg/sql/opt/exec/execbuilder/testdata/prepare index d79ef3825daf..784433d906f8 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/prepare +++ b/pkg/sql/opt/exec/execbuilder/testdata/prepare @@ -115,6 +115,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, reused rows decoded from KV: 1 (8 B, 2 KVs, 1 gRPC calls) maximum memory usage: network usage: @@ -145,6 +146,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: generic, reused maximum memory usage: network usage: regions: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/unique b/pkg/sql/opt/exec/execbuilder/testdata/unique index d29040c4fe01..fa4f136b8d2e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/unique +++ b/pkg/sql/opt/exec/execbuilder/testdata/unique @@ -605,6 +605,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: @@ -720,6 +721,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 6 (48 B, 12 KVs, 6 gRPC calls) maximum memory usage: network usage: @@ -2602,6 +2604,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 10 (80 B, 20 KVs, 10 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local b/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local index a8076c8be0d9..9299a6206d4b 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local +++ b/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local @@ -42,6 +42,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 2,001 (16 KiB, 4,002 KVs, 2,001 gRPC calls) maximum memory usage: network usage: @@ -74,6 +75,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -123,6 +125,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 3 (24 B, 6 KVs, 3 gRPC calls) maximum memory usage: network usage: @@ -209,6 +212,7 @@ planning time: 10µs execution time: 100µs distribution: vectorized: +plan type: custom rows decoded from KV: 4 (32 B, 8 KVs, 4 gRPC calls) maximum memory usage: network usage: diff --git a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go index 834d6eee8359..7dc02ad0b4f2 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go @@ -258,6 +258,13 @@ func TestExecBuild_forecast1401( runExecBuildLogicTest(t, "forecast1401") } +func TestExecBuild_generic( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runExecBuildLogicTest(t, "generic") +} + func TestExecBuild_geospatial( t *testing.T, ) { diff --git a/pkg/sql/opt/exec/explain/output.go b/pkg/sql/opt/exec/explain/output.go index c97a70da8e4e..0b8305789871 100644 --- a/pkg/sql/opt/exec/explain/output.go +++ b/pkg/sql/opt/exec/explain/output.go @@ -307,6 +307,19 @@ func (ob *OutputBuilder) AddVectorized(value bool) { ob.AddFlakyTopLevelField(DeflakeVectorized, "vectorized", fmt.Sprintf("%t", value)) } +// AddGeneric adds a top-level generic field, if value is true. Cannot be called +// while inside a node. +func (ob *OutputBuilder) AddPlanType(generic, optimized bool) { + switch { + case generic && optimized: + ob.AddTopLevelField("plan type", "generic, re-optimized") + case generic && !optimized: + ob.AddTopLevelField("plan type", "generic, reused") + default: + ob.AddTopLevelField("plan type", "custom") + } +} + // AddPlanningTime adds a top-level planning time field. Cannot be called // while inside a node. func (ob *OutputBuilder) AddPlanningTime(delta time.Duration) { diff --git a/pkg/sql/plan.go b/pkg/sql/plan.go index d5b593c1b2f5..83a0af2969c6 100644 --- a/pkg/sql/plan.go +++ b/pkg/sql/plan.go @@ -486,7 +486,11 @@ func (p *planTop) savePlanInfo() { distribution = physicalplan.PartiallyDistributedPlan } containsMutation := p.flags.IsSet(planFlagContainsMutation) - p.instrumentation.RecordPlanInfo(distribution, vectorized, containsMutation) + generic := p.flags.IsSet(planFlagGeneric) + optimized := p.flags.IsSet(planFlagOptimized) + p.instrumentation.RecordPlanInfo( + distribution, vectorized, containsMutation, generic, optimized, + ) } // startExec calls startExec() on each planNode using a depth-first, post-order @@ -634,6 +638,15 @@ const ( // planFlagSessionMigration is set if the plan is being created during // a session migration. planFlagSessionMigration + + // planFlagGeneric is set if a generic query plan was used. A generic query + // plan is a plan that is fully-optimized once and can be reused without + // being re-optimized. + planFlagGeneric + + // planFlagOptimized is set if optimization was performed during the + // current execution of the query. + planFlagOptimized ) func (pf planFlags) IsSet(flag planFlags) bool { diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index b7e9a3c84bbd..268931287b58 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -166,7 +166,8 @@ func (p *planner) prepareUsingOptimizer( opc.flags.Set(planFlagOptCacheMiss) } - memo, err := opc.buildReusableMemo(ctx) + // Build the memo. Do not attempt to build a generic plan at PREPARE-time. + memo, _, err := opc.buildReusableMemo(ctx, false /* allowGeneric */) if err != nil { return 0, err } @@ -404,28 +405,38 @@ func (opc *optPlanningCtx) log(ctx context.Context, msg redact.SafeString) { // buildReusableMemo builds the statement into a memo that can be stored for // prepared statements and can later be used as a starting point for -// optimization. The returned memo is fully detached from the planner and can be -// used with reuseMemo independently and concurrently by multiple threads. -func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, _ error) { +// optimization. The returned memo is fully optimized if: +// +// 1. The statement does not contain placeholders nor fold-able stable +// operators. +// 2. Or, the placeholder fast path is used. +// 3. Or, useGeneric is true and the plan is fully optimized as best as +// possible in the presence of placeholders. +// +// The returned memo is fully detached from the planner and can be used with +// reuseMemo independently and concurrently by multiple threads. +func (opc *optPlanningCtx) buildReusableMemo( + ctx context.Context, useGeneric bool, +) (_ *memo.Memo, generic bool, _ error) { p := opc.p _, isCanned := opc.p.stmt.AST.(*tree.CannedOptPlan) if isCanned { if !p.EvalContext().SessionData().AllowPrepareAsOptPlan { - return nil, pgerror.New(pgcode.InsufficientPrivilege, + return nil, false, pgerror.New(pgcode.InsufficientPrivilege, "PREPARE AS OPT PLAN is a testing facility that should not be used directly", ) } if !p.SessionData().User().IsRootUser() { - return nil, pgerror.New(pgcode.InsufficientPrivilege, + return nil, false, pgerror.New(pgcode.InsufficientPrivilege, "PREPARE AS OPT PLAN may only be used by root", ) } } if p.SessionData().SaveTablesPrefix != "" && !p.SessionData().User().IsRootUser() { - return nil, pgerror.New(pgcode.InsufficientPrivilege, + return nil, false, pgerror.New(pgcode.InsufficientPrivilege, "sub-expression tables creation may only be used by root", ) } @@ -443,7 +454,7 @@ func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, bld.SkipAOST = true } if err := bld.Build(); err != nil { - return nil, err + return nil, false, err } if bld.DisableMemoReuse { @@ -457,38 +468,48 @@ func (opc *optPlanningCtx) buildReusableMemo(ctx context.Context) (_ *memo.Memo, // We don't support placeholders inside the canned plan. The main reason // is that they would be invisible to the parser (which reports the number // of placeholders, used to initialize the relevant structures). - return nil, pgerror.Newf(pgcode.Syntax, + return nil, false, pgerror.Newf(pgcode.Syntax, "placeholders are not supported with PREPARE AS OPT PLAN") } // With a canned plan, we don't want to optimize the memo. - return opc.optimizer.DetachMemo(ctx), nil + return opc.optimizer.DetachMemo(ctx), false, nil } - if f.Memo().HasPlaceholders() { - // Try the placeholder fast path. - _, ok, err := opc.optimizer.TryPlaceholderFastPath() - if err != nil { - return nil, err - } - if ok { - opc.log(ctx, "placeholder fast path") + // If the memo doesn't have placeholders and did not encounter any stable + // operators that can be constant-folded, then fully optimize it now - it + // can be reused without further changes to build the execution tree. + if !f.Memo().HasPlaceholders() && !f.FoldingControl().PreventedStableFold() { + opc.log(ctx, "optimizing (no placeholders)") + if _, err := opc.optimizer.Optimize(); err != nil { + return nil, false, err } - } else { - // If the memo doesn't have placeholders and did not encounter any stable - // operators that can be constant folded, then fully optimize it now - it - // can be reused without further changes to build the execution tree. - if !f.FoldingControl().PreventedStableFold() { - opc.log(ctx, "optimizing (no placeholders)") - if _, err := opc.optimizer.Optimize(); err != nil { - return nil, err - } + opc.flags.Set(planFlagOptimized) + return opc.optimizer.DetachMemo(ctx), false, nil + } + + // If the memo has placeholders, first try the placeholder fast path. + _, ok, err := opc.optimizer.TryPlaceholderFastPath() + if err != nil { + return nil, false, err + } + if ok { + opc.log(ctx, "placeholder fast path") + opc.flags.Set(planFlagOptimized) + } else if useGeneric { + // Build a generic query plan if the placeholder fast path failed and a + // generic plan was requested. + opc.log(ctx, "optimizing (generic)") + if _, err := opc.optimizer.Optimize(); err != nil { + return nil, false, err } + opc.flags.Set(planFlagOptimized) + return opc.optimizer.DetachMemo(ctx), true, nil } // Detach the prepared memo from the factory and transfer its ownership // to the prepared statement. DetachMemo will re-initialize the optimizer // to an empty memo. - return opc.optimizer.DetachMemo(ctx), nil + return opc.optimizer.DetachMemo(ctx), false, nil } // reuseMemo returns an optimized memo using a cached memo as a starting point. @@ -502,9 +523,9 @@ func (opc *optPlanningCtx) reuseMemo( ctx context.Context, cachedMemo *memo.Memo, ) (*memo.Memo, error) { if cachedMemo.IsOptimized() { - // The query could have been already fully optimized if there were no - // placeholders or the placeholder fast path succeeded (see - // buildReusableMemo). + // The query could have been already fully optimized in + // buildReusableMemo, in which case it is considered a "generic" plan. + opc.flags.Set(planFlagGeneric) return cachedMemo, nil } f := opc.optimizer.Factory() @@ -519,9 +540,124 @@ func (opc *optPlanningCtx) reuseMemo( if _, err := opc.optimizer.Optimize(); err != nil { return nil, err } + opc.flags.Set(planFlagOptimized) return f.Memo(), nil } +// chooseValidPreparedMemo returns an optimized memo that is equal to, or built +// from, baseMemo or genericMemo. It returns nil if both memos are stale. It +// selects baseMemo or genericMemo based on the following rules, in order: +// +// 1. If baseMemo is fully optimized and not stale, it is returned as-is. +// 2. If useGeneric is true and genericMemo is not stale, it is returned +// as-is. +// 3. If useGeneric is true and genericMemo is stale or nil, nil is returned. +// The caller is responsible for building a new generic memo. +// 4. If baseMemo is not stale and unoptimized, optimize and return it. +// 5. Otherwise, nil is returned. The caller is responsible for building a new +// memo. +// +// The logic is structured to avoid unnecessary (*memo.Memo).IsStale calls, +// since they can be expensive. +func (opc *optPlanningCtx) chooseValidPreparedMemo( + ctx context.Context, baseMemo *memo.Memo, genericMemo *memo.Memo, useGeneric bool, +) (*memo.Memo, error) { + // First check for a fully optimized, non-stale, base memo. + if baseMemo != nil && baseMemo.IsOptimized() { + isStale, err := baseMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) + if err != nil { + return nil, err + } else if !isStale { + return baseMemo, nil + } + } + + // Next check for a non-stale, generic memo. + if useGeneric && genericMemo != nil { + isStale, err := genericMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) + if err != nil { + return nil, err + } else if !isStale { + return genericMemo, nil + } + } + + // Next, check for a non-stale, normalized memo, if a generic memo is + // not allowed. + if !useGeneric && baseMemo != nil && !baseMemo.IsOptimized() { + isStale, err := baseMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) + if err != nil { + return nil, err + } else if !isStale { + return baseMemo, nil + } + } + + // A valid memo was not found. + return nil, nil +} + +// fetchPreparedMemo attempts to fetch a memo from the prepared statement +// struct. If a valid (i.e., non-stale) memo is found, it is used. Otherwise, a +// new statement will be built. +// +// The plan_cache_mode session setting controls how this function decides +// between what type of memo to use or reuse: +// +// - force_custom_plan: A fully optimized generic memo will be used if it +// either has no placeholders nor fold-able stable expressions, or it +// utilizes the placeholder fast-path. Otherwise, a normalized memo will be +// fetched or rebuilt, copied into a new memo with placeholders replaced +// with values, and re-optimized. +// +// - force_generic_plan: A fully optimized generic memo will always be used. +// The BaseMemo will be used if it is fully optimized. Otherwise, the +// GenericMemo will be used. +// +// - auto: This currently behaves the same as force_custom_plan. +// +// TODO(mgartner): Implement "auto". +func (opc *optPlanningCtx) fetchPreparedMemo(ctx context.Context) (_ *memo.Memo, err error) { + p := opc.p + prep := p.stmt.Prepared + if !opc.allowMemoReuse || prep == nil { + return nil, nil + } + + useGeneric := opc.p.SessionData().PlanCacheMode == sessiondatapb.PlanCacheModeForceGeneric + + // If the statement was previously prepared, check for a reusable memo. + // First check for a valid (non-stale) memo. + validMemo, err := opc.chooseValidPreparedMemo(ctx, prep.BaseMemo, prep.GenericMemo, useGeneric) + if err != nil { + return nil, err + } + if validMemo != nil { + opc.log(ctx, "reusing cached memo") + return opc.reuseMemo(ctx, validMemo) + } + + // Otherwise, we need to rebuild the memo. + // + // TODO(mgartner): If we have a non-stale, normalized base memo, we can + // build a generic memo from it instead of building the memo from + // scratch. + opc.log(ctx, "rebuilding cached memo") + newMemo, generic, err := opc.buildReusableMemo(ctx, useGeneric) + if err != nil { + return nil, err + } + if generic { + // TODO(mgartner): Add the generic memo to the query cache so that it + // can be reused by future prepared statements. + prep.GenericMemo = newMemo + } else { + prep.BaseMemo = newMemo + } + // Re-optimize the memo, if necessary. + return opc.reuseMemo(ctx, newMemo) +} + // fetchPreparedMemoLegacy attempts to fetch a prepared memo. If a valid (i.e., // non-stale) memo is found, it is used. Otherwise, a new statement will be // built. If memo reuse is not allowed, nil is returned. @@ -538,7 +674,7 @@ func (opc *optPlanningCtx) fetchPreparedMemoLegacy(ctx context.Context) (_ *memo return nil, err } else if isStale { opc.log(ctx, "rebuilding cached memo") - prepared.BaseMemo, err = opc.buildReusableMemo(ctx) + prepared.BaseMemo, _, err = opc.buildReusableMemo(ctx, false /* useGeneric */) if err != nil { return nil, err } @@ -556,15 +692,29 @@ func (opc *optPlanningCtx) fetchPreparedMemoLegacy(ctx context.Context) (_ *memo // The returned memo is only safe to use in one thread, during execution of the // current statement. func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ error) { - m, err := opc.fetchPreparedMemoLegacy(ctx) - if err != nil { - return nil, err - } - if m != nil { - return m, nil + p := opc.p + if p.SessionData().PlanCacheMode == sessiondatapb.PlanCacheModeForceCustom { + // Fallback to the legacy logic for reusing memos if plan_cache_mode is + // set to force_custom_plan. + m, err := opc.fetchPreparedMemoLegacy(ctx) + if err != nil { + return nil, err + } + if m != nil { + return m, nil + } + } else { + // Use new logic for reusing memos if plan_cache_mode is set to + // force_generic_plan or auto. + m, err := opc.fetchPreparedMemo(ctx) + if err != nil { + return nil, err + } + if m != nil { + return m, nil + } } - p := opc.p if opc.useCache { // Consult the query cache. cachedData, ok := p.execCfg.QueryCache.Find(&p.queryCacheSession, opc.p.stmt.SQL) @@ -573,7 +723,7 @@ func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ e return nil, err } else if isStale { opc.log(ctx, "query cache hit but needed update") - cachedData.Memo, err = opc.buildReusableMemo(ctx) + cachedData.Memo, _, err = opc.buildReusableMemo(ctx, false /* allowGeneric */) if err != nil { return nil, err } diff --git a/pkg/sql/prepared_stmt.go b/pkg/sql/prepared_stmt.go index 815a9ce74b5f..3754512b1641 100644 --- a/pkg/sql/prepared_stmt.go +++ b/pkg/sql/prepared_stmt.go @@ -61,6 +61,12 @@ type PreparedStatement struct { // stable expressions. Otherwise, it is an unoptimized, normalized memo. BaseMemo *memo.Memo + // GenericMemo, if present, is a fully-optimized memo that can be executed + // as-is. + // TODO(mgartner): Put all fully-optimized plans in the GenericMemo field to + // reduce confusion. + GenericMemo *memo.Memo + // refCount keeps track of the number of references to this PreparedStatement. // New references are registered through incRef(). // Once refCount hits 0 (through calls to decRef()), the following memAcc is @@ -88,6 +94,9 @@ func (p *PreparedStatement) MemoryEstimate() int64 { if p.BaseMemo != nil { size += p.BaseMemo.MemoryEstimate() } + if p.GenericMemo != nil { + size += p.GenericMemo.MemoryEstimate() + } return size } diff --git a/pkg/sql/testdata/explain_tree b/pkg/sql/testdata/explain_tree index 5803523fc74b..4fbef824d764 100644 --- a/pkg/sql/testdata/explain_tree +++ b/pkg/sql/testdata/explain_tree @@ -11,6 +11,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -46,6 +47,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -81,6 +83,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -162,6 +165,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -197,6 +201,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable @@ -289,6 +294,7 @@ planning time: 0µs execution time: 0µs distribution: local vectorized: false +plan type: custom maximum memory usage: 0 B network usage: 0 B (0 messages) isolation level: serializable From 48e3b967eab84b325dcfe70d7e6e6f67622e769e Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 8 Jul 2024 16:37:44 -0400 Subject: [PATCH 10/15] sql: implement plan_cache_mode=auto The `plan_cache_mode=auto` session setting has been implemented which instructs the database to pick generic query plans for prepared statements when they have a cost less than or equal to custom plans. The current strategy matches Postgres's strategy quite closely. Under `plan_cache_mode=auto`, custom query plans will be generated for the first five executions of a prepared statement. On the sixth execution, a generic query plan will be generated. If the cost of the generic query plan is less than the average cost of the custom plans (plus some overhead for re-optimization), then the generic query plan will be used. Release note (sql change): The session setting `plan_cache_mode=auto` can now be used to instruct the system to automatically determine whether to use "custom" or "generic" query plans for the execution of a prepared statement. Custom query plans are optimized on every execution, while generic plans are optimized once and reused on future executions as-is. Generic query plans are beneficial in cases where query optimization contributes significant overhead to the total cost of executing a query. --- pkg/sql/BUILD.bazel | 1 + pkg/sql/opt/exec/execbuilder/testdata/generic | 529 +++++++++++++++++- pkg/sql/opt/memo/memo.go | 11 + pkg/sql/opt/metadata.go | 5 + pkg/sql/plan_opt.go | 166 ++++-- pkg/sql/prepared_stmt.go | 85 ++- pkg/sql/prepared_stmt_test.go | 51 ++ 7 files changed, 793 insertions(+), 55 deletions(-) create mode 100644 pkg/sql/prepared_stmt_test.go diff --git a/pkg/sql/BUILD.bazel b/pkg/sql/BUILD.bazel index 3abba9f2ab71..2cc6d505adf6 100644 --- a/pkg/sql/BUILD.bazel +++ b/pkg/sql/BUILD.bazel @@ -684,6 +684,7 @@ go_test( "pg_oid_test.go", "pgwire_internal_test.go", "plan_opt_test.go", + "prepared_stmt_test.go", "privileged_accessor_test.go", "rand_test.go", "region_util_test.go", diff --git a/pkg/sql/opt/exec/execbuilder/testdata/generic b/pkg/sql/opt/exec/execbuilder/testdata/generic index a1c4824add6c..1358b39518c0 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/generic +++ b/pkg/sql/opt/exec/execbuilder/testdata/generic @@ -11,18 +11,35 @@ CREATE TABLE t ( a INT, b INT, c INT, + s STRING, t TIMESTAMPTZ, INDEX (a), + INDEX (s), INDEX (t) ) +statement ok +CREATE TABLE c ( + k INT PRIMARY KEY, + a INT, + INDEX (a) +) + +statement ok +CREATE TABLE g ( + k INT PRIMARY KEY, + a INT, + b INT, + INDEX (a, b) +) + statement ok SET plan_cache_mode = force_generic_plan statement ok PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 -# A generic, reusable plan can be built during PREPARE for queries with no +# An ideal generic plan can be built during PREPARE for queries with no # placeholders nor stable expressions. query T EXPLAIN ANALYZE EXECUTE p @@ -75,8 +92,8 @@ quality of service: regular statement ok SET plan_cache_mode = force_custom_plan -# The generic plan is reused even when forcing a custom plan because it will -# always be optimal. +# The ideal generic plan is reused even when forcing a custom plan because it +# will always be optimal. query T EXPLAIN ANALYZE EXECUTE p ---- @@ -184,8 +201,8 @@ quality of service: regular statement ok SET plan_cache_mode = force_generic_plan -# The plan is generic (it has no placeholders), so it can be reused with -# plan_cache_mode set to force_generic_plan. +# The plan is an ideal generic plan (it has no placeholders), so it can be +# reused with plan_cache_mode set to force_generic_plan. query T EXPLAIN ANALYZE EXECUTE p ---- @@ -240,7 +257,7 @@ DEALLOCATE p statement ok PREPARE p AS SELECT * FROM t WHERE k = $1 -# A generic, reusable plan can be built during PREPARE when the placeholder +# An ideal generic plan can be built during PREPARE when the placeholder # fast-path can be used. query T EXPLAIN ANALYZE EXECUTE p(33) @@ -274,8 +291,8 @@ quality of service: regular statement ok SET plan_cache_mode = force_custom_plan -# The generic plan is reused even when forcing a custom plan because it will -# always be optimal. +# The ideal generic plan is reused even when forcing a custom plan because it +# will always be optimal. query T EXPLAIN ANALYZE EXECUTE p(33) ---- @@ -550,6 +567,48 @@ quality of service: regular statement ok DEALLOCATE p +statement ok +PREPARE p AS SELECT k FROM t WHERE s LIKE $1 + +# A suboptimal generic query plan is chosen if it is forced. +query T +EXPLAIN ANALYZE EXECUTE p('foo%') +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: s LIKE 'foo%' +│ +└── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_s_idx + spans: (/NULL - ] + +statement ok +DEALLOCATE p + statement ok PREPARE p AS SELECT k FROM t WHERE c = $1 @@ -633,3 +692,457 @@ DEALLOCATE p statement ok ALTER TABLE t DROP COLUMN z + +statement ok +SET plan_cache_mode = auto + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = 1 AND b = 2 AND c = 3 + +# An ideal generic plan is used immediately with plan_cache_mode=auto. +query T +EXPLAIN ANALYZE EXECUTE p +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: (b = 2) AND (c = 3) +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/1 - /1] + +statement ok +DEALLOCATE p + +statement ok +PREPARE p AS SELECT * FROM t WHERE a = $1 AND c = $2 + +statement ok +EXECUTE p(1, 2); +EXECUTE p(10, 20); +EXECUTE p(100, 200); +EXECUTE p(1000, 2000); +EXECUTE p(10000, 20000); + +# On the sixth execution a generic query plan will be generated. The cost of the +# generic plan is more than the average cost of the five custom plans (plus some +# overhead cost of optimization), so the custom plan is chosen. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• filter +│ nodes: +│ regions: +│ actual row count: 0 +│ filter: c = 20000 +│ +└── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/10000 - /10000] + +statement ok +DEALLOCATE p + +# Now use a more complex query that could have multiple join orderings. +statement ok +PREPARE p AS +SELECT * FROM t +JOIN c ON t.k = c.a +JOIN g ON c.k = g.a +WHERE t.a = $1 AND t.c = $2 + +statement ok +EXECUTE p(1, 2); +EXECUTE p(10, 20); +EXECUTE p(100, 200); +EXECUTE p(1000, 2000); + +# The first five executions will use a custom plan. This is the fifth. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join (streamer) +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: g@g_a_b_idx +│ equality: (k) = (a) +│ +└── • lookup join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: c@c_a_idx + │ equality: (k) = (a) + │ + └── • filter + │ nodes: + │ regions: + │ actual row count: 0 + │ filter: c = 20000 + │ + └── • index join (streamer) + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: t@t_pkey + │ + └── • scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: t@t_a_idx + spans: [/10000 - /10000] + +# On the sixth execution a generic query plan will be generated. The cost of the +# generic plan is less than the average cost of the five custom plans (plus some +# overhead cost of optimization), so the generic plan is chosen. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, re-optimized +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: g@g_a_b_idx +│ equality: (k) = (a) +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: c@c_a_idx + │ equality: (k) = (a) + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_pkey + │ equality: (k) = (k) + │ equality cols are key + │ pred: c = "$2" + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +# On the seventh execution the generic plan is reused. +query T +EXPLAIN ANALYZE EXECUTE p(10000, 20000) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• lookup join +│ nodes: +│ regions: +│ actual row count: 0 +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: g@g_a_b_idx +│ equality: (k) = (a) +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: c@c_a_idx + │ equality: (k) = (a) + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_pkey + │ equality: (k) = (k) + │ equality cols are key + │ pred: c = "$2" + │ + └── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_a_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 2 columns, 1 row + +statement ok +DEALLOCATE p + +# Now use a query with a bad generic query plan. +statement ok +PREPARE p AS +SELECT * FROM g WHERE a = $1 ORDER BY b LIMIT 10 + +statement ok +EXECUTE p(1); +EXECUTE p(10); +EXECUTE p(100); +EXECUTE p(1000); +EXECUTE p(10000); + +# The generic plan is generated on the sixth execution, but is more expensive +# than the average custom plan. +query T +EXPLAIN ANALYZE EXECUTE p(10) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: custom +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• scan + nodes: + regions: + actual row count: 0 + KV time: 0µs + KV contention time: 0µs + KV rows decoded: 0 + KV bytes read: 0 B + KV gRPC calls: 0 + estimated max memory allocated: 0 B + missing stats + table: g@g_a_b_idx + spans: [/10 - /10] + limit: 10 + +statement ok +SET plan_cache_mode = force_generic_plan + +# The generic plan previously generated is reused when forcing a generic plan. +query T +EXPLAIN ANALYZE EXECUTE p(10) +---- +planning time: 10µs +execution time: 100µs +distribution: +vectorized: +plan type: generic, reused +maximum memory usage: +network usage: +regions: +isolation level: serializable +priority: normal +quality of service: regular +· +• limit +│ count: 10 +│ +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ estimated max sql temp disk usage: 0 B + │ table: g@g_a_b_idx + │ equality: ($1) = (a) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 1 column, 1 row diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go index 30d45be5b4d2..5789787ce1dd 100644 --- a/pkg/sql/opt/memo/memo.go +++ b/pkg/sql/opt/memo/memo.go @@ -508,6 +508,17 @@ func (m *Memo) IsOptimized() bool { return ok && rel.RequiredPhysical() != nil } +// OptimizationCost returns a rough estimate of the cost of optimization of the +// memo. It is dependent on the number of tables in the metadata, based on the +// reasoning that queries with more tables likely have more joins, which tend +// to be the biggest contributors to optimization overhead. +func (m *Memo) OptimizationCost() Cost { + // This cpuCostFactor is the same as cpuCostFactor in the coster. + // TODO(mgartner): Package these constants up in a shared location. + const cpuCostFactor = 0.01 + return Cost(m.Metadata().NumTables()) * 1000 * cpuCostFactor +} + // NextRank returns a new rank that can be assigned to a scalar expression. func (m *Memo) NextRank() opt.ScalarRank { m.curRank++ diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index 95e1d956f0b5..219db44f78cc 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -765,6 +765,11 @@ func (md *Metadata) AllTables() []TableMeta { return md.tables } +// NumTables returns the number of tables in the metadata. +func (md *Metadata) NumTables() int { + return len(md.tables) +} + // AddColumn assigns a new unique id to a column within the query and records // its alias and type. If the alias is empty, a "column" alias is created. func (md *Metadata) AddColumn(alias string, typ *types.T) ColumnID { diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index 268931287b58..e826a8d83bac 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -167,7 +167,7 @@ func (p *planner) prepareUsingOptimizer( } // Build the memo. Do not attempt to build a generic plan at PREPARE-time. - memo, _, err := opc.buildReusableMemo(ctx, false /* allowGeneric */) + memo, _, err := opc.buildReusableMemo(ctx, false /* buildGeneric */) if err != nil { return 0, err } @@ -403,6 +403,15 @@ func (opc *optPlanningCtx) log(ctx context.Context, msg redact.SafeString) { } } +type memoType int + +const ( + memoTypeUnknown memoType = iota + memoTypeCustom + memoTypeGeneric + memoTypeIdealGeneric +) + // buildReusableMemo builds the statement into a memo that can be stored for // prepared statements and can later be used as a starting point for // optimization. The returned memo is fully optimized if: @@ -410,33 +419,33 @@ func (opc *optPlanningCtx) log(ctx context.Context, msg redact.SafeString) { // 1. The statement does not contain placeholders nor fold-able stable // operators. // 2. Or, the placeholder fast path is used. -// 3. Or, useGeneric is true and the plan is fully optimized as best as +// 3. Or, buildGeneric is true and the plan is fully optimized as best as // possible in the presence of placeholders. // // The returned memo is fully detached from the planner and can be used with // reuseMemo independently and concurrently by multiple threads. func (opc *optPlanningCtx) buildReusableMemo( - ctx context.Context, useGeneric bool, -) (_ *memo.Memo, generic bool, _ error) { + ctx context.Context, buildGeneric bool, +) (*memo.Memo, memoType, error) { p := opc.p _, isCanned := opc.p.stmt.AST.(*tree.CannedOptPlan) if isCanned { if !p.EvalContext().SessionData().AllowPrepareAsOptPlan { - return nil, false, pgerror.New(pgcode.InsufficientPrivilege, + return nil, memoTypeUnknown, pgerror.New(pgcode.InsufficientPrivilege, "PREPARE AS OPT PLAN is a testing facility that should not be used directly", ) } if !p.SessionData().User().IsRootUser() { - return nil, false, pgerror.New(pgcode.InsufficientPrivilege, + return nil, memoTypeUnknown, pgerror.New(pgcode.InsufficientPrivilege, "PREPARE AS OPT PLAN may only be used by root", ) } } if p.SessionData().SaveTablesPrefix != "" && !p.SessionData().User().IsRootUser() { - return nil, false, pgerror.New(pgcode.InsufficientPrivilege, + return nil, memoTypeUnknown, pgerror.New(pgcode.InsufficientPrivilege, "sub-expression tables creation may only be used by root", ) } @@ -454,7 +463,7 @@ func (opc *optPlanningCtx) buildReusableMemo( bld.SkipAOST = true } if err := bld.Build(); err != nil { - return nil, false, err + return nil, memoTypeUnknown, err } if bld.DisableMemoReuse { @@ -468,11 +477,12 @@ func (opc *optPlanningCtx) buildReusableMemo( // We don't support placeholders inside the canned plan. The main reason // is that they would be invisible to the parser (which reports the number // of placeholders, used to initialize the relevant structures). - return nil, false, pgerror.Newf(pgcode.Syntax, + return nil, memoTypeUnknown, pgerror.Newf(pgcode.Syntax, "placeholders are not supported with PREPARE AS OPT PLAN") } - // With a canned plan, we don't want to optimize the memo. - return opc.optimizer.DetachMemo(ctx), false, nil + // With a canned plan, we don't want to optimize the memo. Since we + // won't optimize it, we consider it an ideal generic plan. + return opc.optimizer.DetachMemo(ctx), memoTypeIdealGeneric, nil } // If the memo doesn't have placeholders and did not encounter any stable @@ -481,35 +491,36 @@ func (opc *optPlanningCtx) buildReusableMemo( if !f.Memo().HasPlaceholders() && !f.FoldingControl().PreventedStableFold() { opc.log(ctx, "optimizing (no placeholders)") if _, err := opc.optimizer.Optimize(); err != nil { - return nil, false, err + return nil, memoTypeUnknown, err } opc.flags.Set(planFlagOptimized) - return opc.optimizer.DetachMemo(ctx), false, nil + return opc.optimizer.DetachMemo(ctx), memoTypeIdealGeneric, nil } // If the memo has placeholders, first try the placeholder fast path. _, ok, err := opc.optimizer.TryPlaceholderFastPath() if err != nil { - return nil, false, err + return nil, memoTypeUnknown, err } if ok { opc.log(ctx, "placeholder fast path") opc.flags.Set(planFlagOptimized) - } else if useGeneric { + return opc.optimizer.DetachMemo(ctx), memoTypeIdealGeneric, nil + } else if buildGeneric { // Build a generic query plan if the placeholder fast path failed and a // generic plan was requested. opc.log(ctx, "optimizing (generic)") if _, err := opc.optimizer.Optimize(); err != nil { - return nil, false, err + return nil, memoTypeUnknown, err } opc.flags.Set(planFlagOptimized) - return opc.optimizer.DetachMemo(ctx), true, nil + return opc.optimizer.DetachMemo(ctx), memoTypeGeneric, nil } // Detach the prepared memo from the factory and transfer its ownership // to the prepared statement. DetachMemo will re-initialize the optimizer // to an empty memo. - return opc.optimizer.DetachMemo(ctx), false, nil + return opc.optimizer.DetachMemo(ctx), memoTypeCustom, nil } // reuseMemo returns an optimized memo using a cached memo as a starting point. @@ -541,7 +552,38 @@ func (opc *optPlanningCtx) reuseMemo( return nil, err } opc.flags.Set(planFlagOptimized) - return f.Memo(), nil + mem := f.Memo() + if prep := opc.p.stmt.Prepared; opc.allowMemoReuse && prep != nil { + prep.Costs.AddCustom(mem.RootExpr().(memo.RelExpr).Cost() + mem.OptimizationCost()) + } + return mem, nil +} + +// useGenericPlan returns true if a generic query plan should be used instead of +// a custom plan. +func (opc *optPlanningCtx) useGenericPlan() bool { + switch opc.p.SessionData().PlanCacheMode { + case sessiondatapb.PlanCacheModeForceGeneric: + return true + case sessiondatapb.PlanCacheModeAuto: + prep := opc.p.stmt.Prepared + // We need to build CustomPlanThreshold custom plans before considering + // a generic plan. + if prep.Costs.NumCustom() < CustomPlanThreshold { + return false + } + // A generic plan should be used if we have CustomPlanThreshold custom + // plan costs and: + // + // 1. The generic cost is unknown because a generic plan has not been + // built. + // 2. Or, the cost of the generic plan is less than or equal to the + // average cost of the custom plans. + // + return prep.Costs.Generic() == 0 || prep.Costs.Generic() < prep.Costs.AvgCustom() + default: + return false + } } // chooseValidPreparedMemo returns an optimized memo that is equal to, or built @@ -549,18 +591,23 @@ func (opc *optPlanningCtx) reuseMemo( // selects baseMemo or genericMemo based on the following rules, in order: // // 1. If baseMemo is fully optimized and not stale, it is returned as-is. -// 2. If useGeneric is true and genericMemo is not stale, it is returned -// as-is. -// 3. If useGeneric is true and genericMemo is stale or nil, nil is returned. -// The caller is responsible for building a new generic memo. -// 4. If baseMemo is not stale and unoptimized, optimize and return it. -// 5. Otherwise, nil is returned. The caller is responsible for building a new -// memo. +// 2. If plan_cache_mode=force_generic_plan is true then genericMemo is +// returned as-is if it is not stale. +// 3. If plan_cache_mode=auto, there have been at least 5 custom plans +// generated, and the cost of the generic memo is less than the average cost +// of the custom plans, then the generic memo is returned as-is if it is not +// stale. If the cost of the generic memo is greater than or equal to the +// average cost of the custom plans, then the baseMemo is returned if it is +// not stale. +// 4. If plan_cache_mode=force_custom_plan, baseMemo is returned if it is not +// stale. +// 5. Otherwise, nil is returned and the caller is responsible for building a +// new memo. // // The logic is structured to avoid unnecessary (*memo.Memo).IsStale calls, // since they can be expensive. func (opc *optPlanningCtx) chooseValidPreparedMemo( - ctx context.Context, baseMemo *memo.Memo, genericMemo *memo.Memo, useGeneric bool, + ctx context.Context, baseMemo *memo.Memo, genericMemo *memo.Memo, ) (*memo.Memo, error) { // First check for a fully optimized, non-stale, base memo. if baseMemo != nil && baseMemo.IsOptimized() { @@ -572,24 +619,37 @@ func (opc *optPlanningCtx) chooseValidPreparedMemo( } } + prep := opc.p.stmt.Prepared + reuseGeneric := opc.useGenericPlan() + // Next check for a non-stale, generic memo. - if useGeneric && genericMemo != nil { + if reuseGeneric && genericMemo != nil { isStale, err := genericMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) if err != nil { return nil, err } else if !isStale { return genericMemo, nil + } else { + // Clear the generic cost if the memo is stale. DDL or new stats + // could drastically change the cost of generic and custom plans, so + // we should re-consider which to use. + prep.Costs.ClearGeneric() } } - // Next, check for a non-stale, normalized memo, if a generic memo is - // not allowed. - if !useGeneric && baseMemo != nil && !baseMemo.IsOptimized() { + // Next, check for a non-stale, normalized memo, if a generic memo should + // not be reused. + if !reuseGeneric && baseMemo != nil && !baseMemo.IsOptimized() { isStale, err := baseMemo.IsStale(ctx, opc.p.EvalContext(), opc.catalog) if err != nil { return nil, err } else if !isStale { return baseMemo, nil + } else { + // Clear the custom costs if the memo is stale. DDL or new stats + // could drastically change the cost of generic and custom plans, so + // we should re-consider which to use. + prep.Costs.ClearCustom() } } @@ -614,9 +674,11 @@ func (opc *optPlanningCtx) chooseValidPreparedMemo( // The BaseMemo will be used if it is fully optimized. Otherwise, the // GenericMemo will be used. // -// - auto: This currently behaves the same as force_custom_plan. -// -// TODO(mgartner): Implement "auto". +// - auto: A "custom plan" will be optimized for first five executions of the +// prepared statement. On the sixth execution, a "generic plan" will be +// generated. If its cost is less than the average cost of the custom plans +// (plus some optimization overhead cost), then the generic plan will be +// used. Otherwise, a custom plan will be used. func (opc *optPlanningCtx) fetchPreparedMemo(ctx context.Context) (_ *memo.Memo, err error) { p := opc.p prep := p.stmt.Prepared @@ -624,11 +686,9 @@ func (opc *optPlanningCtx) fetchPreparedMemo(ctx context.Context) (_ *memo.Memo, return nil, nil } - useGeneric := opc.p.SessionData().PlanCacheMode == sessiondatapb.PlanCacheModeForceGeneric - // If the statement was previously prepared, check for a reusable memo. // First check for a valid (non-stale) memo. - validMemo, err := opc.chooseValidPreparedMemo(ctx, prep.BaseMemo, prep.GenericMemo, useGeneric) + validMemo, err := opc.chooseValidPreparedMemo(ctx, prep.BaseMemo, prep.GenericMemo) if err != nil { return nil, err } @@ -643,17 +703,35 @@ func (opc *optPlanningCtx) fetchPreparedMemo(ctx context.Context) (_ *memo.Memo, // build a generic memo from it instead of building the memo from // scratch. opc.log(ctx, "rebuilding cached memo") - newMemo, generic, err := opc.buildReusableMemo(ctx, useGeneric) + buildGeneric := opc.useGenericPlan() + newMemo, typ, err := opc.buildReusableMemo(ctx, buildGeneric) if err != nil { return nil, err } - if generic { - // TODO(mgartner): Add the generic memo to the query cache so that it - // can be reused by future prepared statements. + switch typ { + case memoTypeIdealGeneric: + // If we have an "ideal" generic memo, store it as a base memo. It will + // always be used regardless of plan_cache_mode, so there is no need to + // set GenericCost. + prep.BaseMemo = newMemo + case memoTypeGeneric: prep.GenericMemo = newMemo - } else { + prep.Costs.SetGeneric(newMemo.RootExpr().(memo.RelExpr).Cost()) + // Now that the cost of the generic plan is known, we need to + // re-evaluate the decision to use a generic or custom plan. + if !opc.useGenericPlan() { + // The generic plan that we just built is too expensive, so we need + // to build a custom plan. We recursively call fetchPreparedMemo in + // case we have a custom plan that can be reused as a starting point + // for optimization. The function should not recurse more than once. + return opc.fetchPreparedMemo(ctx) + } + case memoTypeCustom: prep.BaseMemo = newMemo + default: + return nil, errors.AssertionFailedf("unexpected memo type %v", typ) } + // Re-optimize the memo, if necessary. return opc.reuseMemo(ctx, newMemo) } @@ -674,7 +752,7 @@ func (opc *optPlanningCtx) fetchPreparedMemoLegacy(ctx context.Context) (_ *memo return nil, err } else if isStale { opc.log(ctx, "rebuilding cached memo") - prepared.BaseMemo, _, err = opc.buildReusableMemo(ctx, false /* useGeneric */) + prepared.BaseMemo, _, err = opc.buildReusableMemo(ctx, false /* buildGeneric */) if err != nil { return nil, err } @@ -723,7 +801,7 @@ func (opc *optPlanningCtx) buildExecMemo(ctx context.Context) (_ *memo.Memo, _ e return nil, err } else if isStale { opc.log(ctx, "query cache hit but needed update") - cachedData.Memo, _, err = opc.buildReusableMemo(ctx, false /* allowGeneric */) + cachedData.Memo, _, err = opc.buildReusableMemo(ctx, false /* buildGeneric */) if err != nil { return nil, err } diff --git a/pkg/sql/prepared_stmt.go b/pkg/sql/prepared_stmt.go index 3754512b1641..edc1123bf75e 100644 --- a/pkg/sql/prepared_stmt.go +++ b/pkg/sql/prepared_stmt.go @@ -56,9 +56,16 @@ type PreparedStatement struct { querycache.PrepareMetadata // BaseMemo is the memoized data structure constructed by the cost-based - // optimizer during prepare of a SQL statement. It may be a fully-optimized - // memo if the prepared statement has no placeholders and no fold-able - // stable expressions. Otherwise, it is an unoptimized, normalized memo. + // optimizer during prepare of a SQL statement. + // + // It may be a fully-optimized memo if it contains an "ideal generic plan" + // that is guaranteed to be optimal across all executions of the prepared + // statement. Ideal generic plans are generated when the statement has no + // placeholders nor fold-able stable expressions, or when the placeholder + // fast-path is utilized. + // + // If it is not an ideal generic plan, it is an unoptimized, normalized + // memo that is used as a starting point for optimization of custom plans. BaseMemo *memo.Memo // GenericMemo, if present, is a fully-optimized memo that can be executed @@ -67,6 +74,9 @@ type PreparedStatement struct { // reduce confusion. GenericMemo *memo.Memo + // Costs tracks the costs of previously optimized custom and generic plans. + Costs planCosts + // refCount keeps track of the number of references to this PreparedStatement. // New references are registered through incRef(). // Once refCount hits 0 (through calls to decRef()), the following memAcc is @@ -117,6 +127,75 @@ func (p *PreparedStatement) incRef(ctx context.Context) { p.refCount++ } +const ( + // CustomPlanThreshold is the maximum number of custom plan costs tracked by + // planCosts. It is also the number of custom plans executed when + // plan_cache_mode=auto before attempting to generate a generic plan. + CustomPlanThreshold = 5 +) + +// planCosts tracks costs of generic and custom plans. +type planCosts struct { + generic memo.Cost + custom struct { + nextIdx int + length int + costs [CustomPlanThreshold]memo.Cost + } +} + +// Generic returns the cost of the generic plan. +func (p *planCosts) Generic() memo.Cost { + return p.generic +} + +// SetGeneric sets the cost of the generic plan. +func (p *planCosts) SetGeneric(cost memo.Cost) { + p.generic = cost +} + +// AddCustom adds a custom plan cost to the planCosts, evicting the oldest cost +// if necessary. +func (p *planCosts) AddCustom(cost memo.Cost) { + p.custom.costs[p.custom.nextIdx] = cost + p.custom.nextIdx++ + if p.custom.nextIdx >= CustomPlanThreshold { + p.custom.nextIdx = 0 + } + if p.custom.length < CustomPlanThreshold { + p.custom.length++ + } +} + +// NumCustom returns the number of custom plan costs in the planCosts. +func (p *planCosts) NumCustom() int { + return p.custom.length +} + +// AvgCustom returns the average cost of all the custom plan costs in planCosts. +// If there are no custom plan costs, it returns 0. +func (p *planCosts) AvgCustom() memo.Cost { + if p.custom.length == 0 { + return 0 + } + var sum memo.Cost + for i := 0; i < p.custom.length; i++ { + sum += p.custom.costs[i] + } + return sum / memo.Cost(p.custom.length) +} + +// ClearGeneric clears the generic cost. +func (p *planCosts) ClearGeneric() { + p.generic = 0 +} + +// ClearCustom clears any previously added custom costs. +func (p *planCosts) ClearCustom() { + p.custom.nextIdx = 0 + p.custom.length = 0 +} + // preparedStatementsAccessor gives a planner access to a session's collection // of prepared statements. type preparedStatementsAccessor interface { diff --git a/pkg/sql/prepared_stmt_test.go b/pkg/sql/prepared_stmt_test.go new file mode 100644 index 000000000000..f0293e1f29d3 --- /dev/null +++ b/pkg/sql/prepared_stmt_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package sql + +import ( + "testing" + + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" +) + +func TestPlanCosts(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + type testCase struct { + input []memo.Cost + expectedNum int + expectedAvg memo.Cost + } + testCases := []testCase{ + {input: []memo.Cost{}, expectedNum: 0, expectedAvg: 0}, + {input: []memo.Cost{0, 0}, expectedNum: 2, expectedAvg: 0}, + {input: []memo.Cost{1, 1}, expectedNum: 2, expectedAvg: 1}, + {input: []memo.Cost{1, 2, 3, 4, 5}, expectedNum: 5, expectedAvg: 3}, + {input: []memo.Cost{1, 2, 3, 4, 5, 6}, expectedNum: 5, expectedAvg: 4}, + {input: []memo.Cost{9, 9, 9, 9, 9, 1, 2, 3, 4, 5}, expectedNum: 5, expectedAvg: 3}, + } + var pc planCosts + for _, tc := range testCases { + pc.ClearCustom() + for _, cost := range tc.input { + pc.AddCustom(cost) + } + if pc.NumCustom() != tc.expectedNum { + t.Errorf("expected Len() to be %d, got %d", tc.expectedNum, pc.NumCustom()) + } + if pc.AvgCustom() != tc.expectedAvg { + t.Errorf("expected Avg() to be %f, got %f", tc.expectedAvg, pc.AvgCustom()) + } + } +} From 231c019bda4a9e782314d0d02586f2caeedf8a63 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 17 Jun 2024 16:52:20 -0400 Subject: [PATCH 11/15] opttester: allow no-stable-folds for opt directive Prior to this commit, the `no-stable-folds` directive only worked with the "norm", "exprnorm", and "expropt" directives. It now also works with the "opt" directive. This is required for testing generic query plans where plans must be fully optimized without folding stable expressions. Release note: None --- pkg/sql/opt/testutils/opttester/opt_tester.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/sql/opt/testutils/opttester/opt_tester.go b/pkg/sql/opt/testutils/opttester/opt_tester.go index 9fe6f2bb7a42..75dce5db6cc8 100644 --- a/pkg/sql/opt/testutils/opttester/opt_tester.go +++ b/pkg/sql/opt/testutils/opttester/opt_tester.go @@ -463,7 +463,7 @@ func New(catalog cat.Catalog, sql string) *OptTester { // modifies the existing set of the flags. // // - no-stable-folds: disallows constant folding for stable operators; only -// used with "norm". +// used with "norm", "opt", "exprnorm", and "expropt". // // - fully-qualify-names: fully qualify all column names in the test output. // @@ -1209,7 +1209,9 @@ func (ot *OptTester) OptimizeWithTables(tables map[cat.StableID]cat.Table) (opt. o.NotifyOnMatchedRule(func(ruleName opt.RuleName) bool { return !ot.Flags.DisableRules.Contains(int(ruleName)) }) - o.Factory().FoldingControl().AllowStableFolds() + if !ot.Flags.NoStableFolds { + o.Factory().FoldingControl().AllowStableFolds() + } return ot.optimizeExpr(o, tables) } From 8ab1cf741b65a1dcf2d580f8fae83f97d0dcbe45 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 17 Jun 2024 16:58:27 -0400 Subject: [PATCH 12/15] opt: optimize generic query plans with stable expressions The `ConvertSelectWithPlaceholdersToJoin` rule has been renamed to `GenerateParameterizedJoin` and modified to also operate on stable expressions. Stable expressions cannot be folded in generic query plans because their value can only be determined at execution time. By transforming a Select with a stable filter expression into a Join with a Values input, the optimizer can potentially plan a lookup join with similar performance characteristics to a constrained scan that would be planned if the stable expression could be folded. Release note: None --- pkg/sql/opt/exec/execbuilder/testdata/generic | 86 +++-- pkg/sql/opt/xform/BUILD.bazel | 1 + pkg/sql/opt/xform/generic_funcs.go | 145 ++++---- pkg/sql/opt/xform/rules/generic.opt | 40 ++- pkg/sql/opt/xform/testdata/rules/generic | 313 +++++++++++++++++- 5 files changed, 447 insertions(+), 138 deletions(-) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/generic b/pkg/sql/opt/exec/execbuilder/testdata/generic index 1358b39518c0..239f19396a55 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/generic +++ b/pkg/sql/opt/exec/execbuilder/testdata/generic @@ -508,25 +508,38 @@ isolation level: serializable priority: normal quality of service: regular · -• filter +• lookup join │ nodes: │ regions: │ actual row count: 0 -│ filter: t = now() +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key │ -└── • scan - nodes: - regions: - actual row count: 0 - KV time: 0µs - KV contention time: 0µs - KV rows decoded: 0 - KV bytes read: 0 B - KV gRPC calls: 0 - estimated max memory allocated: 0 B - missing stats - table: t@t_pkey - spans: FULL SCAN +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_t_idx + │ equality: (column9) = (t) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 1 column, 1 row # The generic plan can be reused. query T @@ -544,25 +557,38 @@ isolation level: serializable priority: normal quality of service: regular · -• filter +• lookup join │ nodes: │ regions: │ actual row count: 0 -│ filter: t = now() +│ KV time: 0µs +│ KV contention time: 0µs +│ KV rows decoded: 0 +│ KV bytes read: 0 B +│ KV gRPC calls: 0 +│ estimated max memory allocated: 0 B +│ table: t@t_pkey +│ equality: (k) = (k) +│ equality cols are key │ -└── • scan - nodes: - regions: - actual row count: 0 - KV time: 0µs - KV contention time: 0µs - KV rows decoded: 0 - KV bytes read: 0 B - KV gRPC calls: 0 - estimated max memory allocated: 0 B - missing stats - table: t@t_pkey - spans: FULL SCAN +└── • lookup join + │ nodes: + │ regions: + │ actual row count: 0 + │ KV time: 0µs + │ KV contention time: 0µs + │ KV rows decoded: 0 + │ KV bytes read: 0 B + │ KV gRPC calls: 0 + │ estimated max memory allocated: 0 B + │ table: t@t_t_idx + │ equality: (column9) = (t) + │ + └── • values + nodes: + regions: + actual row count: 1 + size: 1 column, 1 row statement ok DEALLOCATE p diff --git a/pkg/sql/opt/xform/BUILD.bazel b/pkg/sql/opt/xform/BUILD.bazel index c1a8d266152a..0555dedc8f22 100644 --- a/pkg/sql/opt/xform/BUILD.bazel +++ b/pkg/sql/opt/xform/BUILD.bazel @@ -49,6 +49,7 @@ go_library( "//pkg/sql/rowinfra", "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", + "//pkg/sql/sem/volatility", "//pkg/sql/types", "//pkg/util/buildutil", "//pkg/util/cancelchecker", diff --git a/pkg/sql/opt/xform/generic_funcs.go b/pkg/sql/opt/xform/generic_funcs.go index a6e2795029a5..362e7d34ef78 100644 --- a/pkg/sql/opt/xform/generic_funcs.go +++ b/pkg/sql/opt/xform/generic_funcs.go @@ -16,74 +16,94 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" "github.com/cockroachdb/cockroach/pkg/sql/types" - "github.com/cockroachdb/cockroach/pkg/util/intsets" - "github.com/cockroachdb/errors" ) -// HasPlaceholders returns true if the given relational expression's subtree has +// HasPlaceholdersOrStableExprs returns true if the given relational expression's subtree has // at least one placeholder. -func (c *CustomFuncs) HasPlaceholders(e memo.RelExpr) bool { - return e.Relational().HasPlaceholder +func (c *CustomFuncs) HasPlaceholdersOrStableExprs(e memo.RelExpr) bool { + return e.Relational().HasPlaceholder || e.Relational().VolatilitySet.HasStable() } -// GeneratePlaceholderValuesAndJoinFilters returns a single-row Values -// expression containing placeholders in the given filters. It also returns a -// new set of filters where the placeholders have been replaced with variables -// referencing the columns produced by the returned Values expression. If the -// given filters have no placeholders, ok=false is returned. -func (c *CustomFuncs) GeneratePlaceholderValuesAndJoinFilters( +// GenerateParameterizedJoinValuesAndFilters returns a single-row Values +// expression containing placeholders and stable expressions in the given +// filters. It also returns a new set of filters where the placeholders and +// stable expressions have been replaced with variables referencing the columns +// produced by the returned Values expression. If the given filters have no +// placeholders or stable expressions, ok=false is returned. +func (c *CustomFuncs) GenerateParameterizedJoinValuesAndFilters( filters memo.FiltersExpr, ) (values memo.RelExpr, newFilters memo.FiltersExpr, ok bool) { - // Collect all the placeholders in the filters. - // - // collectPlaceholders recursively walks the scalar expression and collects - // placeholder expressions into the placeholders slice. - var placeholders []*memo.PlaceholderExpr - var seenIndexes intsets.Fast - var collectPlaceholders func(e opt.Expr) - collectPlaceholders = func(e opt.Expr) { - if p, ok := e.(*memo.PlaceholderExpr); ok { - idx := int(p.Value.(*tree.Placeholder).Idx) - // Don't include the same placeholder multiple times. - if !seenIndexes.Contains(idx) { - seenIndexes.Add(idx) - placeholders = append(placeholders, p) + var exprs memo.ScalarListExpr + var cols opt.ColList + placeholderCols := make(map[tree.PlaceholderIdx]opt.ColumnID) + + // replace recursively walks the expression tree and replaces placeholders + // and stable expressions. It collects the replaced expressions and creates + // columns representing those expressions. Those expressions and columns + // will be used in the Values expression created below. + var replace func(e opt.Expr) opt.Expr + replace = func(e opt.Expr) opt.Expr { + switch t := e.(type) { + case *memo.PlaceholderExpr: + idx := t.Value.(*tree.Placeholder).Idx + // Reuse the same column for duplicate placeholder references. + if col, ok := placeholderCols[idx]; ok { + return c.e.f.ConstructVariable(col) + } + col := c.e.f.Metadata().AddColumn(fmt.Sprintf("$%d", idx+1), t.DataType()) + placeholderCols[idx] = col + exprs = append(exprs, t) + cols = append(cols, col) + return c.e.f.ConstructVariable(col) + + case *memo.FunctionExpr: + // TODO(mgartner): Consider including other expressions that could + // be stable: casts, assignment casts, UDFCallExprs, unary ops, + // comparisons, binary ops. + // TODO(mgartner): Include functions with arguments if they are all + // constants or placeholders. + if t.Overload.Volatility == volatility.Stable && len(t.Args) == 0 { + col := c.e.f.Metadata().AddColumn("", t.DataType()) + exprs = append(exprs, t) + cols = append(cols, col) + return c.e.f.ConstructVariable(col) } - return - } - for i, n := 0, e.ChildCount(); i < n; i++ { - collectPlaceholders(e.Child(i)) } + + return c.e.f.Replace(e, replace) } + // Replace placeholders and stable expressions in each filter. for i := range filters { - // Only traverse the scalar expression if it contains a placeholder. - if filters[i].ScalarProps().HasPlaceholder { - collectPlaceholders(filters[i].Condition) + cond := filters[i].Condition + if newCond := replace(cond).(opt.ScalarExpr); newCond != cond { + if newFilters == nil { + // Lazily allocate newFilters. + newFilters = make(memo.FiltersExpr, len(filters)) + copy(newFilters, filters[:i]) + } + // Construct a new filter if placeholders were replaced. + newFilters[i] = c.e.f.ConstructFiltersItem(newCond) + } else if newFilters != nil { + // Otherwise copy the filter if newFilters has been allocated. + newFilters[i] = filters[i] } } - // If there are no placeholders in the filters, there is nothing to do. - if len(placeholders) == 0 { + // If no placeholders or stable expressions were replaced, there is nothing + // to do. + if len(exprs) == 0 { return nil, nil, false } // Create the Values expression with one row and one column for each - // placeholder. - cols := make(opt.ColList, len(placeholders)) - colIDs := make(map[tree.PlaceholderIdx]opt.ColumnID, len(placeholders)) - typs := make([]*types.T, len(placeholders)) - exprs := make(memo.ScalarListExpr, len(placeholders)) - for i, p := range placeholders { - idx := p.Value.(*tree.Placeholder).Idx - col := c.e.f.Metadata().AddColumn(fmt.Sprintf("$%d", idx+1), p.DataType()) - cols[i] = col - colIDs[idx] = col - exprs[i] = p - typs[i] = p.DataType() + // replaced expression. + typs := make([]*types.T, len(exprs)) + for i, e := range exprs { + typs[i] = e.DataType() } - tupleTyp := types.MakeTuple(typs) rows := memo.ScalarListExpr{c.e.f.ConstructTuple(exprs, tupleTyp)} values = c.e.f.ConstructValues(rows, &memo.ValuesPrivate{ @@ -91,39 +111,12 @@ func (c *CustomFuncs) GeneratePlaceholderValuesAndJoinFilters( ID: c.e.f.Metadata().NextUniqueID(), }) - // Create new filters by replacing the placeholders in the filters with - // variables. - var replace func(e opt.Expr) opt.Expr - replace = func(e opt.Expr) opt.Expr { - if p, ok := e.(*memo.PlaceholderExpr); ok { - idx := p.Value.(*tree.Placeholder).Idx - col, ok := colIDs[idx] - if !ok { - panic(errors.AssertionFailedf("unknown placeholder %d", idx)) - } - return c.e.f.ConstructVariable(col) - } - return c.e.f.Replace(e, replace) - } - - newFilters = make(memo.FiltersExpr, len(filters)) - for i := range newFilters { - cond := filters[i].Condition - if newCond := replace(cond).(opt.ScalarExpr); newCond != cond { - // Construct a new filter if placeholders were replaced. - newFilters[i] = c.e.f.ConstructFiltersItem(newCond) - } else { - // Otherwise copy the filter. - newFilters[i] = filters[i] - } - } - return values, newFilters, true } -// GenericJoinPrivate returns JoinPrivate that disabled join reordering and +// ParameterizedJoinPrivate returns JoinPrivate that disabled join reordering and // merge join exploration. -func (c *CustomFuncs) GenericJoinPrivate() *memo.JoinPrivate { +func (c *CustomFuncs) ParameterizedJoinPrivate() *memo.JoinPrivate { return &memo.JoinPrivate{ Flags: memo.DisallowMergeJoin, SkipReorderJoins: true, diff --git a/pkg/sql/opt/xform/rules/generic.opt b/pkg/sql/opt/xform/rules/generic.opt index 3816a14be24b..c49631511e65 100644 --- a/pkg/sql/opt/xform/rules/generic.opt +++ b/pkg/sql/opt/xform/rules/generic.opt @@ -2,47 +2,55 @@ # generic.opt contains exploration rules for optimizing generic query plans. # ============================================================================= -# ConvertSelectWithPlaceholdersToJoin is an exploration rule that converts a -# Select expression with placeholders in the filters into an InnerJoin that -# joins the Select's input with a Values expression that produces the -# placeholder values. +# GenerateParameterizedJoin is an exploration rule that converts a Select +# expression with placeholders and stable expression in the filters into an +# InnerJoin that joins the Select's input with a Values expression that produces +# the placeholder values and stable expressions. # # This rule allows generic query plans, in which placeholder values are not -# known, to be optimized. By converting the Select into an InnerJoin, the -# optimizer can plan a lookup join, in many cases, which has similar performance -# characteristics to the constrained Scan that would be planned if the -# placeholder values were known. For example, consider a schema and query like: +# known and stable expressions are not folded, to be optimized. By converting +# the Select into an InnerJoin, the optimizer can, in many cases, plan a lookup +# join which has similar performance characteristics to the constrained Scan +# that would be planned if the placeholder values were known. +# +# For example, consider a schema and query like: # # CREATE TABLE t (i INT PRIMARY KEY) # SELECT * FROM t WHERE i = $1 # -# ConvertSelectWithPlaceholdersToJoin will perform the first conversion below, -# from a Select into a Join. GenerateLookupJoins will perform the second -# conversion from a (hash) Join into a LookupJoin. -# +# GenerateParameterizedJoin will perform the first transformation below, from a +# Select into a Join. GenerateLookupJoins will perform the second transformation +# from a (hash) Join into a LookupJoin. # # Select (i=$1) Join (i=col_$1) LookupJoin (t@t_pkey) # | -> / \ -> | # | / \ | # Scan t Values ($1) Scan t Values ($1) # -[ConvertSelectWithPlaceholdersToJoin, Explore] +[GenerateParameterizedJoin, Explore] (Select $scan:(Scan $scanPrivate:*) & (IsCanonicalScan $scanPrivate) $filters:* & - (HasPlaceholders (Root)) & + (HasPlaceholdersOrStableExprs (Root)) & (Let ( $values $newFilters $ok - ):(GeneratePlaceholderValuesAndJoinFilters $filters) + ):(GenerateParameterizedJoinValuesAndFilters + $filters + ) $ok ) ) => (Project - (InnerJoin $values $scan $newFilters (GenericJoinPrivate)) + (InnerJoin + $values + $scan + $newFilters + (ParameterizedJoinPrivate) + ) [] (OutputCols (Root)) ) diff --git a/pkg/sql/opt/xform/testdata/rules/generic b/pkg/sql/opt/xform/testdata/rules/generic index dbd694762060..93df29d979a5 100644 --- a/pkg/sql/opt/xform/testdata/rules/generic +++ b/pkg/sql/opt/xform/testdata/rules/generic @@ -4,16 +4,18 @@ CREATE TABLE t ( i INT, s STRING, b BOOL, - t TIMESTAMP, - INDEX (i, s, b) + t TIMESTAMPTZ, + INDEX (i, s, b), + INDEX (i, t), + INDEX (t) ) ---- # -------------------------------------------------- -# ConvertSelectWithPlaceholdersToJoin +# GenerateParameterizedJoin # -------------------------------------------------- -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = $1 ---- project @@ -40,7 +42,7 @@ project │ └── ($1,) └── filters (true) -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = $1::INT ---- project @@ -67,7 +69,7 @@ project │ └── ($1,) └── filters (true) -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND s = $2 AND b = $3 ---- project @@ -101,7 +103,7 @@ project # A placeholder referenced multiple times in the filters should only appear once # in the Values expression. -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = $1 AND i = $1 ---- project @@ -131,7 +133,7 @@ project # The generated join should not be reordered and merge joins should not be # explored on it. -opt expect=ConvertSelectWithPlaceholdersToJoin expect-not=(ReorderJoins,GenerateMergeJoins) +opt expect=GenerateParameterizedJoin expect-not=(ReorderJoins,GenerateMergeJoins) SELECT * FROM t WHERE i = $1 ---- project @@ -146,13 +148,13 @@ project ├── has-placeholder ├── key: (1) ├── fd: ()-->(2,8), (1)-->(3-5), (2)==(8), (8)==(2) - ├── inner-join (lookup t@t_i_s_b_idx) - │ ├── columns: k:1!null i:2!null s:3 b:4 "$1":8!null + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5 "$1":8!null │ ├── flags: disallow merge join │ ├── key columns: [8] = [2] │ ├── has-placeholder │ ├── key: (1) - │ ├── fd: ()-->(2,8), (1)-->(3,4), (2)==(8), (8)==(2) + │ ├── fd: ()-->(2,8), (1)-->(5), (2)==(8), (8)==(2) │ ├── values │ │ ├── columns: "$1":8 │ │ ├── cardinality: [1 - 1] @@ -163,7 +165,7 @@ project │ └── filters (true) └── filters (true) -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = (SELECT i FROM t WHERE k = $1) ---- project @@ -240,7 +242,7 @@ exec-ddl CREATE INDEX partial_idx ON t(t) WHERE t IS NOT NULL ---- -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE t = $1 ---- project @@ -280,7 +282,7 @@ exec-ddl CREATE INDEX partial_idx ON t(i, t) WHERE i IS NOT NULL AND t IS NOT NULL ---- -opt expect=ConvertSelectWithPlaceholdersToJoin +opt expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND t = $2 ---- project @@ -360,8 +362,287 @@ exec-ddl DROP INDEX partial_idx ---- -# The rule does not match if there are no placeholders in the filters. -opt expect-not=ConvertSelectWithPlaceholdersToJoin +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE t = now() +---- +project + ├── columns: k:1!null i:2 s:3 b:4 t:5!null + ├── stable + ├── key: (1) + ├── fd: ()-->(5), (1)-->(2-4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2 s:3 b:4 t:5!null column8:8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable + ├── key: (1) + ├── fd: ()-->(5,8), (1)-->(2-4), (5)==(8), (8)==(5) + ├── inner-join (lookup t@t_t_idx) + │ ├── columns: k:1!null t:5!null column8:8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [5] + │ ├── stable + │ ├── key: (1) + │ ├── fd: ()-->(5,8), (5)==(8), (8)==(5) + │ ├── values + │ │ ├── columns: column8:8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── (now(),) + │ └── filters (true) + └── filters (true) + +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = now() +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8,9), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 9] = [2 5] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8,9), (2)==(8), (8)==(2), (5)==(9), (9)==(5) + │ ├── values + │ │ ├── columns: "$1":8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, now()) + │ └── filters (true) + └── filters (true) + +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t > now() +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2), (1)-->(3-5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,8,9), (1)-->(3-5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9!null + │ ├── flags: disallow merge join + │ ├── lookup expression + │ │ └── filters + │ │ ├── "$1":8 = i:2 [outer=(2,8), constraints=(/2: (/NULL - ]; /8: (/NULL - ]), fd=(2)==(8), (8)==(2)] + │ │ └── t:5 > column9:9 [outer=(5,9), constraints=(/5: (/NULL - ]; /9: (/NULL - ])] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,8,9), (1)-->(5), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8,9) + │ │ └── ($1, now()) + │ └── filters (true) + └── filters (true) + +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = now() + $2 +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 "$2":10 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8-10), (1)-->(3,4), (2)==(8), (8)==(2) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 "$2":10 column11:11!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8-11), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(11), (11)==(5) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9 "$2":10 column11:11!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 11] = [2 5] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8-11), (2)==(8), (8)==(2), (5)==(11), (11)==(5) + │ ├── project + │ │ ├── columns: column11:11 "$1":8 column9:9 "$2":10 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8-11) + │ │ ├── values + │ │ │ ├── columns: "$1":8 column9:9 "$2":10 + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── stable, has-placeholder + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(8-10) + │ │ │ └── ($1, now(), $2) + │ │ └── projections + │ │ └── column9:9 + "$2":10 [as=column11:11, outer=(9,10), stable] + │ └── filters (true) + └── filters (true) + +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND t = now() + '1 hr'::INTERVAL +---- +project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5), (1)-->(3,4) + └── project + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8,9), (1)-->(3,4), (2)==(8), (8)==(2) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3 b:4 t:5!null "$1":8!null column9:9 column10:10!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,5,8-10), (1)-->(3,4), (2)==(8), (8)==(2), (5)==(10), (10)==(5) + ├── inner-join (lookup t@t_i_t_idx) + │ ├── columns: k:1!null i:2!null t:5!null "$1":8!null column9:9 column10:10!null + │ ├── flags: disallow merge join + │ ├── key columns: [8 10] = [2 5] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,5,8-10), (2)==(8), (8)==(2), (5)==(10), (10)==(5) + │ ├── project + │ │ ├── columns: column10:10 "$1":8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── stable, has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8-10) + │ │ ├── values + │ │ │ ├── columns: "$1":8 column9:9 + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── stable, has-placeholder + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(8,9) + │ │ │ └── ($1, now()) + │ │ └── projections + │ │ └── column9:9 + '01:00:00' [as=column10:10, outer=(9), stable] + │ └── filters (true) + └── filters (true) + +# TODO(mgartner): Apply the rule to stable, non-leaf expressions. +opt no-stable-folds +SELECT * FROM t WHERE t = '2024-01-01 12:00:00'::TIMESTAMP::TIMESTAMPTZ +---- +select + ├── columns: k:1!null i:2 s:3 b:4 t:5!null + ├── stable + ├── key: (1) + ├── fd: ()-->(5), (1)-->(2-4) + ├── scan t + │ ├── columns: k:1!null i:2 s:3 b:4 t:5 + │ ├── key: (1) + │ └── fd: (1)-->(2-5) + └── filters + └── t:5 = '2024-01-01 12:00:00'::TIMESTAMPTZ [outer=(5), stable, constraints=(/5: (/NULL - ]), fd=()-->(5)] + +# A stable function is not included in the Values expression if it has +# arguments. +# TODO(mgartner): We should be able to relax this restriction as long as all the +# arguments are constants or placeholders. +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND s = quote_literal(1::INT) +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3), (1)-->(4,5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3,8), (1)-->(4,5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3!null b:4 "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [2] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,3,8), (1)-->(4), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters + │ └── s:3 = quote_literal(1) [outer=(3), stable, constraints=(/3: (/NULL - ]), fd=()-->(3)] + └── filters (true) + +# A stable function is not included in the Values expression if its arguments +# reference a column from the table. This would create an illegal outer column +# reference in a non-apply-join. +opt no-stable-folds expect=GenerateParameterizedJoin +SELECT * FROM t WHERE i = $1 AND s = quote_literal(i) +---- +project + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3), (1)-->(4,5) + └── inner-join (lookup t) + ├── columns: k:1!null i:2!null s:3!null b:4 t:5 "$1":8!null + ├── key columns: [1] = [1] + ├── lookup columns are key + ├── stable, has-placeholder + ├── key: (1) + ├── fd: ()-->(2,3,8), (1)-->(4,5), (2)==(8), (8)==(2) + ├── inner-join (lookup t@t_i_s_b_idx) + │ ├── columns: k:1!null i:2!null s:3!null b:4 "$1":8!null + │ ├── flags: disallow merge join + │ ├── key columns: [8] = [2] + │ ├── stable, has-placeholder + │ ├── key: (1) + │ ├── fd: ()-->(2,3,8), (1)-->(4), (2)==(8), (8)==(2) + │ ├── values + │ │ ├── columns: "$1":8 + │ │ ├── cardinality: [1 - 1] + │ │ ├── has-placeholder + │ │ ├── key: () + │ │ ├── fd: ()-->(8) + │ │ └── ($1,) + │ └── filters + │ └── s:3 = quote_literal(i:2) [outer=(2,3), stable, constraints=(/3: (/NULL - ]), fd=(2)-->(3)] + └── filters (true) + +# The rule does not match if there are no placeholders or stable expressions in +# the filters. +opt expect-not=GenerateParameterizedJoin SELECT * FROM t WHERE i = 1 AND s = 'foo' ---- index-join t From b85764ce4abdb9f8d6d3bfe7d5d2e465f770df81 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 15 Jul 2024 17:28:21 -0700 Subject: [PATCH 13/15] sql: require an enterprise license to use generic query plans Release note: None --- .github/CODEOWNERS | 1 + pkg/BUILD.bazel | 2 ++ pkg/ccl/BUILD.bazel | 1 + pkg/ccl/ccl_init.go | 1 + pkg/ccl/gqpccl/BUILD.bazel | 14 +++++++++++ pkg/ccl/gqpccl/gpq.go | 24 +++++++++++++++++++ .../logictestccl/testdata/logic_test}/generic | 0 pkg/ccl/logictestccl/tests/local/BUILD.bazel | 2 +- .../tests/local/generated_test.go | 7 ++++++ pkg/sql/BUILD.bazel | 1 + pkg/sql/gpq/BUILD.bazel | 14 +++++++++++ pkg/sql/gpq/gpq.go | 24 +++++++++++++++++++ .../testdata/logic_test/generic_license | 7 ++++++ .../logictest/tests/local/generated_test.go | 7 ++++++ .../execbuilder/tests/local/generated_test.go | 7 ------ .../comparator_generated_test.go | 5 ++++ pkg/sql/vars.go | 16 ++++++++++--- 17 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 pkg/ccl/gqpccl/BUILD.bazel create mode 100644 pkg/ccl/gqpccl/gpq.go rename pkg/{sql/opt/exec/execbuilder/testdata => ccl/logictestccl/testdata/logic_test}/generic (100%) create mode 100644 pkg/sql/gpq/BUILD.bazel create mode 100644 pkg/sql/gpq/gpq.go create mode 100644 pkg/sql/logictest/testdata/logic_test/generic_license diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1caffd486013..2f32f81bafec 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -371,6 +371,7 @@ #!/pkg/ccl/cliccl/ @cockroachdb/unowned /pkg/ccl/cmdccl/stub-schema-registry/ @cockroachdb/cdc-prs /pkg/ccl/cmdccl/clusterrepl/ @cockroachdb/disaster-recovery +/pkg/ccl/gqpccl/ @cockroachdb/sql-queries-prs #!/pkg/ccl/gssapiccl/ @cockroachdb/unowned /pkg/ccl/jwtauthccl/ @cockroachdb/cloud-identity #!/pkg/ccl/kvccl/ @cockroachdb/kv-noreview diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 128548c40765..1f30790af929 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -841,6 +841,7 @@ GO_TARGETS = [ "//pkg/ccl/cmdccl/clusterrepl:clusterrepl_lib", "//pkg/ccl/cmdccl/stub-schema-registry:stub-schema-registry", "//pkg/ccl/cmdccl/stub-schema-registry:stub-schema-registry_lib", + "//pkg/ccl/gqpccl:gqpccl", "//pkg/ccl/gssapiccl:gssapiccl", "//pkg/ccl/importerccl:importerccl_test", "//pkg/ccl/jobsccl/jobsprotectedtsccl:jobsprotectedtsccl_test", @@ -1836,6 +1837,7 @@ GO_TARGETS = [ "//pkg/sql/gcjob:gcjob", "//pkg/sql/gcjob:gcjob_test", "//pkg/sql/gcjob_test:gcjob_test_test", + "//pkg/sql/gpq:gpq", "//pkg/sql/idxrecommendations:idxrecommendations", "//pkg/sql/idxrecommendations:idxrecommendations_test", "//pkg/sql/idxusage:idxusage", diff --git a/pkg/ccl/BUILD.bazel b/pkg/ccl/BUILD.bazel index 494a6fd5eef6..180b79a553db 100644 --- a/pkg/ccl/BUILD.bazel +++ b/pkg/ccl/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//pkg/ccl/buildccl", "//pkg/ccl/changefeedccl", "//pkg/ccl/cliccl", + "//pkg/ccl/gqpccl", "//pkg/ccl/gssapiccl", "//pkg/ccl/jwtauthccl", "//pkg/ccl/kvccl", diff --git a/pkg/ccl/ccl_init.go b/pkg/ccl/ccl_init.go index fc315a0a087f..d14239571ddc 100644 --- a/pkg/ccl/ccl_init.go +++ b/pkg/ccl/ccl_init.go @@ -18,6 +18,7 @@ import ( _ "github.com/cockroachdb/cockroach/pkg/ccl/buildccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/cliccl" + _ "github.com/cockroachdb/cockroach/pkg/ccl/gqpccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/gssapiccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl" _ "github.com/cockroachdb/cockroach/pkg/ccl/kvccl" diff --git a/pkg/ccl/gqpccl/BUILD.bazel b/pkg/ccl/gqpccl/BUILD.bazel new file mode 100644 index 000000000000..d4723e260a2b --- /dev/null +++ b/pkg/ccl/gqpccl/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "gqpccl", + srcs = ["gpq.go"], + importpath = "github.com/cockroachdb/cockroach/pkg/ccl/gqpccl", + visibility = ["//visibility:public"], + deps = [ + "//pkg/ccl/utilccl", + "//pkg/settings/cluster", + "//pkg/sql/gpq", + "//pkg/util/uuid", + ], +) diff --git a/pkg/ccl/gqpccl/gpq.go b/pkg/ccl/gqpccl/gpq.go new file mode 100644 index 000000000000..05b6e3b8e633 --- /dev/null +++ b/pkg/ccl/gqpccl/gpq.go @@ -0,0 +1,24 @@ +// Copyright 2024 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package gpqccl + +import ( + "github.com/cockroachdb/cockroach/pkg/ccl/utilccl" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/gpq" + "github.com/cockroachdb/cockroach/pkg/util/uuid" +) + +func init() { + gpq.CheckClusterSupportsGenericQueryPlans = checkClusterSupportsGenericQueryPlans +} + +func checkClusterSupportsGenericQueryPlans(settings *cluster.Settings, cluster uuid.UUID) error { + return utilccl.CheckEnterpriseEnabled(settings, cluster, "generic query plans") +} diff --git a/pkg/sql/opt/exec/execbuilder/testdata/generic b/pkg/ccl/logictestccl/testdata/logic_test/generic similarity index 100% rename from pkg/sql/opt/exec/execbuilder/testdata/generic rename to pkg/ccl/logictestccl/testdata/logic_test/generic diff --git a/pkg/ccl/logictestccl/tests/local/BUILD.bazel b/pkg/ccl/logictestccl/tests/local/BUILD.bazel index 456f817c845b..905ee5708707 100644 --- a/pkg/ccl/logictestccl/tests/local/BUILD.bazel +++ b/pkg/ccl/logictestccl/tests/local/BUILD.bazel @@ -13,7 +13,7 @@ go_test( "//pkg/ccl/logictestccl:testdata", # keep ], exec_properties = {"Pool": "large"}, - shard_count = 31, + shard_count = 32, tags = [ "ccl_test", "cpu:1", diff --git a/pkg/ccl/logictestccl/tests/local/generated_test.go b/pkg/ccl/logictestccl/tests/local/generated_test.go index 702d7e66af78..b50124dc8610 100644 --- a/pkg/ccl/logictestccl/tests/local/generated_test.go +++ b/pkg/ccl/logictestccl/tests/local/generated_test.go @@ -134,6 +134,13 @@ func TestCCLLogic_fips_ready( runCCLLogicTest(t, "fips_ready") } +func TestCCLLogic_generic( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runCCLLogicTest(t, "generic") +} + func TestCCLLogic_new_schema_changer( t *testing.T, ) { diff --git a/pkg/sql/BUILD.bazel b/pkg/sql/BUILD.bazel index 2cc6d505adf6..827235bdfab5 100644 --- a/pkg/sql/BUILD.bazel +++ b/pkg/sql/BUILD.bazel @@ -427,6 +427,7 @@ go_library( "//pkg/sql/faketreeeval", "//pkg/sql/flowinfra", "//pkg/sql/gcjob/gcjobnotifier", + "//pkg/sql/gpq", "//pkg/sql/idxrecommendations", "//pkg/sql/idxusage", "//pkg/sql/inverted", diff --git a/pkg/sql/gpq/BUILD.bazel b/pkg/sql/gpq/BUILD.bazel new file mode 100644 index 000000000000..0cae8159f9b5 --- /dev/null +++ b/pkg/sql/gpq/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "gpq", + srcs = ["gpq.go"], + importpath = "github.com/cockroachdb/cockroach/pkg/sql/gpq", + visibility = ["//visibility:public"], + deps = [ + "//pkg/settings/cluster", + "//pkg/sql/sqlerrors", + "//pkg/util/uuid", + "@com_github_cockroachdb_errors//:errors", + ], +) diff --git a/pkg/sql/gpq/gpq.go b/pkg/sql/gpq/gpq.go new file mode 100644 index 000000000000..dac94c9b68ca --- /dev/null +++ b/pkg/sql/gpq/gpq.go @@ -0,0 +1,24 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package gpq + +import ( + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/sqlerrors" + "github.com/cockroachdb/cockroach/pkg/util/uuid" + "github.com/cockroachdb/errors" +) + +var CheckClusterSupportsGenericQueryPlans = func(settings *cluster.Settings, cluster uuid.UUID) error { + return sqlerrors.NewCCLRequiredError( + errors.New("plan_cache_mode=force_generic_plan and plan_cache_mode=auto require a CCL binary"), + ) +} diff --git a/pkg/sql/logictest/testdata/logic_test/generic_license b/pkg/sql/logictest/testdata/logic_test/generic_license new file mode 100644 index 000000000000..d86c4f353038 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/generic_license @@ -0,0 +1,7 @@ +# LogicTest: local + +statement error pgcode XXC01 pq: plan_cache_mode=force_generic_plan and plan_cache_mode=auto require a CCL binary +SET plan_cache_mode = force_generic_plan + +statement error pgcode XXC01 pq: plan_cache_mode=force_generic_plan and plan_cache_mode=auto require a CCL binary +SET plan_cache_mode = auto diff --git a/pkg/sql/logictest/tests/local/generated_test.go b/pkg/sql/logictest/tests/local/generated_test.go index 3100cc9efa8e..c85ba3356cba 100644 --- a/pkg/sql/logictest/tests/local/generated_test.go +++ b/pkg/sql/logictest/tests/local/generated_test.go @@ -904,6 +904,13 @@ func TestLogic_generator_probe_ranges( runLogicTest(t, "generator_probe_ranges") } +func TestLogic_generic_license( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runLogicTest(t, "generic_license") +} + func TestLogic_geospatial( t *testing.T, ) { diff --git a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go index 7dc02ad0b4f2..834d6eee8359 100644 --- a/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go +++ b/pkg/sql/opt/exec/execbuilder/tests/local/generated_test.go @@ -258,13 +258,6 @@ func TestExecBuild_forecast1401( runExecBuildLogicTest(t, "forecast1401") } -func TestExecBuild_generic( - t *testing.T, -) { - defer leaktest.AfterTest(t)() - runExecBuildLogicTest(t, "generic") -} - func TestExecBuild_geospatial( t *testing.T, ) { diff --git a/pkg/sql/schemachanger/comparator_generated_test.go b/pkg/sql/schemachanger/comparator_generated_test.go index 8eb1d7d25e0f..66f7ce202cea 100644 --- a/pkg/sql/schemachanger/comparator_generated_test.go +++ b/pkg/sql/schemachanger/comparator_generated_test.go @@ -688,6 +688,11 @@ func TestSchemaChangeComparator_generator_probe_ranges(t *testing.T) { var logicTestFile = "pkg/sql/logictest/testdata/logic_test/generator_probe_ranges" runSchemaChangeComparatorTest(t, logicTestFile) } +func TestSchemaChangeComparator_generic_license(t *testing.T) { + defer leaktest.AfterTest(t)() + var logicTestFile = "pkg/sql/logictest/testdata/logic_test/generic_license" + runSchemaChangeComparatorTest(t, logicTestFile) +} func TestSchemaChangeComparator_geospatial(t *testing.T) { defer leaktest.AfterTest(t)() var logicTestFile = "pkg/sql/logictest/testdata/logic_test/geospatial" diff --git a/pkg/sql/vars.go b/pkg/sql/vars.go index f63806d0094c..bca0e31955e6 100644 --- a/pkg/sql/vars.go +++ b/pkg/sql/vars.go @@ -31,6 +31,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/colfetcher" "github.com/cockroachdb/cockroach/pkg/sql/delegate" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" + "github.com/cockroachdb/cockroach/pkg/sql/gpq" "github.com/cockroachdb/cockroach/pkg/sql/lex" "github.com/cockroachdb/cockroach/pkg/sql/paramparse" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" @@ -3352,7 +3353,7 @@ var varGen = map[string]sessionVar{ }, `plan_cache_mode`: { - Set: func(_ context.Context, m sessionDataMutator, s string) error { + SetWithPlanner: func(ctx context.Context, p *planner, local bool, s string) error { mode, ok := sessiondatapb.PlanCacheModeFromString(s) if !ok { return newVarValueError( @@ -3363,8 +3364,17 @@ var varGen = map[string]sessionVar{ sessiondatapb.PlanCacheModeAuto.String(), ) } - m.SetPlanCacheMode(mode) - return nil + if mode == sessiondatapb.PlanCacheModeForceGeneric || + mode == sessiondatapb.PlanCacheModeAuto { + evalCtx := p.EvalContext() + if err := gpq.CheckClusterSupportsGenericQueryPlans(evalCtx.Settings, evalCtx.ClusterID); err != nil { + return err + } + } + return p.applyOnSessionDataMutators(ctx, local, func(m sessionDataMutator) error { + m.SetPlanCacheMode(mode) + return nil + }) }, Get: func(evalCtx *extendedEvalContext, _ *kv.Txn) (string, error) { return evalCtx.SessionData().PlanCacheMode.String(), nil From d14710ac56bfcc5672eaa264203cd860128b3d8b Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 30 Jul 2024 12:52:44 -0400 Subject: [PATCH 14/15] sql: add telemetry counters for generic query plans Fixes #127651 Release note: None --- .../telemetryccl/testdata/telemetry/generic | 134 ++++++++++++++++++ pkg/sql/plan_opt.go | 19 +++ pkg/sql/sqltelemetry/planning.go | 16 +++ 3 files changed, 169 insertions(+) create mode 100644 pkg/ccl/telemetryccl/testdata/telemetry/generic diff --git a/pkg/ccl/telemetryccl/testdata/telemetry/generic b/pkg/ccl/telemetryccl/testdata/telemetry/generic new file mode 100644 index 000000000000..cdf25c994b70 --- /dev/null +++ b/pkg/ccl/telemetryccl/testdata/telemetry/generic @@ -0,0 +1,134 @@ +# This file contains telemetry tests for CCL-related generic query plan +# counters. + +exec +CREATE TABLE kv ( + k INT PRIMARY KEY, + v INT, + INDEX (v) +) +---- + +exec +CREATE TABLE ab ( + a INT PRIMARY KEY, + b INT +) +---- + +exec +PREPARE p_pk AS +SELECT * FROM kv WHERE k = $1 +---- + +exec +PREPARE p_join AS +SELECT * FROM kv +JOIN ab ON a = k +WHERE v = $1 +---- + +feature-list +sql.plan.type.* +---- + +exec +SET plan_cache_mode = force_custom_plan +---- + +exec +SET index_recommendations_enabled = false +---- + +exec +SET application_name = 'testuser' +---- + +feature-usage +EXECUTE p_pk(100) +---- +sql.plan.type.force-custom + +feature-usage +EXECUTE p_join(1) +---- +sql.plan.type.force-custom + +# Non-prepared statements do not increment plan type counters. +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +exec +SET plan_cache_mode = force_generic_plan +---- + +feature-usage +EXECUTE p_pk(100) +---- +sql.plan.type.force-generic + +feature-usage +EXECUTE p_join(0) +---- +sql.plan.type.force-generic + +# Non-prepared statements do not increment plan type counters. +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +exec +SET plan_cache_mode = auto +---- + +# If the placeholder fast-path is used, the plan is always generic. +feature-usage +EXECUTE p_pk(100) +---- +sql.plan.type.auto-generic + +# The first five executions of p_join have custom plans while establishing an +# average cost. One of the custom executions occurred above. +feature-usage +EXECUTE p_join(2) +---- +sql.plan.type.auto-custom + +feature-usage +EXECUTE p_join(3) +---- +sql.plan.type.auto-custom + +feature-usage +EXECUTE p_join(4) +---- +sql.plan.type.auto-custom + +feature-usage +EXECUTE p_join(5) +---- +sql.plan.type.auto-custom + +# The sixth execution uses a generic plan. +feature-usage +EXECUTE p_join(6) +---- +sql.plan.type.auto-generic + +# Non-prepared statements do not increment plan type counters. +feature-usage +SELECT * FROM kv WHERE v = 100 +---- + +feature-usage +SELECT * FROM kv WHERE v = 100 +---- diff --git a/pkg/sql/plan_opt.go b/pkg/sql/plan_opt.go index e826a8d83bac..195863d9d712 100644 --- a/pkg/sql/plan_opt.go +++ b/pkg/sql/plan_opt.go @@ -533,6 +533,7 @@ func (opc *optPlanningCtx) buildReusableMemo( func (opc *optPlanningCtx) reuseMemo( ctx context.Context, cachedMemo *memo.Memo, ) (*memo.Memo, error) { + opc.incPlanTypeTelemetry(cachedMemo) if cachedMemo.IsOptimized() { // The query could have been already fully optimized in // buildReusableMemo, in which case it is considered a "generic" plan. @@ -559,6 +560,24 @@ func (opc *optPlanningCtx) reuseMemo( return mem, nil } +// incPlanTypeTelemetry increments the telemetry counters for the type of the +// plan: generic or custom. +func (opc *optPlanningCtx) incPlanTypeTelemetry(cachedMemo *memo.Memo) { + switch opc.p.SessionData().PlanCacheMode { + case sessiondatapb.PlanCacheModeForceCustom: + telemetry.Inc(sqltelemetry.PlanTypeForceCustomCounter) + case sessiondatapb.PlanCacheModeForceGeneric: + telemetry.Inc(sqltelemetry.PlanTypeForceGenericCounter) + case sessiondatapb.PlanCacheModeAuto: + if cachedMemo.IsOptimized() { + // A fully optimized memo is generic. + telemetry.Inc(sqltelemetry.PlanTypeAutoGenericCounter) + } else { + telemetry.Inc(sqltelemetry.PlanTypeAutoCustomCounter) + } + } +} + // useGenericPlan returns true if a generic query plan should be used instead of // a custom plan. func (opc *optPlanningCtx) useGenericPlan() bool { diff --git a/pkg/sql/sqltelemetry/planning.go b/pkg/sql/sqltelemetry/planning.go index a82bb0fe1365..ccc8ec510d65 100644 --- a/pkg/sql/sqltelemetry/planning.go +++ b/pkg/sql/sqltelemetry/planning.go @@ -205,6 +205,22 @@ var CancelQueriesUseCounter = telemetry.GetCounterOnce("sql.session.cancel-queri // CANCEL SESSIONS is run. var CancelSessionsUseCounter = telemetry.GetCounterOnce("sql.session.cancel-sessions") +// PlanTypeForceCustomCounter is to be incremented whenever a custom plan is +// used when plan_cache_mode=force_custom_plan. +var PlanTypeForceCustomCounter = telemetry.GetCounterOnce("sql.plan.type.force-custom") + +// PlanTypeForceGenericCounter is to be incremented whenever a generic plan is used +// when plan_cache_mode=force_generic_plan. +var PlanTypeForceGenericCounter = telemetry.GetCounterOnce("sql.plan.type.force-generic") + +// PlanTypeAutoCustomCounter is to be incremented whenever a generic plan is +// used when plan_cache_mode=auto. +var PlanTypeAutoCustomCounter = telemetry.GetCounterOnce("sql.plan.type.auto-custom") + +// PlanTypeAutoGenericCounter is to be incremented whenever a custom plan is +// used when plan_cache_mode=auto. +var PlanTypeAutoGenericCounter = telemetry.GetCounterOnce("sql.plan.type.auto-generic") + // We can't parameterize these telemetry counters, so just make a bunch of // buckets for setting the join reorder limit since the range of reasonable // values for the join reorder limit is quite small. From 8e980f3416828e6c19a7711071c717f4e3ffbd90 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Thu, 15 Aug 2024 10:09:57 -0400 Subject: [PATCH 15/15] opt: disable `GenerateParameterizedJoin` when forcing custom plans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exploration rule `GenerateParameterizedJoin` is now disabled when the `plan_cache_mode` session setting is set to `force_custom_plan`. This prevents possible regressions in plans for this mode in the rare case that a stable function is not folded during normalization (due to an error encountered during folding). In most cases, the check for placeholders and stable functions in `GenerateParameterizedJoin` is sufficient for preventing the rule from firing when a generic query plan is not being built—before optimizing a custom plan placeholders are always replaced with constants and stable functions are usually folded to constants. Release note: None --- pkg/sql/opt/norm/BUILD.bazel | 1 + pkg/sql/opt/norm/factory_test.go | 2 + pkg/sql/opt/testutils/opttester/BUILD.bazel | 1 + pkg/sql/opt/testutils/opttester/opt_tester.go | 12 ++++ pkg/sql/opt/xform/BUILD.bazel | 1 + pkg/sql/opt/xform/generic_funcs.go | 7 +++ pkg/sql/opt/xform/rules/generic.opt | 4 +- pkg/sql/opt/xform/testdata/external/hibernate | 6 +- pkg/sql/opt/xform/testdata/external/nova | 16 +++--- pkg/sql/opt/xform/testdata/rules/generic | 56 ++++++++++++------- 10 files changed, 75 insertions(+), 31 deletions(-) diff --git a/pkg/sql/opt/norm/BUILD.bazel b/pkg/sql/opt/norm/BUILD.bazel index 5cf9b2c64ab4..b42907b18c5f 100644 --- a/pkg/sql/opt/norm/BUILD.bazel +++ b/pkg/sql/opt/norm/BUILD.bazel @@ -91,6 +91,7 @@ go_test( "//pkg/sql/sem/catconstants", "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", + "//pkg/sql/sessiondatapb", "//pkg/sql/types", "//pkg/testutils/datapathutils", "//pkg/util/leaktest", diff --git a/pkg/sql/opt/norm/factory_test.go b/pkg/sql/opt/norm/factory_test.go index 756f45e8a7b6..d8adff216cda 100644 --- a/pkg/sql/opt/norm/factory_test.go +++ b/pkg/sql/opt/norm/factory_test.go @@ -24,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/catconstants" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb" "github.com/cockroachdb/cockroach/pkg/sql/types" ) @@ -84,6 +85,7 @@ func TestCopyAndReplace(t *testing.T) { } evalCtx := eval.MakeTestingEvalContext(cluster.MakeTestingClusterSettings()) + evalCtx.SessionData().PlanCacheMode = sessiondatapb.PlanCacheModeAuto var o xform.Optimizer testutils.BuildQuery(t, &o, cat, &evalCtx, "SELECT * FROM ab WHERE a = $1") diff --git a/pkg/sql/opt/testutils/opttester/BUILD.bazel b/pkg/sql/opt/testutils/opttester/BUILD.bazel index e927b3219e85..7bd28ca49c6f 100644 --- a/pkg/sql/opt/testutils/opttester/BUILD.bazel +++ b/pkg/sql/opt/testutils/opttester/BUILD.bazel @@ -46,6 +46,7 @@ go_library( "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", "//pkg/sql/sem/volatility", + "//pkg/sql/sessiondatapb", "//pkg/sql/stats", "//pkg/testutils/datapathutils", "//pkg/testutils/floatcmp", diff --git a/pkg/sql/opt/testutils/opttester/opt_tester.go b/pkg/sql/opt/testutils/opttester/opt_tester.go index 75dce5db6cc8..3d826524276e 100644 --- a/pkg/sql/opt/testutils/opttester/opt_tester.go +++ b/pkg/sql/opt/testutils/opttester/opt_tester.go @@ -61,6 +61,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb" "github.com/cockroachdb/cockroach/pkg/sql/stats" "github.com/cockroachdb/cockroach/pkg/testutils/datapathutils" "github.com/cockroachdb/cockroach/pkg/testutils/floatcmp" @@ -165,6 +166,9 @@ type Flags struct { // DisableRules is a set of rules that are not allowed to run. DisableRules RuleSet + // Generic enables optimizations for generic query plans. + Generic bool + // ExploreTraceRule restricts the ExploreTrace output to only show the effects // of a specific rule. ExploreTraceRule opt.RuleName @@ -479,6 +483,11 @@ func New(catalog cat.Catalog, sql string) *OptTester { // opt disable=ConstrainScan // norm disable=(NegateOr,NegateAnd) // +// - generic: enables optimizations for generic query plans. +// NOTE: This flag sets the plan_cache_mode session setting to "auto", which +// cannot be done via the "set" flag because it requires a CCL license, +// which optimizer tests are not set up to utilize. +// // - rule: used with exploretrace; the value is the name of a rule. When // specified, the exploretrace output is filtered to only show expression // changes due to that specific rule. @@ -988,6 +997,9 @@ func (f *Flags) Set(arg datadriven.CmdArg) error { f.DisableRules.Add(int(r)) } + case "generic": + f.evalCtx.SessionData().PlanCacheMode = sessiondatapb.PlanCacheModeAuto + case "rule": if len(arg.Vals) != 1 { return fmt.Errorf("rule requires one argument") diff --git a/pkg/sql/opt/xform/BUILD.bazel b/pkg/sql/opt/xform/BUILD.bazel index 0555dedc8f22..60477e7dcb0a 100644 --- a/pkg/sql/opt/xform/BUILD.bazel +++ b/pkg/sql/opt/xform/BUILD.bazel @@ -50,6 +50,7 @@ go_library( "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", "//pkg/sql/sem/volatility", + "//pkg/sql/sessiondatapb", "//pkg/sql/types", "//pkg/util/buildutil", "//pkg/util/cancelchecker", diff --git a/pkg/sql/opt/xform/generic_funcs.go b/pkg/sql/opt/xform/generic_funcs.go index 362e7d34ef78..805d431f728f 100644 --- a/pkg/sql/opt/xform/generic_funcs.go +++ b/pkg/sql/opt/xform/generic_funcs.go @@ -17,9 +17,16 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sem/volatility" + "github.com/cockroachdb/cockroach/pkg/sql/sessiondatapb" "github.com/cockroachdb/cockroach/pkg/sql/types" ) +// GenericRulesEnabled returns true if rules for optimizing generic query plans +// are enabled, based on the plan_cache_mode session setting. +func (c *CustomFuncs) GenericRulesEnabled() bool { + return c.e.evalCtx.SessionData().PlanCacheMode != sessiondatapb.PlanCacheModeForceCustom +} + // HasPlaceholdersOrStableExprs returns true if the given relational expression's subtree has // at least one placeholder. func (c *CustomFuncs) HasPlaceholdersOrStableExprs(e memo.RelExpr) bool { diff --git a/pkg/sql/opt/xform/rules/generic.opt b/pkg/sql/opt/xform/rules/generic.opt index c49631511e65..4b89e4ab00c9 100644 --- a/pkg/sql/opt/xform/rules/generic.opt +++ b/pkg/sql/opt/xform/rules/generic.opt @@ -29,7 +29,9 @@ # [GenerateParameterizedJoin, Explore] (Select - $scan:(Scan $scanPrivate:*) & (IsCanonicalScan $scanPrivate) + $scan:(Scan $scanPrivate:*) & + (GenericRulesEnabled) & + (IsCanonicalScan $scanPrivate) $filters:* & (HasPlaceholdersOrStableExprs (Root)) & (Let diff --git a/pkg/sql/opt/xform/testdata/external/hibernate b/pkg/sql/opt/xform/testdata/external/hibernate index 5a8fad72719c..9fe7cc3fdecc 100644 --- a/pkg/sql/opt/xform/testdata/external/hibernate +++ b/pkg/sql/opt/xform/testdata/external/hibernate @@ -886,7 +886,7 @@ project └── filters └── min:14 = $1 [outer=(14), constraints=(/14: (/NULL - ]), fd=()-->(14)] -opt +opt generic select person0_.id as id1_2_, person0_.address as address2_2_, @@ -951,7 +951,7 @@ project └── filters └── max:16 = 0 [outer=(16), constraints=(/16: [/0 - /0]; tight), fd=()-->(16)] -opt +opt generic select person0_.id as id1_2_, person0_.address as address2_2_, @@ -1016,7 +1016,7 @@ project │ └── filters (true) └── filters (true) -opt +opt generic select person0_.id as id1_2_, person0_.address as address2_2_, diff --git a/pkg/sql/opt/xform/testdata/external/nova b/pkg/sql/opt/xform/testdata/external/nova index a36bb9fd3dae..6ac5889c87e7 100644 --- a/pkg/sql/opt/xform/testdata/external/nova +++ b/pkg/sql/opt/xform/testdata/external/nova @@ -107,7 +107,7 @@ create table instance_type_extra_specs ) ---- -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -904,7 +904,7 @@ sort └── filters └── instance_type_extra_specs_1.instance_type_id:22 = instance_types.id:1 [outer=(1,22), constraints=(/1: (/NULL - ]; /22: (/NULL - ]), fd=(1)==(22), (22)==(1)] -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -1096,7 +1096,7 @@ project │ └── instance_type_extra_specs_1.deleted:36 = $7 [outer=(36), constraints=(/36: (/NULL - ]), fd=()-->(36)] └── filters (true) -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -1302,7 +1302,7 @@ project │ └── instance_type_extra_specs_1.deleted:36 = $7 [outer=(36), constraints=(/36: (/NULL - ]), fd=()-->(36)] └── filters (true) -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -1479,7 +1479,7 @@ project │ └── filters (true) └── filters (true) -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, @@ -2038,7 +2038,7 @@ sort └── filters └── instance_type_extra_specs_1.instance_type_id:35 = instance_types.id:1 [outer=(1,35), constraints=(/1: (/NULL - ]; /35: (/NULL - ]), fd=(1)==(35), (35)==(1)] -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -3025,7 +3025,7 @@ sort └── filters └── instance_type_extra_specs_1.instance_type_id:48 = instance_types.id:1 [outer=(1,48), constraints=(/1: (/NULL - ]; /48: (/NULL - ]), fd=(1)==(48), (48)==(1)] -opt +opt generic select anon_1.instance_types_created_at as anon_1_instance_types_created_at, anon_1.instance_types_updated_at as anon_1_instance_types_updated_at, anon_1.instance_types_deleted_at as anon_1_instance_types_deleted_at, @@ -3220,7 +3220,7 @@ project │ └── instance_type_extra_specs_1.deleted:36 = $7 [outer=(36), constraints=(/36: (/NULL - ]), fd=()-->(36)] └── filters (true) -opt +opt generic select anon_1.flavors_created_at as anon_1_flavors_created_at, anon_1.flavors_updated_at as anon_1_flavors_updated_at, anon_1.flavors_id as anon_1_flavors_id, diff --git a/pkg/sql/opt/xform/testdata/rules/generic b/pkg/sql/opt/xform/testdata/rules/generic index 93df29d979a5..fed232cac6d0 100644 --- a/pkg/sql/opt/xform/testdata/rules/generic +++ b/pkg/sql/opt/xform/testdata/rules/generic @@ -15,7 +15,7 @@ CREATE TABLE t ( # GenerateParameterizedJoin # -------------------------------------------------- -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = $1 ---- project @@ -42,7 +42,7 @@ project │ └── ($1,) └── filters (true) -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = $1::INT ---- project @@ -69,7 +69,7 @@ project │ └── ($1,) └── filters (true) -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND s = $2 AND b = $3 ---- project @@ -103,7 +103,7 @@ project # A placeholder referenced multiple times in the filters should only appear once # in the Values expression. -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = $1 AND i = $1 ---- project @@ -133,7 +133,7 @@ project # The generated join should not be reordered and merge joins should not be # explored on it. -opt expect=GenerateParameterizedJoin expect-not=(ReorderJoins,GenerateMergeJoins) +opt generic expect=GenerateParameterizedJoin expect-not=(ReorderJoins,GenerateMergeJoins) SELECT * FROM t WHERE i = $1 ---- project @@ -165,7 +165,7 @@ project │ └── filters (true) └── filters (true) -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE k = (SELECT i FROM t WHERE k = $1) ---- project @@ -209,7 +209,7 @@ project # TODO(mgartner): The rule doesn't apply because the filters do not reference # the placeholder directly. Consider ways to handle cases like this. -opt +opt generic SELECT * FROM t WHERE k = (SELECT $1::INT) ---- project @@ -242,7 +242,7 @@ exec-ddl CREATE INDEX partial_idx ON t(t) WHERE t IS NOT NULL ---- -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE t = $1 ---- project @@ -282,7 +282,7 @@ exec-ddl CREATE INDEX partial_idx ON t(i, t) WHERE i IS NOT NULL AND t IS NOT NULL ---- -opt expect=GenerateParameterizedJoin +opt generic expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND t = $2 ---- project @@ -322,7 +322,7 @@ exec-ddl CREATE INDEX partial_idx ON t(s) WHERE k = i ---- -opt +opt generic SELECT * FROM t@partial_idx WHERE s = $1 AND k = $2 AND i = $2 ---- project @@ -362,7 +362,7 @@ exec-ddl DROP INDEX partial_idx ---- -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE t = now() ---- project @@ -394,7 +394,7 @@ project │ └── filters (true) └── filters (true) -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND t = now() ---- project @@ -426,7 +426,7 @@ project │ └── filters (true) └── filters (true) -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND t > now() ---- project @@ -461,7 +461,7 @@ project │ └── filters (true) └── filters (true) -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND t = now() + $2 ---- project @@ -506,7 +506,7 @@ project │ └── filters (true) └── filters (true) -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND t = now() + '1 hr'::INTERVAL ---- project @@ -552,7 +552,7 @@ project └── filters (true) # TODO(mgartner): Apply the rule to stable, non-leaf expressions. -opt no-stable-folds +opt generic no-stable-folds SELECT * FROM t WHERE t = '2024-01-01 12:00:00'::TIMESTAMP::TIMESTAMPTZ ---- select @@ -571,7 +571,7 @@ select # arguments. # TODO(mgartner): We should be able to relax this restriction as long as all the # arguments are constants or placeholders. -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND s = quote_literal(1::INT) ---- project @@ -607,7 +607,7 @@ project # A stable function is not included in the Values expression if its arguments # reference a column from the table. This would create an illegal outer column # reference in a non-apply-join. -opt no-stable-folds expect=GenerateParameterizedJoin +opt generic no-stable-folds expect=GenerateParameterizedJoin SELECT * FROM t WHERE i = $1 AND s = quote_literal(i) ---- project @@ -642,7 +642,7 @@ project # The rule does not match if there are no placeholders or stable expressions in # the filters. -opt expect-not=GenerateParameterizedJoin +opt generic expect-not=GenerateParameterizedJoin SELECT * FROM t WHERE i = 1 AND s = 'foo' ---- index-join t @@ -654,3 +654,21 @@ index-join t ├── constraint: /2/3/4/1: [/1/'foo' - /1/'foo'] ├── key: (1) └── fd: ()-->(2,3), (1)-->(4) + +# The rule does not match if generic optimizations are disabled. +opt expect-not=GenerateParameterizedJoin +SELECT * FROM t WHERE k = $1 AND s = quote_literal(1::INT) +---- +select + ├── columns: k:1!null i:2 s:3!null b:4 t:5 + ├── cardinality: [0 - 1] + ├── has-placeholder + ├── key: () + ├── fd: ()-->(1-5) + ├── scan t + │ ├── columns: k:1!null i:2 s:3 b:4 t:5 + │ ├── key: (1) + │ └── fd: (1)-->(2-5) + └── filters + ├── k:1 = $1 [outer=(1), constraints=(/1: (/NULL - ]), fd=()-->(1)] + └── s:3 = e'\'1\'' [outer=(3), constraints=(/3: [/e'\'1\'' - /e'\'1\'']; tight), fd=()-->(3)]