Merge pull request #123152 from rytaft/backportstaging-v23.1.20-123083

staging-v23.1.20: opt/memo: improve zigzag join cost and selectivity estimation with multi-column stats (#123152)
cockroachdb · Apr 26, 2024 · 057f6d8 · 057f6d8
2 parents 68aeac6 + 30e7aab
commit 057f6d8
Show file tree

Hide file tree

Showing 18 changed files with 785 additions and 93 deletions.
diff --git a/pkg/ccl/backupccl/testdata/backup-restore/in-progress-restores b/pkg/ccl/backupccl/testdata/backup-restore/in-progress-restores
@@ -155,7 +155,10 @@ SELECT * FROM b2.me.baz;
 
 
 query-sql
-SELECT table_name,index_name,non_unique,seq_in_index,column_name FROM [SHOW INDEX FROM b2.me.baz] WHERE index_name = 'greeting_idx';
+SELECT table_name,index_name,non_unique,seq_in_index,column_name
+FROM [SHOW INDEX FROM b2.me.baz]
+WHERE index_name = 'greeting_idx'
+ORDER BY seq_in_index;
 ----
 baz greeting_idx true 1 y
 baz greeting_idx true 2 rowid

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior
@@ -3568,8 +3568,7 @@ ALTER TABLE xyz INJECT STATISTICS '[
   }
 ]';
 
-# Regression test for #105942
-# The following should produce 2 lookup joins with a local distribution.
+# Regression test for #105942.
 query T retry
 EXPLAIN SELECT
   xyz.str,
@@ -3612,7 +3611,7 @@ vectorized: true
               table: xyz@xyz_id2_str_abc_id_idx
               spans: [/'ap-southeast-2'/'68088706-02c6-47d1-b993-a421cd761f2b' - /'ap-southeast-2'/'68088706-02c6-47d1-b993-a421cd761f2b']
 
-# The following should use a string of 4 lookup joins with a cost under 200.
+# The following should use a string of 4 lookup/index joins with a cost under 200.
 query T retry
 EXPLAIN(opt,verbose) SELECT
   x.str,
@@ -3717,4 +3716,3 @@ project
       │         ├── x.id2:12 = '68088706-02c6-47d1-b993-a421cd761f2b' [outer=(12), constraints=(/12: [/'68088706-02c6-47d1-b993-a421cd761f2b' - /'68088706-02c6-47d1-b993-a421cd761f2b']; tight), fd=()-->(12)]
       │         └── x.crdb_region:13 = 'ap-southeast-2' [outer=(13), immutable, constraints=(/13: [/'ap-southeast-2' - /'ap-southeast-2']; tight), fd=()-->(13)]
       └── filters (true)
-
diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go
@@ -3595,6 +3595,14 @@ func (m *sessionDataMutator) SetOptimizerUseProvidedOrderingFix(val bool) {
 	m.data.OptimizerUseProvidedOrderingFix = val
 }
 
+func (m *sessionDataMutator) SetOptimizerUseImprovedZigzagJoinCosting(val bool) {
+	m.data.OptimizerUseImprovedZigzagJoinCosting = val
+}
+
+func (m *sessionDataMutator) SetOptimizerUseImprovedMultiColumnSelectivityEstimate(val bool) {
+	m.data.OptimizerUseImprovedMultiColumnSelectivityEstimate = val
+}
+
 // Utility functions related to scrubbing sensitive information on SQL Stats.
 
 // quantizeCounts ensures that the Count field in the

diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema
@@ -5340,7 +5340,9 @@ optimizer_use_forecasts                                    on
 optimizer_use_histograms                                   on
 optimizer_use_improved_computed_column_filters_derivation  off
 optimizer_use_improved_disjunction_stats                   on
+optimizer_use_improved_multi_column_selectivity_estimate   off
 optimizer_use_improved_split_disjunction_for_joins         on
+optimizer_use_improved_zigzag_join_costing                 off
 optimizer_use_limit_ordering_for_streaming_group_by        on
 optimizer_use_multicol_stats                               on
 optimizer_use_not_visible_indexes                          off

diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog
@@ -2825,7 +2825,9 @@ optimizer_use_forecasts                                    on                  N
 optimizer_use_histograms                                   on                  NULL      NULL        NULL        string
 optimizer_use_improved_computed_column_filters_derivation  off                 NULL      NULL        NULL        string
 optimizer_use_improved_disjunction_stats                   on                  NULL      NULL        NULL        string
+optimizer_use_improved_multi_column_selectivity_estimate   off                 NULL      NULL        NULL        string
 optimizer_use_improved_split_disjunction_for_joins         on                  NULL      NULL        NULL        string
+optimizer_use_improved_zigzag_join_costing                 off                 NULL      NULL        NULL        string
 optimizer_use_limit_ordering_for_streaming_group_by        on                  NULL      NULL        NULL        string
 optimizer_use_multicol_stats                               on                  NULL      NULL        NULL        string
 optimizer_use_not_visible_indexes                          off                 NULL      NULL        NULL        string
@@ -2987,7 +2989,9 @@ optimizer_use_forecasts                                    on                  N
 optimizer_use_histograms                                   on                  NULL  user     NULL      on                  on
 optimizer_use_improved_computed_column_filters_derivation  off                 NULL  user     NULL      off                 off
 optimizer_use_improved_disjunction_stats                   on                  NULL  user     NULL      on                  on
+optimizer_use_improved_multi_column_selectivity_estimate   off                 NULL  user     NULL      off                 off
 optimizer_use_improved_split_disjunction_for_joins         on                  NULL  user     NULL      on                  on
+optimizer_use_improved_zigzag_join_costing                 off                 NULL  user     NULL      off                 off
 optimizer_use_limit_ordering_for_streaming_group_by        on                  NULL  user     NULL      on                  on
 optimizer_use_multicol_stats                               on                  NULL  user     NULL      on                  on
 optimizer_use_not_visible_indexes                          off                 NULL  user     NULL      off                 off
@@ -3148,7 +3152,9 @@ optimizer_use_forecasts                                    NULL    NULL     NULL
 optimizer_use_histograms                                   NULL    NULL     NULL     NULL        NULL
 optimizer_use_improved_computed_column_filters_derivation  NULL    NULL     NULL     NULL        NULL
 optimizer_use_improved_disjunction_stats                   NULL    NULL     NULL     NULL        NULL
+optimizer_use_improved_multi_column_selectivity_estimate   NULL    NULL     NULL     NULL        NULL
 optimizer_use_improved_split_disjunction_for_joins         NULL    NULL     NULL     NULL        NULL
+optimizer_use_improved_zigzag_join_costing                 NULL    NULL     NULL     NULL        NULL
 optimizer_use_limit_ordering_for_streaming_group_by        NULL    NULL     NULL     NULL        NULL
 optimizer_use_multicol_stats                               NULL    NULL     NULL     NULL        NULL
 optimizer_use_not_visible_indexes                          NULL    NULL     NULL     NULL        NULL

diff --git a/pkg/sql/logictest/testdata/logic_test/show_source b/pkg/sql/logictest/testdata/logic_test/show_source
@@ -118,7 +118,9 @@ optimizer_use_forecasts                                    on
 optimizer_use_histograms                                   on
 optimizer_use_improved_computed_column_filters_derivation  off
 optimizer_use_improved_disjunction_stats                   on
+optimizer_use_improved_multi_column_selectivity_estimate   off
 optimizer_use_improved_split_disjunction_for_joins         on
+optimizer_use_improved_zigzag_join_costing                 off
 optimizer_use_limit_ordering_for_streaming_group_by        on
 optimizer_use_multicol_stats                               on
 optimizer_use_not_visible_indexes                          off

diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go
@@ -167,6 +167,8 @@ type Memo struct {
 	hoistUncorrelatedEqualitySubqueries        bool
 	useImprovedComputedColumnFiltersDerivation bool
 	useProvidedOrderingFix                     bool
+	useImprovedZigzagJoinCosting               bool
+	useImprovedMultiColumnSelectivityEstimate  bool
 
 	// curRank is the highest currently in-use scalar expression rank.
 	curRank opt.ScalarRank
@@ -229,6 +231,8 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
 		hoistUncorrelatedEqualitySubqueries:        evalCtx.SessionData().OptimizerHoistUncorrelatedEqualitySubqueries,
 		useImprovedComputedColumnFiltersDerivation: evalCtx.SessionData().OptimizerUseImprovedComputedColumnFiltersDerivation,
 		useProvidedOrderingFix:                     evalCtx.SessionData().OptimizerUseProvidedOrderingFix,
+		useImprovedZigzagJoinCosting:               evalCtx.SessionData().OptimizerUseImprovedZigzagJoinCosting,
+		useImprovedMultiColumnSelectivityEstimate:  evalCtx.SessionData().OptimizerUseImprovedMultiColumnSelectivityEstimate,
 	}
 	m.metadata.Init()
 	m.logPropsBuilder.init(ctx, evalCtx, m)
@@ -374,7 +378,9 @@ func (m *Memo) IsStale(
 		m.alwaysUseHistograms != evalCtx.SessionData().OptimizerAlwaysUseHistograms ||
 		m.hoistUncorrelatedEqualitySubqueries != evalCtx.SessionData().OptimizerHoistUncorrelatedEqualitySubqueries ||
 		m.useImprovedComputedColumnFiltersDerivation != evalCtx.SessionData().OptimizerUseImprovedComputedColumnFiltersDerivation ||
-		m.useProvidedOrderingFix != evalCtx.SessionData().OptimizerUseProvidedOrderingFix {
+		m.useProvidedOrderingFix != evalCtx.SessionData().OptimizerUseProvidedOrderingFix ||
+		m.useImprovedZigzagJoinCosting != evalCtx.SessionData().OptimizerUseImprovedZigzagJoinCosting ||
+		m.useImprovedMultiColumnSelectivityEstimate != evalCtx.SessionData().OptimizerUseImprovedMultiColumnSelectivityEstimate {
 		return true, nil
 	}
 

diff --git a/pkg/sql/opt/memo/memo_test.go b/pkg/sql/opt/memo/memo_test.go
@@ -379,6 +379,18 @@ func TestMemoIsStale(t *testing.T) {
 	evalCtx.SessionData().OptimizerUseProvidedOrderingFix = false
 	notStale()
 
+	// Stale optimizer_use_improved_zigzag_join_costing.
+	evalCtx.SessionData().OptimizerUseImprovedZigzagJoinCosting = true
+	stale()
+	evalCtx.SessionData().OptimizerUseImprovedZigzagJoinCosting = false
+	notStale()
+
+	// Stale optimizer_use_improved_multi_column_selectivity_estimate.
+	evalCtx.SessionData().OptimizerUseImprovedMultiColumnSelectivityEstimate = true
+	stale()
+	evalCtx.SessionData().OptimizerUseImprovedMultiColumnSelectivityEstimate = false
+	notStale()
+
 	// User no longer has access to view.
 	catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
 	_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)