From 491690a4566b54108245fe457150548d8b5646fa Mon Sep 17 00:00:00 2001 From: Radu Berinde Date: Mon, 14 Jun 2021 11:32:13 -0700 Subject: [PATCH] opt: improve FDs for Intersect and Except variants We can pass through FDs from the left side on Intersect/IntersectAll, and we can pass through FDs from both sides on Except/ExceptAll. Release note: None --- pkg/sql/opt/exec/execbuilder/testdata/fk | 5 ++ pkg/sql/opt/exec/execbuilder/testdata/union | 12 ++-- pkg/sql/opt/memo/logical_props_builder.go | 53 +++++++++++------ pkg/sql/opt/memo/testdata/logprops/set | 12 ++-- pkg/sql/opt/memo/testdata/stats/set | 59 ++++++++++--------- pkg/sql/opt/memo/testdata/stats_quality/tpcc | 6 ++ pkg/sql/opt/norm/testdata/rules/select | 12 ++++ pkg/sql/opt/norm/testdata/rules/set | 3 + pkg/sql/opt/props/func_dep.go | 17 ++++++ pkg/sql/opt/props/func_dep_test.go | 10 ++++ pkg/sql/opt/xform/testdata/coster/set | 8 +++ pkg/sql/opt/xform/testdata/external/tpcc | 6 ++ .../xform/testdata/external/tpcc-later-stats | 6 ++ .../opt/xform/testdata/external/tpcc-no-stats | 6 ++ pkg/sql/opt/xform/testdata/physprops/ordering | 4 +- 15 files changed, 162 insertions(+), 57 deletions(-) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/fk b/pkg/sql/opt/exec/execbuilder/testdata/fk index 9cdfe818cd71..2dd37eee7159 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/fk +++ b/pkg/sql/opt/exec/execbuilder/testdata/fk @@ -487,6 +487,7 @@ vectorized: true │ │ │ └── • hash join │ │ equality: (p) = (p) +│ │ left cols are key │ │ right cols are key │ │ │ ├── • except all @@ -512,6 +513,7 @@ vectorized: true │ └── • hash join │ equality: (p) = (p) + │ left cols are key │ right cols are key │ ├── • except all @@ -618,6 +620,7 @@ vectorized: true │ └── • hash join │ equality: (c) = (c) + │ left cols are key │ right cols are key │ ├── • except all @@ -758,6 +761,7 @@ vectorized: true │ └── • hash join │ equality: (c) = (c) + │ left cols are key │ right cols are key │ ├── • except all @@ -890,6 +894,7 @@ vectorized: true │ └── • hash join │ equality: (x) = (y) + │ left cols are key │ right cols are key │ ├── • except all diff --git a/pkg/sql/opt/exec/execbuilder/testdata/union b/pkg/sql/opt/exec/execbuilder/testdata/union index 00690b075f75..93aa01e8566e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/union +++ b/pkg/sql/opt/exec/execbuilder/testdata/union @@ -595,7 +595,7 @@ vectorized: true · • intersect │ columns: (b, c, d, e) -│ ordering: +b,+c,+d,+e +│ ordering: +b,+d,+c,+e │ estimated row count: 1 (missing stats) │ ├── • filter @@ -624,7 +624,7 @@ vectorized: true table: abcde@abcde_b_c_d_e_idx spans: FULL SCAN · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0lF-L2kAUxd_7KS73KcEpbv7olgFhdmuWBlyzTUL_UERi5tYG0iSdjLBF_O4lSdtVu6Z12b4Ic2Z-95x7kGyx_pYjR-_D3ezKn4Mx9aM4ejtj8M4Lr4PIMyHyZt7rGFYMUgaSAcFNGNyCcUJPVqkk8OexF0bNfd8zE96_8UIPjBQmYJlwNZ-CIWECZEIQTr0Qrj_CChkWpaR58pVq5J_QwgXDSpUp1XWpGmnbPvDlPfILhllRbXQjLximpSLkW9SZzgk5xskqp5ASSWp4gQwl6STL27FtItH-LlfLdCmXtMzkPTKMqqSoObxEhsFGcxA2Ew4TLhMjXOwYlhv94FjrZE3IrR3791Q3Wa5JkRpah5E6nYMh7KYgzrk_j1_97Ek4MAHhmicj2OdE2C_G_p_FOE8qxnnOYtxzIkyzWmdFqofuYQRhsV_rNusrSYokh0Z2TjqPnuQ8egbn8UnnB8NNUXbDDvwWDfm3J4_EvyW1poh0UA3HhwvE3yvie5-Iq9kMGeb0WRvCGjBhD5hwBky4A3OisvWXP-Xff7i9Kk5tfnlO5yHVVVnUdNzAo5MvmrVJrqmrsS43KqU7VaatTXcMWq4VJNW6u7W6g190V03AfdjqhUf9sN0LO_2w0wu7_bDbC48PYOsYHp0B28fwuBe-PIq92L34EQAA___K-yzi +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0lF-L2kAUxd_7KS73KcEpbv7olgFhdmuWBlyzTUL_UERi5tYG0iSdjLBF_O4lSdtVu6Z12b4Ic2Z-95x7kGyx_pYjR-_D3ezKn4Mx9aM4ejtj8M4Lr4PIMyHyZt7rGFYMUgaSAcFNGNyCcUJPVqkk8OexF0bNfd8zE96_8UIPjBQmYJlwNZ-CIWECZEIQTr0Qrj_CChkWpaR58pVq5J_QwgXDSpUp1XWpGmnbPvDlPfILhllRbXQjLximpSLkW9SZzgk5xskqp5ASSWp4gQwl6STL27FtItH-LlfLdCmXtMzkPTKMqqSoObxEhsFGcxA2Ew4TLhMjXOwYlhv94FjrZE3IrR3791Q3Wa5JkRpah5E6nYMh7KYgzrk_j1_97Ek4MAHhmicj2OdE2C_G_p_FOE8qxnnOYtxzIkyzWmdFqofuYQRhsV_rNusrSYokh0Z2TjqPnuQ8egbn8UnnB8NNUXbDDvwWDfm3J4_EvyW1poh0UA3HhwvE3yvie5-Iq9kMGeb0WRvCGjDhDJiwB0y4A3OisvWXP-Xff7i9Kk5tfnlO5yHVVVnUdNzAo5MvmrVJrqmrsS43KqU7VaatTXcMWq4VJNW6u7W6g190V03AfdjqhUf9sN0LO_2w0wu7_bDbC48PYOsYHp0B28fwuBe-PIq92L34EQAA___LAyzi query T EXPLAIN (DISTSQL,VERBOSE) SELECT b, c, d, e FROM (SELECT b, c, d, e FROM abcde EXCEPT SELECT b, c, d, e FROM abcde) WHERE c = 1 AND d = e ORDER BY b, c, d, e @@ -634,7 +634,7 @@ vectorized: true · • except │ columns: (b, c, d, e) -│ ordering: +b,+c,+d,+e +│ ordering: +b,+d,+c,+e │ estimated row count: 1 (missing stats) │ ├── • filter @@ -663,7 +663,7 @@ vectorized: true table: abcde@abcde_b_c_d_e_idx spans: FULL SCAN · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0lN9q20wQxe-_pxjmSsL74eiPnbJg2CRWqMGxUkm0KcUYWTt1BaqkrtaQEvzuRVLT2G6k1iG9MezZ_c05czB6wOpbhhy9u9v5xWwBxnQWRuG7OYP3XnDph54JoTf3riJYM0gYSAYE14F_A0aHHq8TSeDdXXm3URfbvDHhw1sv8MBIYAKWCReLKRgSJkAm-MHUC-Dy4x6JDPNC0iL-ShXyT2jhkmGpioSqqlC19NA8mMl75GcM07zc6lpeMkwKRcgfUKc6I-QYxeuMAoolqeEZMpSk4zRrxjbRRPO7Wq-SlVzRKpX3yDAs47zi8D8y9Leag7CZcJhwmRjhcsew2Oonx0rHG0Ju7djfp7pOM02K1NA6jNTqHAxh101xzmeL6M3PwoQDExCu2RnBPiXCfjH2vyzGeVExzmsW454SYZpWOs0TPXQPIwiLPa5br68kKZIcatnpdB69yHn0Cs7jTucnw21etMMO_JY1-acnz8S_IbWhkLRfDseHC0TfS-KPH4qL-RwZZvRZG8IaMGEPmHAGTLgDc6LSzZff5V__tr0eutY-P6XwgKqyyCs6Xv_ZyWf1ziQ31HZYFVuV0K0qksamPfoN1wiSKt3eWu1hlrdXdcB92OqF3X7Y7oWdftjphUf9sNsLjw9g6xgenQDbx_C4Fz4_ir3c_fcjAAD__7UbLSI= +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0lN9q20wQxe-_pxjmSsL74eiPnbJg2CRWqMGxUkm0KcUYWTt1BaqkrtaQEvzuRVLT2G6k1iG9MezZ_c05czB6wOpbhhy9u9v5xWwBxnQWRuG7OYP3XnDph54JoTf3riJYM0gYSAYE14F_A0aHHq8TSeDdXXm3URfbvDHhw1sv8MBIYAKWCReLKRgSJkAm-MHUC-Dy4x6JDPNC0iL-ShXyT2jhkmGpioSqqlC19NA8mMl75GcM07zc6lpeMkwKRcgfUKc6I-QYxeuMAoolqeEZMpSk4zRrxjbRRPO7Wq-SlVzRKpX3yDAs47zi8D8y9Leag7CZcJhwmRjhcsew2Oonx0rHG0Ju7djfp7pOM02K1NA6jNTqHAxh101xzmeL6M3PwoQDExCu2RnBPiXCfjH2vyzGeVExzmsW454SYZpWOs0TPXQPIwiLPa5br68kKZIcatnpdB69yHn0Cs7jTucnw21etMMO_JY1-acnz8S_IbWhkLRfDseHC0TfS-KPH4qL-RwZZvRZG8IaMOEMmLAHTLgDc6LSzZff5V__tr0eutY-P6XwgKqyyCs6Xv_ZyWf1ziQ31HZYFVuV0K0qksamPfoN1wiSKt3eWu1hlrdXdcB92OqF3X7Y7oWdftjphUf9sNsLjw9g6xgenQDbx_C4Fz4_ir3c_fcjAAD__7UjLSI= query T EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde EXCEPT ALL SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY a @@ -712,7 +712,7 @@ vectorized: true · • intersect all │ columns: (a, b, c, d, e) -│ ordering: +b,+c,+d,+a,+e +│ ordering: +b,+d,+a,+c,+e │ estimated row count: 1 (missing stats) │ ├── • filter @@ -741,7 +741,7 @@ vectorized: true table: abcde@abcde_b_c_d_e_idx spans: FULL SCAN · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysk9uLm1AQxt_7VwzzpM2UjZdAORA42calQjZuVXqhhGA801Swao8GtoT870Ut7BqStCn7EpjLN993fsE91j9zFOh9fljM_CUYcz-Kow8Lgo9eeBtEngmRt_DexfAa7sLgHoxhmWxSxeAvYy-M2vZssYATGyZ8eu-FHhgpTMEyYbacg6FgCmxCEM69EG6_wIYgJVAETJAgYVEqXiY_uEbxFS1cEVa6TLmuS9229t2Crx5RjAmzoto1bXtFmJaaUeyxyZqcUWCcbHIOOVGsb8ZIqLhJsrw726WT3e96s07Xas3rTD0iYVQlRS3gDa4OhOWueTpfN8mWUVgH-vcId1nesGZ9Yw39-74AQzotGSGEv4zf_gEkXZiCnJhnI9jXRHhOwX4xCs5_UXBekoJ7NsKT864otWLNamC8apV_WznxjnvWW464Caobd_iS-FfFYvg9IGHO3xpD2iOSzoikOyJpjUhORuZUZ9vvp0dIGOwaAdIiaZN0SLokJ2chTK75H0Kuq7Ko-RjGycvjlgCrLfdE63KnU37QZdrZ9GXQ6bqG4rrpp1Zf-EU_agM-F1sXxe5AbB-L7Yti57Kzc4WzdSx2L4onR86rw6vfAQAA___SNa8C +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysk9uLm1AQxt_7VwzzpM2UjZdAORA42calQjZuVXqhhGA801Swao8GtoT870Ut7BqStCn7EpjLN993fsE91j9zFOh9fljM_CUYcz-Kow8Lgo9eeBtEngmRt_DexfAa7sLgHoxhmWxSxeAvYy-M2vZssYATGyZ8eu-FHhgpTMEyYbacg6FgCmxCEM69EG6_wIYgJVAETJAgYVEqXiY_uEbxFS1cEVa6TLmuS9229t2Crx5RjAmzoto1bXtFmJaaUeyxyZqcUWCcbHIOOVGsb8ZIqLhJsrw726WT3e96s07Xas3rTD0iYVQlRS3gDa4OhOWueTpfN8mWUVgH-vcId1nesGZ9Yw39-74AQzotGSGEv4zf_gEkXZiCnJhnI9jXRHhOwX4xCs5_UXBekoJ7NsKT864otWLNamC8apV_WznxjnvWW464Caobd_iS-FfFYvg9IGHO3xpD2iOS7oikNSLpjEhORuZUZ9vvp0dIGOwaAdIiaZN0SLokJ2chTK75H0Kuq7Ko-RjGycvjlgCrLfdE63KnU37QZdrZ9GXQ6bqG4rrpp1Zf-EU_agM-F1sXxe5AbB-L7Yti57Kzc4WzdSx2L4onR86rw6vfAQAA___SLa8C # Regression test for #64181. Ensure that a projection on top of an ordered # UNION ALL correctly projects away ordering columns. diff --git a/pkg/sql/opt/memo/logical_props_builder.go b/pkg/sql/opt/memo/logical_props_builder.go index f3e1a488ace8..e8bdf66cd0ae 100644 --- a/pkg/sql/opt/memo/logical_props_builder.go +++ b/pkg/sql/opt/memo/logical_props_builder.go @@ -685,6 +685,7 @@ func (b *logicalPropsBuilder) buildLocalityOptimizedSearchProps( func (b *logicalPropsBuilder) buildSetProps(setNode RelExpr, rel *props.Relational) { BuildSharedProps(setNode, &rel.Shared) + op := setNode.Op() leftProps := setNode.Child(0).(RelExpr).Relational() rightProps := setNode.Child(1).(RelExpr).Relational() setPrivate := setNode.Private().(*SetPrivate) @@ -720,12 +721,7 @@ func (b *logicalPropsBuilder) buildSetProps(setNode RelExpr, rel *props.Relation // Functional Dependencies // ----------------------- - switch setNode.Op() { - case opt.UnionOp, opt.IntersectOp, opt.ExceptOp: - // These operators eliminate duplicates, so a strict key exists. - rel.FuncDeps.AddStrictKey(rel.OutputCols, rel.OutputCols) - } - switch setNode.Op() { + switch op { case opt.UnionOp, opt.UnionAllOp, opt.LocalityOptimizedSearchOp: // If columns at ordinals (i, j) are equivalent in both the left input // and right input, then the output columns at ordinals at (i, j) are @@ -738,27 +734,48 @@ func (b *logicalPropsBuilder) buildSetProps(setNode RelExpr, rel *props.Relation } } } + case opt.IntersectOp, opt.IntersectAllOp, opt.ExceptOp, opt.ExceptAllOp: - // Intersect, IntersectAll, Except and ExceptAll only output rows from - // the left input, so if columns at ordinals (i, j) are equivalent in - // the left input, then they are equivalent in the output. - // TODO(mgartner): The entire FD set on the left side can be used, but - // columns may need to be mapped. Intersections can combine FD - // information from both the left and the right. - for i := range setPrivate.OutCols { - for j := i + 1; j < len(setPrivate.OutCols); j++ { - if leftProps.FuncDeps.AreColsEquiv(setPrivate.LeftCols[i], setPrivate.LeftCols[j]) { - rel.FuncDeps.AddEquivalency(setPrivate.OutCols[i], setPrivate.OutCols[j]) + // With these operators, the output is a subset of the left input, so all + // the left FDs still hold (similar to a Select). + rel.FuncDeps.RemapFrom(&leftProps.FuncDeps, func(in opt.ColumnID) opt.ColumnID { + for i, leftCol := range setPrivate.LeftCols { + if in == leftCol { + return setPrivate.OutCols[i] } } + panic(errors.AssertionFailedf("invalid column %d in left FD", in)) + }) + + if op == opt.IntersectOp || op == opt.IntersectAllOp { + // With Intersect operators, the output is also a subset of the right input, + // so all the right FDs apply as well. + var remapped props.FuncDepSet + remapped.RemapFrom(&rightProps.FuncDeps, func(in opt.ColumnID) opt.ColumnID { + for i, rightCol := range setPrivate.RightCols { + if in == rightCol { + return setPrivate.OutCols[i] + } + } + panic(errors.AssertionFailedf("invalid column %d in right FD", in)) + }) + rel.FuncDeps.AddFrom(&remapped) } } + // Add a strict key for variants that eliminate duplicates. + switch op { + case opt.UnionOp, opt.IntersectOp, opt.ExceptOp: + rel.FuncDeps.AddStrictKey(rel.OutputCols, rel.OutputCols) + } + // Cardinality // ----------- // Calculate cardinality of the set operator. - rel.Cardinality = b.makeSetCardinality( - setNode.Op(), leftProps.Cardinality, rightProps.Cardinality) + rel.Cardinality = b.makeSetCardinality(op, leftProps.Cardinality, rightProps.Cardinality) + if rel.FuncDeps.HasMax1Row() { + rel.Cardinality = rel.Cardinality.Limit(1) + } // Statistics // ---------- diff --git a/pkg/sql/opt/memo/testdata/logprops/set b/pkg/sql/opt/memo/testdata/logprops/set index 3709cc79d7ca..083a16ad932b 100644 --- a/pkg/sql/opt/memo/testdata/logprops/set +++ b/pkg/sql/opt/memo/testdata/logprops/set @@ -48,7 +48,8 @@ intersect ├── columns: x:1(int!null) y:2(int) x:1(int!null) ├── left columns: x:1(int!null) y:2(int) x:1(int!null) ├── right columns: v:6(int) u:5(int) rowid:7(int) - ├── key: (1,2) + ├── key: (1) + ├── fd: ()-->(2) ├── interesting orderings: (+1) ├── project │ ├── columns: x:1(int!null) y:2(int) @@ -92,7 +93,8 @@ except ├── columns: x:1(int!null) x:1(int!null) y:2(int) ├── left columns: x:1(int!null) x:1(int!null) y:2(int) ├── right columns: u:5(int) v:6(int) v:6(int) - ├── key: (1,2) + ├── key: (1) + ├── fd: (1)-->(2) ├── interesting orderings: (+1) ├── project │ ├── columns: x:1(int!null) y:2(int) @@ -489,7 +491,8 @@ intersect-all ├── columns: a:1(int!null) b:2(int) c:3(int) ├── left columns: a:1(int!null) b:2(int) c:3(int) ├── right columns: a:6(int) b:7(int) c:8(int) - ├── fd: (1)==(3), (3)==(1) + ├── key: (1) + ├── fd: (1)==(2,3), (3)==(1,2), (2)==(1,3) ├── interesting orderings: (+1) ├── select │ ├── columns: a:1(int!null) b:2(int) c:3(int!null) @@ -541,7 +544,8 @@ except-all ├── columns: a:1(int!null) b:2(int) c:3(int) ├── left columns: a:1(int!null) b:2(int) c:3(int) ├── right columns: a:6(int) b:7(int) c:8(int) - ├── fd: (1)==(3), (3)==(1) + ├── key: (1) + ├── fd: (1)-->(2), (1)==(3), (3)==(1) ├── interesting orderings: (+1) ├── select │ ├── columns: a:1(int!null) b:2(int) c:3(int!null) diff --git a/pkg/sql/opt/memo/testdata/stats/set b/pkg/sql/opt/memo/testdata/stats/set index 6cae19f50382..1d68e2d12ae0 100644 --- a/pkg/sql/opt/memo/testdata/stats/set +++ b/pkg/sql/opt/memo/testdata/stats/set @@ -241,7 +241,8 @@ intersect ├── left columns: a.x:1(int!null) y:2(int) a.x:1(int!null) ├── right columns: z:7(int) b.x:6(int) rowid:9(int) ├── stats: [rows=2, distinct(1,2)=2, null(1,2)=0] - ├── key: (1,2) + ├── key: (1) + ├── fd: ()-->(2) ├── project │ ├── columns: a.x:1(int!null) y:2(int) │ ├── stats: [rows=5000, distinct(1,2)=5000, null(1,2)=0] @@ -283,6 +284,8 @@ intersect-all ├── left columns: a.x:1(int!null) y:2(int) a.x:1(int!null) ├── right columns: z:7(int) b.x:6(int) rowid:9(int) ├── stats: [rows=2] + ├── key: (1) + ├── fd: ()-->(2) ├── project │ ├── columns: a.x:1(int!null) y:2(int) │ ├── stats: [rows=5000] @@ -401,7 +404,8 @@ except ├── left columns: a.x:1(int!null) a.x:1(int!null) y:2(int) ├── right columns: b.x:6(int) z:7(int) z:7(int) ├── stats: [rows=5000, distinct(1,2)=5000, null(1,2)=0] - ├── key: (1,2) + ├── key: (1) + ├── fd: (1)-->(2) ├── project │ ├── columns: a.x:1(int!null) y:2(int) │ ├── stats: [rows=5000, distinct(1,2)=5000, null(1,2)=0] @@ -441,6 +445,8 @@ except-all ├── left columns: a.x:1(int!null) a.x:1(int!null) y:2(int) ├── right columns: b.x:6(int) z:7(int) z:7(int) ├── stats: [rows=5000] + ├── key: (1) + ├── fd: (1)-->(2) ├── project │ ├── columns: a.x:1(int!null) y:2(int) │ ├── stats: [rows=5000] @@ -878,9 +884,10 @@ except ├── columns: a:1(int) b:2(bool!null) ├── left columns: column1:1(int) column2:2(bool!null) ├── right columns: column1:3(int) column2:4(bool) - ├── cardinality: [0 - 2] + ├── cardinality: [0 - 1] ├── stats: [rows=1, distinct(1,2)=1, null(1,2)=0, distinct(2,5)=1, null(2,5)=0] - ├── key: (1,2) + ├── key: () + ├── fd: ()-->(1,2) ├── select │ ├── columns: column1:1(int) column2:2(bool!null) │ ├── cardinality: [0 - 2] @@ -922,7 +929,8 @@ except ├── right columns: column1:3(int) column2:4(int) ├── cardinality: [0 - 3] ├── stats: [rows=2, distinct(1,2)=2, null(1,2)=0.666666667] - ├── key: (1,2) + ├── key: (2) + ├── fd: ()-->(1) ├── select │ ├── columns: column1:1(int) column2:2(int) │ ├── cardinality: [0 - 3] @@ -956,29 +964,24 @@ except opt disable=SimplifyIntersectRight VALUES (1), (2) INTERSECT VALUES (NULL) ORDER BY 1 ---- -sort +intersect ├── columns: column1:1(int) + ├── left columns: column1:1(int) + ├── right columns: column1:2(int) ├── cardinality: [0 - 1] ├── stats: [rows=1, distinct(1)=1, null(1)=0] - ├── key: (1) - ├── ordering: +1 - └── intersect - ├── columns: column1:1(int) - ├── left columns: column1:1(int) - ├── right columns: column1:2(int) - ├── cardinality: [0 - 1] - ├── stats: [rows=1, distinct(1)=1, null(1)=0] - ├── key: (1) - ├── values - │ ├── columns: column1:1(int!null) - │ ├── cardinality: [2 - 2] - │ ├── stats: [rows=2, distinct(1)=2, null(1)=0] - │ ├── (1,) [type=tuple{int}] - │ └── (2,) [type=tuple{int}] - └── values - ├── columns: column1:2(int) - ├── cardinality: [1 - 1] - ├── stats: [rows=1, distinct(2)=1, null(2)=1] - ├── key: () - ├── fd: ()-->(2) - └── (NULL,) [type=tuple{int}] + ├── key: () + ├── fd: ()-->(1) + ├── values + │ ├── columns: column1:1(int!null) + │ ├── cardinality: [2 - 2] + │ ├── stats: [rows=2, distinct(1)=2, null(1)=0] + │ ├── (1,) [type=tuple{int}] + │ └── (2,) [type=tuple{int}] + └── values + ├── columns: column1:2(int) + ├── cardinality: [1 - 1] + ├── stats: [rows=1, distinct(2)=1, null(2)=1] + ├── key: () + ├── fd: ()-->(2) + └── (NULL,) [type=tuple{int}] diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpcc b/pkg/sql/opt/memo/testdata/stats_quality/tpcc index 827ab0874eed..73e6a5c6c1a2 100644 --- a/pkg/sql/opt/memo/testdata/stats_quality/tpcc +++ b/pkg/sql/opt/memo/testdata/stats_quality/tpcc @@ -1177,6 +1177,7 @@ except-all ├── left columns: no_w_id:3(int!null) no_d_id:2(int!null) no_o_id:1(int!null) ├── right columns: o_w_id:8(int) o_d_id:7(int) o_id:6(int) ├── stats: [rows=90000, distinct(1)=900, null(1)=0, distinct(2)=10, null(2)=0, distinct(3)=10, null(3)=0] + ├── key: (1-3) ├── scan new_order │ ├── save-table-name: consistency_08_scan_2 │ ├── columns: no_o_id:1(int!null) no_d_id:2(int!null) no_w_id:3(int!null) @@ -1290,6 +1291,7 @@ except-all ├── left columns: o_w_id:3(int!null) o_d_id:2(int!null) o_id:1(int!null) ├── right columns: no_w_id:13(int) no_d_id:12(int) no_o_id:11(int) ├── stats: [rows=90000, distinct(1)=2999, null(1)=0, distinct(2)=10, null(2)=0, distinct(3)=10, null(3)=0] + ├── key: (1-3) ├── project │ ├── save-table-name: consistency_09_project_2 │ ├── columns: o_id:1(int!null) o_d_id:2(int!null) o_w_id:3(int!null) @@ -1412,6 +1414,8 @@ except-all ├── left columns: o_w_id:3(int!null) o_d_id:2(int!null) o_id:1(int!null) o_ol_cnt:7(int) ├── right columns: ol_w_id:13(int) ol_d_id:12(int) ol_o_id:11(int) count_rows:23(int) ├── stats: [rows=300000, distinct(1)=2999, null(1)=0, distinct(2)=10, null(2)=0, distinct(3)=10, null(3)=0, distinct(7)=11, null(7)=0] + ├── key: (1-3) + ├── fd: (1-3)-->(7) ├── scan order │ ├── save-table-name: consistency_10_scan_2 │ ├── columns: o_id:1(int!null) o_d_id:2(int!null) o_w_id:3(int!null) o_ol_cnt:7(int) @@ -1521,6 +1525,8 @@ except-all ├── left columns: ol_w_id:3(int!null) ol_d_id:2(int!null) ol_o_id:1(int!null) count_rows:13(int) ├── right columns: o_w_id:16(int) o_d_id:15(int) o_id:14(int) o_ol_cnt:20(int) ├── stats: [rows=295745, distinct(1)=2999, null(1)=0, distinct(2)=10, null(2)=0, distinct(3)=10, null(3)=0, distinct(13)=295745, null(13)=0] + ├── key: (1-3) + ├── fd: (1-3)-->(13) ├── group-by │ ├── save-table-name: consistency_11_group_by_2 │ ├── columns: ol_o_id:1(int!null) ol_d_id:2(int!null) ol_w_id:3(int!null) count_rows:13(int!null) diff --git a/pkg/sql/opt/norm/testdata/rules/select b/pkg/sql/opt/norm/testdata/rules/select index 553ad989f801..00c74533a6fc 100644 --- a/pkg/sql/opt/norm/testdata/rules/select +++ b/pkg/sql/opt/norm/testdata/rules/select @@ -1525,6 +1525,7 @@ except-all ├── left columns: b.k:1!null ├── right columns: a.i:9 ├── cardinality: [0 - 8] + ├── key: (1) ├── select │ ├── columns: b.k:1!null │ ├── cardinality: [0 - 8] @@ -1553,6 +1554,7 @@ except-all ├── left columns: b.k:1!null ├── right columns: a.i:9 ├── cardinality: [0 - 8] + ├── key: (1) ├── select │ ├── columns: b.k:1!null │ ├── cardinality: [0 - 8] @@ -1581,6 +1583,7 @@ intersect-all ├── left columns: b.k:1!null ├── right columns: a.i:9 ├── cardinality: [0 - 8] + ├── key: (1) ├── select │ ├── columns: b.k:1!null │ ├── cardinality: [0 - 8] @@ -1609,6 +1612,7 @@ intersect-all ├── left columns: b.k:1!null ├── right columns: a.i:9 ├── cardinality: [0 - 8] + ├── key: (1) ├── select │ ├── columns: b.k:1!null │ ├── cardinality: [0 - 8] @@ -1871,6 +1875,8 @@ except-all ├── right columns: column1:3 column2:4 ├── cardinality: [0 - 1] ├── immutable + ├── key: () + ├── fd: ()-->(1,2) ├── values │ ├── columns: column1:1!null column2:2!null │ ├── cardinality: [1 - 1] @@ -1900,11 +1906,15 @@ select ├── columns: column1:1!null ├── cardinality: [0 - 1] ├── immutable + ├── key: () + ├── fd: ()-->(1) ├── except-all │ ├── columns: column1:1!null │ ├── left columns: column1:1!null │ ├── right columns: column1:2 │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(1) │ ├── values │ │ ├── columns: column1:1!null │ │ ├── cardinality: [1 - 1] @@ -1929,6 +1939,8 @@ except-all ├── left columns: column1:1!null ├── right columns: column1:2 ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(1) ├── values │ ├── columns: column1:1!null │ ├── cardinality: [1 - 1] diff --git a/pkg/sql/opt/norm/testdata/rules/set b/pkg/sql/opt/norm/testdata/rules/set index 4879a8bf6909..138f07278c56 100644 --- a/pkg/sql/opt/norm/testdata/rules/set +++ b/pkg/sql/opt/norm/testdata/rules/set @@ -185,6 +185,7 @@ except-all ├── columns: k:1 ├── left columns: k:1 ├── right columns: i:9 + ├── key: (1) ├── scan b │ ├── columns: k:1!null │ └── key: (1) @@ -223,6 +224,7 @@ intersect-all ├── columns: k:1 ├── left columns: k:1 ├── right columns: i:9 + ├── key: (1) ├── scan b │ ├── columns: k:1!null │ └── key: (1) @@ -255,6 +257,7 @@ intersect-all ├── columns: k:1!null ├── left columns: k:1!null ├── right columns: k:8 + ├── key: (1) ├── scan b │ ├── columns: k:1!null │ └── key: (1) diff --git a/pkg/sql/opt/props/func_dep.go b/pkg/sql/opt/props/func_dep.go index 379e17dfd405..ab90c0b9eb41 100644 --- a/pkg/sql/opt/props/func_dep.go +++ b/pkg/sql/opt/props/func_dep.go @@ -551,6 +551,23 @@ func (f *FuncDepSet) CopyFrom(fdset *FuncDepSet) { f.hasKey = fdset.hasKey } +// RemapFrom copies the given FD into this FD, remapping column IDs according to +// the given function. +func (f *FuncDepSet) RemapFrom(fdset *FuncDepSet, remap func(opt.ColumnID) opt.ColumnID) { + f.CopyFrom(fdset) + remapColSet := func(in opt.ColSet) (out opt.ColSet) { + for c, ok := in.Next(0); ok; c, ok = in.Next(c + 1) { + out.Add(remap(c)) + } + return out + } + for i := range f.deps { + f.deps[i].from = remapColSet(f.deps[i].from) + f.deps[i].to = remapColSet(f.deps[i].to) + } + f.key = remapColSet(f.key) +} + // ColsAreStrictKey returns true if the given columns contain a strict key for the // relation. This means that any two rows in the relation will never have the // same values for this set of columns. If the columns are nullable, then at diff --git a/pkg/sql/opt/props/func_dep_test.go b/pkg/sql/opt/props/func_dep_test.go index 96f2f668ae85..e86bf8fe6cc7 100644 --- a/pkg/sql/opt/props/func_dep_test.go +++ b/pkg/sql/opt/props/func_dep_test.go @@ -1224,6 +1224,16 @@ func TestFuncDeps_MakeFullOuter(t *testing.T) { verifyFD(t, outer, "") } +func TestFuncDeps_RemapFrom(t *testing.T) { + var res props.FuncDepSet + abcde := makeAbcdeFD(t) + mnpq := makeMnpqFD(t) + res.RemapFrom(abcde, func(in opt.ColumnID) opt.ColumnID { return in + 100 }) + verifyFD(t, &res, "ey(101); (101)-->(102-105), (102,103)~~>(101,104,105)") + res.RemapFrom(mnpq, func(in opt.ColumnID) opt.ColumnID { return in + 100 }) + verifyFD(t, &res, "key(110,111); (110,111)-->(112,113)") +} + // Construct base table FD from figure 3.3, page 114: // CREATE TABLE abcde (a INT PRIMARY KEY, b INT, c INT, d INT, e INT) // CREATE UNIQUE INDEX ON abcde (b, c) diff --git a/pkg/sql/opt/xform/testdata/coster/set b/pkg/sql/opt/xform/testdata/coster/set index 291cd39d26a2..522dde0d0242 100644 --- a/pkg/sql/opt/xform/testdata/coster/set +++ b/pkg/sql/opt/xform/testdata/coster/set @@ -56,6 +56,8 @@ intersect-all ├── right columns: x:7 z:8 ├── stats: [rows=1000] ├── cost: 2169.13 + ├── key: (1) + ├── fd: (1)-->(2) ├── scan a │ ├── columns: k:1!null i:2 │ ├── stats: [rows=1000] @@ -76,6 +78,8 @@ intersect-all ├── right columns: x:7 z:8 ├── stats: [rows=1000] ├── cost: 2169.13 + ├── key: (1) + ├── fd: (1)-->(2) ├── scan a │ ├── columns: k:1!null i:2 │ ├── stats: [rows=1000] @@ -96,6 +100,8 @@ except-all ├── right columns: x:7 z:8 ├── stats: [rows=1000] ├── cost: 2169.13 + ├── key: (1) + ├── fd: (1)-->(2) ├── scan a │ ├── columns: k:1!null i:2 │ ├── stats: [rows=1000] @@ -116,6 +122,8 @@ except-all ├── right columns: x:7 z:8 ├── stats: [rows=1000] ├── cost: 2169.13 + ├── key: (1) + ├── fd: (1)-->(2) ├── scan a │ ├── columns: k:1!null i:2 │ ├── stats: [rows=1000] diff --git a/pkg/sql/opt/xform/testdata/external/tpcc b/pkg/sql/opt/xform/testdata/external/tpcc index 22af3ea04843..495f3076bee3 100644 --- a/pkg/sql/opt/xform/testdata/external/tpcc +++ b/pkg/sql/opt/xform/testdata/external/tpcc @@ -1243,6 +1243,7 @@ except-all ├── columns: no_w_id:3!null no_d_id:2!null no_o_id:1!null ├── left columns: no_w_id:3!null no_d_id:2!null no_o_id:1!null ├── right columns: o_w_id:8 o_d_id:7 o_id:6 + ├── key: (1-3) ├── scan new_order │ ├── columns: no_o_id:1!null no_d_id:2!null no_w_id:3!null │ └── key: (1-3) @@ -1269,6 +1270,7 @@ except-all ├── columns: o_w_id:3!null o_d_id:2!null o_id:1!null ├── left columns: o_w_id:3!null o_d_id:2!null o_id:1!null ├── right columns: no_w_id:13 no_d_id:12 no_o_id:11 + ├── key: (1-3) ├── project │ ├── columns: o_id:1!null o_d_id:2!null o_w_id:3!null │ ├── key: (1-3) @@ -1304,6 +1306,8 @@ except-all ├── columns: o_w_id:3!null o_d_id:2!null o_id:1!null o_ol_cnt:7 ├── left columns: o_w_id:3!null o_d_id:2!null o_id:1!null o_ol_cnt:7 ├── right columns: ol_w_id:13 ol_d_id:12 ol_o_id:11 count_rows:23 + ├── key: (1-3) + ├── fd: (1-3)-->(7) ├── scan order │ ├── columns: o_id:1!null o_d_id:2!null o_w_id:3!null o_ol_cnt:7 │ ├── key: (1-3) @@ -1338,6 +1342,8 @@ except-all ├── columns: ol_w_id:3!null ol_d_id:2!null ol_o_id:1!null count:13 ├── left columns: ol_w_id:3!null ol_d_id:2!null ol_o_id:1!null count_rows:13 ├── right columns: o_w_id:16 o_d_id:15 o_id:14 o_ol_cnt:20 + ├── key: (1-3) + ├── fd: (1-3)-->(13) ├── group-by │ ├── columns: ol_o_id:1!null ol_d_id:2!null ol_w_id:3!null count_rows:13!null │ ├── grouping columns: ol_o_id:1!null ol_d_id:2!null ol_w_id:3!null diff --git a/pkg/sql/opt/xform/testdata/external/tpcc-later-stats b/pkg/sql/opt/xform/testdata/external/tpcc-later-stats index cddc0cddb0c0..1bfb52f71b3d 100644 --- a/pkg/sql/opt/xform/testdata/external/tpcc-later-stats +++ b/pkg/sql/opt/xform/testdata/external/tpcc-later-stats @@ -1245,6 +1245,7 @@ except-all ├── columns: no_w_id:3!null no_d_id:2!null no_o_id:1!null ├── left columns: no_w_id:3!null no_d_id:2!null no_o_id:1!null ├── right columns: o_w_id:8 o_d_id:7 o_id:6 + ├── key: (1-3) ├── scan new_order │ ├── columns: no_o_id:1!null no_d_id:2!null no_w_id:3!null │ └── key: (1-3) @@ -1271,6 +1272,7 @@ except-all ├── columns: o_w_id:3!null o_d_id:2!null o_id:1!null ├── left columns: o_w_id:3!null o_d_id:2!null o_id:1!null ├── right columns: no_w_id:13 no_d_id:12 no_o_id:11 + ├── key: (1-3) ├── project │ ├── columns: o_id:1!null o_d_id:2!null o_w_id:3!null │ ├── key: (1-3) @@ -1306,6 +1308,8 @@ except-all ├── columns: o_w_id:3!null o_d_id:2!null o_id:1!null o_ol_cnt:7 ├── left columns: o_w_id:3!null o_d_id:2!null o_id:1!null o_ol_cnt:7 ├── right columns: ol_w_id:13 ol_d_id:12 ol_o_id:11 count_rows:23 + ├── key: (1-3) + ├── fd: (1-3)-->(7) ├── scan order │ ├── columns: o_id:1!null o_d_id:2!null o_w_id:3!null o_ol_cnt:7 │ ├── key: (1-3) @@ -1340,6 +1344,8 @@ except-all ├── columns: ol_w_id:3!null ol_d_id:2!null ol_o_id:1!null count:13 ├── left columns: ol_w_id:3!null ol_d_id:2!null ol_o_id:1!null count_rows:13 ├── right columns: o_w_id:16 o_d_id:15 o_id:14 o_ol_cnt:20 + ├── key: (1-3) + ├── fd: (1-3)-->(13) ├── group-by │ ├── columns: ol_o_id:1!null ol_d_id:2!null ol_w_id:3!null count_rows:13!null │ ├── grouping columns: ol_o_id:1!null ol_d_id:2!null ol_w_id:3!null diff --git a/pkg/sql/opt/xform/testdata/external/tpcc-no-stats b/pkg/sql/opt/xform/testdata/external/tpcc-no-stats index 89a9b43efe75..bb9d76d32a1d 100644 --- a/pkg/sql/opt/xform/testdata/external/tpcc-no-stats +++ b/pkg/sql/opt/xform/testdata/external/tpcc-no-stats @@ -1239,6 +1239,7 @@ except-all ├── columns: no_w_id:3!null no_d_id:2!null no_o_id:1!null ├── left columns: no_w_id:3!null no_d_id:2!null no_o_id:1!null ├── right columns: o_w_id:8 o_d_id:7 o_id:6 + ├── key: (1-3) ├── scan new_order │ ├── columns: no_o_id:1!null no_d_id:2!null no_w_id:3!null │ └── key: (1-3) @@ -1265,6 +1266,7 @@ except-all ├── columns: o_w_id:3!null o_d_id:2!null o_id:1!null ├── left columns: o_w_id:3!null o_d_id:2!null o_id:1!null ├── right columns: no_w_id:13 no_d_id:12 no_o_id:11 + ├── key: (1-3) ├── project │ ├── columns: o_id:1!null o_d_id:2!null o_w_id:3!null │ ├── key: (1-3) @@ -1300,6 +1302,8 @@ except-all ├── columns: o_w_id:3!null o_d_id:2!null o_id:1!null o_ol_cnt:7 ├── left columns: o_w_id:3!null o_d_id:2!null o_id:1!null o_ol_cnt:7 ├── right columns: ol_w_id:13 ol_d_id:12 ol_o_id:11 count_rows:23 + ├── key: (1-3) + ├── fd: (1-3)-->(7) ├── scan order │ ├── columns: o_id:1!null o_d_id:2!null o_w_id:3!null o_ol_cnt:7 │ ├── key: (1-3) @@ -1334,6 +1338,8 @@ except-all ├── columns: ol_w_id:3!null ol_d_id:2!null ol_o_id:1!null count:13 ├── left columns: ol_w_id:3!null ol_d_id:2!null ol_o_id:1!null count_rows:13 ├── right columns: o_w_id:16 o_d_id:15 o_id:14 o_ol_cnt:20 + ├── key: (1-3) + ├── fd: (1-3)-->(13) ├── group-by │ ├── columns: ol_o_id:1!null ol_d_id:2!null ol_w_id:3!null count_rows:13!null │ ├── grouping columns: ol_o_id:1!null ol_d_id:2!null ol_w_id:3!null diff --git a/pkg/sql/opt/xform/testdata/physprops/ordering b/pkg/sql/opt/xform/testdata/physprops/ordering index c1dae9ece593..ca15d1fc8b39 100644 --- a/pkg/sql/opt/xform/testdata/physprops/ordering +++ b/pkg/sql/opt/xform/testdata/physprops/ordering @@ -1558,6 +1558,7 @@ intersect-all ├── columns: a:1!null b:2!null c:3!null ├── left columns: a:1!null b:2!null c:3!null ├── right columns: y:7 x:6 z:8 + ├── key: (2,3) ├── fd: (1)==(2), (2)==(1) ├── ordering: +(1|2) [actual: +1] ├── select @@ -1666,7 +1667,8 @@ intersect-all ├── left columns: column1:1!null ├── right columns: column1:2 ├── cardinality: [0 - 1] - ├── ordering: +1 + ├── key: () + ├── fd: ()-->(1) ├── values │ ├── columns: column1:1!null │ ├── cardinality: [1 - 1]