From 80b09da818f914641fc3fd1dca6a7c2e35eef28b Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Wed, 21 Apr 2021 19:35:40 +1000 Subject: [PATCH 1/5] sql: disallow RBR transforms if transitioning indexes are incompatible Release note (bug fix): Previously, if a DROP INDEX failed during a REGIONAL BY ROW transition, the index may be re-inserted back into the REGIONAL BY ROW table but would be invalid if it was sharded or partitioned. This is now rectified. --- .../logictestccl/testdata/logic_test/alter_table_locality | 2 +- pkg/sql/alter_database.go | 2 +- pkg/sql/alter_table_locality.go | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/ccl/logictestccl/testdata/logic_test/alter_table_locality b/pkg/ccl/logictestccl/testdata/logic_test/alter_table_locality index 12e4c218598d..149aec6574d6 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/alter_table_locality +++ b/pkg/ccl/logictestccl/testdata/logic_test/alter_table_locality @@ -2625,7 +2625,7 @@ CREATE TABLE hash_sharded_idx_table ( pk INT PRIMARY KEY USING HASH WITH BUCKET_COUNT = 8 ) -statement error cannot convert a table to REGIONAL BY ROW if table table contains hash sharded indexes +statement error cannot convert hash_sharded_idx_table to REGIONAL BY ROW as the table contains hash sharded indexes ALTER TABLE hash_sharded_idx_table SET LOCALITY REGIONAL BY ROW statement ok diff --git a/pkg/sql/alter_database.go b/pkg/sql/alter_database.go index c8e0a6522f34..3b0dc4d9c84a 100644 --- a/pkg/sql/alter_database.go +++ b/pkg/sql/alter_database.go @@ -746,7 +746,7 @@ func checkCanConvertTableToMultiRegion( tableDesc.GetName(), ) } - for _, idx := range tableDesc.NonDropIndexes() { + for _, idx := range tableDesc.AllIndexes() { if idx.GetPartitioning().NumColumns > 0 { return errors.WithDetailf( pgerror.Newf( diff --git a/pkg/sql/alter_table_locality.go b/pkg/sql/alter_table_locality.go index 43f7fb9d6b12..1c68cd13df6a 100644 --- a/pkg/sql/alter_table_locality.go +++ b/pkg/sql/alter_table_locality.go @@ -240,9 +240,13 @@ func (n *alterTableSetLocalityNode) alterTableLocalityToRegionalByRow( return interleaveOnRegionalByRowError() } - for _, idx := range n.tableDesc.NonDropIndexes() { + for _, idx := range n.tableDesc.AllIndexes() { if idx.IsSharded() { - return pgerror.New(pgcode.FeatureNotSupported, "cannot convert a table to REGIONAL BY ROW if table table contains hash sharded indexes") + return pgerror.Newf( + pgcode.FeatureNotSupported, + "cannot convert %s to REGIONAL BY ROW as the table contains hash sharded indexes", + tree.Name(n.tableDesc.GetName()), + ) } } From 3d40e6ca3b83d8438d1b23763d7c7d60c297e575 Mon Sep 17 00:00:00 2001 From: Andrew Werner Date: Tue, 20 Apr 2021 11:08:46 -0400 Subject: [PATCH 2/5] scripts/gceworker.sh: always populate username In the previous change to this file we were to allow injecting a username. The code before this commit, the code would populate the username as "" leading to a leading `@` in the address. The `ssh` family of commands are happy enough with this but the unison command is not. Before this change, we'd see: ``` Fatal error: ill-formed root specification ssh://@gceworker-ajwerner.us-east1-b.cockroach-workers/go/src/github.com/cockroachdb/cockroach ``` I'm content doing something different here like detecting whether we have a username injected instead. I don't have a strong reason to do that but maybe by explicitly making the user the output of `whoami` I'm preventing some other customization. Release note: None --- scripts/gceworker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gceworker.sh b/scripts/gceworker.sh index d9739302dda9..646ac2b2a71b 100755 --- a/scripts/gceworker.sh +++ b/scripts/gceworker.sh @@ -8,7 +8,7 @@ source build/shlib.sh export CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT-${GCEWORKER_PROJECT-cockroach-workers}} export CLOUDSDK_COMPUTE_ZONE=${GCEWORKER_ZONE-${CLOUDSDK_COMPUTE_ZONE-us-east1-b}} INSTANCE=${GCEWORKER_NAME-gceworker-$(id -un)} -SSH_USER=${GCEWORKER_USER:-""} +SSH_USER=${GCEWORKER_USER:-"$(id -un)"} cmd=${1-} if [[ "${cmd}" ]]; then From f4daa508796f0e330e85a70f09f4df9ffa5f2bbf Mon Sep 17 00:00:00 2001 From: Rebecca Taft Date: Fri, 16 Apr 2021 17:12:25 -0500 Subject: [PATCH 3/5] opt,sql: support streaming set ops This commit adds support for planning streaming set operations in the optimizer. Streaming set operations consist of a UNION, INTERSECT, or EXCEPT (including ALL variants) in which both inputs are ordered on the desired output ordering, and the set operation merges the inputs to maintain the ordering. The execution engine already supported this type of streaming set operation, so the purpose of this commit was to teach the optimizer about this capability and hook it up to the execution engine. This is also a prerequisite for #56201. Fixes #40797 Informs #56201 Release note (performance improvement): Set operations (UNION, UNION ALL, INTERSECT, INTERSECT ALL, EXCEPT and EXCEPT ALL) can now maintain ordering if both inputs are ordered on the desired output ordering. This can eliminate unnecessary sort operations and improve performance. --- pkg/sql/distsql_physical_planner.go | 16 +- pkg/sql/distsql_spec_exec_factory.go | 6 +- .../inverted_filter_json_array_dist | 110 +-- pkg/sql/logictest/testdata/logic_test/union | 145 ++++ pkg/sql/opt/exec/execbuilder/relational.go | 19 +- .../exec/execbuilder/testdata/distsql_union | 63 +- .../testdata/inverted_filter_json_array | 64 +- pkg/sql/opt/exec/execbuilder/testdata/limit | 156 +++-- pkg/sql/opt/exec/execbuilder/testdata/union | 561 ++++++++++++++++ pkg/sql/opt/exec/factory.opt | 9 + pkg/sql/opt/memo/testdata/stats/set | 39 +- pkg/sql/opt/optbuilder/testdata/limit | 60 +- pkg/sql/opt/optbuilder/testdata/union | 196 +++--- pkg/sql/opt/ordering/BUILD.bazel | 1 + pkg/sql/opt/ordering/ordering.go | 30 + pkg/sql/opt/ordering/set.go | 130 ++++ pkg/sql/opt/props/physical/ordering_choice.go | 16 + pkg/sql/opt/xform/testdata/external/trading | 103 +-- .../xform/testdata/external/trading-mutation | 103 +-- pkg/sql/opt/xform/testdata/physprops/ordering | 217 ++++++ pkg/sql/opt/xform/testdata/rules/limit | 624 +++++++++--------- pkg/sql/opt/xform/testdata/rules/select | 46 +- pkg/sql/opt_exec_factory.go | 10 +- pkg/sql/plan_ordering.go | 2 +- pkg/sql/union.go | 23 +- 25 files changed, 1999 insertions(+), 750 deletions(-) create mode 100644 pkg/sql/opt/ordering/set.go diff --git a/pkg/sql/distsql_physical_planner.go b/pkg/sql/distsql_physical_planner.go index 30ae83a8de79..d95f22f155f5 100644 --- a/pkg/sql/distsql_physical_planner.go +++ b/pkg/sql/distsql_physical_planner.go @@ -3331,15 +3331,8 @@ func (dsp *DistSQLPlanner) createPlanForSetOp( return nil, err } - if len(leftPlan.MergeOrdering.Columns) != 0 || len(rightPlan.MergeOrdering.Columns) != 0 { - return nil, errors.AssertionFailedf("set op inputs should have no orderings") - } - - // TODO(radu): for INTERSECT and EXCEPT, the mergeOrdering should be set when - // we can use merge joiners below. The optimizer needs to be modified to take - // advantage of this optimization and pass down merge orderings. Tracked by - // #40797. - var mergeOrdering execinfrapb.Ordering + // Set the merge ordering. + mergeOrdering := dsp.convertOrdering(n.reqOrdering, p.PlanToStreamColMap) // Merge processors, streams, result routers, and stage counter. leftRouters := leftPlan.ResultRouters @@ -3479,11 +3472,6 @@ func (dsp *DistSQLPlanner) createPlanForSetOp( ) } - // An EXCEPT ALL is like a left outer join, so there is no guaranteed ordering. - if n.unionType == tree.ExceptOp { - mergeOrdering = execinfrapb.Ordering{} - } - p.SetMergeOrdering(mergeOrdering) } diff --git a/pkg/sql/distsql_spec_exec_factory.go b/pkg/sql/distsql_spec_exec_factory.go index cc227e87d866..bb0dfd33028a 100644 --- a/pkg/sql/distsql_spec_exec_factory.go +++ b/pkg/sql/distsql_spec_exec_factory.go @@ -592,7 +592,11 @@ func (e *distSQLSpecExecFactory) ConstructDistinct( } func (e *distSQLSpecExecFactory) ConstructSetOp( - typ tree.UnionType, all bool, left, right exec.Node, hardLimit uint64, + typ tree.UnionType, + all bool, + left, right exec.Node, + reqOrdering exec.OutputOrdering, + hardLimit uint64, ) (exec.Node, error) { return nil, unimplemented.NewWithIssue(47473, "experimental opt-driven distsql planning: set op") } diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist index 0fdaa809f51c..bf82eed56d9c 100644 --- a/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist +++ b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist @@ -295,25 +295,25 @@ vectorized: true │ distinct on: a │ order key: a │ -└── • sort - │ order: +a +└── • union all │ - └── • union all - │ - ├── • index join - │ │ table: json_tab@primary - │ │ - │ └── • scan - │ missing stats - │ table: json_tab@json_inv - │ spans: 1 span - │ - └── • scan - missing stats - table: json_tab@primary - spans: [/44 - /44] + ├── • index join + │ │ table: json_tab@primary + │ │ + │ └── • sort + │ │ order: +a + │ │ + │ └── • scan + │ missing stats + │ table: json_tab@json_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: json_tab@primary + spans: [/44 - /44] · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy8lF9v2jwUxu_fT2Gdm7Z6XSVOArSRJtGtmcbEgAWmbeqiyRCvzUTjzDZTK8R3n5Iw_hY7FNYbZOL8znns8zyZgvw1Bh-CL732VauDTq9b_UH_Y_sM9YN28GaAKHobdj-gn5Kn3xUdos_vgjBAp0PU_DaxbZehkxsSnZyhbohOKXqFPC9fXwchev0VUcCQ8ph16D2T4N8AAQwOYHAhwpAJPmJScpFvTYsXW_ED-DaGJM0mKn8cYRhxwcCfgkrUmIEPAzocs5DRmAnLBgwxUzQZF-X_qmxmIrmn4hEw9DOaSh9ZnmfZ58Wv1fnUbgOG7kT5qElw04FohoFP1LKnVPSWgU9muLquPheKCauxLqlJ_t9Z3tun_HUiVZKOlEXszQ75YUTMBIvzA220W1YYPqI7Ku-26Gi2lFTbKWlZh5e9njqnVrdr0L0Yyc4Lqx9dXYef88wi3trru9o31tqT6jYlO2xaLJL094pPiW1dCWGR88XK6gn2I3kI0rjCDTn7SHzPk3Su0DUGqUJc3H2az-NSe25cDOUXtrt4sbSQo6aFHDktTnW7OtXt6izs6hxuV4PEFbt6x7erofncrvXn2tVQfjH3yxezq3NUuzr_8OP-ROOQyYynklX6btu5dBbfsvKokk_EiPUEHxVtyr_dgisexEyqcpeUf1ppuZULXIWJFnb0sLMJk1X4cg0m-8HEO4R29bCrPbShs6e_7pr-ympauq6H61q4oYcbWvhCD18cMmk9bJq0njZM-vKQSRNDskzR0meLGMJFtiy-jrsGfMvk-4zMQJtmZsANQyP6iG32jmb__QkAAP__5vuJ4A== +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy8VF1v0zAUfedXWPdlm_CU2EnbLRJSBwuiqLQjHQI0IuQ2Zgvq4mC7aNPU_46SjH6ttVNa9hI5cc49x_ee4wdQv8YQQPjlonvW6aHD887gcvCxe4QGYTd8c4kYehv1P6CfSmTfNRuiz-_CKESHQ9T-NnFdj6ODKxIfHKF-hA4ZeoV8v1ifhxF6_RUxwJCJhPfYLVcQXAEBDBQweBBjyKUYcaWELLYeyh87yR0ELoY0yye6-BxjGAnJIXgAneoxhwAu2XDMI84SLh0XMCRcs3Rclv-rsp3L9JbJe8AwyFmmAuT4vuMel0-n96nbBQz9iQ5Qm-A2hXiKQUz0nFNpds0hIFNcX9d5qnSajbRzuiyqXRy6LxMueVIQrrDNCwzv0Q1TN0_Q8XSuqLFR0byOqLhW67ysCm2UTahF96xlG_vV3Lu6njgWuUO8pd830beW6El9G5ENNioXafZ7wUfEdc6kdMjxbOVcSP4jvQuzpEaH6DYSB0JqLh1vXa_Wl_e2Kf9epNljAxrWHNVIi78N-cx2rWdLC9lrWtw9p4XWtyutb1c6syvd3a4WiY929f_VrpbyC3Zt7t-uFvLZ3E-eza50r3Yl__FyX0MccZWLTPFa97ZbSOfJNa-OqsREjviFFKOSpnrtl7jyQ8KVrnZJ9dLJqq1C4CKYrILJIvhkCUy2AxNvFzQ1g6nx0BZmz9wx39wy34humMENI7hpBjeN4JYZ3Npl0mawbdJmtGXSJ7tM-tQ8adcSDku0bNl64vBlOLXAn3h8q2ya0dZwmuGWmRFzwla54-mLPwEAAP__0V5WYQ== # We cannot use the index for this query. query T @@ -413,25 +413,25 @@ vectorized: true │ distinct on: a │ order key: a │ -└── • sort - │ order: +a +└── • union all │ - └── • union all - │ - ├── • index join - │ │ table: array_tab@primary - │ │ - │ └── • scan - │ missing stats - │ table: array_tab@arr_inv - │ spans: 1 span - │ - └── • scan - missing stats - table: array_tab@primary - spans: [/1 - /1] + ├── • index join + │ │ table: array_tab@primary + │ │ + │ └── • sort + │ │ order: +a + │ │ + │ └── • scan + │ missing stats + │ table: array_tab@arr_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: array_tab@primary + spans: [/1 - /1] · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0lF9r2zAUxd_3KcR9acsUbCl_ZxhkWzyWkSWdnbGNzRTFvhRDanmSPFZCvvuwXZo61KrTsRcj6ep3z5EO1g70ry144H-7XLyZL8n5bB6uw8-LCxL6C__dmgjyPlh9IkIpcXtlxIZ8_eAHPjnfkOnPwnX7SM52bH92QVYBORfkNWHlcOYH5O13IoBCJhNcihvU4P0ABhQ4UOhDRCFXMkatpSpLu2rjPPkDnkshzfLC1MsmNVsED4pMqgQVJkAhQSPSbVmP9hGFWCoE77B1KXsyd0ZHGynIwty1jShoI64RvMGetkg_0jiUyqByxo3OMGUvoa398JT2s1SbNIuNMzkWAAqr-vQeqWeFqUZtuqOGLrPrrsVmiwGKBJXDmtL3uU-FUldp9hsohLnItEcc5jqsV355B0P8FEMfZZrd-em3-clVeiPU7UGaTnmrer-hzrtfh_u0_P11OG6v_DjLL4tFN1vsubZ495R4lRJ_TkpPGHqQ0uA_p3QQfewRALA8A8NOz4B7yn8aoM5lprFj54gCJtdYn0PLQsV4qWRcydTTVcVVCwlqU1dZPZlndak0-BBmVpjbYW6FX9nhvhUe2OGBFXYbMDuGh1Z4YodHVnhstz3-F-XJSWeO9i_-BgAA____53O4 +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0lF9r2zAUxd_3KcR9acsUbNn5axhkWzyWkSWdk7GNzRTFvhRDanmSPFaCv_uwHdq6JIqTshdj_fndc6500BbU7w144H-_nr2dzsnlZLpcLb_MrsjSn_nvV4STD8HiM-FS8vsbzdfk20c_8Mnlmox_5bbtIrnYsuLiiiwCcsnJG8LK34kfkHc_CAcKqYhxzu9QgfcTGFBwgIILIYVMigiVErJc2lYbp_Ff8GwKSZrlup7Wid4geCBkjBJjoBCj5smmqjdmryEsQgqRkAje4-656IjMGjZ2hwUFketd5ZCC0vwWwesV9ID6nsKTROkkjbQ1eu4EKCxqjx6pR7mu_g7p9hu6zKy74usNBshjlBZrSj_czZhLeZOkf4DCMuOp8ojFbIt1yq_TwpBziqGlkBql5e67kP3l3VPKfxJJumu3d6jdTCZ3XN4_dkbHzkH1bkPdaX_a9nH5h9O27E75seZfZ7N2tti5tpz2IXCqEDjnhOCIoV0IuueG4Ej5JyHo_-cQvOyhGbR6aOxTHpoAVSZShS0rhxQwvsW6FSVyGeG1FFElUw8XFVdNxKh0vcrqwTStl0qDT2Fmhm0z7Rhp1wy7RrhrhrtGuGmbPYd7RnhkhvtGeGC2PTDCQzM8fInt0UkHFhav_gUAAP__CVynkw== # The split disjunction rule allows us to use the index for this query. query T @@ -445,29 +445,29 @@ vectorized: true │ distinct on: a │ order key: a │ -└── • sort - │ order: +a +└── • union all │ - └── • union all - │ - ├── • index join - │ │ table: array_tab@primary - │ │ - │ └── • inverted filter - │ │ inverted column: b_inverted_key - │ │ num spans: 1 - │ │ - │ └── • scan - │ missing stats - │ table: array_tab@arr_inv - │ spans: 1 span - │ - └── • scan - missing stats - table: array_tab@primary - spans: [/1 - /1] + ├── • index join + │ │ table: array_tab@primary + │ │ + │ └── • sort + │ │ order: +a + │ │ + │ └── • inverted filter + │ │ inverted column: b_inverted_key + │ │ num spans: 1 + │ │ + │ └── • scan + │ missing stats + │ table: array_tab@arr_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: array_tab@primary + spans: [/1 - /1] · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJzMVGFv2jAQ_b5fYd2Xtlqq5BwobaRJbGuqMTHooNM2DVQZcmoj0TizTdWq4r9PIQiaQhxA3bQvEfb53Xt397gn0L8nEED447L9vtVhh-et_lX_a_uI9cN2-PGKCXbR635hQinxeG3EiH3_FPZCdng4Ys3B1PN8YgdPODs4Yt0eK1zy2cFRfivYO4bZz_Owxz78ZAIcSGREHXFHGoJfgOAABwd8GDqQKjkmraXKQk_zh63oAQLPgThJpya_NrGZEAQwTaSKSFEEDkRkRDzJ4sPZ0IGxVATB6ul5rE2cjI1bL7yFZkbfnZqANRGGMwfk1Cx4hg5oI24IgtrMKdGygemzjJMeiYiUe1LkWvaxmar4TqjHFbXT5KXs9V3Y-1IZUm7jZZVvS9Of7JJ-2cazTW3Mh7GhlasEo0d2K_TtGno-tIWiRqmiVZ5Ng1-UaZONWKG72gynr66uI49l6qL_wsWb6c8K9Ggf15UYTWhhRiwzo1DqOk7uwYF-KhIdMBc9F4-zr7-dQ_kumlrJPSlD0UU8MaRIucW6l_HwIVVMJqyJAdOZMKaNUCYYwGDAEQfAKIkWJ38ArFSdX1DHt--YV_33XXbM9Y6zj9v51m5v1zTcVxbffpB8Pki-5yArNK0NsvY_DXKxCE_3XYQV6Vcbxftnm5C_6ibkf3ETbiDukU5lommrJedl0im6obxULadqTJdKjuc0-bE7x80vItImj2J-aCV5KBP4HIxWMLeDuRXs28H-SzA-B9cKYNwNjL4dXbPqrtvBdXu7a_aqT6zohh3csII9u-5Tu-4Kn5zZ0V6Fy-werVCOdpNihUtxzaa7OK0CXWU1XHOq1WvD2Zs_AQAA__8i-Qm8 +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJzMVF9v2j4Uff99iqv70la_VIkdKG2kSWxrqjEx6KDTNg1UGXLVRqJxZpuqVcV3n5IwKH_iQNVNe4liXx-f43uO_YT65wQDDL9dtt-2OnB43upf9T-3j6AftsP3VyDgotf9BEIp8XhtxAi-fgh7IRwejqA5mHqeT3DwxGYHR9Dtwcoknx0cFbMC3gDLfs_DHrz7DgIdTGREHXFHGoMfyNBBjg76OHQwVXJMWkuVlZ7yha3oAQPPwThJp6aYNrGZEAY4TaSKSFGEDkZkRDzJ6sPZ0MGxVITBcul5rE2cjI1bX1mLzYy-OzUBNBkOZw7KqZnzDB3URtwQBrWZU6JlC1NfKkPKPVnn-b90-_o-23-UcdIjEZFyG6sUC5uaqYrvhHpcnsxp8lL2k33YF2083dbGwowtrVxuMHqEW6FvN9C5aXNFjVJFy322GT9vsk028yp0V4fh9NXVdeSxTF3G11K8nf5shZ7Z7boSownN08LK0iKUuo6Te3Swn4pEB-Ayz2XH2dffLUJ8H02t5J6UoeginhhSpHKSZ8J-18OHVIFMoMkC0Jkw0EYoEwxwMOCMDRAoieYjf4BQqs5fUcd375hXfb8WHXO94-zjdr6027s1jb1UFt_dSJ4byV9oZIWmDSNr_5KRiyt_9tdeKv6qLxX7gy_VFuIe6VQmmnZ6hLxMOkU3VBxVy6ka06WS45ymGHZzXD4RkTZFlRWDVlKUMoHPwcwK5nYwt4J9O9hfB7Pn4NoKmO0HZtyOrll11-3gur3dFac-saIbdnDDCvbsuk_tuitycmZHexUps2e0QjnbCOk-UalAV2WF2UO-Hpbh7L9fAQAA___OJNYQ # We cannot use the index for this query. query T diff --git a/pkg/sql/logictest/testdata/logic_test/union b/pkg/sql/logictest/testdata/logic_test/union index 2a107d2af9c3..fc9debfa18fd 100644 --- a/pkg/sql/logictest/testdata/logic_test/union +++ b/pkg/sql/logictest/testdata/logic_test/union @@ -367,6 +367,9 @@ SELECT b FROM ab UNION SELECT a FROM ab 1 2 +statement ok +DROP TABLE ab; + # Regression test for the vectorized engine not being able to handle a UNION # between NULL and a tuple (#59611). statement ok @@ -398,3 +401,145 @@ CREATE TABLE t34524 (a INT PRIMARY KEY) query I (SELECT NULL FROM t34524) EXCEPT (VALUES((SELECT 1 FROM t34524 LIMIT 1)), (1)) ---- + +statement ok +CREATE TABLE ab (a INT PRIMARY KEY, b INT, INDEX (b, a)); +INSERT INTO ab VALUES + (1, 1), + (2, 2), + (3, 1), + (4, 2), + (5, 1), + (6, 2), + (7, 3) + +statement ok +CREATE TABLE xy (x INT PRIMARY KEY, y INT, INDEX (y, x)); +INSERT INTO xy VALUES + (1, 1), + (2, 3), + (3, 2), + (4, 3), + (5, 1), + (6, 3) + +# Regression tests for #41245, #40797. Ensure we can plan ordered set ops +# without a sort. +query II +SELECT a, b FROM ab UNION SELECT x AS a, y AS b FROM xy ORDER BY b, a +---- +1 1 +3 1 +5 1 +2 2 +3 2 +4 2 +6 2 +2 3 +4 3 +6 3 +7 3 + +query I +SELECT a FROM ab UNION ALL SELECT x AS a FROM xy ORDER BY a +---- +1 +1 +2 +2 +3 +3 +4 +4 +5 +5 +6 +6 +7 + +query II +SELECT a, b FROM ab INTERSECT SELECT x AS a, y AS b FROM xy ORDER BY b, a +---- +1 1 +5 1 + +query I +SELECT b FROM ab INTERSECT ALL SELECT y AS b FROM xy ORDER BY b +---- +1 +1 +2 +3 + +query II +SELECT b, a FROM ab EXCEPT SELECT y AS b, x AS a FROM xy ORDER BY b, a +---- +1 3 +2 2 +2 4 +2 6 +3 7 + +query I +SELECT b FROM ab EXCEPT ALL SELECT y AS b FROM xy ORDER BY b +---- +1 +2 +2 + +# If the ordering is only required on a subset of the columns, ensure that we +# still produce the correct ordering. +statement ok +TRUNCATE ab; +TRUNCATE xy; +INSERT INTO ab VALUES (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); +INSERT INTO xy VALUES (1, 1), (3, 3), (5, 5), (7, 7); + +query II +SELECT a, b FROM ab UNION SELECT x, y FROM xy ORDER BY a +---- +1 1 +2 2 +3 3 +4 4 +5 5 +7 7 + +query II +SELECT a, b FROM ab UNION ALL SELECT x, y FROM xy ORDER BY a +---- +1 1 +1 1 +2 2 +3 3 +3 3 +4 4 +5 5 +5 5 +7 7 + +query II +SELECT a, b FROM ab INTERSECT SELECT x, y FROM xy ORDER BY a +---- +1 1 +3 3 +5 5 + +query II +SELECT a, b FROM ab INTERSECT ALL SELECT x, y FROM xy ORDER BY a +---- +1 1 +3 3 +5 5 + +query II +SELECT a, b FROM ab EXCEPT SELECT x, y FROM xy ORDER BY a +---- +2 2 +4 4 + +query II +SELECT a, b FROM ab EXCEPT ALL SELECT x, y FROM xy ORDER BY a +---- +2 2 +4 4 diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index df935184ccc9..ffe4492cc25a 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -1357,14 +1357,23 @@ func (b *Builder) buildSetOp(set memo.RelExpr) (execPlan, error) { )) } } - node, err := b.factory.ConstructSetOp(typ, all, left.root, right.root, hardLimit) - if err != nil { - return execPlan{}, err - } - ep := execPlan{root: node} + + ep := execPlan{} for i, col := range private.OutCols { ep.outputCols.Set(int(col), i) } + // TODO(rytaft): This ordering may be stronger than the required output + // ordering in order to guarantee the use of a streaming (merge join or + // distinct) operation. We should probably pass both orderings to + // ConstructSetOp, similar to ConstructGroupBy. + reqOrdering := exec.OutputOrdering( + ep.sqlOrdering(ordering.StreamingSetOpOrdering(set, &set.RequiredPhysical().Ordering)), + ) + + ep.root, err = b.factory.ConstructSetOp(typ, all, left.root, right.root, reqOrdering, hardLimit) + if err != nil { + return execPlan{}, err + } return ep, nil } diff --git a/pkg/sql/opt/exec/execbuilder/testdata/distsql_union b/pkg/sql/opt/exec/execbuilder/testdata/distsql_union index f5642e8e8e94..f484b48e90a3 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/distsql_union +++ b/pkg/sql/opt/exec/execbuilder/testdata/distsql_union @@ -37,22 +37,25 @@ EXPLAIN (DISTSQL) SELECT x FROM xyz UNION ALL SELECT x FROM xyz ORDER BY x distribution: full vectorized: true · -• sort -│ order: +x +• union all │ -└── • union all - │ - ├── • scan - │ missing stats - │ table: xyz@primary - │ spans: FULL SCAN +├── • sort +│ │ order: +x +│ │ +│ └── • scan +│ missing stats +│ table: xyz@primary +│ spans: FULL SCAN +│ +└── • sort + │ order: +x │ └── • scan missing stats table: xyz@primary spans: FULL SCAN · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysVk2L2kAYvvdXDO-ppRPiTOJXTm67FgSrW7XQUnJIzbAIrpPORIgr_vdiguwq5p3JxzEZn3k-kRxB_9tCAONfT9OHyYx8fJwsV8sf009kOZ6Ov65IRr4t5t9JdnglP2eT-Yw8TKd3zuaLx_GCfPlNMqCwk7GYRS9CQ_AHGFDgQMEDCj5Q6EJIIVFyLbSW6vyTYw6YxBkEHQqbXbJPz69DCmupBARHSDfpVkAAq-jvVixEFAvldoBCLNJos81pssPrKFGbl0gdgMIyiXY6II57Zp7v04CMGIQnCnKfvl2v0-hZQMBOtJ6EbmsSeKmEN2apYqFEfM0J4emOyJl0ZOKy64jKuDtV7C-lSoVy2Y31Eftc6s2r6624tNQet7PnX9Ez-3aZRbsud1yv8sQqiOi1KIKXimgyMlZjZIYALiPr1R0Ztw-Y2wTsOa5fueUKIvotiuClIpq0zGu0bAjg0nK_bsuefcCeTcC-k_-jV2u5gohBiyJ4qYgmLXs1WjYEcGl5ULdl3z5g3ybgrlO54woShq1J4KUSmjTs12jYYP_S8LCNT4I79y-ETuROC0vlIQURP4siKC33ai2elFznNMXjPMflL2Kh0-KUFw-TXX6U9_8ezJqAOQr2rsCdW7CHgn2c2cdlM5y6i6L7OHWvCbiPgge47EGTxIYomDHDTPCRGeH4zBjHjTN8aAbnDF8a6xrg-NaMcHxsrGewjs_NZB3fGxsa4PjijHB8c7yDW-f45m6th6cP_wMAAP__BE4uGA== +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0ll-L2kAUxd_7KcJ9aumEeGcm-ydPu-1aEKxu1UJL8SE1wyK4TjqJoCt-92IUdi3mTmZiH5PxnHPPnR-YLRR_FpBA98dj_743CN4_9MaT8bf-h2Dc7Xc_T4J18GU0_BqsNy_B90FvOAju-_0zZ8PRQ3cUfPoZrIHBUmdqkD6rApJfgMCAAwMBDCQwiGHKIDd6popCm_1PtpWgl60h6TCYL_NVuX89ZTDTRkGyhXJeLhQkMEl_L9RIpZkyUQcYZKpM54sqZr15ucvN_Dk1G2AwztNlkQRhtE8ersokuEOY7hjoVflqX5Tpk4IEd6z5CGNtSmWi-DT9Dj_W2nMX-7cN8XIVhUdFdOgoa_1fbbXJlFHZOdMzQwx0qPOIn66gLr7zv-LjRvHxSTw63HCDC454GAlnii1DHK_4ypdil46XLCk8SqJDS1nr3xIk9OCYN18yb7JjEUbSGSTLEMcdX_uC5NARL1lSeJREh5ay1r8lSNwDJNF8yaLJjmVY_f-5gWQZ4rjjG1-QHDriJUsKj5Lo0FLW-rcESXiAJJsvWTbZcRw6Y2QZ4bjhW1-MHBri5SoKj4ro0FHW-reESLb8PDvjPFJFrpeFaug8ZaCyJ3XoUuiVmalHo2dVzOFxWOmqF5kqysMpHh56y8PRfsC3YiTF8kSM_4o5KRZ0smiTLElxTItjUsyv6LmvSPU1Lb6m76pDD35Dqm_p6NtW0UhTZtk40pght0BKg4a2dBo1lJZ0GjZrOo2bTU3jhhbekAaO226dJg4tyCHNnC2dt2KO08xxC3OcZo7b0mnmuIU5TjNnTXdjbrp79zcAAP__3g8zPw== query T EXPLAIN (DISTSQL) SELECT x FROM xyz UNION SELECT x FROM xyz ORDER BY x @@ -165,22 +168,25 @@ EXPLAIN (DISTSQL) (SELECT x FROM xyz ORDER BY y) UNION ALL (SELECT x FROM xyz OR distribution: full vectorized: true · -• sort -│ order: +x +• union all │ -└── • union all - │ - ├── • scan - │ missing stats - │ table: xyz@primary - │ spans: FULL SCAN +├── • sort +│ │ order: +x +│ │ +│ └── • scan +│ missing stats +│ table: xyz@primary +│ spans: FULL SCAN +│ +└── • sort + │ order: +x │ └── • scan missing stats table: xyz@primary spans: FULL SCAN · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysVluL2kwYvv9-xfBerXwT4kziKVduuxYEq1u10FJykZphEVwnnYngAf97MUGssnlncrgzh2eeI5IT6D8bCGD043XyPJ6Sp5fxYrn4NmmRp8VoMvq8JHvyZT77SvaHI5nNX0Zz8uknObTI9-l4NiXPkwn64rF1-70HClsZi2n0LjQEv4ABBQ4UPKDgA4UOhBQSJVdCa6kur5wywDjeQ9CmsN4mu_RyO6SwkkpAcIJ0nW4EBLCMfm_EXESxUG4bKMQijdabjGZ_OA4TtX6P1AEoLJJoqwPiuBfm2S4NyJBBeKYgd-nteJ1GbwICdqbVJHQak8ALJdyYpYqFEvE9J4TnD0ROpSMTl91HVMTdLmN_IVUqlMserA_Z_4XevKre8kML7XE7e_4dPbNvl1m063LH9UpPrISIboMieKGIOiNjFUZmCOA6sm7VkXH7gLlNwJ7j-qVbLiGi16AIXiiiTsu8QsuGAK4t96q27NkH7NkE7DvZP3q5lkuI6DcogheKqNOyV6FlQwDXlvtVW_btA_ZtAu44pTsuIWHQmAReKKFOw36Fhg32rw0Pmvgk-OD8udCJ3GphqTykIOI3kQel5U6txKuSq4wmv5xluOxGLHSaP-X5xXibPcr6_xfM6oA5CvbuwO1HsIeCfZzZx2UznLqDons4dbcOuIeC-7jsfp3EBiiYMcNM8JEZ4fjMGMeNM3xoBucMXxrrGOD41oxwfGysa7COz81kHd8bGxjg-OKMcHxzvI1b5_jmHq2H5__-BgAA__96tDSb +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0ll1v2jwUx--fT2Gdq1aPUTiO05dc0a1MQmLQQSdtmrjIiFUhUZw5QYJWfPeJgETZmmPH1u6al39-53_8k8orlL-WkEL_28PwbjBiF_eD6eP0y_CSXUz7w_7HR7Zhnybjz2yzfWHjyX1_wj58Z9tL9nU0GI_Y3XBIvvhyefp7AxxWOlej7FmVkP4ABA4COMTAQQKHBGYcCqPnqiy12b_yWgcG-QbSLofFqlhX-9szDnNtFKSvUC2qpYIUHrOfSzVRWa5M1AUOuaqyxbLGbLYvvcIsnjOzBQ7TIluVKetEe_J4XaWsh7wXw2zHQa-rE6GssicFKe64-xRTbSplouR8gB7-f2I1gkQb0Nu66NFXNI4Re_RFr8KykXQCaJMro_K_Pz_bvTPOSHd0EYnzfTThu_8KnzjhkzM8tjhuh9OORCeKffy2zHE876twv9sU9mrsarhbY_SqLBtJgYqhh-HCfePCZeFxJ5I-ilnmOC78OlyxFoXRq7GrYm6N0auybCQFKiY8FIvdNx67LFx26n-lrRWzzHFc-E24Yi0Ko1djV8XcGqNXZdlIClQs9lBMum9cuiw86fgIZpniuO7bcMFa1EWPvq56ufVFr8KykRSolwz8GfjOlyeqLPSqVI5fnnFQ-ZM6dCn12szVg9HzGnO4HNe5-kauyurwFA8Xg9Xh0X7At2Ekw_IsjH-GBRmOaXIcQpZkOKHDCRkWV_TcV2T6mg5f02fVpQe_IdO3NPo2CI20ZZaNI60ZCouktGhoo9OqobTQadmsdFo3W5rWDS2-IS2csJ06bRxalEPaORtdBDknaOeExTlBOydsdNo5YXFO0M5Z6e2cm-3--x0AAP__SitYcQ== query T EXPLAIN (DISTSQL) (SELECT x FROM xyz ORDER BY y) UNION (SELECT x FROM xyz ORDER BY z) ORDER BY x @@ -212,22 +218,25 @@ EXPLAIN (DISTSQL) (SELECT x FROM xyz ORDER BY y) UNION ALL (SELECT x FROM xyz OR distribution: full vectorized: true · -• sort -│ order: +x +• union all │ -└── • union all - │ - ├── • scan - │ missing stats - │ table: xyz@primary - │ spans: FULL SCAN +├── • sort +│ │ order: +x +│ │ +│ └── • scan +│ missing stats +│ table: xyz@primary +│ spans: FULL SCAN +│ +└── • sort + │ order: +x │ └── • scan missing stats table: xyz@primary spans: FULL SCAN · -Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysVluL2kwYvv9-xfBerXwT4kziKVduuxYEq1u10FJykZphEVwnnYngAf97MUGssnlncricJM88R8QT6D8bCGD043XyPJ6Sp5fxYrn4NmmRp8VoMvq8JHvyZT77SvaHI5nNX0Zz8uknObTI9-l4NiXPkwn-ISXH1u24BwpbGYtp9C40BL-AAQUOFDyg4AOFDoQUEiVXQmupLp-cMsA43kPQprDeJrv08jiksJJKQHCCdJ1uBASwjH5vxFxEsVBuGyjEIo3Wm4xmfzgOE7V-j9QBKCySaKsD4rgX5tkuDciQQXimIHfp7XqdRm8CAnam1SR0GpPACyXcmKWKhRLxPSeE5w9ETqUjE5fdR1TE3S5jfyFVKpTLHqwP2f-F3ryq3vJLC-1xO3v-HT2zb5dZtOtyx_VKT6yEiG6DInihiDojYxVGZgjgOrJu1ZFx-4C5TcCe4_qlWy4hotegCF4ook7LvELLhgCuLfeqtuzZB-zZBOw72S96uZZLiOg3KIIXiqjTslehZUMA15b7VVv27QP2bQLuOKU7LiFh0JgEXiihTsN-hYYN9q8ND5r4S_DB_XOhE7nVwlJ5SEHEbyIPSsudWolXJVcZTX6cZbjsQSx0mr_l-WG8zV5l_f8LZnXAHAV7d-D2I9hDwT7O7OOyGU7dQdE9nLpbB9xDwX1cdr9OYgMUzJhhJvjIjHB8Zozjxhk-NINzhi-NdQxwfGtGOD421jVYx-dmso7vjQ0McHxxRji-Od7GrXN8c4_Ww_N_fwMAAP__3oE1YA== +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0ll1v2jwUx--fT2Gdq1aPUTiO05dc0a1MQmLQQSdtmrjIiFUhUZw5QYJWfPeJgETZmmPH1i7z8s_v_I9_ErxC-WsJKfS_PQzvBiN2cT-YPk6_DC_ZxbQ_7H98ZBv2aTL-zDbbFzae3Pcn7MN3tr1kX0eD8YjdDYf0i5y9XJ4uN8BhpXM1yp5VCekPQOAggEMMHCRwSGDGoTB6rspSm_0rr3VgkG8g7XJYrIp1tb894zDXRkH6CtWiWipI4TH7uVQTleXKRF3gkKsqWyxrzGb70ivM4jkzW-AwLbJVmbJOtCeP11XKesh7gvdimO046HV1gpRV9qQgxR13H2SqTaVMlJzP0MP_T7hGkGgDetsYfSo3jhF79EWvwrKRdAJokyuj8r8_P9u9M85Id3QRifN9NOG7_wqfOOGTMzy2OG6H045EJ4o9FbeMcjzyq3DF23T2K-0ouVtj9KosG0mBlqGH5MJ948Jl4XEnkp6WWUY57vw63LIWndGvtKNlbo3Rq7JsJAVaJjwsi903HrssXHbqH1QfyyyjHHd-E25Zi87oV9rRMrfG6FVZNpICLYs9LJPuG5cuC086no5ZBjlu_DbcsRaN0aeyo2FufdGrsGwkBRomA_8SvvPliSoLvSqV45dnHFT-pA5dSr02c_Vg9LzGHC7Hda6-kauyOjzFw8VgdXi0H_BtGMmwPAvjn2FBhmOaHIeQJRlO6HBChsUVPfcVmb6mw9f0WXXpwW_I9C2Nvg1CI22ZZeNIa4bCIiktGtrotGooLXRaNiud1s2WpnVDi29ICydsp04bhxblkHbORhdBzgnaOWFxTtDOCRuddk5YnBO0c1Z6O-dmu_9-BwAA__9mrFxM # Only one distinct processor should be used in the single node UNION case. query T diff --git a/pkg/sql/opt/exec/execbuilder/testdata/inverted_filter_json_array b/pkg/sql/opt/exec/execbuilder/testdata/inverted_filter_json_array index 69fc790d2baf..4d5aa67de233 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/inverted_filter_json_array +++ b/pkg/sql/opt/exec/execbuilder/testdata/inverted_filter_json_array @@ -211,23 +211,23 @@ vectorized: true │ distinct on: a │ order key: a │ -└── • sort - │ order: +a +└── • union all │ - └── • union all - │ - ├── • index join - │ │ table: json_tab@primary - │ │ - │ └── • scan - │ missing stats - │ table: json_tab@foo_inv - │ spans: 1 span - │ - └── • scan - missing stats - table: json_tab@primary - spans: [/44 - /44] + ├── • index join + │ │ table: json_tab@primary + │ │ + │ └── • sort + │ │ order: +a + │ │ + │ └── • scan + │ missing stats + │ table: json_tab@foo_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: json_tab@primary + spans: [/44 - /44] # We cannot use the index for this query. query error pq: index "foo_inv" is inverted and cannot be used for this query @@ -295,23 +295,23 @@ vectorized: true │ distinct on: a │ order key: a │ -└── • sort - │ order: +a +└── • union all │ - └── • union all - │ - ├── • index join - │ │ table: array_tab@primary - │ │ - │ └── • scan - │ missing stats - │ table: array_tab@foo_inv - │ spans: 1 span - │ - └── • scan - missing stats - table: array_tab@primary - spans: [/1 - /1] + ├── • index join + │ │ table: array_tab@primary + │ │ + │ └── • sort + │ │ order: +a + │ │ + │ └── • scan + │ missing stats + │ table: array_tab@foo_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: array_tab@primary + spans: [/1 - /1] # We cannot use the index for this query. query error pq: index "foo_inv" is inverted and cannot be used for this query diff --git a/pkg/sql/opt/exec/execbuilder/testdata/limit b/pkg/sql/opt/exec/execbuilder/testdata/limit index 494bd72a5b12..2b8d5825a748 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/limit +++ b/pkg/sql/opt/exec/execbuilder/testdata/limit @@ -411,72 +411,100 @@ vectorized: true │ estimated row count: 5 │ count: 5 │ -└── • sort +└── • union │ estimated row count: 40 - │ order: +date_should_be_completed │ - └── • union - │ estimated row count: 40 - │ - ├── • union - │ │ estimated row count: 35 - │ │ - │ ├── • union - │ │ │ estimated row count: 30 - │ │ │ - │ │ ├── • union - │ │ │ │ estimated row count: 25 - │ │ │ │ - │ │ │ ├── • union - │ │ │ │ │ estimated row count: 20 - │ │ │ │ │ - │ │ │ │ ├── • union - │ │ │ │ │ │ estimated row count: 15 - │ │ │ │ │ │ - │ │ │ │ │ ├── • union - │ │ │ │ │ │ │ estimated row count: 10 - │ │ │ │ │ │ │ - │ │ │ │ │ │ ├── • scan - │ │ │ │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ │ │ │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ │ │ │ │ │ │ spans: [/0/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /0/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ │ │ │ │ │ │ limit: 5 - │ │ │ │ │ │ │ - │ │ │ │ │ │ └── • scan - │ │ │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ │ │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ │ │ │ │ │ spans: [/1/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /1/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ │ │ │ │ │ limit: 5 - │ │ │ │ │ │ - │ │ │ │ │ └── • scan - │ │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ │ │ │ │ spans: [/2/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /2/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ │ │ │ │ limit: 5 - │ │ │ │ │ - │ │ │ │ └── • scan - │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ │ │ │ spans: [/3/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /3/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ │ │ │ limit: 5 - │ │ │ │ - │ │ │ └── • scan - │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ │ │ spans: [/4/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /4/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ │ │ limit: 5 - │ │ │ - │ │ └── • scan - │ │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ │ spans: [/5/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /5/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ │ limit: 5 - │ │ - │ └── • scan - │ estimated row count: 5 (0.05% of the table; stats collected ago) - │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem - │ spans: [/6/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /6/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] - │ limit: 5 + ├── • union + │ │ estimated row count: 35 + │ │ + │ ├── • union + │ │ │ estimated row count: 30 + │ │ │ + │ │ ├── • union + │ │ │ │ estimated row count: 25 + │ │ │ │ + │ │ │ ├── • union + │ │ │ │ │ estimated row count: 20 + │ │ │ │ │ + │ │ │ │ ├── • union + │ │ │ │ │ │ estimated row count: 15 + │ │ │ │ │ │ + │ │ │ │ │ ├── • union + │ │ │ │ │ │ │ estimated row count: 10 + │ │ │ │ │ │ │ + │ │ │ │ │ │ ├── • sort + │ │ │ │ │ │ │ │ estimated row count: 5 + │ │ │ │ │ │ │ │ order: +date_should_be_completed + │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ └── • scan + │ │ │ │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ │ │ │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ │ │ │ │ │ │ spans: [/0/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /0/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ │ │ │ │ │ │ limit: 5 + │ │ │ │ │ │ │ + │ │ │ │ │ │ └── • sort + │ │ │ │ │ │ │ estimated row count: 5 + │ │ │ │ │ │ │ order: +date_should_be_completed + │ │ │ │ │ │ │ + │ │ │ │ │ │ └── • scan + │ │ │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ │ │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ │ │ │ │ │ spans: [/1/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /1/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ │ │ │ │ │ limit: 5 + │ │ │ │ │ │ + │ │ │ │ │ └── • sort + │ │ │ │ │ │ estimated row count: 5 + │ │ │ │ │ │ order: +date_should_be_completed + │ │ │ │ │ │ + │ │ │ │ │ └── • scan + │ │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ │ │ │ │ spans: [/2/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /2/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ │ │ │ │ limit: 5 + │ │ │ │ │ + │ │ │ │ └── • sort + │ │ │ │ │ estimated row count: 5 + │ │ │ │ │ order: +date_should_be_completed + │ │ │ │ │ + │ │ │ │ └── • scan + │ │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ │ │ │ spans: [/3/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /3/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ │ │ │ limit: 5 + │ │ │ │ + │ │ │ └── • sort + │ │ │ │ estimated row count: 5 + │ │ │ │ order: +date_should_be_completed + │ │ │ │ + │ │ │ └── • scan + │ │ │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ │ │ spans: [/4/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /4/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ │ │ limit: 5 + │ │ │ + │ │ └── • sort + │ │ │ estimated row count: 5 + │ │ │ order: +date_should_be_completed + │ │ │ + │ │ └── • scan + │ │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ │ spans: [/5/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /5/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ │ limit: 5 + │ │ + │ └── • sort + │ │ estimated row count: 5 + │ │ order: +date_should_be_completed + │ │ + │ └── • scan + │ estimated row count: 5 (0.05% of the table; stats collected ago) + │ table: user_checklist_items@userchecklistitems_tenantid_userid_dateshouldbecompleted_locationname_orderitem + │ spans: [/6/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'/'2020-10-01' - /6/'a2a0dd49-23cf-4cf2-b823-61701c416e60'/'01603523-c6f0-4e12-a43f-524c76b0fa8f'] + │ limit: 5 + │ + └── • sort + │ estimated row count: 5 + │ order: +date_should_be_completed │ └── • scan estimated row count: 5 (0.05% of the table; stats collected ago) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/union b/pkg/sql/opt/exec/execbuilder/testdata/union index 7451c6d2a2ff..157cec406cd4 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/union +++ b/pkg/sql/opt/exec/execbuilder/testdata/union @@ -194,3 +194,564 @@ vectorized: true estimated row count: 1,000 (missing stats) table: uniontest@primary spans: FULL SCAN + +statement ok +CREATE TABLE ab (a INT PRIMARY KEY, b INT, INDEX (b, a)) + +statement ok +CREATE TABLE xy (x INT PRIMARY KEY, y INT, INDEX (y, x)) + +# Regression tests for #41245, #40797. Ensure we can plan ordered set ops +# without a sort. +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT a FROM ab UNION SELECT x AS a FROM xy ORDER BY a +---- +distribution: local +vectorized: true +· +• union +│ columns: (a) +│ ordering: +a +│ estimated row count: 2,000 (missing stats) +│ +├── • scan +│ columns: (a) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: ab@primary +│ spans: FULL SCAN +│ +└── • scan + columns: (x) + ordering: +x + estimated row count: 1,000 (missing stats) + table: xy@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJykkVFLwzAUhd_9FZf7pBhZ273laZurUJjtbKco0oesuYxC19QkhY7R_y5rGbPiBOfjPTfnfCfJHs1HgRz91-ViGoRwPQ-SVfK0YPDix7Mo8W8g8Rf-_QoEPMTRI4g1PIdBFB7lBqbJcdfsIIrnfgyzNxDIsFSSQrElg_wdXUwZVlplZIzSB2nfHQhkg9xhmJdVbQ9yyjBTmpDv0ea2IOS4EuuCYhKS9MhBhpKsyIsuttlNKp1vhd4hw6QSpeFwhwyj2nKYuJi2DFVtT9nGig0hd1t2Gd8d8sX6Mr53ln_CKi1JkxwCJ-4tpu0PJee5sXmZ2ZH33XC2xPgvjxCTqVRpaJB-Ltk5NCS5of5GRtU6o6VWWYfpx6jzdYIkY_ut1w9B2a26X_pqdv9j9n41jwdmp03bq88AAAD__9FnAD0= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT a, b FROM ab UNION SELECT x AS a, y AS b FROM xy ORDER BY a +---- +distribution: local +vectorized: true +· +• union +│ columns: (a, b) +│ ordering: +a,+b +│ estimated row count: 2,000 (missing stats) +│ +├── • scan +│ columns: (a, b) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: ab@primary +│ spans: FULL SCAN +│ +└── • scan + columns: (x, y) + ordering: +x + estimated row count: 1,000 (missing stats) + table: xy@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJykkUFr4zAQhe_7K8ScdomW2M5NJycbLxhSO7XT0lJ8kK0hGBzLlRSwCf7vxXJL6tIU0p7EzJs335N0Av1cAYPgYbtZhhH5vQ7TXXq7oeQ-SFZxGvwhabAJ_u0IpyQn_5P4hvCc3EVhHL0pLVmmg9wN5-tM25E4WQcJWT0SDhRqKTDiB9TAnsCFjEKjZIFaSzW0TnYgFC0wh0JZN0cztDMKhVQI7ASmNBUCgx3PK0yQC1RzBygINLys7Nq28xtVHrjqgELa8Foz8heynoI8mvNGbfgegbk9_R7VnVJ5fg3Vu0g9w6QSqFBMMb47o743g6z_JN661KasCzP3Ppqo713MsrjmBRLUjaw1TgCXNjtDSBR7HC-m5VEVuFWysJixjK3PNgRqM6reWIS1lewXvTe7PzF7X5oXE7PTZ_2vlwAAAP__UP__WA== + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT a, b FROM ab UNION ALL SELECT x AS a, y AS b FROM xy ORDER BY a +---- +distribution: local +vectorized: true +· +• union all +│ columns: (a, b) +│ ordering: +a +│ estimated row count: 2,000 (missing stats) +│ +├── • scan +│ columns: (a, b) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: ab@primary +│ spans: FULL SCAN +│ +└── • scan + columns: (x, y) + ordering: +x + estimated row count: 1,000 (missing stats) + table: xy@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJykkc9rq0AQx-_vrxjm9B5vQ9TePCVpLAhWU01LS_GwukMQjGt3DSjB_724tqRCU5r2JDPfHx-HPaJ-KdFF73ETLP0Q_q79ZJvcBQwevHgVJd4_SLzAu94CZ5DBTRzdAs_gPvSjEJZB8K62sEwGSzd833xtB1G89mJYPQFHhpUUFPI9aXSf0caUYa1kTlpLNayOxuCLFl2LYVHVh2ZYpwxzqQjdIzZFUxK6uOVZSTFxQWpuIUNBDS9KU9t2i1oVe646ZJjUvNIuzDDtGcpDc2rUDd8RunbPfka1p1SeXUJ1zlJPMKkEKRJTzML-j2n_ya-FcibruTNxn6Nbl9wck65lpembzSlDEjsaT9HyoHLaKJkbzDhGJmcWgnQzqs44-JWRzKN8DNu_CTtfhq8mYatP-z-vAQAA___WJ_y5 + +# TODO(yuzefovich): The synchronizers in the below DistSQL plans are all +# unordered. This is not a problem, but we shouldn't need an input synchronizer +# at all when there is only one incoming stream. We should look into removing +# it. +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT a, b FROM ab INTERSECT SELECT x AS a, y AS b FROM xy ORDER BY b, a +---- +distribution: local +vectorized: true +· +• intersect +│ columns: (a, b) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ +├── • scan +│ columns: (a, b) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ table: ab@ab_b_a_idx +│ spans: FULL SCAN +│ +└── • scan + columns: (x, y) + ordering: +y,+x + estimated row count: 1,000 (missing stats) + table: xy@xy_y_x_idx + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJykkl-Lm0AQwN_7KZZ5uuO2JJrcy0LBXGMh4MWrSmkpIqs7TQXr2t0VlJDvXlYPcoZL2lyfZP785jezuAf9uwIG_tenYLXZkpv1Jk7izwElX_zoIYz9WxL7gf8xIZySnHyKwkfCc7LZJn4U2_RztSOr2Lb09vvc1_UkjNZ-RB6-kZwSDhRqKXDLf6EG9h0cSCk0ShaotVQ2tR8aNqIDNqdQ1k1rbDqlUEiFwPZgSlMhMEh4XmGEXKCazYGCQMPLahjb9V7XZ33WZaXogELc8Foz8h7SAwXZmuNQbfgOgTkH-jaxMxXz3ON5lmf8X8XuNeJ1qU1ZF2bmTq2eQz2bCpVAhYIRz6Wec9a5eJNz8V_O5VnnUdXWchw2MaWW_FvLK4s_otphjCZsZsvp6knfIHvx_66CAChU-MPceO4d9Zy72w-q3P08hvbO1jAyHn3uxvtr3jVC3cha4-mtr06e2wNR7HB8MC1bVeCTksWgGcNw4IaEQG3G6mIMNvVYsgu-hJ2LsHsZdi_CywnsnMKLK2D3FF5ehO9P1k4P7_4EAAD__60NkI0= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT b FROM ab INTERSECT ALL SELECT x AS b FROM xy ORDER BY b +---- +distribution: local +vectorized: true +· +• intersect all +│ columns: (b) +│ ordering: +b +│ estimated row count: 1,000 (missing stats) +│ +├── • scan +│ columns: (b) +│ ordering: +b +│ estimated row count: 1,000 (missing stats) +│ table: ab@ab_b_a_idx +│ spans: FULL SCAN +│ +└── • scan + columns: (x) + ordering: +x + estimated row count: 1,000 (missing stats) + table: xy@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJycUWGLnDAQ_d5fEebTHU059b4FCu71LAjeejVSWoos0Uyt4BqbRFAW_3tRWbYuu2233zJv5s2b93IA87MGBsGX12gTbsndc8hT_imi5HOQPMU8uCc8iIIPKcnJxyR-ISIn4TYNEj5hmyg6tnuy4ceZfiBx8hwk5OkryYFCoyRuxR4NsG_gQkah1apAY5SeoMM8EMoemEOhatrOTnBGoVAagR3AVrZGYJCKvMYEhUT94AAFiVZU9by2H_xWV3uhB6DAW9EYRt4BhbizjPguZCMF1dnTbmNFicDckf6fvrvWF7kv8l2-E7tK9pdO8K6e4F094aTcNUpL1ChXqtnE_NvIBR8vqEvkaOP2wVvbSIcW2fqHgUKN3-2d7769f6-r8sfy_IdsH2_JNkHTqsbgucGLm53JFcoSl5SM6nSBr1oVs8xSxjNvBiQau3S9pQibuTUn_zvZvYHsnpO9P5IfV2RnzMY3vwIAAP__7D8d8w== + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT b, a FROM ab INTERSECT ALL SELECT y AS b, x AS a FROM xy ORDER BY b +---- +distribution: local +vectorized: true +· +• intersect all +│ columns: (b, a) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ +├── • scan +│ columns: (b, a) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ table: ab@ab_b_a_idx +│ spans: FULL SCAN +│ +└── • scan + columns: (y, x) + ordering: +y,+x + estimated row count: 1,000 (missing stats) + table: xy@xy_y_x_idx + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJycUVFrnTAYfd-vCN9TSzN6Y98CA29XB4K9dkbGxhCJ5psTnHFJBOXifx_qtjtLu9L7FL6Tc3LOyXcE-7MBDsHnh2gfHsjFXShS8TGi5FOQ3MYiuCQiiIL3KSkokeRDEt8TWZDwkAaJmOF9FP1hjGQvZtown7-5w0ji5C5IyO0XUgCFVis8yB9ogX8FBhmFzugSrdVmho4LIVQD8B2Fuu16N8MZhVIbBH4EV7sGgUMqiwYTlArN9Q4oKHSybpZnh9EfxnzMh7xWA1AQnWwtJ2-BQtw7TnyP-gyyiYLu3cnBOlkhcDbR81KwbQpZ-LLIi1yelcJ7NsXJvG-1UWhQbYyzWfkS5Ykq92gqFOji7trbNknHDvl25UChwW_uwmdX1PeuLt-Zuvp-Gv92ZNT3nu1485qfTtB2urX4uOuTL-_mgqgqXD_M6t6U-GB0udisY7zoFkChdeuttw5hu1wtS_hXzF4hZo_F3n_FNxvxbsqmN78CAAD__zutJUU= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT b, a FROM ab EXCEPT SELECT y AS b, x AS a FROM xy ORDER BY b +---- +distribution: local +vectorized: true +· +• except +│ columns: (b, a) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ +├── • scan +│ columns: (b, a) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ table: ab@ab_b_a_idx +│ spans: FULL SCAN +│ +└── • scan + columns: (y, x) + ordering: +y,+x + estimated row count: 1,000 (missing stats) + table: xy@xy_y_x_idx + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJykktGLm0AQxt_7VyzzdMdtSTS5l4WCuYuFgHemKuVKEVndaSpY1-6uoIT870XtNTVcLEmfwnwzv_m-2bgH_bMABu7L1lttnsnNehNG4SePks9u8OCH7i0JXc99jEhKCScfA_-J8JS4L4_uNnpttWQVdv2m-_091LTED9ZuQB6-kBQolFLgM_-BGthXsCCmUCmZodZSddK-H9iIBticQl5WtenkmEImFQLbg8lNgcAg4mmBAXKBajYHCgINz4t-LU8dniZpwpNcNEAhrHipGXkPFPzaMOLY1LEgPlCQtTk6aMN3CMw60OtSWOMUTes0bdImzVUp7EtSrHNt8jIzM3scwbGo00m-EqhQMDII5zwXV3ku_stzedbzaFWXclg2coo78l8jbwR_QrXDEI1fzZbj6FFbIXv9pleeBxQK_GZuHOuOOvbd7QeV774fyz9_5OSB95c8aoC6kqXG00Pf3DzvrkOxw-G1tKxVhlsls95mKP2e6wWB2gxdeyg25dDqAv4NW5PwYhq2J-HlCLZO4cUFsH0KLyfh-5PY8eHdrwAAAP__SsSUMg== + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT a, b FROM ab EXCEPT ALL SELECT x AS a, y AS b FROM xy ORDER BY b, a +---- +distribution: local +vectorized: true +· +• except all +│ columns: (a, b) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ +├── • scan +│ columns: (a, b) +│ ordering: +b,+a +│ estimated row count: 1,000 (missing stats) +│ table: ab@ab_b_a_idx +│ spans: FULL SCAN +│ +└── • scan + columns: (x, y) + ordering: +y,+x + estimated row count: 1,000 (missing stats) + table: xy@xy_y_x_idx + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyckVFrnEAQx9_7KYZ5SsiWnOZtoeAl2cKBiVeVklJE1tupFaxrd1dQDr97UVsuhqRt8rTMzP8_v52ZI9qfNXIUD_twu7uHs9tdkiafQgafRXwdJeIcEhGKmxQkgwI-xtEdyALEw43Yp7ANwz_lHrbJpBmm97ewHyCKb0UM11-gYCCRYaMV3csfZJF_RQ8zhq3RB7JWmyl1nAU71SPfMKyatnNTOmN40IaQH9FVribkmMqippikInO5QYaKnKzqua0sAlnkRS7zSvXIMGllYzm8x2xkqDt3amqdLAm5N7K3gb01uB-CfsiHvP9fsP8i-MTrGm0UGVIrVjY5_yV55vd3ZEpKyEXtpb_-fDq0xB8dFhnW9M2dBf4FC7yL8w-mKr-fQmQYdY5D4LHAf3HAq9dsNibb6sbS00Gf7byZpiNV0rItqztzoL3RhxmzhNHsmxOKrFuq_hLsmrk0n_6x2XuF2X9q9v9qvlqZN2M2vvsVAAD__wSzHgA= + +statement ok +CREATE TABLE abcde (a INT PRIMARY KEY, b INT, c INT, d INT, e INT, INDEX (b, c, d, e)) + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde UNION SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY a +---- +distribution: local +vectorized: true +· +• union +│ columns: (a, b, c, d, e) +│ ordering: +a,+b,+c,+d,+e +│ estimated row count: 2 (missing stats) +│ +├── • filter +│ │ columns: (a, b, c, d, e) +│ │ ordering: +a +│ │ estimated row count: 1 (missing stats) +│ │ filter: (c = 1) AND (d = e) +│ │ +│ └── • scan +│ columns: (a, b, c, d, e) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: abcde@primary +│ spans: FULL SCAN +│ +└── • filter + │ columns: (a, b, c, d, e) + │ ordering: +a + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + ordering: +a + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyskF-Lm0AUxd_7KS73SeuUxH9QBgKz27hUSHWr6T-KD65zWQRX7cwEWkK-e1ELu4ZN6DZ9nHvnnN85d4_6R4Mco6-3m6s4AWsd59v844bB5yi7TvPIhjzaRO-28BpusvQDWPNneVdJgk9JnCbwzMaGL--jLAKrghW4Nlwla7AkrIBsSLN1lMH1NyiRYdtJSsoH0si_o4sFw151FWndqWG0Hz_E8ifyJcO67XdmGBcMq04R8j2a2jSEHLflXUMZlZLUYokMJZmybkbbMZHoVf1Qql_IMO_LVnN4g8WBYbczj6balPeE3D2wvwff1I0hRWrhzqnTnIMl_OEGnPM42b79cwoRwApEaJ-M4L0kwtPu3oXd_X_q7v_P7sHJCI_kTklSJOdY4TpMeA4TvsNE4DAROlgcnsm9rrWp28osgmMDJjwmfCYCJsKTAcOX3Cgj3XetphnplPNySEvynqa2utupim5VV42Y6ZmOunEgSZtp606PuJ1WQ8CnYvesOJiJ3WOxd1bsnyf7l5CDs-LwiFwcXv0OAAD__4ZyhRo= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde UNION SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY b, c, d, e, a +---- +distribution: local +vectorized: true +· +• union +│ columns: (a, b, c, d, e) +│ ordering: +b,+c,+d,+a,+e +│ estimated row count: 2 (missing stats) +│ +├── • sort +│ │ columns: (a, b, c, d, e) +│ │ ordering: +b,+d,+a +│ │ estimated row count: 1 (missing stats) +│ │ order: +b,+d,+a +│ │ +│ └── • filter +│ │ columns: (a, b, c, d, e) +│ │ estimated row count: 1 (missing stats) +│ │ filter: (c = 1) AND (d = e) +│ │ +│ └── • scan +│ columns: (a, b, c, d, e) +│ estimated row count: 1,000 (missing stats) +│ table: abcde@primary +│ spans: FULL SCAN +│ +└── • sort + │ columns: (a, b, c, d, e) + │ ordering: +b,+d,+a + │ estimated row count: 1 (missing stats) + │ order: +b,+d,+a + │ + └── • filter + │ columns: (a, b, c, d, e) + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyskF2L2kAUhu_7Kw7nSuspmi-3DAizW7M0YOM2sV8UL2LmsATcxM6M0CL-95KkdFfRUHd7OefkOc-bd4fmxxoFhl_vZtdRDL1plC7SjzOCz2FyM0_DPqThLHy3gNdwm8w_QO_wma1yxfApjuYxnNj04cv7MAmhl8MEnD5cx1PoKZgA92GeTMMEbr7BiiAnUARMkCFhWSmOswc2KL6jg0vCja5yNqbS9WjXfBCpnyhGhEW52dp6vCTMK80odmgLu2YUuMhWa044U6yHIyRUbLNi3Zxt0smNLh4y_QsJ001WGgFvcLknrLb28aix2T2jcPb07-LbYm1Zsx46h9Z2LqAnvboPIUQUL97-qUX6MAEZ9M9GcC-JkFbash66hwGkOyDpD0g6g7Ma7xLN04q9F1bsP6ti_39WHDyj4uDyisdnNY_XK61Yszpx3PsrIBnUkhPZpoWxRZnb4fjogEPSJemR9EkGZwNeXdJDwmZTlYYPTOcuj-q0rO65_VtTbXXOd7rKG037nDdcM1BsbLt12kdUtqs64FPY6YTdbtjthMcHsHMMe52w3232O-GgGw5eEnvcCV8dmZf7V78DAAD__4t08R0= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde UNION ALL SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY a +---- +distribution: local +vectorized: true +· +• union all +│ columns: (a, b, c, d, e) +│ ordering: +a +│ estimated row count: 2 (missing stats) +│ +├── • filter +│ │ columns: (a, b, c, d, e) +│ │ ordering: +a +│ │ estimated row count: 1 (missing stats) +│ │ filter: (c = 1) AND (d = e) +│ │ +│ └── • scan +│ columns: (a, b, c, d, e) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: abcde@primary +│ spans: FULL SCAN +│ +└── • filter + │ columns: (a, b, c, d, e) + │ ordering: +a + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + ordering: +a + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyskN1q20AQhe_7FMNcSe0GW5YCZcEgp95QgSqlkvtH0cVGOwSBolV319Bi_O7FUiFRiE3S5HJ-znznzA7trxY5iu9X6SrJwFsn5ab8nDL4KoqLvBQ-lCIVHzbwFi6L_BN401Je14rgS5bkGazSFB6Z-vDtoygEeDUsIfBhla3BU7AE8iEv1qKAix8gkWGnFWXylizynxhgxbA3uiZrtTm0dsNCon4jnzNsun7rDu2KYa0NId-ha1xLyHEjr1sqSCoyszkyVORk0w5nB0dxb5pbaf4gw7KXneVwhtWeod66u6PWyRtCHuzZ08GXTevIkJkFU-rY5-DF4eEHnPMk27z_94o4giXE5_5RC4vnWLifffHC7OF_ZQ9fM3t01MIdWRtFhtQUGwfvsNo_4jPTZ7qfRZPtY_T5cx5QkO11Z-mJlyuGpG5ojGL11tR0ZXQ9YMYyH3RDQ5F14zQYi6QbRweD98XBSXE0EQcPxYuT4vA0OXwJOTopPn9ArvZv_gYAAP__aIR-OA== + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde UNION ALL SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY b, c, d, e, a +---- +distribution: local +vectorized: true +· +• union all +│ columns: (a, b, c, d, e) +│ ordering: +b,+c,+d,+a +│ estimated row count: 2 (missing stats) +│ +├── • sort +│ │ columns: (a, b, c, d, e) +│ │ ordering: +b,+d,+a +│ │ estimated row count: 1 (missing stats) +│ │ order: +b,+d,+a +│ │ +│ └── • filter +│ │ columns: (a, b, c, d, e) +│ │ estimated row count: 1 (missing stats) +│ │ filter: (c = 1) AND (d = e) +│ │ +│ └── • scan +│ columns: (a, b, c, d, e) +│ estimated row count: 1,000 (missing stats) +│ table: abcde@primary +│ spans: FULL SCAN +│ +└── • sort + │ columns: (a, b, c, d, e) + │ ordering: +b,+d,+a + │ estimated row count: 1 (missing stats) + │ order: +b,+d,+a + │ + └── • filter + │ columns: (a, b, c, d, e) + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyskN1q20AQhe_7FMNcWc0GW39uWTDIqRUqUKVUcv8oupC1QxAoWnVXhhbjdy-WCrGCLeKklzOz3zlnzw71rwo5-t_vwmUQwWQVpOv0c8jgq5_cxKlvQOqH_oc1vIXbJP4Ek-GYbwpB8CUK4giWYQgnrgZ8--gnPkwKWIBpwDJawUTAAsiAOFn5Cdz8gA2DgoFgQAxyZFhLQVH-QBr5TzQxY9goWZDWUh1Wu-5BIH4jnzEs62bbHtYZw0IqQr7DtmwrQo7rfFNRQrkgNZ0hQ0FtXladbJfOa1T5kKs_yDBt8lpzuMZsz1Bu20dR3eb3hNzcs-cb35ZVS4rU1By69nsOE88-9ME5D6L1-3-1eA4swHONsxGsSyKkUrWkptYwgGddMc-5Yp55ddbGvsTmuGL7lRU7L6rY-Z8Vuy-o2L284vlZm0d1qQQpEifE7WODE7kieS2b6XxAnksyu-TDCelG1pqeqZwxJHFP_be03KqC7pQsOpt-jDuuWwjSbX81-yGo-9Mh4DFsjsLWOGyNwvMBbD6F7VHYGXd2RmF3HHZfE3s-Cr974pzt3_wNAAD__wOb7Jc= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde INTERSECT SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY a +---- +distribution: local +vectorized: true +· +• intersect +│ columns: (a, b, c, d, e) +│ ordering: +a,+b,+c,+d,+e +│ estimated row count: 1 (missing stats) +│ +├── • filter +│ │ columns: (a, b, c, d, e) +│ │ ordering: +a +│ │ estimated row count: 1 (missing stats) +│ │ filter: (c = 1) AND (d = e) +│ │ +│ └── • scan +│ columns: (a, b, c, d, e) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: abcde@primary +│ spans: FULL SCAN +│ +└── • filter + │ columns: (a, b, c, d, e) + │ ordering: +a + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + ordering: +a + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysk1uLm0AUx9_7KQ7nSZspibdsGQhMtnGpkI1blV4oeXD1NBVcteMEuoR896IWNobENrv7OOf4O_8LuMP6V44c3a93y7m3Am3hhVH4acngsxtc-6GrQ-gu3Q8RvIWbwL8Frf-M75OUwFtFbhA24xNbHb58dAMXtARmYOgwXy1AS2EGpIMfLNwArr9BjAyLMqVV_EA18u9o4JphJcuE6rqUzWjXfuClv5FPGGZFtVXNeM0wKSUh36HKVE7IMYrvcwooTkmOJ8gwJRVneXu2dSQqmT3E8hEZhlVc1Bze4XrPsNyqp6O1ijeE3Niz_xe-yXJFkuTY6Kt2cw6asJoOOOfeKnr_twphwwyEo5-1YF5i4TC7-cLs1rOyW6-Z3b7EwiKrVVYkamz3LQiDCZMJiwmbCQcZ-jIlSSkHYZxVdp6l7LyC8vSs8pPgtii7Sz29dUP-65MT9m9Jbigk5VfjaT9A9FgRP_i_58slMszph9KEMWLCHDFhjZiwR0w4I30ms83P06sm_FY1wXuVnCvh6pL6A6qrsqjpuIyTlydNA5RuqGu0LrcyoTtZJq1M9_Rbrh2kVKtua3QPr-hWjcFD2BiEnWHYHIStYdgahO1h2B6Epz3YOIadC2DzGJ4OwldHttf7N38CAAD__8p2GHI= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde EXCEPT SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY b, c, d, e, a +---- +distribution: local +vectorized: true +· +• sort +│ columns: (a, b, c, d, e) +│ ordering: +b,+c,+d,+a +│ estimated row count: 1 (missing stats) +│ order: +b,+c,+d,+a +│ +└── • except + │ columns: (a, b, c, d, e) + │ estimated row count: 1 (missing stats) + │ + ├── • filter + │ │ columns: (a, b, c, d, e) + │ │ estimated row count: 1 (missing stats) + │ │ filter: (c = 1) AND (d = e) + │ │ + │ └── • scan + │ columns: (a, b, c, d, e) + │ estimated row count: 1,000 (missing stats) + │ table: abcde@primary + │ spans: FULL SCAN + │ + └── • filter + │ columns: (a, b, c, d, e) + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJysk11vmzwUx--fT3F0ruCJp5S3pLIUyW1D1UhZ6ABtnaZcUDhrkSgw25FWVf3uE7CpJUtYk-3Sx_6d_wviCdW3Ajn6N9fLs8UKjPkiiqMPSwYf_fA8iHwTIn_pX8TwP1yGwXsw-sfkNs0I_JsL_zqGHVcmfLryQx-MFGZgmXC2moORwQzIhCCc-yGcf4ZbBimDjAExSJBhWWW0Sh5IIf-CFq4Z1rJKSalKNqOn9sEi-478hGFe1hvdjNcM00oS8ifUuS4IOcbJbUEhJRnJ8QkyzEgnedGubd2JWuYPiXxEhlGdlIrDO1w_M6w2-mWp0skdIbee2duFL_NCkyQ5tvqq3ZyDIZymD875YhWf_qxFuDAD4Zl7LdiHWHid3f7L7M5R2Z1_md09xMI8VzovUz12-xaExYTNhMOEy4S3V8s7Sss7SmuyV-tFYlNWMiNJWU9h3ZB_erLD8FWi7iPSQT2e9B3HjzXxX__y2XKJDAv6qo1-EnMm87v736bIMNhoDm-MPT2k4qiSmuR4ulWwPWLCGTHhjpiwRnulTg-RCknVValou-mdm0-aeim7o-5zqWojU7qWVdrKdMeg5dpBRkp3t1Z3WJTdVWPwNWwNwu4wbA_CzjDsDMLeMOwOwpMebG3D3gGwvQ1PBuHpsO3pIHy6Ba-f__sRAAD__7MJRLg= + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde EXCEPT ALL SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY a +---- +distribution: local +vectorized: true +· +• except all +│ columns: (a, b, c, d, e) +│ ordering: +a,+b,+c,+d,+e +│ estimated row count: 1 (missing stats) +│ +├── • filter +│ │ columns: (a, b, c, d, e) +│ │ ordering: +a +│ │ estimated row count: 1 (missing stats) +│ │ filter: (c = 1) AND (d = e) +│ │ +│ └── • scan +│ columns: (a, b, c, d, e) +│ ordering: +a +│ estimated row count: 1,000 (missing stats) +│ table: abcde@primary +│ spans: FULL SCAN +│ +└── • filter + │ columns: (a, b, c, d, e) + │ ordering: +a + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + ordering: +a + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyskm-L2kAQxt_3UwzzKqlbNH-EsiCsd-ao4BmbhPZK8UUuO7WBXJLurtBD_O4lSeGMqK3tvZyZ_c0zz5PsUP8okGPwsFpM50uwZvM4iT8uGHwKopswDmyIg0Vwm8BbuIvCe7D6ZfqYSYLg4TZYJTBdLODE2IbPH4IoACuDCTg2TJczsCRMgGwIo1kQwc0XSJFhWUlapk-kkX9FB9cMa1VlpHWlmtaufTCXP5GPGOZlvTVNe80wqxQh36HJTUHIMUkfC4oolaSGI2QoyaR50a5tLxK1yp9S9YwM4zotNYd3uN4zrLbmZak26YaQO3v298J3eWFIkRo6fdWuz8ESXpMB53y-TN7_jkL4MAExts-e4F5zwqF39z-9e__k3XtN7_7ZE16Ut2WlJCmSPeF1Q_7pyQkf96Q2FJMJ66Hfd5I818QPfnZkWNA3YwlnwIQ7YMIbMOEPmBgP7InKN99Pj5BhuDUchMOEy4THhM_E-GwC42s-QkS6rkpNx0mc3Dxq7JPcUBenrrYqo5WqslamK8OWaxuStOmmTlfMy27UHHgIOxdhvwc7x7B7EfYuK3tXKLvHsH8RHh8pr_dvfgUAAP__ApejQg== + +query T +EXPLAIN (DISTSQL,VERBOSE) SELECT * FROM (SELECT * FROM abcde INTERSECT ALL SELECT * FROM abcde) WHERE c = 1 AND d = e ORDER BY b, c, d, e, a +---- +distribution: local +vectorized: true +· +• sort +│ columns: (a, b, c, d, e) +│ ordering: +b,+c,+d,+a +│ estimated row count: 1 (missing stats) +│ order: +b,+c,+d,+a +│ +└── • intersect all + │ columns: (a, b, c, d, e) + │ estimated row count: 1 (missing stats) + │ + ├── • filter + │ │ columns: (a, b, c, d, e) + │ │ estimated row count: 1 (missing stats) + │ │ filter: (c = 1) AND (d = e) + │ │ + │ └── • scan + │ columns: (a, b, c, d, e) + │ estimated row count: 1,000 (missing stats) + │ table: abcde@primary + │ spans: FULL SCAN + │ + └── • filter + │ columns: (a, b, c, d, e) + │ estimated row count: 1 (missing stats) + │ filter: (c = 1) AND (d = e) + │ + └── • scan + columns: (a, b, c, d, e) + estimated row count: 1,000 (missing stats) + table: abcde@primary + spans: FULL SCAN +· +Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJyskt-Lm04Uxd-_f8XlPuk3U7L-WspAYLKNywpp3Kr0ByUPRm-zgqt2ZgJdQv73ohZ2TRPbtH2cM_O555yre1RfS-Tof7xfzoMVGIsgTuJ3Swbv_egmjH0TYn_pv0ngf7iNwrdgDI_pJssJglXiR3Erz5dLOPHChA93fuSDkcEMLBPmqwUYOcyATAijhR_BzSfYMMgY5AyIQYoMqzqnVfpICvlntHDNsJF1RkrVspX23YMg_4b8imFRNTvdymuGWS0J-R51oUtCjkm6KSmiNCc5vUKGOem0KLuxXTrRyOIxlU_IMG7SSnF4hesDw3qnn4cqnW4JuXVgv298W5SaJMmpNXTtdQ6GcNp9cM6DVfL6x1qECzMQnnk2gn1JhJfd7b_s7vxRd-dfdnfPRnh23lW1zElSPjBet-SvnpzocZeqh5h02EzdYZHkqSE-_PWRYUlftCEsJmwmHCZcJjxzJovtw08qMgx3msNQPtvcu2T5cS01yak3TCzsCRPOhAl3woQ1OWt1fYlVRKqpK0XHyz45-ardMOVb6r-Yqncyo3tZZ51Nfww7rhNyUrq_tfpDUPVXbcCXsDUKuwPYPobtUdgZd3YucLaOYXcU9sadvVH4-gheH_77HgAA__-rHNsg diff --git a/pkg/sql/opt/exec/factory.opt b/pkg/sql/opt/exec/factory.opt index 4677b200f792..04aed74e76c5 100644 --- a/pkg/sql/opt/exec/factory.opt +++ b/pkg/sql/opt/exec/factory.opt @@ -168,6 +168,14 @@ define Distinct { # DISTINCT version). The left and right nodes must have the same number of # columns. # +# ReqOrdering specifies the required output ordering, and if not empty, both +# inputs are already ordered according to it. If ReqOrdering is set, it is +# guaranteed to include all columns produced by this SetOp (the one exception is +# UNION ALL, which is implemented with only an ordered synchronizer and does not +# require an ordering over all columns). The execution engine is then guaranteed +# to use a merge join (or streaming DISTINCT for UNION), which is a streaming +# operation that maintains ordering. +# # HardLimit can only be set for UNION ALL operations. It is used to implement # locality optimized search, and instructs the execution engine that it should # execute the left node to completion and possibly short-circuit if the limit is @@ -178,6 +186,7 @@ define SetOp { All bool Left exec.Node Right exec.Node + ReqOrdering exec.OutputOrdering HardLimit uint64 } diff --git a/pkg/sql/opt/memo/testdata/stats/set b/pkg/sql/opt/memo/testdata/stats/set index c0bf1ef14e43..2bfe7dca024e 100644 --- a/pkg/sql/opt/memo/testdata/stats/set +++ b/pkg/sql/opt/memo/testdata/stats/set @@ -941,30 +941,25 @@ except opt VALUES (1) INTERSECT VALUES (NULL) ORDER BY 1 ---- -sort +intersect ├── columns: column1:1(int) + ├── left columns: column1:1(int) + ├── right columns: column1:2(int) ├── cardinality: [0 - 1] ├── stats: [rows=1, distinct(1)=1, null(1)=0] ├── key: (1) ├── ordering: +1 - └── intersect - ├── columns: column1:1(int) - ├── left columns: column1:1(int) - ├── right columns: column1:2(int) - ├── cardinality: [0 - 1] - ├── stats: [rows=1, distinct(1)=1, null(1)=0] - ├── key: (1) - ├── values - │ ├── columns: column1:1(int!null) - │ ├── cardinality: [1 - 1] - │ ├── stats: [rows=1, distinct(1)=1, null(1)=0] - │ ├── key: () - │ ├── fd: ()-->(1) - │ └── (1,) [type=tuple{int}] - └── values - ├── columns: column1:2(int) - ├── cardinality: [1 - 1] - ├── stats: [rows=1, distinct(2)=1, null(2)=1] - ├── key: () - ├── fd: ()-->(2) - └── (NULL,) [type=tuple{int}] + ├── values + │ ├── columns: column1:1(int!null) + │ ├── cardinality: [1 - 1] + │ ├── stats: [rows=1, distinct(1)=1, null(1)=0] + │ ├── key: () + │ ├── fd: ()-->(1) + │ └── (1,) [type=tuple{int}] + └── values + ├── columns: column1:2(int) + ├── cardinality: [1 - 1] + ├── stats: [rows=1, distinct(2)=1, null(2)=1] + ├── key: () + ├── fd: ()-->(2) + └── (NULL,) [type=tuple{int}] diff --git a/pkg/sql/opt/optbuilder/testdata/limit b/pkg/sql/opt/optbuilder/testdata/limit index 160ab6f45e03..fd2185b94811 100644 --- a/pkg/sql/opt/optbuilder/testdata/limit +++ b/pkg/sql/opt/optbuilder/testdata/limit @@ -198,21 +198,27 @@ limit ├── columns: column1:3!null ├── internal-ordering: -3 ├── ordering: -3 - ├── sort + ├── union-all │ ├── columns: column1:3!null + │ ├── left columns: column1:1 + │ ├── right columns: column1:2 │ ├── ordering: -3 │ ├── limit hint: 2.00 - │ └── union-all - │ ├── columns: column1:3!null - │ ├── left columns: column1:1 - │ ├── right columns: column1:2 - │ ├── values - │ │ ├── columns: column1:1!null - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (2,) - │ │ └── (2,) + │ ├── sort + │ │ ├── columns: column1:1!null + │ │ ├── ordering: -1 + │ │ ├── limit hint: 2.00 + │ │ └── values + │ │ ├── columns: column1:1!null + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (2,) + │ │ └── (2,) + │ └── sort + │ ├── columns: column1:2!null + │ ├── ordering: -2 + │ ├── limit hint: 2.00 │ └── values │ ├── columns: column1:2!null │ ├── (1,) @@ -228,21 +234,27 @@ limit ├── columns: column1:3!null ├── internal-ordering: -3 ├── ordering: -3 - ├── sort + ├── union-all │ ├── columns: column1:3!null + │ ├── left columns: column1:1 + │ ├── right columns: column1:2 │ ├── ordering: -3 │ ├── limit hint: 2.00 - │ └── union-all - │ ├── columns: column1:3!null - │ ├── left columns: column1:1 - │ ├── right columns: column1:2 - │ ├── values - │ │ ├── columns: column1:1!null - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (2,) - │ │ └── (2,) + │ ├── sort + │ │ ├── columns: column1:1!null + │ │ ├── ordering: -1 + │ │ ├── limit hint: 2.00 + │ │ └── values + │ │ ├── columns: column1:1!null + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (2,) + │ │ └── (2,) + │ └── sort + │ ├── columns: column1:2!null + │ ├── ordering: -2 + │ ├── limit hint: 2.00 │ └── values │ ├── columns: column1:2!null │ ├── (1,) diff --git a/pkg/sql/opt/optbuilder/testdata/union b/pkg/sql/opt/optbuilder/testdata/union index f2ffbdce4f6f..c74394073678 100644 --- a/pkg/sql/opt/optbuilder/testdata/union +++ b/pkg/sql/opt/optbuilder/testdata/union @@ -147,21 +147,27 @@ limit ├── columns: column1:3!null ├── internal-ordering: -3 ├── ordering: -3 - ├── sort + ├── union-all │ ├── columns: column1:3!null + │ ├── left columns: column1:1 + │ ├── right columns: column1:2 │ ├── ordering: -3 │ ├── limit hint: 2.00 - │ └── union-all - │ ├── columns: column1:3!null - │ ├── left columns: column1:1 - │ ├── right columns: column1:2 - │ ├── values - │ │ ├── columns: column1:1!null - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (2,) - │ │ └── (2,) + │ ├── sort + │ │ ├── columns: column1:1!null + │ │ ├── ordering: -1 + │ │ ├── limit hint: 2.00 + │ │ └── values + │ │ ├── columns: column1:1!null + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (2,) + │ │ └── (2,) + │ └── sort + │ ├── columns: column1:2!null + │ ├── ordering: -2 + │ ├── limit hint: 2.00 │ └── values │ ├── columns: column1:2!null │ ├── (1,) @@ -177,21 +183,27 @@ limit ├── columns: column1:3!null ├── internal-ordering: -3 ├── ordering: -3 - ├── sort + ├── union-all │ ├── columns: column1:3!null + │ ├── left columns: column1:1 + │ ├── right columns: column1:2 │ ├── ordering: -3 │ ├── limit hint: 2.00 - │ └── union-all - │ ├── columns: column1:3!null - │ ├── left columns: column1:1 - │ ├── right columns: column1:2 - │ ├── values - │ │ ├── columns: column1:1!null - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (1,) - │ │ ├── (2,) - │ │ └── (2,) + │ ├── sort + │ │ ├── columns: column1:1!null + │ │ ├── ordering: -1 + │ │ ├── limit hint: 2.00 + │ │ └── values + │ │ ├── columns: column1:1!null + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (1,) + │ │ ├── (2,) + │ │ └── (2,) + │ └── sort + │ ├── columns: column1:2!null + │ ├── ordering: -2 + │ ├── limit hint: 2.00 │ └── values │ ├── columns: column1:2!null │ ├── (1,) @@ -203,23 +215,21 @@ limit build VALUES (NULL) UNION ALL VALUES (1) ORDER BY 1 ---- -sort +union-all ├── columns: column1:4 + ├── left columns: column1:3 + ├── right columns: column1:2 ├── ordering: +4 - └── union-all - ├── columns: column1:4 - ├── left columns: column1:3 - ├── right columns: column1:2 - ├── project - │ ├── columns: column1:3 - │ ├── values - │ │ ├── columns: column1:1 - │ │ └── (NULL,) - │ └── projections - │ └── column1:1::INT8 [as=column1:3] - └── values - ├── columns: column1:2!null - └── (1,) + ├── project + │ ├── columns: column1:3 + │ ├── values + │ │ ├── columns: column1:1 + │ │ └── (NULL,) + │ └── projections + │ └── column1:1::INT8 [as=column1:3] + └── values + ├── columns: column1:2!null + └── (1,) build VALUES (NULL) UNION ALL VALUES (NULL) @@ -449,22 +459,28 @@ limit ├── columns: v:9 ├── internal-ordering: -9 ├── ordering: -9 - ├── sort + ├── union-all │ ├── columns: v:9 + │ ├── left columns: uniontest.v:2 + │ ├── right columns: uniontest.v:6 │ ├── ordering: -9 │ ├── limit hint: 2.00 - │ └── union-all - │ ├── columns: v:9 - │ ├── left columns: uniontest.v:2 - │ ├── right columns: uniontest.v:6 - │ ├── project - │ │ ├── columns: uniontest.v:2 - │ │ └── select - │ │ ├── columns: k:1!null uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 - │ │ ├── scan uniontest - │ │ │ └── columns: k:1 uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 - │ │ └── filters - │ │ └── k:1 = 1 + │ ├── sort + │ │ ├── columns: uniontest.v:2 + │ │ ├── ordering: -2 + │ │ ├── limit hint: 2.00 + │ │ └── project + │ │ ├── columns: uniontest.v:2 + │ │ └── select + │ │ ├── columns: k:1!null uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 + │ │ ├── scan uniontest + │ │ │ └── columns: k:1 uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 + │ │ └── filters + │ │ └── k:1 = 1 + │ └── sort + │ ├── columns: uniontest.v:6 + │ ├── ordering: -6 + │ ├── limit hint: 2.00 │ └── project │ ├── columns: uniontest.v:6 │ └── select @@ -483,22 +499,28 @@ limit ├── columns: v:9 ├── internal-ordering: -9 ├── ordering: -9 - ├── sort + ├── union-all │ ├── columns: v:9 + │ ├── left columns: uniontest.v:2 + │ ├── right columns: uniontest.v:6 │ ├── ordering: -9 │ ├── limit hint: 2.00 - │ └── union-all - │ ├── columns: v:9 - │ ├── left columns: uniontest.v:2 - │ ├── right columns: uniontest.v:6 - │ ├── project - │ │ ├── columns: uniontest.v:2 - │ │ └── select - │ │ ├── columns: k:1!null uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 - │ │ ├── scan uniontest - │ │ │ └── columns: k:1 uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 - │ │ └── filters - │ │ └── k:1 = 1 + │ ├── sort + │ │ ├── columns: uniontest.v:2 + │ │ ├── ordering: -2 + │ │ ├── limit hint: 2.00 + │ │ └── project + │ │ ├── columns: uniontest.v:2 + │ │ └── select + │ │ ├── columns: k:1!null uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 + │ │ ├── scan uniontest + │ │ │ └── columns: k:1 uniontest.v:2 rowid:3!null crdb_internal_mvcc_timestamp:4 + │ │ └── filters + │ │ └── k:1 = 1 + │ └── sort + │ ├── columns: uniontest.v:6 + │ ├── ordering: -6 + │ ├── limit hint: 2.00 │ └── project │ ├── columns: uniontest.v:6 │ └── select @@ -709,17 +731,21 @@ union build (SELECT a FROM abc ORDER BY b) UNION ALL (SELECT b FROM abc) ORDER BY a ---- -sort +union-all ├── columns: a:11 + ├── left columns: abc.a:1 + ├── right columns: b:7 ├── ordering: +11 - └── union-all - ├── columns: a:11 - ├── left columns: abc.a:1 - ├── right columns: b:7 - ├── project - │ ├── columns: abc.a:1 b:2!null - │ └── scan abc - │ └── columns: abc.a:1 b:2!null c:3!null rowid:4!null crdb_internal_mvcc_timestamp:5 + ├── sort + │ ├── columns: abc.a:1 b:2!null + │ ├── ordering: +1 + │ └── project + │ ├── columns: abc.a:1 b:2!null + │ └── scan abc + │ └── columns: abc.a:1 b:2!null c:3!null rowid:4!null crdb_internal_mvcc_timestamp:5 + └── sort + ├── columns: b:7!null + ├── ordering: +7 └── project ├── columns: b:7!null └── scan abc @@ -728,17 +754,21 @@ sort build (SELECT a FROM abc ORDER BY b) UNION ALL (SELECT a FROM abc ORDER BY c) ORDER BY a ---- -sort +union-all ├── columns: a:11 + ├── left columns: abc.a:1 + ├── right columns: abc.a:6 ├── ordering: +11 - └── union-all - ├── columns: a:11 - ├── left columns: abc.a:1 - ├── right columns: abc.a:6 - ├── project - │ ├── columns: abc.a:1 b:2!null - │ └── scan abc - │ └── columns: abc.a:1 b:2!null c:3!null rowid:4!null crdb_internal_mvcc_timestamp:5 + ├── sort + │ ├── columns: abc.a:1 b:2!null + │ ├── ordering: +1 + │ └── project + │ ├── columns: abc.a:1 b:2!null + │ └── scan abc + │ └── columns: abc.a:1 b:2!null c:3!null rowid:4!null crdb_internal_mvcc_timestamp:5 + └── sort + ├── columns: abc.a:6 c:8!null + ├── ordering: +6 └── project ├── columns: abc.a:6 c:8!null └── scan abc diff --git a/pkg/sql/opt/ordering/BUILD.bazel b/pkg/sql/opt/ordering/BUILD.bazel index 787e1a56a617..282bf49fe4fe 100644 --- a/pkg/sql/opt/ordering/BUILD.bazel +++ b/pkg/sql/opt/ordering/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "row_number.go", "scan.go", "select.go", + "set.go", "sort.go", "statement.go", ], diff --git a/pkg/sql/opt/ordering/ordering.go b/pkg/sql/opt/ordering/ordering.go index 5a548887a123..8b7c006e30a6 100644 --- a/pkg/sql/opt/ordering/ordering.go +++ b/pkg/sql/opt/ordering/ordering.go @@ -110,6 +110,36 @@ func init() { buildChildReqOrdering: projectBuildChildReqOrdering, buildProvidedOrdering: projectBuildProvided, } + funcMap[opt.UnionOp] = funcs{ + canProvideOrdering: setOpCanProvideOrdering, + buildChildReqOrdering: setOpBuildChildReqOrdering, + buildProvidedOrdering: setOpBuildProvided, + } + funcMap[opt.UnionAllOp] = funcs{ + canProvideOrdering: setOpCanProvideOrdering, + buildChildReqOrdering: setOpBuildChildReqOrdering, + buildProvidedOrdering: setOpBuildProvided, + } + funcMap[opt.IntersectOp] = funcs{ + canProvideOrdering: setOpCanProvideOrdering, + buildChildReqOrdering: setOpBuildChildReqOrdering, + buildProvidedOrdering: setOpBuildProvided, + } + funcMap[opt.IntersectAllOp] = funcs{ + canProvideOrdering: setOpCanProvideOrdering, + buildChildReqOrdering: setOpBuildChildReqOrdering, + buildProvidedOrdering: setOpBuildProvided, + } + funcMap[opt.ExceptOp] = funcs{ + canProvideOrdering: setOpCanProvideOrdering, + buildChildReqOrdering: setOpBuildChildReqOrdering, + buildProvidedOrdering: setOpBuildProvided, + } + funcMap[opt.ExceptAllOp] = funcs{ + canProvideOrdering: setOpCanProvideOrdering, + buildChildReqOrdering: setOpBuildChildReqOrdering, + buildProvidedOrdering: setOpBuildProvided, + } funcMap[opt.IndexJoinOp] = funcs{ canProvideOrdering: lookupOrIndexJoinCanProvideOrdering, buildChildReqOrdering: lookupOrIndexJoinBuildChildReqOrdering, diff --git a/pkg/sql/opt/ordering/set.go b/pkg/sql/opt/ordering/set.go new file mode 100644 index 000000000000..1e02ba3d05dc --- /dev/null +++ b/pkg/sql/opt/ordering/set.go @@ -0,0 +1,130 @@ +// Copyright 2021 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package ordering + +import ( + "github.com/cockroachdb/cockroach/pkg/sql/opt" + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/opt/props/physical" +) + +func setOpCanProvideOrdering(expr memo.RelExpr, required *physical.OrderingChoice) bool { + // Set operations can provide any ordering by requiring that both inputs have + // the same ordering. + return true +} + +func setOpBuildChildReqOrdering( + parent memo.RelExpr, required *physical.OrderingChoice, childIdx int, +) physical.OrderingChoice { + if childIdx != 0 && childIdx != 1 { + return physical.OrderingChoice{} + } + + required = setOpBuildRequired(parent, required) + private := parent.Private().(*memo.SetPrivate) + var childReq physical.OrderingChoice + switch childIdx { + case 0: + childReq = required.RemapColumns(private.OutCols, private.LeftCols) + + case 1: + childReq = required.RemapColumns(private.OutCols, private.RightCols) + + default: + return physical.OrderingChoice{} + } + + // Try to simplify the required ordering in case some of the ordering columns + // are constant in the input. + fds := &parent.Child(childIdx).(memo.RelExpr).Relational().FuncDeps + if childReq.CanSimplify(fds) { + childReq.Simplify(fds) + } + return childReq +} + +func setOpBuildProvided(expr memo.RelExpr, required *physical.OrderingChoice) opt.Ordering { + // Set operations can always provide the required ordering. Don't use the + // provided ordering from the inputs in case they were trimmed to remove + // constant columns. Call remapProvided to remove columns that are now + // unnecessary (e.g. because the set op is guaranteed to provide at most one + // row). + rel := expr.Relational() + return remapProvided(required.ToOrdering(), &rel.FuncDeps, rel.OutputCols) +} + +// setOpBuildRequired pads the required ordering if needed to ensure that it +// includes all output columns of the set operation. This is necessary because +// the execution engine can only use a streaming (merge join or distinct) +// operation if the ordering involves all columns. +func setOpBuildRequired( + expr memo.RelExpr, required *physical.OrderingChoice, +) *physical.OrderingChoice { + if required.Any() { + return required + } + + // UNION ALL is implemented with only an ordered synchronizer, so there is no + // need to add extra ordering columns. + if expr.Op() == opt.UnionAllOp { + return required + } + + // If required includes some columns but not all, add the remaining columns in + // an arbitrary (but deterministic) order. + // TODO(rytaft): Use an "interesting ordering" provided from left side + // instead. + missing := expr.Relational().OutputCols.Difference(required.ColSet()) + if !missing.Empty() { + copy := required.Copy() + missing.ForEach(func(col opt.ColumnID) { + copy.AppendCol(col, false /* descending */) + }) + fds := &expr.Relational().FuncDeps + if copy.CanSimplify(fds) { + copy.Simplify(fds) + } + required = © + } + + return required +} + +// StreamingSetOpOrdering returns an ordering on the set operation output +// columns that is guaranteed on both inputs. This ordering can be used to +// perform a streaming set operation. +func StreamingSetOpOrdering(expr memo.RelExpr, required *physical.OrderingChoice) opt.Ordering { + required = setOpBuildRequired(expr, required) + ordering := required.ToOrdering() + if ordering.Empty() { + return ordering + } + + // UNION ALL is implemented with only an ordered synchronizer, so there is no + // need to add extra ordering columns. + if expr.Op() == opt.UnionAllOp { + return ordering + } + + // Pad the ordering to make sure every column is accounted for in the + // ordering. This won't change the order of data (setOpBuildRequired already + // ensured the required ordering was fully specified according to the FDs), + // but it's necessary for the execution engine to plan a streaming operation. + // TODO(rytaft): Consider changing the execution engine to accept the + // optimizer's decision to plan a streaming operation even if all columns are + // not included in the ordering. + missing := expr.Relational().OutputCols.Difference(ordering.ColSet()) + missing.ForEach(func(col opt.ColumnID) { + ordering = append(ordering, opt.MakeOrderingColumn(col, false /* descending */)) + }) + return ordering +} diff --git a/pkg/sql/opt/props/physical/ordering_choice.go b/pkg/sql/opt/props/physical/ordering_choice.go index 387cf4edcb50..6836193a922e 100644 --- a/pkg/sql/opt/props/physical/ordering_choice.go +++ b/pkg/sql/opt/props/physical/ordering_choice.go @@ -776,6 +776,22 @@ func (oc OrderingChoice) Format(buf *bytes.Buffer) { } } +// RemapColumns returns a copy of oc with all columns in from mapped to columns +// in to. +func (oc *OrderingChoice) RemapColumns(from, to opt.ColList) OrderingChoice { + var other OrderingChoice + other.Optional = opt.TranslateColSet(oc.Optional, from, to) + other.Columns = make([]OrderingColumnChoice, len(oc.Columns)) + for i := range oc.Columns { + col := &oc.Columns[i] + other.Columns[i] = OrderingColumnChoice{ + Group: opt.TranslateColSet(col.Group, from, to), + Descending: col.Descending, + } + } + return other +} + // AnyID returns the ID of an arbitrary member of the group of equivalent // columns. func (oc *OrderingColumnChoice) AnyID() opt.ColumnID { diff --git a/pkg/sql/opt/xform/testdata/external/trading b/pkg/sql/opt/xform/testdata/external/trading index a1aa94877983..5462396f47c8 100644 --- a/pkg/sql/opt/xform/testdata/external/trading +++ b/pkg/sql/opt/xform/testdata/external/trading @@ -603,62 +603,65 @@ project │ │ ├── stats: [rows=1] │ │ ├── key: () │ │ ├── fd: ()-->(8,16) - │ │ ├── sort + │ │ ├── union │ │ │ ├── columns: dealerid:8!null version:16!null + │ │ │ ├── left columns: dealerid:75 version:83 + │ │ │ ├── right columns: dealerid:85 version:93 │ │ │ ├── cardinality: [0 - 4] │ │ │ ├── stats: [rows=4, distinct(8,16)=4, null(8,16)=0] │ │ │ ├── key: (8,16) │ │ │ ├── ordering: -16 │ │ │ ├── limit hint: 1.00 - │ │ │ └── union - │ │ │ ├── columns: dealerid:8!null version:16!null - │ │ │ ├── left columns: dealerid:75 version:83 - │ │ │ ├── right columns: dealerid:85 version:93 - │ │ │ ├── cardinality: [0 - 4] - │ │ │ ├── stats: [rows=4, distinct(8,16)=4, null(8,16)=0] - │ │ │ ├── key: (8,16) - │ │ │ ├── union - │ │ │ │ ├── columns: dealerid:75!null version:83!null - │ │ │ │ ├── left columns: dealerid:55 version:63 - │ │ │ │ ├── right columns: dealerid:65 version:73 - │ │ │ │ ├── cardinality: [0 - 3] - │ │ │ │ ├── stats: [rows=3, distinct(75,83)=3, null(75,83)=0] - │ │ │ │ ├── key: (75,83) - │ │ │ │ ├── union - │ │ │ │ │ ├── columns: dealerid:55!null version:63!null - │ │ │ │ │ ├── left columns: dealerid:35 version:43 - │ │ │ │ │ ├── right columns: dealerid:45 version:53 - │ │ │ │ │ ├── cardinality: [0 - 2] - │ │ │ │ │ ├── stats: [rows=2, distinct(55,63)=2, null(55,63)=0] - │ │ │ │ │ ├── key: (55,63) - │ │ │ │ │ ├── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ │ │ │ ├── columns: dealerid:35!null version:43!null - │ │ │ │ │ │ ├── constraint: /35/43: [/1 - /1] - │ │ │ │ │ │ ├── limit: 1(rev) - │ │ │ │ │ │ ├── stats: [rows=1, distinct(35)=1, null(35)=0, distinct(35,43)=1, null(35,43)=0] - │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ └── fd: ()-->(35,43) - │ │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ │ │ ├── columns: dealerid:45!null version:53!null - │ │ │ │ │ ├── constraint: /45/53: [/2 - /2] - │ │ │ │ │ ├── limit: 1(rev) - │ │ │ │ │ ├── stats: [rows=1, distinct(45)=1, null(45)=0, distinct(45,53)=1, null(45,53)=0] - │ │ │ │ │ ├── key: () - │ │ │ │ │ └── fd: ()-->(45,53) - │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ │ ├── columns: dealerid:65!null version:73!null - │ │ │ │ ├── constraint: /65/73: [/3 - /3] - │ │ │ │ ├── limit: 1(rev) - │ │ │ │ ├── stats: [rows=1, distinct(65)=1, null(65)=0, distinct(65,73)=1, null(65,73)=0] - │ │ │ │ ├── key: () - │ │ │ │ └── fd: ()-->(65,73) - │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ ├── columns: dealerid:85!null version:93!null - │ │ │ ├── constraint: /85/93: [/4 - /4] - │ │ │ ├── limit: 1(rev) - │ │ │ ├── stats: [rows=1, distinct(85)=1, null(85)=0, distinct(85,93)=1, null(85,93)=0] - │ │ │ ├── key: () - │ │ │ └── fd: ()-->(85,93) + │ │ │ ├── union + │ │ │ │ ├── columns: dealerid:75!null version:83!null + │ │ │ │ ├── left columns: dealerid:55 version:63 + │ │ │ │ ├── right columns: dealerid:65 version:73 + │ │ │ │ ├── cardinality: [0 - 3] + │ │ │ │ ├── stats: [rows=3, distinct(75,83)=3, null(75,83)=0] + │ │ │ │ ├── key: (75,83) + │ │ │ │ ├── ordering: -83,+75 + │ │ │ │ ├── limit hint: 1.00 + │ │ │ │ ├── union + │ │ │ │ │ ├── columns: dealerid:55!null version:63!null + │ │ │ │ │ ├── left columns: dealerid:35 version:43 + │ │ │ │ │ ├── right columns: dealerid:45 version:53 + │ │ │ │ │ ├── cardinality: [0 - 2] + │ │ │ │ │ ├── stats: [rows=2, distinct(55,63)=2, null(55,63)=0] + │ │ │ │ │ ├── key: (55,63) + │ │ │ │ │ ├── ordering: -63,+55 + │ │ │ │ │ ├── limit hint: 1.00 + │ │ │ │ │ ├── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ │ │ │ ├── columns: dealerid:35!null version:43!null + │ │ │ │ │ │ ├── constraint: /35/43: [/1 - /1] + │ │ │ │ │ │ ├── limit: 1(rev) + │ │ │ │ │ │ ├── stats: [rows=1, distinct(35)=1, null(35)=0, distinct(35,43)=1, null(35,43)=0] + │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ ├── fd: ()-->(35,43) + │ │ │ │ │ │ └── limit hint: 1.00 + │ │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ │ │ ├── columns: dealerid:45!null version:53!null + │ │ │ │ │ ├── constraint: /45/53: [/2 - /2] + │ │ │ │ │ ├── limit: 1(rev) + │ │ │ │ │ ├── stats: [rows=1, distinct(45)=1, null(45)=0, distinct(45,53)=1, null(45,53)=0] + │ │ │ │ │ ├── key: () + │ │ │ │ │ ├── fd: ()-->(45,53) + │ │ │ │ │ └── limit hint: 1.00 + │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ │ ├── columns: dealerid:65!null version:73!null + │ │ │ │ ├── constraint: /65/73: [/3 - /3] + │ │ │ │ ├── limit: 1(rev) + │ │ │ │ ├── stats: [rows=1, distinct(65)=1, null(65)=0, distinct(65,73)=1, null(65,73)=0] + │ │ │ │ ├── key: () + │ │ │ │ ├── fd: ()-->(65,73) + │ │ │ │ └── limit hint: 1.00 + │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ ├── columns: dealerid:85!null version:93!null + │ │ │ ├── constraint: /85/93: [/4 - /4] + │ │ │ ├── limit: 1(rev) + │ │ │ ├── stats: [rows=1, distinct(85)=1, null(85)=0, distinct(85,93)=1, null(85,93)=0] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(85,93) + │ │ │ └── limit hint: 1.00 │ │ └── 1 │ └── aggregations │ └── const-agg [as=max:33, outer=(16)] diff --git a/pkg/sql/opt/xform/testdata/external/trading-mutation b/pkg/sql/opt/xform/testdata/external/trading-mutation index 20bc1c93615a..77f5e2de523c 100644 --- a/pkg/sql/opt/xform/testdata/external/trading-mutation +++ b/pkg/sql/opt/xform/testdata/external/trading-mutation @@ -609,62 +609,65 @@ project │ │ ├── stats: [rows=1] │ │ ├── key: () │ │ ├── fd: ()-->(8,16) - │ │ ├── sort + │ │ ├── union │ │ │ ├── columns: dealerid:8!null version:16!null + │ │ │ ├── left columns: dealerid:95 version:103 + │ │ │ ├── right columns: dealerid:109 version:117 │ │ │ ├── cardinality: [0 - 4] │ │ │ ├── stats: [rows=4, distinct(8,16)=4, null(8,16)=0] │ │ │ ├── key: (8,16) │ │ │ ├── ordering: -16 │ │ │ ├── limit hint: 1.00 - │ │ │ └── union - │ │ │ ├── columns: dealerid:8!null version:16!null - │ │ │ ├── left columns: dealerid:95 version:103 - │ │ │ ├── right columns: dealerid:109 version:117 - │ │ │ ├── cardinality: [0 - 4] - │ │ │ ├── stats: [rows=4, distinct(8,16)=4, null(8,16)=0] - │ │ │ ├── key: (8,16) - │ │ │ ├── union - │ │ │ │ ├── columns: dealerid:95!null version:103!null - │ │ │ │ ├── left columns: dealerid:67 version:75 - │ │ │ │ ├── right columns: dealerid:81 version:89 - │ │ │ │ ├── cardinality: [0 - 3] - │ │ │ │ ├── stats: [rows=3, distinct(95,103)=3, null(95,103)=0] - │ │ │ │ ├── key: (95,103) - │ │ │ │ ├── union - │ │ │ │ │ ├── columns: dealerid:67!null version:75!null - │ │ │ │ │ ├── left columns: dealerid:39 version:47 - │ │ │ │ │ ├── right columns: dealerid:53 version:61 - │ │ │ │ │ ├── cardinality: [0 - 2] - │ │ │ │ │ ├── stats: [rows=2, distinct(67,75)=2, null(67,75)=0] - │ │ │ │ │ ├── key: (67,75) - │ │ │ │ │ ├── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ │ │ │ ├── columns: dealerid:39!null version:47!null - │ │ │ │ │ │ ├── constraint: /39/47: [/1 - /1] - │ │ │ │ │ │ ├── limit: 1(rev) - │ │ │ │ │ │ ├── stats: [rows=1, distinct(39)=1, null(39)=0, distinct(39,47)=1, null(39,47)=0] - │ │ │ │ │ │ ├── key: () - │ │ │ │ │ │ └── fd: ()-->(39,47) - │ │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ │ │ ├── columns: dealerid:53!null version:61!null - │ │ │ │ │ ├── constraint: /53/61: [/2 - /2] - │ │ │ │ │ ├── limit: 1(rev) - │ │ │ │ │ ├── stats: [rows=1, distinct(53)=1, null(53)=0, distinct(53,61)=1, null(53,61)=0] - │ │ │ │ │ ├── key: () - │ │ │ │ │ └── fd: ()-->(53,61) - │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ │ ├── columns: dealerid:81!null version:89!null - │ │ │ │ ├── constraint: /81/89: [/3 - /3] - │ │ │ │ ├── limit: 1(rev) - │ │ │ │ ├── stats: [rows=1, distinct(81)=1, null(81)=0, distinct(81,89)=1, null(81,89)=0] - │ │ │ │ ├── key: () - │ │ │ │ └── fd: ()-->(81,89) - │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev - │ │ │ ├── columns: dealerid:109!null version:117!null - │ │ │ ├── constraint: /109/117: [/4 - /4] - │ │ │ ├── limit: 1(rev) - │ │ │ ├── stats: [rows=1, distinct(109)=1, null(109)=0, distinct(109,117)=1, null(109,117)=0] - │ │ │ ├── key: () - │ │ │ └── fd: ()-->(109,117) + │ │ │ ├── union + │ │ │ │ ├── columns: dealerid:95!null version:103!null + │ │ │ │ ├── left columns: dealerid:67 version:75 + │ │ │ │ ├── right columns: dealerid:81 version:89 + │ │ │ │ ├── cardinality: [0 - 3] + │ │ │ │ ├── stats: [rows=3, distinct(95,103)=3, null(95,103)=0] + │ │ │ │ ├── key: (95,103) + │ │ │ │ ├── ordering: -103,+95 + │ │ │ │ ├── limit hint: 1.00 + │ │ │ │ ├── union + │ │ │ │ │ ├── columns: dealerid:67!null version:75!null + │ │ │ │ │ ├── left columns: dealerid:39 version:47 + │ │ │ │ │ ├── right columns: dealerid:53 version:61 + │ │ │ │ │ ├── cardinality: [0 - 2] + │ │ │ │ │ ├── stats: [rows=2, distinct(67,75)=2, null(67,75)=0] + │ │ │ │ │ ├── key: (67,75) + │ │ │ │ │ ├── ordering: -75,+67 + │ │ │ │ │ ├── limit hint: 1.00 + │ │ │ │ │ ├── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ │ │ │ ├── columns: dealerid:39!null version:47!null + │ │ │ │ │ │ ├── constraint: /39/47: [/1 - /1] + │ │ │ │ │ │ ├── limit: 1(rev) + │ │ │ │ │ │ ├── stats: [rows=1, distinct(39)=1, null(39)=0, distinct(39,47)=1, null(39,47)=0] + │ │ │ │ │ │ ├── key: () + │ │ │ │ │ │ ├── fd: ()-->(39,47) + │ │ │ │ │ │ └── limit hint: 1.00 + │ │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ │ │ ├── columns: dealerid:53!null version:61!null + │ │ │ │ │ ├── constraint: /53/61: [/2 - /2] + │ │ │ │ │ ├── limit: 1(rev) + │ │ │ │ │ ├── stats: [rows=1, distinct(53)=1, null(53)=0, distinct(53,61)=1, null(53,61)=0] + │ │ │ │ │ ├── key: () + │ │ │ │ │ ├── fd: ()-->(53,61) + │ │ │ │ │ └── limit hint: 1.00 + │ │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ │ ├── columns: dealerid:81!null version:89!null + │ │ │ │ ├── constraint: /81/89: [/3 - /3] + │ │ │ │ ├── limit: 1(rev) + │ │ │ │ ├── stats: [rows=1, distinct(81)=1, null(81)=0, distinct(81,89)=1, null(81,89)=0] + │ │ │ │ ├── key: () + │ │ │ │ ├── fd: ()-->(81,89) + │ │ │ │ └── limit hint: 1.00 + │ │ │ └── scan cardsinfo@cardsinfoversionindex,rev + │ │ │ ├── columns: dealerid:109!null version:117!null + │ │ │ ├── constraint: /109/117: [/4 - /4] + │ │ │ ├── limit: 1(rev) + │ │ │ ├── stats: [rows=1, distinct(109)=1, null(109)=0, distinct(109,117)=1, null(109,117)=0] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(109,117) + │ │ │ └── limit hint: 1.00 │ │ └── 1 │ └── aggregations │ └── const-agg [as=max:37, outer=(16)] diff --git a/pkg/sql/opt/xform/testdata/physprops/ordering b/pkg/sql/opt/xform/testdata/physprops/ordering index 5aedf0e7420d..b737288198cb 100644 --- a/pkg/sql/opt/xform/testdata/physprops/ordering +++ b/pkg/sql/opt/xform/testdata/physprops/ordering @@ -1443,6 +1443,223 @@ distinct-on └── first-agg [as=b:2, outer=(2)] └── b:2 +# -------------------------------------------------- +# Set Operations. +# -------------------------------------------------- + +opt +SELECT * FROM (SELECT a, b, c FROM abc UNION SELECT y, x, z FROM xyz) WHERE a = b ORDER BY a +---- +union + ├── columns: a:9!null b:10!null c:11!null + ├── left columns: abc.a:1 abc.b:2 abc.c:3 + ├── right columns: y:6 x:5 z:7 + ├── key: (10,11) + ├── fd: (9)==(10), (10)==(9) + ├── ordering: +(9|10) [actual: +9] + ├── sort (segmented) + │ ├── columns: abc.a:1!null abc.b:2!null abc.c:3!null + │ ├── key: (2,3) + │ ├── fd: (1)==(2), (2)==(1) + │ ├── ordering: +(1|2),+3 [actual: +1,+3] + │ └── select + │ ├── columns: abc.a:1!null abc.b:2!null abc.c:3!null + │ ├── key: (2,3) + │ ├── fd: (1)==(2), (2)==(1) + │ ├── ordering: +1 + │ ├── scan abc + │ │ ├── columns: abc.a:1!null abc.b:2!null abc.c:3!null + │ │ ├── key: (1-3) + │ │ └── ordering: +1 + │ └── filters + │ └── abc.a:1 = abc.b:2 [outer=(1,2), fd=(1)==(2), (2)==(1)] + └── sort (segmented) + ├── columns: x:5!null y:6!null z:7!null + ├── key: (6,7) + ├── fd: (5)==(6), (6)==(5) + ├── ordering: +(5|6),+7 [actual: +5,+7] + └── select + ├── columns: x:5!null y:6!null z:7!null + ├── key: (6,7) + ├── fd: (5)==(6), (6)==(5) + ├── ordering: +5 + ├── scan xyz + │ ├── columns: x:5!null y:6!null z:7!null + │ ├── key: (5-7) + │ └── ordering: +5 + └── filters + └── y:6 = x:5 [outer=(5,6), fd=(5)==(6), (6)==(5)] + +opt +SELECT * FROM (SELECT a, b, c, d FROM abcd UNION ALL SELECT c, d, a, b FROM abcd) ORDER BY a, b +---- +union-all + ├── columns: a:13 b:14 c:15 d:16 + ├── left columns: abcd.a:1 abcd.b:2 abcd.c:3 abcd.d:4 + ├── right columns: abcd.c:9 abcd.d:10 abcd.a:7 abcd.b:8 + ├── ordering: +13,+14 + ├── scan abcd@ab + │ ├── columns: abcd.a:1 abcd.b:2 abcd.c:3 abcd.d:4 + │ └── ordering: +1,+2 + └── scan abcd@cd + ├── columns: abcd.a:7 abcd.b:8 abcd.c:9 abcd.d:10 + └── ordering: +9,+10 + +opt +SELECT * FROM (SELECT a, b, c, d FROM abcd INTERSECT SELECT c, d, a, b FROM abcd) ORDER BY c, d +---- +intersect + ├── columns: a:1 b:2 c:3 d:4 + ├── left columns: a:1 b:2 c:3 d:4 + ├── right columns: c:9 d:10 a:7 b:8 + ├── key: (1-4) + ├── ordering: +3,+4 + ├── sort (segmented) + │ ├── columns: a:1 b:2 c:3 d:4 + │ ├── ordering: +3,+4,+1,+2 + │ └── scan abcd@cd + │ ├── columns: a:1 b:2 c:3 d:4 + │ └── ordering: +3,+4 + └── sort (segmented) + ├── columns: a:7 b:8 c:9 d:10 + ├── ordering: +7,+8,+9,+10 + └── scan abcd@ab + ├── columns: a:7 b:8 c:9 d:10 + └── ordering: +7,+8 + +opt +SELECT * FROM (SELECT a, b, c FROM abc INTERSECT ALL SELECT y, x, z FROM xyz) WHERE a = b ORDER BY b +---- +sort + ├── columns: a:1!null b:2!null c:3!null + ├── fd: (1)==(2), (2)==(1) + ├── ordering: +(1|2) [actual: +1] + └── intersect-all + ├── columns: a:1!null b:2!null c:3!null + ├── left columns: a:1!null b:2!null c:3!null + ├── right columns: y:6 x:5 z:7 + ├── fd: (1)==(2), (2)==(1) + ├── select + │ ├── columns: a:1!null b:2!null c:3!null + │ ├── key: (2,3) + │ ├── fd: (1)==(2), (2)==(1) + │ ├── scan abc + │ │ ├── columns: a:1!null b:2!null c:3!null + │ │ └── key: (1-3) + │ └── filters + │ └── a:1 = b:2 [outer=(1,2), fd=(1)==(2), (2)==(1)] + └── select + ├── columns: x:5!null y:6!null z:7!null + ├── key: (6,7) + ├── fd: (5)==(6), (6)==(5) + ├── scan xyz + │ ├── columns: x:5!null y:6!null z:7!null + │ └── key: (5-7) + └── filters + └── y:6 = x:5 [outer=(5,6), fd=(5)==(6), (6)==(5)] + +opt +SELECT * FROM (SELECT a, b, c FROM abc EXCEPT SELECT z, y, x FROM xyz) WHERE a = c ORDER BY a, b +---- +except + ├── columns: a:1!null b:2!null c:3!null + ├── left columns: a:1!null b:2!null c:3!null + ├── right columns: z:7 y:6 x:5 + ├── key: (2,3) + ├── fd: (1)==(3), (3)==(1) + ├── ordering: +(1|3),+2 [actual: +1,+2] + ├── select + │ ├── columns: a:1!null b:2!null c:3!null + │ ├── key: (2,3) + │ ├── fd: (1)==(3), (3)==(1) + │ ├── ordering: +(1|3),+2 [actual: +1,+2] + │ ├── scan abc + │ │ ├── columns: a:1!null b:2!null c:3!null + │ │ ├── key: (1-3) + │ │ └── ordering: +1,+2 + │ └── filters + │ └── a:1 = c:3 [outer=(1,3), fd=(1)==(3), (3)==(1)] + └── select + ├── columns: x:5!null y:6!null z:7!null + ├── key: (6,7) + ├── fd: (5)==(7), (7)==(5) + ├── ordering: +(5|7),+6 [actual: +5,+6] + ├── scan xyz + │ ├── columns: x:5!null y:6!null z:7!null + │ ├── key: (5-7) + │ └── ordering: +5,+6 + └── filters + └── z:7 = x:5 [outer=(5,7), fd=(5)==(7), (7)==(5)] + +opt +SELECT * FROM (SELECT a, b, c, d FROM abcd EXCEPT ALL SELECT c, d, a, b FROM abcd) ORDER BY a, b +---- +except-all + ├── columns: a:1 b:2 c:3 d:4 + ├── left columns: a:1 b:2 c:3 d:4 + ├── right columns: c:9 d:10 a:7 b:8 + ├── ordering: +1,+2 + ├── sort (segmented) + │ ├── columns: a:1 b:2 c:3 d:4 + │ ├── ordering: +1,+2,+3,+4 + │ └── scan abcd@ab + │ ├── columns: a:1 b:2 c:3 d:4 + │ └── ordering: +1,+2 + └── sort (segmented) + ├── columns: a:7 b:8 c:9 d:10 + ├── ordering: +9,+10,+7,+8 + └── scan abcd@cd + ├── columns: a:7 b:8 c:9 d:10 + └── ordering: +9,+10 + +opt +VALUES (1) UNION ALL VALUES (NULL) ORDER BY 1 +---- +union-all + ├── columns: column1:3 + ├── left columns: column1:1 + ├── right columns: column1:2 + ├── cardinality: [2 - 2] + ├── ordering: +3 + ├── values + │ ├── columns: column1:1!null + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(1) + │ └── (1,) + └── values + ├── columns: column1:2 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(2) + └── (NULL,) + +# TODO(rytaft): We could remove the ordering from the INTERSECT operation +# if we updated the FDs to show that column1 is constant (the cardinality +# proves this to be true). +opt +VALUES (1) INTERSECT VALUES (1) ORDER BY 1 +---- +intersect + ├── columns: column1:1!null + ├── left columns: column1:1!null + ├── right columns: column1:2 + ├── cardinality: [0 - 1] + ├── key: (1) + ├── ordering: +1 + ├── values + │ ├── columns: column1:1!null + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(1) + │ └── (1,) + └── values + ├── columns: column1:2!null + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(2) + └── (1,) + # -------------------------------------------------- # Insert operator. # -------------------------------------------------- diff --git a/pkg/sql/opt/xform/testdata/rules/limit b/pkg/sql/opt/xform/testdata/rules/limit index f1b926445d63..1e8088d717a4 100644 --- a/pkg/sql/opt/xform/testdata/rules/limit +++ b/pkg/sql/opt/xform/testdata/rules/limit @@ -305,30 +305,30 @@ index-join kuv ├── key: (4) ├── fd: (4)-->(1,2) ├── ordering: +2 - ├── sort + ├── union │ ├── columns: k:1!null u:2 rowid:4!null + │ ├── left columns: k:6 u:7 rowid:9 + │ ├── right columns: k:11 u:12 rowid:14 │ ├── cardinality: [0 - 10] │ ├── key: (1,2,4) │ ├── ordering: +2 │ ├── limit hint: 5.00 - │ └── union - │ ├── columns: k:1!null u:2 rowid:4!null - │ ├── left columns: k:6 u:7 rowid:9 - │ ├── right columns: k:11 u:12 rowid:14 - │ ├── cardinality: [0 - 10] - │ ├── key: (1,2,4) - │ ├── scan kuv@secondary - │ │ ├── columns: k:6!null u:7 rowid:9!null - │ │ ├── constraint: /6/7/9: [/1 - /1] - │ │ ├── limit: 5 - │ │ ├── key: (9) - │ │ └── fd: ()-->(6), (9)-->(7) - │ └── scan kuv@secondary - │ ├── columns: k:11!null u:12 rowid:14!null - │ ├── constraint: /11/12/14: [/2 - /2] - │ ├── limit: 5 - │ ├── key: (14) - │ └── fd: ()-->(11), (14)-->(12) + │ ├── scan kuv@secondary + │ │ ├── columns: k:6!null u:7 rowid:9!null + │ │ ├── constraint: /6/7/9: [/1 - /1] + │ │ ├── limit: 5 + │ │ ├── key: (9) + │ │ ├── fd: ()-->(6), (9)-->(7) + │ │ ├── ordering: +7,+9 opt(6) [actual: +7,+9] + │ │ └── limit hint: 5.00 + │ └── scan kuv@secondary + │ ├── columns: k:11!null u:12 rowid:14!null + │ ├── constraint: /11/12/14: [/2 - /2] + │ ├── limit: 5 + │ ├── key: (14) + │ ├── fd: ()-->(11), (14)-->(12) + │ ├── ordering: +12,+14 opt(11) [actual: +12,+14] + │ └── limit hint: 5.00 └── 5 # Ensure that the limit is not pushed down when the ordering requires columns @@ -834,39 +834,43 @@ limit ├── internal-ordering: +6 ├── cardinality: [0 - 10] ├── ordering: +6 - ├── sort + ├── union │ ├── columns: val:2!null data1:6!null + │ ├── left columns: val:32 data1:36 + │ ├── right columns: val:42 data1:46 │ ├── cardinality: [0 - 30] │ ├── key: (2,6) │ ├── ordering: +6 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: val:2!null data1:6!null - │ ├── left columns: val:32 data1:36 - │ ├── right columns: val:42 data1:46 - │ ├── cardinality: [0 - 30] - │ ├── key: (2,6) - │ ├── union - │ │ ├── columns: val:32!null data1:36!null - │ │ ├── left columns: val:12 data1:16 - │ │ ├── right columns: val:22 data1:26 - │ │ ├── cardinality: [0 - 20] - │ │ ├── key: (32,36) - │ │ ├── scan index_tab@b - │ │ │ ├── columns: val:12!null data1:16!null - │ │ │ ├── constraint: /12/16/17/11: [/1 - /1] - │ │ │ ├── limit: 10 - │ │ │ └── fd: ()-->(12) - │ │ └── scan index_tab@b - │ │ ├── columns: val:22!null data1:26!null - │ │ ├── constraint: /22/26/27/21: [/2 - /2] - │ │ ├── limit: 10 - │ │ └── fd: ()-->(22) - │ └── scan index_tab@b - │ ├── columns: val:42!null data1:46!null - │ ├── constraint: /42/46/47/41: [/3 - /3] - │ ├── limit: 10 - │ └── fd: ()-->(42) + │ ├── union + │ │ ├── columns: val:32!null data1:36!null + │ │ ├── left columns: val:12 data1:16 + │ │ ├── right columns: val:22 data1:26 + │ │ ├── cardinality: [0 - 20] + │ │ ├── key: (32,36) + │ │ ├── ordering: +36,+32 + │ │ ├── limit hint: 10.00 + │ │ ├── scan index_tab@b + │ │ │ ├── columns: val:12!null data1:16!null + │ │ │ ├── constraint: /12/16/17/11: [/1 - /1] + │ │ │ ├── limit: 10 + │ │ │ ├── fd: ()-->(12) + │ │ │ ├── ordering: +16 opt(12) [actual: +16] + │ │ │ └── limit hint: 10.00 + │ │ └── scan index_tab@b + │ │ ├── columns: val:22!null data1:26!null + │ │ ├── constraint: /22/26/27/21: [/2 - /2] + │ │ ├── limit: 10 + │ │ ├── fd: ()-->(22) + │ │ ├── ordering: +26 opt(22) [actual: +26] + │ │ └── limit hint: 10.00 + │ └── scan index_tab@b + │ ├── columns: val:42!null data1:46!null + │ ├── constraint: /42/46/47/41: [/3 - /3] + │ ├── limit: 10 + │ ├── fd: ()-->(42) + │ ├── ordering: +46 opt(42) [actual: +46] + │ └── limit hint: 10.00 └── 10 # Case with single-key spans. @@ -884,30 +888,28 @@ scalar-group-by │ ├── cardinality: [0 - 1] │ ├── key: () │ ├── fd: ()-->(3,6) - │ ├── sort + │ ├── union │ │ ├── columns: region:3!null data1:6!null + │ │ ├── left columns: region:14 data1:17 + │ │ ├── right columns: region:24 data1:27 │ │ ├── cardinality: [0 - 2] │ │ ├── key: (3,6) │ │ ├── ordering: -6 │ │ ├── limit hint: 1.00 - │ │ └── union - │ │ ├── columns: region:3!null data1:6!null - │ │ ├── left columns: region:14 data1:17 - │ │ ├── right columns: region:24 data1:27 - │ │ ├── cardinality: [0 - 2] - │ │ ├── key: (3,6) - │ │ ├── scan index_tab@c,rev - │ │ │ ├── columns: region:14!null data1:17!null - │ │ │ ├── constraint: /14/17/18/12: [/'US_EAST' - /'US_EAST'] - │ │ │ ├── limit: 1(rev) - │ │ │ ├── key: () - │ │ │ └── fd: ()-->(14,17) - │ │ └── scan index_tab@c,rev - │ │ ├── columns: region:24!null data1:27!null - │ │ ├── constraint: /24/27/28/22: [/'US_WEST' - /'US_WEST'] - │ │ ├── limit: 1(rev) - │ │ ├── key: () - │ │ └── fd: ()-->(24,27) + │ │ ├── scan index_tab@c,rev + │ │ │ ├── columns: region:14!null data1:17!null + │ │ │ ├── constraint: /14/17/18/12: [/'US_EAST' - /'US_EAST'] + │ │ │ ├── limit: 1(rev) + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(14,17) + │ │ │ └── limit hint: 1.00 + │ │ └── scan index_tab@c,rev + │ │ ├── columns: region:24!null data1:27!null + │ │ ├── constraint: /24/27/28/22: [/'US_WEST' - /'US_WEST'] + │ │ ├── limit: 1(rev) + │ │ ├── key: () + │ │ ├── fd: ()-->(24,27) + │ │ └── limit hint: 1.00 │ └── 1 └── aggregations └── const-agg [as=max:11, outer=(6)] @@ -930,30 +932,28 @@ scalar-group-by │ ├── cardinality: [0 - 1] │ ├── key: () │ ├── fd: ()-->(4-6) - │ ├── sort + │ ├── union │ │ ├── columns: latitude:4!null longitude:5!null data1:6!null + │ │ ├── left columns: latitude:15 longitude:16 data1:17 + │ │ ├── right columns: latitude:25 longitude:26 data1:27 │ │ ├── cardinality: [0 - 2] │ │ ├── key: (4-6) │ │ ├── ordering: -6 │ │ ├── limit hint: 1.00 - │ │ └── union - │ │ ├── columns: latitude:4!null longitude:5!null data1:6!null - │ │ ├── left columns: latitude:15 longitude:16 data1:17 - │ │ ├── right columns: latitude:25 longitude:26 data1:27 - │ │ ├── cardinality: [0 - 2] - │ │ ├── key: (4-6) - │ │ ├── scan index_tab@d,rev - │ │ │ ├── columns: latitude:15!null longitude:16!null data1:17!null - │ │ │ ├── constraint: /15/16/17/18/12: [/1/2 - /1/2] - │ │ │ ├── limit: 1(rev) - │ │ │ ├── key: () - │ │ │ └── fd: ()-->(15-17) - │ │ └── scan index_tab@d,rev - │ │ ├── columns: latitude:25!null longitude:26!null data1:27!null - │ │ ├── constraint: /25/26/27/28/22: [/4/5 - /4/5] - │ │ ├── limit: 1(rev) - │ │ ├── key: () - │ │ └── fd: ()-->(25-27) + │ │ ├── scan index_tab@d,rev + │ │ │ ├── columns: latitude:15!null longitude:16!null data1:17!null + │ │ │ ├── constraint: /15/16/17/18/12: [/1/2 - /1/2] + │ │ │ ├── limit: 1(rev) + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(15-17) + │ │ │ └── limit hint: 1.00 + │ │ └── scan index_tab@d,rev + │ │ ├── columns: latitude:25!null longitude:26!null data1:27!null + │ │ ├── constraint: /25/26/27/28/22: [/4/5 - /4/5] + │ │ ├── limit: 1(rev) + │ │ ├── key: () + │ │ ├── fd: ()-->(25-27) + │ │ └── limit hint: 1.00 │ └── 1 └── aggregations └── const-agg [as=max:11, outer=(6)] @@ -974,42 +974,43 @@ scalar-group-by │ ├── cardinality: [0 - 1] │ ├── key: () │ ├── fd: ()-->(2,6) - │ ├── sort + │ ├── union │ │ ├── columns: val:2!null data1:6!null + │ │ ├── left columns: val:33 data1:37 + │ │ ├── right columns: val:43 data1:47 │ │ ├── cardinality: [0 - 3] │ │ ├── key: (2,6) │ │ ├── ordering: -6 │ │ ├── limit hint: 1.00 - │ │ └── union - │ │ ├── columns: val:2!null data1:6!null - │ │ ├── left columns: val:33 data1:37 - │ │ ├── right columns: val:43 data1:47 - │ │ ├── cardinality: [0 - 3] - │ │ ├── key: (2,6) - │ │ ├── union - │ │ │ ├── columns: val:33!null data1:37!null - │ │ │ ├── left columns: val:13 data1:17 - │ │ │ ├── right columns: val:23 data1:27 - │ │ │ ├── cardinality: [0 - 2] - │ │ │ ├── key: (33,37) - │ │ │ ├── scan index_tab@b,rev - │ │ │ │ ├── columns: val:13!null data1:17!null - │ │ │ │ ├── constraint: /13/17/18/12: [/1 - /1] - │ │ │ │ ├── limit: 1(rev) - │ │ │ │ ├── key: () - │ │ │ │ └── fd: ()-->(13,17) - │ │ │ └── scan index_tab@b,rev - │ │ │ ├── columns: val:23!null data1:27!null - │ │ │ ├── constraint: /23/27/28/22: [/2 - /2] - │ │ │ ├── limit: 1(rev) - │ │ │ ├── key: () - │ │ │ └── fd: ()-->(23,27) - │ │ └── scan index_tab@b,rev - │ │ ├── columns: val:43!null data1:47!null - │ │ ├── constraint: /43/47/48/42: [/3 - /3] - │ │ ├── limit: 1(rev) - │ │ ├── key: () - │ │ └── fd: ()-->(43,47) + │ │ ├── union + │ │ │ ├── columns: val:33!null data1:37!null + │ │ │ ├── left columns: val:13 data1:17 + │ │ │ ├── right columns: val:23 data1:27 + │ │ │ ├── cardinality: [0 - 2] + │ │ │ ├── key: (33,37) + │ │ │ ├── ordering: -37,+33 + │ │ │ ├── limit hint: 1.00 + │ │ │ ├── scan index_tab@b,rev + │ │ │ │ ├── columns: val:13!null data1:17!null + │ │ │ │ ├── constraint: /13/17/18/12: [/1 - /1] + │ │ │ │ ├── limit: 1(rev) + │ │ │ │ ├── key: () + │ │ │ │ ├── fd: ()-->(13,17) + │ │ │ │ └── limit hint: 1.00 + │ │ │ └── scan index_tab@b,rev + │ │ │ ├── columns: val:23!null data1:27!null + │ │ │ ├── constraint: /23/27/28/22: [/2 - /2] + │ │ │ ├── limit: 1(rev) + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(23,27) + │ │ │ └── limit hint: 1.00 + │ │ └── scan index_tab@b,rev + │ │ ├── columns: val:43!null data1:47!null + │ │ ├── constraint: /43/47/48/42: [/3 - /3] + │ │ ├── limit: 1(rev) + │ │ ├── key: () + │ │ ├── fd: ()-->(43,47) + │ │ └── limit hint: 1.00 │ └── 1 └── aggregations └── const-agg [as=max:11, outer=(6)] @@ -1028,28 +1029,28 @@ limit ├── internal-ordering: +6,+7 ├── cardinality: [0 - 10] ├── ordering: +6,+7 - ├── sort + ├── union │ ├── columns: region:3!null data1:6!null data2:7!null + │ ├── left columns: region:13 data1:16 data2:17 + │ ├── right columns: region:23 data1:26 data2:27 │ ├── cardinality: [0 - 20] │ ├── key: (3,6,7) │ ├── ordering: +6,+7 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: region:3!null data1:6!null data2:7!null - │ ├── left columns: region:13 data1:16 data2:17 - │ ├── right columns: region:23 data1:26 data2:27 - │ ├── cardinality: [0 - 20] - │ ├── key: (3,6,7) - │ ├── scan index_tab@c - │ │ ├── columns: region:13!null data1:16!null data2:17!null - │ │ ├── constraint: /13/16/17/11: [/'US_EAST' - /'US_EAST'] - │ │ ├── limit: 10 - │ │ └── fd: ()-->(13) - │ └── scan index_tab@c - │ ├── columns: region:23!null data1:26!null data2:27!null - │ ├── constraint: /23/26/27/21: [/'US_WEST' - /'US_WEST'] - │ ├── limit: 10 - │ └── fd: ()-->(23) + │ ├── scan index_tab@c + │ │ ├── columns: region:13!null data1:16!null data2:17!null + │ │ ├── constraint: /13/16/17/11: [/'US_EAST' - /'US_EAST'] + │ │ ├── limit: 10 + │ │ ├── fd: ()-->(13) + │ │ ├── ordering: +16,+17 opt(13) [actual: +16,+17] + │ │ └── limit hint: 10.00 + │ └── scan index_tab@c + │ ├── columns: region:23!null data1:26!null data2:27!null + │ ├── constraint: /23/26/27/21: [/'US_WEST' - /'US_WEST'] + │ ├── limit: 10 + │ ├── fd: ()-->(23) + │ ├── ordering: +26,+27 opt(23) [actual: +26,+27] + │ └── limit hint: 10.00 └── 10 # Case with start key longer than the ordering prefix length. @@ -1066,28 +1067,28 @@ limit ├── internal-ordering: +6,+7 ├── cardinality: [0 - 10] ├── ordering: +6,+7 - ├── sort + ├── union │ ├── columns: region:3!null data1:6!null data2:7!null + │ ├── left columns: region:13 data1:16 data2:17 + │ ├── right columns: region:23 data1:26 data2:27 │ ├── cardinality: [0 - 20] │ ├── key: (3,6,7) │ ├── ordering: +6,+7 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: region:3!null data1:6!null data2:7!null - │ ├── left columns: region:13 data1:16 data2:17 - │ ├── right columns: region:23 data1:26 data2:27 - │ ├── cardinality: [0 - 20] - │ ├── key: (3,6,7) - │ ├── scan index_tab@c - │ │ ├── columns: region:13!null data1:16!null data2:17!null - │ │ ├── constraint: /13/16/17/11: [/'US_EAST'/4 - /'US_EAST'] - │ │ ├── limit: 10 - │ │ └── fd: ()-->(13) - │ └── scan index_tab@c - │ ├── columns: region:23!null data1:26!null data2:27!null - │ ├── constraint: /23/26/27/21: [/'US_WEST'/4 - /'US_WEST'] - │ ├── limit: 10 - │ └── fd: ()-->(23) + │ ├── scan index_tab@c + │ │ ├── columns: region:13!null data1:16!null data2:17!null + │ │ ├── constraint: /13/16/17/11: [/'US_EAST'/4 - /'US_EAST'] + │ │ ├── limit: 10 + │ │ ├── fd: ()-->(13) + │ │ ├── ordering: +16,+17 opt(13) [actual: +16,+17] + │ │ └── limit hint: 10.00 + │ └── scan index_tab@c + │ ├── columns: region:23!null data1:26!null data2:27!null + │ ├── constraint: /23/26/27/21: [/'US_WEST'/4 - /'US_WEST'] + │ ├── limit: 10 + │ ├── fd: ()-->(23) + │ ├── ordering: +26,+27 opt(23) [actual: +26,+27] + │ └── limit hint: 10.00 └── 10 # Case with end key longer than the ordering prefix length. @@ -1104,28 +1105,28 @@ limit ├── internal-ordering: +6,+7 ├── cardinality: [0 - 10] ├── ordering: +6,+7 - ├── sort + ├── union │ ├── columns: region:3!null data1:6!null data2:7!null + │ ├── left columns: region:13 data1:16 data2:17 + │ ├── right columns: region:23 data1:26 data2:27 │ ├── cardinality: [0 - 20] │ ├── key: (3,6,7) │ ├── ordering: +6,+7 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: region:3!null data1:6!null data2:7!null - │ ├── left columns: region:13 data1:16 data2:17 - │ ├── right columns: region:23 data1:26 data2:27 - │ ├── cardinality: [0 - 20] - │ ├── key: (3,6,7) - │ ├── scan index_tab@c - │ │ ├── columns: region:13!null data1:16!null data2:17!null - │ │ ├── constraint: /13/16/17/11: [/'US_EAST' - /'US_EAST'/2] - │ │ ├── limit: 10 - │ │ └── fd: ()-->(13) - │ └── scan index_tab@c - │ ├── columns: region:23!null data1:26!null data2:27!null - │ ├── constraint: /23/26/27/21: [/'US_WEST' - /'US_WEST'/2] - │ ├── limit: 10 - │ └── fd: ()-->(23) + │ ├── scan index_tab@c + │ │ ├── columns: region:13!null data1:16!null data2:17!null + │ │ ├── constraint: /13/16/17/11: [/'US_EAST' - /'US_EAST'/2] + │ │ ├── limit: 10 + │ │ ├── fd: ()-->(13) + │ │ ├── ordering: +16,+17 opt(13) [actual: +16,+17] + │ │ └── limit hint: 10.00 + │ └── scan index_tab@c + │ ├── columns: region:23!null data1:26!null data2:27!null + │ ├── constraint: /23/26/27/21: [/'US_WEST' - /'US_WEST'/2] + │ ├── limit: 10 + │ ├── fd: ()-->(23) + │ ├── ordering: +26,+27 opt(23) [actual: +26,+27] + │ └── limit hint: 10.00 └── 10 # Case with both keys longer than the ordering prefix length. @@ -1143,28 +1144,28 @@ limit ├── internal-ordering: +6,+7 ├── cardinality: [0 - 10] ├── ordering: +6,+7 - ├── sort + ├── union │ ├── columns: region:3!null data1:6!null data2:7!null + │ ├── left columns: region:13 data1:16 data2:17 + │ ├── right columns: region:23 data1:26 data2:27 │ ├── cardinality: [0 - 20] │ ├── key: (3,6,7) │ ├── ordering: +6,+7 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: region:3!null data1:6!null data2:7!null - │ ├── left columns: region:13 data1:16 data2:17 - │ ├── right columns: region:23 data1:26 data2:27 - │ ├── cardinality: [0 - 20] - │ ├── key: (3,6,7) - │ ├── scan index_tab@c - │ │ ├── columns: region:13!null data1:16!null data2:17!null - │ │ ├── constraint: /13/16/17/11: [/'US_EAST'/4 - /'US_EAST'/999] - │ │ ├── limit: 10 - │ │ └── fd: ()-->(13) - │ └── scan index_tab@c - │ ├── columns: region:23!null data1:26!null data2:27!null - │ ├── constraint: /23/26/27/21: [/'US_WEST'/4 - /'US_WEST'/999] - │ ├── limit: 10 - │ └── fd: ()-->(23) + │ ├── scan index_tab@c + │ │ ├── columns: region:13!null data1:16!null data2:17!null + │ │ ├── constraint: /13/16/17/11: [/'US_EAST'/4 - /'US_EAST'/999] + │ │ ├── limit: 10 + │ │ ├── fd: ()-->(13) + │ │ ├── ordering: +16,+17 opt(13) [actual: +16,+17] + │ │ └── limit hint: 10.00 + │ └── scan index_tab@c + │ ├── columns: region:23!null data1:26!null data2:27!null + │ ├── constraint: /23/26/27/21: [/'US_WEST'/4 - /'US_WEST'/999] + │ ├── limit: 10 + │ ├── fd: ()-->(23) + │ ├── ordering: +26,+27 opt(23) [actual: +26,+27] + │ └── limit hint: 10.00 └── 10 # Case where one span can be used for a limited scan, but not the others. Note @@ -1183,43 +1184,54 @@ limit ├── internal-ordering: +6,+7 ├── cardinality: [0 - 10] ├── ordering: +6,+7 - ├── sort + ├── union │ ├── columns: latitude:4!null longitude:5 data1:6!null data2:7!null + │ ├── left columns: latitude:74 longitude:75 data1:76 data2:77 + │ ├── right columns: latitude:84 longitude:85 data1:86 data2:87 │ ├── key: (4-7) │ ├── ordering: +6,+7 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: latitude:4!null longitude:5 data1:6!null data2:7!null - │ ├── left columns: latitude:74 longitude:75 data1:76 data2:77 - │ ├── right columns: latitude:84 longitude:85 data1:86 data2:87 - │ ├── key: (4-7) - │ ├── union - │ │ ├── columns: latitude:74!null longitude:75!null data1:76!null data2:77!null - │ │ ├── left columns: latitude:54 longitude:55 data1:56 data2:57 - │ │ ├── right columns: latitude:64 longitude:65 data1:66 data2:67 - │ │ ├── cardinality: [0 - 30] - │ │ ├── key: (74-77) - │ │ ├── union - │ │ │ ├── columns: latitude:54!null longitude:55!null data1:56!null data2:57!null - │ │ │ ├── left columns: latitude:34 longitude:35 data1:36 data2:37 - │ │ │ ├── right columns: latitude:44 longitude:45 data1:46 data2:47 - │ │ │ ├── cardinality: [0 - 20] - │ │ │ ├── key: (54-57) - │ │ │ ├── scan index_tab@d - │ │ │ │ ├── columns: latitude:34!null longitude:35!null data1:36!null data2:37!null - │ │ │ │ ├── constraint: /34/35/36/37/31: [/10/11 - /10/11] - │ │ │ │ ├── limit: 10 - │ │ │ │ └── fd: ()-->(34,35) - │ │ │ └── scan index_tab@d - │ │ │ ├── columns: latitude:44!null longitude:45!null data1:46!null data2:47!null - │ │ │ ├── constraint: /44/45/46/47/41: [/10/12 - /10/12] - │ │ │ ├── limit: 10 - │ │ │ └── fd: ()-->(44,45) - │ │ └── scan index_tab@d - │ │ ├── columns: latitude:64!null longitude:65!null data1:66!null data2:67!null - │ │ ├── constraint: /64/65/66/67/61: [/10/13 - /10/13] - │ │ ├── limit: 10 - │ │ └── fd: ()-->(64,65) + │ ├── union + │ │ ├── columns: latitude:74!null longitude:75!null data1:76!null data2:77!null + │ │ ├── left columns: latitude:54 longitude:55 data1:56 data2:57 + │ │ ├── right columns: latitude:64 longitude:65 data1:66 data2:67 + │ │ ├── cardinality: [0 - 30] + │ │ ├── key: (74-77) + │ │ ├── ordering: +76,+77,+74,+75 + │ │ ├── limit hint: 10.00 + │ │ ├── union + │ │ │ ├── columns: latitude:54!null longitude:55!null data1:56!null data2:57!null + │ │ │ ├── left columns: latitude:34 longitude:35 data1:36 data2:37 + │ │ │ ├── right columns: latitude:44 longitude:45 data1:46 data2:47 + │ │ │ ├── cardinality: [0 - 20] + │ │ │ ├── key: (54-57) + │ │ │ ├── ordering: +56,+57,+54,+55 + │ │ │ ├── limit hint: 10.00 + │ │ │ ├── scan index_tab@d + │ │ │ │ ├── columns: latitude:34!null longitude:35!null data1:36!null data2:37!null + │ │ │ │ ├── constraint: /34/35/36/37/31: [/10/11 - /10/11] + │ │ │ │ ├── limit: 10 + │ │ │ │ ├── fd: ()-->(34,35) + │ │ │ │ ├── ordering: +36,+37 opt(34,35) [actual: +36,+37] + │ │ │ │ └── limit hint: 10.00 + │ │ │ └── scan index_tab@d + │ │ │ ├── columns: latitude:44!null longitude:45!null data1:46!null data2:47!null + │ │ │ ├── constraint: /44/45/46/47/41: [/10/12 - /10/12] + │ │ │ ├── limit: 10 + │ │ │ ├── fd: ()-->(44,45) + │ │ │ ├── ordering: +46,+47 opt(44,45) [actual: +46,+47] + │ │ │ └── limit hint: 10.00 + │ │ └── scan index_tab@d + │ │ ├── columns: latitude:64!null longitude:65!null data1:66!null data2:67!null + │ │ ├── constraint: /64/65/66/67/61: [/10/13 - /10/13] + │ │ ├── limit: 10 + │ │ ├── fd: ()-->(64,65) + │ │ ├── ordering: +66,+67 opt(64,65) [actual: +66,+67] + │ │ └── limit hint: 10.00 + │ └── sort + │ ├── columns: latitude:84!null longitude:85 data1:86!null data2:87!null + │ ├── ordering: +86,+87,+84,+85 + │ ├── limit hint: 10.00 │ └── scan index_tab@d │ ├── columns: latitude:84!null longitude:85 data1:86!null data2:87!null │ └── constraint: /84/85/86/87/81 @@ -1246,24 +1258,34 @@ index-join index_tab ├── key: (1) ├── fd: (1)-->(3,6,7) ├── ordering: +6 - ├── sort + ├── union │ ├── columns: id:1!null region:3!null data1:6!null data2:7!null + │ ├── left columns: id:11 region:13 data1:16 data2:17 + │ ├── right columns: id:21 region:23 data1:26 data2:27 │ ├── cardinality: [0 - 20] │ ├── key: (1,3,6,7) │ ├── ordering: +6 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: id:1!null region:3!null data1:6!null data2:7!null - │ ├── left columns: id:11 region:13 data1:16 data2:17 - │ ├── right columns: id:21 region:23 data1:26 data2:27 - │ ├── cardinality: [0 - 20] - │ ├── key: (1,3,6,7) - │ ├── scan index_tab@c - │ │ ├── columns: id:11!null region:13!null data1:16!null data2:17!null - │ │ ├── constraint: /13/16/17/11: [/'US_EAST' - /'US_EAST'] - │ │ ├── limit: 10 - │ │ ├── key: (11) - │ │ └── fd: ()-->(13), (11)-->(16,17) + │ ├── sort + │ │ ├── columns: id:11!null region:13!null data1:16!null data2:17!null + │ │ ├── cardinality: [0 - 10] + │ │ ├── key: (11) + │ │ ├── fd: ()-->(13), (11)-->(16,17) + │ │ ├── ordering: +16,+11 opt(13) [actual: +16,+11] + │ │ ├── limit hint: 10.00 + │ │ └── scan index_tab@c + │ │ ├── columns: id:11!null region:13!null data1:16!null data2:17!null + │ │ ├── constraint: /13/16/17/11: [/'US_EAST' - /'US_EAST'] + │ │ ├── limit: 10 + │ │ ├── key: (11) + │ │ └── fd: ()-->(13), (11)-->(16,17) + │ └── sort + │ ├── columns: id:21!null region:23!null data1:26!null data2:27!null + │ ├── cardinality: [0 - 10] + │ ├── key: (21) + │ ├── fd: ()-->(23), (21)-->(26,27) + │ ├── ordering: +26,+21 opt(23) [actual: +26,+21] + │ ├── limit hint: 10.00 │ └── scan index_tab@c │ ├── columns: id:21!null region:23!null data1:26!null data2:27!null │ ├── constraint: /23/26/27/21: [/'US_WEST' - /'US_WEST'] @@ -1283,42 +1305,46 @@ limit ├── key: (1,2) ├── fd: (1,2)-->(3,4) ├── ordering: +2 - ├── sort + ├── union │ ├── columns: p:1!null q:2!null r:3!null s:4!null + │ ├── left columns: p:16 q:17 r:18 s:19 + │ ├── right columns: p:21 q:22 r:23 s:24 │ ├── cardinality: [0 - 15] │ ├── key: (1-4) │ ├── ordering: +2 │ ├── limit hint: 5.00 - │ └── union - │ ├── columns: p:1!null q:2!null r:3!null s:4!null - │ ├── left columns: p:16 q:17 r:18 s:19 - │ ├── right columns: p:21 q:22 r:23 s:24 - │ ├── cardinality: [0 - 15] - │ ├── key: (1-4) - │ ├── union - │ │ ├── columns: p:16!null q:17!null r:18!null s:19!null - │ │ ├── left columns: p:6 q:7 r:8 s:9 - │ │ ├── right columns: p:11 q:12 r:13 s:14 - │ │ ├── cardinality: [0 - 10] - │ │ ├── key: (16-19) - │ │ ├── scan pqrs - │ │ │ ├── columns: p:6!null q:7!null r:8!null s:9!null - │ │ │ ├── constraint: /6/7: [/1 - /1] - │ │ │ ├── limit: 5 - │ │ │ ├── key: (7) - │ │ │ └── fd: ()-->(6), (7)-->(8,9) - │ │ └── scan pqrs - │ │ ├── columns: p:11!null q:12!null r:13!null s:14!null - │ │ ├── constraint: /11/12: [/5 - /5] - │ │ ├── limit: 5 - │ │ ├── key: (12) - │ │ └── fd: ()-->(11), (12)-->(13,14) - │ └── scan pqrs - │ ├── columns: p:21!null q:22!null r:23!null s:24!null - │ ├── constraint: /21/22: [/10 - /10] - │ ├── limit: 5 - │ ├── key: (22) - │ └── fd: ()-->(21), (22)-->(23,24) + │ ├── union + │ │ ├── columns: p:16!null q:17!null r:18!null s:19!null + │ │ ├── left columns: p:6 q:7 r:8 s:9 + │ │ ├── right columns: p:11 q:12 r:13 s:14 + │ │ ├── cardinality: [0 - 10] + │ │ ├── key: (16-19) + │ │ ├── ordering: +17,+16,+18,+19 + │ │ ├── limit hint: 5.00 + │ │ ├── scan pqrs + │ │ │ ├── columns: p:6!null q:7!null r:8!null s:9!null + │ │ │ ├── constraint: /6/7: [/1 - /1] + │ │ │ ├── limit: 5 + │ │ │ ├── key: (7) + │ │ │ ├── fd: ()-->(6), (7)-->(8,9) + │ │ │ ├── ordering: +7 opt(6) [actual: +7] + │ │ │ └── limit hint: 5.00 + │ │ └── scan pqrs + │ │ ├── columns: p:11!null q:12!null r:13!null s:14!null + │ │ ├── constraint: /11/12: [/5 - /5] + │ │ ├── limit: 5 + │ │ ├── key: (12) + │ │ ├── fd: ()-->(11), (12)-->(13,14) + │ │ ├── ordering: +12 opt(11) [actual: +12] + │ │ └── limit hint: 5.00 + │ └── scan pqrs + │ ├── columns: p:21!null q:22!null r:23!null s:24!null + │ ├── constraint: /21/22: [/10 - /10] + │ ├── limit: 5 + │ ├── key: (22) + │ ├── fd: ()-->(21), (22)-->(23,24) + │ ├── ordering: +22 opt(21) [actual: +22] + │ └── limit hint: 5.00 └── 5 # Case where multiple check constraints are combined into one constraint @@ -1333,30 +1359,30 @@ limit ├── key: (1,2) ├── fd: (1,2)-->(3,4) ├── ordering: +4 - ├── sort + ├── union │ ├── columns: p:1!null q:2!null r:3!null s:4!null + │ ├── left columns: p:6 q:7 r:8 s:9 + │ ├── right columns: p:11 q:12 r:13 s:14 │ ├── cardinality: [0 - 20] │ ├── key: (1-4) │ ├── ordering: +4 │ ├── limit hint: 10.00 - │ └── union - │ ├── columns: p:1!null q:2!null r:3!null s:4!null - │ ├── left columns: p:6 q:7 r:8 s:9 - │ ├── right columns: p:11 q:12 r:13 s:14 - │ ├── cardinality: [0 - 20] - │ ├── key: (1-4) - │ ├── scan pqrs@secondary - │ │ ├── columns: p:6!null q:7!null r:8!null s:9!null - │ │ ├── constraint: /8/9/6/7: [/1 - /1] - │ │ ├── limit: 10 - │ │ ├── key: (6,7) - │ │ └── fd: ()-->(8), (6,7)-->(9) - │ └── scan pqrs@secondary - │ ├── columns: p:11!null q:12!null r:13!null s:14!null - │ ├── constraint: /13/14/11/12: [/2 - /2] - │ ├── limit: 10 - │ ├── key: (11,12) - │ └── fd: ()-->(13), (11,12)-->(14) + │ ├── scan pqrs@secondary + │ │ ├── columns: p:6!null q:7!null r:8!null s:9!null + │ │ ├── constraint: /8/9/6/7: [/1 - /1] + │ │ ├── limit: 10 + │ │ ├── key: (6,7) + │ │ ├── fd: ()-->(8), (6,7)-->(9) + │ │ ├── ordering: +9,+6,+7 opt(8) [actual: +9,+6,+7] + │ │ └── limit hint: 10.00 + │ └── scan pqrs@secondary + │ ├── columns: p:11!null q:12!null r:13!null s:14!null + │ ├── constraint: /13/14/11/12: [/2 - /2] + │ ├── limit: 10 + │ ├── key: (11,12) + │ ├── fd: ()-->(13), (11,12)-->(14) + │ ├── ordering: +14,+11,+12 opt(13) [actual: +14,+11,+12] + │ └── limit hint: 10.00 └── 10 # Check constraints are not used because the scan is already constrained (the @@ -1371,30 +1397,30 @@ limit ├── key: (1,2) ├── fd: (1,2)-->(3,4) ├── ordering: +2 - ├── sort + ├── union │ ├── columns: p:1!null q:2!null r:3!null s:4!null + │ ├── left columns: p:6 q:7 r:8 s:9 + │ ├── right columns: p:11 q:12 r:13 s:14 │ ├── cardinality: [0 - 10] │ ├── key: (1-4) │ ├── ordering: +2 │ ├── limit hint: 5.00 - │ └── union - │ ├── columns: p:1!null q:2!null r:3!null s:4!null - │ ├── left columns: p:6 q:7 r:8 s:9 - │ ├── right columns: p:11 q:12 r:13 s:14 - │ ├── cardinality: [0 - 10] - │ ├── key: (1-4) - │ ├── scan pqrs - │ │ ├── columns: p:6!null q:7!null r:8!null s:9!null - │ │ ├── constraint: /6/7: [/1 - /1] - │ │ ├── limit: 5 - │ │ ├── key: (7) - │ │ └── fd: ()-->(6), (7)-->(8,9) - │ └── scan pqrs - │ ├── columns: p:11!null q:12!null r:13!null s:14!null - │ ├── constraint: /11/12: [/5 - /5] - │ ├── limit: 5 - │ ├── key: (12) - │ └── fd: ()-->(11), (12)-->(13,14) + │ ├── scan pqrs + │ │ ├── columns: p:6!null q:7!null r:8!null s:9!null + │ │ ├── constraint: /6/7: [/1 - /1] + │ │ ├── limit: 5 + │ │ ├── key: (7) + │ │ ├── fd: ()-->(6), (7)-->(8,9) + │ │ ├── ordering: +7 opt(6) [actual: +7] + │ │ └── limit hint: 5.00 + │ └── scan pqrs + │ ├── columns: p:11!null q:12!null r:13!null s:14!null + │ ├── constraint: /11/12: [/5 - /5] + │ ├── limit: 5 + │ ├── key: (12) + │ ├── fd: ()-->(11), (12)-->(13,14) + │ ├── ordering: +12 opt(11) [actual: +12] + │ └── limit hint: 5.00 └── 5 # No-op case because the scan has an inverted index. diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 8e8edb811c82..22ec04ed9ec1 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -6740,9 +6740,10 @@ project ├── columns: d.k:1!null d.u:2 d.v:3 ├── key: (1) ├── fd: (1)-->(2,3) - └── inner-join (hash) + └── inner-join (merge) ├── columns: d.k:1!null d.u:2!null d.v:3 a.u:7!null - ├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-more) + ├── left ordering: +2 + ├── right ordering: +7 ├── key: (1) ├── fd: (1)-->(2,3), (2)==(7), (7)==(2) ├── distinct-on @@ -6750,10 +6751,12 @@ project │ ├── grouping columns: d.k:1!null │ ├── key: (1) │ ├── fd: (1)-->(2,3) + │ ├── ordering: +2 │ ├── union-all │ │ ├── columns: d.k:1!null d.u:2 d.v:3 │ │ ├── left columns: d.k:10 d.u:11 d.v:12 │ │ ├── right columns: d.k:15 d.u:16 d.v:17 + │ │ ├── ordering: +2 │ │ ├── index-join d │ │ │ ├── columns: d.k:10!null d.u:11!null d.v:12 │ │ │ ├── key: (10) @@ -6763,15 +6766,20 @@ project │ │ │ ├── constraint: /11/10: [/1 - /1] │ │ │ ├── key: (10) │ │ │ └── fd: ()-->(11) - │ │ └── index-join d + │ │ └── sort │ │ ├── columns: d.k:15!null d.u:16 d.v:17!null │ │ ├── key: (15) │ │ ├── fd: ()-->(17), (15)-->(16) - │ │ └── scan d@v - │ │ ├── columns: d.k:15!null d.v:17!null - │ │ ├── constraint: /17/15: [/1 - /1] + │ │ ├── ordering: +16 opt(17) [actual: +16] + │ │ └── index-join d + │ │ ├── columns: d.k:15!null d.u:16 d.v:17!null │ │ ├── key: (15) - │ │ └── fd: ()-->(17) + │ │ ├── fd: ()-->(17), (15)-->(16) + │ │ └── scan d@v + │ │ ├── columns: d.k:15!null d.v:17!null + │ │ ├── constraint: /17/15: [/1 - /1] + │ │ ├── key: (15) + │ │ └── fd: ()-->(17) │ └── aggregations │ ├── const-agg [as=d.u:2, outer=(2)] │ │ └── d.u:2 @@ -6780,13 +6788,12 @@ project ├── distinct-on │ ├── columns: a.u:7 │ ├── grouping columns: a.u:7 - │ ├── internal-ordering: +7 │ ├── key: (7) + │ ├── ordering: +7 │ └── scan a@u │ ├── columns: a.u:7 │ └── ordering: +7 - └── filters - └── a.u:7 = d.u:2 [outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ]), fd=(2)==(7), (7)==(2)] + └── filters (true) # Correlated subquery with references to outer columns not in the scan columns. opt expect=SplitDisjunction @@ -7548,8 +7555,8 @@ memo (optimized, ~28KB, required=[presentation: a:1] [ordering: +2]) ├── G7: (filters G10) ├── G8: (union-all G13 G14) │ ├── [ordering: +(2|3)] - │ │ ├── best: (sort G8) - │ │ └── cost: 1089.85 + │ │ ├── best: (union-all G13 G14="[ordering: +(14|15)]") + │ │ └── cost: 1089.79 │ └── [] │ ├── best: (union-all G13 G14) │ └── cost: 1089.76 @@ -7568,6 +7575,9 @@ memo (optimized, ~28KB, required=[presentation: a:1] [ordering: +2]) │ ├── best: (select G25 G26) │ └── cost: 5.10 ├── G14: (select G28 G29) (select G30 G31) + │ ├── [ordering: +(14|15)] + │ │ ├── best: (sort G14) + │ │ └── cost: 1084.67 │ └── [] │ ├── best: (select G28 G29) │ └── cost: 1084.64 @@ -7600,11 +7610,17 @@ memo (optimized, ~28KB, required=[presentation: a:1] [ordering: +2]) │ ├── best: (index-join G37 t61795,cols=(9-11)) │ └── cost: 1049.59 ├── G28: (scan t61795 [as=t1],cols=(13-15)) + │ ├── [ordering: +14] + │ │ ├── best: (sort G28) + │ │ └── cost: 1323.94 │ └── [] │ ├── best: (scan t61795 [as=t1],cols=(13-15)) │ └── cost: 1074.61 ├── G29: (filters G38 G39) ├── G30: (index-join G40 t61795,cols=(13-15)) + │ ├── [ordering: +14] + │ │ ├── best: (index-join G40="[ordering: +14]" t61795,cols=(13-15)) + │ │ └── cost: 3040.04 │ └── [] │ ├── best: (index-join G40 t61795,cols=(13-15)) │ └── cost: 3040.04 @@ -7621,6 +7637,9 @@ memo (optimized, ~28KB, required=[presentation: a:1] [ordering: +2]) ├── G38: (eq G47 G48) ├── G39: (ne G48 G49) ├── G40: (select G50 G51) + │ ├── [ordering: +14] + │ │ ├── best: (select G50="[ordering: +14]" G51) + │ │ └── cost: 1043.53 │ └── [] │ ├── best: (select G50 G51) │ └── cost: 1043.53 @@ -7637,6 +7656,9 @@ memo (optimized, ~28KB, required=[presentation: a:1] [ordering: +2]) ├── G48: (variable t1.b) ├── G49: (function G52 abs) ├── G50: (scan t61795@secondary [as=t1],cols=(13,14),constrained) + │ ├── [ordering: +14] + │ │ ├── best: (scan t61795@secondary [as=t1],cols=(13,14),constrained) + │ │ └── cost: 1033.61 │ └── [] │ ├── best: (scan t61795@secondary [as=t1],cols=(13,14),constrained) │ └── cost: 1033.61 diff --git a/pkg/sql/opt_exec_factory.go b/pkg/sql/opt_exec_factory.go index 6eae14b0ea8c..1bcf7d7c2a77 100644 --- a/pkg/sql/opt_exec_factory.go +++ b/pkg/sql/opt_exec_factory.go @@ -497,7 +497,11 @@ func (ef *execFactory) ConstructDistinct( // ConstructSetOp is part of the exec.Factory interface. func (ef *execFactory) ConstructSetOp( - typ tree.UnionType, all bool, left, right exec.Node, hardLimit uint64, + typ tree.UnionType, + all bool, + left, right exec.Node, + reqOrdering exec.OutputOrdering, + hardLimit uint64, ) (exec.Node, error) { if hardLimit != 0 && (typ != tree.UnionOp || !all) { return nil, errors.AssertionFailedf("a hard limit on a set operator is only supported for UNION ALL") @@ -507,7 +511,9 @@ func (ef *execFactory) ConstructSetOp( "locality optimized search is not yet supported for more than one row at a time", ) } - return ef.planner.newUnionNode(typ, all, left.(planNode), right.(planNode), hardLimit) + return ef.planner.newUnionNode( + typ, all, left.(planNode), right.(planNode), ReqOrdering(reqOrdering), hardLimit, + ) } // ConstructSort is part of the exec.Factory interface. diff --git a/pkg/sql/plan_ordering.go b/pkg/sql/plan_ordering.go index 56e0540955aa..e37b171cea9f 100644 --- a/pkg/sql/plan_ordering.go +++ b/pkg/sql/plan_ordering.go @@ -55,7 +55,7 @@ func planReqOrdering(plan planNode) ReqOrdering { case *joinNode: return n.reqOrdering case *unionNode: - // TODO(knz): this can be ordered if the source is ordered already. + return n.reqOrdering case *insertNode, *insertFastPathNode: // TODO(knz): RETURNING is ordered by the PK. case *updateNode, *upsertNode: diff --git a/pkg/sql/union.go b/pkg/sql/union.go index 3ab9ddb5a428..2b71f19c15ee 100644 --- a/pkg/sql/union.go +++ b/pkg/sql/union.go @@ -77,6 +77,10 @@ type unionNode struct { // all indicates if the operation is the ALL or DISTINCT version all bool + // reqOrdering specifies the required output ordering. If not empty, both + // inputs are already ordered according to it. + reqOrdering ReqOrdering + // hardLimit can only be set for UNION ALL operations. It is used to implement // locality optimized search, and instructs the execution engine that it // should execute the left node to completion and possibly short-circuit if @@ -86,7 +90,7 @@ type unionNode struct { } func (p *planner) newUnionNode( - typ tree.UnionType, all bool, left, right planNode, hardLimit uint64, + typ tree.UnionType, all bool, left, right planNode, reqOrdering ReqOrdering, hardLimit uint64, ) (planNode, error) { emitAll := false switch typ { @@ -137,14 +141,15 @@ func (p *planner) newUnionNode( } node := &unionNode{ - right: right, - left: left, - columns: unionColumns, - inverted: inverted, - emitAll: emitAll, - unionType: typ, - all: all, - hardLimit: hardLimit, + right: right, + left: left, + columns: unionColumns, + inverted: inverted, + emitAll: emitAll, + unionType: typ, + all: all, + reqOrdering: reqOrdering, + hardLimit: hardLimit, } return node, nil } From 58131720266c2e4292745f3c11b207ebd4bf3e00 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 15 Apr 2021 13:20:02 -0400 Subject: [PATCH 4/5] sql: batch write event logs for grant/revoke Helps with https://github.com/cockroachdb/cockroach/issues/41930. Previously, if we ran grant/revoke on multiple tables, we would create event logs for each table and write them one by one, resulting in round trips proportional to the number of tables. This patch addresses this by batch writing the event logs, so that 1 write to the event log table occurs regardless of the number of tables updated. Release note: None --- .../testdata/benchmark_expectations | 8 +- pkg/sql/create_stats.go | 6 +- pkg/sql/event_log.go | 239 ++++++++++++++---- pkg/sql/grant_revoke.go | 10 +- .../logictest/testdata/logic_test/event_log | 16 ++ 5 files changed, 224 insertions(+), 55 deletions(-) diff --git a/pkg/bench/ddl_analysis/testdata/benchmark_expectations b/pkg/bench/ddl_analysis/testdata/benchmark_expectations index 1a7280bc57c4..292e800e9ea0 100644 --- a/pkg/bench/ddl_analysis/testdata/benchmark_expectations +++ b/pkg/bench/ddl_analysis/testdata/benchmark_expectations @@ -48,8 +48,8 @@ exp,benchmark 40,DropView/drop_2_views 57,DropView/drop_3_views 19,Grant/grant_all_on_1_table -23,Grant/grant_all_on_2_tables -27,Grant/grant_all_on_3_tables +22,Grant/grant_all_on_2_tables +25,Grant/grant_all_on_3_tables 19,GrantRole/grant_1_role 22,GrantRole/grant_2_roles 2,ORMQueries/activerecord_type_introspection_query @@ -61,8 +61,8 @@ exp,benchmark 2,ORMQueries/pg_namespace 2,ORMQueries/pg_type 19,Revoke/revoke_all_on_1_table -23,Revoke/revoke_all_on_2_tables -27,Revoke/revoke_all_on_3_tables +22,Revoke/revoke_all_on_2_tables +25,Revoke/revoke_all_on_3_tables 18,RevokeRole/revoke_1_role 20,RevokeRole/revoke_2_roles 1,SystemDatabaseQueries/select_system.users_with_empty_database_name diff --git a/pkg/sql/create_stats.go b/pkg/sql/create_stats.go index 49e3cf186d48..4b93126b0c3c 100644 --- a/pkg/sql/create_stats.go +++ b/pkg/sql/create_stats.go @@ -582,15 +582,15 @@ func (r *createStatsResumer) Resume(ctx context.Context, execCtx interface{}) er // See: https://github.com/cockroachdb/cockroach/issues/57739 return evalCtx.ExecCfg.DB.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { return logEventInternalForSQLStatements(ctx, evalCtx.ExecCfg, txn, - details.Table.ID, + descpb.IDs{details.Table.ID}, evalCtx.SessionData.User(), evalCtx.SessionData.ApplicationName, details.Statement, - nil, /* no placeholders known at this point */ + nil, /* no placeholders known at this point */ + true, /* writeToEventLog */ &eventpb.CreateStatistics{ TableName: details.FQTableName, }, - true, /* writeToEventLog */ ) }) } diff --git a/pkg/sql/event_log.go b/pkg/sql/event_log.go index 034de033c3d7..e2f1fb687676 100644 --- a/pkg/sql/event_log.go +++ b/pkg/sql/event_log.go @@ -13,6 +13,8 @@ package sql import ( "context" "encoding/json" + "fmt" + "strings" "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/jobs" @@ -31,9 +33,17 @@ import ( // logEvent emits a cluster event in the context of a regular SQL // statement. func (p *planner) logEvent( - ctx context.Context, descID descpb.ID, event eventpb.EventPayload, + ctx context.Context, descID descpb.ID, events eventpb.EventPayload, +) error { + return p.logEventsWithSystemEventLogOption(ctx, descpb.IDs{descID}, true /* writeToEventLog */, events) +} + +// batchLogEvents is like logEvent, except it takes in slice of events +// to batch write. +func (p *planner) batchLogEvents( + ctx context.Context, descIDs descpb.IDs, events ...eventpb.EventPayload, ) error { - return p.logEventWithSystemEventLogOption(ctx, descID, event, true /* writeToEventLog */) + return p.logEventsWithSystemEventLogOption(ctx, descIDs, true /* writeToEventLog */, events...) } func (p *planner) logEventOnlyExternally( @@ -41,25 +51,23 @@ func (p *planner) logEventOnlyExternally( ) { // The API contract for logEventWithSystemEventLogOption() is that it returns // no error when system.eventlog is not written to. - _ = p.logEventWithSystemEventLogOption(ctx, descID, event, false /* writeToEventLog */) + _ = p.logEventsWithSystemEventLogOption(ctx, descpb.IDs{descID}, false /* writeToEventLog */, event) } -// logEventWithSystemEventLogOption is like logEvent() but it gives +// logEventsWithSystemEventLogOption is like logEvent() but it gives // control to the caller as to whether the entry is written into // system.eventlog. // // If writeToEventLog is false, this function guarantees that it // returns no error. -func (p *planner) logEventWithSystemEventLogOption( - ctx context.Context, descID descpb.ID, event eventpb.EventPayload, writeToEventLog bool, +func (p *planner) logEventsWithSystemEventLogOption( + ctx context.Context, descIDs descpb.IDs, writeToEventLog bool, events ...eventpb.EventPayload, ) error { - // Compute the common fields from data already known to the planner. user := p.User() stmt := tree.AsStringWithFQNames(p.stmt.AST, p.extendedEvalCtx.EvalContext.Annotations) pl := p.extendedEvalCtx.EvalContext.Placeholders.Values appName := p.SessionData().ApplicationName - - return logEventInternalForSQLStatements(ctx, p.extendedEvalCtx.ExecCfg, p.txn, descID, user, appName, stmt, pl, event, writeToEventLog) + return logEventInternalForSQLStatements(ctx, p.extendedEvalCtx.ExecCfg, p.txn, descIDs, user, appName, stmt, pl, writeToEventLog, events...) } // logEventInternalForSchemaChange emits a cluster event in the @@ -107,15 +115,46 @@ func logEventInternalForSQLStatements( ctx context.Context, execCfg *ExecutorConfig, txn *kv.Txn, - descID descpb.ID, + descIDs descpb.IDs, user security.SQLUsername, appName string, stmt string, placeholders tree.QueryArguments, - event eventpb.EventPayload, writeToEventLog bool, + events ...eventpb.EventPayload, ) error { // Inject the common fields into the payload provided by the caller. + for i := range events { + if err := injectCommonFields( + txn, descIDs[i], user, appName, stmt, placeholders, events[i], + ); err != nil { + return err + } + } + + // Delegate the storing of the event to the regular event logic. + if !writeToEventLog || !eventLogEnabled.Get(&execCfg.InternalExecutor.s.cfg.Settings.SV) { + return skipWritePath(ctx, txn, events, !writeToEventLog) + } + + return batchInsertEventRecords(ctx, execCfg.InternalExecutor, + txn, + descIDs, + int32(execCfg.NodeID.SQLInstanceID()), + events..., + ) +} + +// injectCommonFields injects the common fields into the event payload provided by the caller. +func injectCommonFields( + txn *kv.Txn, + descID descpb.ID, + user security.SQLUsername, + appName string, + stmt string, + placeholders tree.QueryArguments, + event eventpb.EventPayload, +) error { event.CommonDetails().Timestamp = txn.ReadTimestamp().WallTime sqlCommon, ok := event.(eventpb.EventWithCommonSQLPayload) if !ok { @@ -132,16 +171,7 @@ func logEventInternalForSQLStatements( m.PlaceholderValues[idx] = val.String() } } - - // Delegate the storing of the event to the regular event logic. - return InsertEventRecord(ctx, execCfg.InternalExecutor, - txn, - int32(descID), - int32(execCfg.NodeID.SQLInstanceID()), - false, /* skipExternalLog */ - event, - !writeToEventLog, - ) + return nil } // LogEventForJobs emits a cluster event in the context of a job. @@ -211,6 +241,87 @@ func InsertEventRecord( skipExternalLog bool, info eventpb.EventPayload, onlyLog bool, +) error { + if onlyLog || !eventLogEnabled.Get(&ex.s.cfg.Settings.SV) { + return skipWritePath(ctx, txn, []eventpb.EventPayload{info}, onlyLog) + } + return batchInsertEventRecords( + ctx, ex, txn, + descpb.IDs{descpb.ID(targetID)}, + reportingID, + info, + ) +} + +// batchInsertEventRecords is like InsertEventRecord except it takes +// a slice of events to batch write. Any insert that calls this function +// will always write to the event table (i.e. it won't only log them, and writing +// to the event table will not be disabled). +func batchInsertEventRecords( + ctx context.Context, + ex *InternalExecutor, + txn *kv.Txn, + descIDs descpb.IDs, + reportingID int32, + events ...eventpb.EventPayload, +) error { + const colsPerEvent = 5 + const baseQuery = ` +INSERT INTO system.eventlog ( + timestamp, "eventType", "targetID", "reportingID", info +) +VALUES($1, $2, $3, $4, $5)` + args := make([]interface{}, 0, len(events)*colsPerEvent) + + // Prepare first row so we can take the fast path if we're only inserting one event log. + if err := prepareRow( + ctx, txn, &args, events[0], descIDs[0], reportingID, + ); err != nil { + return err + } + if len(events) == 1 { + return execEventLogInsert(ctx, ex, txn, baseQuery, args, len(events)) + } + + var additionalRows strings.Builder + for i := 1; i < len(events); i++ { + var placeholderNum = 1 + (i * colsPerEvent) + if err := prepareRow(ctx, txn, &args, events[i], descIDs[i], reportingID); err != nil { + return err + } + additionalRows.WriteString(fmt.Sprintf(", ($%d, $%d, $%d, $%d, $%d)", + placeholderNum, placeholderNum+1, placeholderNum+2, placeholderNum+3, placeholderNum+4)) + } + + rows, err := ex.Exec(ctx, "log-event", txn, baseQuery+additionalRows.String(), args...) + if err != nil { + return err + } + if rows != len(events) { + return errors.Errorf("%d rows affected by log insertion; expected %d rows affected.", rows, len(events)) + } + return nil +} + +// skipWritePath is used when either onlyLog is true, or writes to the event log +// table are disabled. In these cases, we do not write to the event log table. +func skipWritePath( + ctx context.Context, txn *kv.Txn, events []eventpb.EventPayload, onlyLog bool, +) error { + for i := range events { + if err := setupEventAndMaybeLog( + ctx, txn, events[i], onlyLog, + ); err != nil { + return err + } + } + return nil +} + +// setupEventAndMaybeLog prepares the event log to be written. Also, +// if onlyLog is true, it will log the event. +func setupEventAndMaybeLog( + ctx context.Context, txn *kv.Txn, info eventpb.EventPayload, onlyLog bool, ) error { eventType := eventpb.GetEventTypeName(info) @@ -222,6 +333,7 @@ func InsertEventRecord( return errors.AssertionFailedf("programming error: timestamp field in event not populated: %T", info) } + // If we only want to log and not write to the events table, early exit. if onlyLog { log.StructuredEvent(ctx, info) return nil @@ -233,39 +345,76 @@ func InsertEventRecord( log.StructuredEvent(ctx, info) }) - // If writes to the event log table are disabled, take a shortcut. - if !eventLogEnabled.Get(&ex.s.cfg.Settings.SV) { - return nil - } + return nil +} - const insertEventTableStmt = ` -INSERT INTO system.eventlog ( - timestamp, "eventType", "targetID", "reportingID", info -) -VALUES( - $1, $2, $3, $4, $5 -) -` - args := []interface{}{ - timeutil.Unix(0, info.CommonDetails().Timestamp), - eventType, - targetID, +// constructArgs constructs the values for a single event-log row insert. +func constructArgs( + args *[]interface{}, + event eventpb.EventPayload, + eventType string, + descID descpb.ID, + reportingID int32, +) error { + *args = append( + *args, + timeutil.Unix(0, event.CommonDetails().Timestamp), + eventType, int32(descID), reportingID, - nil, // info - } - if info != nil { - infoBytes, err := json.Marshal(info) + ) + var info interface{} + if event != nil { + infoBytes, err := json.Marshal(event) if err != nil { return err } - args[4] = string(infoBytes) + info = string(infoBytes) } - rows, err := ex.Exec(ctx, "log-event", txn, insertEventTableStmt, args...) + *args = append(*args, info) + return nil +} + +// execEventLogInsert executes the insert query to insert the new events +// into the event log table. +func execEventLogInsert( + ctx context.Context, + ex *InternalExecutor, + txn *kv.Txn, + query string, + args []interface{}, + numEvents int, +) error { + rows, err := ex.Exec(ctx, "log-event", txn, query, args...) if err != nil { return err } - if rows != 1 { - return errors.Errorf("%d rows affected by log insertion; expected exactly one row affected.", rows) + if rows != numEvents { + return errors.Errorf("%d rows affected by log insertion; expected %d rows affected.", rows, numEvents) + } + return nil +} + +// prepareRow creates the values of an insert for a row. It populates the +// event payload with additional info, and then adds the values of the row to args. +func prepareRow( + ctx context.Context, + txn *kv.Txn, + args *[]interface{}, + event eventpb.EventPayload, + descID descpb.ID, + reportingID int32, +) error { + // Setup event log. + eventType := eventpb.GetEventTypeName(event) + if err := setupEventAndMaybeLog( + ctx, txn, event, false, /* onlyLog */ + ); err != nil { + return err + } + + // Construct the args for this row. + if err := constructArgs(args, event, eventType, descID, reportingID); err != nil { + return err } return nil } diff --git a/pkg/sql/grant_revoke.go b/pkg/sql/grant_revoke.go index 2f6d57ff442c..1d9a7b1ec5d0 100644 --- a/pkg/sql/grant_revoke.go +++ b/pkg/sql/grant_revoke.go @@ -312,10 +312,14 @@ func (n *changePrivilegesNode) startExec(params runParams) error { // Record the privilege changes in the event log. This is an // auditable log event and is recorded in the same transaction as // the table descriptor update. + descIDs := make(descpb.IDs, 0, len(events)) + eventPayloads := make([]eventpb.EventPayload, 0, len(events)) for _, ev := range events { - if err := params.p.logEvent(params.ctx, ev.descID, ev.event); err != nil { - return err - } + descIDs = append(descIDs, ev.descID) + eventPayloads = append(eventPayloads, ev.event) + } + if err := params.p.batchLogEvents(params.ctx, descIDs, eventPayloads...); err != nil { + return err } return nil } diff --git a/pkg/sql/logictest/testdata/logic_test/event_log b/pkg/sql/logictest/testdata/logic_test/event_log index 91ba19dcf271..cdfc2dea4433 100644 --- a/pkg/sql/logictest/testdata/logic_test/event_log +++ b/pkg/sql/logictest/testdata/logic_test/event_log @@ -598,6 +598,12 @@ REVOKE CREATE ON SCHEMA sc FROM u,v statement ok REVOKE CREATE ON DATABASE dbt FROM u,v +statement ok +GRANT ALL ON * TO u + +statement ok +REVOKE ALL ON * FROM u + query ITT SELECT "reportingID", "info"::JSONB - 'Timestamp' - 'DescriptorID', "eventType" FROM system.eventlog @@ -616,6 +622,16 @@ ORDER BY "timestamp", info 1 {"EventType": "change_schema_privilege", "Grantee": "v", "RevokedPrivileges": ["CREATE"], "SchemaName": "sc", "Statement": "REVOKE CREATE ON SCHEMA \"\".sc FROM u, v", "User": "root"} change_schema_privilege 1 {"DatabaseName": "dbt", "EventType": "change_database_privilege", "Grantee": "u", "RevokedPrivileges": ["CREATE"], "Statement": "REVOKE CREATE ON DATABASE dbt FROM u, v", "User": "root"} change_database_privilege 1 {"DatabaseName": "dbt", "EventType": "change_database_privilege", "Grantee": "v", "RevokedPrivileges": ["CREATE"], "Statement": "REVOKE CREATE ON DATABASE dbt FROM u, v", "User": "root"} change_database_privilege +1 {"EventType": "change_table_privilege", "GrantedPrivileges": ["ALL"], "Grantee": "u", "Statement": "GRANT ALL ON TABLE * TO u", "TableName": "renamedtable", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "GrantedPrivileges": ["ALL"], "Grantee": "u", "Statement": "GRANT ALL ON TABLE * TO u", "TableName": "a", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "GrantedPrivileges": ["ALL"], "Grantee": "u", "Statement": "GRANT ALL ON TABLE * TO u", "TableName": "b", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "GrantedPrivileges": ["ALL"], "Grantee": "u", "Statement": "GRANT ALL ON TABLE * TO u", "TableName": "c", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "GrantedPrivileges": ["ALL"], "Grantee": "u", "Statement": "GRANT ALL ON TABLE * TO u", "TableName": "sq", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "Grantee": "u", "RevokedPrivileges": ["ALL"], "Statement": "REVOKE ALL ON TABLE * FROM u", "TableName": "renamedtable", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "Grantee": "u", "RevokedPrivileges": ["ALL"], "Statement": "REVOKE ALL ON TABLE * FROM u", "TableName": "a", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "Grantee": "u", "RevokedPrivileges": ["ALL"], "Statement": "REVOKE ALL ON TABLE * FROM u", "TableName": "b", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "Grantee": "u", "RevokedPrivileges": ["ALL"], "Statement": "REVOKE ALL ON TABLE * FROM u", "TableName": "c", "User": "root"} change_table_privilege +1 {"EventType": "change_table_privilege", "Grantee": "u", "RevokedPrivileges": ["ALL"], "Statement": "REVOKE ALL ON TABLE * FROM u", "TableName": "sq", "User": "root"} change_table_privilege statement ok DROP DATABASE dbt From 59ce286a2927d76bdbe69f62ad5c59cad765fa57 Mon Sep 17 00:00:00 2001 From: Azhng Date: Thu, 22 Apr 2021 15:21:07 -0400 Subject: [PATCH 5/5] sql: add statement_id to crdb_internal.node_statement_statistics Currently, crdb_internal.node_transaction_statistics uses the statement_ids column to reference statements in crdb_internal.node_statement_statistics. However, the statement statistics view does not have column that shows statement id. This commit introduce a new statement_id column in the statement statistics view. Release note (sql change): crdb_internal.node_statement_statistics now stores statement_id. --- pkg/sql/crdb_internal.go | 18 ++++++++++-------- .../testdata/logic_test/crdb_internal | 4 ++-- .../testdata/logic_test/crdb_internal_tenant | 4 ++-- .../testdata/logic_test/create_statements | 2 ++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index 50dcac8e67ba..10c417778c81 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -840,6 +840,7 @@ CREATE TABLE crdb_internal.node_statement_statistics ( node_id INT NOT NULL, application_name STRING NOT NULL, flags STRING NOT NULL, + statement_id STRING NOT NULL, key STRING NOT NULL, anonymized STRING, count INT NOT NULL, @@ -941,14 +942,15 @@ CREATE TABLE crdb_internal.node_statement_statistics ( flags = "!" + flags } err := addRow( - tree.NewDInt(tree.DInt(nodeID)), // node_id - tree.NewDString(appName), // application_name - tree.NewDString(flags), // flags - tree.NewDString(stmtKey.anonymizedStmt), // key - anonymized, // anonymized - tree.NewDInt(tree.DInt(s.mu.data.Count)), // count - tree.NewDInt(tree.DInt(s.mu.data.FirstAttemptCount)), // first_attempt_count - tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)), // max_retries + tree.NewDInt(tree.DInt(nodeID)), // node_id + tree.NewDString(appName), // application_name + tree.NewDString(flags), // flags + tree.NewDString(strconv.FormatUint(uint64(stmtID), 10)), // statement_id + tree.NewDString(stmtKey.anonymizedStmt), // key + anonymized, // anonymized + tree.NewDInt(tree.DInt(s.mu.data.Count)), // count + tree.NewDInt(tree.DInt(s.mu.data.FirstAttemptCount)), // first_attempt_count + tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)), // max_retries errString, // last_error tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)), // rows_avg tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))), // rows_var diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal b/pkg/sql/logictest/testdata/logic_test/crdb_internal index cfaa50b112d7..2402e3d198f1 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal @@ -154,10 +154,10 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0 ---- node_id table_id name parent_id expiration deleted -query ITTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBB colnames +query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBB colnames SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0 ---- -node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan +node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan query ITTTIIRRRRRRRRRRRRRRRRRR colnames SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0 diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant b/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant index 854569277081..ce6fba1ae709 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant @@ -168,10 +168,10 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0 ---- node_id table_id name parent_id expiration deleted -query ITTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBB colnames +query ITTTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRBB colnames SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0 ---- -node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan +node_id application_name flags statement_id key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn full_scan query ITTTIIRRRRRRRRRRRRRRRRRR colnames SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0 diff --git a/pkg/sql/logictest/testdata/logic_test/create_statements b/pkg/sql/logictest/testdata/logic_test/create_statements index 831232566865..9371cd93dcfb 100644 --- a/pkg/sql/logictest/testdata/logic_test/create_statements +++ b/pkg/sql/logictest/testdata/logic_test/create_statements @@ -661,6 +661,7 @@ CREATE TABLE crdb_internal.node_statement_statistics ( node_id INT8 NOT NULL, application_name STRING NOT NULL, flags STRING NOT NULL, + statement_id STRING NOT NULL, key STRING NOT NULL, anonymized STRING NULL, count INT8 NOT NULL, @@ -699,6 +700,7 @@ CREATE TABLE crdb_internal.node_statement_statistics ( node_id INT8 NOT NULL, application_name STRING NOT NULL, flags STRING NOT NULL, + statement_id STRING NOT NULL, key STRING NOT NULL, anonymized STRING NULL, count INT8 NOT NULL,