sql: add cluster setting to preserve subquery and cte ordering

This patch adds an experimental cluster setting `propagate_input_ordering`
that indicates whether subqueries and CTEs should propagate their orderings
to their parent scopes, when the parent scope is unordered. As an example,
the following two queries should produce the following result when the
cluster setting is true:
```
select * from (select * from generate_series(1, 10) i order by i % 5 asc, i asc) tmp;
with tmp as (select * from generate_series(1, 10) i order by i % 5 asc, i asc) select * from tmp;
----
   5
  10
   1
   6
   2
   7
   3
   8
   4
   9
```
This allows cockroach to imitate postgres behavior - while postgres
does not guarantee to maintain ordering on subqueries, it does in
practice. Some existing applications take advantage of this fact,
and so the ability to toggle this setting can help resolve
incompatibilities in some cases.

Fixes #68211

Release note: None

pkg/sql/exec_util.go

@@ -432,6 +432,12 @@ var copyPartitioningWhenDeinterleavingTable = settings.RegisterBoolSetting(
 	false,
 ).WithPublic()
 
+var propagateInputOrdering = settings.RegisterBoolSetting(
+	`sql.defaults.propagate_input_ordering.enabled`,
+	`default value for the experimental propagate_input_ordering session variable`,
+	false,
+)
+
 // settingWorkMemBytes is a cluster setting that determines the maximum amount
 // of RAM that a processor can use.
 var settingWorkMemBytes = settings.RegisterByteSizeSetting(
@@ -2637,6 +2643,10 @@ func (m *sessionDataMutator) SetExperimentalComputedColumnRewrites(val string) {
 	m.data.ExperimentalComputedColumnRewrites = val
 }
 
+func (m *sessionDataMutator) SetPropagateInputOrdering(b bool) {
+	m.data.PropagateInputOrdering = b
+}
+
 // Utility functions related to scrubbing sensitive information on SQL Stats.
 
 // quantizeCounts ensures that the Count field in the

pkg/sql/logictest/testdata/logic_test/information_schema

@@ -4561,6 +4561,7 @@ optimizer_use_histograms                               on
 optimizer_use_multicol_stats                           on
 override_multi_region_zone_config                      off
 prefer_lookup_joins_for_fks                            off
+propagate_input_ordering                               off
 reorder_joins_limit                                    8
 require_explicit_primary_keys                          off
 results_buffer_size                                    16384

pkg/sql/logictest/testdata/logic_test/pg_catalog

@@ -3880,6 +3880,7 @@ optimizer_use_histograms                               on                  NULL
 optimizer_use_multicol_stats                           on                  NULL      NULL        NULL        string
 override_multi_region_zone_config                      off                 NULL      NULL        NULL        string
 prefer_lookup_joins_for_fks                            off                 NULL      NULL        NULL        string
+propagate_input_ordering                               off                 NULL      NULL        NULL        string
 reorder_joins_limit                                    8                   NULL      NULL        NULL        string
 require_explicit_primary_keys                          off                 NULL      NULL        NULL        string
 results_buffer_size                                    16384               NULL      NULL        NULL        string
@@ -3965,6 +3966,7 @@ optimizer_use_histograms                               on                  NULL
 optimizer_use_multicol_stats                           on                  NULL  user     NULL      on                  on
 override_multi_region_zone_config                      off                 NULL  user     NULL      off                 off
 prefer_lookup_joins_for_fks                            off                 NULL  user     NULL      off                 off
+propagate_input_ordering                               off                 NULL  user     NULL      off                 off
 reorder_joins_limit                                    8                   NULL  user     NULL      8                   8
 require_explicit_primary_keys                          off                 NULL  user     NULL      off                 off
 results_buffer_size                                    16384               NULL  user     NULL      16384               16384
@@ -4047,6 +4049,7 @@ optimizer_use_histograms                               NULL    NULL     NULL
 optimizer_use_multicol_stats                           NULL    NULL     NULL     NULL        NULL
 override_multi_region_zone_config                      NULL    NULL     NULL     NULL        NULL
 prefer_lookup_joins_for_fks                            NULL    NULL     NULL     NULL        NULL
+propagate_input_ordering                               NULL    NULL     NULL     NULL        NULL
 reorder_joins_limit                                    NULL    NULL     NULL     NULL        NULL
 require_explicit_primary_keys                          NULL    NULL     NULL     NULL        NULL
 results_buffer_size                                    NULL    NULL     NULL     NULL        NULL

pkg/sql/logictest/testdata/logic_test/preserve_input_order

@@ -0,0 +1,137 @@
+statement ok
+SET propagate_input_ordering=true;
+
+query I
+WITH tmp AS (SELECT * FROM generate_series(1, 10) i ORDER BY i % 5 ASC, i ASC) SELECT * FROM tmp;
+----
+5
+10
+1
+6
+2
+7
+3
+8
+4
+9
+
+query I
+WITH tmp AS (SELECT * FROM generate_series(1, 10) i ORDER BY i % 5 ASC, i ASC) SELECT * FROM tmp;
+----
+5
+10
+1
+6
+2
+7
+3
+8
+4
+9
+
+# The internal ordering column for i%5 should not be present in the output.
+query T
+SELECT foo FROM (SELECT i, i%2 FROM generate_series(1, 10) i ORDER BY i % 5 ASC, i ASC) AS foo
+----
+(5,1)
+(10,0)
+(1,1)
+(6,0)
+(2,0)
+(7,1)
+(3,1)
+(8,0)
+(4,0)
+(9,1)
+
+# The internal ordering column for i%5 should not be present in the output.
+query II
+SELECT foo.* FROM (SELECT i, i%2 FROM generate_series(1, 10) i ORDER BY i % 5 ASC, i ASC) AS foo
+----
+5   1
+10  0
+1   1
+6   0
+2   0
+7   1
+3   1
+8   0
+4   0
+9   1
+
+# The subquery ordering is propagated to the aggregation.
+query T
+SELECT array_agg(i) FROM (SELECT * FROM generate_series(1, 5) i ORDER BY i%2 DESC, i)
+----
+{1,3,5,2,4}
+
+# The input ordering is not propagated through joins.
+query II
+WITH tmp AS (SELECT * FROM generate_series(1, 2) x),
+     tmp2 AS (SELECT * FROM generate_series(8, 12) i ORDER BY i % 5 ASC, i ASC)
+SELECT * FROM tmp, tmp2;
+----
+1  8
+1  9
+1  10
+1  11
+1  12
+2  8
+2  9
+2  10
+2  11
+2  12
+
+# The input ordering is not propagated through joins.
+query II
+SELECT *
+FROM (SELECT * FROM generate_series(1, 2) x) tmp,
+     (SELECT * FROM generate_series(8, 12) i ORDER BY i % 5 ASC, i ASC) tmp2;
+----
+1  8
+1  9
+1  10
+1  11
+1  12
+2  8
+2  9
+2  10
+2  11
+2  12
+
+# Do not preserve the subquery ordering because the parent scope has its own
+# ordering.
+query I
+WITH tmp AS (SELECT * FROM generate_series(1, 10) i ORDER BY i % 5 ASC, i ASC)
+SELECT * FROM tmp ORDER BY i DESC;
+----
+10
+9
+8
+7
+6
+5
+4
+3
+2
+1
+
+# Do not preserve the subquery ordering because the parent scope has its own
+# ordering.
+query I
+WITH tmp AS (SELECT * FROM generate_series(1, 10) i ORDER BY i % 5 ASC, i ASC)
+SELECT * FROM tmp ORDER BY i DESC;
+----
+10
+9
+8
+7
+6
+5
+4
+3
+2
+1
+
+statement ok
+RESET propagate_input_ordering;

pkg/sql/logictest/testdata/logic_test/show_source

@@ -77,6 +77,7 @@ optimizer_use_histograms                               on
 optimizer_use_multicol_stats                           on
 override_multi_region_zone_config                      off
 prefer_lookup_joins_for_fks                            off
+propagate_input_ordering                               off
 reorder_joins_limit                                    8
 require_explicit_primary_keys                          off
 results_buffer_size                                    16384

pkg/sql/opt/optbuilder/project.go

@@ -24,6 +24,11 @@ import (
 // different set of columns than its input. Either way, it updates
 // projectionsScope.group with the output memo group ID.
 func (b *Builder) constructProjectForScope(inScope, projectionsScope *scope) {
+	if b.evalCtx.SessionData.PropagateInputOrdering && len(projectionsScope.ordering) == 0 {
+		// Preserve the input ordering.
+		projectionsScope.copyOrdering(inScope)
+	}
+
 	// Don't add an unnecessary "pass through" project.
 	if projectionsScope.hasSameColumns(inScope) {
 		projectionsScope.expr = inScope.expr

pkg/sql/opt/optbuilder/select.go

@@ -77,9 +77,9 @@ func (b *Builder) buildDataSource(
 		if cte := inScope.resolveCTE(tn); cte != nil {
 			locking.ignoreLockingForCTE()
 			outScope = inScope.push()
-			inCols := make(opt.ColList, len(cte.cols))
-			outCols := make(opt.ColList, len(cte.cols))
-			outScope.cols = nil
+			inCols := make(opt.ColList, len(cte.cols), len(cte.cols)+len(inScope.ordering))
+			outCols := make(opt.ColList, len(cte.cols), len(cte.cols)+len(inScope.ordering))
+			outScope.cols, outScope.extraCols = nil, nil
 			for i, col := range cte.cols {
 				id := col.ID
 				c := b.factory.Metadata().ColumnMeta(id)
@@ -89,6 +89,34 @@ func (b *Builder) buildDataSource(
 				outCols[i] = newCol.id
 			}
 
+			if b.evalCtx.SessionData.PropagateInputOrdering && len(inScope.ordering) > 0 {
+				var oldToNew opt.ColMap
+				for i := range inCols {
+					oldToNew.Set(int(inCols[i]), int(outCols[i]))
+				}
+				outScope.ordering = make(opt.Ordering, len(inScope.ordering))
+				for i, col := range inScope.ordering {
+					var newID int
+					var ok bool
+					if newID, ok = oldToNew.Get(int(col.ID())); !ok {
+						c := b.factory.Metadata().ColumnMeta(col.ID())
+						outScope.extraCols = append(outScope.extraCols,
+							scopeColumn{
+								name: scopeColName(tree.Name("order_" + c.Alias)),
+								typ:  c.Type,
+							},
+						)
+						newCol := &outScope.extraCols[len(outScope.extraCols)-1]
+						b.populateSynthesizedColumn(newCol, nil)
+						newCol.table = *tn
+						newID = int(newCol.id)
+						inCols = append(inCols, col.ID())
+						outCols = append(outCols, newCol.id)
+					}
+					outScope.ordering[i] = opt.MakeOrderingColumn(opt.ColumnID(newID), col.Descending())
+				}
+			}
+
 			outScope.expr = b.factory.ConstructWithScan(&memo.WithScanPrivate{
 				With:    cte.id,
 				Name:    string(cte.name.Alias),