diff --git a/pkg/sql/logictest/testdata/logic_test/udf b/pkg/sql/logictest/testdata/logic_test/udf index d0b72fa2e282..7f89f0220470 100644 --- a/pkg/sql/logictest/testdata/logic_test/udf +++ b/pkg/sql/logictest/testdata/logic_test/udf @@ -2937,6 +2937,26 @@ SELECT all_fn(1), all_fn(2), all_fn(NULL::INT) NULL false NULL +subtest array_flatten + +statement ok +CREATE FUNCTION arr(x INT) RETURNS INT[] LANGUAGE SQL AS $$ + SELECT ARRAY(VALUES (1), (2), (x)); +$$ + +query T +SELECT arr(10) +---- +{1,2,10} + +query T +SELECT arr(i) FROM generate_series(1, 3) g(i) +---- +{1,2,1} +{1,2,2} +{1,2,3} + + subtest variadic # Variadic UDFS are not currently supported. @@ -3046,10 +3066,10 @@ SELECT oid, proname, pronamespace, proowner, prolang, proleakproof, proisstrict, FROM pg_catalog.pg_proc WHERE proname IN ('f_93314', 'f_93314_alias', 'f_93314_comp', 'f_93314_comp_t') ORDER BY oid; ---- -100271 f_93314 105 1546506610 14 false false false v 0 100270 · {} NULL SELECT i, e FROM test.public.t_93314 ORDER BY i LIMIT 1; -100273 f_93314_alias 105 1546506610 14 false false false v 0 100272 · {} NULL SELECT i, e FROM test.public.t_93314_alias ORDER BY i LIMIT 1; -100277 f_93314_comp 105 1546506610 14 false false false v 0 100274 · {} NULL SELECT (1, 2); -100278 f_93314_comp_t 105 1546506610 14 false false false v 0 100276 · {} NULL SELECT a, c FROM test.public.t_93314_comp LIMIT 1; +100272 f_93314 105 1546506610 14 false false false v 0 100271 · {} NULL SELECT i, e FROM test.public.t_93314 ORDER BY i LIMIT 1; +100274 f_93314_alias 105 1546506610 14 false false false v 0 100273 · {} NULL SELECT i, e FROM test.public.t_93314_alias ORDER BY i LIMIT 1; +100278 f_93314_comp 105 1546506610 14 false false false v 0 100275 · {} NULL SELECT (1, 2); +100279 f_93314_comp_t 105 1546506610 14 false false false v 0 100277 · {} NULL SELECT a, c FROM test.public.t_93314_comp LIMIT 1; # Regression test for #95240. Strict UDFs that are inlined should result in NULL # when presented with NULL arguments. diff --git a/pkg/sql/opt/exec/execbuilder/scalar.go b/pkg/sql/opt/exec/execbuilder/scalar.go index c5eba38fac4b..937ba6775e1c 100644 --- a/pkg/sql/opt/exec/execbuilder/scalar.go +++ b/pkg/sql/opt/exec/execbuilder/scalar.go @@ -495,6 +495,16 @@ func (b *Builder) buildArrayFlatten( panic(errors.AssertionFailedf("input to ArrayFlatten should be uncorrelated")) } + if b.planLazySubqueries { + // The NormalizeArrayFlattenToAgg rule should have converted an + // ArrayFlatten within a UDF into an aggregation. + // We don't yet convert an ArrayFlatten within a correlated subquery + // into an aggregation, so we return a decorrelation error. + // TODO(mgartner): Build an ArrayFlatten within a correlated subquery as + // a Routine, or apply NormalizeArrayFlattenToAgg to all ArrayFlattens. + return nil, b.decorrelationError() + } + root, err := b.buildRelational(af.Input) if err != nil { return nil, err @@ -762,10 +772,6 @@ func (b *Builder) buildSubquery( // because we don't need to optimize the subquery input any further. // It's already been fully optimized because it is uncorrelated and has // no outer columns. - // - // TODO(mgartner): Uncorrelated subqueries only need to be evaluated - // once. We should cache their result to avoid all this overhead for - // every invocation. inputRowCount := int64(input.Relational().Statistics().RowCountIfAvailable()) withExprs := make([]builtWithExpr, len(b.withExprs)) copy(withExprs, b.withExprs) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/subquery b/pkg/sql/opt/exec/execbuilder/testdata/subquery index 9673591b71cb..aad587ab8255 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/subquery +++ b/pkg/sql/opt/exec/execbuilder/testdata/subquery @@ -562,3 +562,10 @@ vectorized: true estimated row count: 1,000 (missing stats) table: corr@corr_pkey spans: FULL SCAN + +# Case where the a correlated subquery contains an uncorrelated array-flatten +# subquery. +statement error could not decorrelate subquery +SELECT + CASE WHEN k < 5 THEN (SELECT array(SELECT 1) FROM corr tmp WHERE k*10 = corr.k) END +FROM corr diff --git a/pkg/sql/opt/exec/execbuilder/testdata/udf b/pkg/sql/opt/exec/execbuilder/testdata/udf index e33456450b96..71ac848d8b33 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/udf +++ b/pkg/sql/opt/exec/execbuilder/testdata/udf @@ -235,6 +235,30 @@ SELECT sub_fn4() FROM generate_series(1, 3) 1 1 +statement ok +CREATE FUNCTION arr() RETURNS INT[] LANGUAGE SQL AS $$ + SELECT ARRAY(VALUES (1), (2)); +$$ + +# A query with a uncorrelated array-flatten within a UDF. +query T +EXPLAIN (VERBOSE) SELECT arr() FROM generate_series(1, 3) +---- +distribution: local +vectorized: true +· +• render +│ columns: (arr) +│ render arr: arr() +│ +└── • project set + │ columns: (generate_series) + │ estimated row count: 10 + │ render 0: generate_series(1, 3) + │ + └── • emptyrow + columns: () + subtest regressions diff --git a/pkg/sql/opt/norm/rules/scalar.opt b/pkg/sql/opt/norm/rules/scalar.opt index 6e9866c3d0ed..d29a362432fa 100644 --- a/pkg/sql/opt/norm/rules/scalar.opt +++ b/pkg/sql/opt/norm/rules/scalar.opt @@ -391,7 +391,10 @@ $input # Max1Row operator we introduce is guaranteed to be eliminated as # MakeArrayAggForFlatten will return a ScalarGroupBy. [NormalizeArrayFlattenToAgg, Normalize] -(ArrayFlatten $input:(HasOuterCols $input) $subquery:*) +(ArrayFlatten + $input:* + $private:* & (CanNormalizeArrayFlatten $input $private) +) => (Coalesce [ @@ -403,7 +406,7 @@ $input (ArrayAgg (Variable $requestedCol:(SubqueryRequestedCol - $subquery + $private ) ) ) @@ -414,7 +417,7 @@ $input ] (MakeGrouping (MakeEmptyColSet) - (SubqueryOrdering $subquery) + (SubqueryOrdering $private) ) ) (MakeUnorderedSubquery) diff --git a/pkg/sql/opt/norm/scalar_funcs.go b/pkg/sql/opt/norm/scalar_funcs.go index 2fc808f78586..cd7125b659c3 100644 --- a/pkg/sql/opt/norm/scalar_funcs.go +++ b/pkg/sql/opt/norm/scalar_funcs.go @@ -387,3 +387,9 @@ func (c *CustomFuncs) SplitTupleEq(lhs, rhs *memo.TupleExpr) memo.FiltersExpr { } return res } + +// CanNormalizeArrayFlatten returns true if the input is correlated or if the +// ArrayFlatten exists within a UDF. +func (c *CustomFuncs) CanNormalizeArrayFlatten(input memo.RelExpr, p *memo.SubqueryPrivate) bool { + return c.HasOuterCols(input) || p.WithinUDF +} diff --git a/pkg/sql/opt/norm/testdata/rules/scalar b/pkg/sql/opt/norm/testdata/rules/scalar index 0bb77892c381..8d710fe008a8 100644 --- a/pkg/sql/opt/norm/testdata/rules/scalar +++ b/pkg/sql/opt/norm/testdata/rules/scalar @@ -1985,6 +1985,50 @@ project ├── columns: k:8!null └── key: (8) +exec-ddl +CREATE FUNCTION arr() RETURNS INT[] LANGUAGE SQL AS $$ + SELECT ARRAY(VALUES (1), (2)); +$$ +---- + +# Should trigger for uncorrelated ArrayFlatten subqueries within a UDF +norm expect=NormalizeArrayFlattenToAgg format=show-scalars +SELECT arr() +---- +values + ├── columns: arr:4 + ├── cardinality: [1 - 1] + ├── volatile + ├── key: () + ├── fd: ()-->(4) + └── tuple + └── udf: arr + └── body + └── values + ├── columns: array:3 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(3) + └── tuple + └── coalesce + ├── subquery + │ └── scalar-group-by + │ ├── columns: array_agg:2 + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(2) + │ ├── values + │ │ ├── columns: column1:1!null + │ │ ├── cardinality: [2 - 2] + │ │ ├── tuple + │ │ │ └── const: 1 + │ │ └── tuple + │ │ └── const: 2 + │ └── aggregations + │ └── array-agg [as=array_agg:2, outer=(1)] + │ └── variable: column1:1 + └── const: ARRAY[] + exec-ddl CREATE TABLE pg_class ( oid OID NULL, diff --git a/pkg/sql/opt/ops/scalar.opt b/pkg/sql/opt/ops/scalar.opt index c35cf2e3b3e6..e5e38e67a6bf 100644 --- a/pkg/sql/opt/ops/scalar.opt +++ b/pkg/sql/opt/ops/scalar.opt @@ -43,6 +43,10 @@ define SubqueryPrivate { # will eventually be output. It is only used for ArrayFlatten expressions. RequestedCol ColumnID + # WithinUDF is set to true if the subquery exists inside a UDFExpr. It is + # only used for ArrayFlatten expressions. + WithinUDF bool + # Cmp is only used for AnyOp. Cmp Operator diff --git a/pkg/sql/opt/optbuilder/scalar.go b/pkg/sql/opt/optbuilder/scalar.go index f74582f335ca..6a3360b16476 100644 --- a/pkg/sql/opt/optbuilder/scalar.go +++ b/pkg/sql/opt/optbuilder/scalar.go @@ -169,6 +169,7 @@ func (b *Builder) buildScalar( OriginalExpr: s.Subquery, Ordering: s.ordering, RequestedCol: inCol, + WithinUDF: b.insideUDF, } out = b.factory.ConstructArrayFlatten(s.node, &subqueryPrivate)