Merge #40644

40644: opt: fix memory corruption causing stack overflow r=rytaft a=rytaft This commit fixes a bug which was causing memory corruption and leading to stack overflow. The fix changes `DerivePruneCols` to always use a deep copy when deriving the `PruneCols` column set from other relational properties. Fixes #40589 Release note (bug fix): Fixed a memory corruption error in the optimizer that could cause stack overflow. Co-authored-by: Rebecca Taft <[email protected]>
cockroachdb · Sep 11, 2019 · f24615a · f24615a
2 parents acdabf6 + c3e6ff4
commit f24615a
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 3 deletions.
diff --git a/pkg/sql/logictest/testdata/logic_test/apply_join b/pkg/sql/logictest/testdata/logic_test/apply_join
@@ -311,3 +311,49 @@ WHERE
             )
         )
     )
+
+# Regression test for #40589.
+statement ok
+CREATE TABLE IF NOT EXISTS t40589 AS
+	SELECT
+		'2001-01-01'::TIMESTAMPTZ + g * '1 day',
+		g * '1 day'::INTERVAL AS _interval,
+		g % 0 = 0 AS _bool,
+		g AS _decimal,
+		g,
+		g AS _bytes,
+		substring(NULL, NULL, NULL)::UUID AS _uuid,
+		'0.0.0.0'::INET + g AS _inet,
+		g AS _jsonb
+	FROM
+		generate_series(NULL, NULL) AS g;
+
+query T
+SELECT
+	(
+		SELECT
+			NULL
+		FROM
+			t40589,
+			t40589 AS t0,
+			t40589 AS t1
+			INNER JOIN t40589 AS t2 ON true
+			JOIN t40589 AS t3
+				RIGHT JOIN t40589 AS t4
+					LEFT JOIN t40589 AS t5 ON
+							t._bool ON false ON
+					t1._uuid = t3._uuid
+			JOIN t40589 AS t6
+				JOIN t40589 AS t7
+					LEFT JOIN t40589 AS t8 ON true
+					JOIN t40589 AS t9
+						JOIN t40589 AS t10 ON true ON
+							true ON true ON true
+		WHERE
+			7.835752314020045477E+27
+			NOT IN (SELECT t6._decimal::DECIMAL)
+			AND true
+	)
+FROM
+	t40589 AS t, t40589;
+----
diff --git a/pkg/sql/opt/norm/prune_cols.go b/pkg/sql/opt/norm/prune_cols.go
@@ -420,7 +420,7 @@ func DerivePruneCols(e memo.RelExpr) opt.ColSet {
 	switch e.Op() {
 	case opt.ScanOp, opt.ValuesOp:
 		// All columns can potentially be pruned from the Scan and Values operators.
-		relProps.Rule.PruneCols = relProps.OutputCols
+		relProps.Rule.PruneCols = relProps.OutputCols.Copy()
 
 	case opt.SelectOp:
 		// Any pruneable input columns can potentially be pruned, as long as they're
@@ -433,7 +433,7 @@ func DerivePruneCols(e memo.RelExpr) opt.ColSet {
 	case opt.ProjectOp:
 		// All columns can potentially be pruned from the Project, if they're never
 		// used in a higher-level expression.
-		relProps.Rule.PruneCols = relProps.OutputCols
+		relProps.Rule.PruneCols = relProps.OutputCols.Copy()
 
 	case opt.InnerJoinOp, opt.LeftJoinOp, opt.RightJoinOp, opt.FullJoinOp,
 		opt.SemiJoinOp, opt.AntiJoinOp, opt.InnerJoinApplyOp, opt.LeftJoinApplyOp,
@@ -462,7 +462,7 @@ func DerivePruneCols(e memo.RelExpr) opt.ColSet {
 		// However, aggregation columns can potentially be pruned.
 		groupingColSet := e.Private().(*memo.GroupingPrivate).GroupingCols
 		if groupingColSet.Empty() {
-			relProps.Rule.PruneCols = relProps.OutputCols
+			relProps.Rule.PruneCols = relProps.OutputCols.Copy()
 		} else {
 			relProps.Rule.PruneCols = relProps.OutputCols.Difference(groupingColSet)
 		}