From c0c5de71ea9427c6b1e5ce3af6fbd6a8471d6ac4 Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Tue, 19 Sep 2023 13:51:37 -0500 Subject: [PATCH 1/3] dev: add support for `podman` Part of: DEVINF-522 Epic: none Release note: None --- dev | 2 +- pkg/cmd/dev/builder.go | 19 +++++++++++++++++++ pkg/cmd/dev/testdata/recorderdriven/builder | 2 ++ .../dev/testdata/recorderdriven/builder.rec | 6 ++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/dev b/dev index 582ce7eeee50..6a0f6723f99e 100755 --- a/dev +++ b/dev @@ -8,7 +8,7 @@ fi set -euo pipefail # Bump this counter to force rebuilding `dev` on all machines. -DEV_VERSION=86 +DEV_VERSION=87 THIS_DIR=$(cd "$(dirname "$0")" && pwd) BINARY_DIR=$THIS_DIR/bin/dev-versions diff --git a/pkg/cmd/dev/builder.go b/pkg/cmd/dev/builder.go index f108cfd17f53..89641a5c0a2f 100644 --- a/pkg/cmd/dev/builder.go +++ b/pkg/cmd/dev/builder.go @@ -11,6 +11,7 @@ package main import ( + "bytes" "context" "fmt" "log" @@ -56,6 +57,17 @@ func (d *dev) builder(cmd *cobra.Command, extraArgs []string) error { return d.exec.CommandContextInheritingStdStreams(ctx, "docker", args...) } +func (d *dev) dockerIsPodman(ctx context.Context) (bool, error) { + output, err := d.exec.CommandContextSilent(ctx, "docker", "help") + if err != nil { + return false, err + } + if bytes.Contains(output, []byte("podman")) { + return true, nil + } + return false, nil +} + func (d *dev) getDockerRunArgs( ctx context.Context, volume string, tty bool, extraArgs []string, ) (args []string, err error) { @@ -119,6 +131,13 @@ func (d *dev) getDockerRunArgs( } args = append(args, "run", "--rm") + isPodman, err := d.dockerIsPodman(ctx) + if err != nil { + return nil, err + } + if isPodman { + args = append(args, "--passwd=false") + } if tty { args = append(args, "-it") } else { diff --git a/pkg/cmd/dev/testdata/recorderdriven/builder b/pkg/cmd/dev/testdata/recorderdriven/builder index 011c1d30be44..6e095c9b6107 100644 --- a/pkg/cmd/dev/testdata/recorderdriven/builder +++ b/pkg/cmd/dev/testdata/recorderdriven/builder @@ -5,6 +5,7 @@ id bazel info workspace --color=no cat crdb-checkout/build/.bazelbuilderversion docker volume inspect bzlhome +docker help mkdir crdb-checkout/artifacts docker run --rm -it -v crdb-checkout:/cockroach --workdir=/cockroach -v crdb-checkout/build/bazelutil/empty.bazelrc:/cockroach/.bazelrc.user -v crdb-checkout/artifacts:/artifacts -v bzlhome:/home/roach:delegated -u 502:502 cockroachdb/bazel:20220328-163955 @@ -15,5 +16,6 @@ id bazel info workspace --color=no cat crdb-checkout/build/.bazelbuilderversion docker volume inspect bzlhome +docker help mkdir crdb-checkout/artifacts docker run --rm -i -v crdb-checkout:/cockroach --workdir=/cockroach -v crdb-checkout/build/bazelutil/empty.bazelrc:/cockroach/.bazelrc.user -v crdb-checkout/artifacts:/artifacts -v bzlhome:/home/roach:delegated -u 502:502 cockroachdb/bazel:20220328-163955 echo hi diff --git a/pkg/cmd/dev/testdata/recorderdriven/builder.rec b/pkg/cmd/dev/testdata/recorderdriven/builder.rec index 5e69c2d07713..533b050e338d 100644 --- a/pkg/cmd/dev/testdata/recorderdriven/builder.rec +++ b/pkg/cmd/dev/testdata/recorderdriven/builder.rec @@ -27,6 +27,9 @@ docker volume inspect bzlhome } ] +docker help +---- + mkdir crdb-checkout/artifacts ---- @@ -62,6 +65,9 @@ docker volume inspect bzlhome } ] +docker help +---- + mkdir crdb-checkout/artifacts ---- From 7ea7df01a49b9e68ca1940cc53e5a90c38abc685 Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Wed, 20 Sep 2023 13:51:54 -0400 Subject: [PATCH 2/3] catpb: make JobID implement fmt.Stringer This will make some things nicer (e.g. in roachtest/tests/jobs.go we used %s format directive). Epic: None Release note: None --- pkg/sql/catalog/catpb/job_id.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/sql/catalog/catpb/job_id.go b/pkg/sql/catalog/catpb/job_id.go index 69b3234dd116..17241b16414f 100644 --- a/pkg/sql/catalog/catpb/job_id.go +++ b/pkg/sql/catalog/catpb/job_id.go @@ -10,6 +10,8 @@ package catpb +import "strconv" + // JobID is the ID of a job. It is defined here and imported in jobspb because // jobs are referenced in descriptors and also jobs reference parts of // descriptors. This avoids any dependency cycles. @@ -20,3 +22,8 @@ const InvalidJobID JobID = 0 // SafeValue implements the redact.SafeValue interface. func (j JobID) SafeValue() {} + +// String implements the fmt.Stringer interface. +func (j JobID) String() string { + return strconv.Itoa(int(j)) +} From eb09145f9217c3194a68713c41b8927ddbd3bcbe Mon Sep 17 00:00:00 2001 From: rharding6373 Date: Wed, 7 Jun 2023 20:25:21 -0700 Subject: [PATCH 3/3] sql: add support for foreign key cascades in udfs This commit adds testing and makes some fixes to support foreign key cascades in UDFs. Epic: CRDB-25388 Informs: #87289 Release note: none --- pkg/sql/apply_join.go | 12 +- pkg/sql/logictest/testdata/logic_test/udf_fk | 411 ++++++++++++++++++- pkg/sql/opt/exec/execbuilder/scalar.go | 3 - 3 files changed, 401 insertions(+), 25 deletions(-) diff --git a/pkg/sql/apply_join.go b/pkg/sql/apply_join.go index 28d474d55be7..aa1daeb1222c 100644 --- a/pkg/sql/apply_join.go +++ b/pkg/sql/apply_join.go @@ -286,6 +286,7 @@ func runPlanInsidePlan( plannerCopy := *params.p plannerCopy.curPlan.planComponents = *plan + // "Pausable portal" execution model is only applicable to the outer // statement since we actually need to execute all inner plans to completion // before we can produce any "outer" rows to be returned to the client, so @@ -333,6 +334,12 @@ func runPlanInsidePlan( // We don't have "inner" subqueries, so the apply join can only refer to // the "outer" ones. plannerCopy.curPlan.subqueryPlans = params.p.curPlan.subqueryPlans + // During cleanup, nil out the inner subquery plans before closing the plan + // components. Otherwise, we may inadvertently close nodes that are needed + // when executing the outer query. + defer func() { + plan.subqueryPlans = nil + }() } distributePlan := getPlanDistribution( @@ -365,10 +372,13 @@ func runPlanInsidePlan( evalCtxFactory2 := func(usedConcurrently bool) *extendedEvalContext { return evalCtxFactory() } - + plannerCopy.autoCommit = false execCfg.DistSQLPlanner.PlanAndRunCascadesAndChecks( ctx, &plannerCopy, evalCtxFactory2, &plannerCopy.curPlan.planComponents, recv, ) + // We might have appended some cascades or checks to the plannerCopy, so we + // need to update the plan for cleanup purposes before proceeding. + *plan = plannerCopy.curPlan.planComponents if recv.commErr != nil { return recv.commErr } diff --git a/pkg/sql/logictest/testdata/logic_test/udf_fk b/pkg/sql/logictest/testdata/logic_test/udf_fk index 0ce5977eb743..e87c81b345c0 100644 --- a/pkg/sql/logictest/testdata/logic_test/udf_fk +++ b/pkg/sql/logictest/testdata/logic_test/udf_fk @@ -36,10 +36,10 @@ CREATE FUNCTION f_fk_p_c(k INT, r INT) RETURNS RECORD AS $$ INSERT INTO child VALUES (k, r) RETURNING *; $$ LANGUAGE SQL; -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c(100, 1); -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_p(100, 1); query T @@ -47,7 +47,7 @@ SELECT f_fk_p_c(100, 1); ---- (100,1) -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" WITH x AS (SELECT f_fk_c(101, 2)) INSERT INTO parent VALUES (2); query T @@ -68,10 +68,10 @@ CREATE FUNCTION f_fk_c_multi(k1 INT, r1 INT, k2 INT, r2 INT) RETURNS SETOF RECOR SELECT * FROM child WHERE c = k1 OR c = k2; $$ LANGUAGE SQL; -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_multi(101, 1, 102, 2); -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_multi(101, 2, 102, 1); query T rowsort @@ -96,13 +96,13 @@ CREATE FUNCTION f_fk_c_seq_last(k INT, r INT) RETURNS RECORD AS $$ SELECT nextval('s'); $$ LANGUAGE SQL; -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_seq_last(103,2); -statement error pq: currval\(\): currval of sequence \"test.public.s\" is not yet defined in this session +statement error pgcode 55000 pq: currval\(\): currval of sequence \"test.public.s\" is not yet defined in this session SELECT currval('s'); -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_seq_first(103,2); query I @@ -110,6 +110,8 @@ SELECT currval('s'); ---- 1 +subtest end + subtest delete statement ok @@ -158,26 +160,36 @@ CREATE FUNCTION f_fk_p_c_del(k INT, r INT) RETURNS RECORD AS $$ DELETE FROM child WHERE c = k RETURNING *; $$ LANGUAGE SQL; -statement ok +query T SELECT f_fk_p_del(4); +---- +(4) -statement error pq: delete on table "parent" violates foreign key constraint "child_p_fkey" on table "child"\nDETAIL: Key \(p\)=\(3\) is still referenced from table "child"\. +statement error pgcode 23503 pq: delete on table "parent" violates foreign key constraint "child_p_fkey" on table "child"\nDETAIL: Key \(p\)=\(3\) is still referenced from table "child"\. SELECT f_fk_p_del(3); -statement ok +query T SELECT f_fk_c_del(102); +---- +(102,3) -statement ok +query T SELECT f_fk_p_del(3); +---- +(3) -statement error pq: delete on table "parent" violates foreign key constraint "child_p_fkey" on table "child"\nDETAIL: Key \(p\)=\(2\) is still referenced from table "child"\. +statement error pgcode 23503 pq: delete on table "parent" violates foreign key constraint "child_p_fkey" on table "child"\nDETAIL: Key \(p\)=\(2\) is still referenced from table "child"\. SELECT f_fk_p_c_del(101,2); -statement ok +query T SELECT f_fk_c_p_del(101,2); +---- +(2) -statement ok +query TT SELECT f_fk_c_del(100), f_fk_p_del(1); +---- +(100,1) (1) query I rowsort SELECT * FROM parent @@ -187,6 +199,7 @@ query II rowsort SELECT * FROM child ---- +subtest end subtest upsert @@ -202,15 +215,17 @@ statement ok INSERT INTO parent VALUES (1), (3); # Insert -statement ok +query T SELECT f_fk_c_ocdu(100,1); +---- +(100,1) # Update to value not in parent fails. -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_ocdu(100,2); # Inserting value not in parent fails. -statement error pq: insert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: insert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_ocdu(101,2); statement ok @@ -218,11 +233,365 @@ CREATE FUNCTION f_fk_c_ups(k INT, r INT) RETURNS RECORD AS $$ UPSERT INTO child VALUES (k, r) RETURNING *; $$ LANGUAGE SQL; -statement ok +query T SELECT f_fk_c_ups(102,3); +---- +(102,3) -statement error pq: upsert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: upsert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_ups(102,4); -statement error pq: upsert on table "child" violates foreign key constraint "child_p_fkey" +statement error pgcode 23503 pq: upsert on table "child" violates foreign key constraint "child_p_fkey" SELECT f_fk_c_ups(103,4); + +subtest end + +subtest cascade + +statement ok +CREATE TABLE parent_cascade (p INT PRIMARY KEY); + +statement ok +CREATE TABLE child_cascade ( + c INT PRIMARY KEY, + p INT NOT NULL REFERENCES parent_cascade(p) ON DELETE CASCADE ON UPDATE CASCADE +); + +statement ok +CREATE FUNCTION f_fk_p_cascade(old INT, new INT) RETURNS RECORD AS $$ + UPDATE parent_cascade SET p = new WHERE p = old RETURNING *; +$$ LANGUAGE SQL; + +statement ok +INSERT INTO parent_cascade VALUES (1); + +statement ok +INSERT INTO child_cascade VALUES (100,1); + +# Test that we can successfully cascade an update one level. +query T +SELECT f_fk_p_cascade(1, 2); +---- +(2) + +query II rowsort +SELECT * FROM child_cascade; +---- +100 2 + +statement ok +INSERT INTO child_cascade VALUES (101,2), (102,2); + +# Test that UDFs can cascade updates to multiple rows. +query T +SELECT f_fk_p_cascade(2, 3); +---- +(3) + +query II rowsort +SELECT * FROM child_cascade; +---- +100 3 +101 3 +102 3 + +# Test that two update cascades to the same row result in the final value. +query TT +SELECT f_fk_p_cascade(3, 4), f_fk_p_cascade(4, 2); +---- +(4) (2) + +query II rowsort +SELECT * FROM child_cascade; +---- +100 2 +101 2 +102 2 + +statement ok +DROP TABLE child_cascade; + +# Make child_cascade with a unique FK so that we can introduce another level of +# FK references. +statement ok +CREATE TABLE child_cascade ( + c INT PRIMARY KEY, + p INT UNIQUE NOT NULL REFERENCES parent_cascade(p) ON DELETE CASCADE ON UPDATE CASCADE +); + +statement ok +CREATE TABLE grandchild_cascade ( + c INT PRIMARY KEY, + p INT NOT NULL REFERENCES child_cascade(p) ON DELETE CASCADE ON UPDATE CASCADE +); + +statement ok +INSERT INTO child_cascade VALUES (100,2); + +statement ok +INSERT INTO grandchild_cascade VALUES (1000,2); + +# Test two levels of cascading updates. +query T +SELECT f_fk_p_cascade(2, 3); +---- +(3) + +query II rowsort +SELECT * FROM child_cascade; +---- +100 3 + +query II rowsort +SELECT * FROM grandchild_cascade; +---- +1000 3 + +statement ok +CREATE OR REPLACE FUNCTION f_fk_c(k INT, r INT) RETURNS RECORD AS $$ + INSERT INTO child_cascade VALUES (k,r) RETURNING *; +$$ LANGUAGE SQL; + +# No updates occur if there is an error in a later UDF. +statement error pgcode 23503 pq: insert on table "child_cascade" violates foreign key constraint "child_cascade_p_fkey" +SELECT f_fk_p_cascade(3, 4), f_fk_c(10, 100); + +query II rowsort +SELECT * FROM child_cascade; +---- +100 3 + +query II rowsort +SELECT * FROM grandchild_cascade; +---- +1000 3 + +statement ok +CREATE FUNCTION f_fk_p_del_cascade(old INT) RETURNS RECORD AS $$ + DELETE FROM parent_cascade WHERE p = old RETURNING *; +$$ LANGUAGE SQL; + +# Test two levels of cascading deletes. +query T +SELECT f_fk_p_del_cascade(3); +---- +(3) + +query II rowsort +SELECT * FROM child_cascade; +---- + +query II rowsort +SELECT * FROM grandchild_cascade; +---- + +statement ok +INSERT INTO parent_cascade VALUES (1), (2); + +statement ok +INSERT INTO child_cascade VALUES (1, 1), (2, 2); + +statement ok +INSERT INTO grandchild_cascade VALUES (11, 1), (12, 2); + +# Test multiple cascading updates to different rows. +query TT rowsort +SELECT f_fk_p_cascade(1, 3), f_fk_p_cascade(2, 4); +---- +(3) (4) + +query II rowsort +SELECT * FROM child_cascade; +---- +1 3 +2 4 + +query II rowsort +SELECT * FROM grandchild_cascade; +---- +11 3 +12 4 + +# Test an update and multiple deletes, including to the updated row. +query TTT rowsort +SELECT f_fk_p_cascade(3, 5), f_fk_p_del_cascade(4), f_fk_p_del_cascade(5); +---- +(5) (4) (5) + +query I rowsort +SELECT * FROM parent_cascade; +---- + +query II rowsort +SELECT * FROM child_cascade; +---- + +query II rowsort +SELECT * FROM grandchild_cascade; +---- + +statement ok +DROP TABLE grandchild_cascade; + +statement ok +DROP TABLE child_cascade; + +statement ok +CREATE TABLE child_cascade ( + c INT PRIMARY KEY, + p INT REFERENCES parent_cascade(p) ON DELETE SET NULL ON UPDATE SET NULL +); + +statement ok +INSERT INTO parent_cascade VALUES (3); + +statement ok +INSERT INTO child_cascade VALUES (100,3); + +# Test cascading updates with `UPDATE SET NULL`. +query T +SELECT f_fk_p_cascade(3, 4); +---- +(4) + +query II rowsort +SELECT * FROM child_cascade; +---- +100 NULL + +statement ok +INSERT INTO child_cascade VALUES(101, 4); + +# Test cascading deletes with `DELETE SET NULL`. +query T +SELECT f_fk_p_del_cascade(4); +---- +(4) + +query II rowsort +SELECT * FROM child_cascade; +---- +100 NULL +101 NULL + +query I rowsort +SELECT * FROM parent_cascade; +---- + +statement ok +DROP TABLE child_cascade + +subtest end + +# Test a query with both an apply join and a UDF with a cascade. +subtest apply_join + +statement ok +CREATE TABLE child_cascade ( + c INT PRIMARY KEY, + p INT UNIQUE NOT NULL REFERENCES parent_cascade(p) ON DELETE CASCADE ON UPDATE CASCADE +); + +statement ok +CREATE TABLE grandchild_cascade ( + c INT PRIMARY KEY, + p INT NOT NULL REFERENCES child_cascade(p) ON DELETE CASCADE ON UPDATE CASCADE +); + +statement ok +CREATE OR REPLACE FUNCTION f_fk_p_cascade(old INT, new INT) RETURNS RECORD AS $$ + UPDATE parent_cascade SET p = new WHERE p = old RETURNING *; +$$ LANGUAGE SQL; + +statement ok +INSERT INTO parent_cascade VALUES (1), (2), (3); + +statement ok +INSERT INTO child_cascade VALUES (1, 1), (2, 2), (3, 3); + +statement ok +INSERT INTO grandchild_cascade VALUES (11, 1), (12, 2), (13, 3); + +query IT rowsort +SELECT + (SELECT * FROM (VALUES ((SELECT x FROM (VALUES (1)) AS s (x)) + y))), + f_fk_p_cascade(y, y+10) +FROM + (VALUES (1), (2), (3)) AS t (y) +---- +2 (11) +3 (12) +4 (13) + +query II rowsort +SELECT * FROM child_cascade +---- +1 11 +2 12 +3 13 + +query II rowsort +SELECT * FROM grandchild_cascade +---- +11 11 +12 12 +13 13 + +# Test multiple cascades in the same function. +statement ok +CREATE OR REPLACE FUNCTION f_fk_swap(a INT, b INT) RETURNS RECORD AS $$ + UPDATE parent_cascade SET p = a+1 WHERE p = b RETURNING *; + UPDATE parent_cascade SET p = b WHERE p = a RETURNING *; + UPDATE parent_cascade SET p = a WHERE p = a+1 RETURNING *; +$$ LANGUAGE SQL; + +query T +SELECT f_fk_swap(13, 12); +---- +(13) + +query II rowsort +SELECT * FROM grandchild_cascade +---- +11 11 +12 13 +13 12 + +statement ok +CREATE TABLE grandchild( + c INT PRIMARY KEY, + p INT NOT NULL REFERENCES child_cascade(p) +); + +statement ok +INSERT INTO grandchild VALUES (11,11), (12,13), (13,12); + +statement error pgcode 23503 pq: update on table "child_cascade" violates foreign key constraint "grandchild_p_fkey" on table "grandchild" +SELECT f_fk_p_cascade(13, 14) + +statement error pgcode 23503 pq: update on table "child_cascade" violates foreign key constraint "grandchild_p_fkey" on table "grandchild" +SELECT f_fk_swap(13, 12); + +statement ok +CREATE TABLE selfref (a INT PRIMARY KEY, b INT NOT NULL REFERENCES selfref(a) ON UPDATE CASCADE) + +statement ok +INSERT INTO selfref VALUES (1,1); + +statement ok +CREATE FUNCTION f_selfref(old INT, new INT) RETURNS RECORD AS $$ + UPDATE selfref SET a = new WHERE a = old RETURNING *; +$$ LANGUAGE SQL; + +# Test a self-referencing FK cascade. +query T +SELECT f_selfref(1,2); +---- +(2,1) + +query II +SELECT * FROM selfref; +---- +2 2 + +subtest end diff --git a/pkg/sql/opt/exec/execbuilder/scalar.go b/pkg/sql/opt/exec/execbuilder/scalar.go index 087452fd7cc3..65e9536ca164 100644 --- a/pkg/sql/opt/exec/execbuilder/scalar.go +++ b/pkg/sql/opt/exec/execbuilder/scalar.go @@ -1159,9 +1159,6 @@ func (b *Builder) buildRoutinePlanGenerator( if len(eb.subqueries) > 0 { return expectedLazyRoutineError("subquery") } - if len(eb.cascades) > 0 { - return expectedLazyRoutineError("cascade") - } isFinalPlan := i == len(stmts)-1 err = fn(plan, isFinalPlan) if err != nil {