Merge #86662

86662: ci: add scripting to generate declarative schema changer corpus r=fqazi a=fqazi

These changes do the following:

Informs: #82643

1. Add a test case for validating schema changer corpuses back into master (was accidentally cloberred)
2. Extend logictest to run certain tests in a mixed version test that are interesting from a declarative schema changer viewpoint (we will extend this more)
3. More importantly for the dev-infra team an initial approach to collect declarative schema changer states from logictest during nightly builds. The scripts included here will generate a corpus and upload them to GCS storage for a nightly build.

Release justification: low risk changes to add scripting for nightly artifacts to allow us to do mixed version testing for the declarative schema changer

Co-authored-by: Faizan Qazi <[email protected]>

build/teamcity/cockroach/nightlies/sqllogic_corpus_nightly.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+tc_start_block "Run SQL Logic Test with Declarative Corpus Generation"
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILD_BRANCH -e GITHUB_API_TOKEN -e GOOGLE_EPHEMERAL_CREDENTIALS -e BUILD_VCS_NUMBER -e TC_BUILD_ID -e TC_SERVER_URL" \
+  run_bazel build/teamcity/cockroach/nightlies/sqllogic_corpus_nightly_impl.sh
+tc_end_block "Run SQL Logic Test High VModule"

build/teamcity/cockroach/nightlies/sqllogic_corpus_nightly_impl.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+
+set -xeuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"  # For process_test_json
+source "$dir/teamcity-support.sh"  # For process_test_json
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
+BAZEL_BIN=$(bazel info bazel-bin --config=ci)
+google_credentials="$GOOGLE_EPHEMERAL_CREDENTIALS"
+
+log_into_gcloud
+
+ARTIFACTS_DIR=/artifacts
+GO_TEST_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test.json.txt
+GO_TEST_GEN_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test-gen.json.txt
+GO_TEST_GEN_CCL_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test-gen-ccl.json.txt
+GO_TEST_VALIDATE_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test-validate.json.txt
+GO_TEST_JSON_OUTPUT_FILE_MIXED=$ARTIFACTS_DIR/test-mixed.json.txt
+GO_TEST_VALIDATE_JSON_OUTPUT_FILE_MIXED=$ARTIFACTS_DIR/test-validate-mixed.json.txt
+
+mkdir -p $ARTIFACTS_DIR/corpus
+mkdir -p $ARTIFACTS_DIR/corpus-mixed
+exit_status=0
+
+# Generate a corpus for all non-mixed version variants
+for config in local multiregion-9node-3region-3azs; do
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci -- --config=ci \
+    test //pkg/sql/logictest/tests/$config/... \
+    --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus \
+    --test_env=GO_TEST_WRAP_TESTV=1 \
+    --test_env=GO_TEST_WRAP=1 \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE.$config \
+    --test_timeout=7200 \
+    || exit_status=$?
+
+process_test_json \
+  $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+  $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+  $ARTIFACTS_DIR \
+  $GO_TEST_JSON_OUTPUT_FILE.$config \
+  $exit_status
+done
+
+for config in local multiregion-9node-3region-3azs multiregion-9node-3region-3azs-no-los multiregion-9node-3region-3azs-tenant multiregion-9node-3region-3azs-vec-off multiregion-15node-5region-3azs 3node-tenant 3node-tenant-multiregion; do
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+    test //pkg/ccl/logictestccl/tests/$config/... -- \
+    --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus \
+    --test_env=GO_TEST_WRAP_TESTV=1 \
+    --test_env=GO_TEST_WRAP=1 \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE.$config \
+    --test_timeout=7200 \
+    || exit_status=$?
+
+process_test_json \
+  $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+  $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+  $ARTIFACTS_DIR \
+  $GO_TEST_JSON_OUTPUT_FILE.$config \
+  $exit_status
+done
+
+# Generate corpuses from end-to-end-schema changer tests
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+  test //pkg/sql/schemachanger:schemachanger_test -- \
+  --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus \
+  --test_filter='^TestGenerateCorpus.*$' \
+  --test_env=GO_TEST_WRAP_TESTV=1 \
+  --test_env=GO_TEST_WRAP=1 \
+  --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_GEN_JSON_OUTPUT_FILE \
+  --test_timeout=7200 \
+  || exit_status=$?
+
+process_test_json \
+$BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+$BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+$ARTIFACTS_DIR \
+$GO_TEST_GEN_JSON_OUTPUT_FILE \
+$exit_status
+
+# Generate corpuses from end-to-end-schema changer tests
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+  test //pkg/ccl/schemachangerccl:schemachangerccl_test -- \
+  --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus \
+  --test_filter='^TestGenerateCorpus.*$' \
+  --test_env=GO_TEST_WRAP_TESTV=1 \
+  --test_env=GO_TEST_WRAP=1 \
+  --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_GEN_CCL_JSON_OUTPUT_FILE \
+  --test_timeout=7200 \
+  || exit_status=$?
+
+process_test_json \
+$BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+$BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+$ARTIFACTS_DIR \
+$GO_TEST_GEN_CCL_JSON_OUTPUT_FILE \
+$exit_status
+
+
+# Any generated corpus should be validated on the current version first, which
+# indicates we can replay it on the same version.
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+  test //pkg/sql/schemachanger/corpus:corpus_test -- \
+  --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus \
+  --test_filter='^TestValidateCorpuses$' \
+  --test_env=GO_TEST_WRAP_TESTV=1 \
+  --test_env=GO_TEST_WRAP=1 \
+  --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_VALIDATE_JSON_OUTPUT_FILE \
+  --test_timeout=7200 \
+  || exit_status=$?
+
+process_test_json \
+$BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+$BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+$ARTIFACTS_DIR \
+$GO_TEST_VALIDATE_JSON_OUTPUT_FILE \
+$exit_status
+
+# If validation passes its safe to update the copy in storage.
+if [ $exit_status = 0 ]; then
+  gsutil cp  $ARTIFACTS_DIR/corpus/* gs://cockroach-corpus/corpus-$TC_BUILD_BRANCH/
+fi
+
+# Generate a corpus for all mixed version variants
+for config in local-mixed-22.1-22.2; do
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+    test //pkg/sql/logictest/tests/$config/... -- \
+    --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus-mixed\
+    --test_env=GO_TEST_WRAP_TESTV=1 \
+    --test_env=GO_TEST_WRAP=1 \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE_MIXED.$config \
+    --test_timeout=7200 \
+    || exit_status=$?
+
+process_test_json \
+  $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+  $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+  $ARTIFACTS_DIR \
+  $GO_TEST_JSON_OUTPUT_FILE_MIXED.$config \
+  $exit_status
+done
+
+# Any generated corpus should be validated on the current version first, which
+# indicates we can replay it on the same version.
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+    test //pkg/sql/schemachanger/corpus:corpus_test -- \
+    --test_arg=--declarative-corpus=$ARTIFACTS_DIR/corpus-mixed \
+    --test_filter='^TestValidateCorpuses$' \
+    --test_env=GO_TEST_WRAP_TESTV=1 \
+    --test_env=GO_TEST_WRAP=1 \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_VALIDATE_JSON_OUTPUT_FILE_MIXED \
+    --test_timeout=7200 \
+    || exit_status=$?
+
+process_test_json \
+  $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+  $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+  $ARTIFACTS_DIR \
+  $GO_TEST_VALIDATE_JSON_OUTPUT_FILE_MIXED \
+  $exit_status
+
+# If validation passes its safe to update the copy in storage.
+if [ $exit_status = 0 ]; then
+  gsutil cp  $ARTIFACTS_DIR/corpus-mixed/* gs://cockroach-corpus/corpus-mixed-$TC_BUILD_BRANCH/
+fi

pkg/BUILD.bazel

@@ -429,6 +429,7 @@ ALL_TESTS = [
     "//pkg/sql/scanner:scanner_test",
     "//pkg/sql/scheduledlogging:scheduledlogging_test",
     "//pkg/sql/schemachange:schemachange_test",
+    "//pkg/sql/schemachanger/corpus:corpus_test",
     "//pkg/sql/schemachanger/rel:rel_test",
     "//pkg/sql/schemachanger/scbuild:scbuild_test",
     "//pkg/sql/schemachanger/scdecomp:scdecomp_test",
@@ -1602,6 +1603,7 @@ GO_TARGETS = [
     "//pkg/sql/schemachange:schemachange",
     "//pkg/sql/schemachange:schemachange_test",
     "//pkg/sql/schemachanger/corpus:corpus",
+    "//pkg/sql/schemachanger/corpus:corpus_test",
     "//pkg/sql/schemachanger/rel/internal/comparetest:comparetest",
     "//pkg/sql/schemachanger/rel/internal/cyclegraphtest:cyclegraphtest",
     "//pkg/sql/schemachanger/rel/internal/entitynodetest:entitynodetest",

pkg/sql/logictest/testdata/logic_test/alter_table

@@ -1,3 +1,4 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
 
 statement ok
 CREATE TABLE other (b INT PRIMARY KEY)
@@ -2066,6 +2067,7 @@ CREATE TABLE t (id INT NOT NULL)
 statement ok
 ALTER TABLE t ADD CONSTRAINT t_pkey PRIMARY KEY (id)
 
+skipif config local-mixed-22.1-22.2
 query TT
 SHOW CREATE TABLE t
 ----

pkg/sql/logictest/testdata/logic_test/create_index

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement ok
 CREATE TABLE t (
   a INT PRIMARY KEY,

pkg/sql/logictest/testdata/logic_test/drop_database

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement ok
 SET CLUSTER SETTING sql.cross_db_views.enabled = TRUE
 
@@ -39,6 +41,7 @@ system     node  NULL  NULL  {}  NULL
 test       root  NULL  NULL  {}  NULL
 
 skipif config local-legacy-schema-changer
+skipif config local-mixed-22.1-22.2
 query TT
 SELECT job_type, status FROM [SHOW JOBS] WHERE user_name = 'root'
 ----

pkg/sql/logictest/testdata/logic_test/drop_index

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement ok
 CREATE TABLE users (
   id    INT PRIMARY KEY,

pkg/sql/logictest/testdata/logic_test/drop_owned_by

@@ -1,5 +1,3 @@
-# LogicTest: !local-legacy-schema-changer
-
 # Test dropping nothing.
 statement ok
 DROP OWNED BY testuser

pkg/sql/logictest/testdata/logic_test/drop_schema

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement error pq: cannot drop schema "public"
 DROP SCHEMA public
 

pkg/sql/logictest/testdata/logic_test/drop_sequence

@@ -1,3 +1,6 @@
+
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 # see also file `sequences`
 
 statement ok

pkg/sql/logictest/testdata/logic_test/drop_table

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement ok
 CREATE TABLE a (id INT PRIMARY KEY)
 
@@ -30,6 +32,7 @@ DROP TABLE a
 # run by an unrelated startup migration.
 # TODO (lucy): Update this if/when we decide to change how these jobs queued by
 # the startup migration are handled.
+skipif config local-mixed-22.1-22.2
 query TT
 SELECT replace(job_type, 'NEW SCHEMA CHANGE', 'SCHEMA CHANGE'), status
   FROM [SHOW JOBS]

pkg/sql/logictest/testdata/logic_test/drop_type

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement ok
 SET sql_safe_updates = false;
 SET enable_experimental_alter_column_type_general = true;

pkg/sql/logictest/testdata/logic_test/drop_view

@@ -1,3 +1,5 @@
+# LogicTest: default-configs local-mixed-22.1-22.2
+
 statement ok
 CREATE TABLE a (k STRING PRIMARY KEY, v STRING)
 

pkg/sql/logictest/tests/local-mixed-22.1-22.2/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
         "//c-deps:libgeos",  # keep
         "//pkg/sql/logictest:testdata",  # keep
     ],
-    shard_count = 2,
+    shard_count = 11,
     tags = ["cpu:1"],
     deps = [
         "//pkg/build/bazel",

pkg/sql/schemachanger/corpus/BUILD.bazel

@@ -1,10 +1,11 @@
 load("//build/bazelutil/unused_checker:unused.bzl", "get_x_data")
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "corpus",
     srcs = ["corpus.go"],
     importpath = "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/corpus",
+    tags = ["integration"],
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/sql/schemachanger/scop",
@@ -18,4 +19,17 @@ go_library(
     ],
 )
 
+go_test(
+    name = "corpus_test",
+    srcs = ["corpus_test.go"],
+    deps = [
+        ":corpus",
+        "//pkg/jobs/jobspb",
+        "//pkg/sql/schemachanger/scop",
+        "//pkg/sql/schemachanger/scplan",
+        "//pkg/testutils/skip",
+        "@com_github_stretchr_testify//require",
+    ],
+)
+
 get_x_data(name = "get_x_data")