Merge #75585

75585: ci: update pebble metamorphic nightly to use bazel r=nicktrav a=rickystewart

Closes #67335.

Release note: None

Co-authored-by: Ricky Stewart <[email protected]>

DEPS.bzl

@@ -1216,8 +1216,13 @@ def go_deps():
     )
     go_repository(
         name = "com_github_cockroachdb_pebble",
+        build_directives = ["gazelle:build_tags invariants"],
         build_file_proto_mode = "disable_global",
         importpath = "github.com/cockroachdb/pebble",
+        patch_args = ["-p1"],
+        patches = [
+            "@cockroach//build/patches:com_github_cockroachdb_pebble.patch",
+        ],
         sha256 = "0018bcef357bf7bba06d5e3eb35277709b5fd98ee437924001531fa935d8c76d",
         strip_prefix = "github.com/cockroachdb/[email protected]",
         urls = [

build/bazelutil/bazel-generate.sh

@@ -42,7 +42,7 @@ files_unchanged_from_upstream () {
 # be blocked by the existence of a file before the bazel command is
 # invoked. For now, this is left as an exercise for the user.
 
-if files_unchanged_from_upstream go.mod go.sum DEPS.bzl $(find ./pkg/cmd/generate-staticcheck -name BUILD.bazel -or -name '*.go'); then
+if files_unchanged_from_upstream go.mod go.sum DEPS.bzl $(find ./pkg/cmd/mirror -name BUILD.bazel -or -name '*.go') $(find ./pkg/cmd/generate-staticcheck -name BUILD.bazel -or -name '*.go') $(find ./build/patches -name '*.patch'); then
   echo "Skipping //pkg/cmd/mirror (relevant files are unchanged from upstream)."
   echo "Skipping //pkg/cmd/generate-staticcheck (relevant files are unchanged from upstream)."
 else

build/patches/README.md

@@ -10,6 +10,10 @@ Generally speaking, the patches here "fix" `BUILD` files, which were either
 originally present in the repo, or which will be generated by Gazelle. A couple
 examples:
 
+* `com_github_cockroachdb_pebble.patch` adds the file
+  `internal/invariants/off.go` to the `go_library` for the `invariants`
+  package -- otherwise, Gazelle omits it, causing builds to fail unless you pass
+  the `invariants` build tag.
 * `com_github_gogo_protobuf.patch` adds a rule, `gogo_proto`, so that we can
   import `gogoproto/gogo.proto` in our own `.proto` files, while not updating
   the existing `gogoproto` rule that uses the pre-generated `.pb.go` file.

build/patches/com_github_cockroachdb_pebble.patch

@@ -0,0 +1,11 @@
+diff -urN a/internal/invariants/BUILD.bazel b/internal/invariants/BUILD.bazel
+--- a/internal/invariants/BUILD.bazel	1969-12-31 19:00:00.000000000 -0500
++++ b/internal/invariants/BUILD.bazel	2000-01-01 00:00:00.000000000 -0000
+@@ -5,6 +5,7 @@
+     srcs = [
+         "finalizer_off.go",
+         "finalizer_on.go",
++        "off.go",
+         "on.go",
+         "race_off.go",
+         "race_on.go",

build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# This script is run by the Pebble Nightly Metamorphic - TeamCity build
+# configuration.
+
+set -euo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+mkdir -p artifacts
+
+# Pull in the latest version of Pebble from upstream. The benchmarks run
+# against the tip of the 'master' branch. We do this by `go get`ting the
+# latest version of the module, and then running `mirror` to update `DEPS.bzl`
+# accordingly.
+bazel run @go_sdk//:bin/go get github.com/cockroachdb/pebble@latest
+NEW_DEPS_BZL_CONTENT=$(bazel run //pkg/cmd/mirror)
+echo "$NEW_DEPS_BZL_CONTENT" > DEPS.bzl
+
+PEBBLE_SUM=$(grep 'version =' DEPS.bzl | cut -d'"' -f2)
+echo "Pebble module sum: $PEBBLE_SUM"
+
+env=(
+  "GITHUB_REPO=$GITHUB_REPO"
+  "GITHUB_API_TOKEN=$GITHUB_API_TOKEN"
+  "BUILD_VCS_NUMBER=$PEBBLE_SUM"
+  "TC_BUILD_ID=$TC_BUILD_ID"
+  "TC_SERVER_URL=$TC_SERVER_URL"
+  "TC_BUILD_BRANCH=$TC_BUILD_BRANCH"
+  "PKG=internal/metamorphic"
+  "STRESSFLAGS=-maxtime 3h -maxfails 1 -stderr -p 1"
+  "TZ=America/New_York"
+)
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e BUILD_VCS_NUMBER=$PEBBLE_SUM -e GITHUB_API_TOKEN -e GITHUB_REPO -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL" \
+                               run_bazel build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh
+

build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"  # For process_test_json
+
+set -euxo pipefail
+ARTIFACTS_DIR=/artifacts/meta
+mkdir -p $ARTIFACTS_DIR
+GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
+
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+
+exit_status=0
+# NB: If adjusting the metamorphic test flags below, be sure to also update
+# pkg/cmd/github-post/main.go to ensure the GitHub issue poster includes the
+# correct flags in the reproduction command.
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci test \
+                                      @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test -- \
+                                      --test_timeout=11000 '--test_filter=TestMeta$' \
+                                      --define gotags=bazel,invariants \
+                                      "--test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE" \
+                                      --run_under "@com_github_cockroachdb_stress//:stress -bazel -shardable-artifacts 'GO_TEST_JSON_OUTPUT_FILE=cat,XML_OUTPUT_FILE=$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci merge-test-xmls' -maxtime 3h -maxfails 1 -stderr -p 1" \
+                                      --test_arg -dir --test_arg $ARTIFACTS_DIR \
+                                      --test_arg -ops --test_arg "uniform:5000-25000" \
+    || exit_status=$?
+
+process_test_json \
+    $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+    $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+    /artifacts $GO_TEST_JSON_OUTPUT_FILE $exit_status
+
+exit $exit_status

pkg/cmd/github-post/main.go

@@ -541,8 +541,9 @@ func formatPebbleMetamorphicIssue(
 			s = strings.TrimSpace(s)
 			s = strings.TrimSpace(s[:strings.Index(s, "\n")])
 
-			repro = fmt.Sprintf("go test -mod=vendor -tags 'invariants' -exec 'stress -p 1' "+
-				`-timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed %s -ops "uniform:5000-25000"`, s)
+			repro = fmt.Sprintf("dev test @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test "+
+				"-f TestMeta --stress --stress-args='-p 1' -- --define gotags=bazel,invariants --test_timeout=11000 "+
+				`--test_arg -seed --test_arg %s --test_arg -ops --test_arg "uniform:5000-25000"`, s)
 		}
 	}
 	return issues.UnitTestFormatter, issues.PostRequest{

pkg/cmd/github-post/main_test.go

@@ -235,7 +235,7 @@ TestXXA - 1.00s
 					testName: "TestMeta",
 					title:    "internal/metamorphic: TestMeta failed",
 					message:  "panic: induced panic",
-					expRepro: `go test -mod=vendor -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed 1600209371838097000 -ops "uniform:5000-25000"`,
+					expRepro: `dev test @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test -f TestMeta --stress --stress-args='-p 1' -- --define gotags=bazel,invariants --test_timeout=11000 --test_arg -seed --test_arg 1600209371838097000 --test_arg -ops --test_arg "uniform:5000-25000"`,
 				},
 			},
 			formatter: formatPebbleMetamorphicIssue,

pkg/cmd/mirror/mirror.go

@@ -386,6 +386,13 @@ func buildFileProtoModeForRepo(repoName string) string {
 	return "disable_global"
 }
 
+func dumpBuildDirectivesForRepo(repoName string) {
+	if repoName == "com_github_cockroachdb_pebble" {
+		fmt.Printf(`        build_directives = ["gazelle:build_tags invariants"],
+`)
+	}
+}
+
 func dumpBuildNamingConventionArgsForRepo(repoName string) {
 	if repoName == "com_github_envoyproxy_protoc_gen_validate" || repoName == "com_github_grpc_ecosystem_grpc_gateway" {
 		fmt.Printf("        build_naming_convention = \"go_default_library\",\n")
@@ -448,9 +455,10 @@ def go_deps():
 		}
 		fmt.Printf(`    go_repository(
         name = "%s",
-        build_file_proto_mode = "%s",
-`, repoName, buildFileProtoModeForRepo(repoName))
-
+`, repoName)
+		dumpBuildDirectivesForRepo(repoName)
+		fmt.Printf(`        build_file_proto_mode = "%s",
+`, buildFileProtoModeForRepo(repoName))
 		dumpBuildNamingConventionArgsForRepo(repoName)
 		expectedURL := formatURL(replaced.Path, replaced.Version)
 		fmt.Printf("        importpath = \"%s\",\n", mod.Path)