From ad730644e0058e3ee415679de695ab670346a083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?X=C3=B9d=C5=8Dng=20Y=C3=A1ng?= Date: Mon, 12 Feb 2024 14:32:31 -0500 Subject: [PATCH 1/2] Generate a lockfile for the distribution archive on the fly (third_party changes from PR #21302) The MODULE.bazel.lock file we have in our source tree is generated by the version of Bazel specified in the .bazelversion file. We currently use this lockfile for our distribution archives, which means whenever we increment the lockfile version, we need to manually transform the checked-in lockfile to make it work with HEAD Bazel too. This commit generates a lockfile for HEAD Bazel on the fly and uses that for the distribution archive. This lockfile doesn't have any module extension content, since we only need to make sure that no network access happens during tests that use the distribution archive. See https://github.com/bazelbuild/bazel/pull/21283#issuecomment-1938398252 for more context. Change-Id: I5266bd1caf2e4cc3d7d18cb8695af73488064ae1 Signed-off-by: Xudong Yang --- third_party/googleapis/BUILD.bazel | 5 ++++- third_party/remoteapis/BUILD.bazel | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/third_party/googleapis/BUILD.bazel b/third_party/googleapis/BUILD.bazel index 0a3ee0ddc60ab2..fd9ef80b59a527 100644 --- a/third_party/googleapis/BUILD.bazel +++ b/third_party/googleapis/BUILD.bazel @@ -20,7 +20,10 @@ license( license_text = "LICENSE", ) -exports_files(["LICENSE"]) +exports_files([ + "LICENSE", + "MODULE.bazel", +]) filegroup( name = "srcs", diff --git a/third_party/remoteapis/BUILD.bazel b/third_party/remoteapis/BUILD.bazel index 44a308c764bc23..ab36e13fed69f4 100644 --- a/third_party/remoteapis/BUILD.bazel +++ b/third_party/remoteapis/BUILD.bazel @@ -5,7 +5,10 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -exports_files(["LICENSE"]) +exports_files([ + "LICENSE", + "MODULE.bazel", +]) load("@io_bazel//third_party/grpc:build_defs.bzl", "java_grpc_library") load("@io_bazel//tools/build_rules:utilities.bzl", "java_library_srcs") From c40b494be87b5b2aaa9ffa5a665b1c2e30953304 Mon Sep 17 00:00:00 2001 From: Xdng Yng Date: Tue, 13 Feb 2024 13:39:23 -0800 Subject: [PATCH 2/2] Generate a lockfile for the distribution archive on the fly The MODULE.bazel.lock file we have in our source tree is generated by the version of Bazel specified in the .bazelversion file. We currently use this lockfile for our distribution archives, which means whenever we increment the lockfile version, we need to manually transform the checked-in lockfile to make it work with HEAD Bazel too. This commit generates a lockfile for HEAD Bazel on the fly and uses that for the distribution archive. This lockfile doesn't have any module extension content, since we only need to make sure that no network access happens during tests that use the distribution archive. See https://github.com/bazelbuild/bazel/pull/21283#issuecomment-1938398252 for more context. Closes #21302. PiperOrigin-RevId: 606731749 Change-Id: If1f2b23066428b199e646059c76fa44ecfd2044a --- BUILD | 22 +++++++++++++++++++ scripts/bootstrap/compile.sh | 9 +------- src/BUILD | 1 + .../shell/bazel/bazel_determinism_test.sh | 12 ---------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/BUILD b/BUILD index 3a394a8b9828fc..ebf481121853e2 100644 --- a/BUILD +++ b/BUILD @@ -27,6 +27,7 @@ filegroup( srcs = glob( ["*"], exclude = [ + "MODULE.bazel.lock", # Use MODULE.bazel.lock.dist instead "WORKSPACE.bzlmod", # Needs to be filtered. "bazel-*", # convenience symlinks "out", # IntelliJ with setup-intellij.sh @@ -34,6 +35,7 @@ filegroup( ".*", # mainly .git* files ], ) + [ + "//:MODULE.bazel.lock.dist", "//:WORKSPACE.bzlmod.filtered", "//examples:srcs", "//scripts:srcs", @@ -96,6 +98,25 @@ genrule( ]), ) +genrule( + name = "generate_dist_lockfile", + srcs = [ + "MODULE.bazel", + "//third_party/googleapis:MODULE.bazel", + "//third_party/remoteapis:MODULE.bazel", + ], + outs = ["MODULE.bazel.lock.dist"], + cmd = "touch BUILD && " + + "trap 'rm -rf tmp_bazel_root' EXIT && " + + # Instead of `bazel mod deps`, we run a simpler command like `bazel query :all` here + # so that we only trigger module resolution, not extension eval. + # Also use `--batch` so that Bazel doesn't keep a server process alive. + "$(location //src:bazel) --batch --output_user_root=$$PWD/tmp_bazel_root query --check_direct_dependencies=error --lockfile_mode=update :all && " + + "mv MODULE.bazel.lock $@", + tags = ["requires-network"], + tools = ["//src:bazel"], +) + pkg_tar( name = "bootstrap-jars", srcs = [ @@ -151,6 +172,7 @@ pkg_tar( ], # TODO(aiuto): Replace with pkg_filegroup when that is available. remap_paths = { + "MODULE.bazel.lock.dist": "MODULE.bazel.lock", "WORKSPACE.bzlmod.filtered": "WORKSPACE.bzlmod", # Rewrite paths coming from local repositories back into third_party. "external/googleapis~override": "third_party/googleapis", diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index 468cd911113897..44aff9fbdfde98 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -250,15 +250,8 @@ if [ -z "${BAZEL_SKIP_JAVA_COMPILATION}" ]; then workspace(name = 'bazel_tools') EOF - # Set up the MODULE.bazel file for `bazel_tools` and update the hash in the lockfile. + # Set up the MODULE.bazel file for `bazel_tools` link_file "${PWD}/src/MODULE.tools" "${BAZEL_TOOLS_REPO}/MODULE.bazel" - new_hash=$(shasum -a 256 "${BAZEL_TOOLS_REPO}/MODULE.bazel" | awk '{print $1}') - sed -i.bak "/\"bazel_tools\":/s/\"[a-f0-9]*\"/\"$new_hash\"/" MODULE.bazel.lock - # TODO: Temporary hack for lockfile version mismatch, remove these lines after updating to 7.1.0 - sed -i.bak 's/"lockFileVersion": 3/"lockFileVersion": 4/' MODULE.bazel.lock - # Replace canonical repository names and parts thereof of the form rules_foo~1.2.3 with rules_foo~ - sed -i.bak -E 's/([a-z]([a-z0-9._-]*[a-z0-9]){0,1})~[a-zA-Z0-9.]{1,}(-[0-9.-]{1,}){0,1}(\+[0-9.-]{1,}){0,1}/\1/g' MODULE.bazel.lock - rm MODULE.bazel.lock.bak mkdir -p "${BAZEL_TOOLS_REPO}/src/conditions" link_file "${PWD}/src/conditions/BUILD.tools" \ diff --git a/src/BUILD b/src/BUILD index 90c19f27cc0b33..b12e41d753f307 100644 --- a/src/BUILD +++ b/src/BUILD @@ -292,6 +292,7 @@ genrule( executable = 1, output_to_bindir = 1, visibility = [ + "//:__pkg__", # For distribution archive lockfile generation "//scripts:__pkg__", # For bash completion generation "//scripts/packages:__pkg__", # For installer generation "//src/java:__subpackages__", # For command line reference generation diff --git a/src/test/shell/bazel/bazel_determinism_test.sh b/src/test/shell/bazel/bazel_determinism_test.sh index ec90b701a1adc8..a122cd238106c2 100755 --- a/src/test/shell/bazel/bazel_determinism_test.sh +++ b/src/test/shell/bazel/bazel_determinism_test.sh @@ -69,18 +69,6 @@ function test_determinism() { # Set up the maven repository properly. cp derived/maven/BUILD.vendor derived/maven/BUILD - # Update the hash of bazel_tools in lockfile to avoid rerunning module resolution. - new_hash=$(shasum -a 256 "src/MODULE.tools" | awk '{print $1}') - sed -i.bak "/\"bazel_tools\":/s/\"[a-f0-9]*\"/\"$new_hash\"/" MODULE.bazel.lock - # TODO: Temporary hack for lockfile version mismatch, remove these lines after updating to 7.1.0 - sed -i.bak 's/"lockFileVersion": 3/"lockFileVersion": 4/' MODULE.bazel.lock - # Replace canonical repository names and parts thereof of the form rules_foo~1.2.3 with rules_foo~ - sed -i.bak -E 's/([a-z]([a-z0-9._-]*[a-z0-9]){0,1})~[a-zA-Z0-9.]{1,}(-[0-9.-]{1,}){0,1}(\+[0-9.-]{1,}){0,1}/\1/g' MODULE.bazel.lock - rm MODULE.bazel.lock.bak - - # Use @bazel_tools//tools/python:autodetecting_toolchain to avoid - # downloading python toolchain. - # Build Bazel once. bazel \ --output_base="${TEST_TMPDIR}/out1" \