From 866d66c4d5dad50b0801644f4f87fd10b5746002 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 9 Jul 2024 19:40:23 -0700 Subject: [PATCH] wip: checking ci with bootstrap=script by default --- CHANGELOG.md | 5 +- python/private/stage1_bootstrap_template.sh | 11 +- tests/base_rules/BUILD.bazel | 40 +++++++ tests/base_rules/bin.py | 21 ++++ tests/base_rules/run_binary_zip_no_test.sh | 43 +++++++ tests/base_rules/run_binary_zip_yes_test.sh | 44 ++++++++ tests/base_rules/run_zip_test.sh | 38 +++++++ tests/support/sh_py_run_test.bzl | 117 ++++++++++++++++++++ 8 files changed, 316 insertions(+), 3 deletions(-) create mode 100644 tests/base_rules/bin.py create mode 100755 tests/base_rules/run_binary_zip_no_test.sh create mode 100755 tests/base_rules/run_binary_zip_yes_test.sh create mode 100755 tests/base_rules/run_zip_test.sh create mode 100644 tests/support/sh_py_run_test.bzl diff --git a/CHANGELOG.md b/CHANGELOG.md index cc44a47be0..d924f650ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,10 @@ A brief description of the categories of changes: * (rules) Signals are properly received when using {obj}`--bootstrap_impl=script` (for non-zip builds). ([#2043](https://github.com/bazelbuild/rules_python/issues/2043)) -* (rules) Fixes python builds when the `--build_python_zip` is set to `false` on Windows. See [#1840](https://github.com/bazelbuild/rules_python/issues/1840). +* (rules) Fixes Python builds when the `--build_python_zip` is set to `false` on + Windows. See [#1840](https://github.com/bazelbuild/rules_python/issues/1840). +* (rules) Fixes Mac + `--build_python_zip` + {obj}`--bootstrap_impl=script` + ([#2030](https://github.com/bazelbuild/rules_python/issues/2030)). * (pip) Fixed pypi parse_simpleapi_html function for feeds with package metadata containing ">" sign diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index 48711aa92f..46e33b4837 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -16,7 +16,9 @@ PYTHON_BINARY='%python_binary%' IS_ZIPFILE="%is_zipfile%" if [[ "$IS_ZIPFILE" == "1" ]]; then - zip_dir=$(mktemp -d --suffix Bazel.runfiles_) + # NOTE: Macs have an old version of mktemp, so we must use only the + # minimal functionality of it. + zip_dir=$(mktemp -d) if [[ -n "$zip_dir" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then trap 'rm -fr "$zip_dir"' EXIT @@ -27,7 +29,7 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then # The alternative requires having to copy ourselves elsewhere with the prelude # stripped (because zip can't extract from a stream). We avoid that because # it's wasteful. - ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || /bin/true ) + ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || true ) RUNFILES_DIR="$zip_dir/runfiles" if [[ ! -d "$RUNFILES_DIR" ]]; then @@ -105,6 +107,11 @@ declare -a interpreter_args # NOTE: Only works for 3.11+ interpreter_env+=("PYTHONSAFEPATH=1") +if [[ "$IS_ZIPFILE" == "1" ]]; then + interpreter_args+=("-XRULES_PYTHON_ZIP_DIR=$zip_dir") +fi + + export RUNFILES_DIR command=( diff --git a/tests/base_rules/BUILD.bazel b/tests/base_rules/BUILD.bazel index aa21042e25..62d73ac88f 100644 --- a/tests/base_rules/BUILD.bazel +++ b/tests/base_rules/BUILD.bazel @@ -11,3 +11,43 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility +load("//tests/support:sh_py_run_test.bzl", "sh_py_run_test") + +_SUPPORTS_BOOTSTRAP_SCRIPT = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], +}) if IS_BAZEL_7_OR_HIGHER else ["@platforms//:incompatible"] + +sh_py_run_test( + name = "run_binary_zip_no_test", + build_python_zip = "no", + py_src = "bin.py", + sh_src = "run_binary_zip_no_test.sh", +) + +sh_py_run_test( + name = "run_binary_zip_yes_test", + build_python_zip = "yes", + py_src = "bin.py", + sh_src = "run_binary_zip_yes_test.sh", +) + +sh_py_run_test( + name = "run_binary_bootstrap_script_zip_yes_test", + bootstrap_impl = "script", + build_python_zip = "yes", + py_src = "bin.py", + sh_src = "run_binary_zip_yes_test.sh", + target_compatible_with = _SUPPORTS_BOOTSTRAP_SCRIPT, +) + +sh_py_run_test( + name = "run_binary_bootstrap_script_zip_no_test", + bootstrap_impl = "script", + build_python_zip = "no", + py_src = "bin.py", + sh_src = "run_binary_zip_no_test.sh", + target_compatible_with = _SUPPORTS_BOOTSTRAP_SCRIPT, +) diff --git a/tests/base_rules/bin.py b/tests/base_rules/bin.py new file mode 100644 index 0000000000..cffb79ba19 --- /dev/null +++ b/tests/base_rules/bin.py @@ -0,0 +1,21 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys + +print("Hello") +print( + "RULES_PYTHON_ZIP_DIR:{}".format(sys._xoptions.get("RULES_PYTHON_ZIP_DIR", "UNSET")) +) +print("file:", __file__) diff --git a/tests/base_rules/run_binary_zip_no_test.sh b/tests/base_rules/run_binary_zip_no_test.sh new file mode 100755 index 0000000000..2ee69f3f66 --- /dev/null +++ b/tests/base_rules/run_binary_zip_no_test.sh @@ -0,0 +1,43 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- +set +e + +bin=$(rlocation $BIN_RLOCATION) +if [[ -z "$bin" ]]; then + echo "Unable to locate test binary: $BIN_RLOCATION" + exit 1 +fi +actual=$($bin 2>&1) + +# How we detect if a zip file was executed from depends on which bootstrap +# is used. +# bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR= +# bootstrap_impl=system_python outputs file:.*Bazel.runfiles +expected_pattern="Hello" +if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then + echo "expected output to match: $expected_pattern" + echo "but got:\n$actual" + exit 1 +fi diff --git a/tests/base_rules/run_binary_zip_yes_test.sh b/tests/base_rules/run_binary_zip_yes_test.sh new file mode 100755 index 0000000000..ca278083dd --- /dev/null +++ b/tests/base_rules/run_binary_zip_yes_test.sh @@ -0,0 +1,44 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- +set +e + +bin=$(rlocation $BIN_RLOCATION) +if [[ -z "$bin" ]]; then + echo "Unable to locate test binary: $BIN_RLOCATION" + exit 1 +fi +actual=$($bin) + +# How we detect if a zip file was executed from depends on which bootstrap +# is used. +# bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR: +# bootstrap_impl=system_python outputs file:.*Bazel.runfiles +expected_pattern="RULES_PYTHON_ZIP_DIR:/\|file:.*Bazel.runfiles" +if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then + echo "expected output to match: $expected_pattern" + echo "but got: $actual" + exit 1 +fi + diff --git a/tests/base_rules/run_zip_test.sh b/tests/base_rules/run_zip_test.sh new file mode 100755 index 0000000000..64857e6490 --- /dev/null +++ b/tests/base_rules/run_zip_test.sh @@ -0,0 +1,38 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- +set +e + +bin=$(rlocation _main/tests/base_rules/_run_zip_test_bin) +if [[ -z "$bin" ]]; then + echo "Unable to locate test binary" + exit 1 +fi +actual=$($bin) + +if [[ ! "$actual" == RULES_PYTHON_ZIP_DIR=/* ]]; then + echo "expected output: RULES_PYTHON_ZIP_DIR=" + echo "but got: $actual" + exit 1 +fi diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl new file mode 100644 index 0000000000..4b3d22d5bb --- /dev/null +++ b/tests/support/sh_py_run_test.bzl @@ -0,0 +1,117 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run a py_binary with altered config settings in an sh_test. + +This facilitates verify running binaries with different configuration settings +without the overhead of a bazel-in-bazel integration test. +""" + +load("//python:py_binary.bzl", "py_binary") + +def _perform_transition_impl(input_settings, attr): + settings = dict(input_settings) + settings["//command_line_option:build_python_zip"] = attr.build_python_zip + if attr.bootstrap_impl: + settings["//python/config_settings:bootstrap_impl"] = attr.bootstrap_impl + return settings + +_perform_transition = transition( + implementation = _perform_transition_impl, + inputs = [ + "//python/config_settings:bootstrap_impl", + ], + outputs = [ + "//command_line_option:build_python_zip", + "//python/config_settings:bootstrap_impl", + ], +) + +def _transition_impl(ctx): + default_info = ctx.attr.target[DefaultInfo] + exe_ext = default_info.files_to_run.executable.extension + if exe_ext: + exe_ext = "." + exe_ext + exe_name = ctx.label.name + exe_ext + + executable = ctx.actions.declare_file(exe_name) + ctx.actions.symlink(output = executable, target_file = default_info.files_to_run.executable) + + default_outputs = [executable] + + # todo: could probably check target.owner vs src.owner to check if it should + # be symlinked or included as-is + # For simplicity of implementation, we're assuming the target being run is + # py_binary-like. In order for Windows to work, we need to make sure the + # file that the .exe launcher runs (the .zip or underlying non-exe + # executable) is a sibling of the .exe file with the same base name. + for src in default_info.files.to_list(): + if src.extension in ("", "zip"): + ext = ("." if src.extension else "") + src.extension + output = ctx.actions.declare_file(ctx.label.name + ext) + ctx.actions.symlink(output = output, target_file = src) + default_outputs.append(output) + + return [ + DefaultInfo( + executable = executable, + files = depset(default_outputs), + runfiles = default_info.default_runfiles, + ), + testing.TestEnvironment( + environment = ctx.attr.env, + ), + ] + +transition_binary = rule( + implementation = _transition_impl, + attrs = { + "bootstrap_impl": attr.string(), + "build_python_zip": attr.string(default = "auto"), + "env": attr.string_dict(), + "target": attr.label(executable = True, cfg = "target"), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, + cfg = _perform_transition, + executable = True, +) + +def sh_py_run_test(*, name, sh_src, py_src, **kwargs): + bin_name = "_{}_bin".format(name) + native.sh_test( + name = name, + srcs = [sh_src], + data = [bin_name], + deps = [ + "@bazel_tools//tools/bash/runfiles", + ], + env = { + "BIN_RLOCATION": "$(rlocationpath {})".format(bin_name), + }, + ) + + transition_binary( + name = bin_name, + tags = ["manual"], + target = "_{}_plain_bin".format(name), + **kwargs + ) + + py_binary( + name = "_{}_plain_bin".format(name), + srcs = [py_src], + main = py_src, + tags = ["manual"], + )