wip: checking ci with bootstrap=script by default

.bazelci/presubmit.yml

@@ -26,7 +26,7 @@ buildifier:
 .reusable_config: &reusable_config
   build_targets:
     - "--"
-    - "..."
+    - "//tests/base_rules:all"
     # As a regression test for #225, check that wheel targets still build when
     # their package path is qualified with the repo name.
     - "@rules_python//examples/wheel/..."
@@ -35,7 +35,7 @@ buildifier:
     - "--build_tag_filters=-integration-test"
   test_targets:
     - "--"
-    - "..."
+    - "//tests/base_rules:all"
   test_flags:
     - "--test_tag_filters=-integration-test"
 .common_workspace_flags: &common_workspace_flags

python/private/stage1_bootstrap_template.sh

@@ -16,7 +16,9 @@ PYTHON_BINARY='%python_binary%'
 IS_ZIPFILE="%is_zipfile%"
 
 if [[ "$IS_ZIPFILE" == "1" ]]; then
-  zip_dir=$(mktemp -d --suffix Bazel.runfiles_)
+  # NOTE: Macs have an old version of mktemp, so we must use only the
+  # minimal functionality of it.
+  zip_dir=$(mktemp -d)
 
   if [[ -n "$zip_dir" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then
     trap 'rm -fr "$zip_dir"' EXIT
@@ -27,7 +29,7 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then
   # The alternative requires having to copy ourselves elsewhere with the prelude
   # stripped (because zip can't extract from a stream). We avoid that because
   # it's wasteful.
-  ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || /bin/true )
+  ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || true )
 
   RUNFILES_DIR="$zip_dir/runfiles"
   if [[ ! -d "$RUNFILES_DIR" ]]; then
@@ -105,6 +107,11 @@ declare -a interpreter_args
 # NOTE: Only works for 3.11+
 interpreter_env+=("PYTHONSAFEPATH=1")
 
+if [[ "$IS_ZIPFILE" == "1" ]]; then
+  interpreter_args+=("-XRULES_PYTHON_ZIP_DIR=$zip_dir")
+fi
+
+
 export RUNFILES_DIR
 
 command=(

tests/base_rules/BUILD.bazel

@@ -11,3 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+load("//tests/support:sh_py_run_test.bzl", "sh_py_run_test")
+
+sh_py_run_test(
+    name = "run_binary_zip_off_test",
+    build_python_zip = "no",
+    py_src = "bin.py",
+    sh_src = "run_binary_zip_off_test.sh",
+)
+
+sh_py_run_test(
+    name = "run_binary_zip_on_test",
+    build_python_zip = "yes",
+    py_src = "bin.py",
+    sh_src = "run_binary_zip_on_test.sh",
+)

tests/base_rules/bin.py

@@ -0,0 +1,21 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+
+print("Hello")
+print(
+    "RULES_PYTHON_ZIP_DIR:{}".format(sys._xoptions.get("RULES_PYTHON_ZIP_DIR", "UNSET"))
+)
+print("file:", __file__)

tests/base_rules/run_binary_zip_off_test.sh

@@ -0,0 +1,43 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+set +e
+
+bin=$(rlocation $BIN_RLOCATION)
+if [[ -z "$bin" ]]; then
+  echo "Unable to locate test binary: $BIN_RLOCATION"
+  exit 1
+fi
+actual=$($bin 2>&1)
+
+# How we detect if a zip file was executed from depends on which bootstrap
+# is used.
+# bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR=<somepath>
+# bootstrap_impl=system_python outputs file:.*Bazel.runfiles
+expected_pattern="Hello"
+if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then
+  echo "expected output to match: $expected_pattern"
+  echo "but got:\n$actual"
+  exit 1
+fi

tests/base_rules/run_binary_zip_on_test.sh

@@ -0,0 +1,44 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+set +e
+
+bin=$(rlocation $BIN_RLOCATION)
+if [[ -z "$bin" ]]; then
+  echo "Unable to locate test binary: $BIN_RLOCATION"
+  exit 1
+fi
+actual=$($bin)
+
+# How we detect if a zip file was executed from depends on which bootstrap
+# is used.
+# bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR:<somepath>
+# bootstrap_impl=system_python outputs file:.*Bazel.runfiles
+expected_pattern="RULES_PYTHON_ZIP_DIR:/\|file:.*Bazel.runfiles"
+if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then
+  echo "expected output to match: $expected_pattern"
+  echo "but got: $actual"
+  exit 1
+fi
+

tests/base_rules/run_zip_test.sh

@@ -0,0 +1,38 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+set +e
+
+bin=$(rlocation _main/tests/base_rules/_run_zip_test_bin)
+if [[ -z "$bin" ]]; then
+  echo "Unable to locate test binary"
+  exit 1
+fi
+actual=$($bin)
+
+if [[ ! "$actual" == RULES_PYTHON_ZIP_DIR=/* ]]; then
+  echo "expected output: RULES_PYTHON_ZIP_DIR=<some path>"
+  echo "but got: $actual"
+  exit 1
+fi

tests/support/sh_py_run_test.bzl

@@ -0,0 +1,113 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Run a py_binary with altered config settings in an sh_test.
+
+This facilitates verify running binaries with different configuration settings
+without the overhead of a bazel-in-bazel integration test.
+"""
+
+load("//python:py_binary.bzl", "py_binary")
+
+def _perform_transition_impl(_input_settings, attr):
+    settings = {
+        "//command_line_option:build_python_zip": attr.build_python_zip,
+    }
+    return settings
+
+_perform_transition = transition(
+    implementation = _perform_transition_impl,
+    inputs = [],
+    outputs = [
+        "//command_line_option:build_python_zip",
+    ],
+)
+
+def _transition_impl(ctx):
+    default_info = ctx.attr.target[DefaultInfo]
+    exe_ext = default_info.files_to_run.executable.extension
+    if exe_ext:
+        exe_ext = "." + exe_ext
+    exe_name = ctx.label.name + exe_ext
+
+    executable = ctx.actions.declare_file(exe_name)
+    ctx.actions.symlink(output = executable, target_file = default_info.files_to_run.executable)
+
+    default_outputs = [executable]
+
+    # todo: could probably check target.owner vs src.owner to check if it should
+    # be symlinked or included as-is
+    # For simplicity of implementation, we're assuming the target being run is
+    # py_binary-like. In order for Windows to work, we need to make sure the
+    # file that the .exe launcher runs (the .zip or underlying non-exe
+    # executable) is a sibling of the .exe file with the same base name.
+    for src in default_info.files.to_list():
+        if src.extension in ("", "zip"):
+            ext = ("." if src.extension else "") + src.extension
+            output = ctx.actions.declare_file(ctx.label.name + ext)
+            ctx.actions.symlink(output = output, target_file = src)
+            default_outputs.append(output)
+
+    return [
+        DefaultInfo(
+            executable = executable,
+            files = depset(default_outputs),
+            runfiles = default_info.default_runfiles,
+        ),
+        testing.TestEnvironment(
+            environment = ctx.attr.env,
+        ),
+    ]
+
+transition_binary = rule(
+    implementation = _transition_impl,
+    attrs = {
+        "bootstrap_impl": attr.string(),
+        "build_python_zip": attr.string(default = "auto"),
+        "env": attr.string_dict(),
+        "target": attr.label(executable = True, cfg = "target"),
+        "_allowlist_function_transition": attr.label(
+            default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
+        ),
+    },
+    cfg = _perform_transition,
+    executable = True,
+)
+
+def sh_py_run_test(*, name, sh_src, py_src, **kwargs):
+    bin_name = "_{}_bin".format(name)
+    native.sh_test(
+        name = name,
+        srcs = [sh_src],
+        data = [bin_name],
+        deps = [
+            "@bazel_tools//tools/bash/runfiles",
+        ],
+        env = {
+            "BIN_RLOCATION": "$(rlocationpath {})".format(bin_name),
+        },
+    )
+
+    transition_binary(
+        name = bin_name,
+        tags = ["manual"],
+        target = "_{}_plain_bin".format(name),
+        **kwargs
+    )
+
+    py_binary(
+        name = "_{}_plain_bin".format(name),
+        srcs = [py_src],
+        main = py_src,
+        tags = ["manual"],
+    )