From 11631747860c898a08e9be3043bb13bfaf3441c5 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Fri, 26 Jul 2024 19:42:14 +1000 Subject: [PATCH] Allow unesecure node temporarily for manylinux container builds (Cherry-pick of #21172) (#21212) Renders github actions and env settings to continue to use node16 inside the containers that match the glibc (2.17) of manylinux_2014 builder images. The new default node version for gha requires a much higher version. I checked if https://github.com/nodejs/unofficial-builds had node binaries we could use. It does not. Based on https://github.com/pantsbuild/pants/pull/21133 Co-authored-by: Tobias Nilsson --- .github/workflows/release.yaml | 14 ++- .github/workflows/test.yaml | 14 ++- .../generate_github_workflows.py | 106 +++++++++++------- 3 files changed, 84 insertions(+), 50 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 04f3adf73a2..1d21061fa9c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,6 +8,7 @@ jobs: container: image: ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: github.repository_owner == 'pantsbuild' @@ -20,7 +21,7 @@ jobs: - ARM64 steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 10 ref: ${{ needs.release_info.outputs.build-ref }} @@ -57,7 +58,7 @@ jobs: - continue-on-error: true if: always() name: Upload pants.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: logs-wheels-and-pex-Linux-ARM64 overwrite: 'true' @@ -84,6 +85,7 @@ jobs: container: image: quay.io/pypa/manylinux2014_x86_64:latest env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: github.repository_owner == 'pantsbuild' @@ -94,7 +96,7 @@ jobs: - ubuntu-20.04 steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 10 ref: ${{ needs.release_info.outputs.build-ref }} @@ -121,7 +123,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} version: 23.x - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v4 with: go-version: 1.19.5 - env: @@ -135,7 +137,7 @@ jobs: - continue-on-error: true if: always() name: Upload pants.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: logs-wheels-and-pex-Linux-x86_64 overwrite: 'true' @@ -167,6 +169,7 @@ jobs: timeout-minutes: 90 build_wheels_macos10_15_x86_64: env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: github.repository_owner == 'pantsbuild' @@ -252,6 +255,7 @@ jobs: timeout-minutes: 90 build_wheels_macos11_arm64: env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: github.repository_owner == 'pantsbuild' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 794851402dc..5fc58d10687 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -301,6 +301,7 @@ jobs: container: image: ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' @@ -315,7 +316,7 @@ jobs: - ARM64 steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 10 - name: Configure Git @@ -350,7 +351,7 @@ jobs: - continue-on-error: true if: always() name: Upload pants.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: logs-wheels-and-pex-Linux-ARM64 overwrite: 'true' @@ -360,6 +361,7 @@ jobs: container: image: quay.io/pypa/manylinux2014_x86_64:latest env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' @@ -372,7 +374,7 @@ jobs: - ubuntu-20.04 steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 10 - name: Configure Git @@ -397,7 +399,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} version: 23.x - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v4 with: go-version: 1.19.5 - env: @@ -411,7 +413,7 @@ jobs: - continue-on-error: true if: always() name: Upload pants.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: logs-wheels-and-pex-Linux-x86_64 overwrite: 'true' @@ -419,6 +421,7 @@ jobs: timeout-minutes: 90 build_wheels_macos10_15_x86_64: env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' @@ -487,6 +490,7 @@ jobs: timeout-minutes: 90 build_wheels_macos11_arm64: env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' diff --git a/src/python/pants_release/generate_github_workflows.py b/src/python/pants_release/generate_github_workflows.py index e19a518e2df..a26be037e1e 100644 --- a/src/python/pants_release/generate_github_workflows.py +++ b/src/python/pants_release/generate_github_workflows.py @@ -19,22 +19,45 @@ from pants.util.strutil import softwrap -ACTION = { - "action-send-mail": "dawidd6/action-send-mail@v3.8.0", - "cache": "actions/cache@v4", - "checkout": "actions/checkout@v4", - "download-artifact": "actions/download-artifact@v4", - "expose-pythons": "pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe", - "github-action-required-labels": "mheap/github-action-required-labels@v4.0.0", - "rust-cache": "benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1", - "setup-go": "actions/setup-go@v5", - "setup-java": "actions/setup-java@v4", - "setup-node": "actions/setup-node@v4", - "setup-protoc": "arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9", - "setup-python": "actions/setup-python@v5", - "slack-github-action": "slackapi/slack-github-action@v1.24.0", - "upload-artifact": "actions/upload-artifact@v4", -} + +def action(name: str, node16_compat: bool = False) -> str: + # Versions of actions compatible with node16 and the `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION` setting. + # glibc 2.17 is required to build manylinux_2014 wheels, but node.js does do not ship glibc 2.17 compatible + # binaries for node >= v17. + if node16_compat: + version_map = { + "checkout": "actions/checkout@v3", + "upload-artifact": "actions/upload-artifact@v3", + "setup-go": "actions/setup-go@v4", + } + else: + version_map = { + "action-send-mail": "dawidd6/action-send-mail@v3.8.0", + "cache": "actions/cache@v4", + "checkout": "actions/checkout@v4", + "download-artifact": "actions/download-artifact@v4", + "expose-pythons": "pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe", + "github-action-required-labels": "mheap/github-action-required-labels@v4.0.0", + "rust-cache": "benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1", + "setup-go": "actions/setup-go@v5", + "setup-java": "actions/setup-java@v4", + "setup-node": "actions/setup-node@v4", + "setup-protoc": "arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9", + "setup-python": "actions/setup-python@v5", + "slack-github-action": "slackapi/slack-github-action@v1.24.0", + "upload-artifact": "actions/upload-artifact@v4", + } + try: + return version_map[name] + except KeyError as e: + configured_version = ( + "Node 16 compatible version configured" if node16_compat else "version configured" + ) + raise ValueError( + f"Requested github action '{name}' does not have a {configured_version}.\n" + f"Current known versions: {version_map}" + ) from e + HEADER = dedent( """\ @@ -169,7 +192,7 @@ def ensure_category_label() -> Sequence[Step]: { "if": "github.event_name == 'pull_request'", "name": "Ensure category label", - "uses": ACTION["github-action-required-labels"], + "uses": action("github-action-required-labels"), "env": {"GITHUB_TOKEN": gha_expr("secrets.GITHUB_TOKEN")}, "with": { "mode": "exactly", @@ -201,7 +224,7 @@ def checkout( # We need to fetch a few commits back, to be able to access HEAD^2 in the PR case. { "name": "Check out code", - "uses": ACTION["checkout"], + "uses": action("checkout", node16_compat=containerized), "with": { **fetch_depth_opt, **({"ref": ref} if ref else {}), @@ -303,7 +326,7 @@ def install_rustup() -> Step: def install_python(version: str) -> Step: return { "name": f"Set up Python {version}", - "uses": ACTION["setup-python"], + "uses": action("setup-python"), "with": {"python-version": version}, } @@ -311,7 +334,7 @@ def install_python(version: str) -> Step: def install_node(version: str) -> Step: return { "name": f"Set up Node {version}", - "uses": ACTION["setup-node"], + "uses": action("setup-node"), "with": {"node-version": version}, } @@ -319,7 +342,7 @@ def install_node(version: str) -> Step: def install_jdk() -> Step: return { "name": "Install AdoptJDK", - "uses": ACTION["setup-java"], + "uses": action("setup-java"), "with": { "distribution": "adopt", "java-version": "11", @@ -327,10 +350,10 @@ def install_jdk() -> Step: } -def install_go() -> Step: +def install_go(node16_compat: bool = False) -> Step: return { "name": "Install Go", - "uses": ACTION["setup-go"], + "uses": action("setup-go", node16_compat=node16_compat), "with": {"go-version": "1.19.5"}, } @@ -341,7 +364,7 @@ def install_go() -> Step: def install_protoc() -> Step: return { "name": "Install Protoc", - "uses": ACTION["setup-protoc"], + "uses": action("setup-protoc"), "with": { "version": "23.x", "repo-token": "${{ secrets.GITHUB_TOKEN }}", @@ -434,7 +457,7 @@ def wrap_cmd(self, cmd: str) -> str: def native_binaries_upload(self) -> Step: return { "name": "Upload native binaries", - "uses": ACTION["upload-artifact"], + "uses": action("upload-artifact"), "with": { "name": f"native_binaries.{gha_expr('matrix.python-version')}.{self.platform_name()}", "path": "\n".join(NATIVE_FILES), @@ -445,7 +468,7 @@ def native_binaries_download(self) -> Sequence[Step]: return [ { "name": "Download native binaries", - "uses": ACTION["download-artifact"], + "uses": action("download-artifact"), "with": { "name": f"native_binaries.{gha_expr('matrix.python-version')}.{self.platform_name()}", "path": NATIVE_FILES_COMMON_PREFIX, @@ -466,7 +489,7 @@ def rust_caches(self) -> Sequence[Step]: }, { "name": "Cache Rust toolchain", - "uses": ACTION["cache"], + "uses": action("cache"), "with": { "path": f"~/.rustup/toolchains/{rust_channel()}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n", "key": f"{self.platform_name()}-rustup-{hash_files('src/rust/engine/rust-toolchain')}-v2", @@ -474,7 +497,7 @@ def rust_caches(self) -> Sequence[Step]: }, { "name": "Cache Cargo", - "uses": ACTION["rust-cache"], + "uses": action("rust-cache"), "with": { # If set, replaces the job id in the cache key, so that the cache is stable across jobs. # If we don't set this, each job may restore from a previous job's cache entry (via a @@ -504,7 +527,7 @@ def bootstrap_caches(self) -> Sequence[Step]: }, { "name": "Cache native engine", - "uses": ACTION["cache"], + "uses": action("cache"), "with": { "path": "\n".join(NATIVE_FILES), "key": f"{self.platform_name()}-engine-{gha_expr('steps.get-engine-hash.outputs.hash')}-v1", @@ -528,7 +551,7 @@ def expose_all_pythons(self) -> Sequence[Step]: ret.append( { "name": "Expose Pythons", - "uses": ACTION["expose-pythons"], + "uses": action("expose-pythons"), } ) return ret @@ -562,10 +585,10 @@ def bootstrap_pants(self) -> Sequence[Step]: self.native_binaries_upload(), ] - def upload_log_artifacts(self, name: str) -> Step: + def upload_log_artifacts(self, name: str, node16_compat: bool = False) -> Step: return { "name": "Upload pants.log", - "uses": ACTION["upload-artifact"], + "uses": action("upload-artifact", node16_compat=node16_compat), "if": "always()", "continue-on-error": True, "with": { @@ -824,9 +847,7 @@ def build_wheels_job( elif platform == Platform.LINUX_ARM64: # Unfortunately Equinix do not support the CentOS 7 image on the hardware we've been # generously given by the Works on ARM program. So we have to build in this image. - container = { - "image": "ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf", - } + container = {"image": "ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf"} else: container = None @@ -868,6 +889,7 @@ def build_wheels_job( **({"needs": needs} if needs else {}), "timeout-minutes": 90, "env": { + "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION": bool(container), **DISABLE_REMOTE_CACHE_ENV, # If we're not deploying these wheels, build in debug mode, which allows for # incremental compilation across wheels. If this becomes too slow in CI, most likely @@ -878,7 +900,11 @@ def build_wheels_job( "steps": [ *initial_steps, install_protoc(), # for prost crate - *([] if platform == Platform.LINUX_ARM64 else [install_go()]), + *( + [] + if platform == Platform.LINUX_ARM64 + else [install_go(node16_compat=bool(container))] + ), { "name": "Build wheels", "run": "./pants run src/python/pants_release/release.py -- build-wheels", @@ -889,7 +915,7 @@ def build_wheels_job( "run": "./pants package src/python/pants:pants-pex", "env": helper.platform_env(), }, - helper.upload_log_artifacts(name="wheels-and-pex"), + helper.upload_log_artifacts(name="wheels-and-pex", node16_compat=bool(container)), *( [ { @@ -1193,7 +1219,7 @@ def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: "steps": [ { "name": "Checkout Pants at Release Tag", - "uses": ACTION["checkout"], + "uses": action("checkout"), "with": { # N.B.: We need the last few edits to VERSION. Instead of guessing, just # clone the repo, we're not so big as to need to optimize this. @@ -1216,7 +1242,7 @@ def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: }, { "name": "Announce to Slack", - "uses": ACTION["slack-github-action"], + "uses": action("slack-github-action"), "with": { "channel-id": "C18RRR4JK", "payload-file-path": "${{ runner.temp }}/slack_announcement.json", @@ -1225,7 +1251,7 @@ def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: }, { "name": "Announce to pants-devel", - "uses": ACTION["action-send-mail"], + "uses": action("action-send-mail"), "with": { # Note: Email is sent from the dedicated account pants.announce@gmail.com. # The EMAIL_CONNECTION_URL should be of the form: