From 2b46bad8cbe9efe518cd7dd98a1610b22721a469 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 6 Mar 2023 15:03:34 -0500 Subject: [PATCH] OIDC beta support Co-authored-by: Sviatoslav Sydorenko --- .pre-commit-config.yaml | 4 + Dockerfile | 1 + README.md | 48 ++++++++++++ action.yml | 1 + oidc-exchange.py | 156 +++++++++++++++++++++++++++++++++++++++ requirements/runtime.in | 8 ++ requirements/runtime.txt | 12 ++- twine-upload.sh | 6 ++ 8 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 oidc-exchange.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f39d044..5e4e4223 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,11 +105,15 @@ repos: name: flake8 WPS-only args: - --ignore + # NOTE: WPS326: Found implicit string concatenation + # NOTE: WPS332: Found walrus operator - >- WPS102, WPS110, WPS111, WPS305, + WPS326, + WPS332, WPS347, WPS360, WPS421, diff --git a/Dockerfile b/Dockerfile index 241d14ec..988e5570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ WORKDIR /app COPY LICENSE.md . COPY twine-upload.sh . COPY print-hash.py . +COPY oidc-exchange.py . RUN chmod +x twine-upload.sh ENTRYPOINT ["/app/twine-upload.sh"] diff --git a/README.md b/README.md index fddd5a8a..bd6f5a4e 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,51 @@ The secret used in `${{ secrets.PYPI_API_TOKEN }}` needs to be created on the settings page of your project on GitHub. See [Creating & using secrets]. +### Publishing with OpenID Connect + +> **IMPORTANT**: This functionality is in beta, and will not work for you +> unless you're a member of the PyPI OIDC beta testers' group. For more +> information, see [warehouse#12965]. + +This action supports PyPI's [OpenID Connect publishing] +implementation, which allows authentication to PyPI without a manually +configured API token or username/password combination. To perform +[OIDC publishing][OpenID Connect Publishing] with this action, your project's +OIDC publisher must already be configured on PyPI. + +To enter the OIDC flow, configure this action's job with the `id-token: write` +permission and **without** an explicit username or password: + +```yaml +jobs: + pypi-publish: + name: Upload release to PyPI + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for OIDC publishing + steps: + # retrieve your distributions here + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 +``` + +Other indices that support OIDC publishing can also be used, like TestPyPI: + +```yaml +- name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ +``` + +> **Pro tip**: only set the `id-token: write` permission in the job that does +> publishing, not globally. Also, try to separate building from publishing +> — this makes sure that any scripts maliciously injected into the build +> or test environment won't be able to elevate privileges while flying under +> the radar. + + ## Non-goals This GitHub Action [has nothing to do with _building package @@ -221,3 +266,6 @@ https://packaging.python.org/glossary/#term-Distribution-Package https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct-single.svg [SWUdocs]: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md + +[warehouse#12965]: https://github.com/pypi/warehouse/issues/12965 +[OpenID Connect Publishing]: https://pypi.org/help/#openid-connect diff --git a/action.yml b/action.yml index a657ca5c..32837f2b 100644 --- a/action.yml +++ b/action.yml @@ -20,6 +20,7 @@ inputs: The inputs have been normalized to use kebab-case. Use `repository-url` instead. required: false + default: https://pypi.org/legacy/ packages-dir: # Canonical alias for `packages_dir` description: The target directory for distribution required: false diff --git a/oidc-exchange.py b/oidc-exchange.py new file mode 100644 index 00000000..806a609a --- /dev/null +++ b/oidc-exchange.py @@ -0,0 +1,156 @@ +import os +import sys +from http import HTTPStatus +from pathlib import Path +from typing import NoReturn +from urllib.parse import urlparse + +import id # pylint: disable=redefined-builtin +import requests + +_GITHUB_STEP_SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")) + +# Rendered if OIDC identity token retrieval fails for any reason. +_TOKEN_RETRIEVAL_FAILED_MESSAGE = """ +OIDC token retrieval failed: {identity_error} + +This generally indicates a workflow configuration error, such as insufficient +permissions. Make sure that your workflow has `id-token: write` configured +at the job level, e.g.: + +```yaml +permissions: + id-token: write +``` + +Learn more at https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings. +""" + +# Rendered if the package index refuses the given OIDC token. +_SERVER_REFUSED_TOKEN_EXCHANGE_MESSAGE = """ +Token request failed: the server refused the request for the following reasons: + +{reasons} +""" + +# Rendered if the package index's token response isn't valid JSON. +_SERVER_TOKEN_RESPONSE_MALFORMED_JSON = """ +Token request failed: the index produced an unexpected +{status_code} response. + +This strongly suggests a server configuration or downtime issue; wait +a few minutes and try again. +""" + +# Rendered if the package index's token response isn't a valid API token payload. +_SERVER_TOKEN_RESPONSE_MALFORMED_MESSAGE = """ +Token response error: the index gave us an invalid response. + +This strongly suggests a server configuration or downtime issue; wait +a few minutes and try again. +""" + + +def die(msg: str) -> NoReturn: + with _GITHUB_STEP_SUMMARY.open("a", encoding="utf-8") as io: + print(msg, file=io) + + # NOTE: `msg` is Markdown formatted, so we emit only the header line to + # avoid clogging the console log with a full Markdown formatted document. + header = msg.splitlines()[0] + print(f"::error::OIDC exchange failure: {header}", file=sys.stderr) + sys.exit(1) + + +def debug(msg: str): + print(f"::debug::{msg.title()}", file=sys.stderr) + + +def get_normalized_input(name: str) -> str | None: + name = f"INPUT_{name.upper()}" + if val := os.getenv(name): + return val + return os.getenv(name.replace("-", "_")) + + +def assert_successful_audience_call(resp: requests.Response, domain: str): + if resp.ok: + return + + match resp.status_code: + case HTTPStatus.FORBIDDEN: + # This index supports OIDC, but forbids the client from using + # it (either because it's disabled, limited to a beta group, etc.) + die(f"audience retrieval failed: repository at {domain} has OIDC disabled") + case HTTPStatus.NOT_FOUND: + # This index does not support OIDC. + die( + "audience retrieval failed: repository at " + f"{domain} does not indicate OIDC support", + ) + case other: + status = HTTPStatus(other) + # Unknown: the index may or may not support OIDC, but didn't respond with + # something we expect. This can happen if the index is broken, in maintenance mode, + # misconfigured, etc. + die( + "audience retrieval failed: repository at " + f"{domain} responded with unexpected {other}: {status.phrase}", + ) + + +repository_url = get_normalized_input("repository-url") +repository_domain = urlparse(repository_url).netloc +token_exchange_url = f"https://{repository_domain}/_/oidc/github/mint-token" + +# Indices are expected to support `https://{domain}/_/oidc/audience`, +# which tells OIDC exchange clients which audience to use. +audience_url = f"https://{repository_domain}/_/oidc/audience" +audience_resp = requests.get(audience_url) +assert_successful_audience_call(audience_resp, repository_domain) + +oidc_audience = audience_resp.json()["audience"] + +debug(f"selected OIDC token exchange endpoint: {token_exchange_url}") + +try: + oidc_token = id.detect_credential(audience=oidc_audience) +except id.IdentityError as identity_error: + die(_TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error)) + +# Now we can do the actual token exchange. +mint_token_resp = requests.post( + token_exchange_url, + json={"token": oidc_token}, +) + +try: + mint_token_payload = mint_token_resp.json() +except requests.JSONDecodeError: + # Token exchange failure normally produces a JSON error response, but + # we might have hit a server error instead. + die( + _SERVER_TOKEN_RESPONSE_MALFORMED_JSON.format( + status_code=mint_token_resp.status_code, + ), + ) + +# On failure, the JSON response includes the list of errors that +# occurred during minting. +if not mint_token_resp.ok: + reasons = "\n".join( + f"* `{error['code']}`: {error['description']}" + for error in mint_token_payload["errors"] + ) + + die(_SERVER_REFUSED_TOKEN_EXCHANGE_MESSAGE.format(reasons=reasons)) + +pypi_token = mint_token_payload.get("token") +if pypi_token is None: + die(_SERVER_TOKEN_RESPONSE_MALFORMED_MESSAGE) + +# Mask the newly minted PyPI token, so that we don't accidentally leak it in logs. +print(f"::add-mask::{pypi_token}", file=sys.stderr) + +# This final print will be captured by the subshell in `twine-upload.sh`. +print(pypi_token) diff --git a/requirements/runtime.in b/requirements/runtime.in index abf4ef42..387df3c4 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,5 +1,13 @@ twine +# NOTE: Used to detect an ambient OIDC credential for OIDC publishing. +id ~= 1.0 + +# NOTE: This is pulled in transitively through `twine`, but we also declare +# NOTE: it explicitly here because `oidc-exchange.py` uses it. +# Ref: https://github.com/di/id +requests + # NOTE: `pkginfo` is a transitive dependency for us that is coming from Twine. # NOTE: It is declared here only to avoid installing a broken combination of # NOTE: the distribution packages. This should be removed once a fixed version diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 03b99f0d..251fd654 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -18,6 +18,8 @@ cryptography==39.0.1 # via secretstorage docutils==0.19 # via readme-renderer +id==1.0.0 + # via -r requirements/runtime.in idna==3.4 # via requests importlib-metadata==5.1.0 @@ -36,10 +38,12 @@ more-itertools==9.0.0 # via jaraco-classes pkginfo==1.9.2 # via - # -r runtime.in + # -r requirements/runtime.in # twine pycparser==2.21 # via cffi +pydantic==1.10.6 + # via id pygments==2.13.0 # via # readme-renderer @@ -48,6 +52,8 @@ readme-renderer==37.3 # via twine requests==2.28.1 # via + # -r requirements/runtime.in + # id # requests-toolbelt # twine requests-toolbelt==0.10.1 @@ -61,7 +67,9 @@ secretstorage==3.3.3 six==1.16.0 # via bleach twine==4.0.1 - # via -r runtime.in + # via -r requirements/runtime.in +typing-extensions==4.5.0 + # via pydantic urllib3==1.26.13 # via # requests diff --git a/twine-upload.sh b/twine-upload.sh index 935272d6..b549b874 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -40,6 +40,12 @@ INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')" INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')" INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')" +if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then + # No password supplied by the user implies that we're in the OIDC flow; + # retrieve the OIDC credential and exchange it for a PyPI API token. + echo "::notice::In OIDC flow" + INPUT_PASSWORD="$(python /app/oidc-exchange.py)" +fi if [[ "$INPUT_USER" == "__token__" &&