Skip to content

Commit

Permalink
Merge pull request #258 from artichoke/lopopolo/packaging
Browse files Browse the repository at this point in the history
Improve Python packaging, enable some code reuses
  • Loading branch information
lopopolo authored Aug 28, 2024
2 parents c724032 + 93ecf54 commit 270754f
Show file tree
Hide file tree
Showing 8 changed files with 107 additions and 163 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:

- name: GPG sign binary
id: gpg_signing
run: venv/bin/python3 src/gpg_sign.py "nightly-gpg-sign-test" --artifact artichoke/target/release/artichoke
run: venv/bin/python3 -m artichoke_nightly.gpg_sign "nightly-gpg-sign-test" --artifact artichoke/target/release/artichoke

- name: Verify GPG signature
run: gpg --batch --verify "${{ steps.gpg_signing.outputs.signature }}" artichoke/target/release/artichoke
Expand Down Expand Up @@ -100,7 +100,7 @@ jobs:
id: apple_codesigning
if: runner.os == 'macOS'
run: |
venv/bin/python3 src/macos_sign_and_notarize.py "nightly-apple-codesign-test" \
venv/bin/python3 -m artichoke_nightly.macos_sign_and_notarize "nightly-apple-codesign-test" \
--binary "artichoke/target/release/artichoke" \
--binary "artichoke/target/release/airb" \
--resource artichoke/LICENSE \
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/nightly.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -244,16 +244,16 @@ jobs:
if: runner.os == 'macOS'
run: |
python3 -m venv --upgrade-deps venv
./venv/bin/pip install --upgrade pip wheel
./venv/bin/pip install --require-hashes -r requirements.txt
venv/bin/python3 -m pip install --upgrade pip wheel
venv/bin/python3 -m pip install --require-hashes -r requirements.txt
# This will codesign binaries in place which means that the tarballed
# binaries will be codesigned as well.
- name: Run Apple Codesigning and Notarization
id: apple_codesigning
if: runner.os == 'macOS'
run: |
./venv/bin/python3 src/macos_sign_and_notarize.py "artichoke-nightly-${{ matrix.target }}" \
venv/bin/python3 -m artichoke_nightly.macos_sign_and_notarize "artichoke-nightly-${{ matrix.target }}" \
--binary "artichoke/target/${{ matrix.target }}/release/artichoke" \
--binary "artichoke/target/${{ matrix.target }}/release/airb" \
--resource artichoke/LICENSE \
Expand All @@ -269,7 +269,7 @@ jobs:
id: apple_codesigning_gpg
if: runner.os == 'macOS'
run: |
python3 src/gpg_sign.py "artichoke-nightly-${{ matrix.target }}" \
venv/bin/python3 -m artichoke_nightly.gpg_sign "artichoke-nightly-${{ matrix.target }}" \
--artifact "${{ steps.apple_codesigning.outputs.asset }}"
- name: Upload release archive
Expand Down Expand Up @@ -323,7 +323,7 @@ jobs:
- name: GPG sign archive
id: gpg_signing
run: python3 src/gpg_sign.py "artichoke-nightly-${{ matrix.target }}" --artifact "${{ steps.build.outputs.asset }}"
run: venv/bin/python3 -m artichoke_nightly.gpg_sign "artichoke-nightly-${{ matrix.target }}" --artifact "${{ steps.build.outputs.asset }}"

- name: Upload release archive
uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0
Expand Down Expand Up @@ -434,8 +434,8 @@ jobs:
if: runner.os == 'macOS'
run: |
python3 -m venv --upgrade-deps venv
./venv/bin/pip install --upgrade pip wheel
./venv/bin/pip install --require-hashes -r requirements.txt
venv/bin/python3 -m pip install --upgrade pip wheel
venv/bin/python3 -m pip install --require-hashes -r requirements.txt
- name: Build archive
shell: bash
Expand Down
2 changes: 1 addition & 1 deletion Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ GEM
rainbow (3.1.1)
rake (13.2.1)
regexp_parser (2.9.2)
rexml (3.3.5)
rexml (3.3.6)
strscan
rubocop (1.65.1)
json (~> 2.3)
Expand Down
Empty file added artichoke_nightly/__init__.py
Empty file.
60 changes: 60 additions & 0 deletions artichoke_nightly/github_actions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import os
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path


def set_output(*, name: str, value: str) -> None:
"""
Set an output for a GitHub Actions job.
https://docs.github.com/en/actions/using-jobs/defining-outputs-for-jobs
https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
"""

if github_output := os.getenv("GITHUB_OUTPUT"):
with Path(github_output).open("a") as out:
print(f"{name}={value}", file=out)


@contextmanager
def log_group(group: str) -> Iterator[None]:
"""
Create an expandable log group in GitHub Actions job logs.
https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
"""

print(f"::group::{group}")
try:
yield
finally:
print("::endgroup::")


def emit_metadata() -> None:
if os.getenv("CI") != "true":
return
with log_group("Workflow metadata"):
if repository := os.getenv("GITHUB_REPOSITORY"):
print(f"GitHub Repository: {repository}")
if actor := os.getenv("GITHUB_ACTOR"):
print(f"GitHub Actor: {actor}")
if workflow := os.getenv("GITHUB_WORKFLOW"):
print(f"GitHub Workflow: {workflow}")
if job := os.getenv("GITHUB_JOB"):
print(f"GitHub Job: {job}")
if run_id := os.getenv("GITHUB_RUN_ID"):
print(f"GitHub Run ID: {run_id}")
if ref := os.getenv("GITHUB_REF"):
print(f"GitHub Ref: {ref}")
if ref_name := os.getenv("GITHUB_REF_NAME"):
print(f"GitHub Ref Name: {ref_name}")
if sha := os.getenv("GITHUB_SHA"):
print(f"GitHub SHA: {sha}")


def runner_tempdir() -> Path | None:
if temp := os.getenv("RUNNER_TEMP"):
return Path(temp)
return None
80 changes: 4 additions & 76 deletions src/gpg_sign.py → artichoke_nightly/gpg_sign.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
#!/usr/bin/env python3

import argparse
import os
import shutil
import subprocess
import sys
import traceback
from collections.abc import Iterator
from contextlib import contextmanager, suppress
from contextlib import suppress
from dataclasses import dataclass
from pathlib import Path

from .github_actions import emit_metadata, log_group, set_output
from .shell_utils import run_command_with_merged_output

GPG_SIGN_VERSION = "0.3.0"


Expand All @@ -20,79 +21,6 @@ class Args:
release: str


def run_command_with_merged_output(command: list[str]) -> None:
"""
Run the given command as a subprocess and merge its stdout and stderr
streams.
This is useful for funnelling all output of a command into a GitHub Actions
log group.
This command uses `check=True` when delegating to `subprocess`.
"""

proc = subprocess.run(
command,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
for line in proc.stdout.splitlines():
if line:
print(line)


def set_output(*, name: str, value: str) -> None:
"""
Set an output for a GitHub Actions job.
https://docs.github.com/en/actions/using-jobs/defining-outputs-for-jobs
https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
"""

if github_output := os.getenv("GITHUB_OUTPUT"):
with Path(github_output).open("a") as out:
print(f"{name}={value}", file=out)


@contextmanager
def log_group(group: str) -> Iterator[None]:
"""
Create an expandable log group in GitHub Actions job logs.
https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
"""

print(f"::group::{group}")
try:
yield
finally:
print("::endgroup::")


def emit_metadata() -> None:
if os.getenv("CI") != "true":
return
with log_group("Workflow metadata"):
if repository := os.getenv("GITHUB_REPOSITORY"):
print(f"GitHub Repository: {repository}")
if actor := os.getenv("GITHUB_ACTOR"):
print(f"GitHub Actor: {actor}")
if workflow := os.getenv("GITHUB_WORKFLOW"):
print(f"GitHub Workflow: {workflow}")
if job := os.getenv("GITHUB_JOB"):
print(f"GitHub Job: {job}")
if run_id := os.getenv("GITHUB_RUN_ID"):
print(f"GitHub Run ID: {run_id}")
if ref := os.getenv("GITHUB_REF"):
print(f"GitHub Ref: {ref}")
if ref_name := os.getenv("GITHUB_REF_NAME"):
print(f"GitHub Ref Name: {ref_name}")
if sha := os.getenv("GITHUB_SHA"):
print(f"GitHub SHA: {sha}")


def signing_identity() -> str:
"""
Signing identity and GPG key fingerprint.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
import stamina
import validators

from .github_actions import emit_metadata, log_group, runner_tempdir, set_output
from .shell_utils import run_command_with_merged_output

MACOS_SIGN_AND_NOTARIZE_VERSION = "0.6.0"

MACOS_MONTEREY_MAJOR_VERSION = 12
Expand Down Expand Up @@ -98,60 +101,6 @@ def run_notarytool(command: list[str]) -> str:
raise NotaryToolError(proc.stderr)


@stamina.retry(on=subprocess.CalledProcessError, attempts=3)
def run_command_with_merged_output(command: list[str]) -> None:
"""
Run the given command as a subprocess and merge its stdout and stderr
streams. This function will retry the given command on any error, up to 3
times.
This is useful for funnelling all output of a command into a GitHub Actions
log group.
This command uses `check=True` when delegating to `subprocess`.
"""

proc = subprocess.run(
command,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)

for line in proc.stdout.splitlines():
if line:
print(line)


def set_output(*, name: str, value: str) -> None:
"""
Set an output for a GitHub Actions job.
https://docs.github.com/en/actions/using-jobs/defining-outputs-for-jobs
https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
"""

if github_output := os.getenv("GITHUB_OUTPUT"):
with Path(github_output).open("a") as out:
print(f"{name}={value}", file=out)


@contextmanager
def log_group(group: str) -> Iterator[None]:
"""
Create an expandable log group in GitHub Actions job logs.
https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
"""

print(f"::group::{group}")
try:
yield
finally:
print("::endgroup::")


@contextmanager
def attach_disk_image(image: Path, *, readwrite: bool = False) -> Iterator[Path]:
try:
Expand Down Expand Up @@ -215,28 +164,6 @@ def get_image_size(image: Path) -> int:
return (size * 512 // 1000 // 1000) + 1


def emit_metadata() -> None:
if os.getenv("CI") != "true":
return
with log_group("Workflow metadata"):
if repository := os.getenv("GITHUB_REPOSITORY"):
print(f"GitHub Repository: {repository}")
if actor := os.getenv("GITHUB_ACTOR"):
print(f"GitHub Actor: {actor}")
if workflow := os.getenv("GITHUB_WORKFLOW"):
print(f"GitHub Workflow: {workflow}")
if job := os.getenv("GITHUB_JOB"):
print(f"GitHub Job: {job}")
if run_id := os.getenv("GITHUB_RUN_ID"):
print(f"GitHub Run ID: {run_id}")
if ref := os.getenv("GITHUB_REF"):
print(f"GitHub Ref: {ref}")
if ref_name := os.getenv("GITHUB_REF_NAME"):
print(f"GitHub Ref Name: {ref_name}")
if sha := os.getenv("GITHUB_SHA"):
print(f"GitHub SHA: {sha}")


def keychain_path() -> Path:
"""
Absolute path to a keychain used for the codesigning and notarization
Expand All @@ -249,7 +176,7 @@ def keychain_path() -> Path:
#
# `RUNNER_TEMP` is the path to a temporary directory on the runner. This
# directory is emptied at the beginning and end of each job.
if runner_temp := os.getenv("RUNNER_TEMP"):
if runner_temp := runner_tempdir():
return Path(runner_temp).joinpath("notarization.keychain-db")

return Path("notarization.keychain-db").resolve()
Expand Down
29 changes: 29 additions & 0 deletions artichoke_nightly/shell_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import subprocess

import stamina


@stamina.retry(on=subprocess.CalledProcessError, attempts=3)
def run_command_with_merged_output(command: list[str]) -> None:
"""
Run the given command as a subprocess and merge its stdout and stderr
streams. This function will retry the given command on any error, up to 3
times.
This is useful for funnelling all output of a command into a GitHub Actions
log group.
This command uses `check=True` when delegating to `subprocess`.
"""

proc = subprocess.run(
command,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)

for line in proc.stdout.splitlines():
if line:
print(line)

0 comments on commit 270754f

Please sign in to comment.