diff --git a/.github/workflows/_reusable-package-testpypi.yml b/.github/workflows/_reusable-package-testpypi.yml new file mode 100644 index 00000000..9297ff61 --- /dev/null +++ b/.github/workflows/_reusable-package-testpypi.yml @@ -0,0 +1,95 @@ +--- +name: Publish to TestPyPI +on: + workflow_call: + inputs: + package-name: + description: The name of the package. + required: true + type: string + repo-name: + description: The full name of the repository to use to gate uploads, in the + format `owner/repo`. + required: true + type: string +concurrency: + group: pypi +env: + PACKAGE_NAME: ${{ inputs.package-name }} +jobs: + test-pypi-build: + name: Build package with unique version for test.pypi.org + needs: [job-variables] + if: github.repository == inputs.repo-name + runs-on: ubuntu-latest + permissions: + id-token: write + attestations: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ############################################################################################## + # TODO: Convert this into an action + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml # any version will work, just use 3.12 for the action + - name: Install workflow dependencies + run: pip install -r scripts/requirements.txt + - name: Create unique package version + id: create-version + run: | + CURRENT_VERSION=$(python scripts/pypi_latest_version.py --package="$PACKAGE_NAME" --index=test.pypi) + echo CURRENT_VERSION: $CURRENT_VERSION + NEW_VERSION=$(python scripts/create_post_version_for_testpypi.py --version=$CURRENT_VERSION) + echo NEW_VERSION: $NEW_VERSION + python scripts/project_version.py --set-version=$NEW_VERSION + echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_OUTPUT + ############################################################################################## + - name: Build package + uses: hynek/build-and-inspect-python-package@v2.8.0 + with: + attest-build-provenance-github: 'true' + outputs: + built-version: ${{ steps.create-version.outputs.NEW_VERSION }} + test-pypi-upload: + name: Upload package to test.pypi.org + needs: [job-variables, test-pypi-build] + if: github.repository == inputs.repo-name + runs-on: ubuntu-latest + environment: package-testpypi + permissions: + id-token: write + steps: + - name: Download built packages + uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + - name: Upload package to Test PyPI + uses: pypa/gh-action-pypi-publish@v1.9.0 + with: + repository-url: https://test.pypi.org/legacy/ + test-pypi-install: + name: Install package from test.pypi.org + needs: [job-variables, test-pypi-build, test-pypi-upload] + if: github.repository == inputs.repo-name + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + - name: Test installing from test.pypi.org + # A retry is used to allow for some downtime before the package is installable + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 5 + retry_wait_seconds: 30 + warning_on_retry: false + command: pip install --index-url=https://test.pypi.org/simple/ --extra-index-url=https://pypi.org/simple + "$PACKAGE_NAME==${{ needs.test-pypi-build.outputs.built-version }}" diff --git a/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml b/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml index b8131ef1..359d53ec 100644 --- a/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml +++ b/.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml @@ -11,39 +11,41 @@ on: description: The email of the user to use when committing changes to the repository. required: true type: string - dependency-dict: - description: 'Specify a valid dictionary of dependency groups to update, where - each key is a dependency group name, and each value is a tuple of dependencies - to update within that group, e.g. {"dev": ("pylint", "ruff"), "tests": ("ruff")}.' - required: false - type: string - default: '' - update-pre-commit: - description: A boolean indicating if the pre-commit hooks should be updated. - required: false - type: boolean - default: false - run-pre-commit: - description: A boolean indicating to run the pre-commit hooks to perform auto-fixing - after updating the dependencies. Setting this input to `true` will also set - the update-pre-commit input to `true`. - required: false - type: boolean - default: false - pre-commit-hook-skip-list: - description: A comma-separated list of pre-commit hooks to skip (only applicable - when `run-pre-commit=true`). - required: false - default: '' - export-dependency-groups: - description: A comma-separated list of dependency groups that should have their - requirements exported. An output folder can be specified by appending a ":" - followed by the custom output folder path to the provided group name, e.g. - "tests:custom/folder/path". The created file will always be named "requirements.txt", - and the folder will default to matching the group name if no custom folder - path is given. - required: false - default: '' + dependency-dict: + description: 'Specify a valid dictionary of dependency groups to update, where + each key is a dependency group name, and each value is a tuple of dependencies + to update within that group, e.g. {"dev": ("pylint", "ruff"), "tests": ("ruff")}.' + required: false + type: string + default: '' + update-pre-commit: + description: A boolean indicating if the pre-commit hooks should be updated. + required: false + type: boolean + default: false + run-pre-commit: + description: A boolean indicating to run the pre-commit hooks to perform auto-fixing + after updating the dependencies. Setting this input to `true` will also + set the update-pre-commit input to `true`. + required: false + type: boolean + default: false + pre-commit-hook-skip-list: + description: A comma-separated list of pre-commit hooks to skip (only applicable + when `run-pre-commit=true`). + required: false + type: string + default: '' + export-dependency-groups: + description: A comma-separated list of dependency groups that should have + their requirements exported. An output folder can be specified by appending + a ":" followed by the custom output folder path to the provided group name, + e.g. "tests:custom/folder/path". The created file will always be named "requirements.txt", + and the folder will default to matching the group name if no custom folder + path is given. + required: false + type: string + default: '' secrets: checkout-token: description: The token to use for checking out the repository, must have permissions @@ -59,7 +61,7 @@ jobs: update-python-and-pre-commit-deps: name: Update python linters and pre-commit dependencies runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' && contains(github.head_ref, '/pip/') }} +# if: ${{ github.actor == 'dependabot[bot]' && contains(github.head_ref, '/pip/') }} permissions: contents: write steps: @@ -74,7 +76,7 @@ jobs: passphrase: ${{ secrets.gpg-signing-key-passphrase }} git_user_signingkey: true git_commit_gpgsign: true - - uses: tektronix/python-package-ci-cd/actions/update-development-dependencies@main # TODO: pin to a version + - uses: ./actions/update-development-dependencies with: dependency-dict: ${{ inputs.dependency-dict }} update-pre-commit: ${{ inputs.update-pre-commit }} diff --git a/.github/workflows/update-python-and-pre-commit-dependencies.yml b/.github/workflows/update-python-and-pre-commit-dependencies.yml index 3ab958c0..5927613b 100644 --- a/.github/workflows/update-python-and-pre-commit-dependencies.yml +++ b/.github/workflows/update-python-and-pre-commit-dependencies.yml @@ -5,33 +5,17 @@ on: branches: [main] jobs: update-python-and-pre-commit-deps: - # TODO: switch to using the Reusable Workflow - name: Update python linters and pre-commit dependencies - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' && contains(github.head_ref, '/pip/') }} + uses: ./.github/workflows/_reusable-update-python-and-pre-commit-dependencies.yml + with: + commit-user-name: ${{ vars.TEK_OPENSOURCE_NAME }} + commit-user-email: ${{ vars.TEK_OPENSOURCE_EMAIL }} + update-pre-commit: true + run-pre-commit: true + pre-commit-hook-skip-list: pyright,poetry-audit + export-dependency-groups: udd:actions/update-development-dependencies permissions: contents: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.head_ref }} - token: ${{ secrets.TEK_OPENSOURCE_TOKEN }} - - uses: crazy-max/ghaction-import-gpg@v6 - with: - gpg_private_key: ${{ secrets.TEK_OPENSOURCE_GPG_SIGNING_KEY_PRIVATE }} - passphrase: ${{ secrets.TEK_OPENSOURCE_GPG_SIGNING_KEY_PASSPHRASE }} - git_user_signingkey: true - git_commit_gpgsign: true - - uses: ./actions/update-development-dependencies - with: - update-pre-commit: true - run-pre-commit: true - pre-commit-hook-skip-list: pyright,poetry-audit - export-dependency-groups: udd:actions/update-development-dependencies - - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: 'chore: Update python linters and pre-commit dependencies.' - commit_user_name: ${{ vars.TEK_OPENSOURCE_NAME }} - commit_user_email: ${{ vars.TEK_OPENSOURCE_EMAIL }} - commit_author: ${{ vars.TEK_OPENSOURCE_NAME }} <${{ vars.TEK_OPENSOURCE_EMAIL }}> + secrets: + checkout-token: ${{ secrets.TEK_OPENSOURCE_TOKEN }} + gpg-signing-key-private: ${{ secrets.TEK_OPENSOURCE_GPG_SIGNING_KEY_PRIVATE }} + gpg-signing-key-passphrase: ${{ secrets.TEK_OPENSOURCE_GPG_SIGNING_KEY_PASSPHRASE }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18674042..3c0f003f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,7 +59,7 @@ repos: rev: e70baeefd566058716df2f29eae8fe8ffc213a9f # frozen: v2.12.1b3 hooks: - id: hadolint - args: [--ignore=DL3008] + args: [--ignore=DL3008, --ignore=DL3018] - repo: https://github.com/executablebooks/mdformat rev: 08fba30538869a440b5059de90af03e3502e35fb # frozen: 0.7.17 hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 755d54d5..136970cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ Ready to contribute? Here's how to set up `python-package-ci-cd` for local devel - Using the helper script (recommended): ```console - python scripts/contributor_setup.py + python contributor_setup.py ``` 4. Check to see if there are any [open issues](https://github.com/tektronix/python-package-ci-cd/issues) or [pull requests](https://github.com/tektronix/python-package-ci-cd/pulls) that are related to the change you wish to make. diff --git a/actions/update-development-dependencies/Dockerfile b/actions/update-development-dependencies/Dockerfile index 9392e656..411b5939 100644 --- a/actions/update-development-dependencies/Dockerfile +++ b/actions/update-development-dependencies/Dockerfile @@ -1,15 +1,13 @@ -FROM python:3.12-slim-bullseye +FROM python:3.12-alpine # Copy over necessary files COPY requirements.txt /requirements.txt COPY update_development_dependencies.py /update_development_dependencies.py # Install dependencies -RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/00-docker && \ - apt-get update --quiet && \ - apt-get install --quiet --assume-yes --no-install-recommends git && \ - apt-get clean && \ - rm --force --recursive /var/lib/apt/lists/* +RUN apk update && \ + apk add --no-cache git && \ + rm -rf /var/cache/apk/* RUN python -m pip install --no-cache-dir --requirement /requirements.txt # Run the updater script as the entrypoint diff --git a/contributor_setup.py b/contributor_setup.py new file mode 100644 index 00000000..6b52b7ce --- /dev/null +++ b/contributor_setup.py @@ -0,0 +1,98 @@ +"""Set up an environment to use to contribute to this package. + +This script will run through the commands listed in the CONTRIBUTING.md file. +""" + +from __future__ import annotations + +import glob +import os +import platform +import shlex +import subprocess +import sys + +from pathlib import Path + +RUNNING_ON_LINUX = platform.system().upper() != "WINDOWS" +RUNNING_IN_VIRTUALENV = sys.prefix != sys.base_prefix + + +def create_virtual_environment(virtual_env_dir: str | os.PathLike[str]) -> None: + """Create a virtual environment. + + Args: + virtual_env_dir: The directory where the virtual environment should be created + """ + print(f"\nCreating virtualenv located at '{virtual_env_dir}'") + _run_cmd_in_subprocess(f"{sys.executable} -m venv {virtual_env_dir} --clear") + + +def _run_cmd_in_subprocess(command: str) -> None: + """Run the given command in a subprocess. + + Args: + command: The command string to send. + """ + command = command.replace("\\", "/") + print(f"\nExecuting command: {command}") + subprocess.check_call(shlex.split(command)) # noqa: S603 + + +def main() -> None: + """Set up the environment to allow development. + + Raises: + SystemExit: Indicates that the setup failed for some reason. + """ + starting_dir = Path.cwd() + try: + if RUNNING_IN_VIRTUALENV: + raise IndexError # noqa: TRY301 + if sys.version_info[0:2] != (3, 12): + msg = "Unable to set up the environment. Please use Python 3.12." + raise SystemExit(msg) + # Windows systems require the 64 bit python + if platform.system().lower() == "windows" and sys.maxsize <= 2**32: + msg = "Unable to set up the environment. Please use a 64-bit Python version." + raise SystemExit(msg) + # Create the virtual environment + virtual_env_dir = starting_dir / ".venv" + create_virtual_environment(virtual_env_dir) + os.environ["VIRTUAL_ENV"] = virtual_env_dir.as_posix() + + # Delete the previous poetry lock file + lock_file = Path(starting_dir) / "poetry.lock" + if lock_file.exists(): + lock_file.unlink() + + # Find the python executable from the new virtual environment + files = list( + filter( + lambda x: "site-packages" not in x and "pythonw" not in x, + glob.iglob( # noqa: PTH207 + f"{virtual_env_dir}/{'bin' if RUNNING_ON_LINUX else 'Scripts'}/**/python*", + recursive=True, + ), + ) + ) + python_executable = files[0] + commands_to_send = ( + f"{python_executable} -m pip install -U pip wheel poetry", + f"{python_executable} -m poetry install", + f"{python_executable} -m pre_commit install --install-hooks", + ) + for command in commands_to_send: + _run_cmd_in_subprocess(command) + except IndexError: + msg = ( + "Unable to set up the environment. Please run this script from a " + "standard Python installation, not a virtual environment." + ) + raise SystemExit(msg) # noqa: B904 + finally: + os.chdir(starting_dir) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 38166946..159eba6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ poetry-audit-plugin = "^0.4.0" poetry-pre-commit-plugin = "^0.1.2" pyright = "1.1.376" python = "~3.12" +python-semantic-release = "^9.8.7" [tool.poetry.group.udd.dependencies] # dependencies for actions/update-development-dependencies maison = "^1.4.3" # yamlfix is broken with v2.0+