From bf3c1087fbd5357eaea0883f352dbcd2d85253f4 Mon Sep 17 00:00:00 2001 From: Erik Skultety Date: Mon, 25 Nov 2024 12:56:30 +0100 Subject: [PATCH 1/2] .github: workflows: Introduce a new dependabot pip-compile workflow This workflow is a direct consequence of the asynchronous release schedule of pydantic and pydantic core and the fact that pydantic is always pinned to a particular pydantic-core version. Dependabot doesn't see these transitive relations and so can't properly update the versions in this case (it always assumes the latest for every dependency). This will naturally lead to broken CI making these version updates impossible to merge. Since our project directly only cares about pydantic and not pydantic-core, we can ignore pydantic-core updates (future patch) and run a dedicated workflow on every dependabot pull request that would check whether any additional changes (i.e. transitive dependency version locks) to our requirements files are needed. If so, then the GitHub actions bot will comment on the pull request that a change to these files is needed and will provide a patch to the reviewer to apply and update the pull request. The workflow is only executed when changes to the requirements files are proposed (realistically only by dependabot). Note that it's not possible to specify the source branch as the workflow trigger, only the target branch, and so that could not have been used as a better filter for dependabot-proposed pull requests specifically. It is run using a Python Alpine docker image, saves the git diff produced by pip-compile to the default github actions environment followed by a github script action that will pop the diff out of the environment and use it to comment on the pull request. References: - https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings - https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/using-conditions-to-control-job-execution - https://github.com/actions/github-script?tab=readme-ov-file#comment-on-an-issue - https://github.com/actions/github-script/issues/247#issuecomment-1079839739 - https://github.com/actions/github-script/issues/220#issuecomment-1007633429 Signed-off-by: Erik Skultety --- .github/workflows/dependabot-pipcompile.yml | 76 +++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/dependabot-pipcompile.yml diff --git a/.github/workflows/dependabot-pipcompile.yml b/.github/workflows/dependabot-pipcompile.yml new file mode 100644 index 000000000..282b2678e --- /dev/null +++ b/.github/workflows/dependabot-pipcompile.yml @@ -0,0 +1,76 @@ +name: Pip-compile + +on: + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - requirements.txt + - requirements-extras.txt + workflow_dispatch: + inputs: {} + +# Need these permissions for the GITHUB_TOKEN to be able to post a comment to a PR +permissions: + issues: write + pull-requests: write + +jobs: + versions-check: + runs-on: ubuntu-24.04 + container: + image: python:3.9-alpine + + steps: + # Need to install git before running the checkout action in a container + - name: Install dependencies + run: apk update && apk add --no-cache git + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install pip-tools + run: | + pip install --upgrade pip + pip install --no-cache-dir pip-tools + + # This step uses multi-line string injection to GitHub environment [1] + # [1] https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings + - name: Run pip-compile to update requirements.txt + run: | + git config --global --add safe.directory "*" + pip-compile --generate-hashes --output-file=requirements.txt pyproject.toml + pip-compile \ + --all-extras \ + --allow-unsafe \ + --generate-hashes \ + --output-file=requirements-extras.txt \ + pyproject.toml + { + echo 'GIT_DIFF<> "$GITHUB_ENV" + + # Only comment on PRs when changes to requirements files are needed, based on: + # - https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/using-conditions-to-control-job-execution + # - https://github.com/actions/github-script?tab=readme-ov-file#comment-on-an-issue + # - https://github.com/actions/github-script/issues/247#issuecomment-1079839739 + # - https://github.com/actions/github-script/issues/220#issuecomment-1007633429 + - name: Comment on pull request + uses: actions/github-script@v7 + if: env.GIT_DIFF != '' + env: + DIFF: "Changes to requirements files are needed. If you're experiencing CI test failures, please apply the following patch and update the pull request:\n```diff\n${{ env.GIT_DIFF }}\n```" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.DIFF + }) From 2570ebe1b8089c7cd2da43bd0182b25d3534295b Mon Sep 17 00:00:00 2001 From: Erik Skultety Date: Fri, 29 Nov 2024 16:38:19 +0100 Subject: [PATCH 2/2] .github: dependabot: Ignore pydantic-core & consider only pydantic pydantic and pydantic-core have different release cycles and since a pydantic release is always locked to a particular pydantic-core version, a dependabot update to both will cause install conflicts pretty much every time because dependabot doesn't see transitive dependency relations. This patch makes it so that pydantic-core's updates are ignored by dependabot (because it's a transitive dependency for us) and we're only going to consider pydantic. This however, will still break installs because pip-compile needs to be invoked to re-pin this new pydantic version to a correct pydantic-core version. That is handled by the newly introduced github dependabot workflow 'dependabot-pipcompile.yml' introduced in a past patch. Signed-off-by: Erik Skultety --- .github/dependabot.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a5d0a2bd..50015c775 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,20 +19,10 @@ updates: directory: "/" schedule: interval: "monthly" + ignore: + - dependency-name: "pydantic-core" groups: minor-and-patch: update-types: - "minor" - "patch" - exclude-patterns: - - "pydantic*" - - # pydantic is a known violator of version updates where they don't release the core backend - # with the API library at the same time which holds up other legitimate updates, so group - # pydantic deps together - pydantic: - update-types: - - "minor" - - "patch" - patterns: - - "pydantic*"