Automated scheduled releases with changelogs [WIP]

.github/pull_request_template.md

@@ -0,0 +1,3 @@
+
+
+- [x] This change is user-facing

.github/workflows/check-changelog.yml

@@ -0,0 +1,27 @@
+name: Changelog
+on:
+  pull_request:
+    branches:
+      - main
+    # Includes "edited" such that we can detect changes to the description
+    types: [opened, synchronize, reopened, edited]
+
+permissions:
+  pull-requests: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # We need to fetch the parents of the HEAD commit (which is a merge),
+          # because we need to compare the PR against the base branch
+          # to check whether it added a changelog
+          fetch-depth: 2
+
+      - name: check changelog
+        run: scripts/check-changelog.sh . ${{ github.event.pull_request.number }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+

.github/workflows/main.yml

@@ -36,6 +36,68 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
+  # Creates a release commit and combines the changelog files into a single one
+  # For PRs it shows the resulting changelog in the step summary
+  # For pushes to the main branch it updates the release branch
+  #   The release branch is regularly
+  version-changelog:
+    runs-on: ubuntu-latest
+    permissions:
+      # This job only needs this token to read commit objects to figure out what PR they're associated with.
+      # A separate fixed token is used to update the release branch after push events.
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # This fetches the entire Git history.
+          # This is needed so we can determine the commits (and therefore PRs)
+          # where the changelogs have been added
+          fetch-depth: 0
+          # By default, the github.token is used and stored in the Git config,
+          # This would override any authentication provided in the URL,
+          # which we do later to push to a fork.
+          # So we need to prevent that from being stored.
+          persist-credentials: false
+
+      - uses: cachix/install-nix-action@v26
+
+      - name: Increment version and assemble changelog
+        run: |
+          nix-build -A autoVersion
+          # If we're running for a PR, the second argument tells the script to pretend that commits
+          # from this PR are merged already, such that the generated changelog includes it
+          version=$(result/bin/auto-version . ${{ github.event.pull_request.number || '' }})
+          echo "version=$version" >> "$GITHUB_ENV"
+
+          # version is the empty string if there were no user-facing changes for a version bump
+          if [[ -n "$version" ]]; then
+            # While we commit here, it's only pushed conditionally based on it being a push event later
+            git config user.name ${{ github.actor }}
+            git config user.email ${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com
+            git add --all
+            git commit --message "Version $version
+
+          Automated release"
+          fi
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Outputting draft release notes
+        # If we have a new version at all (it's not an empty string)
+        # And it's not a push event (so it's a PR),
+        if: ${{ env.version && github.event_name != 'push' }}
+        # we just output the draft changelog into the step summary
+        run: cat changes/released/${{ env.version }}.md > "$GITHUB_STEP_SUMMARY"
+
+      - name: Update release branch
+        # But if this is a push to the main branch,
+        if: ${{ env.version && github.event_name == 'push' }}
+        # we push to the release branch.
+        # This continuously updates the release branch to contain the latest release notes,
+        # so that one can just merge the release branch into main to do a release.
+        # A PR to do that is opened regularly with another workflow
+        run: git push https://${{ secrets.MACHINE_USER_PAT }}@github.com/infinixbot/nixpkgs-check-by-name.git HEAD:refs/heads/release -f
+
   # Make sure that all links in Markdown documents are valid
   xrefcheck:
     runs-on: ubuntu-latest

.github/workflows/regular-release.yml

@@ -0,0 +1,31 @@
+name: Regular Version
+on:
+  workflow_dispatch: # Allows triggering manually
+  schedule:
+    - cron: '47 14 * * 2' # runs every Tuesday at 14:47 UTC (chosen somewhat randomly)
+
+jobs:
+  version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: infinixbot/nixpkgs-check-by-name
+          ref: release
+
+      - name: Create Pull Request
+        run: |
+          # no-maintainer-edit because manually added commits would get overridden
+          # when the release branch updates again (which is a force push).
+          # Instead maintainers should push any fixes to the main branch.
+          gh pr create \
+            --repo ${{ github.repository }} \
+            --title "$(git log -1 --format=%s HEAD)" \
+            --no-maintainer-edit \
+            --body "Automated release PR.
+
+          - [x] This change is user-facing
+          "
+        env:
+          # Needed so that CI triggers
+          GH_TOKEN: ${{ secrets.MACHINE_USER_PAT }}

CONTRIBUTING.md

@@ -27,6 +27,8 @@ The most important tools and commands in this environment are:
   nix-build -A ci
   ```
 
+Note that pinned dependencies are [regularly updated automatically](./.github/workflows/update.yml).
+
 ### Integration tests
 
 Integration tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs with these files:
@@ -61,9 +63,54 @@ Integration tests are declared in [`./tests`](./tests) as subdirectories imitati
   A file containing the expected standard output.
   The default is expecting an empty standard output.
 
-## Automation
+## Releases and changelogs
+
+The following pipeline is used to ensure a smooth releases process with automated changelogs.
+
+### Pull requests
+
+The default [PR template](./.github/pull_request_template.md) adds this line to the description:
+
+> - [x] This change is user-facing
+
+Unless this field is explicitly unchecked, the PR [is checked to](./.github/workflows/check-changelog.yml)
+add a [changelog entry](#changelog-entries) to describe the user-facing change.
+
+This ensures that all user-facing changes have a changelog entry.
+
+### Changelog entries
+
+In order to avoid conflicts between different PRs,
+a changelog entry is a Markdown file under a directory in
+[`changes/unreleased`](./changes/unreleased).
+Depending on the effort (see [EffVer](https://jacobtomlinson.dev/effver/))
+required for users to update to this change,
+a different directory should be used:
+
+- [`changes/unreleased/major`](./changes/unreleased/major):
+  A large effort. This will cause a version bump from e.g. 0.1.2 to 1.0.0
+- [`changes/unreleased/medium`](./changes/unreleased/medium):
+  Some effort. This will cause a version bump from e.g. 0.1.2 to 1.2.0
+- [`changes/unreleased/minor`](./changes/unreleased/minor):
+  Little/no effort. This will cause a version bump from e.g. 0.1.2 to 0.1.3
+
+The Markdown file must have the `.md` file ending, and be of the form
+
+```markdown
+# Some descriptive title of the change
+
+Optionally more information
+```
+
+### Release branch
 
-Pinned dependencies are [regularly updated automatically](./.github/workflows/update.yml).
+After every push to the main branch, the [infinixbot:release
+branch](https://github.com/infinixbot/nixpkgs-check-by-name/tree/release) is rebased such that it
+contains one commit on top of master, which:
+- Increments the version in `Cargo.toml` according to the unreleased changelog entries.
+- Collects all changelog entries in [`./changes/unreleased`](./changes/unreleased)
+  and combines them into a new `./changes/released/<version>.md` file.
 
-Releases are [automatically created](./.github/workflows/release.yml) when the `version` field in [`Cargo.toml`](./Cargo.toml)
-is updated from a push to the main branch.
+Regularly a PR is [opened automatically](./.github/workflows/regular-release.yml)
+to merge the release branch into the main branch.
+When this PR is merged, a GitHub release is [automatically created](./.github/workflows/release.yml).

default.nix

@@ -127,9 +127,27 @@ let
             echo >&2 "Running ${script}"
             ${lib.getExe script} "$1"
           '') (lib.attrValues updateScripts)}
+          echo ""
+          # To not fail the changelog check
+          printf "%s\n" "- [ ] This change is user-facing"
         '';
       };
 
+    # Run regularly by CI and turned into a PR
+    autoVersion = pkgs.writeShellApplication {
+      name = "auto-version";
+      runtimeInputs = with pkgs; [
+        coreutils
+        git
+        github-cli
+        jq
+        cargo
+        toml-cli
+        cargo-edit
+      ];
+      text = builtins.readFile ./scripts/version.sh;
+    };
+
     # Tests the tool on the pinned Nixpkgs tree, this is a good sanity check
     nixpkgsCheck =
       pkgs.runCommand "test-nixpkgs-check-by-name"

scripts/check-changelog.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s nullglob
+
+root=$1
+prNumber=$2
+
+# The PR template has this, selected by default
+userFacingString="- [x] This change is user-facing"
+nonUserFacingString="- [ ] This change is user-facing"
+
+# Run this first to validate files
+for file in "$root"/changes/unreleased/*/*; do
+  if [[ "$(basename "$file")" == ".gitkeep" ]]; then
+    continue
+  fi
+  if [[ $file != *.md ]]; then
+    echo "File $file: Must be a markdown file with file ending .md"
+    exit 1
+  fi
+  if [[ "$(sed -n '/^#/=' "$file")" != "1" ]]; then
+    echo "File $file: The first line must start with #, while all others must not start with #"
+    exit 1
+  fi
+done
+
+body=$(gh api \
+  -H "Accept: application/vnd.github+json" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  /repos/NixOS/nixpkgs-check-by-name/pulls/"$prNumber" |
+  jq -r '.body')
+
+if grep -F -- "$userFacingString" <<< "$body" > /dev/null; then
+  echo "User-facing change, changelog necessary"
+elif grep -F -- "$nonUserFacingString" <<< "$body" > /dev/null; then
+  echo "Not a user-facing change, no changelog necessary"
+  exit 0
+else
+  echo "Depending on whether this PR has a user-facing change, add one of these lines to the PR description:"
+  printf "%s\n" "$userFacingString"
+  printf "%s\n" "$nonUserFacingString"
+  exit 1
+fi
+
+# This checks whether the most recent commit changed any files in changes/unreleased
+# This works well for PR's CI because there it runs on the merge commit,
+# where HEAD^ is the first parent commit, which is the base branch.
+if [[ -z "$(git -C "$root" log HEAD^..HEAD --name-only "$root"/changes/unreleased)" ]]; then
+  echo "If this PR contains a user-facing change, add a changelog in ./changes/unreleased"
+  echo "Otherwise, check the checkbox:"
+  printf "%s\n" "$nonUserFacingString"
+  exit 1
+else
+  echo "A changelog exists"
+fi

scripts/release.sh

@@ -47,7 +47,10 @@ To import it:
 ```bash
 gzip -cd '"$artifactName"' | nix-store --import | tail -1
 ```
-'
+
+## Changes
+
+'"$(tail -1 "$root"/changes/released/"$version".md)"
 
 echo "Creating draft release"
 if ! release=$(gh api \

scripts/version.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s nullglob
+
+root=$1
+currentPrNumber=${2:-}
+
+[[ "$(toml get --raw "$root"/Cargo.toml package.version)" =~ ([0-9]+)\.([0-9]+)\.([0-9]+) ]]
+splitVersion=("${BASH_REMATCH[@]:1}")
+
+majorChanges=("$root"/changes/unreleased/major/*.md)
+mediumChanges=("$root"/changes/unreleased/medium/*.md)
+minorChanges=("$root"/changes/unreleased/minor/*.md)
+
+if ((${#majorChanges[@]} > 0)); then
+  # If we didn't have `|| true` this would exit the program due to `set -e`,
+  # because (( ... )) returns the incremental value, which is treated as the exit code..
+  ((splitVersion[0]++)) || true
+  splitVersion[1]=0
+  splitVersion[2]=0
+elif ((${#mediumChanges[@]} > 0)); then
+  ((splitVersion[1]++)) || true
+  splitVersion[2]=0
+elif ((${#minorChanges[@]} > 0)); then
+  ((splitVersion[2]++)) || true
+else
+  echo >&2 "No changes"
+  exit 0
+fi
+
+next=${splitVersion[0]}.${splitVersion[1]}.${splitVersion[2]}
+releaseFile=$root/changes/released/${next}.md
+mkdir -p "$(dirname "$releaseFile")"
+
+echo "# Version $next ($(date --iso-8601 --utc))" > "$releaseFile"
+echo "" >> "$releaseFile"
+
+# shellcheck disable=SC2016
+for file in "${majorChanges[@]}" "${mediumChanges[@]}" "${minorChanges[@]}"; do
+  commit=$(git -C "$root" log -1 --format=%H -- "$file")
+  if ! gh api graphql \
+    -f sha="$commit" \
+    -f query='
+    query ($sha: String) {
+      repository(owner: "NixOS", name: "nixpkgs-check-by-name") {
+        commit: object(expression: $sha) {
+          ... on Commit {
+            associatedPullRequests(first: 100) {
+              nodes {
+                merged
+                baseRefName
+                baseRepository { nameWithOwner }
+                number
+                author { login }
+              }
+            }
+          }
+        }
+      }
+    }' |
+    jq --exit-status -r ${currentPrNumber:+--argjson currentPrNumber "$currentPrNumber"} --arg file "$file" '
+    .data.repository.commit.associatedPullRequests?.nodes?[]?
+      | select(
+        # We need to make sure to get the right PR, there can be many
+        (.merged or .number == $ARGS.named.currentPrNumber) and
+        .baseRepository.nameWithOwner == "NixOS/nixpkgs-check-by-name" and
+        .baseRefName == "main")
+      | "\(.number) \(.author.login) \($ARGS.named.file)"'; then
+    echo >&2 "Couldn't get PR for file $file"
+    exit 1
+  fi
+done |
+  sort -n |
+  while read -r number author file; do
+    # Replace the first line `# <title>` by `- <title> by @author in #number`
+    # All other non-empty lines are indented with 2 spaces to make the markdown formatting work
+    sed "$file" \
+      -e "1s|#[[:space:]]\(.*\)|- \1 by [@$author](https://github.com/$author) in [#$number](https://github.com/NixOS/nixpkgs-check-by-name/pull/$number)|" \
+      -e '2,$s/^\(.\)/  \1/' >> "$releaseFile"
+
+    rm "$file"
+  done
+
+cargo set-version --manifest-path "$root"/Cargo.toml "$next"
+echo "$next"