diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 36d9200..d2db5ef 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -3,6 +3,23 @@ on:
   pull_request:
     branches:
       - main
+  push:
+    branches:
+      - main
+
+# This makes sure to only run one instance of this workflow per individual PR and pushes.
+# We need this especially for the release step, because it has side effects.
+# But it's also good in general, there's no need to finish action runs for commits
+# when you push another one right after.
+# Assumption: We push less frequently than it takes for this workflow to finish
+concurrency:
+  # For a push to the main branch, github.ref is refs/heads/<branch_name>
+  # For a pull request, github.ref is refs/pull/<pr_number>/merge
+  group: ${{ github.ref }}
+  # We don't want to cancel in progress workflows for pushes to the main branch,
+  # because it could be in the process of doing a release.
+  # Better wait until it's done
+  cancel-in-progress: ${{ github.event_name != 'pull' }}
 
 jobs:
   build:
@@ -15,6 +32,12 @@ jobs:
       - name: build
         run: nix-build -A ci
 
+      - name: release
+        if: ${{ github.event_name == 'push' }}
+        run: scripts/release.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+
   test-update:
     runs-on: ubuntu-latest
     steps:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6b78354..0104a85 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,43 +1,25 @@
-name: CD
 on:
-  push:
-    branches:
-      - main
-
-jobs:
   release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
+    types: [published]
 
-      - uses: cachix/install-nix-action@v26
-
-      - name: release
-        run: scripts/release.sh
-        env:
-          GH_TOKEN: ${{ github.token }}
-
-  nixpkgs-pr:
+jobs:
+  create-pr:
     runs-on: ubuntu-latest
-    needs: release
     steps:
       - uses: actions/checkout@v4
         with:
           repository: NixOS/nixpkgs
 
-      - name: Get the Nixpkgs rev
-        run: echo "rev=$(git rev-parse HEAD)" >> $GITHUB_ENV
-
-      - name: Get version before
-        run: echo "before=$(<pkgs/test/check-by-name/pinned-version.txt)" >> $GITHUB_ENV
+      - name: Update nixpkgs-check-by-name in Nixpkgs
+        run: |
+          echo "rev=$(git rev-parse HEAD)" >> $GITHUB_ENV
+          echo "before=$(<pkgs/test/check-by-name/pinned-version.txt)" >> $GITHUB_ENV
 
-      - name: Update the version
-        # We run bash explicitly instead of using the files shebang,
-        # because the shebang is using nix-shell with dependencies we already have by default
-        run: bash pkgs/test/check-by-name/update-pinned-tool.sh
+          # We run bash explicitly instead of using the files shebang,
+          # because the shebang is using nix-shell with dependencies we already have by default
+          bash pkgs/test/check-by-name/update-pinned-tool.sh
 
-      - name: Get version after
-        run: echo "after=$(<pkgs/test/check-by-name/pinned-version.txt)" >> $GITHUB_ENV
+          echo "after=$(<pkgs/test/check-by-name/pinned-version.txt)" >> $GITHUB_ENV
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
diff --git a/scripts/release.sh b/scripts/release.sh
index af6813d..d0c4a0a 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -13,6 +13,7 @@ nixeval() {
 # The system to pre-build the release for and distribute artifacts for
 system=x86_64-linux
 root=$(git rev-parse --show-toplevel)
+rev=$(git -C "$root" rev-parse HEAD)
 repository=${GITHUB_REPOSITORY:-NixOS/nixpkgs-check-by-name}
 
 # Get the version from the Cargo.toml file
@@ -55,6 +56,7 @@ if ! release=$(gh api \
   -H "X-GitHub-Api-Version: 2022-11-28" \
   /repos/"$repository"/releases \
   -f tag_name="$version" \
+  -f target_commitish="$rev" \
   -f name="Version $version" \
   -f body="$body" \
   -F draft=true); then