From 7a014b3d693e2ac33a15b75151ea7e9a19687077 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Fri, 5 Apr 2024 00:18:22 +0200
Subject: [PATCH] WIP

---
 .github/workflows/main.yml |  50 +++++++
 scripts/sync-pr.sh         | 263 +++++++++++++++++++++++++++++++++++++
 2 files changed, 313 insertions(+)
 create mode 100755 scripts/sync-pr.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c9b49f81..12c3ac60 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -10,6 +10,10 @@ on:
     branches:
       - master
 
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check:
     runs-on: ubuntu-latest
@@ -33,3 +37,49 @@ jobs:
 
       - name: run tests
         run: nix-shell --run ./test/test.sh
+
+  auto-format:
+    needs: check
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - name: Find Comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Nixpkgs diff
+
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          edit-mode: replace
+          body: |
+            Nixpkgs diff [processing](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})..
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          # required
+          app-id: ${{ vars.app_id }}
+          private-key: ${{ secrets.app_private_key }}
+      - uses: cachix/install-nix-action@v20
+      - uses: actions/checkout@v4
+
+      - run: |
+          ./sync-pr.sh \
+            https://github.com/${{ github.repository }} \
+            ${{ github.event.pull_request.number }} \
+            https://token:${{ steps.app-token.outputs.token }}@github.com/${{ github.repository_owner }}/nixpkgs \
+
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          edit-mode: replace
+          body: |
+            [Nixpkgs diff](https://github.com/${{ github.repository_owner }}/nixpkgs/commits/nixfmt-${{ github.event.pull_request.number }})
diff --git a/scripts/sync-pr.sh b/scripts/sync-pr.sh
new file mode 100755
index 00000000..9cb90309
--- /dev/null
+++ b/scripts/sync-pr.sh
@@ -0,0 +1,263 @@
+#!/usr/bin/env bash
+# This is a fairly intricate script to format Nixpkgs for each commit of a nixfmt PR, pushing the result to a Git repo.
+# Notable features:
+# - It only minimally depends on GitHub: All operations are done using Git directly
+# - Reuse of previous work: Formatting all of Nixpkgs takes some time, we don't want to recompute it if not necessary
+# - Handles force pushes gracefully and linearises merge commits
+#
+
+set -euo pipefail
+
+if (( $# < 3 )); then
+  echo "Usage: $0 NIXFMT_URL NIXFMT_PR_NUMBER NIXPKGS_URL"
+  echo "- NIXFMT_URL: A git remote URL for the nixfmt repo against which the PR is made"
+  echo "- NIXFMT_PR_NUMBER: The PR number"
+  echo "- NIXPKGS_URL: A writable git remote URL for the Nixpkgs repo where branches can be created for the formatted result."
+  echo "  The branch will be called \`nixfmt-<NIXFMT_PR_NUMBER>\`"
+  exit 1
+fi
+
+nixfmtUrl=$1
+nixfmtPrNumber=$2
+nixpkgsUrl=$3
+
+nixpkgsUpstreamUrl=https://github.com/NixOS/nixpkgs
+nixpkgsMirrorBranch=nixfmt-$nixfmtPrNumber
+
+tmp=$(mktemp -d)
+cd "$tmp"
+trap 'rm -rf "$tmp"' exit
+
+
+step() {
+  echo -e "\e[34m$1\e[0m"
+}
+
+isLinear() {
+  local repo=$1
+  local revs=$2
+  for _mergeCommit in $(git 2>/dev/null -C "$repo" log --pretty=format:%H --min-parents=2 "$revs"); do
+    return 1
+  done
+}
+
+
+
+step "Fetching nixfmt pull request and creating a branch for the head commit"
+git init nixfmt
+git -C nixfmt fetch "$nixfmtUrl" "refs/pull/$nixfmtPrNumber/merge"
+nixfmtBaseCommit=$(git -C nixfmt rev-parse FETCH_HEAD^1)
+nixfmtHeadCommit=$(git -C nixfmt rev-parse FETCH_HEAD^2)
+git -C nixfmt switch -c main "$nixfmtHeadCommit"
+
+step "Linearising nixfmt history after the base commit"
+# https://stackoverflow.com/a/17994534
+FILTER_BRANCH_SQUELCH_WARNING=1 git -C nixfmt filter-branch --parent-filter 'cut -f 2,3 -d " "' --msg-filter 'echo $GIT_COMMIT' "$nixfmtBaseCommit"..main
+
+nixfmtCommitCount=$(git -C nixfmt rev-list --count "$nixfmtBaseCommit"..main)
+if (( nixfmtCommitCount == 0 )); then
+  step "No commits, deleting the nixpkgs branch $nixpkgsMirrorBranch if it exists"
+  # git push requires a repository to work at all, _any_ repository
+  git init -q trash
+  git -C trash push "$nixpkgsUrl" :refs/heads/"$nixpkgsMirrorBranch"
+  rm -rf trash
+  exit 0
+else
+  echo "There are $nixfmtCommitCount linearised commits"
+fi
+
+commitsToMirror=("$nixfmtBaseCommit")
+readarray -t -O 1 commitsToMirror < <(git -C nixfmt rev-list --reverse "$nixfmtBaseCommit"..main)
+
+# Computes the commit subject of the Nixpkgs commit that contains the formatted changes
+# Usage: bodyForCommit INDEX
+# - INDEX: The index of the nixfmt commit in the PR that is being used
+#   0 means the PR's base commit
+bodyForCommit() {
+  local index=$1
+  local commit=${commitsToMirror[$index]}
+  local url=$nixfmtUrl/commit/$commit
+  local subject
+  subject=$(git -C nixfmt show -s --format=%s "$commit")
+
+  if (( index == 0 )); then
+    url=$nixfmtUrl/commit/$commit
+    echo -e "base: $subject\n\nFormat using the base commit from nixfmt PR $nixfmtPrNumber: $url"
+  else
+    url=$nixfmtUrl/pull/$nixfmtPrNumber/commits/$commit
+    echo -e "$index: $subject\n\nFormat using commit number $index from nixfmt PR $nixfmtPrNumber: $url"
+  fi
+}
+
+step "Fetching upstream Nixpkgs commit history"
+git init --bare nixpkgs.git
+
+git -C nixpkgs.git remote add upstream "$nixpkgsUpstreamUrl"
+git -C nixpkgs.git config remote.upstream.promisor true
+git -C nixpkgs.git config remote.upstream.partialclonefilter tree:0
+
+git -C nixpkgs.git fetch --no-tags upstream HEAD:master
+
+step "Finding the last Nixpkgs commit before the first commit on nixfmt's branch"
+nixfmtFirstCommit=${commitsToMirror[1]}
+# Commit date, not author date, not sure what's better
+nixfmtFirstCommitDateEpoch=$(git -C nixfmt log -1 --format=%ct "$nixfmtFirstCommit")
+nixfmtFirstCommitDateHuman=$(git -C nixfmt log -1 --format=%ci "$nixfmtFirstCommit")
+echo "The first nixfmt commit is $nixfmtFirstCommit on $nixfmtFirstCommitDateHuman"
+
+nixpkgsBaseCommit=$(git -C nixpkgs.git rev-list -1 master --before="$nixfmtFirstCommitDateEpoch")
+nixpkgsBaseCommitDateHuman=$(git -C nixpkgs.git log -1 --format=%ci "$nixpkgsBaseCommit")
+
+echo "The last Nixpkgs commit before then is $nixpkgsBaseCommit on $nixpkgsBaseCommitDateHuman, which will be used as the Nixpkgs base commit"
+
+step "Fetching mirror Nixpkgs commit history in branch $nixpkgsMirrorBranch if it exists"
+git -C nixpkgs.git remote add mirror "$nixpkgsUrl"
+git -C nixpkgs.git config remote.mirror.promisor true
+git -C nixpkgs.git config remote.mirror.partialclonefilter tree:0
+
+# After this:
+# - $startingCommit should be the nixpkgs commit that the branch should be reset to
+# - $extraCommits should be the number of commits already processed
+if ! git -C nixpkgs.git fetch --no-tags mirror "$nixpkgsMirrorBranch":mirrorBranch; then
+  echo "There is not"
+  startingCommit=$nixpkgsBaseCommit
+  extraCommits=0
+else
+  echo "There is, it points to $(git -C nixpkgs.git rev-parse mirrorBranch)"
+  step "Checking to which extent work from the existing branch can be reused"
+  if [[ -z "$(git -C nixpkgs.git branch --contains="$nixpkgsBaseCommit" mirrorBranch)" ]]; then
+    echo "It is not"
+    startingCommit=$nixpkgsBaseCommit
+    extraCommits=0
+  else
+    echo "It is!"
+    echo "Checking if the branch has a linear history"
+    if ! isLinear nixpkgs.git "$nixpkgsBaseCommit"..mirrorBranch; then
+      echo "It is not linear, resetting the branch"
+      startingCommit=$nixpkgsBaseCommit
+      extraCommits=0
+    else
+      echo "It is linear!"
+      nixpkgsCount=$(git -C nixpkgs.git rev-list --count "$nixpkgsBaseCommit"..mirrorBranch)
+      echo "There's $nixpkgsCount commits in the branch on top of the base commit"
+      extraCommits=0
+      # Check if there's at least 1 commits in nixpkgs and at least 0 commits in nixfmt
+      # Check if commit 1 in nixpkgs corresponds to commit 0 in nixfmt
+      # If true, increase extraCommits by one, otherwise break
+      # Check if there's at least 2 commits in nixpkgs and at least 1 commits in nixfmt
+      # If so, check if commit 2 in nixpkgs corresponds to commit 1 in nixfmt
+      # ...
+      while
+        if (( nixpkgsCount >= extraCommits + 1 && nixfmtCommitCount >= extraCommits)); then
+          echo "Checking whether commit with index $(( extraCommits + 1 )) in nixpkgs corresponds to commit with index $extraCommits in nixfmt"
+          nixpkgsCommit=$(git -C nixpkgs.git rev-parse "mirrorBranch~$((nixpkgsCount - (extraCommits + 1)))")
+          body=$(git -C nixpkgs.git log -1 "$nixpkgsCommit" --pretty=%B)
+          expectedBody=$(bodyForCommit "$extraCommits")
+          if [[ "$body" == "$expectedBody" ]]; then
+            echo "It does!"
+          else
+            echo "It does not, body of nixpkgs commit $nixpkgsCommit is"
+            echo "$body"
+            echo "But expected body is"
+            echo "$expectedBody"
+            false
+          fi
+        else
+          false
+        fi
+      do
+        extraCommits=$(( extraCommits + 1 ))
+      done
+
+      nixpkgsCommit=$(git -C nixpkgs.git rev-parse "mirrorBranch~$(( nixpkgsCount - extraCommits ))")
+      startingCommit="$nixpkgsCommit"
+    fi
+  fi
+fi
+
+echo "Starting commit is $startingCommit, extraCommits is $extraCommits"
+
+step "Fetching contents of Nixpkgs base commit $nixpkgsBaseCommit"
+git init nixpkgs
+git -C nixpkgs fetch --no-tags --depth 1 "$nixpkgsUpstreamUrl" "$nixpkgsBaseCommit":base
+
+step "Fetching contents of the starting commit and updating the mirror branch"
+if (( extraCommits == 0 )); then
+  git -C nixpkgs switch -c mirrorBranch "$startingCommit"
+else
+  git -C nixpkgs fetch --no-tags --depth 1 "$nixpkgsUrl" "$startingCommit":mirrorBranch
+  git -C nixpkgs switch mirrorBranch
+fi
+
+git -C nixpkgs push --force "$nixpkgsUrl" mirrorBranch:"$nixpkgsMirrorBranch"
+
+if (( extraCommits == 0 )); then
+  index=0
+else
+  index=$(( extraCommits - 1 ))
+fi
+
+if (( index == nixfmtCommitCount )); then
+  echo "Nothing to do"
+  exit 0
+fi
+
+git -C nixpkgs config user.name "GitHub Actions"
+git -C nixpkgs config user.email "actions@users.noreply.github.com"
+
+updateToIndex() {
+  nixfmtCommit=${commitsToMirror[$index]}
+
+  step "Checking out nixfmt at $nixfmtCommit"
+  git -C nixfmt checkout -q "$nixfmtCommit"
+
+  step "Building nixfmt"
+  #nix build ./nixfmt
+  nix-build nixfmt -A build
+}
+
+applyNixfmt() {
+  step "Checking out Nixpkgs at the base commit"
+  git -C nixpkgs checkout "$nixpkgsBaseCommit" -- .
+
+  step "Running nixfmt on nixpkgs"
+  if ! xargs -r -0 -P"$(nproc)" -n1 -a <(find nixpkgs -type f -name '*.nix' -print0) result/bin/nixfmt; then
+    echo -e "\e[31mFailed to run nixfmt on some files\e[0m"
+    exit 1
+  fi
+}
+
+commitResult() {
+  step "Committing the formatted result"
+  git -C nixpkgs add -A
+  git -C nixpkgs commit --allow-empty -m "$(bodyForCommit "$index")"
+
+  step "Pushing result"
+  git -C nixpkgs push "$nixpkgsUrl" mirrorBranch:"$nixpkgsMirrorBranch"
+}
+
+
+updateToIndex
+
+appliedNixfmtPath=$(realpath result)
+
+if (( extraCommits == 0 )); then
+  applyNixfmt
+  commitResult
+fi
+
+while (( index != nixfmtCommitCount )); do
+  index=$(( index + 1 ))
+
+  updateToIndex
+
+  step "Formatting nixpkgs"
+  if [[ "$appliedNixfmtPath" != "$(realpath result)" ]]; then
+    applyNixfmt
+    commitResult
+    appliedNixfmtPath=$(realpath result)
+  else
+    echo "The nixfmt store path didn't change, saving ourselves a formatting"
+    commitResult
+  fi
+done