diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fbf9e890..bc306bdc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,10 @@ on: branches: - master +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + jobs: check: runs-on: ubuntu-latest @@ -27,3 +31,45 @@ jobs: - name: run tests run: nix-shell --run ./test/test.sh + + nixpkgs-diff: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Find Comment + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Nixpkgs diff + + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + edit-mode: replace + body: | + Nixpkgs diff [processing](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).. + + Will be available [here](https://github.com/${{ vars.MACHINE_USER }}/nixpkgs/commits/nixfmt-${{ github.event.pull_request.number }}) + + - uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v26 + + - run: | + ./scripts/sync-pr.sh \ + https://github.com/${{ github.repository }} \ + ${{ github.event.pull_request.number }} \ + https://${{ secrets.MACHINE_USER_PAT }}@github.com/${{ vars.MACHINE_USER }}/nixpkgs + + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + edit-mode: replace + body: | + [Nixpkgs diff](https://github.com/${{ vars.MACHINE_USER }}/nixpkgs/commits/nixfmt-${{ github.event.pull_request.number }}) diff --git a/scripts/sync-pr.sh b/scripts/sync-pr.sh new file mode 100755 index 00000000..72a141b2 --- /dev/null +++ b/scripts/sync-pr.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash +# This is a fairly intricate script to format Nixpkgs for each commit of a nixfmt PR, pushing the result to a Git repo. +# Notable features: +# - It only minimally depends on GitHub: All operations are done using Git directly +# - Reuse of previous work: Formatting all of Nixpkgs takes some time, we don't want to recompute it if not necessary +# - Handles force pushes gracefully and linearises merge commits +# + +set -euo pipefail + +if (( $# < 3 )); then + echo "Usage: $0 NIXFMT_URL NIXFMT_PR_NUMBER NIXPKGS_URL" + echo "- NIXFMT_URL: A git remote URL for the nixfmt repo against which the PR is made" + echo "- NIXFMT_PR_NUMBER: The PR number" + echo "- NIXPKGS_URL: A writable git remote URL for the Nixpkgs repo where branches can be created for the formatted result." + echo " The branch will be called \`nixfmt-\`" + exit 1 +fi + +nixfmtUrl=$1 +nixfmtPrNumber=$2 +nixpkgsUrl=$3 + +nixpkgsUpstreamUrl=https://github.com/NixOS/nixpkgs +nixpkgsMirrorBranch=nixfmt-$nixfmtPrNumber + +tmp=$(mktemp -d) +cd "$tmp" +trap 'rm -rf "$tmp"' exit + +step() { + echo -e "\e[34m$1\e[0m" +} + +# Checks whether a revision range is linear, returns 1 if it's not +# Usage: isLinear REPO REV_RANGE +# - REPO: The local Git repo that contains the revisions +# - REV_RANGE: The revision range, see `man git log` +isLinear() { + local repo=$1 + local revs=$2 + # Loops through all merge commits in the range + for _mergeCommit in $(git 2>/dev/null -C "$repo" log --pretty=format:%H --min-parents=2 "$revs"); do + # And returns as soon as the first is found + return 1 + done +} + +step "Fetching nixfmt pull request and creating a branch for the head commit" +git init nixfmt +git -C nixfmt fetch "$nixfmtUrl" "refs/pull/$nixfmtPrNumber/merge" +nixfmtBaseCommit=$(git -C nixfmt rev-parse FETCH_HEAD^1) +nixfmtHeadCommit=$(git -C nixfmt rev-parse FETCH_HEAD^2) +git -C nixfmt switch -c main "$nixfmtHeadCommit" + +# In case the PR contains merge commits, we strip those away, such that the resulting Nixpkgs history is always linear. +step "Linearising nixfmt history after the base commit" +# https://stackoverflow.com/a/17994534 +FILTER_BRANCH_SQUELCH_WARNING=1 git -C nixfmt filter-branch --parent-filter 'cut -f 2,3 -d " "' "$nixfmtBaseCommit"..main + +nixfmtCommitCount=$(git -C nixfmt rev-list --count "$nixfmtBaseCommit"..main) +if (( nixfmtCommitCount == 0 )); then + step "No commits, deleting the nixpkgs branch $nixpkgsMirrorBranch if it exists" + # git push requires a repository to work at all, _any_ repository! + git init -q trash + git -C trash push "$nixpkgsUrl" :refs/heads/"$nixpkgsMirrorBranch" + rm -rf trash + exit 0 +else + echo "There are $nixfmtCommitCount linearised commits" +fi + +# All the commits of the PR, including the base commit (which may change as the base branch is updated) +prCommits=("$nixfmtBaseCommit") +readarray -t -O 1 prCommits < <(git -C nixfmt rev-list --reverse "$nixfmtBaseCommit"..main) + +# Computes the commit subject of the Nixpkgs commit that contains the formatted changes +# Usage: bodyForCommitIndex INDEX +# - INDEX: The index of the nixfmt commit in the PR that is being used +# 0 means the PR's base commit +bodyForCommitIndex() { + local index=$1 + local commit=${prCommits[$index]} + local url=$nixfmtUrl/commit/$commit + local subject + subject=$(git -C nixfmt show -s --format=%s "$commit") + + if (( index == 0 )); then + url=$nixfmtUrl/commit/$commit + echo -e "base: $subject\n\nFormat using the base commit from nixfmt PR $nixfmtPrNumber: $url" + else + url=$nixfmtUrl/pull/$nixfmtPrNumber/commits/$commit + echo -e "$index: $subject\n\nFormat using commit number $index from nixfmt PR $nixfmtPrNumber: $url" + fi +} + +step "Fetching upstream Nixpkgs commit history" +git init --bare nixpkgs.git + +git -C nixpkgs.git remote add upstream "$nixpkgsUpstreamUrl" +# This makes sure that we don't actually have to fetch any contents, otherwise we'd wait forever! +git -C nixpkgs.git config remote.upstream.promisor true +git -C nixpkgs.git config remote.upstream.partialclonefilter tree:0 + +git -C nixpkgs.git fetch --no-tags upstream HEAD:master + +# Instead of e.g. fetching Nixpkgs master, which would continuously move, we "pin" Nixpkgs to the latest commit before the PRs commit +step "Finding the last Nixpkgs commit before the first commit on nixfmt's branch" +nixfmtFirstCommit=${prCommits[1]} +# Commit date, not author date, not sure what's better +nixfmtFirstCommitDateEpoch=$(git -C nixfmt log -1 --format=%ct "$nixfmtFirstCommit") +nixfmtFirstCommitDateHuman=$(git -C nixfmt log -1 --format=%ci --date=iso-local "$nixfmtFirstCommit") +echo "The first nixfmt commit is $nixfmtFirstCommit on $nixfmtFirstCommitDateHuman" + +nixpkgsBaseCommit=$(git -C nixpkgs.git rev-list -1 master --before="$nixfmtFirstCommitDateEpoch") +nixpkgsBaseCommitDateHuman=$(git -C nixpkgs.git log -1 --format=%ci --date=iso-local "$nixpkgsBaseCommit") + +echo "The last Nixpkgs commit before then is $nixpkgsBaseCommit on $nixpkgsBaseCommitDateHuman, which will be used as the Nixpkgs base commit" + +step "Fetching Nixpkgs commit history in branch $nixpkgsMirrorBranch if it exists" +git -C nixpkgs.git remote add mirror "$nixpkgsUrl" +git -C nixpkgs.git config remote.mirror.promisor true +git -C nixpkgs.git config remote.mirror.partialclonefilter tree:0 + +# After this: +# - $nixpkgsCommitCount should be the number of commits that can be reused +# - $startingCommit should be the nixpkgs commit that the branch should be reset to +nixpkgsCommitCount=0 +startingCommit=$nixpkgsBaseCommit +if ! git -C nixpkgs.git fetch --no-tags mirror "$nixpkgsMirrorBranch":mirrorBranch; then + echo "There is not, likely a new PR" +else + echo "There is, it points to $(git -C nixpkgs.git rev-parse mirrorBranch)" + step "Checking to which extent work from the existing branch can be reused" + if [[ -z "$(git -C nixpkgs.git branch --contains="$nixpkgsBaseCommit" mirrorBranch)" ]]; then + echo "It cannot, the desired base commit is not present at all, likely caused by a rebase" + else + if ! isLinear nixpkgs.git "$nixpkgsBaseCommit"..mirrorBranch; then + echo "It cannot, the branch is not linear, this is a bug, but not fatal" + else + + # We need to figure out how many Nixpkgs commits can be reused, + # kind of like detecting when a fork happened, but across repos: + # + # These nixfmt commits are new + # ,-o-o-o <- and haven't been run on Nixpkgs yet (may be 0) + # base-o-o-o-o + # ^ `-o-o-o <- These Nixpkgs commits were formatted + # | with previous nixfmt commits (may be 0) + # these Nixpkgs These commits must be removed + # commits can be reused + + previousNixpkgsCommitCount=$(git -C nixpkgs.git rev-list --count "$nixpkgsBaseCommit"..mirrorBranch) + echo "There's $previousNixpkgsCommitCount commits in the branch on top of the Nixpkgs base commit" + # E.g. if the nixfmt PR has 1 commit, the Nixpkgs PR has two: + # One for the nixfmt base commit (on top of the Nixpkgs base commit) + # and one for the first nixfmt commit + + # Check if there's at least 1 commit in nixpkgs and at least 0 commits in nixfmt + # Check if commit 1 in nixpkgs corresponds to commit 0 in nixfmt + # If true, increase nixpkgsCommitCount by one, otherwise break + # Check if there's at least 2 commits in nixpkgs and at least 1 commit in nixfmt + # If so, check if commit 2 in nixpkgs corresponds to commit 1 in nixfmt + # ... + while + if (( nixpkgsCommitCount > nixfmtCommitCount )); then + echo "All nixfmt commits are already cached" + false + elif (( nixpkgsCommitCount + 1 > previousNixpkgsCommitCount )); then + echo "Not all nixfmt commits are cached" + false + else + # A bit messy, but we kind of go back from the head of the branch via parents. + # This only works correctly because we verified that the branch is linear. + # An alternative would be to read the commits into an array, just like it's done for prCommits + nixpkgsCommit=$(git -C nixpkgs.git rev-parse "mirrorBranch~$((previousNixpkgsCommitCount - (nixpkgsCommitCount + 1)))") + nixfmtCommit=${prCommits[$nixpkgsCommitCount]} + + echo "Checking whether commit with index $(( nixpkgsCommitCount + 1 )) ($nixpkgsCommit) in nixpkgs corresponds to commit with index $nixpkgsCommitCount ($nixfmtCommit) in nixfmt" + + # We generate the bodies of the commits to contain the nixfmt commit so we can check against it here to verify it's the same + body=$(git -C nixpkgs.git log -1 "$nixpkgsCommit" --pretty=%B) + expectedBody=$(bodyForCommitIndex "$nixpkgsCommitCount") + if [[ "$body" == "$expectedBody" ]]; then + echo "It does!" + else + echo "It does not, this indicates a force push was done" + false + fi + fi + do + nixpkgsCommitCount=$(( nixpkgsCommitCount + 1 )) + startingCommit=$(git -C nixpkgs.git rev-parse "mirrorBranch~$(( previousNixpkgsCommitCount - nixpkgsCommitCount ))") + done + + echo "$nixpkgsCommitCount commits can be reused, starting from $startingCommit" + fi + fi +fi + + +git init nixpkgs +git -C nixpkgs config user.name "GitHub Actions" +git -C nixpkgs config user.email "actions@users.noreply.github.com" + +step "Fetching contents of Nixpkgs base commit $nixpkgsBaseCommit" +# This is needed because for every commit we reset Nixpkgs to the base branch before formatting +git -C nixpkgs fetch --no-tags --depth 1 "$nixpkgsUpstreamUrl" "$nixpkgsBaseCommit" + +step "Fetching contents of the starting commit and updating the mirror branch" +# This is only needed so we can push the resulting diff after formatting +if (( nixpkgsCommitCount == 0 )); then + # No reusable commits, create a new branch, starting commit is the same as the base commit here + git -C nixpkgs switch -c mirrorBranch "$startingCommit" +else + # Reusable commits, fetch the starting one and set a local branch to it + git -C nixpkgs fetch --no-tags --depth 1 "$nixpkgsUrl" "$startingCommit":mirrorBranch + git -C nixpkgs switch mirrorBranch +fi + +git -C nixpkgs push --force "$nixpkgsUrl" mirrorBranch:"$nixpkgsMirrorBranch" + +if (( nixpkgsCommitCount - 1 == nixfmtCommitCount )); then + echo "Already up-to-date" + exit 0 +fi + +# Update the result symlink to a specific nixfmt version +# Usage: step INDEX +# - INDEX: The nixfmt commit index to use, 0 is for the PR's base commit +update() { + local index=$1 + nixfmtCommit=${prCommits[$index]} + + step "Checking out nixfmt at $nixfmtCommit" + git -C nixfmt checkout -q "$nixfmtCommit" + + step "Building nixfmt" + nix-build nixfmt +} + +# Format Nixpkgs with a specific nixfmt version and push the result. +# Usage: step INDEX +# - INDEX: The nixfmt commit index to format with, 0 is for the PR's base commit +next() { + local index=$1 + + update "$index" + + if [[ -n "$appliedNixfmtPath" && "$appliedNixfmtPath" == "$(realpath result)" ]]; then + echo "The nixfmt store path didn't change, saving ourselves a formatting" + else + step "Checking out Nixpkgs at the base commit" + git -C nixpkgs checkout "$nixpkgsBaseCommit" -- . + + step "Running nixfmt on nixpkgs" + if ! time xargs -r -0 -P"$(nproc)" -n1 -a <(find nixpkgs -type f -name '*.nix' -print0) result/bin/nixfmt; then + echo -e "\e[31mFailed to run nixfmt on some files\e[0m" + exit 1 + fi + git -C nixpkgs add -A + + appliedNixfmtPath=$(realpath result) + fi + + step "Committing the formatted result" + git -C nixpkgs commit --allow-empty -m "$(bodyForCommitIndex "$index")" + + step "Pushing result" + git -C nixpkgs push "$nixpkgsUrl" mirrorBranch:"$nixpkgsMirrorBranch" + nixpkgsCommitCount=$(( nixpkgsCommitCount + 1 )) +} + +appliedNixfmtPath= +if (( nixpkgsCommitCount == 0 )); then + # If we don't have a base-formatted Nixpkgs commit yet, create it + next 0 +else + # Otherwise, just build the nixfmt that was used for the current commit, such that we know the store path + update "$(( nixpkgsCommitCount - 1 ))" + appliedNixfmtPath=$(realpath result) +fi + +while (( nixpkgsCommitCount - 1 < nixfmtCommitCount )); do + # The number of commits in Nixpkgs is also the index of the nixfmt commit to apply next + # E.g. with 1 Nixpkgs commit (only the base formatted one), we need to run the 1st commit from the nixfmt PR next + next "$nixpkgsCommitCount" +done