Merge BlockstreamResearch/secp256k1-zkp#360: sync-upstream: Simplify

c5be752e2e gha: Simplify sync.yml according to sync-upstream.sh changes (Tim Ruffing)
260fad30aa sync-upstream: Optionally switch to sync branch (Tim Ruffing)
f0ff45f3ab sync-upstream: Simplify (Tim Ruffing)

Pull request description:

  This is a half-rewrite of the sync-upstream script because I still wasn't happy with it. Looking for early feedback on this.

  Advantages:
   - The script does not hardcode any repo URLs, branches, etc.
   - Nicer separation of responsibilities between the script and the GHA workflow. This brings back the usefulness of the script when used locally outside GHA.
   - Much simpler code, and we could simplify also the GHA workflow.

  Still to do:
   - Rewrite the "Tips" section in the generated PR description, e.g., add how to resolve merge conflicts.
   - Change the GHA workflow accordingly.

ACKs for top commit:
  mllwchrry:
    ACK c5be752

Tree-SHA512: 7335b4690642c07965366da952a74065ec404f750e0c6c7e735a9a48285cbe6b9c7db27ff4bdeba1d5ce59717d77be1ecb2334200b56a1a0a2c0fcddc7530d0b

.github/workflows/sync.yml

@@ -9,11 +9,10 @@ jobs:
   sync-upstream:
     runs-on: ubuntu-latest
     env:
-      UPSTREAM: "https://github.com/bitcoin-core/secp256k1.git" 
+      UPSTREAM: "https://github.com/bitcoin-core/secp256k1.git"
-      UPSTREAM_BRANCH: "master"
+      BASE_BRANCH: "master"
-      ORIGIN_BRANCH: "master"
+      UPSTREAM_REF: "upstream/master"
-      MIN_UPSTREAM_MERGES: 1
+
-    
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -26,66 +25,37 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
 
-      - name: Fetch upstream & generate branch name & check for new commits
+      - name: Fetch upstream
-        id: check_commits
-        env:
-          GH_TOKEN: ${{ secrets.SYNC_PAT }}
         run: |
-          gh repo set-default ${{ github.repository }}  # Set the default repo to the origin repository
           git remote add upstream ${{ env.UPSTREAM }}
           git fetch upstream
-          
+          gh repo set-default ${{ github.repository }}
-          MERGES=$(git rev-list --count --merges HEAD..upstream/${{ env.UPSTREAM_BRANCH }})
-          echo "Found $MERGES new merge commits in upstream."
-          
-          if [ "$MERGES" -lt ${{ env.MIN_UPSTREAM_MERGES }} ]; then
-            echo "Exiting."
-            echo "skip=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "skip=false" >> "$GITHUB_OUTPUT"
-          fi
-          
-          # Generate a sync branch name "sync-UPSTREAM_HEAD", where UPSTREAM_HEAD is a commit ID.
-          UPSTREAM_HEAD=$(git rev-parse --short upstream/${{ env.UPSTREAM_BRANCH }})
-          SYNC_BRANCH="sync-$UPSTREAM_HEAD"
-          echo "Sync branch name: $SYNC_BRANCH"
-          echo "SYNC_BRANCH=$SYNC_BRANCH" >> "$GITHUB_ENV"
-
-          # Check if the sync branch already exists in the origin repository
-          if git ls-remote --heads origin "$SYNC_BRANCH" | grep -q "$SYNC_BRANCH"; then
-            echo "Branch $SYNC_BRANCH already exists. Skipping the sync."
-            echo "skip=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "skip=false" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Generate PR metadata
-        id: branch_pr_metadata
-        if: steps.check_commits.outputs.skip == 'false'
-        run: |
-          # Call the sync script to generate PR title and body
-          ./contrib/sync-upstream.sh -b ${{ env.ORIGIN_BRANCH }} "$SYNC_BRANCH"
-
-      - name: Push a sync branch
-        id: push_branch
-        if: steps.check_commits.outputs.skip == 'false'
-        run: |
-          echo "Creating sync branch: $SYNC_BRANCH"
-          git checkout upstream/${{ env.UPSTREAM_BRANCH }}
-          git checkout -b "$SYNC_BRANCH"
-          git push -u origin "$SYNC_BRANCH"
-
-      - name: Create pull request
-        id: create_pr
-        if: steps.check_commits.outputs.skip == 'false'
         env:
           GH_TOKEN: ${{ secrets.SYNC_PAT }}
-        run: |
-          # Execute the generated PR creation script
-          ./contrib/gh-pr-create.sh 
 
-      - name: Cleanup sync branch on failure
+      - name: Run sync-upstream.sh
-        if: steps.push_branch.outcome == 'success' && steps.create_pr.outcome == 'failure'
+        id: sync
         run: |
-          echo "PR creation failed but branch was pushed. Deleting: $SYNC_BRANCH"
+          OUTPUT=$(./contrib/sync-upstream.sh --switch "${{ env.BASE_BRANCH }}" "${{ env.UPSTREAM_REF }}" 2>&1) || { echo "$OUTPUT"; exit 1; }
-          git push origin --delete "$SYNC_BRANCH"
+          echo "$OUTPUT"
+          if echo "$OUTPUT" | grep -qv "^No merge commits"; then
+            echo "newcommits=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "Skipping further workflow steps."
+          fi
+
+      - name: Push sync branch
+        id: push
+        if: steps.sync.outputs.newcommits == 'true'
+        run: |
+          if git push -u origin HEAD; then
+            echo "pushed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "Skipping further workflow steps."
+          fi
+
+      - name: Create pull request
+        if: steps.sync.outputs.newcommits == 'true' && steps.push.outputs.pushed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.SYNC_PAT }}
+        run: ./gh-pr-create.sh

contrib/sync-upstream.sh

@@ -3,112 +3,121 @@
 set -eou pipefail
 
 help() {
-    echo "Sync merge commits from bitcoin-core/secp256k1 into secp256k1-zkp."
+cat <<EOT
-    echo
+$0: Prepare a pull request that syncs a branch with upstream
-    echo "Usage:"
+
-    echo "  $0 [-b <branch>] <pr_branch>"
+Usage:
-    echo "      Find every merge commit present in upstream/master and missing in <branch> (default: master)."
+  $0 [--switch] <base-branch> <upstream-ref>
-    echo
+
-    echo "This tool prepares the title and body for a sync PR"
+This script creates a sync local branch pointing to <upstream-ref>. Moreover, it
-    echo "and generates a helper script contrib/gh-pr-create.sh." 
+generates a helper script for opening a pull request (PR) merging the created
-    echo
+local branch into <base-branch>.
-    echo "Setup:"
+
-    echo "  Requires a remote named 'upstream' pointing to bitcoin-core/secp256k1."
+The synced upstream PRs are listed in the title and the description of the PR.
-    echo
+(This relies on upstream merging PRs using merge commits with titles of the form
-    echo "Listing upstream merge commits:"
+"Merge <repo>#<prnum>: ...".)
-    echo "  To list merge commits in upstream/master that are missing from <branch> (oldest first):"
+
-    echo "    git log --oneline --topo-order --reverse --merges \$(git merge-base upstream/master <branch>)..upstream/master"
+Arguments:
-    exit 1
+  --switch:        Try to switch to the created sync branch
+  <base-branch>:   The branch to sync with upstream
+  <upstream-ref>:  The upstream ref to merge into <base-branch>
+
+Usage examples:
+  $0 --switch master upstream/master
+  $0 master abc1234
+
+To find candidate merge commits from <upstream-ref> (oldest first), use:
+  git log --oneline --topo-order --reverse --merges \$(git merge-base <upstream-ref> <base-branch>)..<upstream-ref>
+EOT
 }
 
-REMOTE=upstream
+### Parse arguments
-REMOTE_BRANCH="$REMOTE/master"
+SWITCH=false
-LOCAL_BRANCH="master"
+if [ "$#" -ge 1 ] && [ "$1" = "--switch" ]; then
-
+    SWITCH=true
-if ! git remote get-url "$REMOTE" &> /dev/null; then
+    shift
-echo "Error: Remote '$REMOTE' not found."
-echo "Add it with: git remote add upstream git@github.com:bitcoin-core/secp256k1.git"
-echo "Then run: git fetch upstream"
-exit 1
 fi
-
+if [ "$#" -ne 2 ]; then
-range() {
-    RANGESTART_COMMIT=$(git merge-base "$REMOTE_BRANCH" "$LOCAL_BRANCH")
-    RANGEEND_COMMIT=$(git rev-parse "$REMOTE_BRANCH")
-    COMMITS=$(git --no-pager log --pretty=format:%H --topo-order --reverse --merges "$RANGESTART_COMMIT".."$RANGEEND_COMMIT")
-}
-
-# Process -b <branch> and -h arguments
-while getopts "b:h" opt; do
-  case $opt in
-    b)
-      LOCAL_BRANCH=$OPTARG
-      ;;
-    h)
-      help
-      ;;
-    *)
-      echo
-      help
-      ;;
-  esac
-done
-
-# Shift off the processed options
-shift $((OPTIND -1))
-if [ "$#" -lt 1 ]; then
-    echo "Error: <pr_branch> argument is required." >&2
-    echo
     help
     exit 1
 fi
+BASE_BRANCH="$1"
+UPSTREAM_REF="$2"
 
-# Extract the PR branch argument
+### Create PR metadata
-PR_BRANCH=$1          
-
-range
-
 TITLE="Upstream PRs"
-BODY="${GITHUB_ACTIONS+This PR has been created by a GitHub Actions workflow without human involvement.}"$'\n'
+RANGESTART_COMMIT=$(git merge-base "$UPSTREAM_REF" "$BASE_BRANCH")
-for COMMIT in $COMMITS
+RANGEEND_COMMIT=$(git rev-parse "$UPSTREAM_REF")
-do
+COMMITS=$(git --no-pager log --pretty=format:%H --topo-order --reverse --merges "$RANGESTART_COMMIT".."$RANGEEND_COMMIT")
-    PRNUM=$(git log -1 "$COMMIT" --pretty=format:%s | sed s/'Merge \(bitcoin-core\/secp256k1\)\?#\([0-9]*\).*'/'\2'/)
+# If there are no commits, exit successfully
+if [ -z "$COMMITS" ]; then
+  echo "No merge commits in range ${RANGESTART_COMMIT}..${RANGEEND_COMMIT}" >&2
+  exit 0
+fi
+BODY="${GITHUB_ACTIONS+*Note: This PR has been created by a GitHub Actions workflow without human involvement.*
+
+}"
+BODY+="This PR syncs the following upstream PRs:"
+for COMMIT in $COMMITS; do
+    PRNUM=$(git log -1 "$COMMIT" --pretty=format:%s | sed s/'Merge .*#\([0-9]*\):.*'/'\1'/)
     TITLE="$TITLE $PRNUM,"
-    BODY=$(printf "%s\n%s" "$BODY" "$(git log -1 "$COMMIT" --pretty=format:%s | sed s/'Merge \(bitcoin-core\/secp256k1\)\?#\([0-9]*\)'/'[bitcoin-core\/secp256k1#\2]'/)")
+    BODY=$(printf "%s\n * %s" "$BODY" "$(git log -1 "$COMMIT" --pretty=format:%s | sed s/'Merge '//)")
-    LAST_COMMIT="$COMMIT"
 done
 # Remove trailing ","
 TITLE=${TITLE%?}
-BODY+=$(cat <<EOF
+BODY+=$(cat <<'EOF'
 
 
-Tips:
+Usage hints:
- * Use \`git show --remerge-diff <pr-branch>\` to show the conflict resolution in the merge commit.
+ * If this PR has merge conflicts, resolve these by switching to the PR branch and merging the base branch into it using `git merge <base-branch>`.
- * Use \`git read-tree --reset -u <pr-branch>\` to replay these resolutions during the conflict resolution stage when recreating the PR branch locally.
+ * To show the conflict resolution diff from an existing merge commit, use `git show --remerge-diff <merge-commit>`.
+ * In case you are recreating the PR branch locally, you can (during the conflict resolution state) replay this conflict resolution diff using `git read-tree --reset -u <merge-commit>`.
    Be aware that this may discard your index as well as the uncommitted changes and untracked files in your worktree.
 EOF
 )
 
+### Create a sync branch locally.
+SYNC_BRANCH="sync-$(git rev-parse --short "$UPSTREAM_REF")"
+# This will error out if the branch already exists, which is what we want.
+git branch --no-track "$SYNC_BRANCH" "$UPSTREAM_REF"
+
+### Print the PR metadata
 echo "-----------------------------------"
 echo "$TITLE"
 echo "-----------------------------------"
 echo "$BODY"
 echo "-----------------------------------"
 
-# Escape single quote
+### Generate the helper script for creating the PR
-# ' -> '\''
+FNAME="gh-pr-create.sh"
+# Escape single quote ' -> '\''
 quote() {
     local quoted=${1//\'/\'\\\'\'}
     printf "%s" "$quoted"
 }
 TITLE=$(quote "$TITLE")
 BODY=$(quote "$BODY")
-
-BASEDIR=$(dirname "$0")
-FNAME="$BASEDIR/gh-pr-create.sh"
 cat <<EOT > "$FNAME"
 #!/bin/sh
-gh pr create -t '$TITLE' -b '$BODY' --base '$LOCAL_BRANCH' --head '$PR_BRANCH'
+TITLE='$TITLE'
+BODY='$BODY'
+SYNC_BRANCH='$SYNC_BRANCH'
+BASE_BRANCH='$BASE_BRANCH'
+
+gh pr create --base "\$BASE_BRANCH" --head "\$SYNC_BRANCH" --title "\$TITLE" --body "\$BODY" "\$@"
 EOT
 chmod +x "$FNAME"
-echo "Generated $FNAME for creating a pull request with the above title and body."
+
+echo "Successfully created local sync branch $SYNC_BRANCH starting at $UPSTREAM_REF."
+echo
+echo "You can now:"
+echo "  1. Optionally resolve merge conflicts by merging $BASE_BRANCH into $SYNC_BRANCH."
+echo "  2. Push $SYNC_BRANCH to some GitHub remote."
+echo "  3. Run ./$FNAME to create a pull request. (Tip: Pass --dry-run first.)"
+
+if [ "${SWITCH:-false}" = true ]; then
+    echo
+    echo "Trying to switch to the sync branch..."
+    echo
+    git switch "$SYNC_BRANCH"
+fi