From 93b31e5020c478ffa759df8be15a97436899061f Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:40:22 +0000 Subject: [PATCH 1/3] fork-follow: fetch the rebase's files first, name every stop (#475) The Plan job checks out with --filter=blob:none, so the rebase fetched each missing file from origin on first read. Git 2.56 (ubuntu-26.04 runners) asked, while applying a pick, for the blob that pick had just merged; origin has no such object ("not our ref"), so the pick died with a git error. There was no REBASE_HEAD and no unmerged file, so the report said "stopped at: unknown" with no reason. - Before the rebase, fetch in one batch every blob it reads that is missing here (each patch's sides, the series' own, the old base's side of upstream's changes). The promisor is extensions.partialclone or the remote marked promisor = true. The rebase then never fetches. - A stop names the patch from REBASE_HEAD, else the last pick in the done list, else says git did not begin; it says whether it was a conflict or a git error and quotes git's own output (last 20 lines, no progress lines). Tests (red on the old script, green now): a partial clone whose origin refuses fetches while a rebase runs follows; a pick stopped by a git error names the patch and quotes git; a rebase git refuses to begin says so; a conflict report quotes git. Refs #475 Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-follow.sh | 72 ++++++++++++++++++++++++++--- .github/scripts/fork-follow.test.sh | 66 +++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 8 deletions(-) diff --git a/.github/scripts/fork-follow.sh b/.github/scripts/fork-follow.sh index eae3c848a4688..8f97599837a77 100755 --- a/.github/scripts/fork-follow.sh +++ b/.github/scripts/fork-follow.sh @@ -427,10 +427,51 @@ fi # are dropped. A stop whose conflicts are all on listed (deleted) workflow files is resolved by # deleting them: upstream edited or brought back a file the fork deletes. before="$(git log --format=%s "$base..HEAD")" + +# The workflow checks out with --filter=blob:none, so git fetches a missing blob from the promisor +# remote when it first reads it. While applying a pick, git 2.56 (the ubuntu-26.04 runners) asked +# that remote in one batch for the blob the pick itself had just merged; no server has it, so the +# fetch failed ("not our ref") and the pick stopped with a git error, not a conflict (#475). Before +# the rebase, fetch in one batch every blob it reads: each patch's two sides, the series' own, and +# the old base's side of every file upstream changed. The rebase then has nothing to fetch. Only +# ids the remote can have are asked for (blobs of the series and the base, never a gitlink), and a +# blob that is already here is not fetched again. The promisor is extensions.partialclone or, as +# `git clone --filter` and `git fetch --filter` leave it, the remote marked `promisor = true`. +prefetch_rebase_blobs() { # prefetch_rebase_blobs BASE TARGET + local promisor + promisor="$(git config --get extensions.partialclone || true)" + [[ -n "$promisor" ]] || promisor="$(git config --get-regexp '^remote\..*\.promisor$' \ + | awk 'tolower($2) == "true" { sub(/^remote\./, "", $1); sub(/\.promisor$/, "", $1); print $1; exit }' || true)" + [[ -n "$promisor" ]] || return 0 + # raw_blobs SIDES: the blob ids (not gitlinks) of `--raw` lines, on the old side or both. + raw_blobs() { awk -v both="$1" '/^:/ { if ($1 ~ /^:(100|120)/) print $3; if (both && $2 ~ /^(100|120)/) print $4 }'; } + { + git rev-list --objects --missing=print "$1..HEAD" | sed -n 's/^?//p' + git log --raw --no-renames --no-abbrev --format= "$1..HEAD" | raw_blobs 1 + git diff-tree -r --no-renames "$1" "$2" | raw_blobs "" + } | awk '$0 !~ /^0+$/' | sort -u > "$logs/prefetch.wanted" + # Objects already here (listing them never fetches); only the rest is asked for. + git cat-file --batch-all-objects --batch-check='%(objectname)' | sort -u > "$logs/prefetch.here" + comm -23 "$logs/prefetch.wanted" "$logs/prefetch.here" > "$logs/prefetch.ids" + [[ -s "$logs/prefetch.ids" ]] || return 0 + git -c fetch.negotiationAlgorithm=noop fetch --quiet "$promisor" --no-tags --no-write-fetch-head \ + --recurse-submodules=no --filter=blob:none --stdin < "$logs/prefetch.ids" > "$logs/prefetch.log" 2>&1 \ + || fail "Could not fetch the $(wc -l < "$logs/prefetch.ids") files the rebase from $base_tag onto $target_tag reads from $promisor (this checkout has no file contents of its own):" \ + '```' "$(tail -n 20 "$logs/prefetch.log")" '```' "Nothing was rebased; the series was left as it is." +} +prefetch_rebase_blobs "$base" "$target" + +# Git's own words for a stop: the last rebase command's output without progress lines, its last +# 20 lines. The output of earlier steps (a resolved listed file) stays in their own logs. +git_said() { + tr '\r' '\n' < "$last_log" \ + | awk '!/^(Rebasing \([0-9]+\/[0-9]+\)|Auto packing the repository|See "git help gc")/ && NF' | tail -n 20 +} as_fork() { git -c user.name=t3code-fork -c user.email=noreply@users.noreply.github.com "$@"; } rebasing() { [[ -d "$(git rev-parse --git-path rebase-merge)" || -d "$(git rev-parse --git-path rebase-apply)" ]]; } rebase_ok=0 -if as_fork rebase --empty=drop --onto "$target" "$base" > "$logs/rebase.log" 2>&1; then +last_log="$logs/rebase.log" +if as_fork rebase --empty=drop --onto "$target" "$base" > "$last_log" 2>&1; then rebase_ok=1 fi steps=0 @@ -440,19 +481,36 @@ while ((!rebase_ok)) && rebasing && ((steps++ <= series_count)); do < <(git -c core.quotepath=off diff -z --name-only --diff-filter=U) ((${#unmerged[@]})) || break for path in "${unmerged[@]}"; do is_removed "$path" || break 2; done - git rm -q --ignore-unmatch -- "${unmerged[@]}" >> "$logs/rebase.log" 2>&1 + last_log="$logs/rebase-step-$steps.log" + git rm -q --ignore-unmatch -- "${unmerged[@]}" > "$last_log" 2>&1 if git diff --cached --quiet HEAD; then - GIT_EDITOR=true as_fork rebase --skip >> "$logs/rebase.log" 2>&1 && rebase_ok=1 + GIT_EDITOR=true as_fork rebase --skip >> "$last_log" 2>&1 && rebase_ok=1 else - GIT_EDITOR=true as_fork rebase --continue >> "$logs/rebase.log" 2>&1 && rebase_ok=1 + GIT_EDITOR=true as_fork rebase --continue >> "$last_log" 2>&1 && rebase_ok=1 fi done if ((!rebase_ok)); then - stopped="$(git log -1 --format='%h %s' REBASE_HEAD 2> /dev/null || echo unknown)" + # REBASE_HEAD names a patch that stopped on a conflict. A pick that stopped on a git error has + # none; the last line of the rebase's done list is then the patch it was applying. + if git rev-parse -q --verify REBASE_HEAD > /dev/null; then + stopped="$(git log -1 --format='%h %s' REBASE_HEAD)" + elif ! rebasing; then + stopped="its start: git did not begin the rebase, so no patch was applied" + else + picked="$(awk '$1 ~ /^(pick|p)$/ { c = $2 } END { print c }' "$(git rev-parse --git-path rebase-merge/done)" 2> /dev/null || true)" + stopped="$(git log -1 --format='%h %s' "${picked:-none}" 2> /dev/null)" \ + || stopped="a patch git did not name (its output is below)" + fi conflicts="$(git diff --name-only --diff-filter=U | sed 's/^/- /')" - git rebase --abort || true + said="$(git_said)" + git rebase --abort > /dev/null 2>&1 || true + if [[ -n "$conflicts" ]]; then + why=("Conflicting files:" "$conflicts") + else + why=("Git stopped with an error, not a conflict: no file is left unmerged.") + fi fail "Rebasing the series from $base_tag onto $target_tag stopped at: $stopped" "" \ - "Conflicting files:" "${conflicts:-(none listed; see the run log)}" "" \ + "${why[@]}" "" "What git said:" '```' "${said:-(git printed nothing)}" '```' "" \ "The series was left as it is. To follow by hand: \`git fetch https://github.com/$UPSTREAM.git tag $target_tag\`," \ "\`git rebase --onto $target_tag $base_tag \`, resolve, run the tests, then push the branch with" \ "\`--force-with-lease\`, and start the follow workflow once (\`gh workflow run fork-follow.yml --repo $REPOSITORY\`):" \ diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index 4810b745bfbdc..af9a747a96d5f 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -214,6 +214,70 @@ check "plan: a conflict names the stopped patch and its file and aborts the reba eval 'has "fork: edit u3" "$root/report.md" && has "u3.txt" "$root/report.md" && [[ ! -d "$root/work/.git/rebase-merge" ]]' check "plan: the conflict advice says to start the follow workflow once after the manual push" \ has "gh workflow run fork-follow.yml" "$root/report.md" +check "plan: a conflict report quotes git's own words" \ + eval 'has "What git said:" "$root/report.md" && has "CONFLICT (" "$root/report.md"' + +# #475: a pick that stops on a git error, not a conflict, leaves no unmerged file. The report names +# the patch from the rebase's own state and quotes git, never "stopped at: unknown". +new_fork v0.1.9 +( + cd "$root/work" + echo early > late.txt && git add -A && git commit -q -m "fork: add late" \ + && git rm -q late.txt && git commit -q -m "fork: drop late" && git push -q origin main + echo untracked > late.txt +) > /dev/null 2>&1 +plan +check "plan: a pick stopped by a git error names the patch and quotes git" \ + eval 'has "stopped at: [0-9a-f]* fork: add late" "$root/report.md" && has "not a conflict" "$root/report.md" \ + && has "untracked working tree files would be overwritten" "$root/report.md" \ + && hasnt "stopped at: unknown" "$root/report.md" && [[ ! -d "$root/work/.git/rebase-merge" ]]' +rm -f "$root/work/late.txt" + +new_fork v0.1.9 +echo untracked > "$root/work/u3.txt" +plan +check "plan: a rebase git refuses to begin says so and quotes git" \ + eval 'has "stopped at: its start" "$root/report.md" && has "u3.txt" "$root/report.md" \ + && has "untracked working tree files would be overwritten" "$root/report.md" && hasnt "stopped at: unknown" "$root/report.md"' +rm -f "$root/work/u3.txt" + +# #475: the workflow checks out a partial clone (--filter=blob:none), so git fetches a blob from +# origin when it first reads it. On git 2.56 such a fetch made while a pick was applied asked for +# the blob the pick had just merged, which origin cannot have, and the rebase stopped. The plan +# fetches every blob the rebase reads before it starts; here origin refuses any fetch made while +# the work tree is rebasing, and the plan still follows. +partial_fork() { # partial_fork: the series of new_fork, checked out again as a partial clone + local bare="$root/gh/me/t3code.git" + git -C "$bare" config uploadpack.allowFilter true + git -C "$bare" config uploadpack.allowAnySHA1InWant true + cat > "$root/bin/upload-pack-not-while-rebasing" << EOF +#!/usr/bin/env bash +if [[ -d "$root/work/.git/rebase-merge" || -d "$root/work/.git/rebase-apply" ]]; then + echo "fetch while rebasing" >> "$root/refused" + echo "fatal: upload-pack: not our ref (a fetch while the rebase runs)" >&2 + exit 1 +fi +exec git upload-pack "\$@" +EOF + chmod +x "$root/bin/upload-pack-not-while-rebasing" + rm -rf "$root/work" "$root/refused" + git clone -q --filter=blob:none --no-checkout "file://$bare" "$root/work" 2> /dev/null + work git config remote.origin.uploadpack "$root/bin/upload-pack-not-while-rebasing" + work git checkout -q --detach origin/main +} +new_fork v0.1.9 +( + cd "$root/work" + echo one > feature.txt && git add -A && git commit -q -m "fork: feature one" + echo two > feature.txt && git add -A && git commit -q -m "fork: feature two" && git push -q origin main +) > /dev/null 2>&1 +partial_fork +check "plan test: the partial clone lacks the blob the first pick needs" \ + eval 'git -C "$root/work" rev-list --objects --missing=print HEAD~2..HEAD | grep -q "^?"' +plan +check "plan: a partial clone follows without fetching while it rebases (#475)" \ + eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.10 ]] && [[ ! -e "$root/refused" ]] \ + && [[ "$(git -C "$root/work" show HEAD:feature.txt)" == two ]]' new_fork main plan @@ -490,7 +554,7 @@ fork_commit "fork: own u4" 'echo mine > u4.txt' before="$(work_head)" plan check "plan: a conflict on a listed file and an unlisted one is refused, naming the unlisted one" \ - eval '[[ -z "$(out action)" ]] && has "u4.txt" "$root/report.md" && hasnt "\.github/workflows/ci.yml" "$root/report.md" && [[ ! -d "$root/work/.git/rebase-merge" && "$(work_head)" == "$before" ]]' + eval '[[ -z "$(out action)" ]] && has "u4.txt" "$root/report.md" && hasnt "^- \.github/workflows/ci.yml" "$root/report.md" && [[ ! -d "$root/work/.git/rebase-merge" && "$(work_head)" == "$before" ]]' # Upstream deleted the file itself (the fork's deleting patch has nothing left to do), then brings # it back: the plan deletes it again with one more commit. From e3705bbe6dcaf169c86534ec6f5ea439ecbeb7b4 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:53:27 +0000 Subject: [PATCH 2/3] fork-follow test: read the whole rev-list, not grep -q (lint) The check that the partial clone lacks a blob piped rev-list into grep -q, which fork-ci-tests.test.cjs refuses (SIGPIPE under pipefail). grep -c reads everything. Refs #475 Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-follow.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index af9a747a96d5f..611da565ab4a1 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -273,7 +273,7 @@ new_fork v0.1.9 ) > /dev/null 2>&1 partial_fork check "plan test: the partial clone lacks the blob the first pick needs" \ - eval 'git -C "$root/work" rev-list --objects --missing=print HEAD~2..HEAD | grep -q "^?"' + eval 'git -C "$root/work" rev-list --objects --missing=print HEAD~2..HEAD | grep -c "^?" > /dev/null' plan check "plan: a partial clone follows without fetching while it rebases (#475)" \ eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.10 ]] && [[ ! -e "$root/refused" ]] \ From 089e2d2c15e30d27810c3b97e431daf003320834 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 18 Nov 2026 16:43:41 +0000 Subject: [PATCH 3/3] fork-follow: quote git safely, prefetch reverted blobs, name every stop (#475) - Git's words, the conflict list and upstream file names go into the report as indented code blocks that no line can end, and every @ in a report gets a zero-width space, so no name mentions anyone. - The prefetch also asks for the target side of upstream's changes: a server with reachability bitmaps does not send a blob the base's history already has (an upstream revert), and the checkout read it. - A promisor remote marked yes/on/1 is found (git config --type=bool). - A stop before the first pick, and a --continue/--skip that fails and ends the rebase, are named as such; a failed rebase --abort is quoted. - Tests: fence in a file name, @ file, failed abort, continue ending the rebase, refused prefetch, mid-pick die (done list), die at checkout, revert prefetch, promisor spellings. Refs #475 Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-follow.sh | 84 +++++++++++------- .github/scripts/fork-follow.test.sh | 132 +++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 34 deletions(-) diff --git a/.github/scripts/fork-follow.sh b/.github/scripts/fork-follow.sh index 8f97599837a77..b9d317253d9ba 100755 --- a/.github/scripts/fork-follow.sh +++ b/.github/scripts/fork-follow.sh @@ -51,8 +51,16 @@ REPOSITORY="${REPOSITORY:-${GITHUB_REPOSITORY:-this repository}}" upstream_url="${UPSTREAM_URL:-https://github.com/$UPSTREAM.git}" reported="" +# quoted TEXT...: the lines of TEXT as they are, for a report GitHub renders (an issue comment): a +# blank line, then every line indented by four spaces, a code block that no line in TEXT can end. A +# fence could be ended: git prints upstream's file names raw, and a name may hold a newline and ```. +quoted() { printf '\n'; printf '%s\n' "$@" | sed 's/^/ /'; } + +# The report names files and patches that upstream and the series chose. Every @ in it gets a +# zero-width space after it, so a name like @someone mentions no one when the report is posted. +zwsp=$'\xe2\x80\x8b' fail() { - printf '%s\n' "$@" > "$REPORT" + printf '%s\n' "$@" | sed "s/@/@$zwsp/g" > "$REPORT" cat "$REPORT" >&2 reported=1 exit 1 @@ -92,9 +100,9 @@ fork_channel_init || fail "FORK_FOLLOW_CHANNEL is '$FORK_FOLLOW_CHANNEL'; it mus # fork's tags (its releases). Pruned, so a tag deleted since an earlier fetch here is gone too. git fetch --quiet --prune --no-tags "$upstream_url" "+refs/tags/v*:refs/upstream-tags/v*" "+HEAD:refs/upstream-head" \ > "$logs/fetch.log" 2>&1 \ - || fail "Could not fetch the tags and default branch of $UPSTREAM ($upstream_url):" '```' "$(tail -n 20 "$logs/fetch.log")" '```' + || fail "Could not fetch the tags and default branch of $UPSTREAM ($upstream_url):" "$(quoted "$(tail -n 20 "$logs/fetch.log")")" git fetch --quiet --prune --no-tags "$ORIGIN" "+refs/tags/v*:refs/fork-tags/v*" > "$logs/fetch.log" 2>&1 \ - || fail "Could not fetch the tags of this fork ($ORIGIN):" '```' "$(tail -n 20 "$logs/fetch.log")" '```' + || fail "Could not fetch the tags of this fork ($ORIGIN):" "$(quoted "$(tail -n 20 "$logs/fetch.log")")" # Newest first by version (fork-channel.sh), not by name or by when a tag was made. upstream_tags="$(git for-each-ref --format='%(refname:strip=2)' refs/upstream-tags | { grep -E "$channel_re" || true; } | newest_first)" @@ -173,10 +181,10 @@ schema_check() { # schema_check BASE WHAT: fails with the check's output, a refu node .github/scripts/fork-schema-check.cjs "$1" HEAD > "$logs/schema-check.log" 2>&1 || status=$? ((status != 0)) || return 0 if ((status == 1)) && grep -qF "$schema_refusal" "$logs/schema-check.log"; then - fail "$2 changes the database schema:" '```' "$(cat "$logs/schema-check.log")" '```' + fail "$2 changes the database schema:" "$(quoted "$(cat "$logs/schema-check.log")")" fi fail "The schema check crashed (exit $status), so it did not say whether the series touches the database schema:" \ - '```' "$(tail -n 20 "$logs/schema-check.log")" '```' "The series was left as it is." + "$(quoted "$(tail -n 20 "$logs/schema-check.log")")" "" "The series was left as it is." } schema_check "$base" "The series" @@ -204,11 +212,11 @@ seam_check() { # seam_check BASE WHAT [--after-rebase] return 0 fi if ((status == 1)) && grep -qF "$seam_refusal" "$logs/seam-check.log"; then - fail "$2 does not fit the seam list (.github/fork-seams.json):" '```' "$(cat "$logs/seam-check.log")" '```' \ + fail "$2 does not fit the seam list (.github/fork-seams.json):" "$(quoted "$(cat "$logs/seam-check.log")")" "" \ "The series was left as it is." fi fail "The seam check crashed (exit $status), so it did not say whether the series stays within the seams:" \ - '```' "$(tail -n 20 "$logs/seam-check.log")" '```' "The series was left as it is." + "$(quoted "$(tail -n 20 "$logs/seam-check.log")")" "" "The series was left as it is." } # The sweep (fork-workflows.sh) takes every .github/workflows/fork-*.y[a]ml as the fork's own, even # one the series does not have yet, and never disables it. A file of that name that the upstream @@ -221,7 +229,7 @@ refuse_upstream_fork_named() { # refuse_upstream_fork_named UPSTREAM_COMMIT TAG local named named="$(comm -12 <(fork_named "$1" | sort) <(fork_named "$3" | sort))" [[ -z "$named" ]] || fail "$UPSTREAM $2 has workflow files named like the fork's own (fork-*.yml), and the candidate keeps them:" \ - "$(sed 's/^/- /' <<< "$named")" "" \ + "$(quoted "$named")" "" \ "The workflow sweep never disables a fork-* workflow, so they would run in the fork. The series was left as it is." \ "Add a fork patch that deletes them (or rename the fork's rule in fork-workflows.sh and fork-follow.sh), then start the follow workflow again." } @@ -281,13 +289,13 @@ refuse_outside_workflows() { # refuse_outside_workflows UPSTREAM_BASE UPSTREAM_T status=0 events="$(node "$here/fork-workflow-triggers.cjs" "$logs/workflow.yml" 2> "$logs/triggers.log")" || status=$? if ((status != 0)); then - found+=("- \`$path\`: its triggers could not be read ($(tr '\n' ' ' < "$logs/triggers.log"| sed 's/ *$//'))") + found+=("$path: its triggers could not be read ($(tr '\n' ' ' < "$logs/triggers.log"| sed 's/ *$//'))") elif grep -qE "$outside_events" <<< "$events" || grep -qwE 'pull_request_target|workflow_run' "$logs/workflow.yml"; then - found+=("- \`$path\`: on $(paste -sd ',' <<< "$events" | sed 's/,/, /g')") + found+=("$path: on $(paste -sd ',' <<< "$events" | sed 's/,/, /g')") fi done ((${#found[@]} == 0)) || fail "$UPSTREAM $3 adds workflow files that someone outside the fork can start, or whose triggers could not be read:" \ - "${found[@]}" "" \ + "$(quoted "${found[@]}")" "" \ "The workflow sweep cannot stop the first run of such a file, so the series was left as it is." \ "Add each one to $removed_file with a fork patch (the next follow then deletes it) and start the follow workflow again;" \ "or, to keep one, rebase onto $3 by hand (see the conflict advice in fork-follow.yml)." @@ -312,8 +320,8 @@ workflow_paths() { # workflow_paths FILTER FROM TO: sets paths, the ','-joined f > "$logs/paths" || fail "Could not list the workflow files that differ between ${2:0:12} and ${3:0:12}." while IFS= read -r -d '' path; do if [[ "$path" == *[,$'\n']* ]]; then - fail "The workflow file \`$path\` (between ${2:0:12} and ${3:0:12}) has a ',' or a newline in its name, so it cannot be named to the workflow sweep." \ - "The series was left as it is. Rename or delete it with a fork patch." + fail "A workflow file between ${2:0:12} and ${3:0:12} has a ',' or a newline in its name, so it cannot be named to the workflow sweep:" \ + "$(quoted "$path")" "" "The series was left as it is. Rename or delete it with a fork patch." fi paths+="${paths:+,}$path" done < "$logs/paths" @@ -396,7 +404,7 @@ if [[ -z "$untagged" && "$target_tag" == "$base_tag" ]]; then # The candidate here is the series head itself, so a listed file in it is refused, not deleted. kept="$(present_removed "$head")" [[ -z "$kept" ]] || fail "The series still has workflow files that $removed_file lists as deleted:" \ - "$(sed 's/^/- /' <<< "$kept")" "" \ + "$(quoted "$kept")" "" \ "The plan does not release them. Delete them with a fork patch (or take them off the list), then start the follow workflow again." read -r released_tag released_commit <<< "$(latest_release "$base_tag")" || true if [[ "${released_commit:-}" == "$head" ]]; then @@ -433,22 +441,24 @@ before="$(git log --format=%s "$base..HEAD")" # that remote in one batch for the blob the pick itself had just merged; no server has it, so the # fetch failed ("not our ref") and the pick stopped with a git error, not a conflict (#475). Before # the rebase, fetch in one batch every blob it reads: each patch's two sides, the series' own, and -# the old base's side of every file upstream changed. The rebase then has nothing to fetch. Only -# ids the remote can have are asked for (blobs of the series and the base, never a gitlink), and a -# blob that is already here is not fetched again. The promisor is extensions.partialclone or, as -# `git clone --filter` and `git fetch --filter` leave it, the remote marked `promisor = true`. +# both sides of every file upstream changed. The rebase then has nothing to fetch. A blob already +# here is not asked for. The upstream fetch above sends every blob of the target except those the +# base's history already has (a file upstream brought back to an earlier content), so whatever is +# asked for is in the series or the base's history, which the remote has; a gitlink never is. The promisor is extensions.partialclone or, as +# `git clone --filter` and `git fetch --filter` leave it, the remote marked `promisor = true` (in +# any spelling git takes for true: yes, on, 1). prefetch_rebase_blobs() { # prefetch_rebase_blobs BASE TARGET local promisor promisor="$(git config --get extensions.partialclone || true)" - [[ -n "$promisor" ]] || promisor="$(git config --get-regexp '^remote\..*\.promisor$' \ - | awk 'tolower($2) == "true" { sub(/^remote\./, "", $1); sub(/\.promisor$/, "", $1); print $1; exit }' || true)" + [[ -n "$promisor" ]] || promisor="$(git config --type=bool --get-regexp '^remote\..*\.promisor$' \ + | awk '$2 == "true" { sub(/^remote\./, "", $1); sub(/\.promisor$/, "", $1); print $1; exit }' || true)" [[ -n "$promisor" ]] || return 0 # raw_blobs SIDES: the blob ids (not gitlinks) of `--raw` lines, on the old side or both. raw_blobs() { awk -v both="$1" '/^:/ { if ($1 ~ /^:(100|120)/) print $3; if (both && $2 ~ /^(100|120)/) print $4 }'; } { git rev-list --objects --missing=print "$1..HEAD" | sed -n 's/^?//p' git log --raw --no-renames --no-abbrev --format= "$1..HEAD" | raw_blobs 1 - git diff-tree -r --no-renames "$1" "$2" | raw_blobs "" + git diff-tree -r --no-renames "$1" "$2" | raw_blobs 1 } | awk '$0 !~ /^0+$/' | sort -u > "$logs/prefetch.wanted" # Objects already here (listing them never fetches); only the rest is asked for. git cat-file --batch-all-objects --batch-check='%(objectname)' | sort -u > "$logs/prefetch.here" @@ -457,7 +467,7 @@ prefetch_rebase_blobs() { # prefetch_rebase_blobs BASE TARGET git -c fetch.negotiationAlgorithm=noop fetch --quiet "$promisor" --no-tags --no-write-fetch-head \ --recurse-submodules=no --filter=blob:none --stdin < "$logs/prefetch.ids" > "$logs/prefetch.log" 2>&1 \ || fail "Could not fetch the $(wc -l < "$logs/prefetch.ids") files the rebase from $base_tag onto $target_tag reads from $promisor (this checkout has no file contents of its own):" \ - '```' "$(tail -n 20 "$logs/prefetch.log")" '```' "Nothing was rebased; the series was left as it is." + "$(quoted "$(tail -n 20 "$logs/prefetch.log")")" "" "Nothing was rebased; the series was left as it is." } prefetch_rebase_blobs "$base" "$target" @@ -475,6 +485,7 @@ if as_fork rebase --empty=drop --onto "$target" "$base" > "$last_log" 2>&1; then rebase_ok=1 fi steps=0 +resumed="" # the patch whose listed files were deleted last, and the command that went on from it while ((!rebase_ok)) && rebasing && ((steps++ <= series_count)); do unmerged=() while IFS= read -r -d '' path; do unmerged+=("$path"); done \ @@ -482,35 +493,48 @@ while ((!rebase_ok)) && rebasing && ((steps++ <= series_count)); do ((${#unmerged[@]})) || break for path in "${unmerged[@]}"; do is_removed "$path" || break 2; done last_log="$logs/rebase-step-$steps.log" + resumed="$(git log -1 --format='%h %s' REBASE_HEAD 2> /dev/null || echo "a patch")" git rm -q --ignore-unmatch -- "${unmerged[@]}" > "$last_log" 2>&1 if git diff --cached --quiet HEAD; then + resumed+=$'\n'--skip GIT_EDITOR=true as_fork rebase --skip >> "$last_log" 2>&1 && rebase_ok=1 else + resumed+=$'\n'--continue GIT_EDITOR=true as_fork rebase --continue >> "$last_log" 2>&1 && rebase_ok=1 fi done if ((!rebase_ok)); then + # With no rebase left, git either never began it or ended it in a --continue/--skip above. # REBASE_HEAD names a patch that stopped on a conflict. A pick that stopped on a git error has - # none; the last line of the rebase's done list is then the patch it was applying. - if git rev-parse -q --verify REBASE_HEAD > /dev/null; then - stopped="$(git log -1 --format='%h %s' REBASE_HEAD)" + # none; the last line of the rebase's done list is then the patch it was applying. With no done + # list git stopped before its first pick, while it checked out the target. + done_list="$(git rev-parse --git-path rebase-merge/done)" + if ! rebasing && [[ -n "$resumed" ]]; then + stopped="the patch after ${resumed%$'\n'*}: once its listed files were deleted, \`git rebase ${resumed##*$'\n'}\` failed and left no rebase state that names the patch" elif ! rebasing; then stopped="its start: git did not begin the rebase, so no patch was applied" + elif git rev-parse -q --verify REBASE_HEAD > /dev/null; then + stopped="$(git log -1 --format='%h %s' REBASE_HEAD)" + elif [[ -d "$(git rev-parse --git-path rebase-merge)" && ! -s "$done_list" ]]; then + stopped="its start: git stopped while it checked out $target_tag, before it applied any patch" else - picked="$(awk '$1 ~ /^(pick|p)$/ { c = $2 } END { print c }' "$(git rev-parse --git-path rebase-merge/done)" 2> /dev/null || true)" + picked="$(awk '$1 ~ /^(pick|p)$/ { c = $2 } END { print c }' "$done_list" 2> /dev/null || true)" stopped="$(git log -1 --format='%h %s' "${picked:-none}" 2> /dev/null)" \ || stopped="a patch git did not name (its output is below)" fi - conflicts="$(git diff --name-only --diff-filter=U | sed 's/^/- /')" + conflicts="$(git diff --name-only --diff-filter=U)" said="$(git_said)" - git rebase --abort > /dev/null 2>&1 || true + aborted=() + git rebase --abort > "$logs/rebase-abort.log" 2>&1 \ + || aborted=("\`git rebase --abort\` failed as well, so this checkout is still rebasing. It said:" \ + "$(quoted "$(tail -n 20 "$logs/rebase-abort.log")")" "") if [[ -n "$conflicts" ]]; then - why=("Conflicting files:" "$conflicts") + why=("Conflicting files:" "$(quoted "$conflicts")") else why=("Git stopped with an error, not a conflict: no file is left unmerged.") fi fail "Rebasing the series from $base_tag onto $target_tag stopped at: $stopped" "" \ - "${why[@]}" "" "What git said:" '```' "${said:-(git printed nothing)}" '```' "" \ + "${why[@]}" "" "What git said:" "$(quoted "${said:-(git printed nothing)}")" "" ${aborted[@]+"${aborted[@]}"} \ "The series was left as it is. To follow by hand: \`git fetch https://github.com/$UPSTREAM.git tag $target_tag\`," \ "\`git rebase --onto $target_tag $base_tag \`, resolve, run the tests, then push the branch with" \ "\`--force-with-lease\`, and start the follow workflow once (\`gh workflow run fork-follow.yml --repo $REPOSITORY\`):" \ diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index 611da565ab4a1..3621ac97b01b2 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -245,14 +245,14 @@ rm -f "$root/work/u3.txt" # origin when it first reads it. On git 2.56 such a fetch made while a pick was applied asked for # the blob the pick had just merged, which origin cannot have, and the rebase stopped. The plan # fetches every blob the rebase reads before it starts; here origin refuses any fetch made while -# the work tree is rebasing, and the plan still follows. +# the work tree is rebasing (or any fetch, while $root/refuse-all exists), and the plan still follows. partial_fork() { # partial_fork: the series of new_fork, checked out again as a partial clone local bare="$root/gh/me/t3code.git" git -C "$bare" config uploadpack.allowFilter true git -C "$bare" config uploadpack.allowAnySHA1InWant true cat > "$root/bin/upload-pack-not-while-rebasing" << EOF #!/usr/bin/env bash -if [[ -d "$root/work/.git/rebase-merge" || -d "$root/work/.git/rebase-apply" ]]; then +if [[ -e "$root/refuse-all" || -d "$root/work/.git/rebase-merge" || -d "$root/work/.git/rebase-apply" ]]; then echo "fetch while rebasing" >> "$root/refused" echo "fatal: upload-pack: not our ref (a fetch while the rebase runs)" >&2 exit 1 @@ -260,7 +260,7 @@ fi exec git upload-pack "\$@" EOF chmod +x "$root/bin/upload-pack-not-while-rebasing" - rm -rf "$root/work" "$root/refused" + rm -rf "$root/work" "$root/refused" "$root/refuse-all" git clone -q --filter=blob:none --no-checkout "file://$bare" "$root/work" 2> /dev/null work git config remote.origin.uploadpack "$root/bin/upload-pack-not-while-rebasing" work git checkout -q --detach origin/main @@ -504,6 +504,130 @@ fork_remove() { # fork_remove FILE...: a fork patch that lists and deletes upstr for f in "$@"; do cmd+="echo .github/workflows/$f >> .github/fork-removed-workflows.txt && rm -f .github/workflows/$f && "; done fork_commit "fork: delete $*" "${cmd}true" } +# --- plan: every stop of the rebase is named, and the report quotes git safely (#475) --------- +# A git that does what FORK_TEST_GIT names and is git otherwise: skip the prefetch (no-prefetch), +# fail `rebase --abort` (abort-fails), or end the rebase in a --continue/--skip and fail that +# (continue-quits). +real_git="$(command -v git)" +mkdir -p "$root/testgit" +cat > "$root/testgit/git" << G +#!/usr/bin/env bash +args=" \$* " +case "\${FORK_TEST_GIT:-}" in + no-prefetch) [[ "\$args" != *" fetch "*" --stdin "* ]] || exit 0 ;; + abort-fails) [[ "\$args" != *" rebase --abort "* ]] || { echo "error: the test git refused the abort" >&2; exit 1; } ;; + continue-quits) + if [[ "\$args" == *" rebase --continue "* || "\$args" == *" rebase --skip "* ]]; then + "$real_git" rebase --quit; echo "error: the test git ended the rebase" >&2; exit 1 + fi ;; +esac +exec "$real_git" "\$@" +G +chmod +x "$root/testgit/git" +tplan() { PATH="$root/testgit:$PATH" FORK_TEST_GIT="$1" plan; } +zwsp=$'\xe2\x80\x8b' + +# Git prints upstream's file names raw. A name with a newline and ``` stays inside the quote of +# git's words, and no name in the report mentions anyone. +new_upstream +fenced="$(printf '.github/workflows/x\n```\n**@octocat injected**\n```\ny.yml')" +up_release v0.1.11 'git mv .github/workflows/ci.yml "$fenced"' +new_fork v0.1.10 +fork_remove ci.yml +plan +check "plan: a file name that holds a fence stays in the quote of git's words and mentions no one" \ + eval 'has "^ \*\*@${zwsp}octocat injected\*\*$" "$root/report.md" && hasnt "@octocat" "$root/report.md" \ + && [[ -z "$(grep -E "octocat|^ {0,3}\`\`\`" "$root/report.md" | grep -v "^ ")" ]]' + +new_upstream +new_fork v0.1.10 +fork_commit "fork: add the octo file" 'echo fork > @octo.txt' +up_release v0.1.11 'echo upstream > @octo.txt' +plan +check "plan: a conflicting file named @name is listed in a quote and mentions no one" \ + eval 'has "^Conflicting files:" "$root/report.md" && has "^ @${zwsp}octo.txt$" "$root/report.md" && hasnt "@octo" "$root/report.md"' + +# A failed `git rebase --abort` is quoted, not hidden. +new_upstream +new_fork v0.1.9 +fork_commit "fork: edit u3" 'echo fork > u3.txt' +tplan abort-fails +check "plan: a failed rebase --abort is quoted in the report" \ + eval 'has "rebase --abort. failed as well" "$root/report.md" && has "^ error: the test git refused the abort" "$root/report.md"' +work git rebase --abort + +# A --continue that fails and ends the rebase is not reported as a rebase git never began. +new_upstream +up_release v0.1.11 'echo "on: [push] # edited" > .github/workflows/ci.yml' +new_fork v0.1.10 +fork_remove ci.yml +tplan continue-quits +check "plan: a --continue that fails and ends the rebase names the patch it went on from" \ + eval 'has "stopped at: the patch after [0-9a-f]* fork: delete ci.yml: once its listed files were deleted, .git rebase --continue. failed" "$root/report.md" \ + && hasnt "did not begin" "$root/report.md" && has "the test git ended the rebase" "$root/report.md"' + +# Partial clones (see partial_fork). Prefetch refused: the plan stops before the rebase, quoting git. +# ORIGIN is the fork's path, so only the promisor remote (origin) refuses. +new_upstream +new_fork v0.1.9 +fork_commit "fork: feature one" 'echo one > feature.txt' +fork_commit "fork: feature two" 'echo two > feature.txt' +partial_fork +before="$(work_head)" +touch "$root/refuse-all" +ORIGIN="$root/gh/me/t3code.git" plan +rm -f "$root/refuse-all" +check "plan: a refused prefetch stops before the rebase and quotes git" \ + eval 'has "^Could not fetch the [0-9]* files the rebase from v0.1.9 onto v0.1.10 reads from origin" "$root/report.md" \ + && has "^ fatal: upload-pack: not our ref" "$root/report.md" && has "Nothing was rebased" "$root/report.md" \ + && [[ ! -d "$root/work/.git/rebase-merge" && "$(work_head)" == "$before" ]]' + +# The #475 stop itself: git dies on a lazy fetch in a later pick and leaves no REBASE_HEAD; the done +# list names the patch. Here the prefetch is skipped and origin refuses while the rebase runs. +new_fork v0.1.9 +fork_commit "fork: add f" 'echo v1 > f.txt' +fork_commit "fork: change f" 'echo v2 > f.txt' +partial_fork +tplan no-prefetch +check "plan: a pick that dies on a fetch, with no REBASE_HEAD, is named from the done list" \ + eval 'has "stopped at: [0-9a-f]* fork: add f$" "$root/report.md" && has "not a conflict" "$root/report.md" \ + && has "could not fetch" "$root/report.md" && [[ -e "$root/refused" && ! -d "$root/work/.git/rebase-merge" ]]' + +# Upstream brings a file back to an earlier content (a revert). A server with reachability bitmaps +# (GitHub) does not send that blob, since the base's history has it, so the checkout of the target +# reads it: the prefetch asks for it, and the plan follows without fetching while it rebases. +new_upstream +up_release v0.1.11 'echo old > z.txt' +up_release v0.1.12 'echo new > z.txt' +new_fork v0.1.12 +partial_fork +up_release v0.1.13 'echo old > z.txt' +git -C "$root/gh/up/t3code.git" repack -q -a -d -b +plan +check "plan: a partial clone follows an upstream revert without fetching while it rebases" \ + eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.13 && ! -e "$root/refused" ]] \ + && [[ "$(git -C "$root/work" show "$(out candidate):z.txt")" == old ]]' + +# Without the prefetch git dies while it checks out the target, before its first pick: no patch +# was applied, and the report says so. +partial_fork +tplan no-prefetch +check "plan: git dying before the first pick says no patch was applied" \ + eval 'has "stopped at: its start: git stopped while it checked out v0.1.13, before it applied any patch" "$root/report.md" \ + && has "could not fetch" "$root/report.md" && [[ -e "$root/refused" && ! -d "$root/work/.git/rebase-merge" ]]' + +# Git takes yes, on and 1 for true: a remote marked so is a promisor, and the prefetch runs. +for yes in yes on 1; do + new_fork v0.1.9 + fork_commit "fork: feature one" 'echo one > feature.txt' + fork_commit "fork: feature two" 'echo two > feature.txt' + partial_fork + work git config remote.origin.promisor "$yes" + plan + check "plan: a promisor remote marked '$yes' is prefetched for" \ + eval 'action_is rebase && [[ ! -e "$root/refused" ]]' +done + # Upstream adopts a "pending" fork line word for word: the rebase drops it, so the series is below # its ceiling. Lowering the ceiling first fails the check on the old base, so after a rebase the # ceiling is only a note; the next run, from the new base, refuses it (a pull request lowers it). @@ -554,7 +678,7 @@ fork_commit "fork: own u4" 'echo mine > u4.txt' before="$(work_head)" plan check "plan: a conflict on a listed file and an unlisted one is refused, naming the unlisted one" \ - eval '[[ -z "$(out action)" ]] && has "u4.txt" "$root/report.md" && hasnt "^- \.github/workflows/ci.yml" "$root/report.md" && [[ ! -d "$root/work/.git/rebase-merge" && "$(work_head)" == "$before" ]]' + eval '[[ -z "$(out action)" ]] && has "u4.txt" "$root/report.md" && hasnt "^ \.github/workflows/ci.yml$" "$root/report.md" && has "^ u4.txt$" "$root/report.md" && [[ ! -d "$root/work/.git/rebase-merge" && "$(work_head)" == "$before" ]]' # Upstream deleted the file itself (the fork's deleting patch has nothing left to do), then brings # it back: the plan deletes it again with one more commit.