diff --git a/.github/scripts/fork-follow.sh b/.github/scripts/fork-follow.sh index 488bb7d415402..49f0b21666598 100755 --- a/.github/scripts/fork-follow.sh +++ b/.github/scripts/fork-follow.sh @@ -172,15 +172,26 @@ schema_check() { # schema_check BASE WHAT: fails with the check's output, a refu schema_check "$base" "The series" # Fork lines in upstream files (.github/fork-seams.json): only the pinned seams, so every fork -# setting stays in the fork's own files. Exit 1 is a refusal; anything else is a crash. After a -# rebase (--after-rebase), a "pending" ceiling above the file's fork lines is only a note: upstream -# taking a fork line drops it in the rebase, and the next run (from the new base) refuses it. +# setting stays in the fork's own files. Exit 1 is a refusal; anything else is a crash. BASE is an +# upstream commit, which --from-upstream says (the series' own list judges it, even when upstream +# is a fork with a list of its own). After a rebase (--after-rebase), a "pending" ceiling or "lost" +# count above the file's lines is only a note: upstream taking a fork line drops it in the rebase, +# and the next run (from the new base) refuses it. seam_refusal="Fork changes to upstream files must stay within the seam lines" seam_check() { # seam_check BASE WHAT [--after-rebase] local status=0 - node .github/scripts/fork-seam-check.cjs ${3:+"$3"} "$1" HEAD > "$logs/seam-check.log" 2>&1 || status=$? + node .github/scripts/fork-seam-check.cjs --from-upstream ${3:+"$3"} "$1" HEAD > "$logs/seam-check.log" 2>&1 || status=$? if ((status == 0)); then - grep '^Note: ' "$logs/seam-check.log" >&2 || true + local notes + notes="$(grep '^Note: ' "$logs/seam-check.log" || true)" + [[ -n "$notes" ]] || return 0 + printf '%s\n' "$notes" >&2 + # The log is seen only when the plan fails: the notes go to the run's summary too. + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + printf '%s\n' "### Seam check notes: $2" "" "$(sed 's/^Note: /- /' <<< "$notes")" "" \ + "A ceiling or count named here is refused by the next run; lower it with a pull request to main." \ + "" >> "$GITHUB_STEP_SUMMARY" + fi return 0 fi if ((status == 1)) && grep -qF "$seam_refusal" "$logs/seam-check.log"; then diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index 56b7d7ee258ba..360d05abd7cff 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -170,9 +170,11 @@ new_fork() { work() { (cd "$root/work" && "$@" > /dev/null 2>&1); } fork_tag() { work git tag -f "$1" "${2:-HEAD}" && work git push -q --force origin "refs/tags/$1"; } -plan() { # runs the plan in $root/work; output in $root/plan.out, report in $root/report.md - rm -f "$root/report.md" "$root/plan.out" - (cd "$root/work" && REPORT="$root/report.md" GITHUB_OUTPUT="$root/plan.out" bash "$dir/fork-follow.sh") > "$root/out" 2>&1 +plan() { # runs the plan in $root/work; output in $root/plan.out, report in $root/report.md, step summary in $root/summary.md (first $summary_before, if set) + rm -f "$root/report.md" "$root/plan.out" "$root/summary.md" + [[ -z "${summary_before:-}" ]] || printf '%s\n' "$summary_before" > "$root/summary.md" + (cd "$root/work" && REPORT="$root/report.md" GITHUB_OUTPUT="$root/plan.out" GITHUB_STEP_SUMMARY="$root/summary.md" \ + bash "$dir/fork-follow.sh") > "$root/out" 2>&1 } out() { sed -n "s/^$1=//p" "$root/plan.out" 2> /dev/null | tail -n1; } action_is() { [[ "$(out action)" == "$1" ]]; } @@ -439,9 +441,14 @@ new_fork v0.1.10 (cd "$root/work" && echo '{"files":{},"pending":{"u2.txt":1}}' > .github/fork-seams.json && echo "fork pending line" >> u2.txt \ && git add -A && git commit -q --amend --no-edit && git push -q --force origin main) > /dev/null 2>&1 up_release v0.1.11 'echo "fork pending line" >> u2.txt' -plan +summary_before="earlier step line" plan check "plan: upstream taking a pending fork line rebases, noting the ceiling" \ eval 'action_is rebase && grep -q "Note: u2.txt: \"pending\" with no fork lines left" "$root/out"' +# The plan log is seen only on a failure; the note also goes to the run's summary. +check "plan: the after-rebase note is also in the step summary" \ + eval 'has "u2.txt: \"pending\" with no fork lines left" "$root/summary.md" && has "lower it with a pull request to main" "$root/summary.md"' +# The runner's summary file may already hold lines: the notes are added, never written over. +check "plan: the step summary notes are added after what was there" has "earlier step line" "$root/summary.md" work git reset -q --hard "$(out candidate)" work git push -q --force origin HEAD:main plan diff --git a/.github/scripts/fork-seam-check.cjs b/.github/scripts/fork-seam-check.cjs index c731790d4fb22..c61a432778c70 100644 --- a/.github/scripts/fork-seam-check.cjs +++ b/.github/scripts/fork-seam-check.cjs @@ -1,7 +1,7 @@ -// Usage: fork-seam-check.cjs [--config-ref ] [--after-rebase] . Refuses 's -// upstream-file changes since the merge-base beyond .github/fork-seams.json, read from (or -// --config-ref; never ); a without it is upstream. --after-rebase (fork-follow): a -// "pending" file below its ceiling is a note (see findUnlistedChanges). Reads git objects only. +// Usage: [--config-ref ] | --from-upstream [--after-rebase] . +// Refuses 's upstream-file changes beyond .github/fork-seams.json. A pull request's list is +// 's (or --config-ref's), never 's; --from-upstream (fork-follow) reads 's, even if +// upstream has its own. --after-rebase: a ceiling above the count is a note. Reads git objects only. const { execFileSync } = require("node:child_process"); @@ -10,9 +10,6 @@ const CONFIG = ".github/fork-seams.json"; // fork-*.yml, which fork-follow makes the fork delete (that list cannot name them), the only upstream // files a change may delete. const REMOVED = ".github/fork-removed-workflows.txt"; -// The refusals of a "pending" ceiling left above a file's fork lines (fixed by lowering it). -const CEILING_TOO_HIGH = - /: "pending" with (no fork lines left|\d+ fork lines, fewer than its ceiling)/; const REMOVED_PATH = /^\.github\/workflows\/[^/,\s]+\.ya?ml$/; const FORK_WORKFLOW = /^\.github\/workflows\/fork-[^/]*\.ya?ml$/; @@ -62,19 +59,52 @@ function addedLines(base, head) { const total = (lines) => lines.reduce((sum, [, extra]) => sum + extra, 0); /** - * Whether `head` keeps `base`'s lines in their order, whitespace and blank lines aside (as - * `git diff --ignore-all-space --ignore-blank-lines` sees no change). + * The non-blank lines in order, with their indentation (in YAML or Python it is meaning) but no + * trailing blanks or CR (as `git diff --ignore-space-at-eol --ignore-blank-lines` compares them). */ +const orderedLines = (text) => + text + .split("\n") + .map((line) => line.trimEnd()) + .filter(Boolean); + +/** Whether `head` keeps `base`'s lines, in their order and indentation (blank lines aside). */ function sameLines(base, head) { - const lines = (text) => - text - .split("\n") - .map((line) => line.replace(/\s+/g, "")) - .filter(Boolean); - const [a, b] = [lines(base), lines(head)]; + const [a, b] = [orderedLines(base), orderedLines(head)]; return a.length === b.length && a.every((line, i) => line === b[i]); } +/** + * How many of the lines `base` lists `head` does not keep in their order: deleted, moved or changed + * (`base`'s length less the longest common subsequence). Common ends are matched first, so the work + * is the changed middle squared (a few ms for a 10,000-line file with fork lines at both ends). + */ +function lostLines(base, head) { + let start = 0; + while (start < base.length && start < head.length && base[start] === head[start]) start++; + let end = 0; + while ( + end < base.length - start && + end < head.length - start && + base[base.length - 1 - end] === head[head.length - 1 - end] + ) { + end++; + } + const ids = new Map(); + const id = (line) => ids.get(line) ?? ids.set(line, ids.size).get(line); + const a = base.slice(start, base.length - end).map(id); + const b = head.slice(start, head.length - end).map(id); + let previous = new Uint32Array(b.length + 1); + let row = new Uint32Array(b.length + 1); + for (const line of a) { + for (let j = 1; j <= b.length; j++) { + row[j] = line === b[j - 1] ? previous[j - 1] + 1 : Math.max(previous[j], row[j - 1]); + } + [previous, row] = [row, previous]; + } + return a.length - previous[b.length]; +} + /** The upstream workflow paths a REMOVED list names; other lines are ignored. */ function parseRemoved(text = "") { return new Set( @@ -93,6 +123,9 @@ function findSeamBreaches(files, { atBase, atHead, upstream = false }) { const reasons = []; for (const [file, seam] of Object.entries(files)) { const base = atBase(file); + const head = atHead(file); + // Neither has it (upstream renamed it): nothing to guard, a note (findListedGone). + if (base === undefined && head === undefined) continue; // A listed file upstream does not have guards nothing: fail closed, so the list is fixed. if (base === undefined) { reasons.push( @@ -100,7 +133,6 @@ function findSeamBreaches(files, { atBase, atHead, upstream = false }) { ); continue; } - const head = atHead(file); if (head === undefined) { reasons.push(`${file}: this change deletes or renames it.`); continue; @@ -155,15 +187,16 @@ function forkFileTest(patterns = []) { /** * Changed files outside the seams. An unlisted upstream file may not change or go (unless `removed` - * names it). A "pending" one holds exactly its ceiling from upstream; on a PR (`isForkFile` - * names fork files) it may change but not grow. `afterRebase`: below the ceiling (an adopted fork - * line drops in the rebase; the old base forbids lowering first) is a note, refused on the next run. + * names it). A "pending" one holds exactly its ceiling from upstream (one too high goes to + * `ceilings`; after a rebase to `notes`, as the old base forbade lowering it first); on a PR + * (`isForkFile` names fork files) it may change but not grow. */ function findUnlistedChanges( config, { changed, atBase, atHead, upstream, isForkFile, removed = new Set(), afterRebase = false }, ) { const reasons = []; + const ceilings = []; const notes = []; const pending = config.pending ?? {}; const deleted = (file) => @@ -187,19 +220,20 @@ function findUnlistedChanges( ); } else if (!sameLines(base, head)) { reasons.push( - `${file}: upstream file whose lines this change deletes or moves, but no entry in ${CONFIG}${entry}`, + `${file}: upstream file whose lines this change deletes, moves or re-indents, but no entry in ${CONFIG}${entry}`, ); } } for (const [file, ceiling] of Object.entries(pending)) { const base = atBase(file); + const head = atHead(file); + if (base === undefined && head === undefined) continue; if (base === undefined) { reasons.push( `${file}: "pending" in ${CONFIG}, but the base does not have it (upstream moved or removed it?).`, ); continue; } - const head = atHead(file); if (head === undefined) { reasons.push(deleted(file)); continue; @@ -214,7 +248,7 @@ function findUnlistedChanges( // A ceiling left too high is room to swap upstream lines for fork lines later, and a note // in the nightly log is never seen: refuse, naming the fix (only noted right after a // rebase; the next run refuses it). - (afterRebase ? notes : reasons).push( + (afterRebase ? notes : ceilings).push( added === 0 ? `${file}: "pending" with no fork lines left; remove it from the list.` : `${file}: "pending" with ${added} fork lines, fewer than its ceiling of ${ceiling}; lower it to ${added}.`, @@ -229,7 +263,90 @@ function findUnlistedChanges( ); } } - return { reasons, notes }; + return { reasons, ceilings, notes }; +} + +/** + * Upstream lines a seam or pending file lost: `lines` not kept in order and indentation, `gone` + * whose text is gone (so a line already moved or re-indented cannot be deleted later for free). + * From upstream both are held exactly to its "lost" entry (0 when unlisted), like a ceiling. + */ +function findLostLines(config, { atBase, atHead, upstream, afterRebase = false }) { + const reasons = []; + const ceilings = []; + const notes = []; + const lost = config.lost ?? {}; + const files = Object.entries(config.files).map(([file, seam]) => [ + file, + lineCounts(seam.added.join("\n")), + ]); + if (upstream) for (const file of Object.keys(config.pending ?? {})) files.push([file, new Map()]); + const say = (count, what) => `${count} upstream line${count === 1 ? "" : "s"} ${what}`; + const what = { + lines: "deleted, moved or re-indented", + gone: "deleted or changed (its text is gone)", + }; + for (const [file, listed] of files) { + const base = atBase(file); + const head = atHead(file); + if (base === undefined || head === undefined) continue; // Refused (or noted) elsewhere. + const lines = lostLines(orderedLines(base), orderedLines(head)); + const gone = addedLines(head, base); // [line, copies the head has fewer of] + if (!upstream) { + // Main's pending lines mix with upstream's, so only seam files: their listed lines may go, + // not move (a moved `}` looks like upstream's). + const dropped = gone.reduce( + (sum, [line, copies]) => sum + Math.min(copies, listed.get(line) ?? 0), + 0, + ); + const count = lines - dropped; + if (count > 0) { + reasons.push( + `${file}: ${say(count, what.lines)} by this change (only its listed seam lines may go, and none may move: a moved copy looks like upstream's line).`, + ); + } + continue; + } + const counts = { lines, gone: total(gone) }; + const held = lost[file] ?? { lines: 0, gone: 0 }; + const below = []; + for (const key of ["lines", "gone"]) { + const text = say(counts[key], what[key]); + if (counts[key] > held[key]) { + reasons.push( + `${file}: ${text}, more than its "lost" ${key} count of ${held[key]} (keep upstream's lines; a fork change goes in a file of the fork's own).`, + ); + } else if (counts[key] < held[key]) { + below.push( + `${file}: ${text}, below its "lost" ${key} count of ${held[key]}; lower it to ${counts[key]}.`, + ); + } + } + if (below.length > 0) { + (afterRebase ? notes : ceilings).push( + ...(lines === 0 + ? [ + `${file}: no upstream lines ${what.lines}, below its "lost" entry; remove it from "lost".`, + ] + : below), + ); + } + } + return { reasons, ceilings, notes }; +} + +/** + * Listed seam and pending paths that neither the base nor the head has: nothing there to guard. + * Upstream renamed (or removed) the file, so the list names both paths until the base moves on, and + * the checks before and after fork-follow's rebase each see one. A note, not a refusal. + */ +function findListedGone(config, { atBase, atHead }) { + return [...Object.keys(config.files), ...Object.keys(config.pending ?? {})] + .filter((file) => atBase(file) === undefined && atHead(file) === undefined) + .map( + (file) => + `${file}: listed in ${CONFIG}, but neither the base nor the head has it (renamed or removed upstream?); remove the entry once the fork's base no longer has it.`, + ); } /** @@ -277,6 +394,29 @@ function parseConfig(text) { if (Object.hasOwn(files, file)) throw new Error(`${CONFIG}: ${file} is in "files" and "pending"`); } + const lost = config.lost ?? {}; + if (typeof lost !== "object" || lost === null || Array.isArray(lost)) { + throw new Error(`${CONFIG}: "lost" must map each path to its count of upstream lines lost`); + } + for (const [file, entry] of Object.entries(lost)) { + if (!Object.hasOwn(files, file) && !Object.hasOwn(pending, file)) { + throw new Error(`${CONFIG}: "lost": ${file} is not in "files" or "pending"`); + } + // `gone` (text gone) is at most `lines` (not kept in order) for any file, so more never matches. + const { lines, gone, ...other } = entry ?? {}; + if ( + Object.keys(other).length > 0 || + !Number.isSafeInteger(lines) || + lines < 1 || + !Number.isSafeInteger(gone) || + gone < 0 || + gone > lines + ) { + throw new Error( + `${CONFIG}: "lost": ${file} needs {"lines": N, "gone": M}, with N at least 1 and M from 0 to N`, + ); + } + } for (const pattern of config.forkFiles ?? []) { const path = pattern.replace(/^!/, ""); if ( @@ -288,7 +428,7 @@ function parseConfig(text) { throw new Error(`${CONFIG}: "forkFiles": not a repository path pattern: ${pattern}`); } } - return { files, pending, forkFiles: config.forkFiles ?? [] }; + return { files, pending, forkFiles: config.forkFiles ?? [], lost }; } const hasConfig = (rev) => entryAt(rev, CONFIG)?.type === "blob"; @@ -302,8 +442,13 @@ function readConfigAt(rev) { function parseArgs(argv) { const args = [...argv]; - const afterRebase = args.includes("--after-rebase"); - if (afterRebase) args.splice(args.indexOf("--after-rebase"), 1); + const flag = (name) => { + const at = args.indexOf(name); + if (at !== -1) args.splice(at, 1); + return at !== -1; + }; + const fromUpstream = flag("--from-upstream"); + const afterRebase = flag("--after-rebase"); let configRef; const at = args.indexOf("--config-ref"); if (at !== -1) { @@ -312,25 +457,30 @@ function parseArgs(argv) { args.splice(at, 2); } if (args.length !== 2 || args.some((arg) => arg.startsWith("--"))) return undefined; - return { configRef, afterRebase, base: args[0], head: args[1] }; + // A pull request's run never runs from upstream; only a run from upstream has a rebase behind it. + if (fromUpstream ? configRef !== undefined : afterRebase) return undefined; + return { configRef, fromUpstream, afterRebase, base: args[0], head: args[1] }; } function main(argv) { const parsed = parseArgs(argv); if (!parsed) { - console.error("Usage: fork-seam-check.cjs [--config-ref ] [--after-rebase] "); + console.error( + "Usage: fork-seam-check.cjs [--config-ref ] \n" + + " fork-seam-check.cjs --from-upstream [--after-rebase] ", + ); return 2; } const { base, head } = parsed; - // `base` without the list is upstream; the fork's main always has it. --config-ref is only for - // a pull request's run, so it never runs from upstream. - const upstream = !parsed.configRef && !hasConfig(base); + // Only the caller knows that `base` is upstream (fork-follow says so): a base without the list + // used to be taken for upstream, and an upstream that has one (a fork of this fork) for main. + const upstream = parsed.fromUpstream; const mergeBase = git(["merge-base", base, head]).trim(); if (!upstream) { // A pull request's base and merge-base are the fork's: without the list, the whole old series // (or a base branch that dropped the list) would count as this change's lines. const stale = !hasConfig(base) - ? `The base ${base} has no ${CONFIG}: open the pull request against main or a branch of it.` + ? `The base ${base} has no ${CONFIG}: open the pull request against main or a branch of it (a run from an upstream commit passes --from-upstream).` : !hasConfig(mergeBase) ? `This branch predates the last rebase of its base onto upstream (the merge-base ${mergeBase.slice(0, 12)} has no ${CONFIG}). Update the branch from its base; the seam check then judges only its own lines.` : undefined; @@ -357,8 +507,12 @@ function main(argv) { removed, afterRebase: parsed.afterRebase, }); + const lost = findLostLines(config, { atBase, atHead, upstream, afterRebase: parsed.afterRebase }); + // A ceiling above the count is fixed by lowering it, not by moving lines: its own hint. + const ceilings = [...unlisted.ceilings, ...lost.ceilings]; const reasons = [ ...findSeamBreaches(config.files, { atBase, atHead, upstream }), + ...lost.reasons, ...unlisted.reasons, ...(upstream ? findHiddenUpstreamFiles( @@ -369,7 +523,8 @@ function main(argv) { : []), ]; const from = parsed.configRef ? `--config-ref ${configRef}` : upstream ? "the head" : "the base"; - for (const note of unlisted.notes) console.log(`Note: ${note}`); + const notes = [...findListedGone(config, { atBase, atHead }), ...unlisted.notes, ...lost.notes]; + for (const note of notes) console.log(`Note: ${note}`); if (!upstream && textAt(head, CONFIG) !== textAt(mergeBase, CONFIG)) { console.log( `Note: this change edits ${CONFIG}; it is judged by the list in ${from}, not its own. ` + @@ -377,7 +532,7 @@ function main(argv) { ); } const counted = upstream ? "every upstream file" : "every upstream file it changes"; - if (reasons.length === 0) { + if (reasons.length === 0 && ceilings.length === 0) { console.log( `Fork lines in upstream files: only the pinned seams (${counted}; list from ${from}).`, ); @@ -386,18 +541,17 @@ function main(argv) { console.error( `Fork changes to upstream files must stay within the seam lines in ${CONFIG} (list from ${from}):`, ); - for (const reason of reasons) console.error(`- ${reason}`); - const ceilings = reasons.filter((reason) => CEILING_TOO_HIGH.test(reason)).length; - if (ceilings < reasons.length) { + for (const reason of [...reasons, ...ceilings]) console.error(`- ${reason}`); + if (reasons.length > 0) { console.error( "Put the change in a file of the fork's own (a fork setting goes in " + "packages/contracts/src/forkSettings.ts; see docs/fork/updates.md). A new seam line needs " + `its own entry in ${CONFIG}, which the owner approves.`, ); } - if (ceilings > 0) { + if (ceilings.length > 0) { console.error( - `Lower each "pending" ceiling named above (or remove the entry) in ${CONFIG} with a pull request to main.`, + `Lower each "pending" ceiling and "lost" count named above (or remove the entry) in ${CONFIG} with a pull request to main.`, ); } return 1; @@ -407,10 +561,13 @@ module.exports = { CONFIG, compileForkPattern, findHiddenUpstreamFiles, + findListedGone, + findLostLines, findSeamBreaches, findUnlistedChanges, forkFileTest, lineCounts, + lostLines, parseConfig, parseRemoved, sameLines, diff --git a/.github/scripts/fork-seam-check.test.cjs b/.github/scripts/fork-seam-check.test.cjs index 1d25a885737ea..0084221c4f219 100644 --- a/.github/scripts/fork-seam-check.test.cjs +++ b/.github/scripts/fork-seam-check.test.cjs @@ -9,8 +9,11 @@ const { compileForkPattern, findHiddenUpstreamFiles, findSeamBreaches, + findListedGone, + findLostLines, findUnlistedChanges, forkFileTest, + lostLines, parseConfig, parseRemoved, sameLines, @@ -91,8 +94,8 @@ test("an upstream file with no entry refuses every fork line, from upstream and assert.equal(reasons.length, 1); assert.match(reasons[0], /^u\.ts: upstream file with 1 fork line but no entry/); } - // Unchanged, re-indented, blank lines or CRLF line ends: no change to judge. - for (const after of [UP, ` ${UP}\n\n`, UP.replace("\n", "\r\n")]) { + // Unchanged, blank lines, trailing blanks or CRLF line ends: no change to judge. + for (const after of [UP, `${UP}\n\n`, UP.replace("\n", " \r\n")]) { assert.deepEqual(unlisted({}, { "u.ts": UP }, { "u.ts": after }, false).reasons, []); } }); @@ -104,10 +107,13 @@ test("an upstream file with no entry refuses lines it loses or moves, from upstr for (const upstream of [true, false]) { const { reasons } = unlisted({}, { "u.ts": guarded }, { "u.ts": after }, upstream); assert.equal(reasons.length, 1, JSON.stringify(after)); - assert.match(reasons[0], /^u\.ts: upstream file whose lines this change deletes or moves/); + assert.match( + reasons[0], + /^u\.ts: upstream file whose lines this change deletes, moves or re-indents/, + ); } } - assert.equal(sameLines("a b\n\nc\n", " ab\r\nc"), true); + assert.equal(sameLines("a b\n\nc\n", "a b \r\n\t\nc"), true); assert.equal(sameLines("a\nb\n", "b\na\n"), false); }); @@ -202,16 +208,21 @@ test("from upstream, a fork pattern that matches an upstream file fails until ex test("a pending file: held to its ceiling from upstream; on a pull request it may change but not grow", () => { const config = { pending: { "p.ts": 2 } }; const main = { "p.ts": `${UP}forkA();\nforkB();\n` }; - assert.deepEqual(unlisted(config, { "p.ts": UP }, main, true), { reasons: [], notes: [] }); + assert.deepEqual(unlisted(config, { "p.ts": UP }, main, true), { + reasons: [], + ceilings: [], + notes: [], + }); // From upstream, a pending file with no fork lines left, or fewer than its ceiling, is refused: // a ceiling left too high is room to swap upstream lines for fork lines later, and a note in - // the nightly log is never seen. + // the nightly log is never seen. They are their own kind of refusal, not matched by text. assert.match( - unlisted(config, { "p.ts": UP }, { "p.ts": UP }, true).reasons[0], + unlisted(config, { "p.ts": UP }, { "p.ts": UP }, true).ceilings[0], /^p\.ts: "pending" with no fork lines left/, ); assert.deepEqual(unlisted(config, { "p.ts": UP }, { "p.ts": `${UP}forkA();\n` }, true), { - reasons: ['p.ts: "pending" with 1 fork lines, fewer than its ceiling of 2; lower it to 1.'], + reasons: [], + ceilings: ['p.ts: "pending" with 1 fork lines, fewer than its ceiling of 2; lower it to 1.'], notes: [], }); const swap = { "p.ts": main["p.ts"].replace("forkB();", "forkC();") }; @@ -258,7 +269,7 @@ test("from upstream, a ceiling left too high after lines moved out is refused, s const config = { pending: { "p.ts": 2 } }; const afterMoveOut = `${upstreamText}forkA();\n`; assert.deepEqual( - unlisted(config, { "p.ts": upstreamText }, { "p.ts": afterMoveOut }, true).reasons, + unlisted(config, { "p.ts": upstreamText }, { "p.ts": afterMoveOut }, true).ceilings, ['p.ts: "pending" with 1 fork lines, fewer than its ceiling of 2; lower it to 1.'], ); // (So the nightly stops right after PR 1, before a PR 2 could use the spare line.) @@ -266,6 +277,7 @@ test("from upstream, a ceiling left too high after lines moved out is refused, s const one = { pending: { "p.ts": 1 } }; assert.deepEqual(unlisted(one, { "p.ts": upstreamText }, { "p.ts": afterMoveOut }, true), { reasons: [], + ceilings: [], notes: [], }); const zero = { "p.ts": upstreamText }; @@ -290,16 +302,18 @@ test("after a rebase, a pending file below its ceiling is a note; over it is sti }); assert.deepEqual(judge(`${UP}forkA();\n`, true), { reasons: [], + ceilings: [], notes: ['p.ts: "pending" with 1 fork lines, fewer than its ceiling of 2; lower it to 1.'], }); assert.deepEqual(judge(UP, true), { reasons: [], + ceilings: [], notes: ['p.ts: "pending" with no fork lines left; remove it from the list.'], }); assert.deepEqual(judge(`${UP}forkA();\nforkB();\nforkC();\n`, true).reasons, [ 'p.ts: "pending" with 3 fork lines, more than its ceiling of 2 (fork lines to move out, not add).', ]); - assert.equal(judge(UP, false).reasons.length, 1); + assert.equal(judge(UP, false).ceilings.length, 1); }); test("a malformed config throws instead of guarding nothing", () => { @@ -316,6 +330,21 @@ test("a malformed config throws instead of guarding nothing", () => { { files: {}, pending: { "a.ts": "3" } }, { files: {}, forkFiles: [1] }, { files: SEAM, pending: { "a.ts": 1 } }, + // "lost": a count of 1 or more for a listed seam or pending file. + { files: SEAM, lost: [] }, + { files: SEAM, lost: "a.ts" }, + { files: SEAM, lost: { "a.ts": 2 } }, + { files: SEAM, lost: { "a.ts": null } }, + { files: SEAM, lost: { "a.ts": [2, 1] } }, + { files: SEAM, lost: { "a.ts": { lines: 0, gone: 0 } } }, + { files: SEAM, lost: { "a.ts": { lines: 1.5, gone: 0 } } }, + { files: SEAM, lost: { "a.ts": { lines: "2", gone: 0 } } }, + { files: SEAM, lost: { "a.ts": { lines: 2 } } }, + { files: SEAM, lost: { "a.ts": { lines: 2, gone: -1 } } }, + { files: SEAM, lost: { "a.ts": { lines: 2, gone: 0.5 } } }, + { files: SEAM, lost: { "a.ts": { lines: 2, gone: 3 } } }, + { files: SEAM, lost: { "a.ts": { lines: 2, gone: 1, why: "x" } } }, + { files: SEAM, lost: { "b.ts": { lines: 1, gone: 0 } } }, ...["/abs", "../up", "**", "*", "!", "a/../b"].map((pattern) => ({ files: {}, forkFiles: [pattern], @@ -327,7 +356,13 @@ test("a malformed config throws instead of guarding nothing", () => { files: SEAM, pending: {}, forkFiles: [], + lost: {}, }); + const lost = { "a.ts": { lines: 2, gone: 0 }, "p.ts": { lines: 1, gone: 1 } }; + assert.deepEqual( + parseConfig(JSON.stringify({ files: SEAM, pending: { "p.ts": 1 }, lost })).lost, + lost, + ); }); // The real files: read from the working tree, or from HEAD in a sparse checkout that leaves them out. @@ -393,9 +428,11 @@ function withRepo(body) { return git("rev-parse", "HEAD").trim(); }; const run = (...args) => spawnSync("node", [SCRIPT, ...args], { cwd: dir, encoding: "utf8" }); + // As fork-follow runs it: from an upstream commit, the list read from the head. + const fromUpstream = (...args) => run("--from-upstream", ...args); try { git("init", "-q"); - body({ dir, git, commit, run }); + body({ dir, git, commit, run, fromUpstream }); } finally { fs.rmSync(dir, { recursive: true, force: true }); } @@ -469,7 +506,7 @@ test("a pull request that widens or drops its own list is judged by main's list" // Every upstream file on the command line, from upstream (no list there: the series' own list judges it) and // from main with a list that predates "pending" and "forkFiles" (the first run after this change). test("every upstream file counts on the command line, from upstream and from main", () => { - withRepo(({ git, commit, run }) => { + withRepo(({ git, commit, run, fromUpstream }) => { const upstream = commit({ "u.ts": UP, "p.ts": UP, ...BASE }, "upstream"); const series = commit( { @@ -480,14 +517,14 @@ test("every upstream file counts on the command line, from upstream and from mai "fork series", ); const unlistedLine = commit({ "u.ts": `${UP}forkNew();\n` }, "fork line in u.ts"); - const refused = run(upstream, unlistedLine); + const refused = fromUpstream(upstream, unlistedLine); assert.equal(refused.status, 1, refused.stdout); assert.match(refused.stderr, /- u\.ts: upstream file with 1 fork line but no entry/); - const first = run(upstream, series); + const first = fromUpstream(upstream, series); assert.equal(first.status, 0, first.stderr); assert.match(first.stdout, /every upstream file; list from the head/); // Runs again: the same answer. - const again = run(upstream, series); + const again = fromUpstream(upstream, series); assert.deepEqual([again.status, again.stdout, again.stderr], [0, first.stdout, first.stderr]); // From main whose list has no "pending" or "forkFiles" (older list): fork-named files pass, // the unlisted upstream file is refused. @@ -504,23 +541,23 @@ test("every upstream file counts on the command line, from upstream and from mai // Pins the hidden-file call in main(); turning it off used to pass every test. test("from upstream, an upstream file named like a fork file fails on the command line until excluded", () => { - withRepo(({ commit, run }) => { + withRepo(({ commit, fromUpstream }) => { const upstream = commit({ "src/forkJoin.ts": UP, ...BASE }, "upstream"); const series = commit(seams({ files: SEAM }), "fork series"); - const refused = run(upstream, series); + const refused = fromUpstream(upstream, series); assert.equal(refused.status, 1, refused.stdout); assert.match( refused.stderr, /- src\/forkJoin\.ts: upstream has it, but .* "!src\/forkJoin\.ts"/, ); const fixed = commit(seams({ files: SEAM, forkFiles: ["!src/forkJoin.ts"] }), "exclude it"); - assert.equal(run(upstream, fixed).status, 0); + assert.equal(fromUpstream(upstream, fixed).status, 0); }); }); // Both cases passed in both modes before. test("a rename or delete cannot hide an upstream file, from upstream or on a pull request", () => { - withRepo(({ dir, git, commit, run }) => { + withRepo(({ dir, git, commit, run, fromUpstream }) => { const workflow = ".github/workflows/up.yml"; const upstream = commit( { @@ -557,7 +594,7 @@ test("a rename or delete cannot hide an upstream file, from upstream or on a pul [moved, "src/unlisted.ts"], ]) { for (const base of [main, upstream]) { - const result = run(base, head); + const result = base === main ? run(base, head) : fromUpstream(base, head); assert.equal(result.status, 1, `${file} from ${base === main ? "main" : "upstream"}`); assert.match(result.stderr, new RegExp(`- ${file}: upstream file this change deletes`)); } @@ -568,13 +605,13 @@ test("a rename or delete cannot hide an upstream file, from upstream or on a pul { [workflow]: null, ".github/fork-removed-workflows.txt": `${workflow}\n` }, "delete a listed upstream workflow", ); - assert.equal(run(upstream, dropped).status, 0); + assert.equal(fromUpstream(upstream, dropped).status, 0); }); }); // A submodule pointer read as no file, so moving it passed. test("a moved submodule pointer in an upstream path is a changed line", () => { - withRepo(({ dir, git, commit, run }) => { + withRepo(({ dir, git, commit, run, fromUpstream }) => { // Staged by hand: `git add -A` in commit() would drop a pointer with no folder on disk. const pointer = (sha, files = {}) => { for (const [file, text] of Object.entries(files)) { @@ -591,7 +628,7 @@ test("a moved submodule pointer in an upstream path is a changed line", () => { const main = pointer("1".repeat(40), seams({ files: SEAM })); const moved = pointer("2".repeat(40)); for (const base of [upstream, main]) { - const result = run(base, moved); + const result = base === main ? run(base, moved) : fromUpstream(base, moved); assert.equal(result.status, 1, result.stdout); assert.match(result.stderr, /- \.repos\/lib: upstream file with 1 fork line/); } @@ -643,29 +680,486 @@ test("with --config-ref, a base without the list is refused, not taken for upstr }); test("--after-rebase notes a ceiling above the fork lines; without it the refusal names the fix", () => { - withRepo(({ commit, run }) => { + withRepo(({ commit, fromUpstream }) => { const upstream = commit( { "p.ts": `${UP}forkA();\n`, "u.ts": UP }, "upstream adopts the fork line", ); const series = commit(seams({ files: {}, pending: { "p.ts": 1 } }), "fork series"); - const after = run("--after-rebase", upstream, series); + const after = fromUpstream("--after-rebase", upstream, series); assert.equal(after.status, 0, after.stderr); assert.match(after.stdout, /^Note: p\.ts: "pending" with no fork lines left/m); - const next = run(upstream, series); + const next = fromUpstream(upstream, series); assert.equal(next.status, 1); assert.match(next.stderr, /- p\.ts: "pending" with no fork lines left/); assert.match(next.stderr, /Lower each "pending" ceiling/); assert.doesNotMatch(next.stderr, /a file of the fork's own/); // A ceiling refusal next to another refusal keeps both hints. const line = commit({ "u.ts": `${UP}forkNew();\n` }, "fork line in u.ts"); - const mixed = run(upstream, line); + const mixed = fromUpstream(upstream, line); assert.equal(mixed.status, 1); assert.match(mixed.stderr, /- u\.ts: upstream file with 1 fork line/); assert.match(mixed.stderr, /a file of the fork's own/); assert.match(mixed.stderr, /Lower each "pending" ceiling/); // After a rebase, the other refusal still fails it. - assert.equal(run("--after-rebase", upstream, line).status, 1); - assert.equal(run("--after-rebase", "--after-rebase", upstream, series).status, 2); + assert.equal(fromUpstream("--after-rebase", upstream, line).status, 1); + assert.equal(fromUpstream("--after-rebase", "--after-rebase", upstream, series).status, 2); + }); +}); + +// In YAML and Python, indentation is meaning. Moving `if: false` from a job to one of +// its steps runs the job; the old compare stripped every space, so it passed. +test("re-indenting an upstream file's lines is a change; trailing blanks and blank lines are not", () => { + const job = "jobs:\n build:\n if: false\n steps:\n - run: make\n"; + const step = "jobs:\n build:\n steps:\n - run: make\n if: false\n"; + const nested = "def main():\n check()\n run()\n"; + for (const [file, before, after] of [ + [".github/workflows/up.yml", job, step], + ["tool.py", "def main():\n run()\n", "def main():\nrun()\n"], + ["tool.py", nested, nested.replace(" run()", " run()")], + ]) { + for (const upstream of [true, false]) { + const { reasons } = unlisted({}, { [file]: before }, { [file]: after }, upstream); + assert.equal(reasons.length, 1, `${file} ${after}`); + assert.match(reasons[0], /whose lines this change deletes, moves or re-indents/); + } + } + assert.equal(sameLines(" a\n", "a\n"), false); + assert.equal(sameLines("\ta\n", " a\n"), false); + assert.equal(sameLines(" a\nb\n", " a \r\n\n \nb"), true); +}); + +// A seam file's lines other than its listed seam lines are upstream's. A guard deleted, +// or moved below what it guards, added no line, so it passed both runs. +const GUARD_BASE = "import { x } from './x';\nif (!authorized) return;\ndoSensitive();\n"; +const GUARD_SEAM = { "a.ts": { why: "test", added: ["...fork,", "forkHook();"] } }; +const lostIn = (config, base, head, upstream, afterRebase = false) => + findLostLines( + { files: {}, pending: {}, lost: {}, ...config }, + { + atBase: (file) => base[file], + atHead: (file) => head[file], + upstream, + afterRebase, + }, + ); +test("a seam file keeps upstream's lines in order: deleting, moving or re-indenting one is refused", () => { + const config = { files: GUARD_SEAM }; + const main = { "a.ts": `${GUARD_BASE}forkHook();\n` }; + for (const head of [ + "import { x } from './x';\ndoSensitive();\nforkHook();\n", + "import { x } from './x';\ndoSensitive();\nif (!authorized) return;\nforkHook();\n", + "import { x } from './x';\n if (!authorized) return;\ndoSensitive();\nforkHook();\n", + ]) { + // On a pull request (from main) and from upstream with no "lost" count. + for (const [base, upstream] of [ + [main, false], + [{ "a.ts": GUARD_BASE }, true], + ]) { + const { reasons } = lostIn(config, base, { "a.ts": head }, upstream); + // From upstream a deleted line also counts as one whose text is gone. + const deleted = upstream && !head.includes("authorized"); + assert.equal(reasons.length, deleted ? 2 : 1, `${upstream} ${head}`); + assert.match(reasons[0], /^a\.ts: 1 upstream line deleted, moved or re-indented/); + } + } + // Its own seam lines may go, and lines may be added around upstream's. + for (const head of [ + GUARD_BASE, + `import { x } from './x';\n\nif (!authorized) return;\n...fork,\ndoSensitive();\nforkHook();\n`, + ]) { + assert.deepEqual(lostIn(config, main, { "a.ts": head }, false), { + reasons: [], + ceilings: [], + notes: [], + }); + } + // A moved seam line is refused on a pull request: a moved copy of a listed line looks the same + // as upstream's line with that text (`}`). Moving one takes two pull requests (drop, then add). + const moved = "forkHook();\nimport { x } from './x';\nif (!authorized) return;\ndoSensitive();\n"; + assert.match( + lostIn(config, main, { "a.ts": moved }, false).reasons[0], + /^a\.ts: 1 upstream line deleted, moved or re-indented by this change/, + ); +}); + +test('from upstream, a seam or pending file\'s lost upstream lines are held to its "lost" counts', () => { + const pendingConfig = (lost) => ({ pending: { "p.ts": 1 }, lost }); + const base = { "p.ts": GUARD_BASE }; + const dropGuard = { "p.ts": "import { x } from './x';\ndoSensitive();\nforkA();\n" }; + const moveGuard = { + "p.ts": "import { x } from './x';\ndoSensitive();\nif (!authorized) return;\nforkA();\n", + }; + for (const [head, gone] of [ + [dropGuard, 1], + [moveGuard, 0], + ]) { + assert.deepEqual(lostIn(pendingConfig({}), base, head, true).reasons, [ + "p.ts: 1 upstream line deleted, moved or re-indented, more than its \"lost\" lines count of 0 (keep upstream's lines; a fork change goes in a file of the fork's own).", + ...(gone + ? [ + "p.ts: 1 upstream line deleted or changed (its text is gone), more than its \"lost\" gone count of 0 (keep upstream's lines; a fork change goes in a file of the fork's own).", + ] + : []), + ]); + // Exactly the counts pass. + assert.deepEqual(lostIn(pendingConfig({ "p.ts": { lines: 1, gone } }), base, head, true), { + reasons: [], + ceilings: [], + notes: [], + }); + } + // One over a count is refused. + const both = { "p.ts": "doSensitive();\nforkA();\n" }; + assert.equal( + lostIn(pendingConfig({ "p.ts": { lines: 1, gone: 1 } }), base, both, true).reasons.length, + 2, + ); + // A count above the lines lost is a ceiling to lower (a note right after a rebase). + const kept = { "p.ts": `${GUARD_BASE}forkA();\n` }; + assert.deepEqual(lostIn(pendingConfig({ "p.ts": { lines: 1, gone: 1 } }), base, kept, true), { + reasons: [], + ceilings: [ + 'p.ts: no upstream lines deleted, moved or re-indented, below its "lost" entry; remove it from "lost".', + ], + notes: [], + }); + assert.deepEqual( + lostIn(pendingConfig({ "p.ts": { lines: 2, gone: 1 } }), base, dropGuard, true, true), + { + reasons: [], + ceilings: [], + notes: [ + 'p.ts: 1 upstream line deleted, moved or re-indented, below its "lost" lines count of 2; lower it to 1.', + ], + }, + ); + // On a pull request a pending file's own lines cannot be told from upstream's: the PR may move a + // fork line out, so only the run from upstream counts them. + assert.deepEqual(lostIn(pendingConfig({}), kept, dropGuard, false).reasons, []); + // A seam file from upstream: the same counts. + const seamConfig = (lost) => ({ files: GUARD_SEAM, lost }); + const seamHead = { "a.ts": "import { x } from './x';\ndoSensitive();\nforkHook();\n" }; + assert.equal(lostIn(seamConfig({}), { "a.ts": GUARD_BASE }, seamHead, true).reasons.length, 2); + const held = { "a.ts": { lines: 1, gone: 1 } }; + assert.deepEqual(lostIn(seamConfig(held), { "a.ts": GUARD_BASE }, seamHead, true).reasons, []); + // On a pull request the count does not widen a seam file: no upstream line may go. + assert.equal(lostIn(seamConfig(held), { "a.ts": GUARD_BASE }, seamHead, false).reasons.length, 1); +}); + +// Right after a rebase only a count left too high is a note; one exceeded still stops the run. +test("after a rebase, a lost count exceeded is still refused", () => { + const config = { pending: { "p.ts": 1 }, lost: { "p.ts": { lines: 1, gone: 1 } } }; + const base = { "p.ts": GUARD_BASE }; + const head = { "p.ts": "doSensitive();\nforkA();\n" }; + const { reasons, notes } = lostIn(config, base, head, true, true); + assert.equal(reasons.length, 2, reasons.join("\n")); + assert.match(reasons[0], /^p\.ts: 2 upstream lines deleted, moved or re-indented, more than/); + assert.deepEqual(notes, []); +}); + +// Main's list pins generic seam lines (`}`, `/**`) in serverSettings.ts. Filtering every copy of +// them out let an upstream line move past an upstream `}` unseen, in both runs. +const BRACE_UP = [ + "for (const key of keys) {", + " if (!Equal.equals(currentRecord[key], defaultsRecord[key])) {", + " next[key] = currentRecord[key];", + " }", + "}", + "", +].join("\n"); +const BRACE_MOVED = BRACE_UP.replace( + " next[key] = currentRecord[key];\n }\n", + " }\n next[key] = currentRecord[key];\n", +); +const BRACE_SEAM = { "s.ts": { why: "test", added: ["}", "/**", "forkHook();"] } }; +test("a line moved past an upstream copy of a listed seam line is refused on a pull request", () => { + // Main: upstream's text plus the fork's listed `}` and hook. + const main = { "s.ts": `${BRACE_UP}forkHook();\n}\n` }; + const head = { "s.ts": `${BRACE_MOVED}forkHook();\n}\n` }; + assert.match( + lostIn({ files: BRACE_SEAM }, main, head, false).reasons[0] ?? "", + /^s\.ts: 1 upstream line deleted, moved or re-indented by this change/, + ); + // Dropping the fork's own `}` (a copy main has beyond upstream's) still passes ... + assert.deepEqual( + lostIn({ files: BRACE_SEAM }, main, { "s.ts": `${BRACE_UP}forkHook();\n` }, false).reasons, + [], + ); + // ... but only as many copies as the list names: dropping upstream's `}` too is refused. + const both = { "s.ts": `${BRACE_UP.replace(/\}\n$/, "")}forkHook();\n` }; + assert.match( + lostIn({ files: BRACE_SEAM }, main, both, false).reasons[0] ?? "", + /^s\.ts: 1 upstream line deleted, moved or re-indented by this change/, + ); +}); + +test("from upstream, a line moved past a listed seam line is counted", () => { + const up = { "s.ts": BRACE_UP }; + assert.match( + lostIn({ files: BRACE_SEAM }, up, { "s.ts": BRACE_MOVED }, true).reasons[0] ?? "", + /^s\.ts: 1 upstream line deleted, moved or re-indented, more than its "lost" lines count of 0/, + ); + // The fork's added seam lines (another `}`) lose no upstream line. + assert.deepEqual( + lostIn({ files: BRACE_SEAM }, up, { "s.ts": `${BRACE_UP}forkHook();\n}\n` }, true).reasons, + [], + ); +}); + +// A seam line the file indents still matches its listed (trimmed) text when a pull request drops it. +test("a pull request may drop an indented seam line", () => { + const main = { "a.ts": `${GUARD_BASE} forkHook();\n` }; + assert.deepEqual(lostIn({ files: GUARD_SEAM }, main, { "a.ts": GUARD_BASE }, false).reasons, []); +}); + +test("lostLines counts the base lines not kept in order (longest common subsequence)", () => { + assert.equal(lostLines(["a", "b", "c"], ["a", "b", "c"]), 0); + assert.equal(lostLines(["a", "b", "c"], ["x", "a", "y", "b", "c", "z"]), 0); + assert.equal(lostLines(["a", "b", "c"], ["b", "a", "c"]), 1); + assert.equal(lostLines(["a", "b", "c", "d"], ["d", "c", "b", "a"]), 3); + assert.equal(lostLines(["a", "a", "b"], ["a", "b"]), 1); + assert.equal(lostLines(["a"], []), 1); + assert.equal(lostLines([], ["a"]), 0); +}); + +// Upstream renames a listed file. The rebase follows the rename; the check before it +// refused the new name (the base lacks it) and the check after it the old one, so no list passed. +test("a listed path that neither the base nor the head has is a note, not a refusal", () => { + const config = { files: { "old.ts": { why: "t", added: [] } }, pending: { "gone.ts": 1 } }; + const full = { files: {}, pending: {}, forkFiles: [], lost: {}, ...config }; + const none = () => undefined; + assert.deepEqual( + findSeamBreaches(full.files, { atBase: none, atHead: none, upstream: true }), + [], + ); + assert.deepEqual( + findUnlistedChanges(full, { + changed: [], + atBase: none, + atHead: none, + upstream: true, + isForkFile: () => false, + }), + { reasons: [], ceilings: [], notes: [] }, + ); + assert.deepEqual(findListedGone(full, { atBase: none, atHead: none }), [ + "old.ts: listed in .github/fork-seams.json, but neither the base nor the head has it (renamed or removed upstream?); remove the entry once the fork's base no longer has it.", + "gone.ts: listed in .github/fork-seams.json, but neither the base nor the head has it (renamed or removed upstream?); remove the entry once the fork's base no longer has it.", + ]); + // Only one side missing still fails closed. + const has = () => UP; + assert.equal(findSeamBreaches(full.files, { atBase: none, atHead: has }).length, 1); + assert.equal(findSeamBreaches(full.files, { atBase: has, atHead: none }).length, 1); + for (const [atBase, atHead] of [ + [none, has], + [has, none], + ]) { + const result = findUnlistedChanges(full, { changed: [], atBase, atHead, upstream: true }); + assert.equal(result.reasons.length, 1); + assert.deepEqual(findListedGone(full, { atBase, atHead }), []); + } +}); + +test("upstream renames a pending file: the list with both names passes before and after the rebase", () => { + withRepo(({ git, commit, fromUpstream }) => { + const fork = "if (forkGate()) return;\n"; + const u1 = commit({ "old.ts": GUARD_BASE }, "upstream 1"); + git("checkout", "-q", "-b", "upstream", u1); + git("mv", "old.ts", "new.ts"); + const u2 = commit({}, "upstream 2 renames old.ts"); + const both = seams({ files: {}, pending: { "old.ts": 1, "new.ts": 1 } }); + git("checkout", "-q", "-b", "series", u1); + const before = commit({ ...both, "old.ts": `${GUARD_BASE}${fork}` }, "fork series"); + git("checkout", "-q", "-b", "rebased", u2); + const after = commit({ ...both, "new.ts": `${GUARD_BASE}${fork}` }, "fork series, rebased"); + const first = fromUpstream(u1, before); + assert.equal(first.status, 0, first.stderr); + assert.match(first.stdout, /^Note: new\.ts: listed in .*, but neither the base nor the head/m); + const second = fromUpstream("--after-rebase", u2, after); + assert.equal(second.status, 0, second.stderr); + assert.match(second.stdout, /^Note: old\.ts: listed in .*, but neither the base nor the head/m); + // Runs again: the same answer. + assert.equal(fromUpstream("--after-rebase", u2, after).stdout, second.stdout); + // The rebased series' fork line in the renamed file is still counted. + const grown = commit({ "new.ts": `${GUARD_BASE}${fork}forkMore();\n` }, "grow it"); + assert.equal(fromUpstream(u2, grown).status, 1); + }); +}); + +// Guards that a mutation used to survive. +test("a pull request that lists an upstream workflow in its own removed list cannot delete it", () => { + withRepo(({ git, commit, run }) => { + const ci = ".github/workflows/ci.yml"; + commit({ [ci]: "on: push\n", ...BASE }, "upstream"); + const main = commit(seams({ files: SEAM }), "fork main"); + const pr = commit( + { [ci]: null, ".github/fork-removed-workflows.txt": `${ci}\n` }, + "list and delete ci.yml", + ); + git("checkout", "-q", "-f", pr); + for (const args of [ + [main, pr], + ["--config-ref", main, main, pr], + ]) { + const result = run(...args); + assert.equal(result.status, 1, args.join(" ")); + assert.match( + result.stderr, + /- \.github\/workflows\/ci\.yml: upstream file this change deletes/, + ); + } + }); +}); + +test("from upstream, only a workflow named fork-* may go, not any upstream file with a fork name", () => { + const files = ["src/forkJoin.ts", "src/fork-thing.ts", ".github/fork-up.txt", "fork/up.ts"]; + const base = Object.fromEntries(files.map((file) => [file, UP])); + assert.deepEqual( + unlisted({}, base, {}, true).reasons.map((reason) => reason.split(":")[0]), + files, + ); + for (const file of [".github/workflows/fork-x.yml", ".github/workflows/fork-y.yaml"]) { + assert.deepEqual(unlisted({}, { [file]: "on: push\n" }, {}, true).reasons, [], file); + } + // A fork-* file in another folder, or a nested workflow folder, is not a workflow the fork deletes. + for (const file of [".github/actions/fork-x.yml", ".github/workflows/sub/fork-x.yml"]) { + assert.equal(unlisted({}, { [file]: "on: push\n" }, {}, true).reasons.length, 1, file); + } +}); + +// Ceiling refusals were found by their message text, so a file named like that text got +// only the "lower the ceiling" advice. +test("a ceiling refusal is known by its kind, not its text", () => { + withRepo(({ commit, fromUpstream }) => { + const odd = 'x: "pending" with no fork lines left'; + const upstream = commit({ [odd]: UP }, "upstream"); + const series = commit({ ...seams({ files: {} }), [odd]: `${UP}forkLine();\n` }, "fork series"); + const result = fromUpstream(upstream, series); + assert.equal(result.status, 1); + assert.match(result.stderr, /upstream file with 1 fork line but no entry/); + assert.match(result.stderr, /a file of the fork's own/); + assert.doesNotMatch(result.stderr, /Lower each "pending" ceiling/); + }); +}); + +// "From upstream" was guessed from a base without the list, so a fork of this fork +// (whose upstream carries a list) was judged by upstream's list as a pull request. +test("--from-upstream reads the head's list even when the upstream commit has one; without it a base with no list is refused", () => { + withRepo(({ git, commit, run, fromUpstream }) => { + // The fork this fork follows allows `inline: 1,` in a.ts; this fork's list does not. + const wide = { files: { "a.ts": { why: "theirs", added: ["inline: 1,"] } } }; + const upstream = commit({ ...seams(wide), ...BASE }, "upstream fork"); + const series = commit( + { + ...seams({ files: SEAM }), + "a.ts": BASE["a.ts"].replace(" a: 1,", " inline: 1,\n a: 1,"), + }, + "this fork's series", + ); + const refused = fromUpstream(upstream, series); + assert.equal(refused.status, 1, refused.stdout); + assert.match(refused.stderr, /a\.ts: adds `inline: 1,`/); + assert.match(refused.stderr, /list from the head/); + // Without the flag a base with no list is not taken for upstream: it is refused. + git("checkout", "-q", "--orphan", "plain"); + git("rm", "-rfq", "."); + const plain = commit(BASE, "upstream without a list"); + const onPlain = commit(seams({ files: SEAM }), "series"); + const noFlag = run(plain, onPlain); + assert.equal(noFlag.status, 1); + assert.match(noFlag.stderr, /^The base .* has no \.github\/fork-seams\.json/); + assert.equal(fromUpstream(plain, onPlain).status, 0); + // Bad flag use is a usage error, never a pass. + for (const args of [ + ["--from-upstream", "--from-upstream", plain, onPlain], + ["--from-upstream", "--config-ref", onPlain, plain, onPlain], + ["--after-rebase", plain, onPlain], + ["--from-upstream=1", plain, onPlain], + ]) { + assert.equal(run(...args).status, 2, args.join(" ")); + } + }); +}); + +// A list from before "lost" (main's today): from upstream, every lost line is counted against 0. +test('from upstream, a list with no "lost" counts refuses a seam file that lost an upstream line', () => { + withRepo(({ commit, fromUpstream, run }) => { + const upstream = commit({ "a.ts": GUARD_BASE }, "upstream"); + const series = commit( + { + ...seams({ files: GUARD_SEAM }), + "a.ts": "import { x } from './x';\ndoSensitive();\nforkHook();\n", + }, + "fork series drops the guard", + ); + const result = fromUpstream(upstream, series); + assert.equal(result.status, 1); + assert.match(result.stderr, /- a\.ts: 1 upstream line deleted, moved or re-indented/); + const recorded = commit( + seams({ files: GUARD_SEAM, lost: { "a.ts": { lines: 1, gone: 1 } } }), + "record it", + ); + assert.equal(fromUpstream(upstream, recorded).status, 0); + // On a pull request to that main, moving an upstream line adds none, and is still refused. + const pr = commit({ "a.ts": "doSensitive();\nimport { x } from './x';\nforkHook();\n" }, "PR"); + const moved = run(recorded, pr); + assert.equal(moved.status, 1); + assert.match( + moved.stderr, + /- a\.ts: 1 upstream line deleted, moved or re-indented by this change/, + ); + assert.doesNotMatch(moved.stderr, /adds/); + }); +}); + +// A count is a number, not a set of lines: an upstream line the fork re-indented already counts +// as lost, so deleting it later left the ordered count the same, and both runs passed. +test("from upstream, deleting an upstream line already counted as re-indented is refused", () => { + withRepo(({ git, commit, run, fromUpstream }) => { + const guard = "if (!req.authorized) {\n return deny();\n}\nserve();\n"; + const upstream = commit({ "p.ts": guard }, "upstream"); + const reindented = " if (!req.authorized) {\n return deny();\n }\nserve();\nforkA();\n"; + const config = { files: {}, pending: { "p.ts": 1 }, lost: { "p.ts": { lines: 3, gone: 0 } } }; + const main = commit({ ...seams(config), "p.ts": reindented }, "fork re-indents the guard"); + const first = fromUpstream(upstream, main); + assert.equal(first.status, 0, first.stderr); + git("checkout", "-q", "-b", "pr", main); + const pr = commit({ "p.ts": reindented.replace(" return deny();\n", "") }, "delete it"); + // The pull request's run cannot tell upstream's lines from the fork's in a pending file ... + assert.equal(run(main, pr).status, 0); + // ... and the run from upstream refuses it: the line's text is gone. + const result = fromUpstream(upstream, pr); + assert.equal(result.status, 1, result.stdout); + assert.match( + result.stderr, + /- p\.ts: 1 upstream line deleted or changed \(its text is gone\), more than its "lost" gone count of 0/, + ); + }); +}); + +// The "lost" ceilings used to be dropped in main() with every test still passing. +test("from upstream, a lost count left too high is refused, naming the count; after a rebase it is a note", () => { + withRepo(({ commit, fromUpstream }) => { + const upstream = commit({ "p.ts": GUARD_BASE }, "upstream"); + const config = { files: {}, pending: { "p.ts": 1 }, lost: { "p.ts": { lines: 3, gone: 2 } } }; + const series = commit( + { ...seams(config), "p.ts": "import { x } from './x';\nforkA();\n" }, + "fork series drops two lines", + ); + const result = fromUpstream(upstream, series); + assert.equal(result.status, 1, result.stdout); + assert.match( + result.stderr, + /- p\.ts: 2 upstream lines deleted, moved or re-indented, below its "lost" lines count of 3; lower it to 2\./, + ); + assert.match(result.stderr, /Lower each "pending" ceiling and "lost" count/); + assert.doesNotMatch(result.stderr, /a file of the fork's own/); + const after = fromUpstream("--after-rebase", upstream, series); + assert.equal(after.status, 0, after.stderr); + assert.match( + after.stdout, + /^Note: p\.ts: 2 upstream lines .* below its "lost" lines count of 3; lower it to 2\.$/m, + ); }); }); diff --git a/docs/fork/updates.md b/docs/fork/updates.md index 42b2276016600..0233d8ab6ec90 100644 --- a/docs/fork/updates.md +++ b/docs/fork/updates.md @@ -48,11 +48,22 @@ watches; it changes nothing on the machine, and `0` turns it off. whose fork lines predate the rule to a ceiling: the most fork lines each may hold, counted from upstream (fork-follow fails above it and below it, so a pull request that moves fork lines out also lowers the ceiling in the same change). Right after a rebase, below the ceiling is only a - note in the run log: when upstream takes a fork line word for word, the rebase drops it, and the - ceiling could not have been lowered first. The next run, from the new base, fails until a pull - request lowers it. A pull request may change a pending file but not grow it. An upstream file in no list may gain no - line, lose no line and keep its lines in order (whitespace and blank lines aside), and no - upstream file may be deleted or renamed, except the workflows + note in the run log and the run's summary: when upstream takes a fork line word for word, the + rebase drops it, and the ceiling could not have been lowered first. The next run, from the new + base, fails until a pull request lowers it. A pull request may change a pending file but not grow + it. Upstream lines a seam or pending file no longer has (deleted, moved below the code they + guard, or re-indented) add no line, so they are counted too, two ways: `lines`, the upstream lines + it does not keep in order and indentation, and `gone`, those whose text it no longer has at all, + so a line already counted as moved or re-indented cannot later be deleted unseen. `lost` maps such + a file to both (`{"lines": N, "gone": M}`), counted from upstream (none when unlisted) and held + exactly like a `pending` ceiling. A pull request may drop a seam file's listed seam lines (no more + copies than listed) but lose no other line, and may not move a seam line: a moved copy of `}` + looks the same as upstream's (move one with two pull requests). In a pending file it cannot tell + upstream's lines from the fork's (moving a fork line out loses one), so there fork-follow counts + them. A listed path that neither the base nor the head has (upstream + renamed it) is a note: list both names until the old one is gone from the base. An upstream file + in no list may gain no line, lose no line and keep its lines in order and indentation (trailing + blanks and blank lines aside), and no upstream file may be deleted or renamed, except the workflows [`fork-removed-workflows.txt`](../../.github/fork-removed-workflows.txt) lists and upstream `fork-*.yml`. A submodule pointer counts as a line. A pull request's run has no upstream commit, so it takes files named as the fork's (`fork`, `fork-*`, `forkName*`, `FORK_*`) and `forkFiles` @@ -60,7 +71,9 @@ watches; it changes nothing on the machine, and `0` turns it off. reason a pull request's run counts upstream's copies of a seam line against its allowance: a seam line upstream also has (`}`, `/**`) cannot be added by a pull request, only checked by fork-follow, so keep seam lines specific. A branch made before fork-follow last rebased main - is refused with one "update the branch" error. + is refused with one "update the branch" error. Only `--from-upstream` (which fork-follow passes) + runs it from an upstream commit and reads the head's list; without it a base with no list is + refused, and an upstream that is itself a fork with a list is still judged by this fork's list. - [`serverSettings.forkDefaults.test.ts`](../../apps/server/src/serverSettings.forkDefaults.test.ts) loads the settings the way a server does and checks every fork setting is at its row's default with no file, `{}`, a file an upstream build wrote, a file cut off half-way and a file with a bad @@ -112,7 +125,7 @@ change, or a fork line beyond the seams, it stops and opens one issue. Then: 1. Rebase by hand as the workflow's report says. Never take upstream's side of a fork setting's default: keep the fork's value (and the `Fork:` comment of the inline `automaticUpdates`). 2. Check that the fork's lines in upstream files are only the seams: - `node .github/scripts/fork-seam-check.cjs HEAD`. A refused line moves into the + `node .github/scripts/fork-seam-check.cjs --from-upstream HEAD`. A refused line moves into the fork's own file (a setting into `forkSettings.ts`, a server layer into `apps/server/src/fork/forkLayers.ts`); a changed seam line is fixed in `.github/fork-seams.json`.