Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/fork-ci-tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
# ... ok lines two) its format: its tests are unittest tests, counted through
# fork-ci-unittest.py, or it prints ok lines, below. The runner judges
# it by this word alone: a unittest test that wrote no plan line, or an
# ok-lines test without its end line, counts nothing
# ok-lines test without its end line, counts nothing. A test the runner
# does not run (upstream's, run by a plain step) is not judged by its
# word: fork-ci-tests.test.cjs checks the word against its source
# A test is listed once.
# A test's own `fork-ci:` comment lines say more (the runner reads them):
# fork-ci: run [as root] [with <NAME>_TESTS=value ...] [only <TestClass>[.<test_method>]]
# an own run, by `fork-ci-tests.cjs --run <test>` in fork-tests.yml;
Expand All @@ -35,7 +38,8 @@
# node through fork-ci-node-counts.cjs (a file that reports no test of its own, or stops
# half-way, counts nothing); an ok-lines test (sh, or Python with "ok lines": `ok <what>`,
# `FAIL <what>`, `skip <name> [why]` on stdout) ends with "all passed: <n> ok, <n> skipped"
# (its own count), else it counts nothing.
# (its own count), else it counts nothing. Its count is trusted: one that prints a matching end
# line and stops before its other checks passes, so a new Python test is best a unittest test.
# A present test that is not listed fails. A listed file that is not present yet
# is a note; one that was present and then deleted fails.
.github/scripts/fork-ci-tests.test.cjs ~11s
Expand Down Expand Up @@ -162,6 +166,7 @@ fork-kit/schedule-kick/tests/late.test.py unittest
.github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml
.github/scripts/mobile-preview-comment.test.cjs run by .github/workflows/fork-tests.yml
.github/scripts/mobile-ios-preview-record.test.cjs run by .github/workflows/fork-tests.yml
# A plain `python3` step runs this one: its word is checked against its source, not by the runner.
.github/scripts/stage-preview-bundle.test.py run by .github/workflows/fork-tests.yml unittest
.github/scripts/test-delta.test.cjs run by .github/workflows/fork-tests.yml
.github/scripts/vercel-deployment-output.test.cjs run by .github/workflows/fork-tests.yml
Expand Down
9 changes: 7 additions & 2 deletions .github/scripts/fork-ci-tests.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ function plan(entries, present, ever, read = () => null, allowed = RUN_ELSEWHERE
}

function readList(text) {
const seen = new Set();
return text
.split("\n")
.map((line) => line.replace(/#.*/, "").trim())
Expand All @@ -176,9 +177,13 @@ function readList(text) {
);
if (!match)
throw new Error(
`${LIST}: cannot read "${line}" (expected "<test>", "<test> ~<seconds>s" or "<test> run by <file>", ` +
`a Python test's then ending in its format: "unittest" or "ok lines")`,
`${LIST}: cannot read "${line}" (expected "<test>", "<test> ~<seconds>s" or "<test> run by <file>"; ` +
`a Python test's line then ends in its format: "unittest" or "ok lines")`,
);
// One line per test: two would let a part and `--run` each read a different one.
if (seen.has(match[1]))
throw new Error(`${LIST}: "${match[1]}" is listed twice (one line per test)`);
seen.add(match[1]);
// A Python test has two formats (countTests()), and says which: none is read as the other.
const python = match[1].endsWith(".py");
if (match[4] && !python)
Expand Down
43 changes: 43 additions & 0 deletions .github/scripts/fork-ci-tests.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,49 @@ test("the list ignores comments and blank lines and reads `run by` and weights",
"a.test.py Unittest",
])
assert.throws(() => readList(`${bad}\n`), /cannot read/, bad);
assert.throws(
() => readList("a.test.py unittest ~12s\n"),
/"; a Python test's line then ends in its format: "unittest" or "ok lines"\)$/,
);
// One line per test: a second line for a file is refused, whatever it says.
for (const twice of [
"a.test.sh\nb.test.sh\na.test.sh\n",
"a.test.py unittest\na.test.py ok lines\n",
"a.test.py ~40s unittest\na.test.py run by c.yml unittest\n",
])
assert.throws(
() => readList(twice),
/"a\.test\.(sh|py)" is listed twice \(one line per test\)/,
twice,
);
});

// A Python test the runner never runs (upstream's, which a plain step runs) leaves no record, so the runner never reads
// its format word: its word is checked here against its source instead.
test("a listed Python test the runner does not judge says the format its source has", () => {
const root = path.resolve(__dirname, "../..");
const read = (file) => fs.readFileSync(path.join(root, file), "utf8");
const entries = readList(read(".github/fork-ci-tests.txt"));
const present = entries.map((e) => e.file).filter((file) => fs.existsSync(path.join(root, file)));
const keys = expectedKeys(entries, present, read);
const unjudged = entries.filter(
(e) =>
e.format &&
present.includes(e.file) &&
!keys.some((key) => key === e.file || key.startsWith(`${e.file} --run`)),
);
assert.ok(unjudged.some((e) => e.file === ".github/scripts/stage-preview-bundle.test.py"));
for (const { file, format } of unjudged) {
const text = read(file);
const unittest = /\bunittest\.main\(|\bunittest\.TestCase\b|\bTextTestRunner\b/.test(text);
const okLines = /all passed: /.test(text);
assert.equal(unittest !== okLines, true, `${file}: its source shows no single format`);
assert.equal(
format,
unittest ? "unittest" : "ok lines",
`${file}: its source is ${unittest ? "unittest" : "ok lines"}`,
);
}
});

test("the heaviest tests start first, tests without a weight before them, ties in list order", () => {
Expand Down
17 changes: 11 additions & 6 deletions fork-ci/actions/doc-check/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,12 @@ action fails a pull request that:
- puts an instruction file over its **size budget** (lines or bytes; above `warn_at` of a budget it only warns, and
says when this change moved the file past it);
- writes **history** into an instruction file (a changelog block, an "is live on" claim, a dated "Updated" banner);
- has a **broken relative link** in a Markdown file it adds or changes (code fences skipped; `/x` is from the
repository root, as GitHub reads it; a link that leaves the repository is broken);
- has a **broken relative link** in a Markdown file it adds or changes (code fences, opened as CommonMark does, and inline code
skipped; `/x` is from the repository root, as GitHub reads it; a link that leaves the repository is broken), or deletes or renames a
file that another Markdown file still links to;
- has a checked file that is **not UTF-8** or cannot be read, or is a **symlink** out of the repository, into
`.git` or to no file (a symlink is checked as the file it points to and reported once when its target is checked
too; sizes are the bytes on disk, CRLF included), or deletes, renames or moves away the file a symlink points to;
- **adds** a file whose name looks like a dated report, handoff or review (those go in the pull request or an issue).

The general rules this enforces, and the rest of the agent rules, are in
Expand Down Expand Up @@ -76,11 +80,12 @@ comments. An unknown key, a value of the wrong type or a pattern that is not a r
- `links`: `false` turns link checks off.

The config is read at the base commit, never from the checkout, so a pull request cannot loosen the rules that judge
it: a change to the config applies from the next pull request. `mode: all` and `--files` read it from the checkout.
it through the config: a change to the config applies from the next pull request. `mode: all` and `--files` read it
from the checkout. The workflow that calls this action is not covered by that: GitHub runs a pull request's own
workflow file, so a change that edits the step or removes it is stopped only by review of `.github/workflows/`.

## Not covered

- A deleted or renamed file breaks the links that point to it from files the change did not touch; `mode: all`
- Anchors (`#section`) and queries (`?plain=1`) are not checked, and neither are backticked paths in the text.
- A broken link that was already on the base in a file the change does not touch is not this change's: `mode: all`
finds those.
- Anchors (`#section`) are not checked, and neither are backticked paths in the text.
- A symlinked Markdown file is listed as a warning and not checked.
3 changes: 2 additions & 1 deletion fork-ci/actions/doc-check/action.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
name: doc-check
description: >-
Doc rules for the Markdown files agents read: size budgets for instruction files, no history in them, every
relative link resolves, no dated reports added to the repository. Rules from the project's optional config, read at
relative link resolves, no dated reports added to the repository; a checked file is UTF-8, and a symlink is
checked as its target, which must be a file in the repository (not in .git). Rules from the project's optional config, read at
the base commit. Fails on a broken rule; warns near a budget. Usage: README.md next to this file.
inputs:
config:
Expand Down
Loading
Loading