Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 98 additions & 10 deletions .github/scripts/fork-contracts.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1004,11 +1004,11 @@ def gate_step(wf, job, name_part):
# within hours (review B). The read step runs only for the sweep, holds only the job's read-only token
# (no secret), takes the record only from a run of fork-secret-sweep.yml (that path, schedule or
# workflow_dispatch, this branch, this repository and not a fork, ended success or failure, an ancestor
# commit, not this run but for a re-run's own first attempt, in a run of the sweep's workflow) and its
# commit, not this run but for a re-run's own newest earlier attempt, in a run of the sweep's workflow) and its
# one artifact of that name, never unpacks the archive (one plain file read in memory, every line
# checked), and falls back to a first run on any doubt; it reads nothing while a tag is named like the
# branch (#703); reset_record leaves an empty record instead (a reset run's re-run: its own first
# attempt's record, else the empty one). Its output `read` (found, none, reset, failed) tells the scan
# branch (#703); reset_record leaves an empty record instead (a reset run's re-run: its own newest
# earlier attempt's record, else, or on any failed or doubtful read, the empty one, #791). Its output `read` (found, none, reset, failed) tells the scan
# step whether a partial record may replace the last one (#703). Its behaviour: the reader e2e below.
# The keep step uploads only on a branch, once the scan step has written the record whole (its output
# `record`) or reset_record left the empty one, for 90 days (the repository's most). What the scan step
Expand All @@ -1022,9 +1022,15 @@ def gate_step(wf, job, name_part):
mkdir -m 700 -- "$dir"
# none STATE WHY: no record. STATE, the output `read`: "none" when no earlier sweep kept one,
# "failed" when one may exist but cannot be read or trusted; then a partial record of this
# run must not replace it (#703).
# run must not replace it (#703). A reset run on a branch (its re-run reads its own record)
# takes the empty record instead of "failed": it trusts nothing, so it is always safe, and a
# failed call cannot leave the record from before the reset in use (#791).
none() {
rm -rf -- "$zip" "$dir/deleted-tips"
if [[ "$1" == failed && "$RESET" == true && "$GITHUB_REF" == refs/heads/* ]]; then
echo "::warning title=Secret scan::$2" >&2
empty_record
fi
echo "read=$1" >> "${GITHUB_OUTPUT:?}"
echo "::warning title=Secret scan::$2, so every listed tip is checked as on a first run" >&2
exit 0
Expand All @@ -1040,8 +1046,9 @@ def gate_step(wf, job, name_part):
fi
# reset_record: an empty record (the header only), which the last step keeps even when this
# run fails before the scan step writes its own, so no later sweep reads the older one (#703).
# A re-run of a reset run (it keeps the input) first takes its own earlier attempt's record,
# which was written after the reset, and no older run's; without one, the empty record.
# A re-run of a reset run (it keeps the input) first takes its own newest earlier attempt's
# record, which was written after the reset, and no older run's; without one, or when it cannot
# be read, the empty record.
empty_record() {
echo "fork secret sweep record 2" > "$dir/deleted-tips"
echo "read=reset" >> "${GITHUB_OUTPUT:?}"
Expand Down Expand Up @@ -1508,8 +1515,12 @@ def lookup_problems(scan):
problems.append(f"{SCAN_FILE}/{j}: LOOKUP_TOKEN is not the job's token")
# The lookups' time limit (#703) must leave the steps before this one, and the lookup under
# way when it is passed, inside the job's time-out (else the job is killed and keeps nothing).
# The limit line missing or reworded fails too, never skips the check (#791).
limit, timeout = re.search(r"\(\(SECONDS > (\d+)\)\)", run), job.get("timeout-minutes")
if limit and not (type(timeout) is int and int(limit.group(1)) + LOOKUP_LIMIT_MARGIN <= timeout * 60):
if not limit:
problems.append(f"{SCAN_FILE}/{j}: no lookups' time limit in the form ((SECONDS > N)), so it cannot be "
"checked against the job's timeout-minutes (#703, #791)")
elif not (type(timeout) is int and int(limit.group(1)) + LOOKUP_LIMIT_MARGIN <= timeout * 60):
problems.append(f"{SCAN_FILE}/{j}: the lookups' time limit ({limit.group(1)} s) is not at least "
f"{LOOKUP_LIMIT_MARGIN} s inside the job's timeout-minutes ({timeout!r}), #703")
if env.get("RECORD_READ") != "${{ steps.read.outputs.read }}":
Expand Down Expand Up @@ -1597,6 +1608,15 @@ def tests_job_problems(scan):

# The sweep's one input to the scan: reset_record, only when a hand run asks for it (#701).
SWEEP_WITH = {"reset_record": "${{ inputs.reset_record == true }}"}
# Sweeps of one ref one at a time, the running one never cancelled (#791): a sweep that starts while
# a reset run runs waits for it and reads its record, so it cannot undo the reset unseen.
SWEEP_CONCURRENCY = {"group": "fork-secret-sweep-${{ github.ref }}", "cancel-in-progress": False}


def same_group(group, *others):
"""True when GROUP is one of OTHERS, spaces aside (`${{ github.ref }}` and `${{github.ref}}` are one group)."""
key = lambda g: re.sub(r"\s+", "", str(g))
return any(o is not None and key(group) == key(o) for o in others)


def sweep_problems(wfs):
Expand All @@ -1616,6 +1636,9 @@ def sweep_problems(wfs):
f"({', '.join(sorted(events - {'schedule', 'workflow_dispatch'}))})")
if sweep.get("permissions") != {"contents": "read"}:
problems.append(f"{SWEEP_FILE}: workflow permissions are not exactly contents: read")
if sweep.get("concurrency") != SWEEP_CONCURRENCY:
problems.append(f"{SWEEP_FILE}: the workflow's concurrency is not one group per ref that never cancels the "
"running sweep (a sweep started during a reset run would undo the reset, #791)")
jobs = sweep.get("jobs") or {}
callers = [j for j, job in jobs.items() if job.get("uses") == SCAN_CALL]
if len(callers) != 1:
Expand Down Expand Up @@ -1652,6 +1675,11 @@ def sweep_problems(wfs):
conc = job.get("concurrency")
if not (isinstance(conc, dict) and conc.get("group") and conc.get("cancel-in-progress") is False):
problems.append(f"{SWEEP_FILE}/{j}: no concurrency group that queues (two sweeps at once would each open an issue)")
elif same_group(conc["group"], SWEEP_CONCURRENCY["group"], (sweep.get("concurrency") or {}).get("group")):
# A job group equal to its own run's workflow group: GitHub sees a deadlock and cancels the
# job, so a failed sweep would alert no one (#791).
problems.append(f"{SWEEP_FILE}/{j}: its concurrency group is the workflow's (GitHub cancels the alert "
"as a deadlock)")
for step in job.get("steps") or []:
run = " ".join(str(step.get("run", "")).split())
if "fork-run-alert.sh" not in run:
Expand Down Expand Up @@ -1955,6 +1983,18 @@ def keep_step(wfs):
"if", "${{ github.event_name == 'schedule' }}"),
"an alert not after the sweep": lambda w: w[SWEEP_FILE]["jobs"]["alert"].pop("needs"),
"an alert without its concurrency group": lambda w: w[SWEEP_FILE]["jobs"]["alert"].pop("concurrency"),
# #791: the sweeps of one ref one at a time, the running one never cancelled.
"a sweep without its concurrency group": lambda w: w[SWEEP_FILE].pop("concurrency"),
"a sweep whose concurrency cancels the running sweep": lambda w: w[SWEEP_FILE]["concurrency"].__setitem__(
"cancel-in-progress", True),
"a sweep whose concurrency group is one run's own (nothing waits)": lambda w: w[SWEEP_FILE]["concurrency"].__setitem__(
"group", "fork-secret-sweep-${{ github.run_id }}"),
"a sweep whose concurrency group is the alert's": lambda w: w[SWEEP_FILE]["concurrency"].__setitem__(
"group", "fork-secret-sweep-alert"),
"an alert whose concurrency group is the sweep's (a deadlock: GitHub cancels the alert)": lambda w: w[SWEEP_FILE][
"jobs"]["alert"]["concurrency"].__setitem__("group", "fork-secret-sweep-${{ github.ref }}"),
"an alert whose concurrency group is the sweep's, spaced otherwise": lambda w: w[SWEEP_FILE][
"jobs"]["alert"]["concurrency"].__setitem__("group", "fork-secret-sweep-${{github.ref}}"),
"an alert whose concurrency cancels a running one": lambda w: w[SWEEP_FILE]["jobs"]["alert"]["concurrency"].__setitem__(
"cancel-in-progress", True),
"an alert that may fail quietly": lambda w: w[SWEEP_FILE]["jobs"]["alert"].__setitem__("continue-on-error", True),
Expand All @@ -1970,6 +2010,18 @@ def keep_step(wfs):
check(f"secret-scan: {SCAN_FILE} with {label} fails" if "fork-tests" not in label and "sweep" not in label
and "alert" not in label else f"secret-scan: {label} fails", scan_mutant(mutate))

# #791: the lookups' time-limit line reworded or gone: its own check against the job's time-out fails
# (not only the exact-text contract), so it can never pass quietly by not finding the line.
for what, new in (("reworded (>=)", "if ((SECONDS >= 1200)); then"), ("gone", "if false; then")):
tl_wfs = copy.deepcopy(real)
tl_step = scan_step(tl_wfs)
tl_run = tl_step["run"].replace("if ((SECONDS > 1200)); then", new)
tl_step["run"] = tl_run
tl_found = [p for p in lookup_problems(tl_wfs[SCAN_FILE]) if "no lookups' time limit" in p]
check(f"secret-scan: the lookups' time-limit line {what}: its check against the job's timeout-minutes fails, "
"not skips (#791)", tl_run != scan_step(real)["run"] and bool(tl_found),
"the line not found in the real file" if tl_run == scan_step(real)["run"] else "no time-limit problem")

check("secret-scan: the sweep's activity lookup (#239) in its exact form passes",
not scan_mutant(lambda w: scan_step(w).__setitem__("run", with_sweep_lookup(scan_step(w)["run"])))
and SWEEP_LOOKUP.search(with_sweep_lookup(scan_step(real)["run"])) is not None)
Expand Down Expand Up @@ -2699,6 +2751,12 @@ def ok(s):
return ok


def reset_empty(s):
"""A reset run's empty record (the header only), read=reset, no older run listed, nothing left."""
return (s.r.returncode == 0 and s.record == RECORD_HEAD and s.output == "read=reset\n" and s.left == []
and "actions/workflows" not in s.calls and "an empty record is kept even if this run fails" in s.out)


def two_runs(bad):
"""The newest run (802) is one the step must not read, with a record of its own; 801 is good."""
return {"runs": [a_run(802, **bad), a_run(801)],
Expand Down Expand Up @@ -2771,10 +2829,32 @@ def stale_before(temp):
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2"},
lambda s: s.r.returncode == 0 and s.record == RECORD_HEAD and s.output == "read=reset\n"
and "runs/900/artifacts" in s.calls and "actions/workflows" not in s.calls and s.left == []),
"a re-run of a reset_record run whose own record cannot be listed: no record, read=failed, so its first "
"attempt's stays the one in use (#703)": (
# #791: any failed or doubtful read in a reset run's re-run gives the empty record, never read=failed:
# if its first attempt kept nothing, read=failed would leave the record from before the reset in use.
"a re-run of a reset_record run whose own record cannot be listed: the empty record, no older run "
"listed (#791)": (
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "FAKE_FAIL": "/artifacts -f",
"artifacts": {900: [an_artifact(903, 900)]}, "zips": {903: good_zip}}, read_as(None)),
"artifacts": {900: [an_artifact(903, 900)]}, "zips": {903: good_zip}},
lambda s: reset_empty(s) and "runs/900/artifacts" in s.calls and "cannot be listed" in s.out),
"a re-run of a reset_record run whose tag check fails: the empty record (#791)": (
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "FAKE_FAIL": "matching-refs",
"artifacts": {900: [an_artifact(903, 900)]}, "zips": {903: good_zip}},
lambda s: reset_empty(s) and "runs/900/" not in s.calls and "cannot be checked" in s.out),
"a re-run of a reset_record run while a tag is named like the branch: the empty record (#791)": (
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "tags": ["refs/tags/main"],
"artifacts": {900: [an_artifact(903, 900)]}, "zips": {903: good_zip}},
lambda s: reset_empty(s) and "runs/900/" not in s.calls),
"a re-run of a reset_record run whose own record cannot be downloaded: the empty record (#791)": (
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "artifacts": {900: [an_artifact(903, 900)]}, "zips": {}},
lambda s: reset_empty(s) and "artifacts/903/zip" in s.calls),
"a re-run of a reset_record run whose own record cannot be read: the empty record, not a partial bad one "
"(#791)": (
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "artifacts": {900: [an_artifact(903, 900)]},
"zips": {903: a_zip([("deleted-tips", good_record + "junk\n")])}},
lambda s: reset_empty(s) and "artifacts/903/zip" in s.calls),
"a re-run of a reset_record run whose commit check fails: the empty record (#791)": (
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "FAKE_FAIL": "compare/",
"artifacts": {900: [an_artifact(903, 900)]}, "zips": {903: good_zip}}, reset_empty),
"a first attempt: its own run never asked about": (
{}, lambda s: read_as(good_record)(s) and "runs/900/" not in s.calls),
"no sweep run kept one yet (or it expired): a first run": ({"runs": []}, read_as(None, state="none")),
Expand Down Expand Up @@ -2881,6 +2961,9 @@ def read_results(code, names=None, first_red=False):
for n, (what, setup, want_record, want_output) in enumerate((
("reset_record", {"RESET": "true"}, RECORD_HEAD + f"{budget_tips[0]} scanned\n", "record=written\n"),
("a failed read (the runs cannot be listed)", {"FAKE_FAIL": "actions/workflows"}, None, ""),
("a reset run's re-run whose own record cannot be listed (#791)",
{"RESET": "true", "GITHUB_RUN_ATTEMPT": "2", "FAKE_FAIL": "/artifacts -f", "artifacts": {900: [an_artifact(903, 900)]}},
RECORD_HEAD + f"{budget_tips[0]} scanned\n", "record=written\n"),
)):
r = read_record(read_code, **setup)
read = re.fullmatch(r"read=(\w+)\n", r.output)
Expand Down Expand Up @@ -2931,6 +3014,11 @@ def read_results(code, names=None, first_red=False):
("the whole-text match", "re.fullmatch(", "re.match("),
("the duplicate check", "if len(set(tips)) != len(tips):", "if False:"),
("the fresh folder", 'rm -rf -- "$dir" "$zip"\n', ""),
("the empty record on a failed read in a reset run (#791)",
' if [[ "$1" == failed && "$RESET" == true && "$GITHUB_REF" == refs/heads/* ]]; then\n'
' echo "::warning title=Secret scan::$2" >&2\n empty_record\n fi\n', ""),
("the branch check of the empty record on a failed read (#791)",
' && "$GITHUB_REF" == refs/heads/* ]]; then', " ]]; then"),
):
mutated = read_code.replace(old, new)
results = read_results(mutated, first_red=True) if mutated != read_code else {}
Expand Down
26 changes: 18 additions & 8 deletions .github/workflows/fork-secret-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@ name: Fork secret scan
# request that check also reads every earlier head of the PR's branch (the scan step), since only
# the newest head's check counts.
# - workflow_call from fork-secret-sweep.yml (daily, and by hand): the sweep, every ref but pull
# request heads, for what no push run read. Same script and settings; no concurrency group. That
# workflow opens an issue when the sweep fails (a schedule run alerts no one otherwise).
# request heads, for what no push run read. Same script and settings; no concurrency group here
# (that workflow runs its sweeps on one ref one at a time, for the record). That workflow opens
# an issue when the sweep fails (a schedule run alerts no one otherwise).
# The scan script always comes from the default branch's tip, never from the checkout: a pull
# request or a branch cannot change how it is scanned by editing the script, and an old branch
# gets today's scan. The checkout's own copy runs only in the `tests` job, on a runner of its own:
Expand Down Expand Up @@ -72,9 +73,11 @@ jobs:
# (on main: a merge after green CI; admins may bypass), and it can only spare tips GitHub's
# API calls gone. A run on a tag named like the branch has that name as its head_branch too:
# while such a tag exists no record is read (#703); a ruleset that blocks creating it closes
# the case of one created and deleted again. A re-run of a sweep run first takes its own
# earlier attempt's record (#703), a reset run's re-run too. The output `read` says how the
# read ended, for the scan step's partial record.
# the case of one created and deleted again. A re-run of a sweep run is meant to take its own
# newest earlier attempt's record first (#703), a reset run's re-run too; not yet confirmed
# live (#950): when it is not found, a reset run's re-run takes the empty record and any other
# re-run the newest earlier sweep's. Never re-run a finished reset run; start a new reset run
# by hand. The output `read` says how the read ended, for the scan step's partial record.
- name: Read the sweep's record
id: read
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
Expand All @@ -89,9 +92,15 @@ jobs:
mkdir -m 700 -- "$dir"
# none STATE WHY: no record. STATE, the output `read`: "none" when no earlier sweep kept one,
# "failed" when one may exist but cannot be read or trusted; then a partial record of this
# run must not replace it (#703).
# run must not replace it (#703). A reset run on a branch (its re-run reads its own record)
# takes the empty record instead of "failed": it trusts nothing, so it is always safe, and a
# failed call cannot leave the record from before the reset in use (#791).
none() {
rm -rf -- "$zip" "$dir/deleted-tips"
if [[ "$1" == failed && "$RESET" == true && "$GITHUB_REF" == refs/heads/* ]]; then
echo "::warning title=Secret scan::$2" >&2
empty_record
fi
echo "read=$1" >> "${GITHUB_OUTPUT:?}"
echo "::warning title=Secret scan::$2, so every listed tip is checked as on a first run" >&2
exit 0
Expand All @@ -107,8 +116,9 @@ jobs:
fi
# reset_record: an empty record (the header only), which the last step keeps even when this
# run fails before the scan step writes its own, so no later sweep reads the older one (#703).
# A re-run of a reset run (it keeps the input) first takes its own earlier attempt's record,
# which was written after the reset, and no older run's; without one, the empty record.
# A re-run of a reset run (it keeps the input) first takes its own newest earlier attempt's
# record, which was written after the reset, and no older run's; without one, or when it cannot
# be read, the empty record.
empty_record() {
echo "fork secret sweep record 2" > "$dir/deleted-tips"
echo "read=reset" >> "${GITHUB_OUTPUT:?}"
Expand Down
Loading
Loading