Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion .github/fork-ci-release.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"_comment": "The fork-ci release (.github/scripts/fork-ci-release.py, fork-ci/README.md). paths: the only files a release commit holds (fork-kit: host kits, installed from a release commit), at the same paths as on sourceBranch. legacyTags: tags made before release commits existed (they name a sourceBranch commit, not a release commit, and stay where they are).",
"_comment": "The fork-ci release (.github/scripts/fork-ci-release.py, fork-ci/README.md). paths: the only files a release commit holds (fork-kit: host kits, installed from a release commit; the desk and heavy kits also read the fleet scripts and the capacity rule files listed here), at the same paths as on sourceBranch; fork-ci-release.test.py fails when a kit installer reads a file outside them. legacyTags: tags made before release commits existed (they name a sourceBranch commit, not a release commit, and stay where they are).",
"sourceBranch": "main",
"requiredCheck": "fork-ci-result",
"branch": "ci-release",
Expand All @@ -16,6 +16,19 @@
".github/scripts/fork-fleet-walls",
".github/scripts/fork-fleet-launch.sh",
".github/scripts/fork-fleet-job-paths.txt",
".github/scripts/fork-fleet-desk-agent.py",
".github/scripts/fork-fleet-desk-agent.service",
".github/scripts/fork-fleet-desk-seed.py",
".github/scripts/fork-fleet-desk-wire.py",
".github/scripts/fork-fleet-hook.sh",
".github/scripts/fork-fleet-refresh.service",
".github/scripts/fork-fleet-refresh.timer",
".github/scripts/fork-fleet-seed.service",
".github/scripts/fork-fleet-seed.timer",
".github/fork-fleet.json",
"packages/shared/src/capacity/capacity-rule.json",
"packages/shared/src/capacity/capacity-format.json",
"packages/shared/src/capacity/capacity-vectors.json",
"scripts/install.sh"
]
}
3 changes: 2 additions & 1 deletion .github/scripts/fork-ci-release.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
onto upstream, so only a tag keeps one of its commits alive.

A release commit holds only the files in .github/fork-ci-release.json `paths` (fork-ci/, the
reusable workflows consumers call, scripts an action runs from outside fork-ci/, LICENSE), at the same paths as on the source branch, so a
reusable workflows consumers call, scripts an action runs from outside fork-ci/, the host kits of fork-kit/ and
the files their installers read from outside it, LICENSE), at the same paths as on the source branch, so a
consumer's `uses:` path never changes. It is built with git plumbing from one source-branch commit,
its parent is the previous release commit, and it is pushed as the branch `branch` (ci-release) and
the annotated tag <tagPrefix><N> (ci/vN) in ONE atomic push. Consumers pin `@<release sha> # ci/vN`.
Expand Down
108 changes: 108 additions & 0 deletions .github/scripts/fork-ci-release.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,114 @@ def unpacked(name):
real_ok and entries, why)
check("real tree: fork-ci-release.yml itself is not released (it starts on schedule)",
real_ok and not any(p.endswith("fork-ci-release.yml") for _, _, p in entries))
# --- every kit installs from this commit's release ---------------------------------------------------
# A kit's bootstrap downloads a release commit, which holds only the config's paths, and runs the kit's installer
# on it; the kits' own tests run from a full checkout, so a file the installer reads from outside those paths
# passes them and fails every real install (heavy-install refused "capacity/capacity-rule.json is missing" on every
# release after the capacity rule moved to packages/shared/src/capacity). Here the release's files are written to a
# folder exactly as release_entries picks them, and each kit's installer, loaded from that folder, runs its own
# file check on it: the same list and the same path rules install uses. A new kit with an installer needs a row.
KIT_CHECK = r"""
import importlib.machinery, importlib.util, json, os, sys
top, kit, how = sys.argv[1:4]
src = os.path.join(top, 'fork-kit', kit)
def load(rel, name):
loader = importlib.machinery.SourceFileLoader(name, os.path.join(src, rel))
mod = importlib.util.module_from_spec(importlib.util.spec_from_loader(name, loader))
loader.exec_module(mod)
return mod
installer = sys.argv[4] if len(sys.argv) > 4 else ''
if how == 'kit_install': # agent-guards, cleaner, golive, schedule-kick: check_options' file part
m = load(installer, 'kit_installer')
inst, missing = m.INSTALLER, []
for rel in inst.source_files():
try:
inst.read_source(os.path.realpath(src), rel)
except m.kit_install.Refused as e:
missing.append(f'{rel}: {e}')
if not missing:
inst.check_same_installer(os.path.realpath(src))
elif how == 'source_file': # heavy (check_options), model-guard (install): SOURCE_FILES through source_file
m = load(installer, 'kit_installer')
missing = [rel for rel in m.SOURCE_FILES if not os.path.isfile(m.source_file(src, rel))]
elif how == 'gh-apps': # copy_tool: SOURCE_FILES under the kit folder
sys.path.insert(0, os.path.join(src, 'lib'))
from ghapps import install
missing = [rel for rel in install.SOURCE_FILES if not os.path.isfile(os.path.join(src, rel))]
elif how == 'desk': # install: missing_sources of the release's top folder
missing = load(installer, 'kit_installer').missing_sources(top)
elif how == 'pc-agents': # setup: CODE sources under the release's top folder
missing = [s for s in load(installer, 'kit_installer').k.CODE if not os.path.isfile(os.path.join(top, s))]
print(json.dumps(missing))
"""
# kit -> (how its installer checks its files, the installer as the bootstrap runs it)
KITS = {"agent-guards": ("kit_install", "sbin/agent-guards-install"),
"cleaner": ("kit_install", "sbin/fork-cleaner-install"),
"golive": ("kit_install", "sbin/t3-golive-install"),
"schedule-kick": ("kit_install", "sbin/schedule-kick-install"),
"heavy": ("source_file", "sbin/heavy-install"),
"model-guard": ("source_file", "sbin/model-guard-install"),
"gh-apps": ("gh-apps", "sbin/gh-apps-install"),
"desk": ("desk", "sbin/desk-install"),
"pc-agents": ("pc-agents", "sbin/pc-agents")}


def write_release(repo, entries, dest):
"""The files of a release tree, as a downloaded release commit holds them (plain files, their modes)."""
batch = subprocess.run(["git", "-C", str(repo), "cat-file", "--batch"], capture_output=True, check=True,
input="".join(f"{oid}\n" for _, oid, _ in entries).encode()).stdout
at = 0
for mode, oid, path in entries:
header_end = batch.index(b"\n", at)
size = int(batch[at:header_end].split()[2])
data = batch[header_end + 1:header_end + 1 + size]
at = header_end + 2 + size
p = dest / path
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
os.chmod(p, 0o755 if mode == "100755" else 0o644)


def kit_release_problems(top):
"""{kit: [what its installer misses in the release at TOP]}, plus kits with an installer and no row."""
problems = {}
for kit_dir in sorted((top / "fork-kit").iterdir()) if (top / "fork-kit").is_dir() else []:
if (kit_dir / "sbin").is_dir() and kit_dir.name not in KITS:
problems[kit_dir.name] = ["has an installer (sbin/) but no row in KITS of fork-ci-release.test.py"]
for kit, (how, installer) in KITS.items():
if not (top / "fork-kit" / kit / installer).is_file():
problems[kit] = [f"fork-kit/{kit}/{installer} is not in the release"]
continue
r = subprocess.run([sys.executable, "-I", "-B", "-c", KIT_CHECK, str(top), kit, how, installer],
capture_output=True, text=True, cwd=work)
try:
missing = json.loads(r.stdout) if r.returncode == 0 else None
except ValueError:
missing = None
if missing is None:
problems[kit] = [f"its file check failed to run: {(r.stderr or r.stdout).strip()[-400:]}"]
elif missing:
problems[kit] = missing
return problems


if real_ok:
rel_top = work / "release-of-head"
write_release(root, entries, rel_top)
kit_problems = kit_release_problems(rel_top)
for kit in sorted(set(KITS) | set(kit_problems)):
check(f"release holds what the {kit} kit's installer reads", kit not in kit_problems,
"; ".join(kit_problems.get(kit, []))[:1500] + " (add the path(s) to .github/fork-ci-release.json paths)")
else:
check("release holds what each kit's installer reads (the release tree above was refused)", False, why)
# The check itself: a kit whose file is not in the release fails (heavy without its capacity files, as on ci/v21).
if real_ok:
mutated = work / "release-without-capacity"
write_release(root, [e for e in entries if not e[2].startswith("packages/shared/src/capacity/")], mutated)
caught = kit_release_problems(mutated).get("heavy", [])
check("release check: a release without packages/shared/src/capacity fails the heavy kit",
any("capacity/capacity-rule.json" in m for m in caught), str(caught))

for bad in ("/abs", "a/../b", "dir/", ""):
t = work / f"cfg-{abs(hash(bad))}.json"
t.write_text(json.dumps({**CFG, "paths": [bad]}))
Expand Down
11 changes: 7 additions & 4 deletions fork-ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,14 @@ upstream, so its old commits disappear. `ci/v1` is such a main commit: it is kep
do not use it.

A release commit holds only what consumers use: `LICENSE`, `fork-ci/`, the host kits in `fork-kit/`
(installed only from a release commit, [fork-kit/README.md](../fork-kit/README.md)), the scripts an action runs from
outside `fork-ci/` (fleet-route runs `.github/scripts/fork-fleet.py`) and the reusable workflows that start
only on `workflow_call` (the list is `paths` in
(installed only from a release commit, [fork-kit/README.md](../fork-kit/README.md)) and the files their installers
read from outside `fork-kit/` (the desk kit's fleet scripts and units, the heavy kit's capacity rule files in
`packages/shared/src/capacity/`), the scripts an action runs from outside `fork-ci/` (fleet-route runs
`.github/scripts/fork-fleet.py`) and the reusable workflows that start only on `workflow_call` (the list is `paths` in
[`.github/fork-ci-release.json`](../.github/fork-ci-release.json)). The paths are the same as on main.
Release commits contain no symlinks and no workflow that starts by itself.
Release commits contain no symlinks and no workflow that starts by itself. `fork-ci-release.test.py` writes this
commit's release files to a folder and runs each kit installer's file check on it, so a kit that reads a file outside
the paths fails CI.

## Using the actions

Expand Down
8 changes: 7 additions & 1 deletion fork-kit/desk/sbin/desk-install
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,12 @@ def lock(stack):
raise Refused('another desk-install install or uninstall is running: wait for it and run again') from None


def missing_sources(source):
"""The files (and the walls folder) of a release that install reads and SOURCE lacks; [] when it has them all.
.github/scripts/fork-ci-release.test.py runs it on the files a release commit holds."""
return [s for s in [*FILES.values(), *UNITS.values(), WALLS] if not os.path.exists(os.path.join(source, s))]


def install(a, stop_after=None, stack=None):
def done(step):
if step == stop_after:
Expand All @@ -519,7 +525,7 @@ def install(a, stop_after=None, stack=None):
raise Refused('--fleet-config and --release-fleet-config: one or the other')
if not SHA.fullmatch(a.sha or ''):
raise Refused('--sha must be the full 40-character release commit')
missing = [s for s in [*FILES.values(), *UNITS.values(), WALLS] if not os.path.exists(os.path.join(a.source, s))]
missing = missing_sources(a.source)
if missing:
raise Refused(f'--source {a.source}: {missing[0]} is missing (point it at the release\'s top folder)')
if a.fleet_config and not os.path.isfile(a.fleet_config):
Expand Down
14 changes: 1 addition & 13 deletions fork-kit/pc-agents/tests/setup.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -972,19 +972,7 @@ def test_owner_steps_run_as_written_in_powershell(self):
self.assertTrue(line.startswith('wsl -d <distro> -u root -e '), line)


class ReleaseHolds(unittest.TestCase):
"""Setup copies k.CODE out of the release commit the bootstrap downloads, and a release commit holds only the
paths of .github/fork-ci-release.json: every source must be in one of them (setup crashed on a real distro
with FileNotFoundError for fork-fleet-walls.py, since the tests run from a full checkout)."""

def test_every_code_source_is_in_a_release(self):
root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..'))
with open(os.path.join(root, '.github', 'fork-ci-release.json')) as f:
paths = json.load(f)['paths']
missing = [src for src in k.CODE if not any(src == p or src.startswith(p.rstrip('/') + '/') for p in paths)]
self.assertEqual(missing, [], 'add these to .github/fork-ci-release.json paths')
for src in k.CODE:
self.assertTrue(os.path.isfile(os.path.join(root, src)), src)
# Whether a release holds every CODE source: .github/scripts/fork-ci-release.test.py, for every kit (one place).


if __name__ == '__main__':
Expand Down
Loading