diff --git a/.github/scripts/render_schema.py b/.github/scripts/render_schema.py index 044ee22..a558cc4 100644 --- a/.github/scripts/render_schema.py +++ b/.github/scripts/render_schema.py @@ -15,6 +15,12 @@ authored there plus its own index, since the client tries its bases in order and a 404 falls straight through to the tree below. +A schema may also introduce a definition outright, listing it under `introduces`. +That definition is not rendered into any tree below and does not appear in their +indexes, so a binary reading an older schema neither lists it nor can fetch it. +This is the version gate: a service that offers an older lerd nothing at all, +because what drives it shipped in a later release, is simply not published to it. + Usage: render_schema.py """ @@ -119,7 +125,7 @@ def index_for(docs): return {"services": entries} -def write_index(path, published, docs, owned): +def write_index(path, published, docs, owned, drop=None): """Rewrite only the entries this render owns. The published index is hand-maintained and does not always match what a @@ -130,13 +136,16 @@ def write_index(path, published, docs, owned): """ entries = index_for(docs)["services"] fresh = {e["name"]: e for e in entries if e.get("name") in owned} + drop = drop or set() out, seen = [], set() if published.exists(): with open(published, "r", encoding="utf-8") as handle: for entry in json.load(handle).get("services", []): name = entry.get("name") - out.append(fresh.get(name, entry)) seen.add(name) + if name in drop: + continue + out.append(fresh.get(name, entry)) for entry in entries: if entry.get("name") not in seen: out.append(entry) @@ -153,6 +162,16 @@ def main(): oldest = 1 newest = max(v for v, _ in specs) + # name -> the schema that introduced it. A tree below that schema neither + # carries the definition nor lists it. + introduced_at = {} + for version, spec in specs: + for name in spec.get("introduces", []) or []: + introduced_at[name] = version + + def visible_at(name, version): + return introduced_at.get(name, oldest) <= version + # A definition is authored in the highest schema tree it needs, and every # tree below renders down from it. One that needs nothing newer is authored # in services/ and published as it stands. @@ -160,17 +179,23 @@ def main(): sources = {p.stem: load_yaml(p) for p in sorted(top.glob("*.yaml"))} if top.exists() else {} plain = {p.stem: load_yaml(p) for p in sorted(LEGACY.glob("*.yaml")) if p.stem not in sources} - inert = [] + inert, withheld = [], [] for name, doc in sorted(sources.items()): + if not visible_at(name, oldest): + withheld.append(name) + (LEGACY / f"{name}.yaml").unlink(missing_ok=True) + continue low = render_to(doc, oldest, specs) dump(low, LEGACY / f"{name}.yaml") if low == doc: inert.append(name) write_index(LEGACY / "index.json", LEGACY / "index.json", - list(plain.values()) + [render_to(d, oldest, specs) for d in sources.values()], - set(sources)) - print(f"schema {oldest}: {len(plain)} authored in place, {len(sources)} rendered -> services") + list(plain.values()) + + [render_to(d, oldest, specs) for n, d in sources.items() if visible_at(n, oldest)], + set(sources), drop={n for n in sources if not visible_at(n, oldest)}) + print(f"schema {oldest}: {len(plain)} authored in place, " + f"{len(sources) - len(withheld)} rendered -> services") for version, _ in sorted(specs): if version <= oldest: @@ -179,6 +204,9 @@ def main(): out_dir.mkdir(parents=True, exist_ok=True) carried = 0 for name, doc in sorted(sources.items()): + if not visible_at(name, version): + (out_dir / f"{name}.yaml").unlink(missing_ok=True) + continue # The newest tree is authored, not rendered: leave its bytes alone. if version < newest: high = render_to(doc, version, specs) @@ -188,10 +216,13 @@ def main(): else: carried += 1 write_index(out_dir / "index.json", LEGACY / "index.json", - list(plain.values()) + [render_to(d, version, specs) for d in sources.values()], - set(sources)) + list(plain.values()) + + [render_to(d, version, specs) for n, d in sources.items() if visible_at(n, version)], + set(sources), drop={n for n in sources if not visible_at(n, version)}) print(f"schema {version}: {carried} definition(s) -> {out_dir.relative_to(ROOT)}") + if withheld: + print(f"withheld from older schemas: {', '.join(sorted(withheld))}") if inert: print(f"note: {', '.join(inert)} render the same in every schema and need no source") print(f"authored schema is {newest}") diff --git a/.github/scripts/test_render_schema.py b/.github/scripts/test_render_schema.py new file mode 100644 index 0000000..b0c5da7 --- /dev/null +++ b/.github/scripts/test_render_schema.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Checks on the renderer, run in CI beside the render itself. + +Each case builds a throwaway store in a temp dir, renders it, and reads the +published trees back, so the checks describe what a binary would fetch rather +than how the renderer is written. +""" + +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + +SCRIPT = pathlib.Path(__file__).resolve() + + +def build(root, schema_yaml, legacy=None, top=None): + (root / ".github" / "scripts").mkdir(parents=True) + shutil.copy2(SCRIPT.parent / "render_schema.py", root / ".github" / "scripts" / "render_schema.py") + (root / "schema").mkdir() + (root / "schema" / "2.yaml").write_text(schema_yaml) + (root / "services").mkdir() + for name, body in (legacy or {}).items(): + (root / "services" / f"{name}.yaml").write_text(body) + top_dir = root / "schema" / "2" / "services" + top_dir.mkdir(parents=True) + for name, body in (top or {}).items(): + (top_dir / f"{name}.yaml").write_text(body) + out = subprocess.run([sys.executable, str(root / ".github" / "scripts" / "render_schema.py")], + capture_output=True, text=True) + if out.returncode != 0: + raise SystemExit(f"render failed:\n{out.stdout}\n{out.stderr}") + return out.stdout + + +def names_in(path): + with open(path, "r", encoding="utf-8") as handle: + return {e["name"] for e in json.load(handle).get("services", [])} + + +def case_introduces_is_withheld_from_older_schemas(): + """A definition a schema introduces reaches that schema's tree and no lower one.""" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + build(root, + "schema: 2\nsince_lerd: \"1.36.0\"\nintroduces:\n - newthing\nchanges: []\n", + legacy={"oldthing": "name: oldthing\nimage: x\ndescription: d\ncategory: c\nicon: i\n"}, + top={"newthing": "name: newthing\nimage: y\ndescription: d\ncategory: c\nicon: i\n"}) + + assert not (root / "services" / "newthing.yaml").exists(), \ + "an introduced definition must not be published to the older tree" + assert (root / "schema" / "2" / "services" / "newthing.yaml").exists(), \ + "an introduced definition must stay in the tree that introduced it" + assert "newthing" not in names_in(root / "services" / "index.json"), \ + "an introduced definition must not be listed in the older index" + assert "newthing" in names_in(root / "schema" / "2" / "services" / "index.json"), \ + "an introduced definition must be listed in its own index" + assert "oldthing" in names_in(root / "services" / "index.json"), \ + "an ordinary definition stays listed everywhere" + assert "oldthing" in names_in(root / "schema" / "2" / "services" / "index.json"), \ + "an ordinary definition is listed in every schema's index" + + +def case_key_downgrade_still_applies(): + """A key a schema adds is still rendered away for the older tree.""" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + build(root, + "schema: 2\nsince_lerd: \"1.36.0\"\nchanges:\n - path: fancy\n downgrade: drop\n", + top={"thing": "name: thing\nimage: y\ndescription: d\ncategory: c\nicon: i\nfancy: true\n"}) + low = (root / "services" / "thing.yaml").read_text() + high = (root / "schema" / "2" / "services" / "thing.yaml").read_text() + assert "fancy" not in low, "the older tree must not carry a key the schema added" + assert "fancy" in high, "the introducing tree keeps the key" + + +def case_guarded_drop_reads_the_source(): + """A `when` guard reads the document as authored, not as half rendered.""" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + build(root, + "schema: 2\nsince_lerd: \"1.36.0\"\nchanges:\n" + " - path: flag\n downgrade: drop\n" + " - path: dashboard\n downgrade: drop\n when: flag\n", + top={"thing": "name: thing\nimage: y\ndescription: d\ncategory: c\nicon: i\n" + "flag: true\ndashboard: http://localhost:1/\n"}) + low = (root / "services" / "thing.yaml").read_text() + assert "dashboard" not in low, \ + "a guarded drop must fire even when an earlier rule removed the key it reads" + + +def main(): + cases = [v for k, v in sorted(globals().items()) if k.startswith("case_")] + for case in cases: + case() + print(f"ok {case.__name__}") + print(f"\n{len(cases)} check(s) passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/schema-guard.yml b/.github/workflows/schema-guard.yml index ac1fa57..fd7cda6 100644 --- a/.github/workflows/schema-guard.yml +++ b/.github/workflows/schema-guard.yml @@ -17,6 +17,9 @@ jobs: python-version: "3.12" - run: pip install pyyaml + - name: Check the renderer + run: python3 .github/scripts/test_render_schema.py + - name: Render every schema run: python3 .github/scripts/render_schema.py diff --git a/schema/2.yaml b/schema/2.yaml index a5a424d..4ba7840 100644 --- a/schema/2.yaml +++ b/schema/2.yaml @@ -8,6 +8,13 @@ schema: 2 since_lerd: "1.36.0" +# Definitions this schema introduces. One listed here is published to this tree +# and no lower one, so a binary reading an older schema neither lists it nor can +# fetch it. For a service that would offer that binary nothing at all, because +# whatever drives it shipped later, that is kinder than publishing a preset which +# installs and then sits there doing nothing. +introduces: [] + changes: # Keys schema 1 has no field for. An old binary ignores them, so dropping is # about publishing a tree that means what it says rather than about safety.