From 5c54dcc1be95ddcfb300a80e3bff0c3f0c6f3d97 Mon Sep 17 00:00:00 2001 From: George Dumitrescu Date: Sun, 20 Sep 2026 22:15:09 +0300 Subject: [PATCH] feat: let a schema introduce a definition, not just change one A schema could say a key was new and render it away for an older binary, but not that a whole definition was. So a preset whose point is something that shipped in a later release had no way to be held back: publish it and every install a day later can see it, install it, and find it does nothing. A schema now lists what it introduces. Such a definition is published to that schema's tree and no lower one, so a binary reading an older schema neither lists it in the catalogue nor can fetch its file. That is the version gate, expressed where the rest of the schema differences already live rather than as a field on the definition. The renderer gains checks of its own, run in CI before the render: the introduced case, a key downgrade, and the guarded drop that reads the document as authored. --- .github/scripts/render_schema.py | 47 ++++++++++-- .github/scripts/test_render_schema.py | 104 ++++++++++++++++++++++++++ .github/workflows/schema-guard.yml | 3 + schema/2.yaml | 7 ++ 4 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/test_render_schema.py diff --git a/.github/scripts/render_schema.py b/.github/scripts/render_schema.py index 044ee22..a558cc4 100644 --- a/.github/scripts/render_schema.py +++ b/.github/scripts/render_schema.py @@ -15,6 +15,12 @@ authored there plus its own index, since the client tries its bases in order and a 404 falls straight through to the tree below. +A schema may also introduce a definition outright, listing it under `introduces`. +That definition is not rendered into any tree below and does not appear in their +indexes, so a binary reading an older schema neither lists it nor can fetch it. +This is the version gate: a service that offers an older lerd nothing at all, +because what drives it shipped in a later release, is simply not published to it. + Usage: render_schema.py """ @@ -119,7 +125,7 @@ def index_for(docs): return {"services": entries} -def write_index(path, published, docs, owned): +def write_index(path, published, docs, owned, drop=None): """Rewrite only the entries this render owns. The published index is hand-maintained and does not always match what a @@ -130,13 +136,16 @@ def write_index(path, published, docs, owned): """ entries = index_for(docs)["services"] fresh = {e["name"]: e for e in entries if e.get("name") in owned} + drop = drop or set() out, seen = [], set() if published.exists(): with open(published, "r", encoding="utf-8") as handle: for entry in json.load(handle).get("services", []): name = entry.get("name") - out.append(fresh.get(name, entry)) seen.add(name) + if name in drop: + continue + out.append(fresh.get(name, entry)) for entry in entries: if entry.get("name") not in seen: out.append(entry) @@ -153,6 +162,16 @@ def main(): oldest = 1 newest = max(v for v, _ in specs) + # name -> the schema that introduced it. A tree below that schema neither + # carries the definition nor lists it. + introduced_at = {} + for version, spec in specs: + for name in spec.get("introduces", []) or []: + introduced_at[name] = version + + def visible_at(name, version): + return introduced_at.get(name, oldest) <= version + # A definition is authored in the highest schema tree it needs, and every # tree below renders down from it. One that needs nothing newer is authored # in services/ and published as it stands. @@ -160,17 +179,23 @@ def main(): sources = {p.stem: load_yaml(p) for p in sorted(top.glob("*.yaml"))} if top.exists() else {} plain = {p.stem: load_yaml(p) for p in sorted(LEGACY.glob("*.yaml")) if p.stem not in sources} - inert = [] + inert, withheld = [], [] for name, doc in sorted(sources.items()): + if not visible_at(name, oldest): + withheld.append(name) + (LEGACY / f"{name}.yaml").unlink(missing_ok=True) + continue low = render_to(doc, oldest, specs) dump(low, LEGACY / f"{name}.yaml") if low == doc: inert.append(name) write_index(LEGACY / "index.json", LEGACY / "index.json", - list(plain.values()) + [render_to(d, oldest, specs) for d in sources.values()], - set(sources)) - print(f"schema {oldest}: {len(plain)} authored in place, {len(sources)} rendered -> services") + list(plain.values()) + + [render_to(d, oldest, specs) for n, d in sources.items() if visible_at(n, oldest)], + set(sources), drop={n for n in sources if not visible_at(n, oldest)}) + print(f"schema {oldest}: {len(plain)} authored in place, " + f"{len(sources) - len(withheld)} rendered -> services") for version, _ in sorted(specs): if version <= oldest: @@ -179,6 +204,9 @@ def main(): out_dir.mkdir(parents=True, exist_ok=True) carried = 0 for name, doc in sorted(sources.items()): + if not visible_at(name, version): + (out_dir / f"{name}.yaml").unlink(missing_ok=True) + continue # The newest tree is authored, not rendered: leave its bytes alone. if version < newest: high = render_to(doc, version, specs) @@ -188,10 +216,13 @@ def main(): else: carried += 1 write_index(out_dir / "index.json", LEGACY / "index.json", - list(plain.values()) + [render_to(d, version, specs) for d in sources.values()], - set(sources)) + list(plain.values()) + + [render_to(d, version, specs) for n, d in sources.items() if visible_at(n, version)], + set(sources), drop={n for n in sources if not visible_at(n, version)}) print(f"schema {version}: {carried} definition(s) -> {out_dir.relative_to(ROOT)}") + if withheld: + print(f"withheld from older schemas: {', '.join(sorted(withheld))}") if inert: print(f"note: {', '.join(inert)} render the same in every schema and need no source") print(f"authored schema is {newest}") diff --git a/.github/scripts/test_render_schema.py b/.github/scripts/test_render_schema.py new file mode 100644 index 0000000..b0c5da7 --- /dev/null +++ b/.github/scripts/test_render_schema.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Checks on the renderer, run in CI beside the render itself. + +Each case builds a throwaway store in a temp dir, renders it, and reads the +published trees back, so the checks describe what a binary would fetch rather +than how the renderer is written. +""" + +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + +SCRIPT = pathlib.Path(__file__).resolve() + + +def build(root, schema_yaml, legacy=None, top=None): + (root / ".github" / "scripts").mkdir(parents=True) + shutil.copy2(SCRIPT.parent / "render_schema.py", root / ".github" / "scripts" / "render_schema.py") + (root / "schema").mkdir() + (root / "schema" / "2.yaml").write_text(schema_yaml) + (root / "services").mkdir() + for name, body in (legacy or {}).items(): + (root / "services" / f"{name}.yaml").write_text(body) + top_dir = root / "schema" / "2" / "services" + top_dir.mkdir(parents=True) + for name, body in (top or {}).items(): + (top_dir / f"{name}.yaml").write_text(body) + out = subprocess.run([sys.executable, str(root / ".github" / "scripts" / "render_schema.py")], + capture_output=True, text=True) + if out.returncode != 0: + raise SystemExit(f"render failed:\n{out.stdout}\n{out.stderr}") + return out.stdout + + +def names_in(path): + with open(path, "r", encoding="utf-8") as handle: + return {e["name"] for e in json.load(handle).get("services", [])} + + +def case_introduces_is_withheld_from_older_schemas(): + """A definition a schema introduces reaches that schema's tree and no lower one.""" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + build(root, + "schema: 2\nsince_lerd: \"1.36.0\"\nintroduces:\n - newthing\nchanges: []\n", + legacy={"oldthing": "name: oldthing\nimage: x\ndescription: d\ncategory: c\nicon: i\n"}, + top={"newthing": "name: newthing\nimage: y\ndescription: d\ncategory: c\nicon: i\n"}) + + assert not (root / "services" / "newthing.yaml").exists(), \ + "an introduced definition must not be published to the older tree" + assert (root / "schema" / "2" / "services" / "newthing.yaml").exists(), \ + "an introduced definition must stay in the tree that introduced it" + assert "newthing" not in names_in(root / "services" / "index.json"), \ + "an introduced definition must not be listed in the older index" + assert "newthing" in names_in(root / "schema" / "2" / "services" / "index.json"), \ + "an introduced definition must be listed in its own index" + assert "oldthing" in names_in(root / "services" / "index.json"), \ + "an ordinary definition stays listed everywhere" + assert "oldthing" in names_in(root / "schema" / "2" / "services" / "index.json"), \ + "an ordinary definition is listed in every schema's index" + + +def case_key_downgrade_still_applies(): + """A key a schema adds is still rendered away for the older tree.""" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + build(root, + "schema: 2\nsince_lerd: \"1.36.0\"\nchanges:\n - path: fancy\n downgrade: drop\n", + top={"thing": "name: thing\nimage: y\ndescription: d\ncategory: c\nicon: i\nfancy: true\n"}) + low = (root / "services" / "thing.yaml").read_text() + high = (root / "schema" / "2" / "services" / "thing.yaml").read_text() + assert "fancy" not in low, "the older tree must not carry a key the schema added" + assert "fancy" in high, "the introducing tree keeps the key" + + +def case_guarded_drop_reads_the_source(): + """A `when` guard reads the document as authored, not as half rendered.""" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + build(root, + "schema: 2\nsince_lerd: \"1.36.0\"\nchanges:\n" + " - path: flag\n downgrade: drop\n" + " - path: dashboard\n downgrade: drop\n when: flag\n", + top={"thing": "name: thing\nimage: y\ndescription: d\ncategory: c\nicon: i\n" + "flag: true\ndashboard: http://localhost:1/\n"}) + low = (root / "services" / "thing.yaml").read_text() + assert "dashboard" not in low, \ + "a guarded drop must fire even when an earlier rule removed the key it reads" + + +def main(): + cases = [v for k, v in sorted(globals().items()) if k.startswith("case_")] + for case in cases: + case() + print(f"ok {case.__name__}") + print(f"\n{len(cases)} check(s) passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/schema-guard.yml b/.github/workflows/schema-guard.yml index ac1fa57..fd7cda6 100644 --- a/.github/workflows/schema-guard.yml +++ b/.github/workflows/schema-guard.yml @@ -17,6 +17,9 @@ jobs: python-version: "3.12" - run: pip install pyyaml + - name: Check the renderer + run: python3 .github/scripts/test_render_schema.py + - name: Render every schema run: python3 .github/scripts/render_schema.py diff --git a/schema/2.yaml b/schema/2.yaml index a5a424d..4ba7840 100644 --- a/schema/2.yaml +++ b/schema/2.yaml @@ -8,6 +8,13 @@ schema: 2 since_lerd: "1.36.0" +# Definitions this schema introduces. One listed here is published to this tree +# and no lower one, so a binary reading an older schema neither lists it nor can +# fetch it. For a service that would offer that binary nothing at all, because +# whatever drives it shipped later, that is kinder than publishing a preset which +# installs and then sits there doing nothing. +introduces: [] + changes: # Keys schema 1 has no field for. An old binary ignores them, so dropping is # about publishing a tree that means what it says rather than about safety.