feat: let a schema introduce a definition, not just change one

.github/scripts/render_schema.py

@@ -15,6 +15,12 @@
 authored there plus its own index, since the client tries its bases in order and
 a 404 falls straight through to the tree below.
 
+A schema may also introduce a definition outright, listing it under `introduces`.
+That definition is not rendered into any tree below and does not appear in their
+indexes, so a binary reading an older schema neither lists it nor can fetch it.
+This is the version gate: a service that offers an older lerd nothing at all,
+because what drives it shipped in a later release, is simply not published to it.
+
 Usage: render_schema.py
 """
 
@@ -119,7 +125,7 @@ def index_for(docs):
     return {"services": entries}
 
 
-def write_index(path, published, docs, owned):
+def write_index(path, published, docs, owned, drop=None):
     """Rewrite only the entries this render owns.
 
     The published index is hand-maintained and does not always match what a
@@ -130,13 +136,16 @@ def write_index(path, published, docs, owned):
     """
     entries = index_for(docs)["services"]
     fresh = {e["name"]: e for e in entries if e.get("name") in owned}
+    drop = drop or set()
     out, seen = [], set()
     if published.exists():
         with open(published, "r", encoding="utf-8") as handle:
             for entry in json.load(handle).get("services", []):
                 name = entry.get("name")
-                out.append(fresh.get(name, entry))
                 seen.add(name)
+                if name in drop:
+                    continue
+                out.append(fresh.get(name, entry))
     for entry in entries:
         if entry.get("name") not in seen:
             out.append(entry)
@@ -153,24 +162,40 @@ def main():
     oldest = 1
     newest = max(v for v, _ in specs)
 
+    # name -> the schema that introduced it. A tree below that schema neither
+    # carries the definition nor lists it.
+    introduced_at = {}
+    for version, spec in specs:
+        for name in spec.get("introduces", []) or []:
+            introduced_at[name] = version
+
+    def visible_at(name, version):
+        return introduced_at.get(name, oldest) <= version
+
     # A definition is authored in the highest schema tree it needs, and every
     # tree below renders down from it. One that needs nothing newer is authored
     # in services/ and published as it stands.
     top = SCHEMA_DIR / str(newest) / "services"
     sources = {p.stem: load_yaml(p) for p in sorted(top.glob("*.yaml"))} if top.exists() else {}
     plain = {p.stem: load_yaml(p) for p in sorted(LEGACY.glob("*.yaml")) if p.stem not in sources}
 
-    inert = []
+    inert, withheld = [], []
     for name, doc in sorted(sources.items()):
+        if not visible_at(name, oldest):
+            withheld.append(name)
+            (LEGACY / f"{name}.yaml").unlink(missing_ok=True)
+            continue
         low = render_to(doc, oldest, specs)
         dump(low, LEGACY / f"{name}.yaml")
         if low == doc:
             inert.append(name)
 
     write_index(LEGACY / "index.json", LEGACY / "index.json",
-                list(plain.values()) + [render_to(d, oldest, specs) for d in sources.values()],
-                set(sources))
-    print(f"schema {oldest}: {len(plain)} authored in place, {len(sources)} rendered -> services")
+                list(plain.values())
+                + [render_to(d, oldest, specs) for n, d in sources.items() if visible_at(n, oldest)],
+                set(sources), drop={n for n in sources if not visible_at(n, oldest)})
+    print(f"schema {oldest}: {len(plain)} authored in place, "
+          f"{len(sources) - len(withheld)} rendered -> services")
 
     for version, _ in sorted(specs):
         if version <= oldest:
@@ -179,6 +204,9 @@ def main():
         out_dir.mkdir(parents=True, exist_ok=True)
         carried = 0
         for name, doc in sorted(sources.items()):
+            if not visible_at(name, version):
+                (out_dir / f"{name}.yaml").unlink(missing_ok=True)
+                continue
             # The newest tree is authored, not rendered: leave its bytes alone.
             if version < newest:
                 high = render_to(doc, version, specs)
@@ -188,10 +216,13 @@ def main():
             else:
                 carried += 1
         write_index(out_dir / "index.json", LEGACY / "index.json",
-                    list(plain.values()) + [render_to(d, version, specs) for d in sources.values()],
-                    set(sources))
+                    list(plain.values())
+                    + [render_to(d, version, specs) for n, d in sources.items() if visible_at(n, version)],
+                    set(sources), drop={n for n in sources if not visible_at(n, version)})
         print(f"schema {version}: {carried} definition(s) -> {out_dir.relative_to(ROOT)}")
 
+    if withheld:
+        print(f"withheld from older schemas: {', '.join(sorted(withheld))}")
     if inert:
         print(f"note: {', '.join(inert)} render the same in every schema and need no source")
     print(f"authored schema is {newest}")

.github/scripts/test_render_schema.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Checks on the renderer, run in CI beside the render itself.
+
+Each case builds a throwaway store in a temp dir, renders it, and reads the
+published trees back, so the checks describe what a binary would fetch rather
+than how the renderer is written.
+"""
+
+import json
+import pathlib
+import shutil
+import subprocess
+import sys
+import tempfile
+
+SCRIPT = pathlib.Path(__file__).resolve()
+
+
+def build(root, schema_yaml, legacy=None, top=None):
+    (root / ".github" / "scripts").mkdir(parents=True)
+    shutil.copy2(SCRIPT.parent / "render_schema.py", root / ".github" / "scripts" / "render_schema.py")
+    (root / "schema").mkdir()
+    (root / "schema" / "2.yaml").write_text(schema_yaml)
+    (root / "services").mkdir()
+    for name, body in (legacy or {}).items():
+        (root / "services" / f"{name}.yaml").write_text(body)
+    top_dir = root / "schema" / "2" / "services"
+    top_dir.mkdir(parents=True)
+    for name, body in (top or {}).items():
+        (top_dir / f"{name}.yaml").write_text(body)
+    out = subprocess.run([sys.executable, str(root / ".github" / "scripts" / "render_schema.py")],
+                         capture_output=True, text=True)
+    if out.returncode != 0:
+        raise SystemExit(f"render failed:\n{out.stdout}\n{out.stderr}")
+    return out.stdout
+
+
+def names_in(path):
+    with open(path, "r", encoding="utf-8") as handle:
+        return {e["name"] for e in json.load(handle).get("services", [])}
+
+
+def case_introduces_is_withheld_from_older_schemas():
+    """A definition a schema introduces reaches that schema's tree and no lower one."""
+    with tempfile.TemporaryDirectory() as tmp:
+        root = pathlib.Path(tmp)
+        build(root,
+              "schema: 2\nsince_lerd: \"1.36.0\"\nintroduces:\n  - newthing\nchanges: []\n",
+              legacy={"oldthing": "name: oldthing\nimage: x\ndescription: d\ncategory: c\nicon: i\n"},
+              top={"newthing": "name: newthing\nimage: y\ndescription: d\ncategory: c\nicon: i\n"})
+
+        assert not (root / "services" / "newthing.yaml").exists(), \
+            "an introduced definition must not be published to the older tree"
+        assert (root / "schema" / "2" / "services" / "newthing.yaml").exists(), \
+            "an introduced definition must stay in the tree that introduced it"
+        assert "newthing" not in names_in(root / "services" / "index.json"), \
+            "an introduced definition must not be listed in the older index"
+        assert "newthing" in names_in(root / "schema" / "2" / "services" / "index.json"), \
+            "an introduced definition must be listed in its own index"
+        assert "oldthing" in names_in(root / "services" / "index.json"), \
+            "an ordinary definition stays listed everywhere"
+        assert "oldthing" in names_in(root / "schema" / "2" / "services" / "index.json"), \
+            "an ordinary definition is listed in every schema's index"
+
+
+def case_key_downgrade_still_applies():
+    """A key a schema adds is still rendered away for the older tree."""
+    with tempfile.TemporaryDirectory() as tmp:
+        root = pathlib.Path(tmp)
+        build(root,
+              "schema: 2\nsince_lerd: \"1.36.0\"\nchanges:\n  - path: fancy\n    downgrade: drop\n",
+              top={"thing": "name: thing\nimage: y\ndescription: d\ncategory: c\nicon: i\nfancy: true\n"})
+        low = (root / "services" / "thing.yaml").read_text()
+        high = (root / "schema" / "2" / "services" / "thing.yaml").read_text()
+        assert "fancy" not in low, "the older tree must not carry a key the schema added"
+        assert "fancy" in high, "the introducing tree keeps the key"
+
+
+def case_guarded_drop_reads_the_source():
+    """A `when` guard reads the document as authored, not as half rendered."""
+    with tempfile.TemporaryDirectory() as tmp:
+        root = pathlib.Path(tmp)
+        build(root,
+              "schema: 2\nsince_lerd: \"1.36.0\"\nchanges:\n"
+              "  - path: flag\n    downgrade: drop\n"
+              "  - path: dashboard\n    downgrade: drop\n    when: flag\n",
+              top={"thing": "name: thing\nimage: y\ndescription: d\ncategory: c\nicon: i\n"
+                            "flag: true\ndashboard: http://localhost:1/\n"})
+        low = (root / "services" / "thing.yaml").read_text()
+        assert "dashboard" not in low, \
+            "a guarded drop must fire even when an earlier rule removed the key it reads"
+
+
+def main():
+    cases = [v for k, v in sorted(globals().items()) if k.startswith("case_")]
+    for case in cases:
+        case()
+        print(f"ok  {case.__name__}")
+    print(f"\n{len(cases)} check(s) passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/schema-guard.yml

@@ -17,6 +17,9 @@ jobs:
           python-version: "3.12"
       - run: pip install pyyaml
 
+      - name: Check the renderer
+        run: python3 .github/scripts/test_render_schema.py
+
       - name: Render every schema
         run: python3 .github/scripts/render_schema.py
 

schema/2.yaml

@@ -8,6 +8,13 @@
 schema: 2
 since_lerd: "1.36.0"
 
+# Definitions this schema introduces. One listed here is published to this tree
+# and no lower one, so a binary reading an older schema neither lists it nor can
+# fetch it. For a service that would offer that binary nothing at all, because
+# whatever drives it shipped later, that is kinder than publishing a preset which
+# installs and then sits there doing nothing.
+introduces: []
+
 changes:
   # Keys schema 1 has no field for. An old binary ignores them, so dropping is
   # about publishing a tree that means what it says rather than about safety.