fix: find a definition in whichever schema tree it is authored in

.github/scripts/render_schema.py

@@ -175,8 +175,20 @@ def visible_at(name, version):
     # A definition is authored in the highest schema tree it needs, and every
     # tree below renders down from it. One that needs nothing newer is authored
     # in services/ and published as it stands.
-    top = SCHEMA_DIR / str(newest) / "services"
-    sources = {p.stem: load_yaml(p) for p in sorted(top.glob("*.yaml"))} if top.exists() else {}
+    #
+    # Every schema tree is searched, not just the newest: a definition authored
+    # at schema 2 stays there when schema 3 arrives, and looking only at the top
+    # would leave it in neither set and drop it from the published trees.
+    sources, authored_at = {}, {}
+    for version in sorted({v for v, _ in specs}, reverse=True):
+        tree = SCHEMA_DIR / str(version) / "services"
+        if not tree.exists():
+            continue
+        for path in sorted(tree.glob("*.yaml")):
+            if path.stem in sources:
+                continue
+            sources[path.stem] = load_yaml(path)
+            authored_at[path.stem] = version
     plain = {p.stem: load_yaml(p) for p in sorted(LEGACY.glob("*.yaml")) if p.stem not in sources}
 
     inert, withheld = [], []
@@ -207,8 +219,8 @@ def visible_at(name, version):
             if not visible_at(name, version):
                 (out_dir / f"{name}.yaml").unlink(missing_ok=True)
                 continue
-            # The newest tree is authored, not rendered: leave its bytes alone.
-            if version < newest:
+            # The tree a definition is authored in keeps its bytes.
+            if version < authored_at.get(name, newest):
                 high = render_to(doc, version, specs)
                 if high != render_to(doc, oldest, specs):
                     dump(high, out_dir / f"{name}.yaml")

.github/scripts/test_render_schema.py

@@ -16,6 +16,28 @@
 SCRIPT = pathlib.Path(__file__).resolve()
 
 
+def build_multi(root, schemas, legacy=None, trees=None):
+    """A store with several schema trees. `schemas` maps version to its yaml."""
+    (root / ".github" / "scripts").mkdir(parents=True)
+    shutil.copy2(SCRIPT.parent / "render_schema.py", root / ".github" / "scripts" / "render_schema.py")
+    (root / "schema").mkdir()
+    for version, body in schemas.items():
+        (root / "schema" / f"{version}.yaml").write_text(body)
+    (root / "services").mkdir()
+    for name, body in (legacy or {}).items():
+        (root / "services" / f"{name}.yaml").write_text(body)
+    for version, files in (trees or {}).items():
+        tree = root / "schema" / str(version) / "services"
+        tree.mkdir(parents=True)
+        for name, body in files.items():
+            (tree / f"{name}.yaml").write_text(body)
+    out = subprocess.run([sys.executable, str(root / ".github" / "scripts" / "render_schema.py")],
+                         capture_output=True, text=True)
+    if out.returncode != 0:
+        raise SystemExit(f"render failed:\n{out.stdout}\n{out.stderr}")
+    return out.stdout
+
+
 def build(root, schema_yaml, legacy=None, top=None):
     (root / ".github" / "scripts").mkdir(parents=True)
     shutil.copy2(SCRIPT.parent / "render_schema.py", root / ".github" / "scripts" / "render_schema.py")
@@ -91,6 +113,29 @@ def case_guarded_drop_reads_the_source():
             "a guarded drop must fire even when an earlier rule removed the key it reads"
 
 
+def case_a_definition_authored_below_the_top_survives():
+    """A definition authored at schema 2 is still found once schema 3 exists."""
+    with tempfile.TemporaryDirectory() as tmp:
+        root = pathlib.Path(tmp)
+        build_multi(root,
+                    {2: "schema: 2\nsince_lerd: \"1.36.0\"\nintroduces:\n  - gated\nchanges: []\n",
+                     3: "schema: 3\nsince_lerd: \"1.40.0\"\nintroduces: []\nchanges:\n"
+                        "  - path: newkey\n    downgrade: drop\n"},
+                    trees={2: {"gated": "name: gated\nimage: g\ndescription: d\ncategory: c\nicon: i\n"},
+                           3: {"fresh": "name: fresh\nimage: f\ndescription: d\ncategory: c\nicon: i\nnewkey: 1\n"}})
+
+        assert (root / "schema" / "2" / "services" / "gated.yaml").exists(), \
+            "a definition authored below the top tree must not be dropped"
+        assert not (root / "services" / "gated.yaml").exists(), \
+            "an introduced definition stays withheld from the tree below it"
+        assert "gated" in names_in(root / "schema" / "2" / "services" / "index.json"), \
+            "it stays listed in the schema that introduced it"
+        assert "gated" in names_in(root / "schema" / "3" / "services" / "index.json"), \
+            "a later schema still lists a definition introduced earlier"
+        assert "gated" not in names_in(root / "services" / "index.json"), \
+            "and the oldest tree still does not"
+
+
 def main():
     cases = [v for k, v in sorted(globals().items()) if k.startswith("case_")]
     for case in cases: