diff --git a/.github/scripts/render_schema.py b/.github/scripts/render_schema.py new file mode 100644 index 0000000..044ee22 --- /dev/null +++ b/.github/scripts/render_schema.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Render the definitions that differ between schemas into their published trees. + +Every binary in the field computes its own store URL and cannot be taught a new +one, so the unprefixed path has to keep carrying what those binaries expect. +That is the oldest schema still supported, and it is where a definition lives by +default: services/.yaml is authored, published as it stands, and copied +nowhere. + +A definition is authored in the highest schema tree it needs, and every tree +below renders down from it. One needing nothing newer is authored in services/ +and published as it stands; one carrying a key that would be wrong to publish to +an older binary is authored in schema/N/services/ instead, and services/ gets the +downgrade rendered from it. A schema tree is sparse, holding only the definitions +authored there plus its own index, since the client tries its bases in order and +a 404 falls straight through to the tree below. + +Usage: render_schema.py +""" + +import copy +import json +import pathlib +import sys +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[2] +LEGACY = ROOT / "services" +SCHEMA_DIR = ROOT / "schema" + +INDEX_FIELDS = ["name", "description", "family", "dashboard", "image", "category", + "icon", "color", "admin_for", "admin_rank", "depends_on", + "versions", "default_version", "env_role"] + + +def load_yaml(path): + with open(path, "r", encoding="utf-8") as handle: + return yaml.safe_load(handle) or {} + + +def schemas(): + out = [] + for path in sorted(SCHEMA_DIR.glob("*.yaml"), key=lambda p: int(p.stem)): + spec = load_yaml(path) + out.append((int(spec["schema"]), spec)) + return out + + +def resolve(doc, path): + node = doc + for part in path.split("."): + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + +def drop(doc, path): + parts = path.split(".") + node = doc + for part in parts[:-1]: + if not isinstance(node, dict) or part not in node: + return + node = node[part] + if isinstance(node, dict): + node.pop(parts[-1], None) + + +def apply_change(doc, change, guard): + """Render one change backwards, newer schema to older. + + A `when` guard reads the document as it entered this schema's step, not the + half-rendered one: a rule commonly depends on a key an earlier rule in the + same step has already dropped. + """ + if "when" in change and not resolve(guard, change["when"]): + return + path, how = change["path"], change["downgrade"] + if how == "drop": + drop(doc, path) + elif isinstance(how, dict) and "join" in how: + value = resolve(doc, path) + if isinstance(value, list): + parts = path.split(".") + node = doc + for part in parts[:-1]: + node = node[part] + node[parts[-1]] = how["join"].join(str(v) for v in value) + elif isinstance(how, dict) and "rename" in how: + value = resolve(doc, path) + if value is not None: + drop(doc, path) + doc[how["rename"]] = value + else: + raise SystemExit(f"unknown downgrade {how!r} for {path}") + + +def render_to(doc, target, specs): + out = copy.deepcopy(doc) + for version, spec in sorted(specs, reverse=True): + if version <= target: + continue + guard = copy.deepcopy(out) + for change in spec.get("changes", []): + apply_change(out, change, guard) + return out + + +def dump(doc, path): + with open(path, "w", encoding="utf-8") as handle: + yaml.safe_dump(doc, handle, sort_keys=False, default_flow_style=False, allow_unicode=True) + + +def index_for(docs): + entries = [] + for doc in sorted(docs, key=lambda d: d.get("name", "")): + entries.append({f: doc[f] for f in INDEX_FIELDS + if f in doc and doc[f] not in (None, "", [], {})}) + return {"services": entries} + + +def write_index(path, published, docs, owned): + """Rewrite only the entries this render owns. + + The published index is hand-maintained and does not always match what a + projection of the YAML would produce. Regenerating it wholesale would push + those differences to every install as a change nobody asked for, so entries + for definitions this render does not touch are carried through exactly as + they stand. + """ + entries = index_for(docs)["services"] + fresh = {e["name"]: e for e in entries if e.get("name") in owned} + out, seen = [], set() + if published.exists(): + with open(published, "r", encoding="utf-8") as handle: + for entry in json.load(handle).get("services", []): + name = entry.get("name") + out.append(fresh.get(name, entry)) + seen.add(name) + for entry in entries: + if entry.get("name") not in seen: + out.append(entry) + with open(path, "w", encoding="utf-8") as handle: + json.dump({"services": out}, handle, indent=2) + handle.write("\n") + + +def main(): + specs = schemas() + if not specs: + print("no schema deltas; nothing to render") + return 0 + oldest = 1 + newest = max(v for v, _ in specs) + + # A definition is authored in the highest schema tree it needs, and every + # tree below renders down from it. One that needs nothing newer is authored + # in services/ and published as it stands. + top = SCHEMA_DIR / str(newest) / "services" + sources = {p.stem: load_yaml(p) for p in sorted(top.glob("*.yaml"))} if top.exists() else {} + plain = {p.stem: load_yaml(p) for p in sorted(LEGACY.glob("*.yaml")) if p.stem not in sources} + + inert = [] + for name, doc in sorted(sources.items()): + low = render_to(doc, oldest, specs) + dump(low, LEGACY / f"{name}.yaml") + if low == doc: + inert.append(name) + + write_index(LEGACY / "index.json", LEGACY / "index.json", + list(plain.values()) + [render_to(d, oldest, specs) for d in sources.values()], + set(sources)) + print(f"schema {oldest}: {len(plain)} authored in place, {len(sources)} rendered -> services") + + for version, _ in sorted(specs): + if version <= oldest: + continue + out_dir = SCHEMA_DIR / str(version) / "services" + out_dir.mkdir(parents=True, exist_ok=True) + carried = 0 + for name, doc in sorted(sources.items()): + # The newest tree is authored, not rendered: leave its bytes alone. + if version < newest: + high = render_to(doc, version, specs) + if high != render_to(doc, oldest, specs): + dump(high, out_dir / f"{name}.yaml") + carried += 1 + else: + carried += 1 + write_index(out_dir / "index.json", LEGACY / "index.json", + list(plain.values()) + [render_to(d, version, specs) for d in sources.values()], + set(sources)) + print(f"schema {version}: {carried} definition(s) -> {out_dir.relative_to(ROOT)}") + + if inert: + print(f"note: {', '.join(inert)} render the same in every schema and need no source") + print(f"authored schema is {newest}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/schema_guard.py b/.github/scripts/schema_guard.py index 4051259..14ac466 100755 --- a/.github/scripts/schema_guard.py +++ b/.github/scripts/schema_guard.py @@ -75,6 +75,26 @@ def profile(root, pattern): return out +def schema_dropped_paths(root): + """Paths a schema delta renders away, which are removals by design. + + The published legacy tree is the oldest schema, so every key a later schema + added is absent from it on purpose. Without this the guard would read each + of those as a key that disappeared. + """ + dropped = set() + for path in sorted(pathlib.Path(root, "schema").glob("*.yaml")): + try: + with open(path, "r", encoding="utf-8") as handle: + spec = yaml.safe_load(handle) or {} + except (OSError, yaml.YAMLError): + continue + for change in spec.get("changes", []): + if change.get("downgrade") == "drop" and "path" in change: + dropped.add(change["path"]) + return dropped + + def closed_sets(profiles): """Union every file's observed values per path, keeping the small ones.""" merged = {} @@ -96,6 +116,7 @@ def main(): published = profile(published_dir, pattern) candidate = profile(candidate_dir, pattern) enums = closed_sets(published) + by_design = schema_dropped_paths(candidate_dir) failures, warnings = [], [] @@ -110,6 +131,8 @@ def main(): for path, kinds in sorted(old_types.items()): if path not in new_types: + if path.split("[]")[0] in by_design: + continue failures.append(f"{rel}: key `{path}` was removed, an older lerd still reads it") continue if kinds != new_types[path] and not kinds & new_types[path]: diff --git a/.github/workflows/schema-guard.yml b/.github/workflows/schema-guard.yml index c8bcd6d..ac1fa57 100644 --- a/.github/workflows/schema-guard.yml +++ b/.github/workflows/schema-guard.yml @@ -3,22 +3,40 @@ name: Schema guard on: pull_request: -# A store definition reaches every install within a day, whatever version of -# lerd it runs, so a pull request may only grow the schema. This refuses a key -# that disappeared or changed type, and reports a value no published definition -# has used before. +# A definition is authored in the highest schema tree it needs and rendered down +# into every tree below. Two things have to hold on every pull request: the lower +# trees are what the authored ones render to, and the lowest stays readable by +# the binaries that fetch it. jobs: + render: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml + + - name: Render every schema + run: python3 .github/scripts/render_schema.py + + - name: The published trees must match the sources + run: | + if ! git diff --quiet; then + echo "The rendered trees are stale. Run .github/scripts/render_schema.py and commit the result." + git diff --stat + exit 1 + fi + schema-guard: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install pyyaml - name: Extract the published tree diff --git a/schema/2.yaml b/schema/2.yaml new file mode 100644 index 0000000..a5a424d --- /dev/null +++ b/schema/2.yaml @@ -0,0 +1,41 @@ +# Schema 2, as understood by lerd 1.36.0 and later. +# +# A schema states only what changed since the one before it, and how to render a +# document back down to that one. A definition is authored in the highest schema +# tree it needs and rendered down into every tree below, the lowest being the +# unprefixed path: that is what every binary up to 1.35.0 computes for itself and +# cannot be taught otherwise. +schema: 2 +since_lerd: "1.36.0" + +changes: + # Keys schema 1 has no field for. An old binary ignores them, so dropping is + # about publishing a tree that means what it says rather than about safety. + - path: admin_rank + downgrade: drop + - path: dashboard_follows_color_scheme + downgrade: drop + - path: dashboard_login + downgrade: drop + - path: dashboard_proxy_at_path + downgrade: drop + - path: dashboard_proxy_keep_host + downgrade: drop + - path: dashboard_proxy_rebase + downgrade: drop + - path: dashboard_proxy_reroute + downgrade: drop + - path: dashboard_proxy_strip + downgrade: drop + - path: dashboard_scheme_key + downgrade: drop + + # This one is not cosmetic. Schema 2 proxies a dashboard at /_svc// and + # strips the upstream's X-Frame-Options and frame-ancestors on the way through. + # Schema 1 has no such proxy and opens the dashboard URL directly in an iframe, + # so a UI that refuses framing, which is every upstream needing the strip flag, + # renders a blocked panel. Better to publish no dashboard to schema 1 than a + # card that cannot work: the service still installs, runs and serves its port. + - path: dashboard + downgrade: drop + when: dashboard_proxy_strip diff --git a/schema/2/services/adminer.yaml b/schema/2/services/adminer.yaml new file mode 100644 index 0000000..c7ef790 --- /dev/null +++ b/schema/2/services/adminer.yaml @@ -0,0 +1,147 @@ +name: adminer +image: docker.io/library/adminer:6 +update_strategy: rolling +description: Adminer web UI for every database lerd runs +ports: +- 8081:8080 +dynamic_env: + ADMINER_MYSQL_HOSTS: discover_family:mysql,mariadb + ADMINER_PGSQL_HOSTS: discover_family:postgres +files: +- target: /var/www/html/index.php + content: "servers[\\Adminer\\SERVER];\n\ + \t\t\t\treturn array($s['server'], $s['username'], $s['password']);\n\t\t\t}\n\ + \t\t\tfunction login($login, $password) { return true; }\n\t\t}\n\t\treturn new\ + \ \\Adminer\\Plugins(array(new LerdServers(\\lerd_servers())));\n\t}\n}\nnamespace\ + \ {\n\tfunction lerd_servers() {\n\t\tstatic $out = null;\n\t\tif ($out !== null)\ + \ { return $out; }\n\t\t$families = array(\n\t\t\t'ADMINER_MYSQL_HOSTS' =>\ + \ array('server', 'root', 'lerd'),\n\t\t\t'ADMINER_PGSQL_HOSTS' =>\ + \ array('pgsql', 'postgres', 'lerd'),\n\t\t);\n\t\t$out = array();\n\t\t\ + foreach ($families as $env => $spec) {\n\t\t\tforeach (array_filter(explode(',',\ + \ (string) getenv($env))) as $host) {\n\t\t\t\t$out[$host] = array('server' =>\ + \ $host, 'driver' => $spec[0], 'username' => $spec[1], 'password' => $spec[2]);\n\ + \t\t\t}\n\t\t}\n\t\tif (!$out) {\n\t\t\t$out['lerd-mysql'] = array('server' =>\ + \ 'lerd-mysql', 'driver' => 'server', 'username' => 'root', 'password' => 'lerd');\n\ + \t\t}\n\t\treturn $out;\n\t}\n\n\tfunction lerd_query_key(array $s) {\n\t\treturn\ + \ $s['driver'] === 'server' ? 'server' : $s['driver'];\n\t}\n\n\t// Which server\ + \ actually holds this database. The dashboard deep-links a\n\t// database by name\ + \ alone, so without this a Postgres database opens against\n\t// MySQL and Adminer\ + \ answers \"Invalid database\".\n\tfunction lerd_server_with_db($db) {\n\t\tforeach\ + \ (lerd_servers() as $host => $s) {\n\t\t\ttry {\n\t\t\t\tif ($s['driver'] ===\ + \ 'server' && extension_loaded('mysqli')) {\n\t\t\t\t\t$c = @new mysqli($host,\ + \ $s['username'], $s['password']);\n\t\t\t\t\tif ($c->connect_errno) { continue;\ + \ }\n\t\t\t\t\t$q = $c->query(\"SHOW DATABASES LIKE '\" . $c->real_escape_string($db)\ + \ . \"'\");\n\t\t\t\t\t$hit = $q && $q->num_rows > 0;\n\t\t\t\t\t$c->close();\n\ + \t\t\t\t\tif ($hit) { return $host; }\n\t\t\t\t} elseif ($s['driver'] === 'pgsql'\ + \ && extension_loaded('pdo_pgsql')) {\n\t\t\t\t\t$p = new PDO(\"pgsql:host=$host;dbname=postgres\"\ + , $s['username'], $s['password'], array(PDO::ATTR_TIMEOUT => 2));\n\t\t\t\t\t\ + $st = $p->prepare('SELECT 1 FROM pg_database WHERE datname = ?');\n\t\t\t\t\t\ + $st->execute(array($db));\n\t\t\t\t\tif ($st->fetchColumn()) { return $host; }\n\ + \t\t\t\t}\n\t\t\t} catch (\\Throwable $e) {\n\t\t\t\t// A server that is down\ + \ must not stop the search.\n\t\t\t}\n\t\t}\n\t\treturn null;\n\t}\n\n\t// The\ + \ dashboard proxy forwards its own /_svc// prefix, and Adminer's own\n\t\ + // assets are query strings rather than paths, so they survive it. A design\n\t\ + // stylesheet is the exception: Adminer links it as a plain file name, which\n\ + \t// arrives here under the prefix, misses on disk and falls through to this\n\ + \t// script. Serve it rather than treat it as a page to redirect.\n\t$asset =\ + \ basename(parse_url($_SERVER['REQUEST_URI'] ?? '', PHP_URL_PATH) ?: '');\n\t\ + if (preg_match('~^[\\\\w.-]+\\\\.(css|js|png|gif|ico|svg|woff2?)$~', $asset) &&\ + \ is_file(__DIR__ . '/' . $asset)) {\n\t\t$types = array('css' => 'text/css',\ + \ 'js' => 'application/javascript', 'png' => 'image/png',\n\t\t\t'gif' => 'image/gif',\ + \ 'ico' => 'image/x-icon', 'svg' => 'image/svg+xml',\n\t\t\t'woff' => 'font/woff',\ + \ 'woff2' => 'font/woff2');\n\t\t$ext = strtolower(pathinfo($asset, PATHINFO_EXTENSION));\n\ + \t\theader('Content-Type: ' . ($types[$ext] ?? 'application/octet-stream'));\n\ + \t\treadfile(__DIR__ . '/' . $asset);\n\t\texit;\n\t}\n\n\t$servers = lerd_servers();\n\ + \t// Seeded and closed before adminer.php is included, with the cache limiter\n\ + \t// off, so its own session_start sends no headers after ours have gone out.\n\ + \tini_set('session.cache_limiter', '');\n\tob_start();\n\tsession_name('adminer_sid');\n\ + \tsession_start();\n\tforeach ($servers as $host => $s) {\n\t\tif (!isset($_SESSION['pwds'][$s['driver']][$host][$s['username']]))\ + \ {\n\t\t\t$_SESSION['pwds'][$s['driver']][$host][$s['username']] = $s['password'];\n\ + \t\t\t$_SESSION['db'][$s['driver']][$host][$s['username']] = array();\n\t\t}\n\ + \t}\n\tsession_write_close();\n\tob_end_clean();\n\n\t// Which server this request\ + \ is for. A request that already names one is left\n\t// alone; anything else\ + \ is answered with a redirect to the canonical URL rather\n\t// than by rewriting\ + \ the request in place, because the versions disagree about\n\t// whether they\ + \ build their own links from $_GET or from the raw query string,\n\t// and a rewrite\ + \ that one of them ignores turns into a redirect loop.\n\t$named = false;\n\t\ + foreach ($servers as $host => $s) {\n\t\tif (isset($_GET[lerd_query_key($s)]))\ + \ { $named = true; break; }\n\t}\n\t// An asset request carries file=; it needs\ + \ no server and redirecting it only\n\t// costs a round trip before Adminer serves\ + \ the same bytes.\n\tif (!$named && !isset($_GET['file'])) {\n\t\t$pick = null;\n\ + \t\t// lerd_server is how the dashboard says which engine it opened this for,\n\ + \t\t// without having to know the driver or the credentials that go with it.\n\ + \t\tif (isset($_GET['lerd_server']) && isset($servers[$_GET['lerd_server']]))\ + \ {\n\t\t\t$pick = $_GET['lerd_server'];\n\t\t} elseif (isset($_GET['db']) &&\ + \ $_GET['db'] !== '') {\n\t\t\t$pick = lerd_server_with_db($_GET['db']);\n\t\t\ + }\n\t\tif ($pick === null) { $pick = key($servers); }\n\t\t$s = $servers[$pick];\n\ + \t\t$q = $_GET;\n\t\tunset($q['lerd_server']);\n\t\t$q[lerd_query_key($s)] = $pick;\n\ + \t\t$q['username'] = $s['username'];\n\t\t// A Postgres database is only half\ + \ an address; without a schema the hub\n\t\t// of the URL is incomplete and 6\ + \ bounces it back rather than opening it.\n\t\tif ($s['driver'] === 'pgsql' &&\ + \ !empty($q['db']) && !isset($q['ns'])) {\n\t\t\t$q['ns'] = 'public';\n\t\t}\n\ + \t\theader('Location: ?' . http_build_query($q), true, 302);\n\t\texit;\n\t}\n\ + \n\t// Adminer 5 registers a plugin hook only for methods that exist on its core\n\ + \t// object, and navigation() is a plain function there, so a plugin cannot add\n\ + \t// to the sidebar. The switcher is spliced into the rendered page instead, from\n\ + \t// an output-buffer callback: Adminer exits on most paths, and a callback still\n\ + \t// runs at shutdown where code after the include would not.\n\tfunction lerd_switcher($html)\ + \ {\n\t\t$servers = lerd_servers();\n\t\tif (count($servers) < 2 || strpos($html,\ + \ \"