ADFA-6740: docdb-studio and the db writers read and write schema 3.0.0

CLAUDE.md

@@ -85,16 +85,18 @@ edit that database** — it contains no part of the production Android app itsel
 > every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the
 > re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0.
 >
-> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` write plain templates,
-> so each refuses a database declaring 3.0.0 or later before writing anything, backup included.
-> `load_android_json_db.py` keeps the major it finds, so a late run would otherwise leave a file
-> declaring 3.1.0 with plain templates in it, which the app cannot read.
+> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` read and write both
+> (ADFA-6740): below 3.0.0 they write plain templates, from it compressed ones (`brotli -D -w 24`,
+> checked by decoding back, never large-window), and they leave a 3.x file at the version it
+> declares. They refuse a major above 3, a format they do not know, before writing anything.
+> `docdb-studio` reads and writes both shapes too (tooltips through the keyed tables, templates
+> decoded and re-encoded with the dictionary), choosing by the declared major.
 >
-> **Every tool in this repo still reads and writes the 2.x shape**: `check-tools` (whose daily
+> **The other tools still read and write only the 2.x shape**: `check-tools` (whose daily
 > `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`),
-> `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`), `TooltipManager.py` and
-> `docdb-studio`. So the step runs on the copy that ships, after all of them, and never on the
-> master database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH.
+> `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`) and `TooltipManager.py`. Until they
+> catch up the step runs on the copy that ships, after all of them, and never on the master
+> database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH.
 >
 > In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the
 > id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)`

ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py

@@ -45,8 +45,8 @@
 sys.path.insert(0, str(_KOTLIN_PIPELINE))
 
 from populate_db import (  # noqa: E402  (the path has to be set up first)
-    DictionaryCompressor, compressed_templates_refusal, fragment_chain, get_content_type, get_id,
-    insert_chunked_content, load_dictionary, upsert_template,
+    DictionaryCompressor, fragment_chain, get_content_type, get_id, insert_chunked_content,
+    load_dictionary, templates_compressed, unknown_format_refusal, upsert_template,
 )
 from migrate_content_to_dictionary_brotli import write_item  # noqa: E402
 
@@ -288,7 +288,17 @@ def check_schema(conn) -> list[str]:
 
 def record_version(conn, stats: dict) -> None:
     """A row saying what changed, as the schema's own comment asks for. Minor, not major: nothing
-    about the schema or the server's contract changes, a documentation set is re-expressed."""
+    about the schema or the server's contract changes, a documentation set is re-expressed.
+
+    From 3.0.0 on the version is left as it is (ADFA-6740): that table holds the one row that says
+    what the file is, and the app and remint_dictionary.py read 3.0.0 as the format they know."""
+    if not templates_compressed(conn):
+        _append_version(conn, stats)
+    conn.execute("INSERT INTO LastChange (documentationSet, who) VALUES (?, ?)",
+                 ("android", "ADFA-5548"))
+
+
+def _append_version(conn, stats: dict) -> None:
     row = conn.execute(
         "SELECT major, minor, patch FROM DocumentationDatabaseVersion "
         "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone()
@@ -300,8 +310,6 @@ def record_version(conn, stats: dict) -> None:
          f"Serve the Android reference from JSON through Pebble templates "
          f"({stats['updated']:,} rows replaced, {stats['inserted']:,} added)"),
     )
-    conn.execute("INSERT INTO LastChange (documentationSet, who) VALUES (?, ?)",
-                 ("android", "ADFA-5548"))
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -328,8 +336,7 @@ def main(argv: list[str] | None = None) -> int:
     source_conn = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True)
     try:
         missing = check_schema(source_conn)
-        # record_version keeps the major, so a 3.x source would come out 3.(x+1) with plain templates in it.
-        refusal = None if missing else compressed_templates_refusal(source_conn)
+        refusal = None if missing else unknown_format_refusal(source_conn)
     finally:
         source_conn.close()
     if missing:

ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py

@@ -12,6 +12,7 @@
 import pytest
 
 import load_android_json_db as loader
+from populate_db import DictionaryCompressor, decode_template, load_dictionary  # loader put its directory on sys.path
 
 
 # --------------------------------------------------------------------------------------------
@@ -263,8 +264,9 @@ def test_the_comment_carries_the_counts(self, db):
 
 
 class TestCompressedTemplates:
-    """From documentation 3.0.0 every template is stored compressed and the app decodes every one. record_version
-    keeps the source's major, so loading a 3.x source would leave 3.(x+1) with plain Android templates in it."""
+    """From documentation 3.0.0 every template is stored compressed against the dictionary, with no per-row marker,
+    and the app decodes every one (ADFA-6740). A 3.0.0 source loads with compressed templates and stays 3.0.0; a
+    later major, a format this loader does not know, is refused before anything is written."""
 
     def run(self, db, tmp_path, *versions):
         db.executescript(VERSION_TABLES)
@@ -276,21 +278,34 @@ def run(self, db, tmp_path, *versions):
         return loader.main([str(tmp_path / "json"), "--source", str(tmp_path / "test.db"),
                             "--out", str(tmp_path / "out.db"), "--workers", "2"])
 
-    @pytest.mark.parametrize("major", [3, 4])
-    def test_a_source_declaring_3_or_later_is_refused_before_anything_is_written(self, db, tmp_path, capsys, major):
+    def test_a_source_declaring_a_later_major_is_refused_before_anything_is_written(self, db, tmp_path, capsys):
         with pytest.raises(SystemExit) as exited:
-            self.run(db, tmp_path, (2, "2026-01-01 00:00:00"), (major, "2026-10-19 00:00:00"))
+            self.run(db, tmp_path, (3, "2026-01-01 00:00:00"), (4, "2026-10-19 00:00:00"))
         assert exited.value.code == 2
-        assert f"declares documentation version {major}" in capsys.readouterr().err
+        assert "declares documentation version 4" in capsys.readouterr().err
         assert not (tmp_path / "out.db").exists()
 
+    def test_a_source_declaring_3_loads_compressed_templates_and_stays_3_0_0(self, db, tmp_path):
+        assert self.run(db, tmp_path, (3, "2026-10-19 00:00:00")) == 0
+        out = sqlite3.connect(tmp_path / "out.db")
+        try:
+            assert out.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall() == [(3, 0, 0)]
+            with DictionaryCompressor(load_dictionary(out)) as compressor:
+                for name, stored in out.execute("SELECT name, content FROM Templates").fetchall():
+                    assert name.startswith("android-")
+                    assert "{%" in decode_template(stored, compressor) or "{{" in decode_template(stored, compressor)
+        finally:
+            out.close()
+
     def test_a_source_declaring_2_still_loads(self, db, tmp_path):
         # The newest row by changeTime is 2, though a 3 was inserted after it.
         assert self.run(db, tmp_path, (2, "2026-10-19 00:00:00"), (3, "2026-01-01 00:00:00")) == 0
         out = sqlite3.connect(tmp_path / "out.db")
         try:
             assert out.execute("SELECT major, minor FROM DocumentationDatabaseVersion "
                                "ORDER BY rowid DESC LIMIT 1").fetchone() == (2, 1)
-            assert out.execute("SELECT COUNT(*) FROM Templates").fetchone() == (3,)
+            templates = out.execute("SELECT content FROM Templates").fetchall()
+            assert len(templates) == 3
+            assert all(decode_template(content, None) for (content,) in templates)
         finally:
             out.close()

ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py

@@ -27,7 +27,7 @@
                     / "ProcessKotlinDocs" / "ProcessKotlinWebsiteJSON")
 sys.path.insert(0, str(_KOTLIN_PIPELINE))
 
-from populate_db import DictionaryCompressor, load_dictionary  # noqa: E402
+from populate_db import DictionaryCompressor, decode_template, load_dictionary, templates_compressed  # noqa: E402
 from migrate_content_to_dictionary_brotli import read_item  # noqa: E402
 
 RENDERER_CLASSPATH = (Path(__file__).resolve().parent
@@ -81,7 +81,10 @@ def main(argv: list[str] | None = None) -> int:
     conn = sqlite3.connect(args.db)
     failures = 0
 
-    templates = {row[0]: row[1].decode("utf-8")
+    compressor = DictionaryCompressor(load_dictionary(conn))
+    # From 3.0.0 every template is compressed against the dictionary (ADFA-6740).
+    template_compressor = compressor if templates_compressed(conn) else None
+    templates = {row[0]: decode_template(row[1], template_compressor)
                  for row in conn.execute("SELECT id, content FROM Templates").fetchall()}
     rows = android_rows(conn)
     print(f"==> {len(rows):,} Android pages")
@@ -94,7 +97,6 @@ def main(argv: list[str] | None = None) -> int:
             print(f"    ! {path}")
 
     kinds: dict[str, int] = {}
-    compressor = DictionaryCompressor(load_dictionary(conn))
     known = {path for path, _t, _c in rows}
     known |= {path for (path,) in conn.execute(
         "SELECT path FROM Content WHERE path NOT LIKE '%-1'").fetchall()}

ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py

@@ -267,7 +267,13 @@ def declare_database_version(
     lower than the file already declares: rebuilding from an older pipeline
     genuinely produces an older format, and the row has to say what the file
     contains now rather than the highest it ever contained.
+
+    Except from COMPRESSED_TEMPLATES_MAJOR on: this step writes that format too
+    (upsert_template compresses, and it writes no tooltips), so a 3.x file keeps
+    the version it declares (ADFA-6740).
     """
+    if (declared_major(conn) or 0) >= COMPRESSED_TEMPLATES_MAJOR:
+        return False
     conn.execute(VERSION_TABLE_SQL)
     declared = conn.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall()
     if len(declared) == 1 and tuple(declared[0]) == DATABASE_FORMAT_VERSION:
@@ -321,6 +327,16 @@ def train_dictionary(samples: list, dict_size: int = DEFAULT_DICT_SIZE) -> bytes
         shutil.rmtree(work_dir, ignore_errors=True)
 
 
+# Brotli's largest RFC 7932 window (see large_window).
+WINDOW_BITS = 24
+
+
+def large_window(stream: bytes) -> bool:
+    """Whether a Brotli stream uses the large-window extension (window over 24 bits), which the CLI decodes and
+    the app's brotli4j refuses. Same test as migrate_tooltip_schema.large_window."""
+    return bool(stream) and stream[0] & 0x7F == 0x11
+
+
 class DictionaryCompressor:
     """Compresses/decompresses bytes against a fixed raw Brotli dictionary,
     shelling out to the `brotli` CLI (the installed Python `brotli` package
@@ -375,7 +391,9 @@ def _run(self, *extra_args: str, data: bytes) -> bytes:
         return result.stdout
 
     def compress(self, data: bytes) -> bytes:
-        return self._run(data=data)
+        # -w 24 is the CLI's default for stdin, made explicit as migrate_tooltip_schema.py does: never a
+        # large-window stream, which the app's brotli4j refuses.
+        return self._run("-w", str(WINDOW_BITS), data=data)
 
     def decompress(self, data: bytes) -> bytes:
         return self._run("-d", data=data)
@@ -569,9 +587,8 @@ def insert_file(conn, data: bytes, name: str, db_path: str, language_id: int, co
     return True
 
 
-# From this documentation major on, migrate_tooltip_schema.py (ADFA-3305) has stored every template compressed
-# against the dictionary, and the app decodes every one: a plain template written into such a database would not
-# render, and the file would still declare 3.x.
+# From this documentation major on, every template is stored compressed against the dictionary, with no per-row
+# marker (migrate_tooltip_schema.py, ADFA-3305), and the app decodes every one; below it, plain UTF-8.
 COMPRESSED_TEMPLATES_MAJOR = 3
 
 
@@ -585,25 +602,55 @@ def declared_major(conn) -> int | None:
     return None if row is None else int(row[0])
 
 
-def compressed_templates_refusal(conn) -> str | None:
-    """Why no plain template may be written into this database, or None."""
+# The newest documentation major this step knows how to write. A later one may store rows in a way it would
+# misread, so it refuses that file before changing anything.
+NEWEST_KNOWN_MAJOR = 3
+
+
+def unknown_format_refusal(conn) -> str | None:
+    """Why this step must not write into this database, or None."""
     major = declared_major(conn)
-    if major is None or major < COMPRESSED_TEMPLATES_MAJOR:
+    if major is None or major <= NEWEST_KNOWN_MAJOR:
         return None
-    return (f"the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 every "
-            "template is stored compressed, and this step writes plain ones. Run it before migrate_tooltip_schema.py")
+    return (f"the database declares documentation version {major}, and this step writes versions up to "
+            f"{NEWEST_KNOWN_MAJOR}.x")
+
+
+def templates_compressed(conn) -> bool:
+    return (declared_major(conn) or 0) >= COMPRESSED_TEMPLATES_MAJOR
+
+
+def encode_template(conn, content: str) -> bytes:
+    """What Templates stores for `content`: plain UTF-8, or from COMPRESSED_TEMPLATES_MAJOR on, Brotli against
+    the database's dictionary, checked by decoding it back. Raises RuntimeError rather than store a template the
+    app cannot read."""
+    data = content.encode("utf-8")
+    if not templates_compressed(conn):
+        return data
+    with DictionaryCompressor(load_dictionary(conn)) as compressor:
+        stream = compressor.compress(data)
+        if large_window(stream) or compressor.decompress(stream) != data:
+            raise RuntimeError("a compressed template did not decode back to its text")
+    return stream
+
+
+def decode_template(stored: bytes, compressor: "DictionaryCompressor | None") -> str:
+    """A Templates row's text: `compressor` is the database's DictionaryCompressor if its templates are
+    compressed (templates_compressed), else None."""
+    stored = bytes(stored)
+    if compressor is not None:
+        if large_window(stored):
+            raise RuntimeError("a template is a large-window Brotli stream, which the app cannot decode")
+        stored = compressor.decompress(stored)
+    return stored.decode("utf-8")
 
 
 def upsert_template(conn, name: str, content: str) -> int:
-    """Writes one plain template. Raises RuntimeError, writing nothing, if the database's templates are stored
-    compressed."""
-    refusal = compressed_templates_refusal(conn)
-    if refusal:
-        raise RuntimeError(refusal)
+    """Writes one template, compressed if the database's templates are (encode_template)."""
     conn.execute(
         "INSERT INTO Templates (name, content) VALUES (?, ?) "
         "ON CONFLICT(name) DO UPDATE SET content = excluded.content",
-        (name, content.encode("utf-8")),
+        (name, encode_template(conn, content)),
     )
     return conn.execute("SELECT id FROM Templates WHERE name = ?", (name,)).fetchone()[0]
 
@@ -751,7 +798,7 @@ def main():
     # Checked before the conversion and the backup, so a refusal costs nothing and writes nothing.
     conn = sqlite3.connect(f"file:{args.db_path}?mode=ro", uri=True)
     try:
-        refusal = compressed_templates_refusal(conn)
+        refusal = unknown_format_refusal(conn)
     finally:
         conn.close()
     if refusal:
@@ -981,11 +1028,11 @@ def main():
         # Declared next to the dictionary because the dictionary is what the
         # version means (ADFA-5220): the two have to land in the same
         # transaction, or a reader can see one without the other.
-        version = ".".join(str(part) for part in DATABASE_FORMAT_VERSION)
         if declare_database_version(conn):
+            version = ".".join(str(part) for part in DATABASE_FORMAT_VERSION)
             print(f"Declared documentation database version {version}", file=sys.stderr)
         else:
-            print(f"Documentation database already declares version {version}", file=sys.stderr)
+            print(f"Documentation database keeps the version it declares, {declared_major(conn)}.x", file=sys.stderr)
         with DictionaryCompressor(dictionary_data) as compressor:
             for page, json_bytes in zip(pages, page_json_bytes):
                 path = f"{page['id']}.html"

ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py

@@ -53,24 +53,23 @@ def test_an_older_declaration_is_replaced_not_appended(self):
         self.assertTrue(declare_database_version(self.conn))
         self.assertEqual([row[:3] for row in self.declared()], [DATABASE_FORMAT_VERSION])
 
-    def test_a_downgrade_replaces_a_higher_declaration(self):
-        """Rebuilding from an older pipeline really does produce an older format.
-        The row has to say what the file contains now, not the highest version it
-        ever contained."""
+    def test_a_3x_declaration_is_kept(self):
+        """This step writes schema 3 too (compressed templates, no tooltips), so a
+        3.0.0 file stays 3.0.0 rather than being declared 2.0.0 (ADFA-6740)."""
         self.conn.execute(VERSION_TABLE_SQL)
         self.conn.execute(
             "INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) "
-            "VALUES (9, 0, 0, 'future', 'a format this pipeline does not produce')"
+            "VALUES (3, 0, 0, 'ADFA-3305', 'keyed tooltips')"
         )
-        self.assertTrue(declare_database_version(self.conn))
-        self.assertEqual([row[:3] for row in self.declared()], [DATABASE_FORMAT_VERSION])
+        self.assertFalse(declare_database_version(self.conn))
+        self.assertEqual(self.declared(), [(3, 0, 0, "ADFA-3305")])
 
     def test_several_rows_are_collapsed_to_one(self):
         """A database that picked up extra rows -- an older tool that appended, a
         hand-edit -- is repaired rather than read around, so "the version" can
         never be ambiguous."""
         self.conn.execute(VERSION_TABLE_SQL)
-        for major in (1, 2, 3):
+        for major in (1, 2, 1):
             self.conn.execute(
                 "INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) "
                 "VALUES (?, 0, 0, 'older tool', 'appended')",