diff --git a/CLAUDE.md b/CLAUDE.md index f9510fd9..cd02a896 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,16 +85,18 @@ edit that database** — it contains no part of the production Android app itsel > every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the > re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0. > -> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` write plain templates, -> so each refuses a database declaring 3.0.0 or later before writing anything, backup included. -> `load_android_json_db.py` keeps the major it finds, so a late run would otherwise leave a file -> declaring 3.1.0 with plain templates in it, which the app cannot read. +> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` read and write both +> (ADFA-6740): below 3.0.0 they write plain templates, from it compressed ones (`brotli -D -w 24`, +> checked by decoding back, never large-window), and they leave a 3.x file at the version it +> declares. They refuse a major above 3, a format they do not know, before writing anything. +> `docdb-studio` reads and writes both shapes too (tooltips through the keyed tables, templates +> decoded and re-encoded with the dictionary), choosing by the declared major. > -> **Every tool in this repo still reads and writes the 2.x shape**: `check-tools` (whose daily +> **The other tools still read and write only the 2.x shape**: `check-tools` (whose daily > `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`), -> `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`), `TooltipManager.py` and -> `docdb-studio`. So the step runs on the copy that ships, after all of them, and never on the -> master database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH. +> `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`) and `TooltipManager.py`. Until they +> catch up the step runs on the copy that ships, after all of them, and never on the master +> database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH. > > In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the > id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)` diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py index 11aec50a..3ff11f17 100755 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py @@ -45,8 +45,8 @@ sys.path.insert(0, str(_KOTLIN_PIPELINE)) from populate_db import ( # noqa: E402 (the path has to be set up first) - DictionaryCompressor, compressed_templates_refusal, fragment_chain, get_content_type, get_id, - insert_chunked_content, load_dictionary, upsert_template, + DictionaryCompressor, fragment_chain, get_content_type, get_id, insert_chunked_content, + load_dictionary, templates_compressed, unknown_format_refusal, upsert_template, ) from migrate_content_to_dictionary_brotli import write_item # noqa: E402 @@ -288,7 +288,17 @@ def check_schema(conn) -> list[str]: def record_version(conn, stats: dict) -> None: """A row saying what changed, as the schema's own comment asks for. Minor, not major: nothing - about the schema or the server's contract changes, a documentation set is re-expressed.""" + about the schema or the server's contract changes, a documentation set is re-expressed. + + From 3.0.0 on the version is left as it is (ADFA-6740): that table holds the one row that says + what the file is, and the app and remint_dictionary.py read 3.0.0 as the format they know.""" + if not templates_compressed(conn): + _append_version(conn, stats) + conn.execute("INSERT INTO LastChange (documentationSet, who) VALUES (?, ?)", + ("android", "ADFA-5548")) + + +def _append_version(conn, stats: dict) -> None: row = conn.execute( "SELECT major, minor, patch FROM DocumentationDatabaseVersion " "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() @@ -300,8 +310,6 @@ def record_version(conn, stats: dict) -> None: f"Serve the Android reference from JSON through Pebble templates " f"({stats['updated']:,} rows replaced, {stats['inserted']:,} added)"), ) - conn.execute("INSERT INTO LastChange (documentationSet, who) VALUES (?, ?)", - ("android", "ADFA-5548")) def main(argv: list[str] | None = None) -> int: @@ -328,8 +336,7 @@ def main(argv: list[str] | None = None) -> int: source_conn = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True) try: missing = check_schema(source_conn) - # record_version keeps the major, so a 3.x source would come out 3.(x+1) with plain templates in it. - refusal = None if missing else compressed_templates_refusal(source_conn) + refusal = None if missing else unknown_format_refusal(source_conn) finally: source_conn.close() if missing: diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py index 0af304b0..c1ee7239 100644 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py @@ -12,6 +12,7 @@ import pytest import load_android_json_db as loader +from populate_db import DictionaryCompressor, decode_template, load_dictionary # loader put its directory on sys.path # -------------------------------------------------------------------------------------------- @@ -263,8 +264,9 @@ def test_the_comment_carries_the_counts(self, db): class TestCompressedTemplates: - """From documentation 3.0.0 every template is stored compressed and the app decodes every one. record_version - keeps the source's major, so loading a 3.x source would leave 3.(x+1) with plain Android templates in it.""" + """From documentation 3.0.0 every template is stored compressed against the dictionary, with no per-row marker, + and the app decodes every one (ADFA-6740). A 3.0.0 source loads with compressed templates and stays 3.0.0; a + later major, a format this loader does not know, is refused before anything is written.""" def run(self, db, tmp_path, *versions): db.executescript(VERSION_TABLES) @@ -276,14 +278,25 @@ def run(self, db, tmp_path, *versions): return loader.main([str(tmp_path / "json"), "--source", str(tmp_path / "test.db"), "--out", str(tmp_path / "out.db"), "--workers", "2"]) - @pytest.mark.parametrize("major", [3, 4]) - def test_a_source_declaring_3_or_later_is_refused_before_anything_is_written(self, db, tmp_path, capsys, major): + def test_a_source_declaring_a_later_major_is_refused_before_anything_is_written(self, db, tmp_path, capsys): with pytest.raises(SystemExit) as exited: - self.run(db, tmp_path, (2, "2026-01-01 00:00:00"), (major, "2026-10-19 00:00:00")) + self.run(db, tmp_path, (3, "2026-01-01 00:00:00"), (4, "2026-10-19 00:00:00")) assert exited.value.code == 2 - assert f"declares documentation version {major}" in capsys.readouterr().err + assert "declares documentation version 4" in capsys.readouterr().err assert not (tmp_path / "out.db").exists() + def test_a_source_declaring_3_loads_compressed_templates_and_stays_3_0_0(self, db, tmp_path): + assert self.run(db, tmp_path, (3, "2026-10-19 00:00:00")) == 0 + out = sqlite3.connect(tmp_path / "out.db") + try: + assert out.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall() == [(3, 0, 0)] + with DictionaryCompressor(load_dictionary(out)) as compressor: + for name, stored in out.execute("SELECT name, content FROM Templates").fetchall(): + assert name.startswith("android-") + assert "{%" in decode_template(stored, compressor) or "{{" in decode_template(stored, compressor) + finally: + out.close() + def test_a_source_declaring_2_still_loads(self, db, tmp_path): # The newest row by changeTime is 2, though a 3 was inserted after it. assert self.run(db, tmp_path, (2, "2026-10-19 00:00:00"), (3, "2026-01-01 00:00:00")) == 0 @@ -291,6 +304,8 @@ def test_a_source_declaring_2_still_loads(self, db, tmp_path): try: assert out.execute("SELECT major, minor FROM DocumentationDatabaseVersion " "ORDER BY rowid DESC LIMIT 1").fetchone() == (2, 1) - assert out.execute("SELECT COUNT(*) FROM Templates").fetchone() == (3,) + templates = out.execute("SELECT content FROM Templates").fetchall() + assert len(templates) == 3 + assert all(decode_template(content, None) for (content,) in templates) finally: out.close() diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py index fca72836..f12b56a6 100755 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py @@ -27,7 +27,7 @@ / "ProcessKotlinDocs" / "ProcessKotlinWebsiteJSON") sys.path.insert(0, str(_KOTLIN_PIPELINE)) -from populate_db import DictionaryCompressor, load_dictionary # noqa: E402 +from populate_db import DictionaryCompressor, decode_template, load_dictionary, templates_compressed # noqa: E402 from migrate_content_to_dictionary_brotli import read_item # noqa: E402 RENDERER_CLASSPATH = (Path(__file__).resolve().parent @@ -81,7 +81,10 @@ def main(argv: list[str] | None = None) -> int: conn = sqlite3.connect(args.db) failures = 0 - templates = {row[0]: row[1].decode("utf-8") + compressor = DictionaryCompressor(load_dictionary(conn)) + # From 3.0.0 every template is compressed against the dictionary (ADFA-6740). + template_compressor = compressor if templates_compressed(conn) else None + templates = {row[0]: decode_template(row[1], template_compressor) for row in conn.execute("SELECT id, content FROM Templates").fetchall()} rows = android_rows(conn) print(f"==> {len(rows):,} Android pages") @@ -94,7 +97,6 @@ def main(argv: list[str] | None = None) -> int: print(f" ! {path}") kinds: dict[str, int] = {} - compressor = DictionaryCompressor(load_dictionary(conn)) known = {path for path, _t, _c in rows} known |= {path for (path,) in conn.execute( "SELECT path FROM Content WHERE path NOT LIKE '%-1'").fetchall()} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py index e7e58921..6ce6b339 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py @@ -267,7 +267,13 @@ def declare_database_version( lower than the file already declares: rebuilding from an older pipeline genuinely produces an older format, and the row has to say what the file contains now rather than the highest it ever contained. + + Except from COMPRESSED_TEMPLATES_MAJOR on: this step writes that format too + (upsert_template compresses, and it writes no tooltips), so a 3.x file keeps + the version it declares (ADFA-6740). """ + if (declared_major(conn) or 0) >= COMPRESSED_TEMPLATES_MAJOR: + return False conn.execute(VERSION_TABLE_SQL) declared = conn.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall() if len(declared) == 1 and tuple(declared[0]) == DATABASE_FORMAT_VERSION: @@ -321,6 +327,16 @@ def train_dictionary(samples: list, dict_size: int = DEFAULT_DICT_SIZE) -> bytes shutil.rmtree(work_dir, ignore_errors=True) +# Brotli's largest RFC 7932 window (see large_window). +WINDOW_BITS = 24 + + +def large_window(stream: bytes) -> bool: + """Whether a Brotli stream uses the large-window extension (window over 24 bits), which the CLI decodes and + the app's brotli4j refuses. Same test as migrate_tooltip_schema.large_window.""" + return bool(stream) and stream[0] & 0x7F == 0x11 + + class DictionaryCompressor: """Compresses/decompresses bytes against a fixed raw Brotli dictionary, shelling out to the `brotli` CLI (the installed Python `brotli` package @@ -375,7 +391,9 @@ def _run(self, *extra_args: str, data: bytes) -> bytes: return result.stdout def compress(self, data: bytes) -> bytes: - return self._run(data=data) + # -w 24 is the CLI's default for stdin, made explicit as migrate_tooltip_schema.py does: never a + # large-window stream, which the app's brotli4j refuses. + return self._run("-w", str(WINDOW_BITS), data=data) def decompress(self, data: bytes) -> bytes: return self._run("-d", data=data) @@ -569,9 +587,8 @@ def insert_file(conn, data: bytes, name: str, db_path: str, language_id: int, co return True -# From this documentation major on, migrate_tooltip_schema.py (ADFA-3305) has stored every template compressed -# against the dictionary, and the app decodes every one: a plain template written into such a database would not -# render, and the file would still declare 3.x. +# From this documentation major on, every template is stored compressed against the dictionary, with no per-row +# marker (migrate_tooltip_schema.py, ADFA-3305), and the app decodes every one; below it, plain UTF-8. COMPRESSED_TEMPLATES_MAJOR = 3 @@ -585,25 +602,55 @@ def declared_major(conn) -> int | None: return None if row is None else int(row[0]) -def compressed_templates_refusal(conn) -> str | None: - """Why no plain template may be written into this database, or None.""" +# The newest documentation major this step knows how to write. A later one may store rows in a way it would +# misread, so it refuses that file before changing anything. +NEWEST_KNOWN_MAJOR = 3 + + +def unknown_format_refusal(conn) -> str | None: + """Why this step must not write into this database, or None.""" major = declared_major(conn) - if major is None or major < COMPRESSED_TEMPLATES_MAJOR: + if major is None or major <= NEWEST_KNOWN_MAJOR: return None - return (f"the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 every " - "template is stored compressed, and this step writes plain ones. Run it before migrate_tooltip_schema.py") + return (f"the database declares documentation version {major}, and this step writes versions up to " + f"{NEWEST_KNOWN_MAJOR}.x") + + +def templates_compressed(conn) -> bool: + return (declared_major(conn) or 0) >= COMPRESSED_TEMPLATES_MAJOR + + +def encode_template(conn, content: str) -> bytes: + """What Templates stores for `content`: plain UTF-8, or from COMPRESSED_TEMPLATES_MAJOR on, Brotli against + the database's dictionary, checked by decoding it back. Raises RuntimeError rather than store a template the + app cannot read.""" + data = content.encode("utf-8") + if not templates_compressed(conn): + return data + with DictionaryCompressor(load_dictionary(conn)) as compressor: + stream = compressor.compress(data) + if large_window(stream) or compressor.decompress(stream) != data: + raise RuntimeError("a compressed template did not decode back to its text") + return stream + + +def decode_template(stored: bytes, compressor: "DictionaryCompressor | None") -> str: + """A Templates row's text: `compressor` is the database's DictionaryCompressor if its templates are + compressed (templates_compressed), else None.""" + stored = bytes(stored) + if compressor is not None: + if large_window(stored): + raise RuntimeError("a template is a large-window Brotli stream, which the app cannot decode") + stored = compressor.decompress(stored) + return stored.decode("utf-8") def upsert_template(conn, name: str, content: str) -> int: - """Writes one plain template. Raises RuntimeError, writing nothing, if the database's templates are stored - compressed.""" - refusal = compressed_templates_refusal(conn) - if refusal: - raise RuntimeError(refusal) + """Writes one template, compressed if the database's templates are (encode_template).""" conn.execute( "INSERT INTO Templates (name, content) VALUES (?, ?) " "ON CONFLICT(name) DO UPDATE SET content = excluded.content", - (name, content.encode("utf-8")), + (name, encode_template(conn, content)), ) return conn.execute("SELECT id FROM Templates WHERE name = ?", (name,)).fetchone()[0] @@ -751,7 +798,7 @@ def main(): # Checked before the conversion and the backup, so a refusal costs nothing and writes nothing. conn = sqlite3.connect(f"file:{args.db_path}?mode=ro", uri=True) try: - refusal = compressed_templates_refusal(conn) + refusal = unknown_format_refusal(conn) finally: conn.close() if refusal: @@ -981,11 +1028,11 @@ def main(): # Declared next to the dictionary because the dictionary is what the # version means (ADFA-5220): the two have to land in the same # transaction, or a reader can see one without the other. - version = ".".join(str(part) for part in DATABASE_FORMAT_VERSION) if declare_database_version(conn): + version = ".".join(str(part) for part in DATABASE_FORMAT_VERSION) print(f"Declared documentation database version {version}", file=sys.stderr) else: - print(f"Documentation database already declares version {version}", file=sys.stderr) + print(f"Documentation database keeps the version it declares, {declared_major(conn)}.x", file=sys.stderr) with DictionaryCompressor(dictionary_data) as compressor: for page, json_bytes in zip(pages, page_json_bytes): path = f"{page['id']}.html" diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py index 774c69d8..ecc1cc65 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py @@ -53,24 +53,23 @@ def test_an_older_declaration_is_replaced_not_appended(self): self.assertTrue(declare_database_version(self.conn)) self.assertEqual([row[:3] for row in self.declared()], [DATABASE_FORMAT_VERSION]) - def test_a_downgrade_replaces_a_higher_declaration(self): - """Rebuilding from an older pipeline really does produce an older format. - The row has to say what the file contains now, not the highest version it - ever contained.""" + def test_a_3x_declaration_is_kept(self): + """This step writes schema 3 too (compressed templates, no tooltips), so a + 3.0.0 file stays 3.0.0 rather than being declared 2.0.0 (ADFA-6740).""" self.conn.execute(VERSION_TABLE_SQL) self.conn.execute( "INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) " - "VALUES (9, 0, 0, 'future', 'a format this pipeline does not produce')" + "VALUES (3, 0, 0, 'ADFA-3305', 'keyed tooltips')" ) - self.assertTrue(declare_database_version(self.conn)) - self.assertEqual([row[:3] for row in self.declared()], [DATABASE_FORMAT_VERSION]) + self.assertFalse(declare_database_version(self.conn)) + self.assertEqual(self.declared(), [(3, 0, 0, "ADFA-3305")]) def test_several_rows_are_collapsed_to_one(self): """A database that picked up extra rows -- an older tool that appended, a hand-edit -- is repaired rather than read around, so "the version" can never be ambiguous.""" self.conn.execute(VERSION_TABLE_SQL) - for major in (1, 2, 3): + for major in (1, 2, 1): self.conn.execute( "INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) " "VALUES (?, 0, 0, 'older tool', 'appended')", diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py index 8ee4ec21..d8454867 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -"""Tests that populate_db.py writes no plain template into a database whose templates are compressed (ADFA-3305). +"""Tests that populate_db.py writes templates the way the database stores them (ADFA-6740). -From documentation 3.0.0 migrate_tooltip_schema.py stores every template compressed, and the app decodes every one, -so a plain page.peb or nav.peb written afterwards would not render while the file still declared 3.x. +From documentation 3.0.0 every template is stored Brotli-compressed against CompressionDictionary, with no per-row +marker, and the app decodes every one; below it they are plain. A 3.0.0 file stays 3.0.0, and a later major, a +format this step does not know, is refused before anything is written. Run directly: python3 test_populate_db_templates.py """ @@ -15,7 +16,8 @@ import zipfile from pathlib import Path -from populate_db import compressed_templates_refusal, declared_major, upsert_template +from populate_db import (DictionaryCompressor, declared_major, decode_template, large_window, templates_compressed, + unknown_format_refusal, upsert_template) HERE = Path(__file__).resolve().parent @@ -36,7 +38,7 @@ """ -class CompressedTemplatesRefusalTest(unittest.TestCase): +class TemplatesTest(unittest.TestCase): def setUp(self): directory = tempfile.TemporaryDirectory() self.addCleanup(directory.cleanup) @@ -62,23 +64,28 @@ def connect(self): def templates(self) -> list: return self.connect().execute("SELECT name, content FROM Templates ORDER BY id").fetchall() - def test_upsert_template_refuses_major_3(self): + def compressor(self): + dictionary = self.connect().execute("SELECT data FROM CompressionDictionary").fetchone()[0] + compressor = DictionaryCompressor(dictionary) + self.addCleanup(compressor.close) + return compressor + + def test_upsert_template_compresses_at_major_3(self): self.declare((2, "2026-01-01 00:00:00"), (3, "2026-10-19 00:00:00")) - before = self.templates() conn = self.connect() - with self.assertRaises(RuntimeError) as raised: - upsert_template(conn, "page.peb", "new") - self.assertIn("declares documentation version 3", str(raised.exception)) - self.assertIn("Run it before migrate_tooltip_schema.py", str(raised.exception)) + upsert_template(conn, "page.peb", "new \u00e9t\u00e9") conn.commit() - self.assertEqual(self.templates(), before) + (name, stored), = self.templates() + self.assertEqual(name, "page.peb") + self.assertFalse(large_window(stored)) + self.assertEqual(self.compressor().decompress(stored), "new \u00e9t\u00e9".encode()) + self.assertEqual(decode_template(stored, self.compressor()), "new \u00e9t\u00e9") - def test_upsert_template_refuses_a_later_major(self): - self.declare((4, "2026-10-19 00:00:00")) + def test_decode_template_refuses_a_large_window_stream(self): with self.assertRaises(RuntimeError): - upsert_template(self.connect(), "nav.peb", "