From 0a0ec4e5a6c65eb781c4e814df5155d655ec3a65 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 20 Oct 2026 14:56:29 -0700 Subject: [PATCH 01/17] ADFA-6683: Store a/, j/ and k/ pages N to a compressed block (prototype) store_pages_in_blocks.py DB --pages-per-block N [--dry-run] writes the format in ADFA-6683-block-format.md, which the app reader follows too: - ContentBlocks(id, data): Brotli q11 w24 against CompressionDictionary of N pages' decoded bytes, concatenated. - ContentBlockItems(contentId, blockId, offset, length), WITHOUT ROWID with declared foreign keys, so rebuild_database.py --pack Content renumbers contentId. - Three sets, each cut in path order (BINARY), the last block of a set shorter, never mixed: a/ templated (not a/devsite/), j/ text/html, k/ templated. Brotli types only; a row with continuation rows and each continuation row stay as they are. - A blocked row keeps path, languageID, contentTypeID and templateId; its content becomes x''. Each block is decoded again before it is stored, and before the commit every item is sliced from its stored block and compared by sha256 with its row's decoded bytes. One transaction, then VACUUM; refuses a database that already has ContentBlockItems; --dry-run rolls every write back. test_store_pages_in_blocks.py covers the seven cases on a synthetic database. Each test was checked to fail against a script broken in the way it names: id order or NOCASE order, mixed sets, no continuation skip, a stuck offset, rows not emptied, templateId cleared, no refusal, a dry run that commits. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/store_pages_in_blocks.py | 303 ++++++++++++++++++++++++++ scripts/test_store_pages_in_blocks.py | 284 ++++++++++++++++++++++++ 2 files changed, 587 insertions(+) create mode 100644 scripts/store_pages_in_blocks.py create mode 100644 scripts/test_store_pages_in_blocks.py diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py new file mode 100644 index 00000000..30f81a7b --- /dev/null +++ b/scripts/store_pages_in_blocks.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# ] +# /// +""" +store_pages_in_blocks.py + +Stores the pages of three documentation sets N to a compressed block instead +of one compressed Content row each (ADFA-6683, prototype): + + uv run scripts/store_pages_in_blocks.py documentation.db --pages-per-block 16 --dry-run + uv run scripts/store_pages_in_blocks.py documentation.db --pages-per-block 16 + +Why: each Content row is compressed on its own, so what neighbouring pages +share - the same member tables, inherited-method lists and markup - is paid +for again in every row, and the shared CompressionDictionary only covers what +all pages share. Compressed together, N pages in path order share the rest. +The 2026-10-20 size analysis put the release .br saving at 7,865,965 bytes for +N = 16 and 10,062,166 for N = 64. The cost is at read time: serving one page +decodes its whole block, so the app caches decoded blocks. + +The format (ADFA-6683-block-format.md, which the app reader follows too): + + ContentBlocks(id, data) + data: Brotli of the block's plain items, concatenated - quality 11, + window 24, against CompressionDictionary, as Content rows are. + ContentBlockItems(contentId, blockId, offset, length) WITHOUT ROWID + The item is [offset, offset + length) of the decoded block. Its + foreign key to Content lets rebuild_database.py --pack Content + renumber contentId with Content. + +Which rows: three sets, each cut in ORDER BY path into blocks of N +consecutive items, the last block of a set maybe shorter; a block never mixes +sets. Block ids follow the sets in this order: + + 1. a/ templated JSON: path LIKE 'a/%' AND path NOT LIKE 'a/devsite/%' AND templateId > 0 + 2. j/ HTML: path LIKE 'j/%' AND content type text/html + 3. k/ templated JSON: path LIKE 'k/%' AND templateId > 0 + +Only rows whose content type is compressed with brotli, and only items held in +one row: a row that owns continuation rows ("path-1", ... after a 1 MiB row), +and each continuation row, stay as they are. An item is exactly its row's +decoded bytes, before any template rendering. A blocked row keeps its path, +languageID, contentTypeID and templateId, so every lookup by path and every +page's template work as before; only its content becomes an empty blob. + +Every block is decoded again before it is stored and compared with the items +it was built from, and before the commit every item is sliced from its stored +block and compared, by sha256, with its row's decoded bytes. + +A database that already has ContentBlockItems is refused: its blocked rows +are empty, so a second run would store empty pages over the blocked ones. + +Runs in one transaction and VACUUMs afterwards. Takes no backup: run it on a +copy, then pack that with rebuild_database.py --pack Templates,Content,TooltipTexts. +--dry-run does all the work, writes included, and rolls it back. +""" +from __future__ import annotations + +import argparse +import hashlib +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from optimize_db_media import human, is_continuation_path, load_dictionary, owned_fragment_paths + +# (set, condition on Content AS c joined to ContentTypes AS t), in block-id order. +SETS = ( + ("a/", "c.path LIKE 'a/%' AND c.path NOT LIKE 'a/devsite/%' AND c.templateId > 0"), + ("j/", "c.path LIKE 'j/%' AND t.value = 'text/html'"), + ("k/", "c.path LIKE 'k/%' AND c.templateId > 0"), +) + +SCHEMA = ( + """CREATE TABLE ContentBlocks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data BLOB NOT NULL -- Brotli of the block's plain items, concatenated +)""", + """CREATE TABLE ContentBlockItems ( + contentId INTEGER NOT NULL PRIMARY KEY REFERENCES Content(id), + blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), + offset INTEGER NOT NULL, -- into the decoded block, in bytes + length INTEGER NOT NULL -- in bytes +) WITHOUT ROWID""", +) + + +class DictionaryBrotli: + """The `brotli` CLI against one dictionary. No Python binding takes a + custom dictionary, so every tool here shells out the same way.""" + + def __init__(self, dictionary: bytes): + self._brotli = shutil.which("brotli") + if self._brotli is None: + raise RuntimeError("the `brotli` CLI is not on PATH; install it and retry") + self._work_dir = Path(tempfile.mkdtemp(prefix="store_pages_in_blocks_")) + self._dictionary = self._work_dir / "dict.bin" + self._dictionary.write_bytes(dictionary) + + def _run(self, args: list, data: bytes) -> bytes: + result = subprocess.run([self._brotli, *args, "-D", str(self._dictionary)], input=data, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if result.returncode != 0: + raise RuntimeError(f"brotli failed: {result.stderr.decode(errors='replace').strip()}") + return result.stdout + + def encode(self, data: bytes) -> bytes: + return self._run(["-c", "-q", "11", "-w", "24"], data) + + def decode(self, data: bytes) -> bytes: + return self._run(["-d", "-c"], data) + + def close(self) -> None: + shutil.rmtree(self._work_dir, ignore_errors=True) + + +def table_exists(conn, name: str) -> bool: + return conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone() is not None + + +def select_sets(conn) -> list: + """[(set, [(content id, path, stored bytes)] in path order, items held in + more than one row, their continuation rows)] for each of SETS.""" + lengths = dict(conn.execute("SELECT path, LENGTH(content) FROM Content")) + selected = [] + for name, condition in SETS: + rows, chunked, continuations = [], 0, 0 + for content_id, path, stored in conn.execute( + "SELECT c.id, c.path, c.content FROM Content AS c JOIN ContentTypes AS t ON t.id = c.contentTypeID " + f"WHERE t.compression = 'brotli' AND {condition} ORDER BY c.path COLLATE BINARY"): + if is_continuation_path(lengths, path): + continuations += 1 + elif owned_fragment_paths(conn, path, len(stored)): + chunked += 1 + else: + rows.append((content_id, path, stored)) + selected.append((name, rows, chunked, continuations)) + return selected + + +def build_block(codec: DictionaryBrotli, rows: list) -> tuple: + """(compressed block, plain length, [(content id, offset, length, sha256)]) + for one block's rows, the block decoded again and checked.""" + plain, items, offset = [], [], 0 + for content_id, path, stored in rows: + try: + raw = codec.decode(stored) + except RuntimeError as exc: + raise RuntimeError(f"{path}: {exc}") from exc + items.append((content_id, offset, len(raw), hashlib.sha256(raw).digest())) + plain.append(raw) + offset += len(raw) + plain = b"".join(plain) + data = codec.encode(plain) + if codec.decode(data) != plain: + raise RuntimeError(f"the block starting at {rows[0][1]} did not decode back to its items") + return data, len(plain), items + + +def check_stored(conn, codec: DictionaryBrotli, expected: dict, workers: int) -> None: + """Slices every item from its block as stored and compares it with its + row's decoded bytes (`expected`: {content id: sha256}); every blocked row + must now be empty.""" + items = {} + for content_id, block_id, offset, length in conn.execute( + "SELECT contentId, blockId, offset, length FROM ContentBlockItems"): + items.setdefault(block_id, []).append((content_id, offset, length)) + if sum(len(group) for group in items.values()) != len(expected): + raise RuntimeError("ContentBlockItems does not hold one row per blocked item") + blocks = conn.execute("SELECT id, data FROM ContentBlocks ORDER BY id").fetchall() + + def check(block): + block_id, data = block + plain = codec.decode(data) + for content_id, offset, length in items.get(block_id, ()): + if hashlib.sha256(plain[offset:offset + length]).digest() != expected.get(content_id): + raise RuntimeError(f"Content id {content_id} does not read back from block {block_id}") + + with ThreadPoolExecutor(max_workers=workers) as pool: + list(pool.map(check, blocks)) + left = conn.execute("SELECT COUNT(*) FROM Content WHERE LENGTH(content) > 0 AND id IN " + "(SELECT contentId FROM ContentBlockItems)").fetchone()[0] + if left: + raise RuntimeError(f"{left} blocked row(s) still hold their content") + + +def run(cfg: dict) -> int: + db_path, per_block, workers = cfg["db_path"], cfg["pages_per_block"], cfg.get("workers") or 1 + if per_block < 1: + print(f"error: --pages-per-block must be at least 1, not {per_block}", file=sys.stderr) + return 1 + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + size_before = db_path.stat().st_size + + conn = sqlite3.connect(db_path, isolation_level=None) + codec = None + summary = [] + try: + for table in ("ContentBlockItems", "ContentBlocks"): + if table_exists(conn, table): + print(f"error: {db_path} already has {table}: it has been through this script, and its blocked " + "rows are empty, so a second run would store empty pages over them. Start again from a " + "database that has not.", file=sys.stderr) + return 1 + dictionary = load_dictionary(conn) + if not dictionary: + print(f"error: {db_path} has no CompressionDictionary, which blocks are compressed against", + file=sys.stderr) + return 1 + codec = DictionaryBrotli(dictionary) + selected = select_sets(conn) + + if cfg["dry_run"]: + print(f"Dry run: every write is rolled back. Storing {db_path}'s pages {per_block} to a block ...") + else: + print(f"Storing {db_path}'s pages {per_block} to a block ...") + conn.execute("BEGIN") + try: + for sql in SCHEMA: + conn.execute(sql) + expected = {} + with ThreadPoolExecutor(max_workers=workers) as pool: + for name, rows, chunked, continuations in selected: + groups = [rows[start:start + per_block] for start in range(0, len(rows), per_block)] + stored = compressed = plain_total = 0 + for group, (data, plain_length, items) in zip( + groups, pool.map(lambda group: build_block(codec, group), groups)): + block_id = conn.execute("INSERT INTO ContentBlocks (data) VALUES (?)", (data,)).lastrowid + for content_id, offset, length, digest in items: + conn.execute("INSERT INTO ContentBlockItems (contentId, blockId, offset, length) " + "VALUES (?, ?, ?, ?)", (content_id, block_id, offset, length)) + conn.execute("UPDATE Content SET content = x'' WHERE id = ?", (content_id,)) + expected[content_id] = digest + stored += sum(len(row[2]) for row in group) + compressed += len(data) + plain_total += plain_length + summary.append((name, len(rows), len(groups), stored, compressed, plain_total, chunked, + continuations)) + check_stored(conn, codec, expected, workers) + if cfg["dry_run"]: + conn.execute("ROLLBACK") + else: + conn.execute("COMMIT") + except BaseException: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + except (sqlite3.Error, RuntimeError) as exc: + print(f"error: {exc}; nothing was written", file=sys.stderr) + return 1 + finally: + if codec is not None: + codec.close() + conn.close() + + for name, items, blocks, stored, compressed, plain, chunked, continuations in summary: + print(f" {name}: {human(items)} page(s) in {human(blocks)} block(s); their rows held {human(stored)} " + f"bytes, the blocks hold {human(compressed)} ({human(plain)} decoded).") + if chunked or continuations: + print(f" {name}: left as they are: {human(chunked)} page(s) held in more than one row, and " + f"{human(continuations)} continuation row(s).") + saved = sum(row[3] - row[4] for row in summary) + verb = "Would store" if cfg["dry_run"] else "Stored" + print(f"{verb} {human(sum(row[1] for row in summary))} page(s) in {human(sum(row[2] for row in summary))} " + f"block(s) of up to {per_block}, {human(saved)} bytes smaller than their rows.") + + if not cfg["dry_run"]: + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + print(f"{db_path}: {human(size_before)} -> {human(db_path.stat().st_size)} bytes.") + return 0 + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to change in place; run it on a copy") + p.add_argument("--pages-per-block", type=int, required=True, metavar="N", + help="Pages per block; the last block of each set may hold fewer") + p.add_argument("--dry-run", action="store_true", help="Do all the work, then roll back without writing") + p.add_argument("--workers", type=int, default=8, + help="brotli processes run at once to decode rows and compress blocks (default: 8)") + args = p.parse_args() + sys.exit(run({"db_path": args.db_path, "pages_per_block": args.pages_per_block, "dry_run": args.dry_run, + "workers": args.workers})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py new file mode 100644 index 00000000..a2163c3b --- /dev/null +++ b/scripts/test_store_pages_in_blocks.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_store_pages_in_blocks.py + +Covers store_pages_in_blocks.py on a small synthetic database, three pages to +a block: each set in path order with its own blocks, the last one shorter; +rows held in more than one row left alone; every item sliced from its block +equal to its row's decoded bytes; blocked rows emptied and nothing else +changed; a second run refused; a dry run writing nothing. + + uv run --with pytest --with-requirements requirements.txt python -m pytest -q scripts/test_store_pages_in_blocks.py +""" +from __future__ import annotations + +import hashlib +import random +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import optimize_db_media as opt # noqa: E402 +import store_pages_in_blocks as spb # noqa: E402 + +PER_BLOCK = 3 +DICTIONARY = b'{"title": "", "members": [{"name": "", "summary": ""}]}
' * 64 + +# (path, content type, templateId). Inserted in this order, so ids do not follow paths. +BLOCKED = { + "a/": [("a/z.html", "text/html", 7), ("a/android/widget/TextView.html", "text/html", 7), + # Looks like a continuation row, but a/z.html is not 1 MiB, so it is a page of its own. + ("a/z.html-1", "text/html", 7), ("a/androidx/a.html", "text/html", 7), + ("a/Zeta.html", "text/html", 7), ("a/androidx/B.html", "text/html", 7), + ("a/android/util/Pair.html", "text/html", 7)], + "j/": [("j/index.html", "text/html", 0), ("j/api/b.html", "text/html", 0), + ("j/Overview.html", "text/html", 0), ("j/api/A.html", "text/html", 0)], + "k/": [("k/html/y.html", "text/html", 9), ("k/html/x.html", "text/html", 8)], +} +LEFT = [ + ("a/devsite/reference/x.html", "text/html", 7), # a/devsite/ is not in the a/ set + ("a/plain.html", "text/html", 0), # no template + ("a/raw.json", "text", 7), # its type is stored uncompressed + ("j/style.css", "text/css", 0), # not HTML + ("j/logo.png", "image/png", 0), + ("k/html/nav.html", "text/html", 0), # no template + ("i/topic.html", "text/html", 7), # not one of the sets +] +CHUNKED = "a/big.html" # a/ templated, but stored as a 1 MiB row and a continuation row +TYPES = [("text/html", "brotli"), ("text/css", "brotli"), ("text", "none"), ("image/png", "none")] +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, + compression TEXT NOT NULL); +CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, + UNIQUE(name)); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (languageID) REFERENCES Languages(id), FOREIGN KEY (contentTypeID) REFERENCES ContentTypes(id), + UNIQUE(path) +); +CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT ''); +""" + + +def _brotli(args: list, data: bytes, dictionary: Path) -> bytes: + return subprocess.run(["brotli", *args, "-D", str(dictionary)], input=data, stdout=subprocess.PIPE, + check=True).stdout + + +def _plain(path: str, rng: random.Random) -> bytes: + words = " ".join(rng.choice(["member", "summary", "inherited", "View", "int", "String"]) + for _ in range(rng.randrange(20, 400))) + return f'{{"path": "{path}", "members": [{{"name": "{words}"}}]}}'.encode() + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + """(database, {path: plain bytes}): the pristine synthetic database.""" + work = tmp_path_factory.mktemp("blocks") + dictionary = work / "dict.bin" + dictionary.write_bytes(DICTIONARY) + rng = random.Random(6683) + db_path = work / "documentation.db" + plain = {} + conn = sqlite3.connect(db_path) + conn.executescript(SCHEMA) + conn.execute("INSERT INTO Languages (value) VALUES ('en')") + conn.executemany("INSERT INTO ContentTypes (value, compression) VALUES (?, ?)", TYPES) + conn.execute("INSERT INTO CompressionDictionary VALUES (1, ?)", (DICTIONARY,)) + conn.executemany("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", + [(7, "android-reference.peb", b"t7"), (8, "page.peb", b"t8"), (9, "nav.peb", b"t9")]) + types = {value: (type_id, compression) for type_id, value, compression in conn.execute("SELECT * FROM ContentTypes")} + pages = [page for group in BLOCKED.values() for page in group] + LEFT + rng.shuffle(pages) + for path, content_type, template_id in pages: + plain[path] = _plain(path, rng) + type_id, compression = types[content_type] + stored = _brotli(["-c", "-q", "5"], plain[path], dictionary) if compression == "brotli" else plain[path] + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, 1, ?, ?, ?)", (path, stored, type_id, template_id)) + # One Brotli stream split across "" and "-1", as a real chunked row is. + plain[CHUNKED] = rng.randbytes(opt.CHUNK_SIZE + 100_000) + stored = _brotli(["-c", "-q", "0"], plain[CHUNKED], dictionary) + assert len(stored) > opt.CHUNK_SIZE + for path, chunk in ((CHUNKED, stored[:opt.CHUNK_SIZE]), (f"{CHUNKED}-1", stored[opt.CHUNK_SIZE:])): + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, 1, ?, ?, 7)", (path, chunk, types["text/html"][0])) + conn.execute("INSERT INTO Bookshelf (contentID, title) SELECT id, 'Pair' FROM Content " + "WHERE path = 'a/android/util/Pair.html'") + conn.commit() + conn.close() + return db_path, plain + + +@pytest.fixture +def database(built, tmp_path): + """A fresh copy of the synthetic database, and the plain bytes of its pages.""" + db_path, plain = built + copy = tmp_path / "documentation.db" + shutil.copyfile(db_path, copy) + return copy, plain + + +def _run(db_path: Path, dry_run: bool = False) -> int: + return spb.run({"db_path": db_path, "pages_per_block": PER_BLOCK, "dry_run": dry_run, "workers": 4}) + + +def _blocks(db_path: Path) -> list: + """[(decoded block, [(path, offset, length)] by offset)] in block-id order.""" + dictionary = db_path.with_name("read.dict.bin") + conn = sqlite3.connect(db_path) + try: + dictionary.write_bytes(conn.execute("SELECT data FROM CompressionDictionary").fetchone()[0]) + blocks = [] + for block_id, data in conn.execute("SELECT id, data FROM ContentBlocks ORDER BY id").fetchall(): + items = conn.execute("SELECT c.path, i.offset, i.length FROM ContentBlockItems AS i JOIN Content AS c " + "ON c.id = i.contentId WHERE i.blockId = ? ORDER BY i.offset", (block_id,)).fetchall() + blocks.append((_brotli(["-d", "-c"], data, dictionary), items)) + return blocks + finally: + conn.close() + + +def _tables(db_path: Path) -> dict: + """Every row of every table, by table. Content's columns: id, path, + languageID, content, contentTypeID, templateId.""" + conn = sqlite3.connect(db_path) + try: + names = [row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")] + return {name: sorted(conn.execute(f'SELECT * FROM "{name}"').fetchall()) for name in names + if not name.startswith("ContentBlock")} + finally: + conn.close() + + +def _in_path_order(paths) -> list: + """ORDER BY path in SQLite's BINARY collation: by UTF-8 bytes.""" + return sorted(paths, key=lambda path: path.encode("utf-8")) + + +def test_blocks_follow_path_order_and_never_mix_sets(database): + db_path, _plain = database + assert _run(db_path) == 0 + blocks = _blocks(db_path) + expected = [path for name in ("a/", "j/", "k/") for path in _in_path_order(p for p, _t, _i in BLOCKED[name])] + assert [path for _data, items in blocks for path, _offset, _length in items] == expected + # Binary, not case-blind: a/Zeta.html sorts before a/android/..., a/androidx/B before a/androidx/a. + assert expected[:3] == ["a/Zeta.html", "a/android/util/Pair.html", "a/android/widget/TextView.html"] + for data, items in blocks: + assert len({path[:2] for path, _offset, _length in items}) == 1 + offset = 0 + for _path, start, length in items: + assert start == offset + offset += length + assert offset == len(data) + + +def test_the_last_block_of_a_set_is_shorter(database): + db_path, _plain = database + assert _run(db_path) == 0 + # a/ has 7 pages, j/ 4 and k/ 2: each set starts a block of its own. + assert [len(items) for _data, items in _blocks(db_path)] == [3, 3, 1, 3, 1, 2] + + +def test_continuation_rows_are_left_alone(database): + db_path, plain = database + before = _tables(db_path)["Content"] + assert _run(db_path) == 0 + blocked = {path for _data, items in _blocks(db_path) for path, _offset, _length in items} + assert CHUNKED not in blocked and f"{CHUNKED}-1" not in blocked + assert "a/z.html-1" in blocked + after = _tables(db_path)["Content"] + for path in (CHUNKED, f"{CHUNKED}-1"): + assert [row for row in after if row[1] == path] == [row for row in before if row[1] == path] + conn = sqlite3.connect(db_path) + try: + first = conn.execute("SELECT content FROM Content WHERE path = ?", (CHUNKED,)).fetchone()[0] + stored = opt.reassemble(conn, CHUNKED, first) + finally: + conn.close() + dictionary = db_path.with_name("chunked.dict.bin") + dictionary.write_bytes(DICTIONARY) + assert _brotli(["-d", "-c"], stored, dictionary) == plain[CHUNKED] + + +def test_every_slice_equals_the_rows_decoded_bytes(database): + db_path, plain = database + assert _run(db_path) == 0 + sliced = 0 + for data, items in _blocks(db_path): + for path, offset, length in items: + assert data[offset:offset + length] == plain[path], path + sliced += 1 + assert sliced == sum(len(group) for group in BLOCKED.values()) + + +def test_blocked_rows_are_emptied_and_nothing_else_changes(database): + db_path, _plain = database + before = _tables(db_path) + assert _run(db_path) == 0 + after = _tables(db_path) + blocked = {path for group in BLOCKED.values() for path, _type, _template in group} + old, new = before.pop("Content"), after.pop("Content") + assert len(new) == len(old) + for was, now in zip(old, new): + if was[1] in blocked: + # id, path, languageID, contentTypeID and templateId stay; content is an empty blob. + assert now[3] == b"" and now[:3] + now[4:] == was[:3] + was[4:] + else: + assert now == was + sequence = dict(before.pop("sqlite_sequence")) + assert dict(after.pop("sqlite_sequence")) == {**sequence, "ContentBlocks": 6} + assert after == before + conn = sqlite3.connect(db_path) + try: + assert conn.execute("SELECT DISTINCT typeof(content) FROM Content WHERE length(content) = 0").fetchall() == \ + [("blob",)] + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] + assert conn.execute("PRAGMA integrity_check").fetchall() == [("ok",)] + finally: + conn.close() + + +def test_a_second_run_is_refused(database, capsys): + db_path, _plain = database + assert _run(db_path) == 0 + once = _sha(db_path) + capsys.readouterr() + assert _run(db_path) == 1 + assert "already has ContentBlockItems" in capsys.readouterr().err + assert _sha(db_path) == once + + +def test_dry_run_writes_nothing(database, capsys): + db_path, _plain = database + pristine = _sha(db_path) + assert _run(db_path, dry_run=True) == 0 + assert "Would store 13 page(s) in 6 block(s)" in capsys.readouterr().out + assert _sha(db_path) == pristine + assert not db_path.with_name(db_path.name + "-journal").exists() + conn = sqlite3.connect(db_path) + try: + assert conn.execute("SELECT name FROM sqlite_master WHERE name LIKE 'ContentBlock%'").fetchall() == [] + finally: + conn.close() From ac11493c8dd3ea656bc13f383c8e5549af5e7c01 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 20 Oct 2026 15:14:49 -0700 Subject: [PATCH 02/17] ADFA-6683: Record the measured saving; say --pack ContentBlocks renumbers blocks Measured on the 2026-10-20 G02 ship-time build (final.db, sha256 c7de1d8a), each database packed with rebuild_database.py --page-size 4096: - Release .br (brotli -q 11 -w 24) 7,904,471 bytes smaller for N = 16 and 10,236,536 for N = 64, of 66,283,003 for the base. - Adding ContentBlocks to --pack: file 823,296 / 200,704 bytes smaller; .br 34,947 bytes larger for 16, 101,421 smaller for 64. - j/'s Javadoc index pages decode to up to 5,065,005 bytes, so two j/ blocks decode to more than the app's 8 MiB cache for N = 16 (largest 14,169,872) and one for N = 64 (28,055,018). The docstring said block ids follow the sets in order. That holds as written, but --pack ContentBlocks renumbers them; nothing reads the order. Comments only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018CMZQuXT3GT7nEct2w4tg3 --- scripts/store_pages_in_blocks.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index 30f81a7b..209e6ac2 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -19,9 +19,16 @@ share - the same member tables, inherited-method lists and markup - is paid for again in every row, and the shared CompressionDictionary only covers what all pages share. Compressed together, N pages in path order share the rest. -The 2026-10-20 size analysis put the release .br saving at 7,865,965 bytes for -N = 16 and 10,062,166 for N = 64. The cost is at read time: serving one page -decodes its whole block, so the app caches decoded blocks. +On the 2026-10-20 G02 ship-time build, packed afterwards (below), the release +.br (brotli -q 11 -w 24) came out 7,904,471 bytes smaller for N = 16 and +10,236,536 for N = 64, of 66,283,003. + +The cost is at read time: serving one page decodes its whole block, so the +app caches decoded blocks, up to 8 MiB of them. Most blocks are far smaller +(on G02 the median a/ block decodes to 259,426 bytes for N = 16), but j/'s +Javadoc index pages decode to as much as 5,065,005 bytes each, so two j/ +blocks decode to more than 8 MiB for N = 16 (the larger 14,169,872 bytes) and +one for N = 64 (28,055,018). The format (ADFA-6683-block-format.md, which the app reader follows too): @@ -35,7 +42,8 @@ Which rows: three sets, each cut in ORDER BY path into blocks of N consecutive items, the last block of a set maybe shorter; a block never mixes -sets. Block ids follow the sets in this order: +sets. Blocks are numbered in this order as written; rebuild_database.py +--pack ContentBlocks renumbers them, and nothing reads the order: 1. a/ templated JSON: path LIKE 'a/%' AND path NOT LIKE 'a/devsite/%' AND templateId > 0 2. j/ HTML: path LIKE 'j/%' AND content type text/html @@ -56,8 +64,11 @@ are empty, so a second run would store empty pages over the blocked ones. Runs in one transaction and VACUUMs afterwards. Takes no backup: run it on a -copy, then pack that with rebuild_database.py --pack Templates,Content,TooltipTexts. ---dry-run does all the work, writes included, and rolls it back. +copy, then pack that with rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks. On G02, adding ContentBlocks to +--pack took the file 823,296 bytes smaller for N = 16 and 200,704 for N = 64, +and its .br 34,947 bytes larger and 101,421 smaller. --dry-run does all the +work, writes included, and rolls it back. """ from __future__ import annotations From 670095fb1e06b1a9a4a85692cbba2b492fb2aa81 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 26 Oct 2026 19:25:34 -0700 Subject: [PATCH 03/17] ADFA-6683: Refuse a 4.x database in the re-mint and its verifier From documentation version 4.0.0, store_pages_in_blocks.py stores most pages in ContentBlocks, compressed against the dictionary, and empties their Content rows. The re-mint recompresses Content rows and templates, not blocks, so a re-mint of a 4.x file would leave every blocked page undecodable, and the verifier compares no blocks either. Both now refuse a database whose newest DocumentationDatabaseVersion row, read as the app reads it, declares major 4 or later: the re-mint before training and before its backup, the verifier for either database. Each says so on one "error:" line naming the blocks step and the ship-time order: re-mint, verify, store_pages_in_blocks.py, then the pack with ContentBlocks. This is the plan's "re-mint before" variant. B02, which would measure whether a dictionary trained on the blocks stores less, has not run; if it pays, this refusal becomes recompress-and-verify code instead. Tests: ContentBlocksRefusalTest (majors 4 and 5 refused before training with nothing written; an older 4 under a newer 3 re-minted and verified; the verifier refusing either side) and two command-line cases (no backup; an error line and no "Byte-identical"). Each fails against the pair broken that way: no re-mint refusal, the threshold at 5, the greatest major read instead of the newest row, the refusal after training and backup, a message without the order, the verifier checking the re-minted file only, and no verifier refusal. Co-Authored-By: Claude Sonnet 5.5 --- .../remint_dictionary.py | 36 ++++++++- .../test_remint_dictionary.py | 77 +++++++++++++++++++ .../verify_remint_dictionary.py | 12 ++- 3 files changed, 122 insertions(+), 3 deletions(-) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py index 540b1f1c..14098ff2 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py @@ -107,6 +107,17 @@ decided the outcome: seed 20821, the only one to beat the dictionary in place, stored 73 bytes more Content than it and 51,657 bytes less in templates. +From documentation version 4.0.0 on (ADFA-6683), scripts/store_pages_in_blocks.py +stores most pages in ContentBlocks, compressed against the same dictionary, +and leaves their Content rows empty. This re-mint recompresses Content rows +and templates, not blocks, so it refuses a database whose newest +DocumentationDatabaseVersion row declares major 4 or later, before training +and with nothing written, backup included; the verifier refuses one too. On +the copy that ships, the re-mint and its verify go before the blocks step: +remint_dictionary.py --seeds 4, verify_remint_dictionary.py, +store_pages_in_blocks.py, then rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks. + Every row is proved to round-trip against the new dictionary before it is written. The backup (VACUUM INTO, unless --no-backup) is taken only then, just before the write: a run that refuses, or keeps the dictionary in place, writes @@ -154,6 +165,12 @@ TEMPLATES_COMPRESSED_FROM_MAJOR = 3 # Templates are sampled under this pseudo path, so they form a doc set of their own. TEMPLATE_SAMPLE_PREFIX = "(templates)/" +# From this documentation major on, most pages are stored in ContentBlocks (ADFA-6683), which neither this re-mint +# nor its verifier handles, so both refuse such a database. +CONTENT_BLOCKS_FROM_MAJOR = 4 +# Where the re-mint and its verify go on the copy that ships, relative to the blocks step and the pack. +SHIP_TIME_ORDER = ("remint_dictionary.py --seeds 4, verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, " + "then scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks") def declared_major(conn) -> int | None: @@ -172,6 +189,19 @@ def templates_compressed(conn) -> bool: return major is not None and major >= TEMPLATES_COMPRESSED_FROM_MAJOR +def content_blocks_refusal(conn, handles: str) -> str | None: + """Why the re-mint or its verifier (`handles`: what it does, as "This re-mint recompresses") must leave this + database alone, or None: from CONTENT_BLOCKS_FROM_MAJOR on, most pages are in ContentBlocks, which neither + handles.""" + major = declared_major(conn) + if major is None or major < CONTENT_BLOCKS_FROM_MAJOR: + return None + return (f"the database declares documentation major {major}, and from {CONTENT_BLOCKS_FROM_MAJOR}.0.0 (ADFA-6683) " + "most pages are stored in ContentBlocks, compressed against the dictionary, with their Content rows " + f"empty. {handles} Content rows and templates, not blocks. Re-mint before " + f"scripts/store_pages_in_blocks.py, never after it: on the copy that ships the order is {SHIP_TIME_ORDER}") + + def load_templates(conn, old_dictionary: bytes) -> list: """Every Templates row of a database whose templates are compressed, as (id, name, stored, plaintext). Raises RuntimeError naming each one that does not decode against the outgoing dictionary.""" @@ -305,7 +335,11 @@ def remint(conn, sample_size: int = REMINT_SAMPLE_SIZE, dict_size: int = REMINT_ with the Content rows, in each candidate's total and the current dictionary's alike. `before_write` is called once every check has passed, just before the write, and not at all when nothing is to be written: main() takes its backup - there.""" + there. A database with blocks (see CONTENT_BLOCKS_FROM_MAJOR) is refused + before anything else.""" + refusal = content_blocks_refusal(conn, "This re-mint recompresses") + if refusal: + raise RuntimeError(f"{refusal}; nothing written") row = conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone() if row is None or not row[0]: raise RuntimeError("this database has no CompressionDictionary to re-mint; " diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py index 87c60c9d..ad4b972d 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py @@ -726,6 +726,66 @@ def test_verifier_catches_a_changed_missing_stale_or_extra_template(self): "template page.peb"]) +class ContentBlocksRefusalTest(_RemintFixture): + """From documentation version 4.0.0 on (ADFA-6683) most pages are in ContentBlocks, compressed against the + dictionary, which neither the re-mint nor its verifier handles: both refuse a database declaring major 4 or later, + as the app reads its version, write nothing, and say the re-mint goes before the blocks step.""" + + def remint(self, **kwargs): + return remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20, seed=999, **kwargs) + + def assertNamesTheBlocksStepAndTheOrder(self, message: str): + self.assertIn("Re-mint before scripts/store_pages_in_blocks.py", message) + self.assertIn("verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, then " + "scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks", message) + + def test_the_remint_refuses_major_4_or_later_before_training(self): + for majors in ((3, 4), (5,)): + with self.subTest(majors=majors): + self.tearDown() + self.setUp() + self.add_versions_and_templates(majors=majors) + content, templates = dict(self.conn.execute("SELECT path, content FROM Content")), \ + self.stored_templates() + with mock.patch.object(remint_dictionary, "train_dictionary", + side_effect=AssertionError("trained on a database with blocks")): + with self.assertRaises(RuntimeError) as raised: + self.remint() + message = str(raised.exception) + self.assertIn(f"declares documentation major {majors[-1]}", message) + self.assertIn("nothing written", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway") + self.assertEqual(dict(self.conn.execute("SELECT path, content FROM Content")), content) + self.assertEqual(self.stored_templates(), templates) + + def test_the_newest_version_row_decides(self): + """An older row declaring 4 under a newer one declaring 3 is a 3.x database, as the app reads it: re-minted, + templates and all, and verified.""" + self.add_versions_and_templates(majors=(4, 3)) + before = self.snapshot() + stats = self.remint() + self.assertEqual(stats["kept"], "candidate") + self.assertEqual(stats["templates"], len(TEMPLATE_NAMES)) + self.assertEqual(verify(before, self.db_path), []) + + def test_the_verifier_refuses_either_database_declaring_major_4(self): + self.add_versions_and_templates(majors=(3,)) + original = self.snapshot() + self.conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (4, 0, 0, 'ADFA-6683', 'blocks', '2026-10-26 00:00:00')") + self.conn.commit() + for old, new in ((original, self.db_path), (self.db_path, original)): + with self.subTest(old=old.name, new=new.name): + with self.assertRaises(RuntimeError) as raised: + verify(old, new) + message = str(raised.exception) + self.assertTrue(message.startswith(f"{self.db_path}: the database declares documentation major 4"), + message) + self.assertIn("This check compares Content rows and templates, not blocks", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + + class CommandLineRefusalTest(_RemintFixture): """A run that refuses writes nothing at all, the backup included, and says why on one "error:" line rather than in a traceback.""" @@ -779,6 +839,23 @@ def test_a_content_path_in_the_templates_doc_set_refuses_with_no_backup(self): sha = self.sha() self.assertRefusedWithNothingWritten(self.run_remint(), sha, "(templates)/page.peb") + def test_a_database_with_blocks_refuses_with_no_backup(self): + self.add_versions_and_templates(majors=(3, 4)) + sha = self.sha() + self.assertRefusedWithNothingWritten(self.run_remint(), sha, "Re-mint before scripts/store_pages_in_blocks.py") + + def test_the_verifier_reports_a_database_with_blocks_on_an_error_line(self): + self.add_versions_and_templates(majors=(3, 4)) + original = self.dir / "original.db" + shutil.copy2(self.db_path, original) + result = self.run_script("verify_remint_dictionary.py", str(original), str(self.db_path)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertTrue(result.stderr.startswith(f"error: {original}: the database declares documentation major 4"), + result.stderr) + self.assertIn("store_pages_in_blocks.py", result.stderr) + self.assertNotIn("Byte-identical", result.stdout) + def test_no_dictionary_refuses_with_no_backup(self): self.conn.execute("DELETE FROM CompressionDictionary") self.conn.commit() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py index e03cda0b..87e466f6 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py @@ -30,7 +30,11 @@ against the original, before the file is put in place. Reads only: neither database is modified. A database it cannot check (no -CompressionDictionary row) is reported on an "error:" line, exit 1. +CompressionDictionary row) is reported on an "error:" line, exit 1. So is a +pair where either database declares documentation major 4 or later +(ADFA-6683): most of its pages are in ContentBlocks, which this check does not +compare, and which remint_dictionary.py refuses to re-mint. Re-mint and verify +before scripts/store_pages_in_blocks.py. """ import argparse import sys @@ -42,7 +46,7 @@ from populate_db import DictionaryCompressor from migrate_content_to_dictionary_brotli import load_base_rows, read_item -from remint_dictionary import templates_compressed +from remint_dictionary import content_blocks_refusal, templates_compressed _thread_local = threading.local() @@ -142,6 +146,10 @@ def verify(old_db: Path, new_db: Path, max_workers: int | None = None) -> list: import sqlite3 old_conn = sqlite3.connect(f"file:{old_db}?mode=ro", uri=True) new_conn = sqlite3.connect(f"file:{new_db}?mode=ro", uri=True) + for path, conn in ((old_db, old_conn), (new_db, new_conn)): + refusal = content_blocks_refusal(conn, "This check compares") + if refusal: + raise RuntimeError(f"{path}: {refusal}") old_dictionary = dictionary_of(old_conn, old_db) new_dictionary = dictionary_of(new_conn, new_db) if old_dictionary == new_dictionary: From 1bb6afe02b0f839dd82574bcdf9170049b1c3eb8 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 26 Oct 2026 19:25:48 -0700 Subject: [PATCH 04/17] ADFA-6683: Block five sets, items up to 1 MiB, blocks up to 4 MiB, 64 by default - Five sets, in this order, each in path order and never mixed in a block: a/ templated JSON (not a/devsite/), j/ HTML, k/ templated JSON, then every Brotli row under a/devsite/ and under i/. Paths are matched with GLOB, case-sensitive as paths are, where LIKE is not. - The cap: an item over 1 MiB decoded stays a row. A block closes at --pages-per-block items, now 64 by default, or before the next item would take it past 4 MiB decoded, so two blocks fit the app's 8 MiB cache. Rows are decoded in the thread pool ahead of the grouping walk, a window at a time, and blocks are built a few ahead of the writes, so no set is held decoded all at once. - Before a row is emptied, its block is decoded back and each item's slice compared byte for byte with the row's decoded bytes. The sha256 pass over the stored blocks before the commit stays. - The summary shows every set with its blocked content types, the pages the cap kept (named, up to 10 a set) and the rows left for their continuation rows. - Docstring: the five sets, the cap, where the step goes, and G02's figures with them, replacing the prototype's 16/64 figures and its reference to a format file the repo does not have. On the 2026-10-20 G02 ship-time build (the prototype's input, sha256 c7de1d8a): 22,478 pages in 356 blocks, rows 34,834,228 -> blocks 24,608,732 bytes; packed with --pack Templates,Content,TooltipTexts,ContentBlocks, the .br went from 66,283,003 to 55,914,367 (10,368,636 smaller). The cap kept 18 pages (5 a/, 12 j/, 1 k/); the largest block decodes to 4,172,562 bytes. integrity_check and foreign_key_check were clean before and after the pack, and the pack's identity check passed. Tests, on the synthetic database: untemplated, non-HTML a/devsite/ and i/ rows blocked; an A/ path left out; nine blocks in set order. On a second database: an item one byte over 1 MiB kept, one exactly at it blocked; a block of exactly 4 MiB, and one closed at 3.5 MiB; 65 k/ pages making 64 + 1 through main()'s default; a codec garbling only the decode-back, and one garbling only the decode before the commit, each refusing with nothing written. Each fails against the script broken that way: three sets, LIKE, sets reordered, a/devsite/ or i/ narrowed, no item cap, the cap at instead of over, on stored size, or at 2 MiB, no block cap, at instead of past, or at 8 MiB, a default of 16 or none, no byte-for-byte check, a length-only check, and no sha256 pass. Co-Authored-By: Claude Sonnet 5.5 --- scripts/store_pages_in_blocks.py | 293 ++++++++++++++++--------- scripts/test_store_pages_in_blocks.py | 302 +++++++++++++++++++++----- 2 files changed, 444 insertions(+), 151 deletions(-) diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index 209e6ac2..6404527c 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -9,66 +9,90 @@ """ store_pages_in_blocks.py -Stores the pages of three documentation sets N to a compressed block instead -of one compressed Content row each (ADFA-6683, prototype): +Stores the pages of five documentation sets up to 64 to a compressed block, +instead of one compressed Content row each (ADFA-6683): - uv run scripts/store_pages_in_blocks.py documentation.db --pages-per-block 16 --dry-run - uv run scripts/store_pages_in_blocks.py documentation.db --pages-per-block 16 + uv run scripts/store_pages_in_blocks.py documentation.db --dry-run + uv run scripts/store_pages_in_blocks.py documentation.db Why: each Content row is compressed on its own, so what neighbouring pages share - the same member tables, inherited-method lists and markup - is paid for again in every row, and the shared CompressionDictionary only covers what -all pages share. Compressed together, N pages in path order share the rest. -On the 2026-10-20 G02 ship-time build, packed afterwards (below), the release -.br (brotli -q 11 -w 24) came out 7,904,471 bytes smaller for N = 16 and -10,236,536 for N = 64, of 66,283,003. - -The cost is at read time: serving one page decodes its whole block, so the -app caches decoded blocks, up to 8 MiB of them. Most blocks are far smaller -(on G02 the median a/ block decodes to 259,426 bytes for N = 16), but j/'s -Javadoc index pages decode to as much as 5,065,005 bytes each, so two j/ -blocks decode to more than 8 MiB for N = 16 (the larger 14,169,872 bytes) and -one for N = 64 (28,055,018). - -The format (ADFA-6683-block-format.md, which the app reader follows too): +all pages share. Compressed together, the pages of a block in path order share +the rest. On the 2026-10-20 G02 ship-time build (3.0.0, after its re-mint), +this step stored 22,478 pages in 356 blocks: their rows held 34,834,228 bytes, +the blocks hold 24,608,732 (a/ 5,457,314 less, j/ 3,549,974, k/ 758,039, +a/devsite/ 296,984, i/ 163,185). Packed with rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096, the file went +from 84,688,896 bytes (the same build packed without blocks) to 74,616,832, +and its release .br (brotli -q 11 -w 24) from 66,283,003 to 55,914,367: +10,368,636 bytes smaller. The cap kept 18 pages as rows (5 a/, 12 j/, 1 k/), +and the largest block decodes to 4,172,562 bytes. With 16 workers the step +took 53 seconds. + +The format: ContentBlocks(id, data) - data: Brotli of the block's plain items, concatenated - quality 11, - window 24, against CompressionDictionary, as Content rows are. + data: Brotli of the block's items, concatenated - quality 11, window + 24, against CompressionDictionary, as Content rows are. ContentBlockItems(contentId, blockId, offset, length) WITHOUT ROWID The item is [offset, offset + length) of the decoded block. Its foreign key to Content lets rebuild_database.py --pack Content renumber contentId with Content. -Which rows: three sets, each cut in ORDER BY path into blocks of N -consecutive items, the last block of a set maybe shorter; a block never mixes -sets. Blocks are numbered in this order as written; rebuild_database.py ---pack ContentBlocks renumbers them, and nothing reads the order: +An item is exactly its row's decoded bytes, before any template is applied. A +blocked row keeps its path, languageID, contentTypeID and templateId, so every +lookup by path and every page's template work as before; only its content +becomes an empty blob. - 1. a/ templated JSON: path LIKE 'a/%' AND path NOT LIKE 'a/devsite/%' AND templateId > 0 - 2. j/ HTML: path LIKE 'j/%' AND content type text/html - 3. k/ templated JSON: path LIKE 'k/%' AND templateId > 0 +Which rows: five sets, each in ORDER BY path COLLATE BINARY, in this order; a +block never mixes sets. Blocks are numbered in this order as written; +rebuild_database.py --pack ContentBlocks renumbers them, and nothing reads the +order. Paths are matched with GLOB, which is case-sensitive as paths are: + + 1. a/ templated JSON: path GLOB 'a/*' AND NOT GLOB 'a/devsite/*' AND templateId > 0 + 2. j/ HTML: path GLOB 'j/*' AND content type text/html + 3. k/ templated JSON: path GLOB 'k/*' AND templateId > 0 + 4. a/devsite/: every row under it + 5. i/: every row under it Only rows whose content type is compressed with brotli, and only items held in one row: a row that owns continuation rows ("path-1", ... after a 1 MiB row), -and each continuation row, stay as they are. An item is exactly its row's -decoded bytes, before any template rendering. A blocked row keeps its path, -languageID, contentTypeID and templateId, so every lookup by path and every -page's template work as before; only its content becomes an empty blob. - -Every block is decoded again before it is stored and compared with the items -it was built from, and before the commit every item is sliced from its stored -block and compared, by sha256, with its row's decoded bytes. - -A database that already has ContentBlockItems is refused: its blocked rows -are empty, so a second run would store empty pages over the blocked ones. +and each continuation row, stay as they are. + +The cap: an item over MAX_ITEM_BYTES (1 MiB) decoded stays a row. A block +closes at --pages-per-block items (default 64), or before the next item would +take it past MAX_BLOCK_BYTES (4 MiB) decoded. Serving a page decodes its whole +block, and the app caches decoded blocks up to 8 MiB, so two blocks always fit. +Without the cap, j/'s Javadoc index pages (up to 5,065,005 bytes each) made a +64-page block decode to 28,055,018 bytes. The summary counts, for each set, +the pages in blocks and their content types, the pages the cap kept as rows, +and the rows left for their continuation rows. + +Checks: every block is decoded again before its rows are emptied, and each +item's slice of it compared byte for byte with its row's decoded bytes. Before +the commit every item is sliced from its block as stored and compared, by +sha256, with its row's decoded bytes, and every blocked row must be empty. Any +failure rolls everything back. + +A database that already has ContentBlocks or ContentBlockItems is refused +too: its blocked rows are empty, so a second run would store empty pages over +the blocked ones. + +Where it goes: on the copy that ships only, never the master, after the last +dictionary re-mint and its verify (remint_dictionary.py and +verify_remint_dictionary.py refuse a 4.x database: they do not recompress +blocks) and before the pack. On that copy the order is: minify, row drops, +Kotlin trim, a/ pages-to-templates, migrate_tooltip_schema.py, +remint_dictionary.py --seeds 4 and verify_remint_dictionary.py, this step, then +rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks +--page-size 4096. A database with blocks goes to dev-assets or Drive only +once the app's block reader is on CodeOnTheGo's stage: builds take whatever +dev-assets holds, and an app without the reader would show every blocked page +empty. Runs in one transaction and VACUUMs afterwards. Takes no backup: run it on a -copy, then pack that with rebuild_database.py --pack -Templates,Content,TooltipTexts,ContentBlocks. On G02, adding ContentBlocks to ---pack took the file 823,296 bytes smaller for N = 16 and 200,704 for N = 64, -and its .br 34,947 bytes larger and 101,421 smaller. --dry-run does all the -work, writes included, and rolls it back. +copy. --dry-run does all the work, writes included, and rolls it back. """ from __future__ import annotations @@ -79,22 +103,33 @@ import subprocess import sys import tempfile +from collections import Counter, deque from concurrent.futures import ThreadPoolExecutor from pathlib import Path from optimize_db_media import human, is_continuation_path, load_dictionary, owned_fragment_paths -# (set, condition on Content AS c joined to ContentTypes AS t), in block-id order. +# (set, condition on Content AS c joined to ContentTypes AS t), in block-id order. GLOB, not LIKE: LIKE ignores case. SETS = ( - ("a/", "c.path LIKE 'a/%' AND c.path NOT LIKE 'a/devsite/%' AND c.templateId > 0"), - ("j/", "c.path LIKE 'j/%' AND t.value = 'text/html'"), - ("k/", "c.path LIKE 'k/%' AND c.templateId > 0"), + ("a/", "c.path GLOB 'a/*' AND c.path NOT GLOB 'a/devsite/*' AND c.templateId > 0"), + ("j/", "c.path GLOB 'j/*' AND t.value = 'text/html'"), + ("k/", "c.path GLOB 'k/*' AND c.templateId > 0"), + ("a/devsite/", "c.path GLOB 'a/devsite/*'"), + ("i/", "c.path GLOB 'i/*'"), ) +# Decoded sizes. An item over MAX_ITEM_BYTES stays a row; a block closes before an item would take it past +# MAX_BLOCK_BYTES, so two blocks fit the app's 8 MiB cache of decoded blocks. +MAX_ITEM_BYTES = 1024 * 1024 +MAX_BLOCK_BYTES = 4 * 1024 * 1024 +PAGES_PER_BLOCK = 64 +# Rows decoded ahead of the grouping walk, at most, so a set is never held decoded all at once. +DECODE_WINDOW = 256 + SCHEMA = ( """CREATE TABLE ContentBlocks ( id INTEGER PRIMARY KEY AUTOINCREMENT, - data BLOB NOT NULL -- Brotli of the block's plain items, concatenated + data BLOB NOT NULL -- Brotli (q11, w24) of the block's items, concatenated, against CompressionDictionary )""", """CREATE TABLE ContentBlockItems ( contentId INTEGER NOT NULL PRIMARY KEY REFERENCES Content(id), @@ -139,44 +174,110 @@ def table_exists(conn, name: str) -> bool: def select_sets(conn) -> list: - """[(set, [(content id, path, stored bytes)] in path order, items held in - more than one row, their continuation rows)] for each of SETS.""" + """[(set, [(content id, path, content type, stored bytes)] in path order, items held in more than one row, + their continuation rows)] for each of SETS.""" lengths = dict(conn.execute("SELECT path, LENGTH(content) FROM Content")) selected = [] for name, condition in SETS: rows, chunked, continuations = [], 0, 0 - for content_id, path, stored in conn.execute( - "SELECT c.id, c.path, c.content FROM Content AS c JOIN ContentTypes AS t ON t.id = c.contentTypeID " - f"WHERE t.compression = 'brotli' AND {condition} ORDER BY c.path COLLATE BINARY"): + for content_id, path, content_type, stored in conn.execute( + "SELECT c.id, c.path, t.value, c.content FROM Content AS c JOIN ContentTypes AS t " + f"ON t.id = c.contentTypeID WHERE t.compression = 'brotli' AND {condition} " + "ORDER BY c.path COLLATE BINARY"): if is_continuation_path(lengths, path): continuations += 1 elif owned_fragment_paths(conn, path, len(stored)): chunked += 1 else: - rows.append((content_id, path, stored)) + rows.append((content_id, path, content_type, stored)) selected.append((name, rows, chunked, continuations)) return selected -def build_block(codec: DictionaryBrotli, rows: list) -> tuple: - """(compressed block, plain length, [(content id, offset, length, sha256)]) - for one block's rows, the block decoded again and checked.""" - plain, items, offset = [], [], 0 - for content_id, path, stored in rows: - try: - raw = codec.decode(stored) - except RuntimeError as exc: - raise RuntimeError(f"{path}: {exc}") from exc +def decode_row(codec: DictionaryBrotli, row: tuple) -> bytes: + try: + return codec.decode(row[3]) + except RuntimeError as exc: + raise RuntimeError(f"{row[1]}: {exc}") from exc + + +def decoded_in_order(pool, codec: DictionaryBrotli, rows: list, window: int = DECODE_WINDOW): + """(row, its decoded bytes) for every row, in order, decoded in the pool at most `window` rows ahead.""" + for start in range(0, len(rows), window): + batch = rows[start:start + window] + yield from zip(batch, pool.map(lambda row: decode_row(codec, row), batch)) + + +def group_items(decoded, per_block: int, kept: list): + """Walks (row, decoded bytes) in path order and yields the blocks, each a list of them. A block closes at + `per_block` items, or before the next item would take it past MAX_BLOCK_BYTES decoded. An item over + MAX_ITEM_BYTES decoded goes to `kept` instead, and stays a row.""" + group, size = [], 0 + for row, plain in decoded: + if len(plain) > MAX_ITEM_BYTES: + kept.append(row) + continue + if group and (len(group) == per_block or size + len(plain) > MAX_BLOCK_BYTES): + yield group + group, size = [], 0 + group.append((row, plain)) + size += len(plain) + if group: + yield group + + +def build_block(codec: DictionaryBrotli, group: list) -> tuple: + """(compressed block, decoded length, [(content id, offset, length, sha256)]) for one block's (row, decoded + bytes), once the block has been decoded back and each item's slice of it matched its row's decoded bytes byte + for byte.""" + plain = b"".join(raw for _row, raw in group) + data = codec.encode(plain) + back = codec.decode(data) + first = group[0][0][1] + if len(back) != len(plain): + raise RuntimeError(f"the block starting at {first} decoded back to {human(len(back))} bytes, not " + f"{human(len(plain))}") + view, items, offset = memoryview(back), [], 0 + for (content_id, path, _type, _stored), raw in group: + if view[offset:offset + len(raw)] != raw: + raise RuntimeError(f"{path} does not read back byte for byte from the block starting at {first}") items.append((content_id, offset, len(raw), hashlib.sha256(raw).digest())) - plain.append(raw) offset += len(raw) - plain = b"".join(plain) - data = codec.encode(plain) - if codec.decode(data) != plain: - raise RuntimeError(f"the block starting at {rows[0][1]} did not decode back to its items") return data, len(plain), items +def store_set(conn, pool, codec: DictionaryBrotli, rows: list, per_block: int, workers: int, expected: dict) -> dict: + """Stores one set's rows in blocks, written in order as they are built, a few blocks ahead at most. Adds each + blocked item's sha256 to `expected` ({content id: sha256}); returns the set's figures.""" + stored_length = {row[0]: len(row[3]) for row in rows} + content_type = {row[0]: row[2] for row in rows} + figures = {"blocks": 0, "items": 0, "stored": 0, "compressed": 0, "decoded": 0, "types": Counter(), "kept": []} + pending = deque() + + def write(future): + data, plain_length, items = future.result() + block_id = conn.execute("INSERT INTO ContentBlocks (data) VALUES (?)", (data,)).lastrowid + for content_id, offset, length, digest in items: + conn.execute("INSERT INTO ContentBlockItems (contentId, blockId, offset, length) VALUES (?, ?, ?, ?)", + (content_id, block_id, offset, length)) + conn.execute("UPDATE Content SET content = x'' WHERE id = ?", (content_id,)) + expected[content_id] = digest + figures["stored"] += stored_length[content_id] + figures["types"][content_type[content_id]] += 1 + figures["blocks"] += 1 + figures["items"] += len(items) + figures["compressed"] += len(data) + figures["decoded"] += plain_length + + for group in group_items(decoded_in_order(pool, codec, rows), per_block, figures["kept"]): + pending.append(pool.submit(build_block, codec, group)) + while len(pending) > 2 * workers: + write(pending.popleft()) + while pending: + write(pending.popleft()) + return figures + + def check_stored(conn, codec: DictionaryBrotli, expected: dict, workers: int) -> None: """Slices every item from its block as stored and compares it with its row's decoded bytes (`expected`: {content id: sha256}); every blocked row @@ -204,6 +305,26 @@ def check(block): raise RuntimeError(f"{left} blocked row(s) still hold their content") +def report(summary: list, per_block: int, dry_run: bool) -> None: + for name, figures, chunked, continuations in summary: + types = ", ".join(f"{value} {human(count)}" for value, count in figures["types"].most_common()) + print(f" {name}: {human(figures['items'])} page(s) in {human(figures['blocks'])} block(s); their rows held " + f"{human(figures['stored'])} bytes, the blocks hold {human(figures['compressed'])} " + f"({human(figures['decoded'])} decoded). Content types: {types or 'none'}.") + kept = figures["kept"] + print(f" {name}: left as rows: {human(len(kept))} page(s) over {human(MAX_ITEM_BYTES)} bytes decoded, " + f"{human(chunked)} page(s) held in more than one row, and {human(continuations)} continuation row(s).") + for _id, path, _type, _stored in kept[:10]: + print(f" kept by the cap: {path}") + if len(kept) > 10: + print(f" ... and {human(len(kept) - 10)} more") + saved = sum(figures["stored"] - figures["compressed"] for _name, figures, _c, _n in summary) + verb = "Would store" if dry_run else "Stored" + print(f"{verb} {human(sum(figures['items'] for _n, figures, _c, _k in summary))} page(s) in " + f"{human(sum(figures['blocks'] for _n, figures, _c, _k in summary))} block(s) of up to {per_block} pages and " + f"{human(MAX_BLOCK_BYTES)} bytes decoded, {human(saved)} bytes smaller than their rows.") + + def run(cfg: dict) -> int: db_path, per_block, workers = cfg["db_path"], cfg["pages_per_block"], cfg.get("workers") or 1 if per_block < 1: @@ -233,9 +354,9 @@ def run(cfg: dict) -> int: selected = select_sets(conn) if cfg["dry_run"]: - print(f"Dry run: every write is rolled back. Storing {db_path}'s pages {per_block} to a block ...") + print(f"Dry run: every write is rolled back. Storing {db_path}'s pages up to {per_block} to a block ...") else: - print(f"Storing {db_path}'s pages {per_block} to a block ...") + print(f"Storing {db_path}'s pages up to {per_block} to a block ...") conn.execute("BEGIN") try: for sql in SCHEMA: @@ -243,21 +364,8 @@ def run(cfg: dict) -> int: expected = {} with ThreadPoolExecutor(max_workers=workers) as pool: for name, rows, chunked, continuations in selected: - groups = [rows[start:start + per_block] for start in range(0, len(rows), per_block)] - stored = compressed = plain_total = 0 - for group, (data, plain_length, items) in zip( - groups, pool.map(lambda group: build_block(codec, group), groups)): - block_id = conn.execute("INSERT INTO ContentBlocks (data) VALUES (?)", (data,)).lastrowid - for content_id, offset, length, digest in items: - conn.execute("INSERT INTO ContentBlockItems (contentId, blockId, offset, length) " - "VALUES (?, ?, ?, ?)", (content_id, block_id, offset, length)) - conn.execute("UPDATE Content SET content = x'' WHERE id = ?", (content_id,)) - expected[content_id] = digest - stored += sum(len(row[2]) for row in group) - compressed += len(data) - plain_total += plain_length - summary.append((name, len(rows), len(groups), stored, compressed, plain_total, chunked, - continuations)) + figures = store_set(conn, pool, codec, rows, per_block, workers, expected) + summary.append((name, figures, chunked, continuations)) check_stored(conn, codec, expected, workers) if cfg["dry_run"]: conn.execute("ROLLBACK") @@ -275,17 +383,7 @@ def run(cfg: dict) -> int: codec.close() conn.close() - for name, items, blocks, stored, compressed, plain, chunked, continuations in summary: - print(f" {name}: {human(items)} page(s) in {human(blocks)} block(s); their rows held {human(stored)} " - f"bytes, the blocks hold {human(compressed)} ({human(plain)} decoded).") - if chunked or continuations: - print(f" {name}: left as they are: {human(chunked)} page(s) held in more than one row, and " - f"{human(continuations)} continuation row(s).") - saved = sum(row[3] - row[4] for row in summary) - verb = "Would store" if cfg["dry_run"] else "Stored" - print(f"{verb} {human(sum(row[1] for row in summary))} page(s) in {human(sum(row[2] for row in summary))} " - f"block(s) of up to {per_block}, {human(saved)} bytes smaller than their rows.") - + report(summary, per_block, cfg["dry_run"]) if not cfg["dry_run"]: print("Vacuuming database...") vac = sqlite3.connect(db_path) @@ -300,8 +398,9 @@ def run(cfg: dict) -> int: def main() -> None: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("db_path", type=Path, help="SQLite database to change in place; run it on a copy") - p.add_argument("--pages-per-block", type=int, required=True, metavar="N", - help="Pages per block; the last block of each set may hold fewer") + p.add_argument("--pages-per-block", type=int, default=PAGES_PER_BLOCK, metavar="N", + help=f"Pages per block at most (default: {PAGES_PER_BLOCK}); a block also closes before it would " + f"decode to more than {human(MAX_BLOCK_BYTES)} bytes") p.add_argument("--dry-run", action="store_true", help="Do all the work, then roll back without writing") p.add_argument("--workers", type=int, default=8, help="brotli processes run at once to decode rows and compress blocks (default: 8)") diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py index a2163c3b..96681f0e 100644 --- a/scripts/test_store_pages_in_blocks.py +++ b/scripts/test_store_pages_in_blocks.py @@ -10,11 +10,15 @@ """ test_store_pages_in_blocks.py -Covers store_pages_in_blocks.py on a small synthetic database, three pages to -a block: each set in path order with its own blocks, the last one shorter; -rows held in more than one row left alone; every item sliced from its block -equal to its row's decoded bytes; blocked rows emptied and nothing else -changed; a second run refused; a dry run writing nothing. +Covers store_pages_in_blocks.py on small synthetic databases. On the first, +three pages to a block: each of the five sets in path order with its own +blocks, the last one shorter, a/devsite/ and i/ rows taken whether templated +or not; rows held in more than one row left alone; every item sliced from its +block equal to its row's decoded bytes; blocked rows emptied and nothing else +changed; a second run refused; a dry run, and a block that does not read back +byte for byte, writing nothing. On +the second, with the default 64: an item over 1 MiB decoded kept as a row, and +a block closed before it would decode past 4 MiB. uv run --with pytest --with-requirements requirements.txt python -m pytest -q scripts/test_store_pages_in_blocks.py """ @@ -26,6 +30,7 @@ import sqlite3 import subprocess import sys +import threading from pathlib import Path import pytest @@ -36,6 +41,9 @@ import store_pages_in_blocks as spb # noqa: E402 PER_BLOCK = 3 +# The cap, written out rather than read from the script, so a script with another cap fails here. +ITEM_CAP = 1024 * 1024 +BLOCK_CAP = 4 * 1024 * 1024 DICTIONARY = b'{"title": "", "members": [{"name": "", "summary": ""}]}
' * 64 # (path, content type, templateId). Inserted in this order, so ids do not follow paths. @@ -48,18 +56,27 @@ "j/": [("j/index.html", "text/html", 0), ("j/api/b.html", "text/html", 0), ("j/Overview.html", "text/html", 0), ("j/api/A.html", "text/html", 0)], "k/": [("k/html/y.html", "text/html", 9), ("k/html/x.html", "text/html", 8)], + # Every Brotli row under a/devsite/ and i/, templated or not, HTML or not. + "a/devsite/": [("a/devsite/reference/x.html", "text/html", 7), ("a/devsite/b.html", "text/html", 0), + ("a/devsite/style.css", "text/css", 0), ("a/devsite/Z.js", "text/javascript", 0)], + "i/": [("i/topic.html", "text/html", 0), ("i/jstopics/topic.js", "text/javascript", 0), + ("i/hm-topic.html", "text/html", 8)], } LEFT = [ - ("a/devsite/reference/x.html", "text/html", 7), # a/devsite/ is not in the a/ set - ("a/plain.html", "text/html", 0), # no template + ("a/plain.html", "text/html", 0), # a/ without a template ("a/raw.json", "text", 7), # its type is stored uncompressed + ("A/Upper.html", "text/html", 7), # not under a/: GLOB tells case apart, as paths do, where LIKE would not ("j/style.css", "text/css", 0), # not HTML ("j/logo.png", "image/png", 0), ("k/html/nav.html", "text/html", 0), # no template - ("i/topic.html", "text/html", 7), # not one of the sets + ("a/devsite/logo.png", "image/png", 0), # under a/devsite/, but not Brotli + ("i/img.png", "image/png", 0), # under i/, but not Brotli + ("d/other.html", "text/html", 7), # in no set ] -CHUNKED = "a/big.html" # a/ templated, but stored as a 1 MiB row and a continuation row -TYPES = [("text/html", "brotli"), ("text/css", "brotli"), ("text", "none"), ("image/png", "none")] +# Each stored as a 1 MiB row and a continuation row: in a set, but left as they are. +CHUNKED = [("a/big.html", "text/html", 7), ("i/book.pdf", "application/pdf", 0)] +TYPES = [("text/html", "brotli"), ("text/css", "brotli"), ("text/javascript", "brotli"), + ("application/pdf", "brotli"), ("text", "none"), ("image/png", "none")] SCHEMA = """ CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, @@ -88,19 +105,21 @@ def _plain(path: str, rng: random.Random) -> bytes: return f'{{"path": "{path}", "members": [{{"name": "{words}"}}]}}'.encode() +def _sized(path: str, size: int) -> bytes: + """`size` bytes naming `path`, cheap to compress at any quality.""" + head = f'{{"path": "{path}", "text": "'.encode() + return (head + b"m" * size)[:size] + + def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -@pytest.fixture(scope="module") -def built(tmp_path_factory): - """(database, {path: plain bytes}): the pristine synthetic database.""" - work = tmp_path_factory.mktemp("blocks") - dictionary = work / "dict.bin" +def _create(db_path: Path) -> tuple: + """A new database with SCHEMA, its types and templates; returns (connection, {content type: (id, + compression)}, the dictionary as a file).""" + dictionary = db_path.with_name("dict.bin") dictionary.write_bytes(DICTIONARY) - rng = random.Random(6683) - db_path = work / "documentation.db" - plain = {} conn = sqlite3.connect(db_path) conn.executescript(SCHEMA) conn.execute("INSERT INTO Languages (value) VALUES ('en')") @@ -109,21 +128,37 @@ def built(tmp_path_factory): conn.executemany("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", [(7, "android-reference.peb", b"t7"), (8, "page.peb", b"t8"), (9, "nav.peb", b"t9")]) types = {value: (type_id, compression) for type_id, value, compression in conn.execute("SELECT * FROM ContentTypes")} + return conn, types, dictionary + + +def _insert(conn, types: dict, dictionary: Path, path: str, content_type: str, template_id: int, plain: bytes, + quality: str = "5") -> None: + type_id, compression = types[content_type] + stored = _brotli(["-c", "-q", quality], plain, dictionary) if compression == "brotli" else plain + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, ?)", + (path, stored, type_id, template_id)) + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + """(database, {path: plain bytes}): the pristine synthetic database.""" + db_path = tmp_path_factory.mktemp("blocks") / "documentation.db" + rng = random.Random(6683) + plain = {} + conn, types, dictionary = _create(db_path) pages = [page for group in BLOCKED.values() for page in group] + LEFT rng.shuffle(pages) for path, content_type, template_id in pages: plain[path] = _plain(path, rng) - type_id, compression = types[content_type] - stored = _brotli(["-c", "-q", "5"], plain[path], dictionary) if compression == "brotli" else plain[path] - conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " - "VALUES (?, 1, ?, ?, ?)", (path, stored, type_id, template_id)) + _insert(conn, types, dictionary, path, content_type, template_id, plain[path]) # One Brotli stream split across "" and "-1", as a real chunked row is. - plain[CHUNKED] = rng.randbytes(opt.CHUNK_SIZE + 100_000) - stored = _brotli(["-c", "-q", "0"], plain[CHUNKED], dictionary) - assert len(stored) > opt.CHUNK_SIZE - for path, chunk in ((CHUNKED, stored[:opt.CHUNK_SIZE]), (f"{CHUNKED}-1", stored[opt.CHUNK_SIZE:])): - conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " - "VALUES (?, 1, ?, ?, 7)", (path, chunk, types["text/html"][0])) + for path, content_type, template_id in CHUNKED: + plain[path] = rng.randbytes(opt.CHUNK_SIZE + 100_000) + stored = _brotli(["-c", "-q", "0"], plain[path], dictionary) + assert len(stored) > opt.CHUNK_SIZE + for part, chunk in ((path, stored[:opt.CHUNK_SIZE]), (f"{path}-1", stored[opt.CHUNK_SIZE:])): + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, 1, ?, ?, ?)", (part, chunk, types[content_type][0], template_id)) conn.execute("INSERT INTO Bookshelf (contentID, title) SELECT id, 'Pair' FROM Content " "WHERE path = 'a/android/util/Pair.html'") conn.commit() @@ -140,8 +175,44 @@ def database(built, tmp_path): return copy, plain -def _run(db_path: Path, dry_run: bool = False) -> int: - return spb.run({"db_path": db_path, "pages_per_block": PER_BLOCK, "dry_run": dry_run, "workers": 4}) +# The cap's database, in path order within each set: +# a/: one item 1 byte over the item cap (kept), one exactly at it and a small one (one block); +# j/: four items at the item cap, exactly the block cap (one block), then three at the item cap and one of half, +# 3.5 MiB, closed before the next item, half and a byte, would take it past the block cap; +# k/: 65 small items, 64 to a block by default. +CAP_PAGES = ( + [("a/cap/huge.html", "text/html", 7, ITEM_CAP + 1), ("a/cap/limit.html", "text/html", 7, ITEM_CAP), + ("a/cap/small.html", "text/html", 7, 100)] + + [(f"j/b{n}.html", "text/html", 0, ITEM_CAP) for n in range(4)] + + [(f"j/c{n}.html", "text/html", 0, ITEM_CAP) for n in range(3)] + + [("j/c3.html", "text/html", 0, ITEM_CAP // 2), ("j/c4.html", "text/html", 0, ITEM_CAP // 2 + 1)] + + [(f"k/{n:02}.html", "text/html", 9, 200) for n in range(65)] +) + + +@pytest.fixture(scope="module") +def cap_built(tmp_path_factory): + db_path = tmp_path_factory.mktemp("cap") / "documentation.db" + conn, types, dictionary = _create(db_path) + plain = {} + for path, content_type, template_id, size in CAP_PAGES: + plain[path] = _sized(path, size) + _insert(conn, types, dictionary, path, content_type, template_id, plain[path], quality="1") + conn.commit() + conn.close() + return db_path, plain + + +@pytest.fixture +def cap_database(cap_built, tmp_path): + db_path, plain = cap_built + copy = tmp_path / "documentation.db" + shutil.copyfile(db_path, copy) + return copy, plain + + +def _run(db_path: Path, dry_run: bool = False, per_block: int = PER_BLOCK) -> int: + return spb.run({"db_path": db_path, "pages_per_block": per_block, "dry_run": dry_run, "workers": 4}) def _blocks(db_path: Path) -> list: @@ -160,6 +231,11 @@ def _blocks(db_path: Path) -> list: conn.close() +def _block_paths(db_path: Path) -> list: + """[[path, ...] by offset] in block-id order.""" + return [[path for path, _offset, _length in items] for _data, items in _blocks(db_path)] + + def _tables(db_path: Path) -> dict: """Every row of every table, by table. Content's columns: id, path, languageID, content, contentTypeID, templateId.""" @@ -172,21 +248,40 @@ def _tables(db_path: Path) -> dict: conn.close() +def _query(db_path: Path, sql: str, args=()) -> list: + conn = sqlite3.connect(db_path) + try: + return conn.execute(sql, args).fetchall() + finally: + conn.close() + + def _in_path_order(paths) -> list: """ORDER BY path in SQLite's BINARY collation: by UTF-8 bytes.""" return sorted(paths, key=lambda path: path.encode("utf-8")) +def _set_of(path: str) -> str: + return "a/devsite/" if path.startswith("a/devsite/") else path[:2] + + +def _assert_nothing_written(db_path: Path, sha: str) -> None: + assert _sha(db_path) == sha + assert not db_path.with_name(db_path.name + "-journal").exists() + assert _query(db_path, "SELECT name FROM sqlite_master WHERE name LIKE 'ContentBlock%'") == [] + + def test_blocks_follow_path_order_and_never_mix_sets(database): db_path, _plain = database assert _run(db_path) == 0 blocks = _blocks(db_path) - expected = [path for name in ("a/", "j/", "k/") for path in _in_path_order(p for p, _t, _i in BLOCKED[name])] + expected = [path for name, _condition in spb.SETS for path in _in_path_order(p for p, _t, _i in BLOCKED[name])] assert [path for _data, items in blocks for path, _offset, _length in items] == expected + assert [name for name, _condition in spb.SETS] == ["a/", "j/", "k/", "a/devsite/", "i/"] # Binary, not case-blind: a/Zeta.html sorts before a/android/..., a/androidx/B before a/androidx/a. assert expected[:3] == ["a/Zeta.html", "a/android/util/Pair.html", "a/android/widget/TextView.html"] for data, items in blocks: - assert len({path[:2] for path, _offset, _length in items}) == 1 + assert len({_set_of(path) for path, _offset, _length in items}) == 1 offset = 0 for _path, start, length in items: assert start == offset @@ -197,29 +292,43 @@ def test_blocks_follow_path_order_and_never_mix_sets(database): def test_the_last_block_of_a_set_is_shorter(database): db_path, _plain = database assert _run(db_path) == 0 - # a/ has 7 pages, j/ 4 and k/ 2: each set starts a block of its own. - assert [len(items) for _data, items in _blocks(db_path)] == [3, 3, 1, 3, 1, 2] + # a/ has 7 pages, j/ 4, k/ 2, a/devsite/ 4 and i/ 3: each set starts a block of its own. + assert [len(items) for _data, items in _blocks(db_path)] == [3, 3, 1, 3, 1, 2, 3, 1, 3] + + +def test_the_summary_names_every_set_its_content_types_and_what_it_left(database, capsys): + db_path, _plain = database + assert _run(db_path) == 0 + out = capsys.readouterr().out + for name, pages, blocks in (("a/", 7, 3), ("j/", 4, 2), ("k/", 2, 1), ("a/devsite/", 4, 2), ("i/", 3, 1)): + assert f" {name}: {pages} page(s) in {blocks} block(s);" in out + devsite = next(line for line in out.splitlines() if line.startswith(" a/devsite/: 4 page(s)")) + assert all(kind in devsite for kind in ("text/html 2", "text/css 1", "text/javascript 1")) + assert (" i/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, and " + "1 continuation row(s).") in out + assert "Stored 20 page(s) in 9 block(s) of up to 3 pages and 4,194,304 bytes decoded" in out def test_continuation_rows_are_left_alone(database): db_path, plain = database before = _tables(db_path)["Content"] assert _run(db_path) == 0 - blocked = {path for _data, items in _blocks(db_path) for path, _offset, _length in items} - assert CHUNKED not in blocked and f"{CHUNKED}-1" not in blocked + blocked = {path for paths in _block_paths(db_path) for path in paths} assert "a/z.html-1" in blocked after = _tables(db_path)["Content"] - for path in (CHUNKED, f"{CHUNKED}-1"): - assert [row for row in after if row[1] == path] == [row for row in before if row[1] == path] - conn = sqlite3.connect(db_path) - try: - first = conn.execute("SELECT content FROM Content WHERE path = ?", (CHUNKED,)).fetchone()[0] - stored = opt.reassemble(conn, CHUNKED, first) - finally: - conn.close() dictionary = db_path.with_name("chunked.dict.bin") dictionary.write_bytes(DICTIONARY) - assert _brotli(["-d", "-c"], stored, dictionary) == plain[CHUNKED] + for chunked, _type, _template in CHUNKED: + for path in (chunked, f"{chunked}-1"): + assert path not in blocked + assert [row for row in after if row[1] == path] == [row for row in before if row[1] == path] + conn = sqlite3.connect(db_path) + try: + first = conn.execute("SELECT content FROM Content WHERE path = ?", (chunked,)).fetchone()[0] + stored = opt.reassemble(conn, chunked, first) + finally: + conn.close() + assert _brotli(["-d", "-c"], stored, dictionary) == plain[chunked] def test_every_slice_equals_the_rows_decoded_bytes(database): @@ -248,7 +357,7 @@ def test_blocked_rows_are_emptied_and_nothing_else_changes(database): else: assert now == was sequence = dict(before.pop("sqlite_sequence")) - assert dict(after.pop("sqlite_sequence")) == {**sequence, "ContentBlocks": 6} + assert dict(after.pop("sqlite_sequence")) == {**sequence, "ContentBlocks": 9} assert after == before conn = sqlite3.connect(db_path) try: @@ -274,11 +383,96 @@ def test_dry_run_writes_nothing(database, capsys): db_path, _plain = database pristine = _sha(db_path) assert _run(db_path, dry_run=True) == 0 - assert "Would store 13 page(s) in 6 block(s)" in capsys.readouterr().out - assert _sha(db_path) == pristine - assert not db_path.with_name(db_path.name + "-journal").exists() - conn = sqlite3.connect(db_path) - try: - assert conn.execute("SELECT name FROM sqlite_master WHERE name LIKE 'ContentBlock%'").fetchall() == [] - finally: - conn.close() + assert "Would store 20 page(s) in 9 block(s)" in capsys.readouterr().out + _assert_nothing_written(db_path, pristine) + + +def test_an_item_over_1_mib_decoded_stays_a_row(cap_database, capsys): + db_path, plain = cap_database + before = dict(_query(db_path, "SELECT path, content FROM Content")) + assert len(before["a/cap/huge.html"]) < ITEM_CAP, "fixture premise: stored small, decoded over the cap" + assert _run(db_path, per_block=64) == 0 + blocks = _block_paths(db_path) + assert blocks[0] == ["a/cap/limit.html", "a/cap/small.html"] # exactly 1 MiB is not over it + assert "a/cap/huge.html" not in {path for paths in blocks for path in paths} + after = dict(_query(db_path, "SELECT path, content FROM Content")) + assert after["a/cap/huge.html"] == before["a/cap/huge.html"] + out = capsys.readouterr().out + assert " a/: left as rows: 1 page(s) over 1,048,576 bytes decoded" in out + assert " kept by the cap: a/cap/huge.html" in out + for data, items in _blocks(db_path): + for path, offset, length in items: + assert data[offset:offset + length] == plain[path], path + + +def test_a_block_closes_before_it_would_decode_past_4_mib(cap_database): + db_path, _plain = cap_database + assert _run(db_path, per_block=64) == 0 + blocks = _blocks(db_path) + j_blocks = [[path for path, _offset, _length in items] for _data, items in blocks + if items[0][0].startswith("j/")] + # Exactly 4 MiB is not past it; 3.5 MiB closes before half a MiB and a byte more. + assert j_blocks == [[f"j/b{n}.html" for n in range(4)], [f"j/c{n}.html" for n in range(4)], ["j/c4.html"]] + sizes = [len(data) for data, _items in blocks] + assert max(sizes) == BLOCK_CAP and all(size <= BLOCK_CAP for size in sizes) + + +def test_64_pages_to_a_block_by_default(cap_database, monkeypatch): + db_path, _plain = cap_database + monkeypatch.setattr(sys, "argv", ["store_pages_in_blocks.py", str(db_path), "--workers", "4"]) + with pytest.raises(SystemExit) as exited: + spb.main() + assert exited.value.code == 0 + k_blocks = [paths for paths in _block_paths(db_path) if paths[0].startswith("k/")] + assert [len(paths) for paths in k_blocks] == [64, 1] + + +class _Garbling(spb.DictionaryBrotli): + """The real codec, except that the `garble`-th decode of each block it encoded comes back with one byte + changed, the length kept.""" + garble = 1 + + def __init__(self, dictionary: bytes): + super().__init__(dictionary) + self._lock = threading.Lock() + self._decodes = {} + + def encode(self, data: bytes) -> bytes: + encoded = super().encode(data) + with self._lock: + self._decodes[encoded] = 0 + return encoded + + def decode(self, data: bytes) -> bytes: + plain = super().decode(data) + with self._lock: + if data not in self._decodes: + return plain + self._decodes[data] += 1 + if self._decodes[data] != self.garble: + return plain + middle = len(plain) // 2 + return plain[:middle] + bytes([plain[middle] ^ 1]) + plain[middle + 1:] + + +def test_a_block_that_does_not_decode_back_byte_for_byte_writes_nothing(database, capsys, monkeypatch): + """Garbled only on the decode-back, so the blocks stored are sound and only the byte-for-byte check before + a row is emptied can stop the run.""" + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(spb, "DictionaryBrotli", type("FirstDecode", (_Garbling,), {"garble": 1})) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert "does not read back byte for byte from the block starting at" in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) + + +def test_a_stored_block_that_does_not_read_back_writes_nothing(database, capsys, monkeypatch): + """Garbled only on the second decode, the one over the stored blocks before the commit.""" + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(spb, "DictionaryBrotli", type("SecondDecode", (_Garbling,), {"garble": 2})) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert "does not read back from block" in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) From 8650521a74415a2bd7614f0135e1589efacb2f88 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 26 Oct 2026 19:26:03 -0700 Subject: [PATCH 05/17] ADFA-6683: Declare 4.0.0, and run only on a database declaring 3 Blocks are documentation version 4.0.0, a new major: an app without the reader would show every blocked page empty, and the app reads blocks by the declared major, never by whether the tables exist. 4.0.0 promises everything 3.0.0 does as well (keyed tooltips, compressed templates). - The step reads the major from the newest DocumentationDatabaseVersion row (ORDER BY changeTime DESC, rowid DESC, as the app and remint_dictionary.declared_major do) and refuses no version, a major below 3, or a major of 4 or more, writing nothing. The refusal of a database that already has the tables stays, and comes first. - In the same transaction as the blocks it replaces that table's rows with one, 4.0.0, who and comment naming ADFA-6683, as migrate_tooltip_schema.py declares 3.0.0. A dry run rolls it back with everything else. Tests: no version table, no version row, 2.x, 4.x and 5.x each refused with nothing written; the newest row deciding (an older changeTime on a later rowid, ties broken by rowid, an older 4 under a newer 3); one 4.0.0 row after a run and the old rows after a dry run. Each fails against the script broken that way: no gate, 4.x let through, no version let through, the newest row by rowid, the greatest major, no declaration, a declaration appended instead of replacing, and a dry run that declares. Co-Authored-By: Claude Sonnet 5.5 --- scripts/store_pages_in_blocks.py | 76 ++++++++++++++++++++--- scripts/test_store_pages_in_blocks.py | 88 +++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 13 deletions(-) diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index 6404527c..ac172d54 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -10,7 +10,8 @@ store_pages_in_blocks.py Stores the pages of five documentation sets up to 64 to a compressed block, -instead of one compressed Content row each (ADFA-6683): +instead of one compressed Content row each, and declares documentation +version 4.0.0 (ADFA-6683): uv run scripts/store_pages_in_blocks.py documentation.db --dry-run uv run scripts/store_pages_in_blocks.py documentation.db @@ -30,7 +31,7 @@ and the largest block decodes to 4,172,562 bytes. With 16 workers the step took 53 seconds. -The format: +The format, documentation version 4.0.0 (CLAUDE.md, "Schema 4.0.0"): ContentBlocks(id, data) data: Brotli of the block's items, concatenated - quality 11, window @@ -45,6 +46,17 @@ lookup by path and every page's template work as before; only its content becomes an empty blob. +The version: 4.0.0 is a new major, because an app that does not read blocks +would serve every blocked page empty. The app reads blocks from a database +declaring major 4 or later, and never decides by whether the tables exist. +4.0.0 promises everything 3.0.0 does too: keyed tooltips and compressed +templates. So this step runs only on a database whose newest +DocumentationDatabaseVersion row (ORDER BY changeTime DESC, rowid DESC, as the +app and remint_dictionary.declared_major read it) declares major 3, and in the +same transaction as the blocks it replaces that table's rows with one, 4.0.0, +as migrate_tooltip_schema.py declares 3.0.0. A database declaring no version, +a major below 3, or a major of 4 or more is refused with nothing written. + Which rows: five sets, each in ORDER BY path COLLATE BINARY, in this order; a block never mixes sets. Blocks are numbered in this order as written; rebuild_database.py --pack ContentBlocks renumbers them, and nothing reads the @@ -86,13 +98,13 @@ Kotlin trim, a/ pages-to-templates, migrate_tooltip_schema.py, remint_dictionary.py --seeds 4 and verify_remint_dictionary.py, this step, then rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks ---page-size 4096. A database with blocks goes to dev-assets or Drive only -once the app's block reader is on CodeOnTheGo's stage: builds take whatever -dev-assets holds, and an app without the reader would show every blocked page -empty. +--page-size 4096. A 4.0.0 database goes to dev-assets or Drive only once the +app's block reader is on CodeOnTheGo's stage: builds take whatever dev-assets +holds, and an app without the reader would show every blocked page empty. Runs in one transaction and VACUUMs afterwards. Takes no backup: run it on a -copy. --dry-run does all the work, writes included, and rolls it back. +copy. --dry-run does all the work, writes and version included, and rolls it +back. """ from __future__ import annotations @@ -126,6 +138,13 @@ # Rows decoded ahead of the grouping walk, at most, so a set is never held decoded all at once. DECODE_WINDOW = 256 +# This step turns a 3.x database into 4.0.0, as one DocumentationDatabaseVersion row. +FROM_MAJOR, TO_MAJOR = 3, 4 +WHO = "ADFA-6683" +COMMENT = ("ADFA-6683: a/, j/, k/, a/devsite/ and i/ pages stored up to 64 to a Brotli block in ContentBlocks, each " + "located by ContentBlockItems; a blocked Content row's content is empty. Keyed tooltips and compressed " + "templates as in 3.0.0") + SCHEMA = ( """CREATE TABLE ContentBlocks ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -173,6 +192,39 @@ def table_exists(conn, name: str) -> bool: return conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone() is not None +def declared_major(conn) -> int | None: + """The major the newest DocumentationDatabaseVersion row declares, read the way the app's + DatabaseVersionResolver and remint_dictionary.declared_major read it (newest changeTime, then newest rowid); + None without one.""" + if not table_exists(conn, "DocumentationDatabaseVersion"): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion ORDER BY changeTime DESC, rowid DESC LIMIT 1" + ).fetchone() + return row[0] if row else None + + +def version_refusal(conn) -> str | None: + """Why this database's declared version keeps it from getting blocks, or None if it declares FROM_MAJOR.""" + major = declared_major(conn) + if major is None: + return (f"it declares no documentation version, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0. " + f"{TO_MAJOR}.0.0 promises everything {FROM_MAJOR}.0.0 does; run migrate_tooltip_schema.py first") + if major < FROM_MAJOR: + return (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0, " + f"which promises everything {FROM_MAJOR}.0.0 does (keyed tooltips, compressed templates); run " + "migrate_tooltip_schema.py first") + if major > FROM_MAJOR: + return (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0; " + f"from {TO_MAJOR} on, pages are already in blocks or in a format this step does not know") + return None + + +def declare_version(conn) -> None: + conn.execute("DELETE FROM DocumentationDatabaseVersion") + conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) VALUES (?, 0, 0, ?, ?)", + (TO_MAJOR, WHO, COMMENT)) + + def select_sets(conn) -> list: """[(set, [(content id, path, content type, stored bytes)] in path order, items held in more than one row, their continuation rows)] for each of SETS.""" @@ -319,10 +371,11 @@ def report(summary: list, per_block: int, dry_run: bool) -> None: if len(kept) > 10: print(f" ... and {human(len(kept) - 10)} more") saved = sum(figures["stored"] - figures["compressed"] for _name, figures, _c, _n in summary) - verb = "Would store" if dry_run else "Stored" + verb, declare = ("Would store", "would declare") if dry_run else ("Stored", "declared") print(f"{verb} {human(sum(figures['items'] for _n, figures, _c, _k in summary))} page(s) in " f"{human(sum(figures['blocks'] for _n, figures, _c, _k in summary))} block(s) of up to {per_block} pages and " - f"{human(MAX_BLOCK_BYTES)} bytes decoded, {human(saved)} bytes smaller than their rows.") + f"{human(MAX_BLOCK_BYTES)} bytes decoded, {human(saved)} bytes smaller than their rows; {declare} " + f"{TO_MAJOR}.0.0.") def run(cfg: dict) -> int: @@ -345,6 +398,10 @@ def run(cfg: dict) -> int: "rows are empty, so a second run would store empty pages over them. Start again from a " "database that has not.", file=sys.stderr) return 1 + refusal = version_refusal(conn) + if refusal: + print(f"error: {db_path}: {refusal}; nothing was written", file=sys.stderr) + return 1 dictionary = load_dictionary(conn) if not dictionary: print(f"error: {db_path} has no CompressionDictionary, which blocks are compressed against", @@ -367,6 +424,7 @@ def run(cfg: dict) -> int: figures = store_set(conn, pool, codec, rows, per_block, workers, expected) summary.append((name, figures, chunked, continuations)) check_stored(conn, codec, expected, workers) + declare_version(conn) if cfg["dry_run"]: conn.execute("ROLLBACK") else: diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py index 96681f0e..73261ad3 100644 --- a/scripts/test_store_pages_in_blocks.py +++ b/scripts/test_store_pages_in_blocks.py @@ -14,9 +14,9 @@ three pages to a block: each of the five sets in path order with its own blocks, the last one shorter, a/devsite/ and i/ rows taken whether templated or not; rows held in more than one row left alone; every item sliced from its -block equal to its row's decoded bytes; blocked rows emptied and nothing else -changed; a second run refused; a dry run, and a block that does not read back -byte for byte, writing nothing. On +block equal to its row's decoded bytes; blocked rows emptied, 4.0.0 declared +and nothing else changed; a second run, no version, 2.x and 4.x refused; a dry +run, and a block that does not read back byte for byte, writing nothing. On the second, with the default 64: an item over 1 MiB decoded kept as a row, and a block closed before it would decode past 4 MiB. @@ -91,7 +91,14 @@ UNIQUE(path) ); CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT ''); +CREATE TABLE DocumentationDatabaseVersion ( + major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL, who TEXT NOT NULL, comment TEXT NOT NULL, + changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +INSERT INTO DocumentationDatabaseVersion VALUES (2, 0, 0, 'ADFA-5220', 'dictionary', '2026-08-20 00:00:00'); +INSERT INTO DocumentationDatabaseVersion VALUES (3, 0, 0, 'ADFA-3305', 'keyed tooltips', '2026-10-20 00:00:00'); """ +VERSION_ROWS = "SELECT major, minor, patch, who, comment FROM DocumentationDatabaseVersion ORDER BY rowid" def _brotli(args: list, data: bytes, dictionary: Path) -> bytes: @@ -256,6 +263,16 @@ def _query(db_path: Path, sql: str, args=()) -> list: conn.close() +def _execute(db_path: Path, *statements: str) -> None: + conn = sqlite3.connect(db_path) + try: + for sql in statements: + conn.execute(sql) + conn.commit() + finally: + conn.close() + + def _in_path_order(paths) -> list: """ORDER BY path in SQLite's BINARY collation: by UTF-8 bytes.""" return sorted(paths, key=lambda path: path.encode("utf-8")) @@ -358,6 +375,8 @@ def test_blocked_rows_are_emptied_and_nothing_else_changes(database): assert now == was sequence = dict(before.pop("sqlite_sequence")) assert dict(after.pop("sqlite_sequence")) == {**sequence, "ContentBlocks": 9} + before.pop("DocumentationDatabaseVersion") + after.pop("DocumentationDatabaseVersion") # test_declares_4_0_0_in_one_row assert after == before conn = sqlite3.connect(db_path) try: @@ -369,6 +388,18 @@ def test_blocked_rows_are_emptied_and_nothing_else_changes(database): conn.close() +def test_declares_4_0_0_in_one_row(database): + db_path, _plain = database + assert _run(db_path) == 0 + assert _query(db_path, VERSION_ROWS) == [(4, 0, 0, "ADFA-6683", spb.COMMENT)] + assert "ADFA-6683" in spb.COMMENT + conn = sqlite3.connect(db_path) + try: + assert spb.declared_major(conn) == 4 + finally: + conn.close() + + def test_a_second_run_is_refused(database, capsys): db_path, _plain = database assert _run(db_path) == 0 @@ -382,9 +413,58 @@ def test_a_second_run_is_refused(database, capsys): def test_dry_run_writes_nothing(database, capsys): db_path, _plain = database pristine = _sha(db_path) + versions = _query(db_path, VERSION_ROWS) assert _run(db_path, dry_run=True) == 0 - assert "Would store 20 page(s) in 9 block(s)" in capsys.readouterr().out + out = capsys.readouterr().out + assert "Would store 20 page(s) in 9 block(s)" in out + assert "would declare 4.0.0" in out _assert_nothing_written(db_path, pristine) + assert _query(db_path, VERSION_ROWS) == versions + + +JAN, OCT = "2026-01-01 00:00:00", "2026-10-21 00:00:00" + + +@pytest.mark.parametrize("statements, says", [ + (["DROP TABLE DocumentationDatabaseVersion"], "declares no documentation version"), + (["DELETE FROM DocumentationDatabaseVersion"], "declares no documentation version"), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (2, 1, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 2"), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (4, 0, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 4"), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (5, 0, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 5"), +], ids=["no version table", "no version row", "2.x", "4.x", "5.x"]) +def test_a_database_not_declaring_3_is_refused_with_nothing_written(database, capsys, statements, says): + db_path, _plain = database + _execute(db_path, *statements) + sha = _sha(db_path) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert says in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) + + +@pytest.mark.parametrize("rows, major", [ + ([(3, OCT), (2, JAN)], 3), # a later rowid with an older changeTime does not decide + ([(2, OCT), (3, OCT)], 3), # a tie goes to the later rowid + ([(3, OCT), (2, OCT)], 2), + ([(4, JAN), (3, OCT)], 3), # an older row declaring 4 does not decide + ([(3, JAN), (4, OCT)], 4), +], ids=["older changeTime, later rowid", "tie, later rowid 3", "tie, later rowid 2", "older 4", "newer 4"]) +def test_the_newest_version_row_decides(database, rows, major): + """Read as the app reads it: ORDER BY changeTime DESC, rowid DESC. Each case is built so that rowid alone, + changeTime alone with the tie broken the other way, or the greatest major would read another major.""" + db_path, _plain = database + _execute(db_path, "DELETE FROM DocumentationDatabaseVersion", + *(f"INSERT INTO DocumentationDatabaseVersion VALUES ({m}, 0, 0, 'test', 'test', '{t}')" for m, t in rows)) + sha = _sha(db_path) + if major == 3: + assert _run(db_path) == 0 + assert _query(db_path, VERSION_ROWS) == [(4, 0, 0, "ADFA-6683", spb.COMMENT)] + else: + assert _run(db_path) == 1 + _assert_nothing_written(db_path, sha) def test_an_item_over_1_mib_decoded_stays_a_row(cap_database, capsys): From 17ef48aa7d05a65b2740137ce1a2337517907b38 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Mon, 26 Oct 2026 19:26:03 -0700 Subject: [PATCH 06/17] ADFA-6683: Run the blocks tests in CI; describe schema 4.0.0 in CLAUDE.md python-tests.yaml lists its test files one by one, so without this line CI never runs scripts/test_store_pages_in_blocks.py. CLAUDE.md gets a "Schema 4.0.0: content blocks, in the shipped copy only" section beside the 3.0.0 one: the two tables, the rule that a blocked Content row is empty, the five sets and the cap, the ship-time order (the step after the last re-mint and its verify, before --pack Templates,Content,TooltipTexts,ContentBlocks), which tools refuse 4.x and which run before the step because they do not know blocks, and the release order. The re-mint paragraph says it refuses 4.x, and the repository tour lists the script. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/python-tests.yaml | 4 +- CLAUDE.md | 59 ++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index c3572405..5dc702ff 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -69,7 +69,8 @@ jobs: # scripts/test_migrate_tooltip_schema.py needs only the standard library, pytest and the brotli CLI (above). # scripts/test_trim_kotlin_json.py needs only brotli and Pillow (in requirements.txt) and the brotli CLI. - - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + optimize_pdfs + Android and Java loaders' + # scripts/test_store_pages_in_blocks.py likewise (through optimize_db_media.py), and the brotli CLI. + - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + store_pages_in_blocks + optimize_pdfs + Android and Java loaders' run: | uv run --with-requirements requirements.txt --with pytest -- \ python -m pytest \ @@ -77,6 +78,7 @@ jobs: scripts/sync_kotlin_stdlib_docs \ scripts/test_migrate_tooltip_schema.py \ scripts/test_trim_kotlin_json.py \ + scripts/test_store_pages_in_blocks.py \ scripts/test_optimize_pdfs.py \ ProcessDocs/AndroidDocs/AndroidDocsToJson \ scripts/sync_java_docs \ diff --git a/CLAUDE.md b/CLAUDE.md index cd02a896..8d150118 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,6 +103,59 @@ edit that database** — it contains no part of the production Android app itsel > index would cost about the 613 pages the conversion saves), and two rows sharing an id share one > text, so deleting either key's text deletes the other's. +> ### Schema 4.0.0: content blocks, in the shipped copy only +> +> `scripts/store_pages_in_blocks.py` (ADFA-6683) stores most pages up to 64 to a compressed block and +> declares **4.0.0**: +> +> ```sql +> CREATE TABLE ContentBlocks ( +> id INTEGER PRIMARY KEY AUTOINCREMENT, +> data BLOB NOT NULL -- Brotli (q11, w24) of the block's items, concatenated, against CompressionDictionary +> ); +> CREATE TABLE ContentBlockItems ( +> contentId INTEGER NOT NULL PRIMARY KEY REFERENCES Content(id), +> blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), +> offset INTEGER NOT NULL, -- into the decoded block, in bytes +> length INTEGER NOT NULL -- in bytes +> ) WITHOUT ROWID; +> ``` +> +> An item is exactly its row's decoded bytes, before any template is applied. **A blocked `Content` +> row is empty**: it keeps `path`, `languageID`, `contentTypeID` and `templateId`, and its `content` +> is `x''`, so anything that reads `Content.content` from a 4.x file reads nothing for that page. The +> blocked rows are five sets, each in path order, never mixed in a block: `a/` templated JSON (not +> `a/devsite/`), `j/` HTML, `k/` templated JSON, and every Brotli row under `a/devsite/` and `i/`. A +> page over 1 MiB decoded, a row with continuation rows, and each continuation row stay rows, and a +> block never decodes to more than 4 MiB, so two fit the app's 8 MiB block cache. 4.0.0 promises +> everything 3.0.0 does too (keyed tooltips, compressed templates). Code On The Go reads blocks only +> from a database declaring major 4 or later, never because the tables exist; one without the reader +> would show every blocked page empty. On the 2026-10-20 ship-time build the step blocked 22,478 pages +> in 356 blocks and took the packed file's `.br` 10,368,636 bytes smaller (66,283,003 to 55,914,367). +> +> Only the copy that ships gets the step, after the last dictionary re-mint and its verify, before the +> pack. With it, the order on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, +> `migrate_tooltip_schema.py`, `remint_dictionary.py --seeds 4` and `verify_remint_dictionary.py`, +> `store_pages_in_blocks.py`, then `rebuild_database.py --pack +> Templates,Content,TooltipTexts,ContentBlocks --page-size 4096` (`ContentBlocks` in the list, so the +> blocks are packed and `ContentBlockItems.blockId` follows them). +> +> Which tools refuse 4.x, writing nothing: `store_pages_in_blocks.py` itself (it runs on a file +> declaring 3, and refuses one that already has the tables); `remint_dictionary.py` and +> `verify_remint_dictionary.py`, which recompress and compare `Content` rows and templates, not +> blocks; `populate_db.py`, `sync_javadoc_json_to_db.py`, `load_android_json_db.py` and `docdb-studio`, +> which refuse a major above 3; and `android_pages_to_templates.py` and `trim_kotlin_json.py`, which +> refuse 3 or later. The other steps that read or write `Content` (`drop_unread_rows.py`, +> `minify_text.py`, `hm_pages_to_templates.py`, the ADFA-6589 and ADFA-4195 steps, +> `svg_images_to_webp.py`, `optimize_pdfs.py`, the media steps) do not read the version and do not know +> blocks: on a 4.x file they would take every blocked page for an empty one. They all run before the +> step, and must never run after it. +> +> Release order: a 4.0.0 database goes to dev-assets or Drive only once the app's block reader is on +> CodeOnTheGo's `stage`. Builds take whatever dev-assets holds, so an early copy would give every build +> empty pages. It ships with the 3.0.0 switch or later. An app update never replaces the installed +> database, so a phone with a 3.x or 2.x file keeps the old read path. + The schema below is what `~/documentation.db` (Alex's current production copy) actually contains, as of 2026-08-05 — predating the 2.0.0 bump described above, which additionally adds the `CompressionDictionary` and `DocumentationDatabaseVersion` tables: @@ -152,7 +205,8 @@ In a database whose newest `DocumentationDatabaseVersion` row declares major 3 o (ADFA-3305). Only the ship-time copy is converted; the master database the tools here edit keeps plain templates. `remint_dictionary.py` and `verify_remint_dictionary.py` read the version the way the app does: from major 3 on they sample, recompress and verify the templates together with the -`Content` rows, and below it they leave the templates exactly as stored. +`Content` rows, and below it they leave the templates exactly as stored. From major 4 on (content +blocks, "Schema 4.0.0" above) they refuse the file, writing nothing. `remint_dictionary.py` trains a 768 KiB dictionary on up to 3,600 rows within 128 MiB by default (ADFA-3305), where the first mint keeps 256 KiB on up to 900 rows within 32 MiB. On the 2026-10-19 @@ -314,6 +368,9 @@ shape: every tool here reads and writes that, and only the shipped copy holds th `a/android/` and `a/androidx/` pages its own way. This step is the one that ships: the workflows that ran that pipeline on the master (`build-android-docs.yaml`, `build-android-docs-local.yaml`) were retired in ADFA-3305, never having run, since after it this step would convert almost nothing. +- **`scripts/store_pages_in_blocks.py`** - stores the `a/`, `j/`, `k/`, `a/devsite/` and `i/` pages up + to 64 to a compressed block and declares documentation version 4.0.0 (ADFA-6683; "Schema 4.0.0" + above). Ship-time copy only, after the last re-mint and its verify, before the pack. - **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and From 6c88cbec1d2db23e45711a0325444588010eb7dc Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 27 Oct 2026 11:09:53 -0700 Subject: [PATCH 07/17] ADFA-6683: Test the decode window, the write-ahead limit and every stored-block check Review of B03, F02-F04, F08, F09: - The decode window is read when a set is decoded, not bound as a default argument, so a test can shrink it. New tests decode ten rows four at a time (the next window only once the walk reaches it), and block the synthetic database with windows of 2 and 5, so a/'s seven pages span windows and a block straddles two. - One page to a block with 1 and 2 workers, so a/'s seven blocks pass the 2 * workers built ahead of the writes: every block is written, in path order, with consecutive ids. - check_stored now requires each block as stored to decode to exactly its items' lengths added up. New tests: a decode-back one byte longer (only the decode-back length check sees it), a stored block one byte longer (only the new check sees it), a lost block whose rows were emptied all the same (only the item count sees it), and a blocked row left full (only the "still hold their content" check sees it). Each writes nothing. - --workers below 1 is refused on an error: line; -1 raised a ValueError traceback and 0 ran as 1. - A database declaring no version, or a major below 2, is no longer told to run migrate_tooltip_schema.py, which converts 2.x only and refuses them. A 2.x database still is. Mutants, each failing the tests: the first window only, the whole set in one window, the window as a default argument, the write dropped or written last-in-first-out under the write-ahead limit, no decode-back length check, no item count, no stored-block length check, no "still hold their content" check, --workers unchecked, 0 run as 1, and the migrate_tooltip_schema.py advice for no version or for 1.x. Co-Authored-By: Claude Sonnet 5.5 --- scripts/store_pages_in_blocks.py | 54 +++++--- scripts/test_store_pages_in_blocks.py | 170 +++++++++++++++++++++++--- 2 files changed, 192 insertions(+), 32 deletions(-) diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index ac172d54..ea3ffc8a 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -81,11 +81,13 @@ the pages in blocks and their content types, the pages the cap kept as rows, and the rows left for their continuation rows. -Checks: every block is decoded again before its rows are emptied, and each -item's slice of it compared byte for byte with its row's decoded bytes. Before -the commit every item is sliced from its block as stored and compared, by -sha256, with its row's decoded bytes, and every blocked row must be empty. Any -failure rolls everything back. +Checks: every block is decoded again before its rows are emptied; it must +decode to its items' total length, and each item's slice of it is compared byte +for byte with its row's decoded bytes. Before the commit ContentBlockItems must +hold one row per blocked item, every block as stored must decode to exactly its +items' lengths added up, every item is sliced from it and compared, by sha256, +with its row's decoded bytes, and every blocked row must be empty. Any failure +rolls everything back. A database that already has ContentBlocks or ContentBlockItems is refused too: its blocked rows are empty, so a second run would store empty pages over @@ -135,11 +137,14 @@ MAX_ITEM_BYTES = 1024 * 1024 MAX_BLOCK_BYTES = 4 * 1024 * 1024 PAGES_PER_BLOCK = 64 -# Rows decoded ahead of the grouping walk, at most, so a set is never held decoded all at once. +# Rows decoded at a time ahead of the grouping walk, so a set is never held decoded all at once. Read when a set is +# decoded, not bound as a default argument, so a test can shrink it. DECODE_WINDOW = 256 # This step turns a 3.x database into 4.0.0, as one DocumentationDatabaseVersion row. FROM_MAJOR, TO_MAJOR = 3, 4 +# migrate_tooltip_schema.py turns this major, and no other, into 3.0.0. +TOOLTIP_MIGRATION_FROM_MAJOR = 2 WHO = "ADFA-6683" COMMENT = ("ADFA-6683: a/, j/, k/, a/devsite/ and i/ pages stored up to 64 to a Brotli block in ContentBlocks, each " "located by ContentBlockItems; a blocked Content row's content is empty. Keyed tooltips and compressed " @@ -207,12 +212,16 @@ def version_refusal(conn) -> str | None: """Why this database's declared version keeps it from getting blocks, or None if it declares FROM_MAJOR.""" major = declared_major(conn) if major is None: - return (f"it declares no documentation version, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0. " - f"{TO_MAJOR}.0.0 promises everything {FROM_MAJOR}.0.0 does; run migrate_tooltip_schema.py first") + return (f"it declares no documentation version (no DocumentationDatabaseVersion row), and this step turns " + f"only a database declaring {FROM_MAJOR}.x into {TO_MAJOR}.0.0. Run it on the copy that ships, once " + f"that declares {FROM_MAJOR}.0.0") if major < FROM_MAJOR: - return (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0, " - f"which promises everything {FROM_MAJOR}.0.0 does (keyed tooltips, compressed templates); run " - "migrate_tooltip_schema.py first") + refusal = (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into " + f"{TO_MAJOR}.0.0, which promises everything {FROM_MAJOR}.0.0 does (keyed tooltips, compressed " + "templates)") + if major == TOOLTIP_MIGRATION_FROM_MAJOR: + refusal += f"; run migrate_tooltip_schema.py first, which turns {major}.x into {FROM_MAJOR}.0.0" + return refusal if major > FROM_MAJOR: return (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0; " f"from {TO_MAJOR} on, pages are already in blocks or in a format this step does not know") @@ -253,8 +262,10 @@ def decode_row(codec: DictionaryBrotli, row: tuple) -> bytes: raise RuntimeError(f"{row[1]}: {exc}") from exc -def decoded_in_order(pool, codec: DictionaryBrotli, rows: list, window: int = DECODE_WINDOW): - """(row, its decoded bytes) for every row, in order, decoded in the pool at most `window` rows ahead.""" +def decoded_in_order(pool, codec: DictionaryBrotli, rows: list): + """(row, its decoded bytes) for every row, in order, decoded in the pool DECODE_WINDOW rows at a time: the next + window only once the walk has taken the last row of this one.""" + window = DECODE_WINDOW for start in range(0, len(rows), window): batch = rows[start:start + window] yield from zip(batch, pool.map(lambda row: decode_row(codec, row), batch)) @@ -332,8 +343,9 @@ def write(future): def check_stored(conn, codec: DictionaryBrotli, expected: dict, workers: int) -> None: """Slices every item from its block as stored and compares it with its - row's decoded bytes (`expected`: {content id: sha256}); every blocked row - must now be empty.""" + row's decoded bytes (`expected`: {content id: sha256}). ContentBlockItems + must hold one row per blocked item, each block must decode to exactly its + items' lengths added up, and every blocked row must now be empty.""" items = {} for content_id, block_id, offset, length in conn.execute( "SELECT contentId, blockId, offset, length FROM ContentBlockItems"): @@ -345,7 +357,12 @@ def check_stored(conn, codec: DictionaryBrotli, expected: dict, workers: int) -> def check(block): block_id, data = block plain = codec.decode(data) - for content_id, offset, length in items.get(block_id, ()): + located = items.get(block_id, ()) + total = sum(length for _content_id, _offset, length in located) + if len(plain) != total: + raise RuntimeError(f"block {block_id} decodes to {human(len(plain))} bytes, but its items add up to " + f"{human(total)}") + for content_id, offset, length in located: if hashlib.sha256(plain[offset:offset + length]).digest() != expected.get(content_id): raise RuntimeError(f"Content id {content_id} does not read back from block {block_id}") @@ -379,10 +396,13 @@ def report(summary: list, per_block: int, dry_run: bool) -> None: def run(cfg: dict) -> int: - db_path, per_block, workers = cfg["db_path"], cfg["pages_per_block"], cfg.get("workers") or 1 + db_path, per_block, workers = cfg["db_path"], cfg["pages_per_block"], cfg.get("workers", 1) if per_block < 1: print(f"error: --pages-per-block must be at least 1, not {per_block}", file=sys.stderr) return 1 + if workers < 1: + print(f"error: --workers must be at least 1, not {workers}", file=sys.stderr) + return 1 if not db_path.is_file(): print(f"error: {db_path} does not exist", file=sys.stderr) return 1 diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py index 73261ad3..a6de19c6 100644 --- a/scripts/test_store_pages_in_blocks.py +++ b/scripts/test_store_pages_in_blocks.py @@ -15,10 +15,13 @@ blocks, the last one shorter, a/devsite/ and i/ rows taken whether templated or not; rows held in more than one row left alone; every item sliced from its block equal to its row's decoded bytes; blocked rows emptied, 4.0.0 declared -and nothing else changed; a second run, no version, 2.x and 4.x refused; a dry -run, and a block that does not read back byte for byte, writing nothing. On -the second, with the default 64: an item over 1 MiB decoded kept as a row, and -a block closed before it would decode past 4 MiB. +and nothing else changed; a second run, no version, 1.x, 2.x and 4.x refused; +a set decoded in several windows, and one page to a block so that blocks are +written while more are pending, each blocked whole and in order; --workers +below 1 refused; a dry run, a block that does not read back byte for byte or +decodes longer than its items, a lost block and a blocked row left full, each +writing nothing. On the second, with the default 64: an item over 1 MiB +decoded kept as a row, and a block closed before it would decode past 4 MiB. uv run --with pytest --with-requirements requirements.txt python -m pytest -q scripts/test_store_pages_in_blocks.py """ @@ -288,11 +291,22 @@ def _assert_nothing_written(db_path: Path, sha: str) -> None: assert _query(db_path, "SELECT name FROM sqlite_master WHERE name LIKE 'ContentBlock%'") == [] +def _blocked_in_order() -> list: + """Every blocked path, set by set in SETS order, each set in path order: the order the items are written in.""" + return [path for name, _condition in spb.SETS for path in _in_path_order(p for p, _t, _i in BLOCKED[name])] + + +def _assert_every_slice_is_its_page(db_path: Path, plain: dict) -> None: + for data, items in _blocks(db_path): + for path, offset, length in items: + assert data[offset:offset + length] == plain[path], path + + def test_blocks_follow_path_order_and_never_mix_sets(database): db_path, _plain = database assert _run(db_path) == 0 blocks = _blocks(db_path) - expected = [path for name, _condition in spb.SETS for path in _in_path_order(p for p, _t, _i in BLOCKED[name])] + expected = _blocked_in_order() assert [path for _data, items in blocks for path, _offset, _length in items] == expected assert [name for name, _condition in spb.SETS] == ["a/", "j/", "k/", "a/devsite/", "i/"] # Binary, not case-blind: a/Zeta.html sorts before a/android/..., a/androidx/B before a/androidx/a. @@ -359,6 +373,64 @@ def test_every_slice_equals_the_rows_decoded_bytes(database): assert sliced == sum(len(group) for group in BLOCKED.values()) +class _RecordingPool: + """Maps in the calling thread, recording how many rows each map() is handed.""" + + def __init__(self): + self.batches = [] + + def map(self, fn, rows): + rows = list(rows) + self.batches.append(len(rows)) + return [fn(row) for row in rows] + + +class _Reversing: + """A codec whose decode reverses the bytes, so a row and its decoded bytes differ.""" + + @staticmethod + def decode(data: bytes) -> bytes: + return data[::-1] + + +def test_rows_are_decoded_a_window_at_a_time_in_order(monkeypatch): + monkeypatch.setattr(spb, "DECODE_WINDOW", 4) + rows = [(n, f"a/{n}.html", "text/html", f"row {n}".encode()) for n in range(10)] + pool = _RecordingPool() + walk = spb.decoded_in_order(pool, _Reversing(), rows) + taken = [next(walk) for _ in range(4)] + assert pool.batches == [4], "rows past the first window were decoded before the walk reached them" + taken += list(walk) + assert pool.batches == [4, 4, 2] + assert taken == [(row, row[3][::-1]) for row in rows] + + +@pytest.mark.parametrize("window", [2, 5]) +def test_a_set_spanning_several_decode_windows_is_blocked_whole(database, monkeypatch, window): + """a/ has 7 pages, so with a window of 2 or 5 it is decoded in more than one window, and a block of 3 straddles + two of them. Every page must still be blocked, in order.""" + db_path, plain = database + monkeypatch.setattr(spb, "DECODE_WINDOW", window) + assert _run(db_path) == 0 + assert [path for paths in _block_paths(db_path) for path in paths] == _blocked_in_order() + _assert_every_slice_is_its_page(db_path, plain) + + +@pytest.mark.parametrize("workers", [1, 2]) +def test_blocks_are_written_in_order_while_more_are_pending(database, capsys, workers): + """One page to a block: a/'s 7 blocks are more than the 2 * workers built ahead of the writes, so blocks are + written while later ones are still pending. Every block must be written, and in path order.""" + db_path, plain = database + assert spb.run({"db_path": db_path, "pages_per_block": 1, "dry_run": False, "workers": workers}) == 0 + expected = _blocked_in_order() + assert len(BLOCKED["a/"]) > 2 * workers, "fixture premise: a/ alone fills the write-ahead limit" + assert _block_paths(db_path) == [[path] for path in expected] + assert _query(db_path, "SELECT id FROM ContentBlocks ORDER BY id") == [(n,) for n in range(1, len(expected) + 1)] + _assert_every_slice_is_its_page(db_path, plain) + assert _query(db_path, "SELECT COUNT(*) FROM Content WHERE LENGTH(content) = 0") == [(len(expected),)] + assert f"Stored {len(expected)} page(s) in {len(expected)} block(s)" in capsys.readouterr().out + + def test_blocked_rows_are_emptied_and_nothing_else_changes(database): db_path, _plain = database before = _tables(db_path) @@ -425,23 +497,27 @@ def test_dry_run_writes_nothing(database, capsys): JAN, OCT = "2026-01-01 00:00:00", "2026-10-21 00:00:00" -@pytest.mark.parametrize("statements, says", [ - (["DROP TABLE DocumentationDatabaseVersion"], "declares no documentation version"), - (["DELETE FROM DocumentationDatabaseVersion"], "declares no documentation version"), +@pytest.mark.parametrize("statements, says, migrate", [ + (["DROP TABLE DocumentationDatabaseVersion"], "declares no documentation version", False), + (["DELETE FROM DocumentationDatabaseVersion"], "declares no documentation version", False), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (1, 0, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 1", False), ([f"INSERT INTO DocumentationDatabaseVersion VALUES (2, 1, 0, 'later', 'later', '{OCT}')"], - "declares documentation major 2"), + "declares documentation major 2", True), ([f"INSERT INTO DocumentationDatabaseVersion VALUES (4, 0, 0, 'later', 'later', '{OCT}')"], - "declares documentation major 4"), + "declares documentation major 4", False), ([f"INSERT INTO DocumentationDatabaseVersion VALUES (5, 0, 0, 'later', 'later', '{OCT}')"], - "declares documentation major 5"), -], ids=["no version table", "no version row", "2.x", "4.x", "5.x"]) -def test_a_database_not_declaring_3_is_refused_with_nothing_written(database, capsys, statements, says): + "declares documentation major 5", False), +], ids=["no version table", "no version row", "1.x", "2.x", "4.x", "5.x"]) +def test_a_database_not_declaring_3_is_refused_with_nothing_written(database, capsys, statements, says, migrate): + """Only a 2.x database is pointed at migrate_tooltip_schema.py, which converts 2.x and refuses anything else.""" db_path, _plain = database _execute(db_path, *statements) sha = _sha(db_path) assert _run(db_path) == 1 err = capsys.readouterr().err - assert says in err and "nothing was written" in err, err + assert err.startswith("error: ") and says in err and "nothing was written" in err, err + assert ("run migrate_tooltip_schema.py first" in err) == migrate, err _assert_nothing_written(db_path, sha) @@ -507,10 +583,23 @@ def test_64_pages_to_a_block_by_default(cap_database, monkeypatch): assert [len(paths) for paths in k_blocks] == [64, 1] +@pytest.mark.parametrize("workers", ["0", "-1"]) +def test_workers_below_1_is_refused_on_an_error_line(database, capsys, monkeypatch, workers): + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(sys, "argv", ["store_pages_in_blocks.py", str(db_path), "--workers", workers]) + with pytest.raises(SystemExit) as exited: + spb.main() + assert exited.value.code == 1 + assert capsys.readouterr().err == f"error: --workers must be at least 1, not {workers}\n" + _assert_nothing_written(db_path, sha) + + class _Garbling(spb.DictionaryBrotli): """The real codec, except that the `garble`-th decode of each block it encoded comes back with one byte - changed, the length kept.""" + changed, the length kept, or with `extend` with one byte more at the end, every item's slice intact.""" garble = 1 + extend = False def __init__(self, dictionary: bytes): super().__init__(dictionary) @@ -531,6 +620,8 @@ def decode(self, data: bytes) -> bytes: self._decodes[data] += 1 if self._decodes[data] != self.garble: return plain + if self.extend: + return plain + b"!" middle = len(plain) // 2 return plain[:middle] + bytes([plain[middle] ^ 1]) + plain[middle + 1:] @@ -556,3 +647,52 @@ def test_a_stored_block_that_does_not_read_back_writes_nothing(database, capsys, err = capsys.readouterr().err assert "does not read back from block" in err and "nothing was written" in err, err _assert_nothing_written(db_path, sha) + + +@pytest.mark.parametrize("garble, says", [ + (1, "decoded back to"), # the decode-back, before a row is emptied + (2, "but its items add up to"), # the decode of the stored blocks, before the commit +], ids=["decode-back", "stored"]) +def test_a_block_that_decodes_longer_than_its_items_writes_nothing(database, capsys, monkeypatch, garble, says): + """One byte more at the end leaves every item's slice intact, so only a length check can stop the run.""" + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(spb, "DictionaryBrotli", type("Longer", (_Garbling,), {"garble": garble, "extend": True})) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert says in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) + + +def _lose_the_last_block(conn) -> None: + """A block and its items gone, as a lost write would leave them: their rows are empty all the same.""" + block_id = conn.execute("SELECT MAX(id) FROM ContentBlocks").fetchone()[0] + conn.execute("DELETE FROM ContentBlockItems WHERE blockId = ?", (block_id,)) + conn.execute("DELETE FROM ContentBlocks WHERE id = ?", (block_id,)) + + +def _refill_a_blocked_row(conn) -> None: + """A blocked row whose content was never emptied.""" + conn.execute("UPDATE Content SET content = x'00' WHERE id = (SELECT MIN(contentId) FROM ContentBlockItems)") + + +@pytest.mark.parametrize("damage, says", [ + (_lose_the_last_block, "ContentBlockItems does not hold one row per blocked item"), + (_refill_a_blocked_row, "1 blocked row(s) still hold their content"), +], ids=["a block lost", "a row not emptied"]) +def test_the_check_before_the_commit_refuses_what_the_writes_got_wrong(database, capsys, monkeypatch, damage, says): + """Each damage leaves every stored block decoding to its items, so only the check that names it can stop the + run.""" + db_path, _plain = database + sha = _sha(db_path) + check_stored = spb.check_stored + + def damaged(conn, *args): + damage(conn) + return check_stored(conn, *args) + + monkeypatch.setattr(spb, "check_stored", damaged) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert says in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) From 7d2424a18170b892d98d079501f3cf4c9ccb5a35 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 27 Oct 2026 11:10:12 -0700 Subject: [PATCH 08/17] ADFA-6683: Re-mint pair refuses block tables whatever the version; record B02 Review of B03, F10, F01 and F06 (the re-mint's part): - remint_dictionary.py and verify_remint_dictionary.py refused a blocked database only by its declared major. A 4.x file whose version row a later step or a hand edit rewrote to 3 (or dropped) would be re-minted, leaving every block undecodable, and the verifier compares no blocks to notice. Both now also refuse a database holding ContentBlocks or ContentBlockItems, whatever it declares, writing nothing. - B02 has run since 670095fb said it had not: re-minting after the blocks step, on the blocks, saved at most 30,880 bytes of .br (25,285 more on the mean of 4 seeds), well inside the docstring's 0.33 MB seed spread, so "re-mint before" stands. The docstring records the measurement, and that whole blocks as zstd samples lose 0.52 to 0.74 MB because zstd trains on a sample's first 128 KiB only. - The ship-time order in the refusal message ends with --page-size 4096, as the pack is run. Tests: the re-mint refuses a 3.x database with both tables, with ContentBlockItems alone, and a database with no version and ContentBlocks, before training and with nothing written; the verifier refuses either side holding ContentBlocks; the command line refuses a 3.x database with both tables, with no backup. The order check includes --page-size 4096. Mutants, each failing the tests: no table check, ContentBlocks only, the table check only with a version declared, the verifier checking the new file only, and the order without --page-size. Co-Authored-By: Claude Sonnet 5.5 --- .../remint_dictionary.py | 48 ++++++++++---- .../test_remint_dictionary.py | 65 ++++++++++++++++++- .../verify_remint_dictionary.py | 3 +- 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py index 14098ff2..799f43aa 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py @@ -111,12 +111,23 @@ stores most pages in ContentBlocks, compressed against the same dictionary, and leaves their Content rows empty. This re-mint recompresses Content rows and templates, not blocks, so it refuses a database whose newest -DocumentationDatabaseVersion row declares major 4 or later, before training -and with nothing written, backup included; the verifier refuses one too. On -the copy that ships, the re-mint and its verify go before the blocks step: +DocumentationDatabaseVersion row declares major 4 or later, or that has +ContentBlocks or ContentBlockItems whatever it declares, before training and +with nothing written, backup included; the verifier refuses one too. On the +copy that ships, the re-mint and its verify go before the blocks step: remint_dictionary.py --seeds 4, verify_remint_dictionary.py, store_pages_in_blocks.py, then rebuild_database.py --pack -Templates,Content,TooltipTexts,ContentBlocks. +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096. + +Re-minting after the blocks step instead, on the blocks, was measured and does +not pay (ADFA-6683's B02, 2026-10-26, on the 2026-10-23 3.0.0 ship-time +database, 4 seeds each, packed as above, .br with brotli -q 11 -w 24). A dictionary +trained on the decoded blocks, cut into 128 KiB pieces, with the rows left and +the templates, stored a .br at most 30,880 bytes smaller than this re-mint run +before the blocks step, and 25,285 bytes larger on the mean of the 4 seeds: +far inside the 0.33 MB seed spread above. Whole blocks as samples did worse, +0.52 to 0.74 MB larger, because zstd trains on the first 128 KiB of a sample +only. Every row is proved to round-trip against the new dictionary before it is written. The backup (VACUUM INTO, unless --no-backup) is taken only then, just @@ -168,9 +179,13 @@ # From this documentation major on, most pages are stored in ContentBlocks (ADFA-6683), which neither this re-mint # nor its verifier handles, so both refuse such a database. CONTENT_BLOCKS_FROM_MAJOR = 4 +# The tables scripts/store_pages_in_blocks.py adds. Both tools refuse a database holding either one, whatever +# version it declares: a version row rewritten after the blocks step would not move the pages back into Content. +CONTENT_BLOCK_TABLES = ("ContentBlocks", "ContentBlockItems") # Where the re-mint and its verify go on the copy that ships, relative to the blocks step and the pack. SHIP_TIME_ORDER = ("remint_dictionary.py --seeds 4, verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, " - "then scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks") + "then scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks " + "--page-size 4096") def declared_major(conn) -> int | None: @@ -192,14 +207,23 @@ def templates_compressed(conn) -> bool: def content_blocks_refusal(conn, handles: str) -> str | None: """Why the re-mint or its verifier (`handles`: what it does, as "This re-mint recompresses") must leave this database alone, or None: from CONTENT_BLOCKS_FROM_MAJOR on, most pages are in ContentBlocks, which neither - handles.""" + handles, and a database holding any of CONTENT_BLOCK_TABLES may have pages there whatever it declares.""" major = declared_major(conn) - if major is None or major < CONTENT_BLOCKS_FROM_MAJOR: - return None - return (f"the database declares documentation major {major}, and from {CONTENT_BLOCKS_FROM_MAJOR}.0.0 (ADFA-6683) " - "most pages are stored in ContentBlocks, compressed against the dictionary, with their Content rows " - f"empty. {handles} Content rows and templates, not blocks. Re-mint before " - f"scripts/store_pages_in_blocks.py, never after it: on the copy that ships the order is {SHIP_TIME_ORDER}") + if major is not None and major >= CONTENT_BLOCKS_FROM_MAJOR: + why = (f"the database declares documentation major {major}, and from {CONTENT_BLOCKS_FROM_MAJOR}.0.0 " + "(ADFA-6683) most pages are stored in ContentBlocks") + else: + tables = [name for name in CONTENT_BLOCK_TABLES if conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone()] + if not tables: + return None + declared = "no documentation version" if major is None else f"documentation major {major}" + why = (f"the database has {' and '.join(tables)}, the tables scripts/store_pages_in_blocks.py (ADFA-6683) " + f"adds, though it declares {declared}, so its pages may be stored in ContentBlocks whatever its " + "version row now says") + return (f"{why}, compressed against the dictionary, with their Content rows empty. {handles} Content rows and " + "templates, not blocks. Re-mint before scripts/store_pages_in_blocks.py, never after it: on the copy " + f"that ships the order is {SHIP_TIME_ORDER}") def load_templates(conn, old_dictionary: bytes) -> list: diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py index ad4b972d..305084a3 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py @@ -39,6 +39,13 @@ CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE(name)); """ TEMPLATE_NAMES = ("layout.pebble", "page.peb", "nav.peb") +# The tables scripts/store_pages_in_blocks.py adds (ADFA-6683), as it creates them. +BLOCK_TABLES = { + "ContentBlocks": "CREATE TABLE ContentBlocks (id INTEGER PRIMARY KEY AUTOINCREMENT, data BLOB NOT NULL)", + "ContentBlockItems": ("CREATE TABLE ContentBlockItems (contentId INTEGER NOT NULL PRIMARY KEY REFERENCES " + "Content(id), blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), offset INTEGER NOT " + "NULL, length INTEGER NOT NULL) WITHOUT ROWID"), +} HERE = Path(__file__).resolve().parent # The dictionary the fixture is first minted with. Small on purpose, so a 16 KiB re-mint beats it by hundreds of @@ -729,7 +736,8 @@ def test_verifier_catches_a_changed_missing_stale_or_extra_template(self): class ContentBlocksRefusalTest(_RemintFixture): """From documentation version 4.0.0 on (ADFA-6683) most pages are in ContentBlocks, compressed against the dictionary, which neither the re-mint nor its verifier handles: both refuse a database declaring major 4 or later, - as the app reads its version, write nothing, and say the re-mint goes before the blocks step.""" + as the app reads its version, or holding the blocks step's tables whatever it declares, write nothing, and say the + re-mint goes before the blocks step.""" def remint(self, **kwargs): return remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20, seed=999, **kwargs) @@ -737,7 +745,8 @@ def remint(self, **kwargs): def assertNamesTheBlocksStepAndTheOrder(self, message: str): self.assertIn("Re-mint before scripts/store_pages_in_blocks.py", message) self.assertIn("verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, then " - "scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks", message) + "scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks --page-size 4096", + message) def test_the_remint_refuses_major_4_or_later_before_training(self): for majors in ((3, 4), (5,)): @@ -785,6 +794,51 @@ def test_the_verifier_refuses_either_database_declaring_major_4(self): self.assertIn("This check compares Content rows and templates, not blocks", message) self.assertNamesTheBlocksStepAndTheOrder(message) + def add_block_tables(self, tables=BLOCK_TABLES): + self.conn.executescript("".join(BLOCK_TABLES[name] + ";" for name in tables)) + self.conn.commit() + + def test_the_remint_refuses_block_tables_whatever_the_database_declares(self): + """A version row rewritten to 3 after the blocks step, or dropped, leaves the pages in blocks all the same.""" + for majors, tables, declares in (((3,), tuple(BLOCK_TABLES), "documentation major 3"), + ((3,), ("ContentBlockItems",), "documentation major 3"), + (None, ("ContentBlocks",), "no documentation version")): + with self.subTest(majors=majors, tables=tables): + self.tearDown() + self.setUp() + if majors is not None: + self.add_versions_and_templates(majors=majors) + self.add_block_tables(tables) + content = dict(self.conn.execute("SELECT path, content FROM Content")) + templates = self.stored_templates() if majors is not None else None + with mock.patch.object(remint_dictionary, "train_dictionary", + side_effect=AssertionError("trained on a database with blocks")): + with self.assertRaises(RuntimeError) as raised: + self.remint() + message = str(raised.exception) + self.assertTrue(message.startswith(f"the database has {' and '.join(tables)}, the tables " + "scripts/store_pages_in_blocks.py (ADFA-6683) adds, though it " + f"declares {declares}"), message) + self.assertIn("nothing written", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway") + self.assertEqual(dict(self.conn.execute("SELECT path, content FROM Content")), content) + if templates is not None: + self.assertEqual(self.stored_templates(), templates) + + def test_the_verifier_refuses_either_database_with_block_tables(self): + self.add_versions_and_templates(majors=(3,)) + original = self.snapshot() + self.add_block_tables(("ContentBlocks",)) + for old, new in ((original, self.db_path), (self.db_path, original)): + with self.subTest(old=old.name, new=new.name): + with self.assertRaises(RuntimeError) as raised: + verify(old, new) + message = str(raised.exception) + self.assertTrue(message.startswith(f"{self.db_path}: the database has ContentBlocks"), message) + self.assertIn("This check compares Content rows and templates, not blocks", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + class CommandLineRefusalTest(_RemintFixture): """A run that refuses writes nothing at all, the backup included, and says why on one "error:" line rather @@ -844,6 +898,13 @@ def test_a_database_with_blocks_refuses_with_no_backup(self): sha = self.sha() self.assertRefusedWithNothingWritten(self.run_remint(), sha, "Re-mint before scripts/store_pages_in_blocks.py") + def test_a_3_x_database_with_block_tables_refuses_with_no_backup(self): + self.add_versions_and_templates(majors=(3,)) + self.conn.executescript("".join(sql + ";" for sql in BLOCK_TABLES.values())) + sha = self.sha() + self.assertRefusedWithNothingWritten(self.run_remint(), sha, "the database has ContentBlocks and " + "ContentBlockItems") + def test_the_verifier_reports_a_database_with_blocks_on_an_error_line(self): self.add_versions_and_templates(majors=(3, 4)) original = self.dir / "original.db" diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py index 87e466f6..ac4422c0 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py @@ -32,7 +32,8 @@ Reads only: neither database is modified. A database it cannot check (no CompressionDictionary row) is reported on an "error:" line, exit 1. So is a pair where either database declares documentation major 4 or later -(ADFA-6683): most of its pages are in ContentBlocks, which this check does not +(ADFA-6683), or has ContentBlocks or ContentBlockItems whatever it declares: +most of its pages are, or may be, in ContentBlocks, which this check does not compare, and which remint_dictionary.py refuses to re-mint. Re-mint and verify before scripts/store_pages_in_blocks.py. """ From e2133381ffc968abf8e4312176ffdd14de617107 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Tue, 27 Oct 2026 11:12:09 -0700 Subject: [PATCH 09/17] ADFA-6683: Correct CLAUDE.md on 4.x tools, ship-time packs and the reader; record B02 Review of B03, F01 and F05-F07, F11: - B02 has run: re-minting after the blocks step, on the blocks, saved at most 30,880 bytes of .br (the mean of 4 seeds 25,285 larger), against 0.33 MB of seed spread, so the re-mint stays before the step. Both the 4.0.0 section and the re-mint paragraph say so, and the re-mint paragraph says the pair also refuses the block tables. - The other Content tools on a 4.x file: drop_unread_rows.py does read the version (3 or more means compressed templates) and does not refuse 4. None of them sees a blocked page as empty: x'' is not a Brotli stream ("corrupt input"), so each refuses, writes what it can read and exits 1, or stops with a traceback, as listed. migrate_tooltip_schema.py (nothing to do) and the master-only writers migrate_content_to_dictionary_brotli.py, insert_optimized_media.py and sync_kdoc_json_to_db.py (which would overwrite blocked k/ rows) are added. - Every ship-time order ends with its pack and --page-size 4096: the 3.0.0 section and the migrate_tooltip_schema.py entry with --pack Templates,Content,TooltipTexts, as compress_docdb.yml runs it (a 3.0.0 file has no ContentBlocks for the 4.0.0 pack to name), plus the 4.0.0 pack with ContentBlocks; the store_pages_in_blocks.py entry with the 4.0.0 pack. - The reader is ADFA-6683's CodeOnTheGo PR to stage, not yet numbered or merged, and a file declaring major 5 would be read as 4. - python-tests.yaml: the comment no longer names the brotli CLI twice. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/python-tests.yaml | 2 +- CLAUDE.md | 71 +++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index 5dc702ff..dcdd6cfc 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -69,7 +69,7 @@ jobs: # scripts/test_migrate_tooltip_schema.py needs only the standard library, pytest and the brotli CLI (above). # scripts/test_trim_kotlin_json.py needs only brotli and Pillow (in requirements.txt) and the brotli CLI. - # scripts/test_store_pages_in_blocks.py likewise (through optimize_db_media.py), and the brotli CLI. + # scripts/test_store_pages_in_blocks.py likewise, brotli and Pillow through optimize_db_media.py. - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + store_pages_in_blocks + optimize_pdfs + Android and Java loaders' run: | uv run --with-requirements requirements.txt --with pytest -- \ diff --git a/CLAUDE.md b/CLAUDE.md index 8d150118..7a51d98a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,10 @@ edit that database** — it contains no part of the production Android app itsel > (the minify and the `a/` step do), so there every one of them runs before the migration. The order > on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, > `migrate_tooltip_schema.py`, PR #48's `remint_dictionary.py --seeds 4` (which recompresses -> templates too in 3.0.0), then `rebuild_database.py --pack Content,TooltipTexts`; +> templates too in 3.0.0) and its verify, then `rebuild_database.py --pack +> Templates,Content,TooltipTexts --page-size 4096`, as CodeOnTheGo's `compress_docdb.yml` packs it +> (for 4.0.0, `store_pages_in_blocks.py` goes between the verify and the pack, which then ends with +> `--pack Templates,Content,TooltipTexts,ContentBlocks --page-size 4096`: "Schema 4.0.0" below); > `hm_pages_to_templates.py` goes after the `i/` page steps it lists, before the migration like > every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the > re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0. @@ -128,28 +131,54 @@ edit that database** — it contains no part of the production Android app itsel > `a/devsite/`), `j/` HTML, `k/` templated JSON, and every Brotli row under `a/devsite/` and `i/`. A > page over 1 MiB decoded, a row with continuation rows, and each continuation row stay rows, and a > block never decodes to more than 4 MiB, so two fit the app's 8 MiB block cache. 4.0.0 promises -> everything 3.0.0 does too (keyed tooltips, compressed templates). Code On The Go reads blocks only -> from a database declaring major 4 or later, never because the tables exist; one without the reader -> would show every blocked page empty. On the 2026-10-20 ship-time build the step blocked 22,478 pages -> in 356 blocks and took the packed file's `.br` 10,368,636 bytes smaller (66,283,003 to 55,914,367). +> everything 3.0.0 does too (keyed tooltips, compressed templates). The app's block reader is +> ADFA-6683's CodeOnTheGo PR to `stage` (B04 in the ticket's plan), not yet numbered or merged. It +> reads blocks only from a database declaring major 4 or later, never because the tables exist, so a +> file declaring major 5 would be read as 4 (the gate is `>=`, as the 3.x gates are). An app without +> the reader would show every blocked page empty. On the 2026-10-20 ship-time build the step blocked +> 22,478 pages in 356 blocks and took the packed file's `.br` 10,368,636 bytes smaller (66,283,003 to +> 55,914,367). > > Only the copy that ships gets the step, after the last dictionary re-mint and its verify, before the > pack. With it, the order on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, > `migrate_tooltip_schema.py`, `remint_dictionary.py --seeds 4` and `verify_remint_dictionary.py`, > `store_pages_in_blocks.py`, then `rebuild_database.py --pack > Templates,Content,TooltipTexts,ContentBlocks --page-size 4096` (`ContentBlocks` in the list, so the -> blocks are packed and `ContentBlockItems.blockId` follows them). +> blocks are packed and `ContentBlockItems.blockId` follows them). The re-mint stays before the step: +> re-minting after it, on the blocks, was measured (ADFA-6683's B02, 2026-10-26, on the 2026-10-23 +> 3.0.0 ship-time database) and saved at most 30,880 bytes of `.br` (the mean of 4 seeds was 25,285 +> bytes larger), against a 0.33 MB spread between re-mint seeds; `remint_dictionary.py`'s docstring +> has the figures. > > Which tools refuse 4.x, writing nothing: `store_pages_in_blocks.py` itself (it runs on a file > declaring 3, and refuses one that already has the tables); `remint_dictionary.py` and > `verify_remint_dictionary.py`, which recompress and compare `Content` rows and templates, not -> blocks; `populate_db.py`, `sync_javadoc_json_to_db.py`, `load_android_json_db.py` and `docdb-studio`, -> which refuse a major above 3; and `android_pages_to_templates.py` and `trim_kotlin_json.py`, which -> refuse 3 or later. The other steps that read or write `Content` (`drop_unread_rows.py`, -> `minify_text.py`, `hm_pages_to_templates.py`, the ADFA-6589 and ADFA-4195 steps, -> `svg_images_to_webp.py`, `optimize_pdfs.py`, the media steps) do not read the version and do not know -> blocks: on a 4.x file they would take every blocked page for an empty one. They all run before the -> step, and must never run after it. +> blocks (they also refuse a file holding `ContentBlocks` or `ContentBlockItems`, whatever it +> declares); `populate_db.py`, `sync_javadoc_json_to_db.py`, `load_android_json_db.py` and +> `docdb-studio`, which refuse a major above 3; and `android_pages_to_templates.py` and +> `trim_kotlin_json.py`, which refuse 3 or later. +> +> No other step that reads or writes `Content` knows blocks or refuses 4.x. Two read the version: +> `drop_unread_rows.py`, only to know that templates are compressed from 3 on, and +> `migrate_tooltip_schema.py`. None of them sees a blocked page as an empty one: its `x''` is not a +> Brotli stream (`brotli -d` reports "corrupt input"), so decoding it fails. What each does then: +> +> - refuses, writing nothing: `drop_unread_rows.py` (it reports the blocked rows it reads as +> unreadable), `minify_text.py` (on the blocked stylesheets), `svg_images_to_webp.py`, and +> `optimize_pdfs.py` if a PDF were blocked; +> - writes what it can read, then exits 1 on `error:` lines: `bundle_remote_images.py`, +> `remove_remote_loads.py`, `remove_skin_font_comment.py`, `share_skin_sprite.py`, +> `optimize_db_media.py`, `update_media_references.py` and `migrate_content_to_dictionary_brotli.py`; +> - stops with a traceback before reaching a block, as on any 3.x file: `hm_pages_to_templates.py` +> (on its compressed template) and `check_remote_loads.py` (on the 3.x tooltip tables); +> `insert_optimized_media.py` stops with one on the first blocked row; +> - `migrate_tooltip_schema.py` leaves the file alone ("nothing to do", exit 0), and +> `sync_kdoc_json_to_db.py`, which never decodes `Content`, would overwrite or delete blocked `k/` +> rows and leave their `ContentBlockItems` pointing at the old page. +> +> The ship-time steps all run before the blocks step and must never run after it; +> `migrate_content_to_dictionary_brotli.py`, `insert_optimized_media.py` and `sync_kdoc_json_to_db.py` +> write the master only, which never gets blocks. > > Release order: a 4.0.0 database goes to dev-assets or Drive only once the app's block reader is on > CodeOnTheGo's `stage`. Builds take whatever dev-assets holds, so an early copy would give every build @@ -206,7 +235,10 @@ In a database whose newest `DocumentationDatabaseVersion` row declares major 3 o plain templates. `remint_dictionary.py` and `verify_remint_dictionary.py` read the version the way the app does: from major 3 on they sample, recompress and verify the templates together with the `Content` rows, and below it they leave the templates exactly as stored. From major 4 on (content -blocks, "Schema 4.0.0" above) they refuse the file, writing nothing. +blocks, "Schema 4.0.0" above) they refuse the file, writing nothing, and so they do a file holding +`ContentBlocks` or `ContentBlockItems` whatever it declares. Re-minting on the blocks instead was +measured (ADFA-6683's B02) and does not pay: at most 30,880 bytes of `.br`, against 0.33 MB of seed +noise. `remint_dictionary.py` trains a 768 KiB dictionary on up to 3,600 rows within 128 MiB by default (ADFA-3305), where the first mint keeps 256 KiB on up to 900 rows within 32 MiB. On the 2026-10-19 @@ -298,9 +330,11 @@ shape: every tool here reads and writes that, and only the shipped copy holds th converts to documentation schema 3.0.0, which Code On The Go reads from 3.0.0 on: the WITHOUT ROWID, lookup-table tooltip shape, and every template Brotli-compressed against the dictionary; run it on the copy that ships, after every step that writes tooltips or templates, since they all write the old - shape and plain templates, never on the master they edit; re-mint afterwards, then pack - `Content,TooltipTexts`, not `Tooltips`), `minify_text.py` (ADFA-6627: minifies the - HTML, CSS, JS and SVG rows and the HTML templates without changing what a page shows; run + shape and plain templates, never on the master they edit; re-mint afterwards, then + `rebuild_database.py --pack Templates,Content,TooltipTexts --page-size 4096` (not `Tooltips`); + for 4.0.0, `store_pages_in_blocks.py` goes before the pack, which then ends with `--pack + Templates,Content,TooltipTexts,ContentBlocks --page-size 4096`), `minify_text.py` (ADFA-6627: + minifies the HTML, CSS, JS and SVG rows and the HTML templates without changing what a page shows; run `ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py` and `verify_remint_dictionary.py` straight after it, or the minified rows compress worse than before), `svg_images_to_webp.py` (ADFA-3305: @@ -370,7 +404,8 @@ shape: every tool here reads and writes that, and only the shipped copy holds th were retired in ADFA-3305, never having run, since after it this step would convert almost nothing. - **`scripts/store_pages_in_blocks.py`** - stores the `a/`, `j/`, `k/`, `a/devsite/` and `i/` pages up to 64 to a compressed block and declares documentation version 4.0.0 (ADFA-6683; "Schema 4.0.0" - above). Ship-time copy only, after the last re-mint and its verify, before the pack. + above). Ship-time copy only, after the last re-mint and its verify, then `rebuild_database.py --pack + Templates,Content,TooltipTexts,ContentBlocks --page-size 4096`. - **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and From 9313890eb6de460c4d114802a190d7c77fff3bcb Mon Sep 17 00:00:00 2001 From: David Schachter Date: Wed, 28 Oct 2026 17:12:17 -0700 Subject: [PATCH 10/17] ADFA-6683 | Say j/ takes templated javadoc JSON rows too; test one Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAaJxvRXKRYkgiDqwGQ --- scripts/store_pages_in_blocks.py | 7 ++++++- scripts/test_store_pages_in_blocks.py | 16 ++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index ea3ffc8a..1006069c 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -63,11 +63,16 @@ order. Paths are matched with GLOB, which is case-sensitive as paths are: 1. a/ templated JSON: path GLOB 'a/*' AND NOT GLOB 'a/devsite/*' AND templateId > 0 - 2. j/ HTML: path GLOB 'j/*' AND content type text/html + 2. j/ HTML: path GLOB 'j/*' AND content type text/html, templated or not 3. k/ templated JSON: path GLOB 'k/*' AND templateId > 0 4. a/devsite/: every row under it 5. i/: every row under it +j/ takes every text/html row, including templated JSON rows: +sync_javadoc_json_to_db.py keeps contentTypeID as text/html and sets templateId +to the javadoc.peb id (its docstring, lines 7-8; the UPDATE at line 375 and the +INSERT at line 398), so there is no templateId condition here. + Only rows whose content type is compressed with brotli, and only items held in one row: a row that owns continuation rows ("path-1", ... after a 1 MiB row), and each continuation row, stay as they are. diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py index a6de19c6..ec5b080c 100644 --- a/scripts/test_store_pages_in_blocks.py +++ b/scripts/test_store_pages_in_blocks.py @@ -57,7 +57,9 @@ ("a/Zeta.html", "text/html", 7), ("a/androidx/B.html", "text/html", 7), ("a/android/util/Pair.html", "text/html", 7)], "j/": [("j/index.html", "text/html", 0), ("j/api/b.html", "text/html", 0), - ("j/Overview.html", "text/html", 0), ("j/api/A.html", "text/html", 0)], + ("j/Overview.html", "text/html", 0), ("j/api/A.html", "text/html", 0), + # A templated javadoc JSON row: sync_javadoc_json_to_db.py keeps text/html and sets templateId. + ("j/api/T.html", "text/html", 7)], "k/": [("k/html/y.html", "text/html", 9), ("k/html/x.html", "text/html", 8)], # Every Brotli row under a/devsite/ and i/, templated or not, HTML or not. "a/devsite/": [("a/devsite/reference/x.html", "text/html", 7), ("a/devsite/b.html", "text/html", 0), @@ -309,6 +311,8 @@ def test_blocks_follow_path_order_and_never_mix_sets(database): expected = _blocked_in_order() assert [path for _data, items in blocks for path, _offset, _length in items] == expected assert [name for name, _condition in spb.SETS] == ["a/", "j/", "k/", "a/devsite/", "i/"] + # j/ takes text/html whatever its templateId: the javadoc JSON rows are templated. + assert "j/api/T.html" in expected # Binary, not case-blind: a/Zeta.html sorts before a/android/..., a/androidx/B before a/androidx/a. assert expected[:3] == ["a/Zeta.html", "a/android/util/Pair.html", "a/android/widget/TextView.html"] for data, items in blocks: @@ -323,21 +327,21 @@ def test_blocks_follow_path_order_and_never_mix_sets(database): def test_the_last_block_of_a_set_is_shorter(database): db_path, _plain = database assert _run(db_path) == 0 - # a/ has 7 pages, j/ 4, k/ 2, a/devsite/ 4 and i/ 3: each set starts a block of its own. - assert [len(items) for _data, items in _blocks(db_path)] == [3, 3, 1, 3, 1, 2, 3, 1, 3] + # a/ has 7 pages, j/ 5, k/ 2, a/devsite/ 4 and i/ 3: each set starts a block of its own. + assert [len(items) for _data, items in _blocks(db_path)] == [3, 3, 1, 3, 2, 2, 3, 1, 3] def test_the_summary_names_every_set_its_content_types_and_what_it_left(database, capsys): db_path, _plain = database assert _run(db_path) == 0 out = capsys.readouterr().out - for name, pages, blocks in (("a/", 7, 3), ("j/", 4, 2), ("k/", 2, 1), ("a/devsite/", 4, 2), ("i/", 3, 1)): + for name, pages, blocks in (("a/", 7, 3), ("j/", 5, 2), ("k/", 2, 1), ("a/devsite/", 4, 2), ("i/", 3, 1)): assert f" {name}: {pages} page(s) in {blocks} block(s);" in out devsite = next(line for line in out.splitlines() if line.startswith(" a/devsite/: 4 page(s)")) assert all(kind in devsite for kind in ("text/html 2", "text/css 1", "text/javascript 1")) assert (" i/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, and " "1 continuation row(s).") in out - assert "Stored 20 page(s) in 9 block(s) of up to 3 pages and 4,194,304 bytes decoded" in out + assert "Stored 21 page(s) in 9 block(s) of up to 3 pages and 4,194,304 bytes decoded" in out def test_continuation_rows_are_left_alone(database): @@ -488,7 +492,7 @@ def test_dry_run_writes_nothing(database, capsys): versions = _query(db_path, VERSION_ROWS) assert _run(db_path, dry_run=True) == 0 out = capsys.readouterr().out - assert "Would store 20 page(s) in 9 block(s)" in out + assert "Would store 21 page(s) in 9 block(s)" in out assert "would declare 4.0.0" in out _assert_nothing_written(db_path, pristine) assert _query(db_path, VERSION_ROWS) == versions From 5a541687f1fe763a584c8827976a6b2cf10bf929 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Wed, 28 Oct 2026 17:15:49 -0700 Subject: [PATCH 11/17] ADFA-6683 | CLAUDE.md: the Drive master will be 4.0.0, so the ship-time writers run on blocks The ADFA-6806-tracked tools (sync_kdoc_json_to_db.py, insert_optimized_media.py, migrate_content_to_dictionary_brotli.py) are no longer said to write a master that never gets blocks. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAaJxvRXKRYkgiDqwGQ --- CLAUDE.md | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7a51d98a..6b1b7137 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,14 +99,15 @@ edit that database** — it contains no part of the production Android app itsel > `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`), > `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`) and `TooltipManager.py`. Until they > catch up the step runs on the copy that ships, after all of them, and never on the master -> database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH. +> database they edit; nothing converts 3.0 back. (The Drive master will itself become 4.0.0, which +> includes 3.0.0: "Schema 4.0.0" below.) It needs the `brotli` CLI on PATH. > > In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the > id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)` > index would cost about the 613 pages the conversion saves), and two rows sharing an id share one > text, so deleting either key's text deletes the other's. -> ### Schema 4.0.0: content blocks, in the shipped copy only +> ### Schema 4.0.0: content blocks > > `scripts/store_pages_in_blocks.py` (ADFA-6683) stores most pages up to 64 to a compressed block and > declares **4.0.0**: @@ -150,6 +151,12 @@ edit that database** — it contains no part of the production Android app itsel > bytes larger), against a 0.33 MB spread between re-mint seeds; `remint_dictionary.py`'s docstring > has the figures. > +> **The Google Drive master database the documentation team edits will become 4.0.0 and will never go +> back to 3.x.** The blocks step itself still runs on the copy that ships, at build time, but from the +> upgrade on every tool below that does not refuse 4.x runs on a blocked database. None of them may be +> left as it is when the Drive copy is upgraded: each must be fixed or guarded first, as ADFA-6806 (the +> docs pipeline scripts), ADFA-6807 (DocDB Studio) and ADFA-6808 (check-tools) track. +> > Which tools refuse 4.x, writing nothing: `store_pages_in_blocks.py` itself (it runs on a file > declaring 3, and refuses one that already has the tables); `remint_dictionary.py` and > `verify_remint_dictionary.py`, which recompress and compare `Content` rows and templates, not @@ -172,16 +179,23 @@ edit that database** — it contains no part of the production Android app itsel > - stops with a traceback before reaching a block, as on any 3.x file: `hm_pages_to_templates.py` > (on its compressed template) and `check_remote_loads.py` (on the 3.x tooltip tables); > `insert_optimized_media.py` stops with one on the first blocked row; -> - `migrate_tooltip_schema.py` leaves the file alone ("nothing to do", exit 0), and -> `sync_kdoc_json_to_db.py`, which never decodes `Content`, would overwrite or delete blocked `k/` -> rows and leave their `ContentBlockItems` pointing at the old page. +> - `migrate_tooltip_schema.py` leaves the file alone ("nothing to do", exit 0); +> - `sync_kdoc_json_to_db.py` has no version check and never decodes `Content`, so it writes into a +> blocked database as if it were 3.x, and silently (code, not run): `write_content` updates a `k/` +> page that still fits in one row in place (`UPDATE Content SET content = ?`, `:147`), leaving a blocked +> row holding its own content beside its `ContentBlockItems` row, a state the blocks step never writes +> (it checks that every blocked row is empty), and a page that has grown past one row is deleted and +> re-inserted under a new id (`:152-160`), which orphans its `ContentBlockItems` row; > -> The ship-time steps all run before the blocks step and must never run after it; -> `migrate_content_to_dictionary_brotli.py`, `insert_optimized_media.py` and `sync_kdoc_json_to_db.py` -> write the master only, which never gets blocks. +> The ship-time steps all run before the blocks step on the copy that ships, and must never run after it +> there. They also run on the Drive master, which is 4.0.0 from the upgrade, and these three do so +> without refusing: `migrate_content_to_dictionary_brotli.py`, `insert_optimized_media.py` and +> `sync_kdoc_json_to_db.py`, with the results above. They must be fixed or guarded (ADFA-6806) before the +> Drive copy is upgraded. > > Release order: a 4.0.0 database goes to dev-assets or Drive only once the app's block reader is on -> CodeOnTheGo's `stage`. Builds take whatever dev-assets holds, so an early copy would give every build +> CodeOnTheGo's `stage`, and the Drive master is upgraded only after ADFA-6806, ADFA-6807 and ADFA-6808 +> are done. Builds take whatever dev-assets holds, so an early copy would give every build > empty pages. It ships with the 3.0.0 switch or later. An app update never replaces the installed > database, so a phone with a 3.x or 2.x file keeps the old read path. @@ -231,7 +245,7 @@ a placeholder. In a database whose newest `DocumentationDatabaseVersion` row declares major 3 or later, every `Templates` row is stored Brotli-compressed against the shared dictionary, with no per-row marker -(ADFA-3305). Only the ship-time copy is converted; the master database the tools here edit keeps +(ADFA-3305). Until the Drive master is upgraded to 4.0.0 ("Schema 4.0.0" above), only the ship-time copy is converted; the master database the tools here edit keeps plain templates. `remint_dictionary.py` and `verify_remint_dictionary.py` read the version the way the app does: from major 3 on they sample, recompress and verify the templates together with the `Content` rows, and below it they leave the templates exactly as stored. From major 4 on (content From 7b92606f072083b1bbf33bde0506408914aa7f78 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Wed, 28 Oct 2026 17:15:49 -0700 Subject: [PATCH 12/17] ADFA-6683 | CLAUDE.md: list the tools a 4.0.0 database breaks, with what each does Run on a 4.0.0 test copy or read from the code, as marked; ADFA-6806, ADFA-6807, ADFA-6808. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAaJxvRXKRYkgiDqwGQ --- CLAUDE.md | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6b1b7137..36d90921 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -168,24 +168,55 @@ edit that database** — it contains no part of the production Android app itsel > No other step that reads or writes `Content` knows blocks or refuses 4.x. Two read the version: > `drop_unread_rows.py`, only to know that templates are compressed from 3 on, and > `migrate_tooltip_schema.py`. None of them sees a blocked page as an empty one: its `x''` is not a -> Brotli stream (`brotli -d` reports "corrupt input"), so decoding it fails. What each does then: +> Brotli stream (`brotli -d` reports "corrupt input"), so decoding it fails. What each does then +> (marked "run" where it was run on a small 4.0.0 test copy made with `store_pages_in_blocks.py +> --pages-per-block 4`, "code" where it is read from the code): > > - refuses, writing nothing: `drop_unread_rows.py` (it reports the blocked rows it reads as > unreadable), `minify_text.py` (on the blocked stylesheets), `svg_images_to_webp.py`, and > `optimize_pdfs.py` if a PDF were blocked; > - writes what it can read, then exits 1 on `error:` lines: `bundle_remote_images.py`, > `remove_remote_loads.py`, `remove_skin_font_comment.py`, `share_skin_sprite.py`, -> `optimize_db_media.py`, `update_media_references.py` and `migrate_content_to_dictionary_brotli.py`; +> `update_media_references.py` (code), `optimize_db_media.py` (run: it commits, then prints "failed to +> optimize" for each blocked SVG and exits 1) and `migrate_content_to_dictionary_brotli.py` (run: an +> `error:` line per blocked row and exit 1; it commits in batches, code); > - stops with a traceback before reaching a block, as on any 3.x file: `hm_pages_to_templates.py` > (on its compressed template) and `check_remote_loads.py` (on the 3.x tooltip tables); -> `insert_optimized_media.py` stops with one on the first blocked row; -> - `migrate_tooltip_schema.py` leaves the file alone ("nothing to do", exit 0); +> - stops with a traceback on the first blocked row: `insert_optimized_media.py` (run: it writes its +> backup, then `rewrite_pages` fails decoding the first blocked page, `insert_optimized_media.py:289`), +> `verify_android_json_db.py` (run: the first blocked `a/` page it samples, decoded with +> `populate_db.py:399` after the read at `verify_android_json_db.py:107`) and +> `set_content_type_compression.py` (run: after its backup, the first blocked row of the type, decoded +> at `:118-121`); `compare_svg_renders.py` fails the same way in `svg_rows` (`:74-76`; code, not run); +> - exits 0 with a wrong answer: `list_database_documents.py` lists a blocked page as +> "Bytes: 0" (run; `:17-20`), and `migrate_tooltip_schema.py` leaves the file alone ("nothing to do"); +> - refuses a table it does not know, writing nothing: `DocumentationDatabase.py` and `ingest.py` +> (run: `ValueError: Database schema contains unexpected tables`, `DocumentationDatabase.py:92-95`; its +> allowed list is only `Content`, `Languages`, `ContentTypes` and `ide_tooltip_table`, so it refuses any +> 2.x or 3.x `documentation.db` too, not only a 4.x one); +> - fails on the 3.x tooltip tables whatever the blocks, so these cannot work on the master after 3.0.0 +> either (ADFA-6808 for `check-tools`): `layout_xml_tooltips.py` (inserts `Tooltips(summary, detail)`, +> `:92-97`) and `TooltipManager.py` (`:52`) read or write columns the 3.0 `Tooltips` table no longer +> has (code; the same SQL on a 3.0.0 test copy gives "no such column: summary"), and so does +> `check-tools/` (`db_health_checker.py` selects `t.summary` at `:130`). `check-tools` also has a +> check of its own that a blocked database fails: the empty-content query at +> `db_health_checker.py:266-269` counts every blocked row (22 of 22 on the test copy) as empty; > - `sync_kdoc_json_to_db.py` has no version check and never decodes `Content`, so it writes into a > blocked database as if it were 3.x, and silently (code, not run): `write_content` updates a `k/` > page that still fits in one row in place (`UPDATE Content SET content = ?`, `:147`), leaving a blocked > row holding its own content beside its `ContentBlockItems` row, a state the blocks step never writes > (it checks that every blocked row is empty), and a page that has grown past one row is deleted and > re-inserted under a new id (`:152-160`), which orphans its `ContentBlockItems` row; +> - works on 4.x: `rebuild_database.py --pack Templates,Content,ContentBlocks` (run: it renumbers the +> blocks, follows `ContentBlockItems.blockId` and `contentId`, and keeps the declared 4.0.0). +> +> `docdb-studio` (ADFA-6807) refuses a major above 3 (`SUPPORTED_DATABASE_MAJOR_VERSION = 3`, +> `docdb_studio.py:191`, checked at `:5220-5221`), and nothing in it mentions `ContentBlocks` or +> `ContentBlockItems`: reading a page (`fetch_content_for_path`, `get_html_anchors_for_path`), listing +> sizes (`get_paths_with_sizes`, `LENGTH(content)`), deleting rows (`_delete_content_by_ids`) and the +> import's orphan pass all treat `Content.content` as the page, so a blocked page would read as empty +> and a delete or overwrite would orphan its `ContentBlockItems` row. DocDB Studio has to gain block +> support before the Drive copy is upgraded, or the team cannot edit the master. > > The ship-time steps all run before the blocks step on the copy that ships, and must never run after it > there. They also run on the Drive master, which is 4.0.0 from the upgrade, and these three do so From aa4f43f8659f438532a1352ca0badfb856090813 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Wed, 28 Oct 2026 17:15:49 -0700 Subject: [PATCH 13/17] ADFA-6683 | CLAUDE.md: drop_unread_rows.py exits 0 with nothing to drop, and refuses only on a blocked row Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAaJxvRXKRYkgiDqwGQ --- CLAUDE.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 36d90921..a140d1b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -172,9 +172,12 @@ edit that database** — it contains no part of the production Android app itsel > (marked "run" where it was run on a small 4.0.0 test copy made with `store_pages_in_blocks.py > --pages-per-block 4`, "code" where it is read from the code): > -> - refuses, writing nothing: `drop_unread_rows.py` (it reports the blocked rows it reads as -> unreadable), `minify_text.py` (on the blocked stylesheets), `svg_images_to_webp.py`, and -> `optimize_pdfs.py` if a PDF were blocked; +> - refuses, writing nothing: `minify_text.py` (on the blocked stylesheets; run), +> `svg_images_to_webp.py` (on a blocked SVG; run), `optimize_pdfs.py` if a PDF were blocked (code), +> and `drop_unread_rows.py` only when it has to read a blocked row (run): given nothing to drop it +> exits 0 with "Nothing to do" without reading one, and given something to drop it reports every +> blocked row it must read (the rows to drop, then the kept rows' references) as unreadable and exits +> 1, changing nothing; > - writes what it can read, then exits 1 on `error:` lines: `bundle_remote_images.py`, > `remove_remote_loads.py`, `remove_skin_font_comment.py`, `share_skin_sprite.py`, > `update_media_references.py` (code), `optimize_db_media.py` (run: it commits, then prints "failed to From 9c094465d9568d707ff0673b22c8773d20ba1017 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Fri, 30 Oct 2026 22:40:41 -0700 Subject: [PATCH 14/17] ADFA-6683 | verify_remint_dictionary.py: close both databases when a pair is refused verify() raised its 4.x refusal with both read-only connections still open, so each refused pair left two handles for the collector (and, on Windows, two files that cannot be deleted). Both now close however it returns. A new test fails without the change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYjFcUx64N8PMRknzL --- .../test_remint_dictionary.py | 23 ++++++++ .../verify_remint_dictionary.py | 54 ++++++++++--------- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py index 305084a3..d676f81e 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py @@ -839,6 +839,29 @@ def test_the_verifier_refuses_either_database_with_block_tables(self): self.assertIn("This check compares Content rows and templates, not blocks", message) self.assertNamesTheBlocksStepAndTheOrder(message) + def test_the_verifier_closes_both_databases_when_it_refuses(self): + """Otherwise each refused pair leaves two read-only handles open until they are collected, and on Windows + neither file can be deleted meanwhile.""" + self.add_versions_and_templates(majors=(3,)) + original = self.snapshot() + self.add_block_tables(("ContentBlocks",)) + opened = [] + connect = sqlite3.connect + + def recording(*args, **kwargs): + opened.append(connect(*args, **kwargs)) + return opened[-1] + + for old, new in ((original, self.db_path), (self.db_path, original)): + with self.subTest(old=old.name, new=new.name): + opened.clear() + with mock.patch.object(sqlite3, "connect", recording), self.assertRaises(RuntimeError): + verify(old, new) + self.assertEqual(len(opened), 2) + for conn in opened: + with self.assertRaises(sqlite3.ProgrammingError): + conn.execute("SELECT 1") + class CommandLineRefusalTest(_RemintFixture): """A run that refuses writes nothing at all, the backup included, and says why on one "error:" line rather diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py index ac4422c0..428ca985 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py @@ -41,6 +41,7 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor +from contextlib import closing from pathlib import Path import brotli @@ -145,32 +146,33 @@ def verify(old_db: Path, new_db: Path, max_workers: int | None = None) -> list: """Every item and template that differs, as (label, problem). Raises RuntimeError for a database it cannot check.""" import sqlite3 - old_conn = sqlite3.connect(f"file:{old_db}?mode=ro", uri=True) - new_conn = sqlite3.connect(f"file:{new_db}?mode=ro", uri=True) - for path, conn in ((old_db, old_conn), (new_db, new_conn)): - refusal = content_blocks_refusal(conn, "This check compares") - if refusal: - raise RuntimeError(f"{path}: {refusal}") - old_dictionary = dictionary_of(old_conn, old_db) - new_dictionary = dictionary_of(new_conn, new_db) - if old_dictionary == new_dictionary: - print("warning: both databases carry the same dictionary; nothing was re-minted", file=sys.stderr) - - items = [row[0] for row in load_base_rows(new_conn)] - print(f"Verifying {len(items):,} item(s)...", file=sys.stderr) - - bad = [] - with ThreadPoolExecutor(max_workers=max_workers) as executor: - for i, (path, problem) in enumerate( - zip(items, executor.map( - lambda p: check(old_dictionary, new_dictionary, old_db, new_db, p), items)), start=1 - ): - if problem: - bad.append((path, problem)) - if i % 6000 == 0: - print(f" {i:,}/{len(items):,} checked, {len(bad)} mismatched", file=sys.stderr) - print(f"Byte-identical: {len(items) - len(bad):,}/{len(items):,}") - return bad + verify_templates(old_conn, new_conn, old_dictionary, new_dictionary) + # Closed however this returns: a refusal raises, and each refused pair would otherwise keep two handles open. + with closing(sqlite3.connect(f"file:{old_db}?mode=ro", uri=True)) as old_conn, \ + closing(sqlite3.connect(f"file:{new_db}?mode=ro", uri=True)) as new_conn: + for path, conn in ((old_db, old_conn), (new_db, new_conn)): + refusal = content_blocks_refusal(conn, "This check compares") + if refusal: + raise RuntimeError(f"{path}: {refusal}") + old_dictionary = dictionary_of(old_conn, old_db) + new_dictionary = dictionary_of(new_conn, new_db) + if old_dictionary == new_dictionary: + print("warning: both databases carry the same dictionary; nothing was re-minted", file=sys.stderr) + + items = [row[0] for row in load_base_rows(new_conn)] + print(f"Verifying {len(items):,} item(s)...", file=sys.stderr) + + bad = [] + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for i, (path, problem) in enumerate( + zip(items, executor.map( + lambda p: check(old_dictionary, new_dictionary, old_db, new_db, p), items)), start=1 + ): + if problem: + bad.append((path, problem)) + if i % 6000 == 0: + print(f" {i:,}/{len(items):,} checked, {len(bad)} mismatched", file=sys.stderr) + print(f"Byte-identical: {len(items) - len(bad):,}/{len(items):,}") + return bad + verify_templates(old_conn, new_conn, old_dictionary, new_dictionary) def main() -> None: From 7a3d63ba59a53410a11e313e9dc0af310364d1b3 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Fri, 30 Oct 2026 22:40:41 -0700 Subject: [PATCH 15/17] ADFA-6683 | store_pages_in_blocks.py: check the writes against what was built, and leave an empty row as it is - The check before the commit counted ContentBlockItems rows but sliced only the blocks it found, so a block lost with its items left behind passed, and those pages would ship empty. It now requires ContentBlocks to hold exactly the blocks written (by sha256) and ContentBlockItems exactly the items written (block, offset, length). That also ends the second q11 decode of every block, and blocks are read from the cursor rather than all at once: build_block has already decoded each block back and matched every item. - An empty Brotli row stopped the whole run ("corrupt input"). It is now left as a row and counted in the summary. - The codec is optimize_db_media.BrotliCodec, which now passes -q 11 -w 24 explicitly (the CLI's defaults for stdin: the same bytes). declared_major and table_exists move to optimize_db_media, and declare_version is migrate_tooltip_schema's, now taking the version. - The docstring says the Drive master stays 3.x until ADFA-6871, as CLAUDE.md now does. Tests: a lost block with its items kept, a changed block, a moved item, an empty row, and each block decoded once; the first four fail without the change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYjFcUx64N8PMRknzL --- scripts/migrate_tooltip_schema.py | 6 +- scripts/optimize_db_media.py | 19 ++- scripts/store_pages_in_blocks.py | 224 +++++++++++--------------- scripts/test_store_pages_in_blocks.py | 115 ++++++++----- 4 files changed, 193 insertions(+), 171 deletions(-) diff --git a/scripts/migrate_tooltip_schema.py b/scripts/migrate_tooltip_schema.py index 5ee066c6..e13436aa 100644 --- a/scripts/migrate_tooltip_schema.py +++ b/scripts/migrate_tooltip_schema.py @@ -606,10 +606,12 @@ def convert(conn) -> list: return dropped -def declare_version(conn) -> None: +def declare_version(conn, major: int = TO_MAJOR, who: str = WHO, comment: str = COMMENT) -> None: + """Replaces every DocumentationDatabaseVersion row with one declaring major.0.0. scripts/store_pages_in_blocks.py + declares 4.0.0 with it.""" conn.execute("DELETE FROM DocumentationDatabaseVersion") conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) VALUES (?, 0, 0, ?, ?)", - (TO_MAJOR, WHO, COMMENT)) + (major, who, comment)) def run(cfg: dict) -> int: diff --git a/scripts/optimize_db_media.py b/scripts/optimize_db_media.py index 757c561b..bd78845e 100644 --- a/scripts/optimize_db_media.py +++ b/scripts/optimize_db_media.py @@ -1279,7 +1279,9 @@ def compress(self, data: bytes, compression: str) -> bytes: return data if self._dictionary is None: return brotli.compress(data) - return self._run(data=data) + # The CLI's own defaults for stdin, written out: the same bytes, and the q11 w24 that rows and + # store_pages_in_blocks.py's blocks are documented as. + return self._run("-q", "11", "-w", "24", data=data) def close(self) -> None: if self._work_dir is not None: @@ -1300,6 +1302,21 @@ def load_dictionary(conn) -> bytes | None: return row[0] if row else None +def table_exists(conn, name: str) -> bool: + return conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone() is not None + + +def declared_major(conn) -> int | None: + """The documentation major the database declares - its newest DocumentationDatabaseVersion row, ordered as + Code On The Go's DatabaseVersionResolver orders them (newest changeTime, then newest rowid) - or None if it + declares none.""" + if not table_exists(conn, "DocumentationDatabaseVersion"): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion ORDER BY changeTime DESC, rowid DESC LIMIT 1" + ).fetchone() + return None if row is None else int(row[0]) + + def human(n: int) -> str: return f"{n:,}" diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index 1006069c..0b0b7755 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -75,7 +75,8 @@ Only rows whose content type is compressed with brotli, and only items held in one row: a row that owns continuation rows ("path-1", ... after a 1 MiB row), -and each continuation row, stay as they are. +and each continuation row, stay as they are. So does an empty row, which has +nothing to decode. The cap: an item over MAX_ITEM_BYTES (1 MiB) decoded stays a row. A block closes at --pages-per-block items (default 64), or before the next item would @@ -84,30 +85,36 @@ Without the cap, j/'s Javadoc index pages (up to 5,065,005 bytes each) made a 64-page block decode to 28,055,018 bytes. The summary counts, for each set, the pages in blocks and their content types, the pages the cap kept as rows, -and the rows left for their continuation rows. +the rows left for their continuation rows, and the empty rows. Checks: every block is decoded again before its rows are emptied; it must decode to its items' total length, and each item's slice of it is compared byte -for byte with its row's decoded bytes. Before the commit ContentBlockItems must -hold one row per blocked item, every block as stored must decode to exactly its -items' lengths added up, every item is sliced from it and compared, by sha256, -with its row's decoded bytes, and every blocked row must be empty. Any failure -rolls everything back. +for byte with its row's decoded bytes. Before the commit ContentBlocks must +hold exactly the blocks written, each the same bytes (by sha256) as the block +that was decoded back, ContentBlockItems exactly the items written, each with +its block, offset and length, and every blocked row must be empty. A block +stored as it was built decodes as it did, so it is not decoded twice. Any +failure rolls everything back. A database that already has ContentBlocks or ContentBlockItems is refused too: its blocked rows are empty, so a second run would store empty pages over the blocked ones. -Where it goes: on the copy that ships only, never the master, after the last -dictionary re-mint and its verify (remint_dictionary.py and -verify_remint_dictionary.py refuse a 4.x database: they do not recompress -blocks) and before the pack. On that copy the order is: minify, row drops, -Kotlin trim, a/ pages-to-templates, migrate_tooltip_schema.py, -remint_dictionary.py --seeds 4 and verify_remint_dictionary.py, this step, then -rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks ---page-size 4096. A 4.0.0 database goes to dev-assets or Drive only once the -app's block reader is on CodeOnTheGo's stage: builds take whatever dev-assets -holds, and an app without the reader would show every blocked page empty. +Where it goes: on the copy that ships, after the last dictionary re-mint and +its verify (remint_dictionary.py and verify_remint_dictionary.py refuse a 4.x +database: they do not recompress blocks) and before the pack. On that copy the +order is: minify, row drops, Kotlin trim, a/ pages-to-templates, +migrate_tooltip_schema.py, remint_dictionary.py --seeds 4 and +verify_remint_dictionary.py, this step, then rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096. That order needs +a 3.x copy, so the Drive master it is made from stays 3.x until ADFA-6871. The +master is to become 4.0.0 too (CLAUDE.md, "Schema 4.0.0"); from then on the +build first turns the copy's blocks back into rows (ADFA-6871), so the same +order runs and this step blocks every page again, those added or edited on the +master since included. A 4.0.0 database goes to dev-assets or Drive only once +the app's block reader is on CodeOnTheGo's stage: builds take whatever +dev-assets holds, and an app without the reader would show every blocked page +empty. Runs in one transaction and VACUUMs afterwards. Takes no backup: run it on a copy. --dry-run does all the work, writes and version included, and rolls it @@ -117,16 +124,22 @@ import argparse import hashlib -import shutil import sqlite3 -import subprocess import sys -import tempfile from collections import Counter, deque from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from optimize_db_media import human, is_continuation_path, load_dictionary, owned_fragment_paths +from migrate_tooltip_schema import declare_version +from optimize_db_media import ( + BrotliCodec, + declared_major, + human, + is_continuation_path, + load_dictionary, + owned_fragment_paths, + table_exists, +) # (set, condition on Content AS c joined to ContentTypes AS t), in block-id order. GLOB, not LIKE: LIKE ignores case. SETS = ( @@ -169,50 +182,6 @@ ) -class DictionaryBrotli: - """The `brotli` CLI against one dictionary. No Python binding takes a - custom dictionary, so every tool here shells out the same way.""" - - def __init__(self, dictionary: bytes): - self._brotli = shutil.which("brotli") - if self._brotli is None: - raise RuntimeError("the `brotli` CLI is not on PATH; install it and retry") - self._work_dir = Path(tempfile.mkdtemp(prefix="store_pages_in_blocks_")) - self._dictionary = self._work_dir / "dict.bin" - self._dictionary.write_bytes(dictionary) - - def _run(self, args: list, data: bytes) -> bytes: - result = subprocess.run([self._brotli, *args, "-D", str(self._dictionary)], input=data, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) - if result.returncode != 0: - raise RuntimeError(f"brotli failed: {result.stderr.decode(errors='replace').strip()}") - return result.stdout - - def encode(self, data: bytes) -> bytes: - return self._run(["-c", "-q", "11", "-w", "24"], data) - - def decode(self, data: bytes) -> bytes: - return self._run(["-d", "-c"], data) - - def close(self) -> None: - shutil.rmtree(self._work_dir, ignore_errors=True) - - -def table_exists(conn, name: str) -> bool: - return conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone() is not None - - -def declared_major(conn) -> int | None: - """The major the newest DocumentationDatabaseVersion row declares, read the way the app's - DatabaseVersionResolver and remint_dictionary.declared_major read it (newest changeTime, then newest rowid); - None without one.""" - if not table_exists(conn, "DocumentationDatabaseVersion"): - return None - row = conn.execute("SELECT major FROM DocumentationDatabaseVersion ORDER BY changeTime DESC, rowid DESC LIMIT 1" - ).fetchone() - return row[0] if row else None - - def version_refusal(conn) -> str | None: """Why this database's declared version keeps it from getting blocks, or None if it declares FROM_MAJOR.""" major = declared_major(conn) @@ -233,41 +202,38 @@ def version_refusal(conn) -> str | None: return None -def declare_version(conn) -> None: - conn.execute("DELETE FROM DocumentationDatabaseVersion") - conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) VALUES (?, 0, 0, ?, ?)", - (TO_MAJOR, WHO, COMMENT)) - - def select_sets(conn) -> list: """[(set, [(content id, path, content type, stored bytes)] in path order, items held in more than one row, - their continuation rows)] for each of SETS.""" + their continuation rows, empty rows)] for each of SETS.""" lengths = dict(conn.execute("SELECT path, LENGTH(content) FROM Content")) selected = [] for name, condition in SETS: - rows, chunked, continuations = [], 0, 0 + rows, chunked, continuations, empty = [], 0, 0, 0 for content_id, path, content_type, stored in conn.execute( "SELECT c.id, c.path, t.value, c.content FROM Content AS c JOIN ContentTypes AS t " f"ON t.id = c.contentTypeID WHERE t.compression = 'brotli' AND {condition} " "ORDER BY c.path COLLATE BINARY"): - if is_continuation_path(lengths, path): + if not stored: + # Not a Brotli stream: `brotli -d` calls it corrupt input, and there is no page to block. + empty += 1 + elif is_continuation_path(lengths, path): continuations += 1 elif owned_fragment_paths(conn, path, len(stored)): chunked += 1 else: rows.append((content_id, path, content_type, stored)) - selected.append((name, rows, chunked, continuations)) + selected.append((name, rows, chunked, continuations, empty)) return selected -def decode_row(codec: DictionaryBrotli, row: tuple) -> bytes: +def decode_row(codec: BrotliCodec, row: tuple) -> bytes: try: - return codec.decode(row[3]) + return codec.decompress(row[3], "brotli") except RuntimeError as exc: raise RuntimeError(f"{row[1]}: {exc}") from exc -def decoded_in_order(pool, codec: DictionaryBrotli, rows: list): +def decoded_in_order(pool, codec: BrotliCodec, rows: list): """(row, its decoded bytes) for every row, in order, decoded in the pool DECODE_WINDOW rows at a time: the next window only once the walk has taken the last row of this one.""" window = DECODE_WINDOW @@ -294,13 +260,13 @@ def group_items(decoded, per_block: int, kept: list): yield group -def build_block(codec: DictionaryBrotli, group: list) -> tuple: - """(compressed block, decoded length, [(content id, offset, length, sha256)]) for one block's (row, decoded - bytes), once the block has been decoded back and each item's slice of it matched its row's decoded bytes byte - for byte.""" +def build_block(codec: BrotliCodec, group: list) -> tuple: + """(compressed block, decoded length, [(content id, offset, length)]) for one block's (row, decoded bytes), + once the block has been decoded back and each item's slice of it matched its row's decoded bytes byte for + byte.""" plain = b"".join(raw for _row, raw in group) - data = codec.encode(plain) - back = codec.decode(data) + data = codec.compress(plain, "brotli") + back = codec.decompress(data, "brotli") first = group[0][0][1] if len(back) != len(plain): raise RuntimeError(f"the block starting at {first} decoded back to {human(len(back))} bytes, not " @@ -309,27 +275,30 @@ def build_block(codec: DictionaryBrotli, group: list) -> tuple: for (content_id, path, _type, _stored), raw in group: if view[offset:offset + len(raw)] != raw: raise RuntimeError(f"{path} does not read back byte for byte from the block starting at {first}") - items.append((content_id, offset, len(raw), hashlib.sha256(raw).digest())) + items.append((content_id, offset, len(raw))) offset += len(raw) return data, len(plain), items -def store_set(conn, pool, codec: DictionaryBrotli, rows: list, per_block: int, workers: int, expected: dict) -> dict: - """Stores one set's rows in blocks, written in order as they are built, a few blocks ahead at most. Adds each - blocked item's sha256 to `expected` ({content id: sha256}); returns the set's figures.""" +def store_set(conn, pool, codec: BrotliCodec, rows: list, per_block: int, workers: int, written: tuple) -> dict: + """Stores one set's rows in blocks, written in order as they are built, a few blocks ahead at most. Adds what + it writes to `written`, ({block id: sha256 of its data}, {content id: (block id, offset, length)}); returns + the set's figures.""" stored_length = {row[0]: len(row[3]) for row in rows} content_type = {row[0]: row[2] for row in rows} figures = {"blocks": 0, "items": 0, "stored": 0, "compressed": 0, "decoded": 0, "types": Counter(), "kept": []} + written_blocks, written_items = written pending = deque() def write(future): data, plain_length, items = future.result() block_id = conn.execute("INSERT INTO ContentBlocks (data) VALUES (?)", (data,)).lastrowid - for content_id, offset, length, digest in items: + written_blocks[block_id] = hashlib.sha256(data).digest() + for content_id, offset, length in items: conn.execute("INSERT INTO ContentBlockItems (contentId, blockId, offset, length) VALUES (?, ?, ?, ?)", (content_id, block_id, offset, length)) conn.execute("UPDATE Content SET content = x'' WHERE id = ?", (content_id,)) - expected[content_id] = digest + written_items[content_id] = (block_id, offset, length) figures["stored"] += stored_length[content_id] figures["types"][content_type[content_id]] += 1 figures["blocks"] += 1 @@ -346,33 +315,28 @@ def write(future): return figures -def check_stored(conn, codec: DictionaryBrotli, expected: dict, workers: int) -> None: - """Slices every item from its block as stored and compares it with its - row's decoded bytes (`expected`: {content id: sha256}). ContentBlockItems - must hold one row per blocked item, each block must decode to exactly its - items' lengths added up, and every blocked row must now be empty.""" - items = {} - for content_id, block_id, offset, length in conn.execute( - "SELECT contentId, blockId, offset, length FROM ContentBlockItems"): - items.setdefault(block_id, []).append((content_id, offset, length)) - if sum(len(group) for group in items.values()) != len(expected): - raise RuntimeError("ContentBlockItems does not hold one row per blocked item") - blocks = conn.execute("SELECT id, data FROM ContentBlocks ORDER BY id").fetchall() - - def check(block): - block_id, data = block - plain = codec.decode(data) - located = items.get(block_id, ()) - total = sum(length for _content_id, _offset, length in located) - if len(plain) != total: - raise RuntimeError(f"block {block_id} decodes to {human(len(plain))} bytes, but its items add up to " - f"{human(total)}") - for content_id, offset, length in located: - if hashlib.sha256(plain[offset:offset + length]).digest() != expected.get(content_id): - raise RuntimeError(f"Content id {content_id} does not read back from block {block_id}") - - with ThreadPoolExecutor(max_workers=workers) as pool: - list(pool.map(check, blocks)) +def check_stored(conn, written: tuple) -> None: + """Compares the tables with what was written (`written`, as store_set fills it): ContentBlocks must hold + exactly the blocks written, each the same bytes by sha256, ContentBlockItems exactly the items written, each + with its block, offset and length, and every blocked row must now be empty. build_block decoded each block + back and matched every item's slice of it, so a block stored as built needs no second decode.""" + written_blocks, written_items = written + items = {content_id: (block_id, offset, length) for content_id, block_id, offset, length in conn.execute( + "SELECT contentId, blockId, offset, length FROM ContentBlockItems")} + wrong = sorted(content_id for content_id in items.keys() | written_items.keys() + if items.get(content_id) != written_items.get(content_id)) + if wrong: + raise RuntimeError(f"ContentBlockItems does not hold {human(len(wrong))} item(s) as written, Content id " + f"{wrong[0]} first") + stored = set() + # One block at a time from the cursor, so the blocks are never all in memory at once. + for block_id, data in conn.execute("SELECT id, data FROM ContentBlocks"): + if hashlib.sha256(data).digest() != written_blocks.get(block_id): + raise RuntimeError(f"block {block_id} as stored is not the block written") + stored.add(block_id) + lost = sorted(written_blocks.keys() - stored) + if lost: + raise RuntimeError(f"{human(len(lost))} block(s) written are not in ContentBlocks, block {lost[0]} first") left = conn.execute("SELECT COUNT(*) FROM Content WHERE LENGTH(content) > 0 AND id IN " "(SELECT contentId FROM ContentBlockItems)").fetchone()[0] if left: @@ -380,22 +344,24 @@ def check(block): def report(summary: list, per_block: int, dry_run: bool) -> None: - for name, figures, chunked, continuations in summary: + for name, figures, chunked, continuations, empty in summary: types = ", ".join(f"{value} {human(count)}" for value, count in figures["types"].most_common()) print(f" {name}: {human(figures['items'])} page(s) in {human(figures['blocks'])} block(s); their rows held " f"{human(figures['stored'])} bytes, the blocks hold {human(figures['compressed'])} " f"({human(figures['decoded'])} decoded). Content types: {types or 'none'}.") kept = figures["kept"] print(f" {name}: left as rows: {human(len(kept))} page(s) over {human(MAX_ITEM_BYTES)} bytes decoded, " - f"{human(chunked)} page(s) held in more than one row, and {human(continuations)} continuation row(s).") + f"{human(chunked)} page(s) held in more than one row, {human(continuations)} continuation row(s), " + f"and {human(empty)} empty row(s).") for _id, path, _type, _stored in kept[:10]: print(f" kept by the cap: {path}") if len(kept) > 10: print(f" ... and {human(len(kept) - 10)} more") - saved = sum(figures["stored"] - figures["compressed"] for _name, figures, _c, _n in summary) + every_set = [figures for _name, figures, *_left in summary] + saved = sum(each["stored"] - each["compressed"] for each in every_set) verb, declare = ("Would store", "would declare") if dry_run else ("Stored", "declared") - print(f"{verb} {human(sum(figures['items'] for _n, figures, _c, _k in summary))} page(s) in " - f"{human(sum(figures['blocks'] for _n, figures, _c, _k in summary))} block(s) of up to {per_block} pages and " + print(f"{verb} {human(sum(each['items'] for each in every_set))} page(s) in " + f"{human(sum(each['blocks'] for each in every_set))} block(s) of up to {per_block} pages and " f"{human(MAX_BLOCK_BYTES)} bytes decoded, {human(saved)} bytes smaller than their rows; {declare} " f"{TO_MAJOR}.0.0.") @@ -432,7 +398,7 @@ def run(cfg: dict) -> int: print(f"error: {db_path} has no CompressionDictionary, which blocks are compressed against", file=sys.stderr) return 1 - codec = DictionaryBrotli(dictionary) + codec = BrotliCodec(dictionary) selected = select_sets(conn) if cfg["dry_run"]: @@ -443,13 +409,13 @@ def run(cfg: dict) -> int: try: for sql in SCHEMA: conn.execute(sql) - expected = {} + written = ({}, {}) with ThreadPoolExecutor(max_workers=workers) as pool: - for name, rows, chunked, continuations in selected: - figures = store_set(conn, pool, codec, rows, per_block, workers, expected) - summary.append((name, figures, chunked, continuations)) - check_stored(conn, codec, expected, workers) - declare_version(conn) + for name, rows, chunked, continuations, empty in selected: + figures = store_set(conn, pool, codec, rows, per_block, workers, written) + summary.append((name, figures, chunked, continuations, empty)) + check_stored(conn, written) + declare_version(conn, TO_MAJOR, WHO, COMMENT) if cfg["dry_run"]: conn.execute("ROLLBACK") else: diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py index ec5b080c..799add72 100644 --- a/scripts/test_store_pages_in_blocks.py +++ b/scripts/test_store_pages_in_blocks.py @@ -17,11 +17,13 @@ block equal to its row's decoded bytes; blocked rows emptied, 4.0.0 declared and nothing else changed; a second run, no version, 1.x, 2.x and 4.x refused; a set decoded in several windows, and one page to a block so that blocks are -written while more are pending, each blocked whole and in order; --workers -below 1 refused; a dry run, a block that does not read back byte for byte or -decodes longer than its items, a lost block and a blocked row left full, each -writing nothing. On the second, with the default 64: an item over 1 MiB -decoded kept as a row, and a block closed before it would decode past 4 MiB. +written while more are pending, each blocked whole and in order; an empty row +left a row; each block decoded once; --workers below 1 refused; a dry run, a +block that does not read back byte for byte or decodes longer than its items, +and, before the commit, a lost block with or without its items, a changed +block, a moved item and a blocked row left full, each writing nothing. On the +second, with the default 64: an item over 1 MiB decoded kept as a row, and a +block closed before it would decode past 4 MiB. uv run --with pytest --with-requirements requirements.txt python -m pytest -q scripts/test_store_pages_in_blocks.py """ @@ -339,11 +341,27 @@ def test_the_summary_names_every_set_its_content_types_and_what_it_left(database assert f" {name}: {pages} page(s) in {blocks} block(s);" in out devsite = next(line for line in out.splitlines() if line.startswith(" a/devsite/: 4 page(s)")) assert all(kind in devsite for kind in ("text/html 2", "text/css 1", "text/javascript 1")) - assert (" i/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, and " - "1 continuation row(s).") in out + assert (" i/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, " + "1 continuation row(s), and 0 empty row(s).") in out assert "Stored 21 page(s) in 9 block(s) of up to 3 pages and 4,194,304 bytes decoded" in out +def test_an_empty_row_stays_a_row(database, capsys): + """An empty Brotli row has nothing to decode (`brotli -d` calls it corrupt input), so it is left as it is, and + every other page of its set is blocked as usual.""" + db_path, plain = database + _execute(db_path, "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "SELECT 'a/empty.html', 1, x'', id, 7 FROM ContentTypes WHERE value = 'text/html'") + assert _run(db_path) == 0 + assert [path for paths in _block_paths(db_path) for path in paths] == _blocked_in_order() + _assert_every_slice_is_its_page(db_path, plain) + assert _query(db_path, "SELECT c.content FROM Content AS c LEFT JOIN ContentBlockItems AS i " + "ON i.contentId = c.id WHERE c.path = 'a/empty.html' AND i.contentId IS NULL") == [(b"",)] + out = capsys.readouterr().out + assert (" a/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, " + "1 continuation row(s), and 1 empty row(s).") in out + + def test_continuation_rows_are_left_alone(database): db_path, plain = database before = _tables(db_path)["Content"] @@ -393,7 +411,7 @@ class _Reversing: """A codec whose decode reverses the bytes, so a row and its decoded bytes differ.""" @staticmethod - def decode(data: bytes) -> bytes: + def decompress(data: bytes, _compression: str) -> bytes: return data[::-1] @@ -599,30 +617,32 @@ def test_workers_below_1_is_refused_on_an_error_line(database, capsys, monkeypat _assert_nothing_written(db_path, sha) -class _Garbling(spb.DictionaryBrotli): +class _Garbling(opt.BrotliCodec): """The real codec, except that the `garble`-th decode of each block it encoded comes back with one byte - changed, the length kept, or with `extend` with one byte more at the end, every item's slice intact.""" + changed, the length kept, or with `extend` with one byte more at the end, every item's slice intact. `decodes` + counts the decodes of each block, across instances.""" garble = 1 extend = False + decodes = None def __init__(self, dictionary: bytes): super().__init__(dictionary) self._lock = threading.Lock() - self._decodes = {} + type(self).decodes = {} - def encode(self, data: bytes) -> bytes: - encoded = super().encode(data) + def compress(self, data: bytes, compression: str) -> bytes: + encoded = super().compress(data, compression) with self._lock: - self._decodes[encoded] = 0 + self.decodes[encoded] = 0 return encoded - def decode(self, data: bytes) -> bytes: - plain = super().decode(data) + def decompress(self, data: bytes, compression: str) -> bytes: + plain = super().decompress(data, compression) with self._lock: - if data not in self._decodes: + if data not in self.decodes: return plain - self._decodes[data] += 1 - if self._decodes[data] != self.garble: + self.decodes[data] += 1 + if self.decodes[data] != self.garble: return plain if self.extend: return plain + b"!" @@ -635,36 +655,33 @@ def test_a_block_that_does_not_decode_back_byte_for_byte_writes_nothing(database a row is emptied can stop the run.""" db_path, _plain = database sha = _sha(db_path) - monkeypatch.setattr(spb, "DictionaryBrotli", type("FirstDecode", (_Garbling,), {"garble": 1})) + monkeypatch.setattr(spb, "BrotliCodec", type("FirstDecode", (_Garbling,), {"garble": 1})) assert _run(db_path) == 1 err = capsys.readouterr().err assert "does not read back byte for byte from the block starting at" in err and "nothing was written" in err, err _assert_nothing_written(db_path, sha) -def test_a_stored_block_that_does_not_read_back_writes_nothing(database, capsys, monkeypatch): - """Garbled only on the second decode, the one over the stored blocks before the commit.""" - db_path, _plain = database - sha = _sha(db_path) - monkeypatch.setattr(spb, "DictionaryBrotli", type("SecondDecode", (_Garbling,), {"garble": 2})) - assert _run(db_path) == 1 - err = capsys.readouterr().err - assert "does not read back from block" in err and "nothing was written" in err, err - _assert_nothing_written(db_path, sha) +def test_each_block_is_decoded_once(database, monkeypatch): + """The decode-back is the only one: the check before the commit compares each stored block with the block + built, by sha256, so a second decode, garbled here, never happens.""" + db_path, plain = database + codec = type("SecondDecode", (_Garbling,), {"garble": 2}) + monkeypatch.setattr(spb, "BrotliCodec", codec) + assert _run(db_path) == 0 + assert sorted(codec.decodes.values()) == [1] * 9 + _assert_every_slice_is_its_page(db_path, plain) -@pytest.mark.parametrize("garble, says", [ - (1, "decoded back to"), # the decode-back, before a row is emptied - (2, "but its items add up to"), # the decode of the stored blocks, before the commit -], ids=["decode-back", "stored"]) -def test_a_block_that_decodes_longer_than_its_items_writes_nothing(database, capsys, monkeypatch, garble, says): - """One byte more at the end leaves every item's slice intact, so only a length check can stop the run.""" +def test_a_block_that_decodes_longer_than_its_items_writes_nothing(database, capsys, monkeypatch): + """One byte more at the end of the decode-back leaves every item's slice intact, so only a length check can + stop the run.""" db_path, _plain = database sha = _sha(db_path) - monkeypatch.setattr(spb, "DictionaryBrotli", type("Longer", (_Garbling,), {"garble": garble, "extend": True})) + monkeypatch.setattr(spb, "BrotliCodec", type("Longer", (_Garbling,), {"garble": 1, "extend": True})) assert _run(db_path) == 1 err = capsys.readouterr().err - assert says in err and "nothing was written" in err, err + assert "decoded back to" in err and "nothing was written" in err, err _assert_nothing_written(db_path, sha) @@ -675,15 +692,35 @@ def _lose_the_last_block(conn) -> None: conn.execute("DELETE FROM ContentBlocks WHERE id = ?", (block_id,)) +def _lose_a_block_keeping_its_items(conn) -> None: + """A block gone and its items left naming it: one item per blocked row still, and their rows empty.""" + conn.execute("DELETE FROM ContentBlocks WHERE id = (SELECT MAX(id) FROM ContentBlocks)") + + +def _change_a_stored_block(conn) -> None: + """A block stored with one byte other than the one built, its length kept.""" + block_id, data = conn.execute("SELECT id, data FROM ContentBlocks ORDER BY id LIMIT 1").fetchone() + conn.execute("UPDATE ContentBlocks SET data = ? WHERE id = ?", (bytes([data[0] ^ 1]) + data[1:], block_id)) + + +def _move_an_item(conn) -> None: + """An item located one byte later than where it was written.""" + conn.execute("UPDATE ContentBlockItems SET offset = offset + 1 WHERE contentId = " + "(SELECT MIN(contentId) FROM ContentBlockItems)") + + def _refill_a_blocked_row(conn) -> None: """A blocked row whose content was never emptied.""" conn.execute("UPDATE Content SET content = x'00' WHERE id = (SELECT MIN(contentId) FROM ContentBlockItems)") @pytest.mark.parametrize("damage, says", [ - (_lose_the_last_block, "ContentBlockItems does not hold one row per blocked item"), + (_lose_the_last_block, "item(s) as written, Content id"), + (_lose_a_block_keeping_its_items, "block(s) written are not in ContentBlocks, block 9 first"), + (_change_a_stored_block, "block 1 as stored is not the block written"), + (_move_an_item, "ContentBlockItems does not hold 1 item(s) as written"), (_refill_a_blocked_row, "1 blocked row(s) still hold their content"), -], ids=["a block lost", "a row not emptied"]) +], ids=["a block lost", "a block lost, its items kept", "a block changed", "an item moved", "a row not emptied"]) def test_the_check_before_the_commit_refuses_what_the_writes_got_wrong(database, capsys, monkeypatch, damage, says): """Each damage leaves every stored block decoding to its items, so only the check that names it can stop the run.""" From bd39a9ef6e2a8302b12eabcfb4745c7cbef9d567 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Fri, 30 Oct 2026 22:40:41 -0700 Subject: [PATCH 16/17] ADFA-6683 | CLAUDE.md: the Drive master stays 3.x until ADFA-6871; the block reader is CodeOnTheGo #2191 The ship-time order needs a 3.x copy, and the re-mint, its verify and the blocks step each refuse 4.x, so the master is upgraded only once ADFA-6871 turns a 4.x copy's blocks back into rows at build time. ADFA-6872 later teaches those tools 4.x. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYjFcUx64N8PMRknzL --- CLAUDE.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a140d1b7..b89aded8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -133,14 +133,14 @@ edit that database** — it contains no part of the production Android app itsel > page over 1 MiB decoded, a row with continuation rows, and each continuation row stay rows, and a > block never decodes to more than 4 MiB, so two fit the app's 8 MiB block cache. 4.0.0 promises > everything 3.0.0 does too (keyed tooltips, compressed templates). The app's block reader is -> ADFA-6683's CodeOnTheGo PR to `stage` (B04 in the ticket's plan), not yet numbered or merged. It -> reads blocks only from a database declaring major 4 or later, never because the tables exist, so a -> file declaring major 5 would be read as 4 (the gate is `>=`, as the 3.x gates are). An app without +> CodeOnTheGo #2191 to `stage` (B04 in the ticket's plan). It reads blocks only from a database +> declaring major 4 or later, never because the tables exist, so a file declaring major 5 would be +> read as 4 (the gate is `>=`, as the 3.x gates are). An app without > the reader would show every blocked page empty. On the 2026-10-20 ship-time build the step blocked > 22,478 pages in 356 blocks and took the packed file's `.br` 10,368,636 bytes smaller (66,283,003 to > 55,914,367). > -> Only the copy that ships gets the step, after the last dictionary re-mint and its verify, before the +> On the copy that ships, the step runs after the last dictionary re-mint and its verify, before the > pack. With it, the order on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, > `migrate_tooltip_schema.py`, `remint_dictionary.py --seeds 4` and `verify_remint_dictionary.py`, > `store_pages_in_blocks.py`, then `rebuild_database.py --pack @@ -152,10 +152,15 @@ edit that database** — it contains no part of the production Android app itsel > has the figures. > > **The Google Drive master database the documentation team edits will become 4.0.0 and will never go -> back to 3.x.** The blocks step itself still runs on the copy that ships, at build time, but from the -> upgrade on every tool below that does not refuse 4.x runs on a blocked database. None of them may be -> left as it is when the Drive copy is upgraded: each must be fixed or guarded first, as ADFA-6806 (the -> docs pipeline scripts), ADFA-6807 (DocDB Studio) and ADFA-6808 (check-tools) track. +> back to 3.x**, but not yet. The order above needs a 3.x copy, and the copy is made from the master: +> the re-mint, its verify and the blocks step each refuse a 4.x file. So the master stays 3.x until +> ADFA-6871 adds a build step that first turns the copy's blocks back into 3.x rows. The order then +> runs as written: the dictionary is re-minted, and the blocks step blocks every page again, including +> pages added or edited on the master since. ADFA-6872 teaches those three tools 4.x, after which that +> build step can go. From the upgrade on, every tool below that does not refuse 4.x runs on a blocked +> database. None of them may be left as it is when the Drive copy is upgraded: each must be fixed or +> guarded first, as ADFA-6806 (the docs pipeline scripts), ADFA-6807 (DocDB Studio) and ADFA-6808 +> (check-tools) track. > > Which tools refuse 4.x, writing nothing: `store_pages_in_blocks.py` itself (it runs on a file > declaring 3, and refuses one that already has the tables); `remint_dictionary.py` and From 583987fa8461a23866c73d5c5bb62dab497ab670 Mon Sep 17 00:00:00 2001 From: David Schachter Date: Fri, 30 Oct 2026 23:46:12 -0700 Subject: [PATCH 17/17] ADFA-6683 | CLAUDE.md: the Drive master is 2.x, and the build order cannot start from a 4.x one bd39a9e said the master is 3.x and that once ADFA-6871 turns a 4.x copy's blocks back into 3.x rows the order runs as written. Both are wrong: the Kotlin trim and the a/ pages-to-templates step refuse 3 or later, migrate_tooltip_schema.py converts only 2.x, and nothing converts 3.0 back to 2.x. So the order starts from a 2.x copy, as the master is today, and ADFA-6871 has to settle how a 4.x master is built into a copy that ships. The release order now waits on ADFA-6871 too. The test docstring for the check before the commit no longer says every damage leaves the blocks decoding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vh2DAYjFcUx64N8PMRknzL --- CLAUDE.md | 20 +++++++++++--------- scripts/store_pages_in_blocks.py | 12 ++++++------ scripts/test_store_pages_in_blocks.py | 4 ++-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b89aded8..eb841468 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -152,13 +152,15 @@ edit that database** — it contains no part of the production Android app itsel > has the figures. > > **The Google Drive master database the documentation team edits will become 4.0.0 and will never go -> back to 3.x**, but not yet. The order above needs a 3.x copy, and the copy is made from the master: -> the re-mint, its verify and the blocks step each refuse a 4.x file. So the master stays 3.x until -> ADFA-6871 adds a build step that first turns the copy's blocks back into 3.x rows. The order then -> runs as written: the dictionary is re-minted, and the blocks step blocks every page again, including -> pages added or edited on the master since. ADFA-6872 teaches those three tools 4.x, after which that -> build step can go. From the upgrade on, every tool below that does not refuse 4.x runs on a blocked -> database. None of them may be left as it is when the Drive copy is upgraded: each must be fixed or +> back to 3.x**, but not yet. The master is 2.x: `migrate_tooltip_schema.py` runs only on the copy +> that ships (above). The order above starts from a 2.x copy: the Kotlin trim and the `a/` +> pages-to-templates step refuse a file declaring 3 or later, `migrate_tooltip_schema.py` converts +> only 2.x, and the re-mint, its verify and the blocks step refuse 4.x. Nothing converts 3.0 back to +> 2.x, so the order cannot start from a 4.x master, even with its blocks turned back into rows. +> ADFA-6871 settles how a 4.x master is built into a copy that ships (which steps run, on what, and +> in what order), and the master is not upgraded before it does. ADFA-6872 teaches the re-mint, its +> verify and the blocks step 4.x. From the upgrade on, every tool below that does not refuse 4.x runs +> on a blocked database. None of them may be left as it is when the Drive copy is upgraded: each must be fixed or > guarded first, as ADFA-6806 (the docs pipeline scripts), ADFA-6807 (DocDB Studio) and ADFA-6808 > (check-tools) track. > @@ -233,8 +235,8 @@ edit that database** — it contains no part of the production Android app itsel > Drive copy is upgraded. > > Release order: a 4.0.0 database goes to dev-assets or Drive only once the app's block reader is on -> CodeOnTheGo's `stage`, and the Drive master is upgraded only after ADFA-6806, ADFA-6807 and ADFA-6808 -> are done. Builds take whatever dev-assets holds, so an early copy would give every build +> CodeOnTheGo's `stage`, and the Drive master is upgraded only after ADFA-6871, ADFA-6806, ADFA-6807 +> and ADFA-6808 are done. Builds take whatever dev-assets holds, so an early copy would give every build > empty pages. It ships with the 3.0.0 switch or later. An app update never replaces the installed > database, so a phone with a 3.x or 2.x file keeps the old read path. diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py index 0b0b7755..3bce9b5f 100644 --- a/scripts/store_pages_in_blocks.py +++ b/scripts/store_pages_in_blocks.py @@ -106,12 +106,12 @@ order is: minify, row drops, Kotlin trim, a/ pages-to-templates, migrate_tooltip_schema.py, remint_dictionary.py --seeds 4 and verify_remint_dictionary.py, this step, then rebuild_database.py --pack -Templates,Content,TooltipTexts,ContentBlocks --page-size 4096. That order needs -a 3.x copy, so the Drive master it is made from stays 3.x until ADFA-6871. The -master is to become 4.0.0 too (CLAUDE.md, "Schema 4.0.0"); from then on the -build first turns the copy's blocks back into rows (ADFA-6871), so the same -order runs and this step blocks every page again, those added or edited on the -master since included. A 4.0.0 database goes to dev-assets or Drive only once +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096. That order +starts from a 2.x copy, as the Drive master is today: the Kotlin trim and the +a/ step refuse 3 or later, and migrate_tooltip_schema.py converts only 2.x. The +master is to become 4.0.0 too (CLAUDE.md, "Schema 4.0.0"), but only once +ADFA-6871 settles how a 4.x master is built into a copy that ships: this order +cannot start from one. A 4.0.0 database goes to dev-assets or Drive only once the app's block reader is on CodeOnTheGo's stage: builds take whatever dev-assets holds, and an app without the reader would show every blocked page empty. diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py index 799add72..2d360cc0 100644 --- a/scripts/test_store_pages_in_blocks.py +++ b/scripts/test_store_pages_in_blocks.py @@ -722,8 +722,8 @@ def _refill_a_blocked_row(conn) -> None: (_refill_a_blocked_row, "1 blocked row(s) still hold their content"), ], ids=["a block lost", "a block lost, its items kept", "a block changed", "an item moved", "a row not emptied"]) def test_the_check_before_the_commit_refuses_what_the_writes_got_wrong(database, capsys, monkeypatch, damage, says): - """Each damage leaves every stored block decoding to its items, so only the check that names it can stop the - run.""" + """Each damage is one a write could leave after its block was built. The check before the commit must name it + and write nothing; a block lost with its items kept once passed it.""" db_path, _plain = database sha = _sha(db_path) check_stored = spb.check_stored