diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index c3572405..dcdd6cfc 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -69,7 +69,8 @@ jobs: # scripts/test_migrate_tooltip_schema.py needs only the standard library, pytest and the brotli CLI (above). # scripts/test_trim_kotlin_json.py needs only brotli and Pillow (in requirements.txt) and the brotli CLI. - - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + optimize_pdfs + Android and Java loaders' + # scripts/test_store_pages_in_blocks.py likewise, brotli and Pillow through optimize_db_media.py. + - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + store_pages_in_blocks + optimize_pdfs + Android and Java loaders' run: | uv run --with-requirements requirements.txt --with pytest -- \ python -m pytest \ @@ -77,6 +78,7 @@ jobs: scripts/sync_kotlin_stdlib_docs \ scripts/test_migrate_tooltip_schema.py \ scripts/test_trim_kotlin_json.py \ + scripts/test_store_pages_in_blocks.py \ scripts/test_optimize_pdfs.py \ ProcessDocs/AndroidDocs/AndroidDocsToJson \ scripts/sync_java_docs \ diff --git a/CLAUDE.md b/CLAUDE.md index cd02a896..eb841468 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,10 @@ edit that database** — it contains no part of the production Android app itsel > (the minify and the `a/` step do), so there every one of them runs before the migration. The order > on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, > `migrate_tooltip_schema.py`, PR #48's `remint_dictionary.py --seeds 4` (which recompresses -> templates too in 3.0.0), then `rebuild_database.py --pack Content,TooltipTexts`; +> templates too in 3.0.0) and its verify, then `rebuild_database.py --pack +> Templates,Content,TooltipTexts --page-size 4096`, as CodeOnTheGo's `compress_docdb.yml` packs it +> (for 4.0.0, `store_pages_in_blocks.py` goes between the verify and the pack, which then ends with +> `--pack Templates,Content,TooltipTexts,ContentBlocks --page-size 4096`: "Schema 4.0.0" below); > `hm_pages_to_templates.py` goes after the `i/` page steps it lists, before the migration like > every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the > re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0. @@ -96,13 +99,147 @@ edit that database** — it contains no part of the production Android app itsel > `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`), > `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`) and `TooltipManager.py`. Until they > catch up the step runs on the copy that ships, after all of them, and never on the master -> database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH. +> database they edit; nothing converts 3.0 back. (The Drive master will itself become 4.0.0, which +> includes 3.0.0: "Schema 4.0.0" below.) It needs the `brotli` CLI on PATH. > > In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the > id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)` > index would cost about the 613 pages the conversion saves), and two rows sharing an id share one > text, so deleting either key's text deletes the other's. +> ### Schema 4.0.0: content blocks +> +> `scripts/store_pages_in_blocks.py` (ADFA-6683) stores most pages up to 64 to a compressed block and +> declares **4.0.0**: +> +> ```sql +> CREATE TABLE ContentBlocks ( +> id INTEGER PRIMARY KEY AUTOINCREMENT, +> data BLOB NOT NULL -- Brotli (q11, w24) of the block's items, concatenated, against CompressionDictionary +> ); +> CREATE TABLE ContentBlockItems ( +> contentId INTEGER NOT NULL PRIMARY KEY REFERENCES Content(id), +> blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), +> offset INTEGER NOT NULL, -- into the decoded block, in bytes +> length INTEGER NOT NULL -- in bytes +> ) WITHOUT ROWID; +> ``` +> +> An item is exactly its row's decoded bytes, before any template is applied. **A blocked `Content` +> row is empty**: it keeps `path`, `languageID`, `contentTypeID` and `templateId`, and its `content` +> is `x''`, so anything that reads `Content.content` from a 4.x file reads nothing for that page. The +> blocked rows are five sets, each in path order, never mixed in a block: `a/` templated JSON (not +> `a/devsite/`), `j/` HTML, `k/` templated JSON, and every Brotli row under `a/devsite/` and `i/`. A +> page over 1 MiB decoded, a row with continuation rows, and each continuation row stay rows, and a +> block never decodes to more than 4 MiB, so two fit the app's 8 MiB block cache. 4.0.0 promises +> everything 3.0.0 does too (keyed tooltips, compressed templates). The app's block reader is +> CodeOnTheGo #2191 to `stage` (B04 in the ticket's plan). It reads blocks only from a database +> declaring major 4 or later, never because the tables exist, so a file declaring major 5 would be +> read as 4 (the gate is `>=`, as the 3.x gates are). An app without +> the reader would show every blocked page empty. On the 2026-10-20 ship-time build the step blocked +> 22,478 pages in 356 blocks and took the packed file's `.br` 10,368,636 bytes smaller (66,283,003 to +> 55,914,367). +> +> On the copy that ships, the step runs after the last dictionary re-mint and its verify, before the +> pack. With it, the order on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, +> `migrate_tooltip_schema.py`, `remint_dictionary.py --seeds 4` and `verify_remint_dictionary.py`, +> `store_pages_in_blocks.py`, then `rebuild_database.py --pack +> Templates,Content,TooltipTexts,ContentBlocks --page-size 4096` (`ContentBlocks` in the list, so the +> blocks are packed and `ContentBlockItems.blockId` follows them). The re-mint stays before the step: +> re-minting after it, on the blocks, was measured (ADFA-6683's B02, 2026-10-26, on the 2026-10-23 +> 3.0.0 ship-time database) and saved at most 30,880 bytes of `.br` (the mean of 4 seeds was 25,285 +> bytes larger), against a 0.33 MB spread between re-mint seeds; `remint_dictionary.py`'s docstring +> has the figures. +> +> **The Google Drive master database the documentation team edits will become 4.0.0 and will never go +> back to 3.x**, but not yet. The master is 2.x: `migrate_tooltip_schema.py` runs only on the copy +> that ships (above). The order above starts from a 2.x copy: the Kotlin trim and the `a/` +> pages-to-templates step refuse a file declaring 3 or later, `migrate_tooltip_schema.py` converts +> only 2.x, and the re-mint, its verify and the blocks step refuse 4.x. Nothing converts 3.0 back to +> 2.x, so the order cannot start from a 4.x master, even with its blocks turned back into rows. +> ADFA-6871 settles how a 4.x master is built into a copy that ships (which steps run, on what, and +> in what order), and the master is not upgraded before it does. ADFA-6872 teaches the re-mint, its +> verify and the blocks step 4.x. From the upgrade on, every tool below that does not refuse 4.x runs +> on a blocked database. None of them may be left as it is when the Drive copy is upgraded: each must be fixed or +> guarded first, as ADFA-6806 (the docs pipeline scripts), ADFA-6807 (DocDB Studio) and ADFA-6808 +> (check-tools) track. +> +> Which tools refuse 4.x, writing nothing: `store_pages_in_blocks.py` itself (it runs on a file +> declaring 3, and refuses one that already has the tables); `remint_dictionary.py` and +> `verify_remint_dictionary.py`, which recompress and compare `Content` rows and templates, not +> blocks (they also refuse a file holding `ContentBlocks` or `ContentBlockItems`, whatever it +> declares); `populate_db.py`, `sync_javadoc_json_to_db.py`, `load_android_json_db.py` and +> `docdb-studio`, which refuse a major above 3; and `android_pages_to_templates.py` and +> `trim_kotlin_json.py`, which refuse 3 or later. +> +> No other step that reads or writes `Content` knows blocks or refuses 4.x. Two read the version: +> `drop_unread_rows.py`, only to know that templates are compressed from 3 on, and +> `migrate_tooltip_schema.py`. None of them sees a blocked page as an empty one: its `x''` is not a +> Brotli stream (`brotli -d` reports "corrupt input"), so decoding it fails. What each does then +> (marked "run" where it was run on a small 4.0.0 test copy made with `store_pages_in_blocks.py +> --pages-per-block 4`, "code" where it is read from the code): +> +> - refuses, writing nothing: `minify_text.py` (on the blocked stylesheets; run), +> `svg_images_to_webp.py` (on a blocked SVG; run), `optimize_pdfs.py` if a PDF were blocked (code), +> and `drop_unread_rows.py` only when it has to read a blocked row (run): given nothing to drop it +> exits 0 with "Nothing to do" without reading one, and given something to drop it reports every +> blocked row it must read (the rows to drop, then the kept rows' references) as unreadable and exits +> 1, changing nothing; +> - writes what it can read, then exits 1 on `error:` lines: `bundle_remote_images.py`, +> `remove_remote_loads.py`, `remove_skin_font_comment.py`, `share_skin_sprite.py`, +> `update_media_references.py` (code), `optimize_db_media.py` (run: it commits, then prints "failed to +> optimize" for each blocked SVG and exits 1) and `migrate_content_to_dictionary_brotli.py` (run: an +> `error:` line per blocked row and exit 1; it commits in batches, code); +> - stops with a traceback before reaching a block, as on any 3.x file: `hm_pages_to_templates.py` +> (on its compressed template) and `check_remote_loads.py` (on the 3.x tooltip tables); +> - stops with a traceback on the first blocked row: `insert_optimized_media.py` (run: it writes its +> backup, then `rewrite_pages` fails decoding the first blocked page, `insert_optimized_media.py:289`), +> `verify_android_json_db.py` (run: the first blocked `a/` page it samples, decoded with +> `populate_db.py:399` after the read at `verify_android_json_db.py:107`) and +> `set_content_type_compression.py` (run: after its backup, the first blocked row of the type, decoded +> at `:118-121`); `compare_svg_renders.py` fails the same way in `svg_rows` (`:74-76`; code, not run); +> - exits 0 with a wrong answer: `list_database_documents.py` lists a blocked page as +> "Bytes: 0" (run; `:17-20`), and `migrate_tooltip_schema.py` leaves the file alone ("nothing to do"); +> - refuses a table it does not know, writing nothing: `DocumentationDatabase.py` and `ingest.py` +> (run: `ValueError: Database schema contains unexpected tables`, `DocumentationDatabase.py:92-95`; its +> allowed list is only `Content`, `Languages`, `ContentTypes` and `ide_tooltip_table`, so it refuses any +> 2.x or 3.x `documentation.db` too, not only a 4.x one); +> - fails on the 3.x tooltip tables whatever the blocks, so these cannot work on the master after 3.0.0 +> either (ADFA-6808 for `check-tools`): `layout_xml_tooltips.py` (inserts `Tooltips(summary, detail)`, +> `:92-97`) and `TooltipManager.py` (`:52`) read or write columns the 3.0 `Tooltips` table no longer +> has (code; the same SQL on a 3.0.0 test copy gives "no such column: summary"), and so does +> `check-tools/` (`db_health_checker.py` selects `t.summary` at `:130`). `check-tools` also has a +> check of its own that a blocked database fails: the empty-content query at +> `db_health_checker.py:266-269` counts every blocked row (22 of 22 on the test copy) as empty; +> - `sync_kdoc_json_to_db.py` has no version check and never decodes `Content`, so it writes into a +> blocked database as if it were 3.x, and silently (code, not run): `write_content` updates a `k/` +> page that still fits in one row in place (`UPDATE Content SET content = ?`, `:147`), leaving a blocked +> row holding its own content beside its `ContentBlockItems` row, a state the blocks step never writes +> (it checks that every blocked row is empty), and a page that has grown past one row is deleted and +> re-inserted under a new id (`:152-160`), which orphans its `ContentBlockItems` row; +> - works on 4.x: `rebuild_database.py --pack Templates,Content,ContentBlocks` (run: it renumbers the +> blocks, follows `ContentBlockItems.blockId` and `contentId`, and keeps the declared 4.0.0). +> +> `docdb-studio` (ADFA-6807) refuses a major above 3 (`SUPPORTED_DATABASE_MAJOR_VERSION = 3`, +> `docdb_studio.py:191`, checked at `:5220-5221`), and nothing in it mentions `ContentBlocks` or +> `ContentBlockItems`: reading a page (`fetch_content_for_path`, `get_html_anchors_for_path`), listing +> sizes (`get_paths_with_sizes`, `LENGTH(content)`), deleting rows (`_delete_content_by_ids`) and the +> import's orphan pass all treat `Content.content` as the page, so a blocked page would read as empty +> and a delete or overwrite would orphan its `ContentBlockItems` row. DocDB Studio has to gain block +> support before the Drive copy is upgraded, or the team cannot edit the master. +> +> The ship-time steps all run before the blocks step on the copy that ships, and must never run after it +> there. They also run on the Drive master, which is 4.0.0 from the upgrade, and these three do so +> without refusing: `migrate_content_to_dictionary_brotli.py`, `insert_optimized_media.py` and +> `sync_kdoc_json_to_db.py`, with the results above. They must be fixed or guarded (ADFA-6806) before the +> Drive copy is upgraded. +> +> Release order: a 4.0.0 database goes to dev-assets or Drive only once the app's block reader is on +> CodeOnTheGo's `stage`, and the Drive master is upgraded only after ADFA-6871, ADFA-6806, ADFA-6807 +> and ADFA-6808 are done. Builds take whatever dev-assets holds, so an early copy would give every build +> empty pages. It ships with the 3.0.0 switch or later. An app update never replaces the installed +> database, so a phone with a 3.x or 2.x file keeps the old read path. + The schema below is what `~/documentation.db` (Alex's current production copy) actually contains, as of 2026-08-05 — predating the 2.0.0 bump described above, which additionally adds the `CompressionDictionary` and `DocumentationDatabaseVersion` tables: @@ -149,10 +286,14 @@ a placeholder. In a database whose newest `DocumentationDatabaseVersion` row declares major 3 or later, every `Templates` row is stored Brotli-compressed against the shared dictionary, with no per-row marker -(ADFA-3305). Only the ship-time copy is converted; the master database the tools here edit keeps +(ADFA-3305). Until the Drive master is upgraded to 4.0.0 ("Schema 4.0.0" above), only the ship-time copy is converted; the master database the tools here edit keeps plain templates. `remint_dictionary.py` and `verify_remint_dictionary.py` read the version the way the app does: from major 3 on they sample, recompress and verify the templates together with the -`Content` rows, and below it they leave the templates exactly as stored. +`Content` rows, and below it they leave the templates exactly as stored. From major 4 on (content +blocks, "Schema 4.0.0" above) they refuse the file, writing nothing, and so they do a file holding +`ContentBlocks` or `ContentBlockItems` whatever it declares. Re-minting on the blocks instead was +measured (ADFA-6683's B02) and does not pay: at most 30,880 bytes of `.br`, against 0.33 MB of seed +noise. `remint_dictionary.py` trains a 768 KiB dictionary on up to 3,600 rows within 128 MiB by default (ADFA-3305), where the first mint keeps 256 KiB on up to 900 rows within 32 MiB. On the 2026-10-19 @@ -244,9 +385,11 @@ shape: every tool here reads and writes that, and only the shipped copy holds th converts to documentation schema 3.0.0, which Code On The Go reads from 3.0.0 on: the WITHOUT ROWID, lookup-table tooltip shape, and every template Brotli-compressed against the dictionary; run it on the copy that ships, after every step that writes tooltips or templates, since they all write the old - shape and plain templates, never on the master they edit; re-mint afterwards, then pack - `Content,TooltipTexts`, not `Tooltips`), `minify_text.py` (ADFA-6627: minifies the - HTML, CSS, JS and SVG rows and the HTML templates without changing what a page shows; run + shape and plain templates, never on the master they edit; re-mint afterwards, then + `rebuild_database.py --pack Templates,Content,TooltipTexts --page-size 4096` (not `Tooltips`); + for 4.0.0, `store_pages_in_blocks.py` goes before the pack, which then ends with `--pack + Templates,Content,TooltipTexts,ContentBlocks --page-size 4096`), `minify_text.py` (ADFA-6627: + minifies the HTML, CSS, JS and SVG rows and the HTML templates without changing what a page shows; run `ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py` and `verify_remint_dictionary.py` straight after it, or the minified rows compress worse than before), `svg_images_to_webp.py` (ADFA-3305: @@ -314,6 +457,10 @@ shape: every tool here reads and writes that, and only the shipped copy holds th `a/android/` and `a/androidx/` pages its own way. This step is the one that ships: the workflows that ran that pipeline on the master (`build-android-docs.yaml`, `build-android-docs-local.yaml`) were retired in ADFA-3305, never having run, since after it this step would convert almost nothing. +- **`scripts/store_pages_in_blocks.py`** - stores the `a/`, `j/`, `k/`, `a/devsite/` and `i/` pages up + to 64 to a compressed block and declares documentation version 4.0.0 (ADFA-6683; "Schema 4.0.0" + above). Ship-time copy only, after the last re-mint and its verify, then `rebuild_database.py --pack + Templates,Content,TooltipTexts,ContentBlocks --page-size 4096`. - **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py index 540b1f1c..799f43aa 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py @@ -107,6 +107,28 @@ decided the outcome: seed 20821, the only one to beat the dictionary in place, stored 73 bytes more Content than it and 51,657 bytes less in templates. +From documentation version 4.0.0 on (ADFA-6683), scripts/store_pages_in_blocks.py +stores most pages in ContentBlocks, compressed against the same dictionary, +and leaves their Content rows empty. This re-mint recompresses Content rows +and templates, not blocks, so it refuses a database whose newest +DocumentationDatabaseVersion row declares major 4 or later, or that has +ContentBlocks or ContentBlockItems whatever it declares, before training and +with nothing written, backup included; the verifier refuses one too. On the +copy that ships, the re-mint and its verify go before the blocks step: +remint_dictionary.py --seeds 4, verify_remint_dictionary.py, +store_pages_in_blocks.py, then rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096. + +Re-minting after the blocks step instead, on the blocks, was measured and does +not pay (ADFA-6683's B02, 2026-10-26, on the 2026-10-23 3.0.0 ship-time +database, 4 seeds each, packed as above, .br with brotli -q 11 -w 24). A dictionary +trained on the decoded blocks, cut into 128 KiB pieces, with the rows left and +the templates, stored a .br at most 30,880 bytes smaller than this re-mint run +before the blocks step, and 25,285 bytes larger on the mean of the 4 seeds: +far inside the 0.33 MB seed spread above. Whole blocks as samples did worse, +0.52 to 0.74 MB larger, because zstd trains on the first 128 KiB of a sample +only. + Every row is proved to round-trip against the new dictionary before it is written. The backup (VACUUM INTO, unless --no-backup) is taken only then, just before the write: a run that refuses, or keeps the dictionary in place, writes @@ -154,6 +176,16 @@ TEMPLATES_COMPRESSED_FROM_MAJOR = 3 # Templates are sampled under this pseudo path, so they form a doc set of their own. TEMPLATE_SAMPLE_PREFIX = "(templates)/" +# From this documentation major on, most pages are stored in ContentBlocks (ADFA-6683), which neither this re-mint +# nor its verifier handles, so both refuse such a database. +CONTENT_BLOCKS_FROM_MAJOR = 4 +# The tables scripts/store_pages_in_blocks.py adds. Both tools refuse a database holding either one, whatever +# version it declares: a version row rewritten after the blocks step would not move the pages back into Content. +CONTENT_BLOCK_TABLES = ("ContentBlocks", "ContentBlockItems") +# Where the re-mint and its verify go on the copy that ships, relative to the blocks step and the pack. +SHIP_TIME_ORDER = ("remint_dictionary.py --seeds 4, verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, " + "then scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks " + "--page-size 4096") def declared_major(conn) -> int | None: @@ -172,6 +204,28 @@ def templates_compressed(conn) -> bool: return major is not None and major >= TEMPLATES_COMPRESSED_FROM_MAJOR +def content_blocks_refusal(conn, handles: str) -> str | None: + """Why the re-mint or its verifier (`handles`: what it does, as "This re-mint recompresses") must leave this + database alone, or None: from CONTENT_BLOCKS_FROM_MAJOR on, most pages are in ContentBlocks, which neither + handles, and a database holding any of CONTENT_BLOCK_TABLES may have pages there whatever it declares.""" + major = declared_major(conn) + if major is not None and major >= CONTENT_BLOCKS_FROM_MAJOR: + why = (f"the database declares documentation major {major}, and from {CONTENT_BLOCKS_FROM_MAJOR}.0.0 " + "(ADFA-6683) most pages are stored in ContentBlocks") + else: + tables = [name for name in CONTENT_BLOCK_TABLES if conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone()] + if not tables: + return None + declared = "no documentation version" if major is None else f"documentation major {major}" + why = (f"the database has {' and '.join(tables)}, the tables scripts/store_pages_in_blocks.py (ADFA-6683) " + f"adds, though it declares {declared}, so its pages may be stored in ContentBlocks whatever its " + "version row now says") + return (f"{why}, compressed against the dictionary, with their Content rows empty. {handles} Content rows and " + "templates, not blocks. Re-mint before scripts/store_pages_in_blocks.py, never after it: on the copy " + f"that ships the order is {SHIP_TIME_ORDER}") + + def load_templates(conn, old_dictionary: bytes) -> list: """Every Templates row of a database whose templates are compressed, as (id, name, stored, plaintext). Raises RuntimeError naming each one that does not decode against the outgoing dictionary.""" @@ -305,7 +359,11 @@ def remint(conn, sample_size: int = REMINT_SAMPLE_SIZE, dict_size: int = REMINT_ with the Content rows, in each candidate's total and the current dictionary's alike. `before_write` is called once every check has passed, just before the write, and not at all when nothing is to be written: main() takes its backup - there.""" + there. A database with blocks (see CONTENT_BLOCKS_FROM_MAJOR) is refused + before anything else.""" + refusal = content_blocks_refusal(conn, "This re-mint recompresses") + if refusal: + raise RuntimeError(f"{refusal}; nothing written") row = conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone() if row is None or not row[0]: raise RuntimeError("this database has no CompressionDictionary to re-mint; " diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py index 87c60c9d..d676f81e 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py @@ -39,6 +39,13 @@ CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE(name)); """ TEMPLATE_NAMES = ("layout.pebble", "page.peb", "nav.peb") +# The tables scripts/store_pages_in_blocks.py adds (ADFA-6683), as it creates them. +BLOCK_TABLES = { + "ContentBlocks": "CREATE TABLE ContentBlocks (id INTEGER PRIMARY KEY AUTOINCREMENT, data BLOB NOT NULL)", + "ContentBlockItems": ("CREATE TABLE ContentBlockItems (contentId INTEGER NOT NULL PRIMARY KEY REFERENCES " + "Content(id), blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), offset INTEGER NOT " + "NULL, length INTEGER NOT NULL) WITHOUT ROWID"), +} HERE = Path(__file__).resolve().parent # The dictionary the fixture is first minted with. Small on purpose, so a 16 KiB re-mint beats it by hundreds of @@ -726,6 +733,136 @@ def test_verifier_catches_a_changed_missing_stale_or_extra_template(self): "template page.peb"]) +class ContentBlocksRefusalTest(_RemintFixture): + """From documentation version 4.0.0 on (ADFA-6683) most pages are in ContentBlocks, compressed against the + dictionary, which neither the re-mint nor its verifier handles: both refuse a database declaring major 4 or later, + as the app reads its version, or holding the blocks step's tables whatever it declares, write nothing, and say the + re-mint goes before the blocks step.""" + + def remint(self, **kwargs): + return remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20, seed=999, **kwargs) + + def assertNamesTheBlocksStepAndTheOrder(self, message: str): + self.assertIn("Re-mint before scripts/store_pages_in_blocks.py", message) + self.assertIn("verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, then " + "scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks --page-size 4096", + message) + + def test_the_remint_refuses_major_4_or_later_before_training(self): + for majors in ((3, 4), (5,)): + with self.subTest(majors=majors): + self.tearDown() + self.setUp() + self.add_versions_and_templates(majors=majors) + content, templates = dict(self.conn.execute("SELECT path, content FROM Content")), \ + self.stored_templates() + with mock.patch.object(remint_dictionary, "train_dictionary", + side_effect=AssertionError("trained on a database with blocks")): + with self.assertRaises(RuntimeError) as raised: + self.remint() + message = str(raised.exception) + self.assertIn(f"declares documentation major {majors[-1]}", message) + self.assertIn("nothing written", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway") + self.assertEqual(dict(self.conn.execute("SELECT path, content FROM Content")), content) + self.assertEqual(self.stored_templates(), templates) + + def test_the_newest_version_row_decides(self): + """An older row declaring 4 under a newer one declaring 3 is a 3.x database, as the app reads it: re-minted, + templates and all, and verified.""" + self.add_versions_and_templates(majors=(4, 3)) + before = self.snapshot() + stats = self.remint() + self.assertEqual(stats["kept"], "candidate") + self.assertEqual(stats["templates"], len(TEMPLATE_NAMES)) + self.assertEqual(verify(before, self.db_path), []) + + def test_the_verifier_refuses_either_database_declaring_major_4(self): + self.add_versions_and_templates(majors=(3,)) + original = self.snapshot() + self.conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (4, 0, 0, 'ADFA-6683', 'blocks', '2026-10-26 00:00:00')") + self.conn.commit() + for old, new in ((original, self.db_path), (self.db_path, original)): + with self.subTest(old=old.name, new=new.name): + with self.assertRaises(RuntimeError) as raised: + verify(old, new) + message = str(raised.exception) + self.assertTrue(message.startswith(f"{self.db_path}: the database declares documentation major 4"), + message) + self.assertIn("This check compares Content rows and templates, not blocks", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + + def add_block_tables(self, tables=BLOCK_TABLES): + self.conn.executescript("".join(BLOCK_TABLES[name] + ";" for name in tables)) + self.conn.commit() + + def test_the_remint_refuses_block_tables_whatever_the_database_declares(self): + """A version row rewritten to 3 after the blocks step, or dropped, leaves the pages in blocks all the same.""" + for majors, tables, declares in (((3,), tuple(BLOCK_TABLES), "documentation major 3"), + ((3,), ("ContentBlockItems",), "documentation major 3"), + (None, ("ContentBlocks",), "no documentation version")): + with self.subTest(majors=majors, tables=tables): + self.tearDown() + self.setUp() + if majors is not None: + self.add_versions_and_templates(majors=majors) + self.add_block_tables(tables) + content = dict(self.conn.execute("SELECT path, content FROM Content")) + templates = self.stored_templates() if majors is not None else None + with mock.patch.object(remint_dictionary, "train_dictionary", + side_effect=AssertionError("trained on a database with blocks")): + with self.assertRaises(RuntimeError) as raised: + self.remint() + message = str(raised.exception) + self.assertTrue(message.startswith(f"the database has {' and '.join(tables)}, the tables " + "scripts/store_pages_in_blocks.py (ADFA-6683) adds, though it " + f"declares {declares}"), message) + self.assertIn("nothing written", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway") + self.assertEqual(dict(self.conn.execute("SELECT path, content FROM Content")), content) + if templates is not None: + self.assertEqual(self.stored_templates(), templates) + + def test_the_verifier_refuses_either_database_with_block_tables(self): + self.add_versions_and_templates(majors=(3,)) + original = self.snapshot() + self.add_block_tables(("ContentBlocks",)) + for old, new in ((original, self.db_path), (self.db_path, original)): + with self.subTest(old=old.name, new=new.name): + with self.assertRaises(RuntimeError) as raised: + verify(old, new) + message = str(raised.exception) + self.assertTrue(message.startswith(f"{self.db_path}: the database has ContentBlocks"), message) + self.assertIn("This check compares Content rows and templates, not blocks", message) + self.assertNamesTheBlocksStepAndTheOrder(message) + + def test_the_verifier_closes_both_databases_when_it_refuses(self): + """Otherwise each refused pair leaves two read-only handles open until they are collected, and on Windows + neither file can be deleted meanwhile.""" + self.add_versions_and_templates(majors=(3,)) + original = self.snapshot() + self.add_block_tables(("ContentBlocks",)) + opened = [] + connect = sqlite3.connect + + def recording(*args, **kwargs): + opened.append(connect(*args, **kwargs)) + return opened[-1] + + for old, new in ((original, self.db_path), (self.db_path, original)): + with self.subTest(old=old.name, new=new.name): + opened.clear() + with mock.patch.object(sqlite3, "connect", recording), self.assertRaises(RuntimeError): + verify(old, new) + self.assertEqual(len(opened), 2) + for conn in opened: + with self.assertRaises(sqlite3.ProgrammingError): + conn.execute("SELECT 1") + + class CommandLineRefusalTest(_RemintFixture): """A run that refuses writes nothing at all, the backup included, and says why on one "error:" line rather than in a traceback.""" @@ -779,6 +916,30 @@ def test_a_content_path_in_the_templates_doc_set_refuses_with_no_backup(self): sha = self.sha() self.assertRefusedWithNothingWritten(self.run_remint(), sha, "(templates)/page.peb") + def test_a_database_with_blocks_refuses_with_no_backup(self): + self.add_versions_and_templates(majors=(3, 4)) + sha = self.sha() + self.assertRefusedWithNothingWritten(self.run_remint(), sha, "Re-mint before scripts/store_pages_in_blocks.py") + + def test_a_3_x_database_with_block_tables_refuses_with_no_backup(self): + self.add_versions_and_templates(majors=(3,)) + self.conn.executescript("".join(sql + ";" for sql in BLOCK_TABLES.values())) + sha = self.sha() + self.assertRefusedWithNothingWritten(self.run_remint(), sha, "the database has ContentBlocks and " + "ContentBlockItems") + + def test_the_verifier_reports_a_database_with_blocks_on_an_error_line(self): + self.add_versions_and_templates(majors=(3, 4)) + original = self.dir / "original.db" + shutil.copy2(self.db_path, original) + result = self.run_script("verify_remint_dictionary.py", str(original), str(self.db_path)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertTrue(result.stderr.startswith(f"error: {original}: the database declares documentation major 4"), + result.stderr) + self.assertIn("store_pages_in_blocks.py", result.stderr) + self.assertNotIn("Byte-identical", result.stdout) + def test_no_dictionary_refuses_with_no_backup(self): self.conn.execute("DELETE FROM CompressionDictionary") self.conn.commit() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py index e03cda0b..428ca985 100755 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/verify_remint_dictionary.py @@ -30,19 +30,25 @@ against the original, before the file is put in place. Reads only: neither database is modified. A database it cannot check (no -CompressionDictionary row) is reported on an "error:" line, exit 1. +CompressionDictionary row) is reported on an "error:" line, exit 1. So is a +pair where either database declares documentation major 4 or later +(ADFA-6683), or has ContentBlocks or ContentBlockItems whatever it declares: +most of its pages are, or may be, in ContentBlocks, which this check does not +compare, and which remint_dictionary.py refuses to re-mint. Re-mint and verify +before scripts/store_pages_in_blocks.py. """ import argparse import sys import threading from concurrent.futures import ThreadPoolExecutor +from contextlib import closing from pathlib import Path import brotli from populate_db import DictionaryCompressor from migrate_content_to_dictionary_brotli import load_base_rows, read_item -from remint_dictionary import templates_compressed +from remint_dictionary import content_blocks_refusal, templates_compressed _thread_local = threading.local() @@ -140,28 +146,33 @@ def verify(old_db: Path, new_db: Path, max_workers: int | None = None) -> list: """Every item and template that differs, as (label, problem). Raises RuntimeError for a database it cannot check.""" import sqlite3 - old_conn = sqlite3.connect(f"file:{old_db}?mode=ro", uri=True) - new_conn = sqlite3.connect(f"file:{new_db}?mode=ro", uri=True) - old_dictionary = dictionary_of(old_conn, old_db) - new_dictionary = dictionary_of(new_conn, new_db) - if old_dictionary == new_dictionary: - print("warning: both databases carry the same dictionary; nothing was re-minted", file=sys.stderr) - - items = [row[0] for row in load_base_rows(new_conn)] - print(f"Verifying {len(items):,} item(s)...", file=sys.stderr) - - bad = [] - with ThreadPoolExecutor(max_workers=max_workers) as executor: - for i, (path, problem) in enumerate( - zip(items, executor.map( - lambda p: check(old_dictionary, new_dictionary, old_db, new_db, p), items)), start=1 - ): - if problem: - bad.append((path, problem)) - if i % 6000 == 0: - print(f" {i:,}/{len(items):,} checked, {len(bad)} mismatched", file=sys.stderr) - print(f"Byte-identical: {len(items) - len(bad):,}/{len(items):,}") - return bad + verify_templates(old_conn, new_conn, old_dictionary, new_dictionary) + # Closed however this returns: a refusal raises, and each refused pair would otherwise keep two handles open. + with closing(sqlite3.connect(f"file:{old_db}?mode=ro", uri=True)) as old_conn, \ + closing(sqlite3.connect(f"file:{new_db}?mode=ro", uri=True)) as new_conn: + for path, conn in ((old_db, old_conn), (new_db, new_conn)): + refusal = content_blocks_refusal(conn, "This check compares") + if refusal: + raise RuntimeError(f"{path}: {refusal}") + old_dictionary = dictionary_of(old_conn, old_db) + new_dictionary = dictionary_of(new_conn, new_db) + if old_dictionary == new_dictionary: + print("warning: both databases carry the same dictionary; nothing was re-minted", file=sys.stderr) + + items = [row[0] for row in load_base_rows(new_conn)] + print(f"Verifying {len(items):,} item(s)...", file=sys.stderr) + + bad = [] + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for i, (path, problem) in enumerate( + zip(items, executor.map( + lambda p: check(old_dictionary, new_dictionary, old_db, new_db, p), items)), start=1 + ): + if problem: + bad.append((path, problem)) + if i % 6000 == 0: + print(f" {i:,}/{len(items):,} checked, {len(bad)} mismatched", file=sys.stderr) + print(f"Byte-identical: {len(items) - len(bad):,}/{len(items):,}") + return bad + verify_templates(old_conn, new_conn, old_dictionary, new_dictionary) def main() -> None: diff --git a/scripts/migrate_tooltip_schema.py b/scripts/migrate_tooltip_schema.py index 5ee066c6..e13436aa 100644 --- a/scripts/migrate_tooltip_schema.py +++ b/scripts/migrate_tooltip_schema.py @@ -606,10 +606,12 @@ def convert(conn) -> list: return dropped -def declare_version(conn) -> None: +def declare_version(conn, major: int = TO_MAJOR, who: str = WHO, comment: str = COMMENT) -> None: + """Replaces every DocumentationDatabaseVersion row with one declaring major.0.0. scripts/store_pages_in_blocks.py + declares 4.0.0 with it.""" conn.execute("DELETE FROM DocumentationDatabaseVersion") conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) VALUES (?, 0, 0, ?, ?)", - (TO_MAJOR, WHO, COMMENT)) + (major, who, comment)) def run(cfg: dict) -> int: diff --git a/scripts/optimize_db_media.py b/scripts/optimize_db_media.py index 757c561b..bd78845e 100644 --- a/scripts/optimize_db_media.py +++ b/scripts/optimize_db_media.py @@ -1279,7 +1279,9 @@ def compress(self, data: bytes, compression: str) -> bytes: return data if self._dictionary is None: return brotli.compress(data) - return self._run(data=data) + # The CLI's own defaults for stdin, written out: the same bytes, and the q11 w24 that rows and + # store_pages_in_blocks.py's blocks are documented as. + return self._run("-q", "11", "-w", "24", data=data) def close(self) -> None: if self._work_dir is not None: @@ -1300,6 +1302,21 @@ def load_dictionary(conn) -> bytes | None: return row[0] if row else None +def table_exists(conn, name: str) -> bool: + return conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone() is not None + + +def declared_major(conn) -> int | None: + """The documentation major the database declares - its newest DocumentationDatabaseVersion row, ordered as + Code On The Go's DatabaseVersionResolver orders them (newest changeTime, then newest rowid) - or None if it + declares none.""" + if not table_exists(conn, "DocumentationDatabaseVersion"): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion ORDER BY changeTime DESC, rowid DESC LIMIT 1" + ).fetchone() + return None if row is None else int(row[0]) + + def human(n: int) -> str: return f"{n:,}" diff --git a/scripts/store_pages_in_blocks.py b/scripts/store_pages_in_blocks.py new file mode 100644 index 00000000..3bce9b5f --- /dev/null +++ b/scripts/store_pages_in_blocks.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# ] +# /// +""" +store_pages_in_blocks.py + +Stores the pages of five documentation sets up to 64 to a compressed block, +instead of one compressed Content row each, and declares documentation +version 4.0.0 (ADFA-6683): + + uv run scripts/store_pages_in_blocks.py documentation.db --dry-run + uv run scripts/store_pages_in_blocks.py documentation.db + +Why: each Content row is compressed on its own, so what neighbouring pages +share - the same member tables, inherited-method lists and markup - is paid +for again in every row, and the shared CompressionDictionary only covers what +all pages share. Compressed together, the pages of a block in path order share +the rest. On the 2026-10-20 G02 ship-time build (3.0.0, after its re-mint), +this step stored 22,478 pages in 356 blocks: their rows held 34,834,228 bytes, +the blocks hold 24,608,732 (a/ 5,457,314 less, j/ 3,549,974, k/ 758,039, +a/devsite/ 296,984, i/ 163,185). Packed with rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096, the file went +from 84,688,896 bytes (the same build packed without blocks) to 74,616,832, +and its release .br (brotli -q 11 -w 24) from 66,283,003 to 55,914,367: +10,368,636 bytes smaller. The cap kept 18 pages as rows (5 a/, 12 j/, 1 k/), +and the largest block decodes to 4,172,562 bytes. With 16 workers the step +took 53 seconds. + +The format, documentation version 4.0.0 (CLAUDE.md, "Schema 4.0.0"): + + ContentBlocks(id, data) + data: Brotli of the block's items, concatenated - quality 11, window + 24, against CompressionDictionary, as Content rows are. + ContentBlockItems(contentId, blockId, offset, length) WITHOUT ROWID + The item is [offset, offset + length) of the decoded block. Its + foreign key to Content lets rebuild_database.py --pack Content + renumber contentId with Content. + +An item is exactly its row's decoded bytes, before any template is applied. A +blocked row keeps its path, languageID, contentTypeID and templateId, so every +lookup by path and every page's template work as before; only its content +becomes an empty blob. + +The version: 4.0.0 is a new major, because an app that does not read blocks +would serve every blocked page empty. The app reads blocks from a database +declaring major 4 or later, and never decides by whether the tables exist. +4.0.0 promises everything 3.0.0 does too: keyed tooltips and compressed +templates. So this step runs only on a database whose newest +DocumentationDatabaseVersion row (ORDER BY changeTime DESC, rowid DESC, as the +app and remint_dictionary.declared_major read it) declares major 3, and in the +same transaction as the blocks it replaces that table's rows with one, 4.0.0, +as migrate_tooltip_schema.py declares 3.0.0. A database declaring no version, +a major below 3, or a major of 4 or more is refused with nothing written. + +Which rows: five sets, each in ORDER BY path COLLATE BINARY, in this order; a +block never mixes sets. Blocks are numbered in this order as written; +rebuild_database.py --pack ContentBlocks renumbers them, and nothing reads the +order. Paths are matched with GLOB, which is case-sensitive as paths are: + + 1. a/ templated JSON: path GLOB 'a/*' AND NOT GLOB 'a/devsite/*' AND templateId > 0 + 2. j/ HTML: path GLOB 'j/*' AND content type text/html, templated or not + 3. k/ templated JSON: path GLOB 'k/*' AND templateId > 0 + 4. a/devsite/: every row under it + 5. i/: every row under it + +j/ takes every text/html row, including templated JSON rows: +sync_javadoc_json_to_db.py keeps contentTypeID as text/html and sets templateId +to the javadoc.peb id (its docstring, lines 7-8; the UPDATE at line 375 and the +INSERT at line 398), so there is no templateId condition here. + +Only rows whose content type is compressed with brotli, and only items held in +one row: a row that owns continuation rows ("path-1", ... after a 1 MiB row), +and each continuation row, stay as they are. So does an empty row, which has +nothing to decode. + +The cap: an item over MAX_ITEM_BYTES (1 MiB) decoded stays a row. A block +closes at --pages-per-block items (default 64), or before the next item would +take it past MAX_BLOCK_BYTES (4 MiB) decoded. Serving a page decodes its whole +block, and the app caches decoded blocks up to 8 MiB, so two blocks always fit. +Without the cap, j/'s Javadoc index pages (up to 5,065,005 bytes each) made a +64-page block decode to 28,055,018 bytes. The summary counts, for each set, +the pages in blocks and their content types, the pages the cap kept as rows, +the rows left for their continuation rows, and the empty rows. + +Checks: every block is decoded again before its rows are emptied; it must +decode to its items' total length, and each item's slice of it is compared byte +for byte with its row's decoded bytes. Before the commit ContentBlocks must +hold exactly the blocks written, each the same bytes (by sha256) as the block +that was decoded back, ContentBlockItems exactly the items written, each with +its block, offset and length, and every blocked row must be empty. A block +stored as it was built decodes as it did, so it is not decoded twice. Any +failure rolls everything back. + +A database that already has ContentBlocks or ContentBlockItems is refused +too: its blocked rows are empty, so a second run would store empty pages over +the blocked ones. + +Where it goes: on the copy that ships, after the last dictionary re-mint and +its verify (remint_dictionary.py and verify_remint_dictionary.py refuse a 4.x +database: they do not recompress blocks) and before the pack. On that copy the +order is: minify, row drops, Kotlin trim, a/ pages-to-templates, +migrate_tooltip_schema.py, remint_dictionary.py --seeds 4 and +verify_remint_dictionary.py, this step, then rebuild_database.py --pack +Templates,Content,TooltipTexts,ContentBlocks --page-size 4096. That order +starts from a 2.x copy, as the Drive master is today: the Kotlin trim and the +a/ step refuse 3 or later, and migrate_tooltip_schema.py converts only 2.x. The +master is to become 4.0.0 too (CLAUDE.md, "Schema 4.0.0"), but only once +ADFA-6871 settles how a 4.x master is built into a copy that ships: this order +cannot start from one. A 4.0.0 database goes to dev-assets or Drive only once +the app's block reader is on CodeOnTheGo's stage: builds take whatever +dev-assets holds, and an app without the reader would show every blocked page +empty. + +Runs in one transaction and VACUUMs afterwards. Takes no backup: run it on a +copy. --dry-run does all the work, writes and version included, and rolls it +back. +""" +from __future__ import annotations + +import argparse +import hashlib +import sqlite3 +import sys +from collections import Counter, deque +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from migrate_tooltip_schema import declare_version +from optimize_db_media import ( + BrotliCodec, + declared_major, + human, + is_continuation_path, + load_dictionary, + owned_fragment_paths, + table_exists, +) + +# (set, condition on Content AS c joined to ContentTypes AS t), in block-id order. GLOB, not LIKE: LIKE ignores case. +SETS = ( + ("a/", "c.path GLOB 'a/*' AND c.path NOT GLOB 'a/devsite/*' AND c.templateId > 0"), + ("j/", "c.path GLOB 'j/*' AND t.value = 'text/html'"), + ("k/", "c.path GLOB 'k/*' AND c.templateId > 0"), + ("a/devsite/", "c.path GLOB 'a/devsite/*'"), + ("i/", "c.path GLOB 'i/*'"), +) + +# Decoded sizes. An item over MAX_ITEM_BYTES stays a row; a block closes before an item would take it past +# MAX_BLOCK_BYTES, so two blocks fit the app's 8 MiB cache of decoded blocks. +MAX_ITEM_BYTES = 1024 * 1024 +MAX_BLOCK_BYTES = 4 * 1024 * 1024 +PAGES_PER_BLOCK = 64 +# Rows decoded at a time ahead of the grouping walk, so a set is never held decoded all at once. Read when a set is +# decoded, not bound as a default argument, so a test can shrink it. +DECODE_WINDOW = 256 + +# This step turns a 3.x database into 4.0.0, as one DocumentationDatabaseVersion row. +FROM_MAJOR, TO_MAJOR = 3, 4 +# migrate_tooltip_schema.py turns this major, and no other, into 3.0.0. +TOOLTIP_MIGRATION_FROM_MAJOR = 2 +WHO = "ADFA-6683" +COMMENT = ("ADFA-6683: a/, j/, k/, a/devsite/ and i/ pages stored up to 64 to a Brotli block in ContentBlocks, each " + "located by ContentBlockItems; a blocked Content row's content is empty. Keyed tooltips and compressed " + "templates as in 3.0.0") + +SCHEMA = ( + """CREATE TABLE ContentBlocks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data BLOB NOT NULL -- Brotli (q11, w24) of the block's items, concatenated, against CompressionDictionary +)""", + """CREATE TABLE ContentBlockItems ( + contentId INTEGER NOT NULL PRIMARY KEY REFERENCES Content(id), + blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), + offset INTEGER NOT NULL, -- into the decoded block, in bytes + length INTEGER NOT NULL -- in bytes +) WITHOUT ROWID""", +) + + +def version_refusal(conn) -> str | None: + """Why this database's declared version keeps it from getting blocks, or None if it declares FROM_MAJOR.""" + major = declared_major(conn) + if major is None: + return (f"it declares no documentation version (no DocumentationDatabaseVersion row), and this step turns " + f"only a database declaring {FROM_MAJOR}.x into {TO_MAJOR}.0.0. Run it on the copy that ships, once " + f"that declares {FROM_MAJOR}.0.0") + if major < FROM_MAJOR: + refusal = (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into " + f"{TO_MAJOR}.0.0, which promises everything {FROM_MAJOR}.0.0 does (keyed tooltips, compressed " + "templates)") + if major == TOOLTIP_MIGRATION_FROM_MAJOR: + refusal += f"; run migrate_tooltip_schema.py first, which turns {major}.x into {FROM_MAJOR}.0.0" + return refusal + if major > FROM_MAJOR: + return (f"it declares documentation major {major}, and this step turns {FROM_MAJOR}.x into {TO_MAJOR}.0.0; " + f"from {TO_MAJOR} on, pages are already in blocks or in a format this step does not know") + return None + + +def select_sets(conn) -> list: + """[(set, [(content id, path, content type, stored bytes)] in path order, items held in more than one row, + their continuation rows, empty rows)] for each of SETS.""" + lengths = dict(conn.execute("SELECT path, LENGTH(content) FROM Content")) + selected = [] + for name, condition in SETS: + rows, chunked, continuations, empty = [], 0, 0, 0 + for content_id, path, content_type, stored in conn.execute( + "SELECT c.id, c.path, t.value, c.content FROM Content AS c JOIN ContentTypes AS t " + f"ON t.id = c.contentTypeID WHERE t.compression = 'brotli' AND {condition} " + "ORDER BY c.path COLLATE BINARY"): + if not stored: + # Not a Brotli stream: `brotli -d` calls it corrupt input, and there is no page to block. + empty += 1 + elif is_continuation_path(lengths, path): + continuations += 1 + elif owned_fragment_paths(conn, path, len(stored)): + chunked += 1 + else: + rows.append((content_id, path, content_type, stored)) + selected.append((name, rows, chunked, continuations, empty)) + return selected + + +def decode_row(codec: BrotliCodec, row: tuple) -> bytes: + try: + return codec.decompress(row[3], "brotli") + except RuntimeError as exc: + raise RuntimeError(f"{row[1]}: {exc}") from exc + + +def decoded_in_order(pool, codec: BrotliCodec, rows: list): + """(row, its decoded bytes) for every row, in order, decoded in the pool DECODE_WINDOW rows at a time: the next + window only once the walk has taken the last row of this one.""" + window = DECODE_WINDOW + for start in range(0, len(rows), window): + batch = rows[start:start + window] + yield from zip(batch, pool.map(lambda row: decode_row(codec, row), batch)) + + +def group_items(decoded, per_block: int, kept: list): + """Walks (row, decoded bytes) in path order and yields the blocks, each a list of them. A block closes at + `per_block` items, or before the next item would take it past MAX_BLOCK_BYTES decoded. An item over + MAX_ITEM_BYTES decoded goes to `kept` instead, and stays a row.""" + group, size = [], 0 + for row, plain in decoded: + if len(plain) > MAX_ITEM_BYTES: + kept.append(row) + continue + if group and (len(group) == per_block or size + len(plain) > MAX_BLOCK_BYTES): + yield group + group, size = [], 0 + group.append((row, plain)) + size += len(plain) + if group: + yield group + + +def build_block(codec: BrotliCodec, group: list) -> tuple: + """(compressed block, decoded length, [(content id, offset, length)]) for one block's (row, decoded bytes), + once the block has been decoded back and each item's slice of it matched its row's decoded bytes byte for + byte.""" + plain = b"".join(raw for _row, raw in group) + data = codec.compress(plain, "brotli") + back = codec.decompress(data, "brotli") + first = group[0][0][1] + if len(back) != len(plain): + raise RuntimeError(f"the block starting at {first} decoded back to {human(len(back))} bytes, not " + f"{human(len(plain))}") + view, items, offset = memoryview(back), [], 0 + for (content_id, path, _type, _stored), raw in group: + if view[offset:offset + len(raw)] != raw: + raise RuntimeError(f"{path} does not read back byte for byte from the block starting at {first}") + items.append((content_id, offset, len(raw))) + offset += len(raw) + return data, len(plain), items + + +def store_set(conn, pool, codec: BrotliCodec, rows: list, per_block: int, workers: int, written: tuple) -> dict: + """Stores one set's rows in blocks, written in order as they are built, a few blocks ahead at most. Adds what + it writes to `written`, ({block id: sha256 of its data}, {content id: (block id, offset, length)}); returns + the set's figures.""" + stored_length = {row[0]: len(row[3]) for row in rows} + content_type = {row[0]: row[2] for row in rows} + figures = {"blocks": 0, "items": 0, "stored": 0, "compressed": 0, "decoded": 0, "types": Counter(), "kept": []} + written_blocks, written_items = written + pending = deque() + + def write(future): + data, plain_length, items = future.result() + block_id = conn.execute("INSERT INTO ContentBlocks (data) VALUES (?)", (data,)).lastrowid + written_blocks[block_id] = hashlib.sha256(data).digest() + for content_id, offset, length in items: + conn.execute("INSERT INTO ContentBlockItems (contentId, blockId, offset, length) VALUES (?, ?, ?, ?)", + (content_id, block_id, offset, length)) + conn.execute("UPDATE Content SET content = x'' WHERE id = ?", (content_id,)) + written_items[content_id] = (block_id, offset, length) + figures["stored"] += stored_length[content_id] + figures["types"][content_type[content_id]] += 1 + figures["blocks"] += 1 + figures["items"] += len(items) + figures["compressed"] += len(data) + figures["decoded"] += plain_length + + for group in group_items(decoded_in_order(pool, codec, rows), per_block, figures["kept"]): + pending.append(pool.submit(build_block, codec, group)) + while len(pending) > 2 * workers: + write(pending.popleft()) + while pending: + write(pending.popleft()) + return figures + + +def check_stored(conn, written: tuple) -> None: + """Compares the tables with what was written (`written`, as store_set fills it): ContentBlocks must hold + exactly the blocks written, each the same bytes by sha256, ContentBlockItems exactly the items written, each + with its block, offset and length, and every blocked row must now be empty. build_block decoded each block + back and matched every item's slice of it, so a block stored as built needs no second decode.""" + written_blocks, written_items = written + items = {content_id: (block_id, offset, length) for content_id, block_id, offset, length in conn.execute( + "SELECT contentId, blockId, offset, length FROM ContentBlockItems")} + wrong = sorted(content_id for content_id in items.keys() | written_items.keys() + if items.get(content_id) != written_items.get(content_id)) + if wrong: + raise RuntimeError(f"ContentBlockItems does not hold {human(len(wrong))} item(s) as written, Content id " + f"{wrong[0]} first") + stored = set() + # One block at a time from the cursor, so the blocks are never all in memory at once. + for block_id, data in conn.execute("SELECT id, data FROM ContentBlocks"): + if hashlib.sha256(data).digest() != written_blocks.get(block_id): + raise RuntimeError(f"block {block_id} as stored is not the block written") + stored.add(block_id) + lost = sorted(written_blocks.keys() - stored) + if lost: + raise RuntimeError(f"{human(len(lost))} block(s) written are not in ContentBlocks, block {lost[0]} first") + left = conn.execute("SELECT COUNT(*) FROM Content WHERE LENGTH(content) > 0 AND id IN " + "(SELECT contentId FROM ContentBlockItems)").fetchone()[0] + if left: + raise RuntimeError(f"{left} blocked row(s) still hold their content") + + +def report(summary: list, per_block: int, dry_run: bool) -> None: + for name, figures, chunked, continuations, empty in summary: + types = ", ".join(f"{value} {human(count)}" for value, count in figures["types"].most_common()) + print(f" {name}: {human(figures['items'])} page(s) in {human(figures['blocks'])} block(s); their rows held " + f"{human(figures['stored'])} bytes, the blocks hold {human(figures['compressed'])} " + f"({human(figures['decoded'])} decoded). Content types: {types or 'none'}.") + kept = figures["kept"] + print(f" {name}: left as rows: {human(len(kept))} page(s) over {human(MAX_ITEM_BYTES)} bytes decoded, " + f"{human(chunked)} page(s) held in more than one row, {human(continuations)} continuation row(s), " + f"and {human(empty)} empty row(s).") + for _id, path, _type, _stored in kept[:10]: + print(f" kept by the cap: {path}") + if len(kept) > 10: + print(f" ... and {human(len(kept) - 10)} more") + every_set = [figures for _name, figures, *_left in summary] + saved = sum(each["stored"] - each["compressed"] for each in every_set) + verb, declare = ("Would store", "would declare") if dry_run else ("Stored", "declared") + print(f"{verb} {human(sum(each['items'] for each in every_set))} page(s) in " + f"{human(sum(each['blocks'] for each in every_set))} block(s) of up to {per_block} pages and " + f"{human(MAX_BLOCK_BYTES)} bytes decoded, {human(saved)} bytes smaller than their rows; {declare} " + f"{TO_MAJOR}.0.0.") + + +def run(cfg: dict) -> int: + db_path, per_block, workers = cfg["db_path"], cfg["pages_per_block"], cfg.get("workers", 1) + if per_block < 1: + print(f"error: --pages-per-block must be at least 1, not {per_block}", file=sys.stderr) + return 1 + if workers < 1: + print(f"error: --workers must be at least 1, not {workers}", file=sys.stderr) + return 1 + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + size_before = db_path.stat().st_size + + conn = sqlite3.connect(db_path, isolation_level=None) + codec = None + summary = [] + try: + for table in ("ContentBlockItems", "ContentBlocks"): + if table_exists(conn, table): + print(f"error: {db_path} already has {table}: it has been through this script, and its blocked " + "rows are empty, so a second run would store empty pages over them. Start again from a " + "database that has not.", file=sys.stderr) + return 1 + refusal = version_refusal(conn) + if refusal: + print(f"error: {db_path}: {refusal}; nothing was written", file=sys.stderr) + return 1 + dictionary = load_dictionary(conn) + if not dictionary: + print(f"error: {db_path} has no CompressionDictionary, which blocks are compressed against", + file=sys.stderr) + return 1 + codec = BrotliCodec(dictionary) + selected = select_sets(conn) + + if cfg["dry_run"]: + print(f"Dry run: every write is rolled back. Storing {db_path}'s pages up to {per_block} to a block ...") + else: + print(f"Storing {db_path}'s pages up to {per_block} to a block ...") + conn.execute("BEGIN") + try: + for sql in SCHEMA: + conn.execute(sql) + written = ({}, {}) + with ThreadPoolExecutor(max_workers=workers) as pool: + for name, rows, chunked, continuations, empty in selected: + figures = store_set(conn, pool, codec, rows, per_block, workers, written) + summary.append((name, figures, chunked, continuations, empty)) + check_stored(conn, written) + declare_version(conn, TO_MAJOR, WHO, COMMENT) + if cfg["dry_run"]: + conn.execute("ROLLBACK") + else: + conn.execute("COMMIT") + except BaseException: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + except (sqlite3.Error, RuntimeError) as exc: + print(f"error: {exc}; nothing was written", file=sys.stderr) + return 1 + finally: + if codec is not None: + codec.close() + conn.close() + + report(summary, per_block, cfg["dry_run"]) + if not cfg["dry_run"]: + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + print(f"{db_path}: {human(size_before)} -> {human(db_path.stat().st_size)} bytes.") + return 0 + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to change in place; run it on a copy") + p.add_argument("--pages-per-block", type=int, default=PAGES_PER_BLOCK, metavar="N", + help=f"Pages per block at most (default: {PAGES_PER_BLOCK}); a block also closes before it would " + f"decode to more than {human(MAX_BLOCK_BYTES)} bytes") + p.add_argument("--dry-run", action="store_true", help="Do all the work, then roll back without writing") + p.add_argument("--workers", type=int, default=8, + help="brotli processes run at once to decode rows and compress blocks (default: 8)") + args = p.parse_args() + sys.exit(run({"db_path": args.db_path, "pages_per_block": args.pages_per_block, "dry_run": args.dry_run, + "workers": args.workers})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_store_pages_in_blocks.py b/scripts/test_store_pages_in_blocks.py new file mode 100644 index 00000000..2d360cc0 --- /dev/null +++ b/scripts/test_store_pages_in_blocks.py @@ -0,0 +1,739 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_store_pages_in_blocks.py + +Covers store_pages_in_blocks.py on small synthetic databases. On the first, +three pages to a block: each of the five sets in path order with its own +blocks, the last one shorter, a/devsite/ and i/ rows taken whether templated +or not; rows held in more than one row left alone; every item sliced from its +block equal to its row's decoded bytes; blocked rows emptied, 4.0.0 declared +and nothing else changed; a second run, no version, 1.x, 2.x and 4.x refused; +a set decoded in several windows, and one page to a block so that blocks are +written while more are pending, each blocked whole and in order; an empty row +left a row; each block decoded once; --workers below 1 refused; a dry run, a +block that does not read back byte for byte or decodes longer than its items, +and, before the commit, a lost block with or without its items, a changed +block, a moved item and a blocked row left full, each writing nothing. On the +second, with the default 64: an item over 1 MiB decoded kept as a row, and a +block closed before it would decode past 4 MiB. + + uv run --with pytest --with-requirements requirements.txt python -m pytest -q scripts/test_store_pages_in_blocks.py +""" +from __future__ import annotations + +import hashlib +import random +import shutil +import sqlite3 +import subprocess +import sys +import threading +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import optimize_db_media as opt # noqa: E402 +import store_pages_in_blocks as spb # noqa: E402 + +PER_BLOCK = 3 +# The cap, written out rather than read from the script, so a script with another cap fails here. +ITEM_CAP = 1024 * 1024 +BLOCK_CAP = 4 * 1024 * 1024 +DICTIONARY = b'{"title": "", "members": [{"name": "", "summary": ""}]}
' * 64 + +# (path, content type, templateId). Inserted in this order, so ids do not follow paths. +BLOCKED = { + "a/": [("a/z.html", "text/html", 7), ("a/android/widget/TextView.html", "text/html", 7), + # Looks like a continuation row, but a/z.html is not 1 MiB, so it is a page of its own. + ("a/z.html-1", "text/html", 7), ("a/androidx/a.html", "text/html", 7), + ("a/Zeta.html", "text/html", 7), ("a/androidx/B.html", "text/html", 7), + ("a/android/util/Pair.html", "text/html", 7)], + "j/": [("j/index.html", "text/html", 0), ("j/api/b.html", "text/html", 0), + ("j/Overview.html", "text/html", 0), ("j/api/A.html", "text/html", 0), + # A templated javadoc JSON row: sync_javadoc_json_to_db.py keeps text/html and sets templateId. + ("j/api/T.html", "text/html", 7)], + "k/": [("k/html/y.html", "text/html", 9), ("k/html/x.html", "text/html", 8)], + # Every Brotli row under a/devsite/ and i/, templated or not, HTML or not. + "a/devsite/": [("a/devsite/reference/x.html", "text/html", 7), ("a/devsite/b.html", "text/html", 0), + ("a/devsite/style.css", "text/css", 0), ("a/devsite/Z.js", "text/javascript", 0)], + "i/": [("i/topic.html", "text/html", 0), ("i/jstopics/topic.js", "text/javascript", 0), + ("i/hm-topic.html", "text/html", 8)], +} +LEFT = [ + ("a/plain.html", "text/html", 0), # a/ without a template + ("a/raw.json", "text", 7), # its type is stored uncompressed + ("A/Upper.html", "text/html", 7), # not under a/: GLOB tells case apart, as paths do, where LIKE would not + ("j/style.css", "text/css", 0), # not HTML + ("j/logo.png", "image/png", 0), + ("k/html/nav.html", "text/html", 0), # no template + ("a/devsite/logo.png", "image/png", 0), # under a/devsite/, but not Brotli + ("i/img.png", "image/png", 0), # under i/, but not Brotli + ("d/other.html", "text/html", 7), # in no set +] +# Each stored as a 1 MiB row and a continuation row: in a set, but left as they are. +CHUNKED = [("a/big.html", "text/html", 7), ("i/book.pdf", "application/pdf", 0)] +TYPES = [("text/html", "brotli"), ("text/css", "brotli"), ("text/javascript", "brotli"), + ("application/pdf", "brotli"), ("text", "none"), ("image/png", "none")] +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, + compression TEXT NOT NULL); +CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, + UNIQUE(name)); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (languageID) REFERENCES Languages(id), FOREIGN KEY (contentTypeID) REFERENCES ContentTypes(id), + UNIQUE(path) +); +CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT ''); +CREATE TABLE DocumentationDatabaseVersion ( + major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL, who TEXT NOT NULL, comment TEXT NOT NULL, + changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +INSERT INTO DocumentationDatabaseVersion VALUES (2, 0, 0, 'ADFA-5220', 'dictionary', '2026-08-20 00:00:00'); +INSERT INTO DocumentationDatabaseVersion VALUES (3, 0, 0, 'ADFA-3305', 'keyed tooltips', '2026-10-20 00:00:00'); +""" +VERSION_ROWS = "SELECT major, minor, patch, who, comment FROM DocumentationDatabaseVersion ORDER BY rowid" + + +def _brotli(args: list, data: bytes, dictionary: Path) -> bytes: + return subprocess.run(["brotli", *args, "-D", str(dictionary)], input=data, stdout=subprocess.PIPE, + check=True).stdout + + +def _plain(path: str, rng: random.Random) -> bytes: + words = " ".join(rng.choice(["member", "summary", "inherited", "View", "int", "String"]) + for _ in range(rng.randrange(20, 400))) + return f'{{"path": "{path}", "members": [{{"name": "{words}"}}]}}'.encode() + + +def _sized(path: str, size: int) -> bytes: + """`size` bytes naming `path`, cheap to compress at any quality.""" + head = f'{{"path": "{path}", "text": "'.encode() + return (head + b"m" * size)[:size] + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _create(db_path: Path) -> tuple: + """A new database with SCHEMA, its types and templates; returns (connection, {content type: (id, + compression)}, the dictionary as a file).""" + dictionary = db_path.with_name("dict.bin") + dictionary.write_bytes(DICTIONARY) + conn = sqlite3.connect(db_path) + conn.executescript(SCHEMA) + conn.execute("INSERT INTO Languages (value) VALUES ('en')") + conn.executemany("INSERT INTO ContentTypes (value, compression) VALUES (?, ?)", TYPES) + conn.execute("INSERT INTO CompressionDictionary VALUES (1, ?)", (DICTIONARY,)) + conn.executemany("INSERT INTO Templates (id, name, content) VALUES (?, ?, ?)", + [(7, "android-reference.peb", b"t7"), (8, "page.peb", b"t8"), (9, "nav.peb", b"t9")]) + types = {value: (type_id, compression) for type_id, value, compression in conn.execute("SELECT * FROM ContentTypes")} + return conn, types, dictionary + + +def _insert(conn, types: dict, dictionary: Path, path: str, content_type: str, template_id: int, plain: bytes, + quality: str = "5") -> None: + type_id, compression = types[content_type] + stored = _brotli(["-c", "-q", quality], plain, dictionary) if compression == "brotli" else plain + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, ?)", + (path, stored, type_id, template_id)) + + +@pytest.fixture(scope="module") +def built(tmp_path_factory): + """(database, {path: plain bytes}): the pristine synthetic database.""" + db_path = tmp_path_factory.mktemp("blocks") / "documentation.db" + rng = random.Random(6683) + plain = {} + conn, types, dictionary = _create(db_path) + pages = [page for group in BLOCKED.values() for page in group] + LEFT + rng.shuffle(pages) + for path, content_type, template_id in pages: + plain[path] = _plain(path, rng) + _insert(conn, types, dictionary, path, content_type, template_id, plain[path]) + # One Brotli stream split across "" and "-1", as a real chunked row is. + for path, content_type, template_id in CHUNKED: + plain[path] = rng.randbytes(opt.CHUNK_SIZE + 100_000) + stored = _brotli(["-c", "-q", "0"], plain[path], dictionary) + assert len(stored) > opt.CHUNK_SIZE + for part, chunk in ((path, stored[:opt.CHUNK_SIZE]), (f"{path}-1", stored[opt.CHUNK_SIZE:])): + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, 1, ?, ?, ?)", (part, chunk, types[content_type][0], template_id)) + conn.execute("INSERT INTO Bookshelf (contentID, title) SELECT id, 'Pair' FROM Content " + "WHERE path = 'a/android/util/Pair.html'") + conn.commit() + conn.close() + return db_path, plain + + +@pytest.fixture +def database(built, tmp_path): + """A fresh copy of the synthetic database, and the plain bytes of its pages.""" + db_path, plain = built + copy = tmp_path / "documentation.db" + shutil.copyfile(db_path, copy) + return copy, plain + + +# The cap's database, in path order within each set: +# a/: one item 1 byte over the item cap (kept), one exactly at it and a small one (one block); +# j/: four items at the item cap, exactly the block cap (one block), then three at the item cap and one of half, +# 3.5 MiB, closed before the next item, half and a byte, would take it past the block cap; +# k/: 65 small items, 64 to a block by default. +CAP_PAGES = ( + [("a/cap/huge.html", "text/html", 7, ITEM_CAP + 1), ("a/cap/limit.html", "text/html", 7, ITEM_CAP), + ("a/cap/small.html", "text/html", 7, 100)] + + [(f"j/b{n}.html", "text/html", 0, ITEM_CAP) for n in range(4)] + + [(f"j/c{n}.html", "text/html", 0, ITEM_CAP) for n in range(3)] + + [("j/c3.html", "text/html", 0, ITEM_CAP // 2), ("j/c4.html", "text/html", 0, ITEM_CAP // 2 + 1)] + + [(f"k/{n:02}.html", "text/html", 9, 200) for n in range(65)] +) + + +@pytest.fixture(scope="module") +def cap_built(tmp_path_factory): + db_path = tmp_path_factory.mktemp("cap") / "documentation.db" + conn, types, dictionary = _create(db_path) + plain = {} + for path, content_type, template_id, size in CAP_PAGES: + plain[path] = _sized(path, size) + _insert(conn, types, dictionary, path, content_type, template_id, plain[path], quality="1") + conn.commit() + conn.close() + return db_path, plain + + +@pytest.fixture +def cap_database(cap_built, tmp_path): + db_path, plain = cap_built + copy = tmp_path / "documentation.db" + shutil.copyfile(db_path, copy) + return copy, plain + + +def _run(db_path: Path, dry_run: bool = False, per_block: int = PER_BLOCK) -> int: + return spb.run({"db_path": db_path, "pages_per_block": per_block, "dry_run": dry_run, "workers": 4}) + + +def _blocks(db_path: Path) -> list: + """[(decoded block, [(path, offset, length)] by offset)] in block-id order.""" + dictionary = db_path.with_name("read.dict.bin") + conn = sqlite3.connect(db_path) + try: + dictionary.write_bytes(conn.execute("SELECT data FROM CompressionDictionary").fetchone()[0]) + blocks = [] + for block_id, data in conn.execute("SELECT id, data FROM ContentBlocks ORDER BY id").fetchall(): + items = conn.execute("SELECT c.path, i.offset, i.length FROM ContentBlockItems AS i JOIN Content AS c " + "ON c.id = i.contentId WHERE i.blockId = ? ORDER BY i.offset", (block_id,)).fetchall() + blocks.append((_brotli(["-d", "-c"], data, dictionary), items)) + return blocks + finally: + conn.close() + + +def _block_paths(db_path: Path) -> list: + """[[path, ...] by offset] in block-id order.""" + return [[path for path, _offset, _length in items] for _data, items in _blocks(db_path)] + + +def _tables(db_path: Path) -> dict: + """Every row of every table, by table. Content's columns: id, path, + languageID, content, contentTypeID, templateId.""" + conn = sqlite3.connect(db_path) + try: + names = [row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")] + return {name: sorted(conn.execute(f'SELECT * FROM "{name}"').fetchall()) for name in names + if not name.startswith("ContentBlock")} + finally: + conn.close() + + +def _query(db_path: Path, sql: str, args=()) -> list: + conn = sqlite3.connect(db_path) + try: + return conn.execute(sql, args).fetchall() + finally: + conn.close() + + +def _execute(db_path: Path, *statements: str) -> None: + conn = sqlite3.connect(db_path) + try: + for sql in statements: + conn.execute(sql) + conn.commit() + finally: + conn.close() + + +def _in_path_order(paths) -> list: + """ORDER BY path in SQLite's BINARY collation: by UTF-8 bytes.""" + return sorted(paths, key=lambda path: path.encode("utf-8")) + + +def _set_of(path: str) -> str: + return "a/devsite/" if path.startswith("a/devsite/") else path[:2] + + +def _assert_nothing_written(db_path: Path, sha: str) -> None: + assert _sha(db_path) == sha + assert not db_path.with_name(db_path.name + "-journal").exists() + assert _query(db_path, "SELECT name FROM sqlite_master WHERE name LIKE 'ContentBlock%'") == [] + + +def _blocked_in_order() -> list: + """Every blocked path, set by set in SETS order, each set in path order: the order the items are written in.""" + return [path for name, _condition in spb.SETS for path in _in_path_order(p for p, _t, _i in BLOCKED[name])] + + +def _assert_every_slice_is_its_page(db_path: Path, plain: dict) -> None: + for data, items in _blocks(db_path): + for path, offset, length in items: + assert data[offset:offset + length] == plain[path], path + + +def test_blocks_follow_path_order_and_never_mix_sets(database): + db_path, _plain = database + assert _run(db_path) == 0 + blocks = _blocks(db_path) + expected = _blocked_in_order() + assert [path for _data, items in blocks for path, _offset, _length in items] == expected + assert [name for name, _condition in spb.SETS] == ["a/", "j/", "k/", "a/devsite/", "i/"] + # j/ takes text/html whatever its templateId: the javadoc JSON rows are templated. + assert "j/api/T.html" in expected + # Binary, not case-blind: a/Zeta.html sorts before a/android/..., a/androidx/B before a/androidx/a. + assert expected[:3] == ["a/Zeta.html", "a/android/util/Pair.html", "a/android/widget/TextView.html"] + for data, items in blocks: + assert len({_set_of(path) for path, _offset, _length in items}) == 1 + offset = 0 + for _path, start, length in items: + assert start == offset + offset += length + assert offset == len(data) + + +def test_the_last_block_of_a_set_is_shorter(database): + db_path, _plain = database + assert _run(db_path) == 0 + # a/ has 7 pages, j/ 5, k/ 2, a/devsite/ 4 and i/ 3: each set starts a block of its own. + assert [len(items) for _data, items in _blocks(db_path)] == [3, 3, 1, 3, 2, 2, 3, 1, 3] + + +def test_the_summary_names_every_set_its_content_types_and_what_it_left(database, capsys): + db_path, _plain = database + assert _run(db_path) == 0 + out = capsys.readouterr().out + for name, pages, blocks in (("a/", 7, 3), ("j/", 5, 2), ("k/", 2, 1), ("a/devsite/", 4, 2), ("i/", 3, 1)): + assert f" {name}: {pages} page(s) in {blocks} block(s);" in out + devsite = next(line for line in out.splitlines() if line.startswith(" a/devsite/: 4 page(s)")) + assert all(kind in devsite for kind in ("text/html 2", "text/css 1", "text/javascript 1")) + assert (" i/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, " + "1 continuation row(s), and 0 empty row(s).") in out + assert "Stored 21 page(s) in 9 block(s) of up to 3 pages and 4,194,304 bytes decoded" in out + + +def test_an_empty_row_stays_a_row(database, capsys): + """An empty Brotli row has nothing to decode (`brotli -d` calls it corrupt input), so it is left as it is, and + every other page of its set is blocked as usual.""" + db_path, plain = database + _execute(db_path, "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "SELECT 'a/empty.html', 1, x'', id, 7 FROM ContentTypes WHERE value = 'text/html'") + assert _run(db_path) == 0 + assert [path for paths in _block_paths(db_path) for path in paths] == _blocked_in_order() + _assert_every_slice_is_its_page(db_path, plain) + assert _query(db_path, "SELECT c.content FROM Content AS c LEFT JOIN ContentBlockItems AS i " + "ON i.contentId = c.id WHERE c.path = 'a/empty.html' AND i.contentId IS NULL") == [(b"",)] + out = capsys.readouterr().out + assert (" a/: left as rows: 0 page(s) over 1,048,576 bytes decoded, 1 page(s) held in more than one row, " + "1 continuation row(s), and 1 empty row(s).") in out + + +def test_continuation_rows_are_left_alone(database): + db_path, plain = database + before = _tables(db_path)["Content"] + assert _run(db_path) == 0 + blocked = {path for paths in _block_paths(db_path) for path in paths} + assert "a/z.html-1" in blocked + after = _tables(db_path)["Content"] + dictionary = db_path.with_name("chunked.dict.bin") + dictionary.write_bytes(DICTIONARY) + for chunked, _type, _template in CHUNKED: + for path in (chunked, f"{chunked}-1"): + assert path not in blocked + assert [row for row in after if row[1] == path] == [row for row in before if row[1] == path] + conn = sqlite3.connect(db_path) + try: + first = conn.execute("SELECT content FROM Content WHERE path = ?", (chunked,)).fetchone()[0] + stored = opt.reassemble(conn, chunked, first) + finally: + conn.close() + assert _brotli(["-d", "-c"], stored, dictionary) == plain[chunked] + + +def test_every_slice_equals_the_rows_decoded_bytes(database): + db_path, plain = database + assert _run(db_path) == 0 + sliced = 0 + for data, items in _blocks(db_path): + for path, offset, length in items: + assert data[offset:offset + length] == plain[path], path + sliced += 1 + assert sliced == sum(len(group) for group in BLOCKED.values()) + + +class _RecordingPool: + """Maps in the calling thread, recording how many rows each map() is handed.""" + + def __init__(self): + self.batches = [] + + def map(self, fn, rows): + rows = list(rows) + self.batches.append(len(rows)) + return [fn(row) for row in rows] + + +class _Reversing: + """A codec whose decode reverses the bytes, so a row and its decoded bytes differ.""" + + @staticmethod + def decompress(data: bytes, _compression: str) -> bytes: + return data[::-1] + + +def test_rows_are_decoded_a_window_at_a_time_in_order(monkeypatch): + monkeypatch.setattr(spb, "DECODE_WINDOW", 4) + rows = [(n, f"a/{n}.html", "text/html", f"row {n}".encode()) for n in range(10)] + pool = _RecordingPool() + walk = spb.decoded_in_order(pool, _Reversing(), rows) + taken = [next(walk) for _ in range(4)] + assert pool.batches == [4], "rows past the first window were decoded before the walk reached them" + taken += list(walk) + assert pool.batches == [4, 4, 2] + assert taken == [(row, row[3][::-1]) for row in rows] + + +@pytest.mark.parametrize("window", [2, 5]) +def test_a_set_spanning_several_decode_windows_is_blocked_whole(database, monkeypatch, window): + """a/ has 7 pages, so with a window of 2 or 5 it is decoded in more than one window, and a block of 3 straddles + two of them. Every page must still be blocked, in order.""" + db_path, plain = database + monkeypatch.setattr(spb, "DECODE_WINDOW", window) + assert _run(db_path) == 0 + assert [path for paths in _block_paths(db_path) for path in paths] == _blocked_in_order() + _assert_every_slice_is_its_page(db_path, plain) + + +@pytest.mark.parametrize("workers", [1, 2]) +def test_blocks_are_written_in_order_while_more_are_pending(database, capsys, workers): + """One page to a block: a/'s 7 blocks are more than the 2 * workers built ahead of the writes, so blocks are + written while later ones are still pending. Every block must be written, and in path order.""" + db_path, plain = database + assert spb.run({"db_path": db_path, "pages_per_block": 1, "dry_run": False, "workers": workers}) == 0 + expected = _blocked_in_order() + assert len(BLOCKED["a/"]) > 2 * workers, "fixture premise: a/ alone fills the write-ahead limit" + assert _block_paths(db_path) == [[path] for path in expected] + assert _query(db_path, "SELECT id FROM ContentBlocks ORDER BY id") == [(n,) for n in range(1, len(expected) + 1)] + _assert_every_slice_is_its_page(db_path, plain) + assert _query(db_path, "SELECT COUNT(*) FROM Content WHERE LENGTH(content) = 0") == [(len(expected),)] + assert f"Stored {len(expected)} page(s) in {len(expected)} block(s)" in capsys.readouterr().out + + +def test_blocked_rows_are_emptied_and_nothing_else_changes(database): + db_path, _plain = database + before = _tables(db_path) + assert _run(db_path) == 0 + after = _tables(db_path) + blocked = {path for group in BLOCKED.values() for path, _type, _template in group} + old, new = before.pop("Content"), after.pop("Content") + assert len(new) == len(old) + for was, now in zip(old, new): + if was[1] in blocked: + # id, path, languageID, contentTypeID and templateId stay; content is an empty blob. + assert now[3] == b"" and now[:3] + now[4:] == was[:3] + was[4:] + else: + assert now == was + sequence = dict(before.pop("sqlite_sequence")) + assert dict(after.pop("sqlite_sequence")) == {**sequence, "ContentBlocks": 9} + before.pop("DocumentationDatabaseVersion") + after.pop("DocumentationDatabaseVersion") # test_declares_4_0_0_in_one_row + assert after == before + conn = sqlite3.connect(db_path) + try: + assert conn.execute("SELECT DISTINCT typeof(content) FROM Content WHERE length(content) = 0").fetchall() == \ + [("blob",)] + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] + assert conn.execute("PRAGMA integrity_check").fetchall() == [("ok",)] + finally: + conn.close() + + +def test_declares_4_0_0_in_one_row(database): + db_path, _plain = database + assert _run(db_path) == 0 + assert _query(db_path, VERSION_ROWS) == [(4, 0, 0, "ADFA-6683", spb.COMMENT)] + assert "ADFA-6683" in spb.COMMENT + conn = sqlite3.connect(db_path) + try: + assert spb.declared_major(conn) == 4 + finally: + conn.close() + + +def test_a_second_run_is_refused(database, capsys): + db_path, _plain = database + assert _run(db_path) == 0 + once = _sha(db_path) + capsys.readouterr() + assert _run(db_path) == 1 + assert "already has ContentBlockItems" in capsys.readouterr().err + assert _sha(db_path) == once + + +def test_dry_run_writes_nothing(database, capsys): + db_path, _plain = database + pristine = _sha(db_path) + versions = _query(db_path, VERSION_ROWS) + assert _run(db_path, dry_run=True) == 0 + out = capsys.readouterr().out + assert "Would store 21 page(s) in 9 block(s)" in out + assert "would declare 4.0.0" in out + _assert_nothing_written(db_path, pristine) + assert _query(db_path, VERSION_ROWS) == versions + + +JAN, OCT = "2026-01-01 00:00:00", "2026-10-21 00:00:00" + + +@pytest.mark.parametrize("statements, says, migrate", [ + (["DROP TABLE DocumentationDatabaseVersion"], "declares no documentation version", False), + (["DELETE FROM DocumentationDatabaseVersion"], "declares no documentation version", False), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (1, 0, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 1", False), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (2, 1, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 2", True), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (4, 0, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 4", False), + ([f"INSERT INTO DocumentationDatabaseVersion VALUES (5, 0, 0, 'later', 'later', '{OCT}')"], + "declares documentation major 5", False), +], ids=["no version table", "no version row", "1.x", "2.x", "4.x", "5.x"]) +def test_a_database_not_declaring_3_is_refused_with_nothing_written(database, capsys, statements, says, migrate): + """Only a 2.x database is pointed at migrate_tooltip_schema.py, which converts 2.x and refuses anything else.""" + db_path, _plain = database + _execute(db_path, *statements) + sha = _sha(db_path) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert err.startswith("error: ") and says in err and "nothing was written" in err, err + assert ("run migrate_tooltip_schema.py first" in err) == migrate, err + _assert_nothing_written(db_path, sha) + + +@pytest.mark.parametrize("rows, major", [ + ([(3, OCT), (2, JAN)], 3), # a later rowid with an older changeTime does not decide + ([(2, OCT), (3, OCT)], 3), # a tie goes to the later rowid + ([(3, OCT), (2, OCT)], 2), + ([(4, JAN), (3, OCT)], 3), # an older row declaring 4 does not decide + ([(3, JAN), (4, OCT)], 4), +], ids=["older changeTime, later rowid", "tie, later rowid 3", "tie, later rowid 2", "older 4", "newer 4"]) +def test_the_newest_version_row_decides(database, rows, major): + """Read as the app reads it: ORDER BY changeTime DESC, rowid DESC. Each case is built so that rowid alone, + changeTime alone with the tie broken the other way, or the greatest major would read another major.""" + db_path, _plain = database + _execute(db_path, "DELETE FROM DocumentationDatabaseVersion", + *(f"INSERT INTO DocumentationDatabaseVersion VALUES ({m}, 0, 0, 'test', 'test', '{t}')" for m, t in rows)) + sha = _sha(db_path) + if major == 3: + assert _run(db_path) == 0 + assert _query(db_path, VERSION_ROWS) == [(4, 0, 0, "ADFA-6683", spb.COMMENT)] + else: + assert _run(db_path) == 1 + _assert_nothing_written(db_path, sha) + + +def test_an_item_over_1_mib_decoded_stays_a_row(cap_database, capsys): + db_path, plain = cap_database + before = dict(_query(db_path, "SELECT path, content FROM Content")) + assert len(before["a/cap/huge.html"]) < ITEM_CAP, "fixture premise: stored small, decoded over the cap" + assert _run(db_path, per_block=64) == 0 + blocks = _block_paths(db_path) + assert blocks[0] == ["a/cap/limit.html", "a/cap/small.html"] # exactly 1 MiB is not over it + assert "a/cap/huge.html" not in {path for paths in blocks for path in paths} + after = dict(_query(db_path, "SELECT path, content FROM Content")) + assert after["a/cap/huge.html"] == before["a/cap/huge.html"] + out = capsys.readouterr().out + assert " a/: left as rows: 1 page(s) over 1,048,576 bytes decoded" in out + assert " kept by the cap: a/cap/huge.html" in out + for data, items in _blocks(db_path): + for path, offset, length in items: + assert data[offset:offset + length] == plain[path], path + + +def test_a_block_closes_before_it_would_decode_past_4_mib(cap_database): + db_path, _plain = cap_database + assert _run(db_path, per_block=64) == 0 + blocks = _blocks(db_path) + j_blocks = [[path for path, _offset, _length in items] for _data, items in blocks + if items[0][0].startswith("j/")] + # Exactly 4 MiB is not past it; 3.5 MiB closes before half a MiB and a byte more. + assert j_blocks == [[f"j/b{n}.html" for n in range(4)], [f"j/c{n}.html" for n in range(4)], ["j/c4.html"]] + sizes = [len(data) for data, _items in blocks] + assert max(sizes) == BLOCK_CAP and all(size <= BLOCK_CAP for size in sizes) + + +def test_64_pages_to_a_block_by_default(cap_database, monkeypatch): + db_path, _plain = cap_database + monkeypatch.setattr(sys, "argv", ["store_pages_in_blocks.py", str(db_path), "--workers", "4"]) + with pytest.raises(SystemExit) as exited: + spb.main() + assert exited.value.code == 0 + k_blocks = [paths for paths in _block_paths(db_path) if paths[0].startswith("k/")] + assert [len(paths) for paths in k_blocks] == [64, 1] + + +@pytest.mark.parametrize("workers", ["0", "-1"]) +def test_workers_below_1_is_refused_on_an_error_line(database, capsys, monkeypatch, workers): + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(sys, "argv", ["store_pages_in_blocks.py", str(db_path), "--workers", workers]) + with pytest.raises(SystemExit) as exited: + spb.main() + assert exited.value.code == 1 + assert capsys.readouterr().err == f"error: --workers must be at least 1, not {workers}\n" + _assert_nothing_written(db_path, sha) + + +class _Garbling(opt.BrotliCodec): + """The real codec, except that the `garble`-th decode of each block it encoded comes back with one byte + changed, the length kept, or with `extend` with one byte more at the end, every item's slice intact. `decodes` + counts the decodes of each block, across instances.""" + garble = 1 + extend = False + decodes = None + + def __init__(self, dictionary: bytes): + super().__init__(dictionary) + self._lock = threading.Lock() + type(self).decodes = {} + + def compress(self, data: bytes, compression: str) -> bytes: + encoded = super().compress(data, compression) + with self._lock: + self.decodes[encoded] = 0 + return encoded + + def decompress(self, data: bytes, compression: str) -> bytes: + plain = super().decompress(data, compression) + with self._lock: + if data not in self.decodes: + return plain + self.decodes[data] += 1 + if self.decodes[data] != self.garble: + return plain + if self.extend: + return plain + b"!" + middle = len(plain) // 2 + return plain[:middle] + bytes([plain[middle] ^ 1]) + plain[middle + 1:] + + +def test_a_block_that_does_not_decode_back_byte_for_byte_writes_nothing(database, capsys, monkeypatch): + """Garbled only on the decode-back, so the blocks stored are sound and only the byte-for-byte check before + a row is emptied can stop the run.""" + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(spb, "BrotliCodec", type("FirstDecode", (_Garbling,), {"garble": 1})) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert "does not read back byte for byte from the block starting at" in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) + + +def test_each_block_is_decoded_once(database, monkeypatch): + """The decode-back is the only one: the check before the commit compares each stored block with the block + built, by sha256, so a second decode, garbled here, never happens.""" + db_path, plain = database + codec = type("SecondDecode", (_Garbling,), {"garble": 2}) + monkeypatch.setattr(spb, "BrotliCodec", codec) + assert _run(db_path) == 0 + assert sorted(codec.decodes.values()) == [1] * 9 + _assert_every_slice_is_its_page(db_path, plain) + + +def test_a_block_that_decodes_longer_than_its_items_writes_nothing(database, capsys, monkeypatch): + """One byte more at the end of the decode-back leaves every item's slice intact, so only a length check can + stop the run.""" + db_path, _plain = database + sha = _sha(db_path) + monkeypatch.setattr(spb, "BrotliCodec", type("Longer", (_Garbling,), {"garble": 1, "extend": True})) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert "decoded back to" in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha) + + +def _lose_the_last_block(conn) -> None: + """A block and its items gone, as a lost write would leave them: their rows are empty all the same.""" + block_id = conn.execute("SELECT MAX(id) FROM ContentBlocks").fetchone()[0] + conn.execute("DELETE FROM ContentBlockItems WHERE blockId = ?", (block_id,)) + conn.execute("DELETE FROM ContentBlocks WHERE id = ?", (block_id,)) + + +def _lose_a_block_keeping_its_items(conn) -> None: + """A block gone and its items left naming it: one item per blocked row still, and their rows empty.""" + conn.execute("DELETE FROM ContentBlocks WHERE id = (SELECT MAX(id) FROM ContentBlocks)") + + +def _change_a_stored_block(conn) -> None: + """A block stored with one byte other than the one built, its length kept.""" + block_id, data = conn.execute("SELECT id, data FROM ContentBlocks ORDER BY id LIMIT 1").fetchone() + conn.execute("UPDATE ContentBlocks SET data = ? WHERE id = ?", (bytes([data[0] ^ 1]) + data[1:], block_id)) + + +def _move_an_item(conn) -> None: + """An item located one byte later than where it was written.""" + conn.execute("UPDATE ContentBlockItems SET offset = offset + 1 WHERE contentId = " + "(SELECT MIN(contentId) FROM ContentBlockItems)") + + +def _refill_a_blocked_row(conn) -> None: + """A blocked row whose content was never emptied.""" + conn.execute("UPDATE Content SET content = x'00' WHERE id = (SELECT MIN(contentId) FROM ContentBlockItems)") + + +@pytest.mark.parametrize("damage, says", [ + (_lose_the_last_block, "item(s) as written, Content id"), + (_lose_a_block_keeping_its_items, "block(s) written are not in ContentBlocks, block 9 first"), + (_change_a_stored_block, "block 1 as stored is not the block written"), + (_move_an_item, "ContentBlockItems does not hold 1 item(s) as written"), + (_refill_a_blocked_row, "1 blocked row(s) still hold their content"), +], ids=["a block lost", "a block lost, its items kept", "a block changed", "an item moved", "a row not emptied"]) +def test_the_check_before_the_commit_refuses_what_the_writes_got_wrong(database, capsys, monkeypatch, damage, says): + """Each damage is one a write could leave after its block was built. The check before the commit must name it + and write nothing; a block lost with its items kept once passed it.""" + db_path, _plain = database + sha = _sha(db_path) + check_stored = spb.check_stored + + def damaged(conn, *args): + damage(conn) + return check_stored(conn, *args) + + monkeypatch.setattr(spb, "check_stored", damaged) + assert _run(db_path) == 1 + err = capsys.readouterr().err + assert says in err and "nothing was written" in err, err + _assert_nothing_written(db_path, sha)