ADFA-6683: Store documentation pages in compressed blocks (schema 4.0.0)

.github/workflows/python-tests.yaml

@@ -69,14 +69,16 @@ jobs:
 
       # scripts/test_migrate_tooltip_schema.py needs only the standard library, pytest and the brotli CLI (above).
       # scripts/test_trim_kotlin_json.py needs only brotli and Pillow (in requirements.txt) and the brotli CLI.
-      - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + optimize_pdfs + Android and Java loaders'
+      # scripts/test_store_pages_in_blocks.py likewise, brotli and Pillow through optimize_db_media.py.
+      - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + trim_kotlin_json + store_pages_in_blocks + optimize_pdfs + Android and Java loaders'
         run: |
           uv run --with-requirements requirements.txt --with pytest -- \
             python -m pytest \
               ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON \
               scripts/sync_kotlin_stdlib_docs \
               scripts/test_migrate_tooltip_schema.py \
               scripts/test_trim_kotlin_json.py \
+              scripts/test_store_pages_in_blocks.py \
               scripts/test_optimize_pdfs.py \
               ProcessDocs/AndroidDocs/AndroidDocsToJson \
               scripts/sync_java_docs \

ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py

@@ -107,6 +107,28 @@
 decided the outcome: seed 20821, the only one to beat the dictionary in place,
 stored 73 bytes more Content than it and 51,657 bytes less in templates.
 
+From documentation version 4.0.0 on (ADFA-6683), scripts/store_pages_in_blocks.py
+stores most pages in ContentBlocks, compressed against the same dictionary,
+and leaves their Content rows empty. This re-mint recompresses Content rows
+and templates, not blocks, so it refuses a database whose newest
+DocumentationDatabaseVersion row declares major 4 or later, or that has
+ContentBlocks or ContentBlockItems whatever it declares, before training and
+with nothing written, backup included; the verifier refuses one too. On the
+copy that ships, the re-mint and its verify go before the blocks step:
+remint_dictionary.py --seeds 4, verify_remint_dictionary.py,
+store_pages_in_blocks.py, then rebuild_database.py --pack
+Templates,Content,TooltipTexts,ContentBlocks --page-size 4096.
+
+Re-minting after the blocks step instead, on the blocks, was measured and does
+not pay (ADFA-6683's B02, 2026-10-26, on the 2026-10-23 3.0.0 ship-time
+database, 4 seeds each, packed as above, .br with brotli -q 11 -w 24). A dictionary
+trained on the decoded blocks, cut into 128 KiB pieces, with the rows left and
+the templates, stored a .br at most 30,880 bytes smaller than this re-mint run
+before the blocks step, and 25,285 bytes larger on the mean of the 4 seeds:
+far inside the 0.33 MB seed spread above. Whole blocks as samples did worse,
+0.52 to 0.74 MB larger, because zstd trains on the first 128 KiB of a sample
+only.
+
 Every row is proved to round-trip against the new dictionary before it is
 written. The backup (VACUUM INTO, unless --no-backup) is taken only then, just
 before the write: a run that refuses, or keeps the dictionary in place, writes
@@ -154,6 +176,16 @@
 TEMPLATES_COMPRESSED_FROM_MAJOR = 3
 # Templates are sampled under this pseudo path, so they form a doc set of their own.
 TEMPLATE_SAMPLE_PREFIX = "(templates)/"
+# From this documentation major on, most pages are stored in ContentBlocks (ADFA-6683), which neither this re-mint
+# nor its verifier handles, so both refuse such a database.
+CONTENT_BLOCKS_FROM_MAJOR = 4
+# The tables scripts/store_pages_in_blocks.py adds. Both tools refuse a database holding either one, whatever
+# version it declares: a version row rewritten after the blocks step would not move the pages back into Content.
+CONTENT_BLOCK_TABLES = ("ContentBlocks", "ContentBlockItems")
+# Where the re-mint and its verify go on the copy that ships, relative to the blocks step and the pack.
+SHIP_TIME_ORDER = ("remint_dictionary.py --seeds 4, verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, "
+                   "then scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks "
+                   "--page-size 4096")
 
 
 def declared_major(conn) -> int | None:
@@ -172,6 +204,28 @@ def templates_compressed(conn) -> bool:
     return major is not None and major >= TEMPLATES_COMPRESSED_FROM_MAJOR
 
 
+def content_blocks_refusal(conn, handles: str) -> str | None:
+    """Why the re-mint or its verifier (`handles`: what it does, as "This re-mint recompresses") must leave this
+    database alone, or None: from CONTENT_BLOCKS_FROM_MAJOR on, most pages are in ContentBlocks, which neither
+    handles, and a database holding any of CONTENT_BLOCK_TABLES may have pages there whatever it declares."""
+    major = declared_major(conn)
+    if major is not None and major >= CONTENT_BLOCKS_FROM_MAJOR:
+        why = (f"the database declares documentation major {major}, and from {CONTENT_BLOCKS_FROM_MAJOR}.0.0 "
+               "(ADFA-6683) most pages are stored in ContentBlocks")
+    else:
+        tables = [name for name in CONTENT_BLOCK_TABLES if conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)).fetchone()]
+        if not tables:
+            return None
+        declared = "no documentation version" if major is None else f"documentation major {major}"
+        why = (f"the database has {' and '.join(tables)}, the tables scripts/store_pages_in_blocks.py (ADFA-6683) "
+               f"adds, though it declares {declared}, so its pages may be stored in ContentBlocks whatever its "
+               "version row now says")
+    return (f"{why}, compressed against the dictionary, with their Content rows empty. {handles} Content rows and "
+            "templates, not blocks. Re-mint before scripts/store_pages_in_blocks.py, never after it: on the copy "
+            f"that ships the order is {SHIP_TIME_ORDER}")
+
+
 def load_templates(conn, old_dictionary: bytes) -> list:
     """Every Templates row of a database whose templates are compressed, as (id, name, stored, plaintext).
     Raises RuntimeError naming each one that does not decode against the outgoing dictionary."""
@@ -305,7 +359,11 @@ def remint(conn, sample_size: int = REMINT_SAMPLE_SIZE, dict_size: int = REMINT_
     with the Content rows, in each candidate's total and the current dictionary's
     alike. `before_write` is called once every check has passed, just before the
     write, and not at all when nothing is to be written: main() takes its backup
-    there."""
+    there. A database with blocks (see CONTENT_BLOCKS_FROM_MAJOR) is refused
+    before anything else."""
+    refusal = content_blocks_refusal(conn, "This re-mint recompresses")
+    if refusal:
+        raise RuntimeError(f"{refusal}; nothing written")
     row = conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone()
     if row is None or not row[0]:
         raise RuntimeError("this database has no CompressionDictionary to re-mint; "

ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py

@@ -39,6 +39,13 @@
 CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE(name));
 """
 TEMPLATE_NAMES = ("layout.pebble", "page.peb", "nav.peb")
+# The tables scripts/store_pages_in_blocks.py adds (ADFA-6683), as it creates them.
+BLOCK_TABLES = {
+    "ContentBlocks": "CREATE TABLE ContentBlocks (id INTEGER PRIMARY KEY AUTOINCREMENT, data BLOB NOT NULL)",
+    "ContentBlockItems": ("CREATE TABLE ContentBlockItems (contentId INTEGER NOT NULL PRIMARY KEY REFERENCES "
+                          "Content(id), blockId INTEGER NOT NULL REFERENCES ContentBlocks(id), offset INTEGER NOT "
+                          "NULL, length INTEGER NOT NULL) WITHOUT ROWID"),
+}
 HERE = Path(__file__).resolve().parent
 
 # The dictionary the fixture is first minted with. Small on purpose, so a 16 KiB re-mint beats it by hundreds of
@@ -726,6 +733,136 @@ def test_verifier_catches_a_changed_missing_stale_or_extra_template(self):
                                             "template page.peb"])
 
 
+class ContentBlocksRefusalTest(_RemintFixture):
+    """From documentation version 4.0.0 on (ADFA-6683) most pages are in ContentBlocks, compressed against the
+    dictionary, which neither the re-mint nor its verifier handles: both refuse a database declaring major 4 or later,
+    as the app reads its version, or holding the blocks step's tables whatever it declares, write nothing, and say the
+    re-mint goes before the blocks step."""
+
+    def remint(self, **kwargs):
+        return remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20, seed=999, **kwargs)
+
+    def assertNamesTheBlocksStepAndTheOrder(self, message: str):
+        self.assertIn("Re-mint before scripts/store_pages_in_blocks.py", message)
+        self.assertIn("verify_remint_dictionary.py, scripts/store_pages_in_blocks.py, then "
+                      "scripts/rebuild_database.py --pack Templates,Content,TooltipTexts,ContentBlocks --page-size 4096",
+                      message)
+
+    def test_the_remint_refuses_major_4_or_later_before_training(self):
+        for majors in ((3, 4), (5,)):
+            with self.subTest(majors=majors):
+                self.tearDown()
+                self.setUp()
+                self.add_versions_and_templates(majors=majors)
+                content, templates = dict(self.conn.execute("SELECT path, content FROM Content")), \
+                    self.stored_templates()
+                with mock.patch.object(remint_dictionary, "train_dictionary",
+                                       side_effect=AssertionError("trained on a database with blocks")):
+                    with self.assertRaises(RuntimeError) as raised:
+                        self.remint()
+                message = str(raised.exception)
+                self.assertIn(f"declares documentation major {majors[-1]}", message)
+                self.assertIn("nothing written", message)
+                self.assertNamesTheBlocksStepAndTheOrder(message)
+                self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway")
+                self.assertEqual(dict(self.conn.execute("SELECT path, content FROM Content")), content)
+                self.assertEqual(self.stored_templates(), templates)
+
+    def test_the_newest_version_row_decides(self):
+        """An older row declaring 4 under a newer one declaring 3 is a 3.x database, as the app reads it: re-minted,
+        templates and all, and verified."""
+        self.add_versions_and_templates(majors=(4, 3))
+        before = self.snapshot()
+        stats = self.remint()
+        self.assertEqual(stats["kept"], "candidate")
+        self.assertEqual(stats["templates"], len(TEMPLATE_NAMES))
+        self.assertEqual(verify(before, self.db_path), [])
+
+    def test_the_verifier_refuses_either_database_declaring_major_4(self):
+        self.add_versions_and_templates(majors=(3,))
+        original = self.snapshot()
+        self.conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) "
+                          "VALUES (4, 0, 0, 'ADFA-6683', 'blocks', '2026-10-26 00:00:00')")
+        self.conn.commit()
+        for old, new in ((original, self.db_path), (self.db_path, original)):
+            with self.subTest(old=old.name, new=new.name):
+                with self.assertRaises(RuntimeError) as raised:
+                    verify(old, new)
+                message = str(raised.exception)
+                self.assertTrue(message.startswith(f"{self.db_path}: the database declares documentation major 4"),
+                                message)
+                self.assertIn("This check compares Content rows and templates, not blocks", message)
+                self.assertNamesTheBlocksStepAndTheOrder(message)
+
+    def add_block_tables(self, tables=BLOCK_TABLES):
+        self.conn.executescript("".join(BLOCK_TABLES[name] + ";" for name in tables))
+        self.conn.commit()
+
+    def test_the_remint_refuses_block_tables_whatever_the_database_declares(self):
+        """A version row rewritten to 3 after the blocks step, or dropped, leaves the pages in blocks all the same."""
+        for majors, tables, declares in (((3,), tuple(BLOCK_TABLES), "documentation major 3"),
+                                         ((3,), ("ContentBlockItems",), "documentation major 3"),
+                                         (None, ("ContentBlocks",), "no documentation version")):
+            with self.subTest(majors=majors, tables=tables):
+                self.tearDown()
+                self.setUp()
+                if majors is not None:
+                    self.add_versions_and_templates(majors=majors)
+                self.add_block_tables(tables)
+                content = dict(self.conn.execute("SELECT path, content FROM Content"))
+                templates = self.stored_templates() if majors is not None else None
+                with mock.patch.object(remint_dictionary, "train_dictionary",
+                                       side_effect=AssertionError("trained on a database with blocks")):
+                    with self.assertRaises(RuntimeError) as raised:
+                        self.remint()
+                message = str(raised.exception)
+                self.assertTrue(message.startswith(f"the database has {' and '.join(tables)}, the tables "
+                                                   "scripts/store_pages_in_blocks.py (ADFA-6683) adds, though it "
+                                                   f"declares {declares}"), message)
+                self.assertIn("nothing written", message)
+                self.assertNamesTheBlocksStepAndTheOrder(message)
+                self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway")
+                self.assertEqual(dict(self.conn.execute("SELECT path, content FROM Content")), content)
+                if templates is not None:
+                    self.assertEqual(self.stored_templates(), templates)
+
+    def test_the_verifier_refuses_either_database_with_block_tables(self):
+        self.add_versions_and_templates(majors=(3,))
+        original = self.snapshot()
+        self.add_block_tables(("ContentBlocks",))
+        for old, new in ((original, self.db_path), (self.db_path, original)):
+            with self.subTest(old=old.name, new=new.name):
+                with self.assertRaises(RuntimeError) as raised:
+                    verify(old, new)
+                message = str(raised.exception)
+                self.assertTrue(message.startswith(f"{self.db_path}: the database has ContentBlocks"), message)
+                self.assertIn("This check compares Content rows and templates, not blocks", message)
+                self.assertNamesTheBlocksStepAndTheOrder(message)
+
+    def test_the_verifier_closes_both_databases_when_it_refuses(self):
+        """Otherwise each refused pair leaves two read-only handles open until they are collected, and on Windows
+        neither file can be deleted meanwhile."""
+        self.add_versions_and_templates(majors=(3,))
+        original = self.snapshot()
+        self.add_block_tables(("ContentBlocks",))
+        opened = []
+        connect = sqlite3.connect
+
+        def recording(*args, **kwargs):
+            opened.append(connect(*args, **kwargs))
+            return opened[-1]
+
+        for old, new in ((original, self.db_path), (self.db_path, original)):
+            with self.subTest(old=old.name, new=new.name):
+                opened.clear()
+                with mock.patch.object(sqlite3, "connect", recording), self.assertRaises(RuntimeError):
+                    verify(old, new)
+                self.assertEqual(len(opened), 2)
+                for conn in opened:
+                    with self.assertRaises(sqlite3.ProgrammingError):
+                        conn.execute("SELECT 1")
+
+
 class CommandLineRefusalTest(_RemintFixture):
     """A run that refuses writes nothing at all, the backup included, and says why on one "error:" line rather
     than in a traceback."""
@@ -779,6 +916,30 @@ def test_a_content_path_in_the_templates_doc_set_refuses_with_no_backup(self):
         sha = self.sha()
         self.assertRefusedWithNothingWritten(self.run_remint(), sha, "(templates)/page.peb")
 
+    def test_a_database_with_blocks_refuses_with_no_backup(self):
+        self.add_versions_and_templates(majors=(3, 4))
+        sha = self.sha()
+        self.assertRefusedWithNothingWritten(self.run_remint(), sha, "Re-mint before scripts/store_pages_in_blocks.py")
+
+    def test_a_3_x_database_with_block_tables_refuses_with_no_backup(self):
+        self.add_versions_and_templates(majors=(3,))
+        self.conn.executescript("".join(sql + ";" for sql in BLOCK_TABLES.values()))
+        sha = self.sha()
+        self.assertRefusedWithNothingWritten(self.run_remint(), sha, "the database has ContentBlocks and "
+                                            "ContentBlockItems")
+
+    def test_the_verifier_reports_a_database_with_blocks_on_an_error_line(self):
+        self.add_versions_and_templates(majors=(3, 4))
+        original = self.dir / "original.db"
+        shutil.copy2(self.db_path, original)
+        result = self.run_script("verify_remint_dictionary.py", str(original), str(self.db_path))
+        self.assertEqual(result.returncode, 1, result.stderr)
+        self.assertNotIn("Traceback", result.stderr)
+        self.assertTrue(result.stderr.startswith(f"error: {original}: the database declares documentation major 4"),
+                        result.stderr)
+        self.assertIn("store_pages_in_blocks.py", result.stderr)
+        self.assertNotIn("Byte-identical", result.stdout)
+
     def test_no_dictionary_refuses_with_no_backup(self):
         self.conn.execute("DELETE FROM CompressionDictionary")
         self.conn.commit()