diff --git a/CLAUDE.md b/CLAUDE.md
index f9510fd9..cd02a896 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -85,16 +85,18 @@ edit that database** — it contains no part of the production Android app itsel
> every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the
> re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0.
>
-> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` write plain templates,
-> so each refuses a database declaring 3.0.0 or later before writing anything, backup included.
-> `load_android_json_db.py` keeps the major it finds, so a late run would otherwise leave a file
-> declaring 3.1.0 with plain templates in it, which the app cannot read.
+> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` read and write both
+> (ADFA-6740): below 3.0.0 they write plain templates, from it compressed ones (`brotli -D -w 24`,
+> checked by decoding back, never large-window), and they leave a 3.x file at the version it
+> declares. They refuse a major above 3, a format they do not know, before writing anything.
+> `docdb-studio` reads and writes both shapes too (tooltips through the keyed tables, templates
+> decoded and re-encoded with the dictionary), choosing by the declared major.
>
-> **Every tool in this repo still reads and writes the 2.x shape**: `check-tools` (whose daily
+> **The other tools still read and write only the 2.x shape**: `check-tools` (whose daily
> `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`),
-> `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`), `TooltipManager.py` and
-> `docdb-studio`. So the step runs on the copy that ships, after all of them, and never on the
-> master database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH.
+> `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`) and `TooltipManager.py`. Until they
+> catch up the step runs on the copy that ships, after all of them, and never on the master
+> database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH.
>
> In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the
> id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)`
diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py
index 11aec50a..3ff11f17 100755
--- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py
+++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py
@@ -45,8 +45,8 @@
sys.path.insert(0, str(_KOTLIN_PIPELINE))
from populate_db import ( # noqa: E402 (the path has to be set up first)
- DictionaryCompressor, compressed_templates_refusal, fragment_chain, get_content_type, get_id,
- insert_chunked_content, load_dictionary, upsert_template,
+ DictionaryCompressor, fragment_chain, get_content_type, get_id, insert_chunked_content,
+ load_dictionary, templates_compressed, unknown_format_refusal, upsert_template,
)
from migrate_content_to_dictionary_brotli import write_item # noqa: E402
@@ -288,7 +288,17 @@ def check_schema(conn) -> list[str]:
def record_version(conn, stats: dict) -> None:
"""A row saying what changed, as the schema's own comment asks for. Minor, not major: nothing
- about the schema or the server's contract changes, a documentation set is re-expressed."""
+ about the schema or the server's contract changes, a documentation set is re-expressed.
+
+ From 3.0.0 on the version is left as it is (ADFA-6740): that table holds the one row that says
+ what the file is, and the app and remint_dictionary.py read 3.0.0 as the format they know."""
+ if not templates_compressed(conn):
+ _append_version(conn, stats)
+ conn.execute("INSERT INTO LastChange (documentationSet, who) VALUES (?, ?)",
+ ("android", "ADFA-5548"))
+
+
+def _append_version(conn, stats: dict) -> None:
row = conn.execute(
"SELECT major, minor, patch FROM DocumentationDatabaseVersion "
"ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone()
@@ -300,8 +310,6 @@ def record_version(conn, stats: dict) -> None:
f"Serve the Android reference from JSON through Pebble templates "
f"({stats['updated']:,} rows replaced, {stats['inserted']:,} added)"),
)
- conn.execute("INSERT INTO LastChange (documentationSet, who) VALUES (?, ?)",
- ("android", "ADFA-5548"))
def main(argv: list[str] | None = None) -> int:
@@ -328,8 +336,7 @@ def main(argv: list[str] | None = None) -> int:
source_conn = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True)
try:
missing = check_schema(source_conn)
- # record_version keeps the major, so a 3.x source would come out 3.(x+1) with plain templates in it.
- refusal = None if missing else compressed_templates_refusal(source_conn)
+ refusal = None if missing else unknown_format_refusal(source_conn)
finally:
source_conn.close()
if missing:
diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py
index 0af304b0..c1ee7239 100644
--- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py
+++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py
@@ -12,6 +12,7 @@
import pytest
import load_android_json_db as loader
+from populate_db import DictionaryCompressor, decode_template, load_dictionary # loader put its directory on sys.path
# --------------------------------------------------------------------------------------------
@@ -263,8 +264,9 @@ def test_the_comment_carries_the_counts(self, db):
class TestCompressedTemplates:
- """From documentation 3.0.0 every template is stored compressed and the app decodes every one. record_version
- keeps the source's major, so loading a 3.x source would leave 3.(x+1) with plain Android templates in it."""
+ """From documentation 3.0.0 every template is stored compressed against the dictionary, with no per-row marker,
+ and the app decodes every one (ADFA-6740). A 3.0.0 source loads with compressed templates and stays 3.0.0; a
+ later major, a format this loader does not know, is refused before anything is written."""
def run(self, db, tmp_path, *versions):
db.executescript(VERSION_TABLES)
@@ -276,14 +278,25 @@ def run(self, db, tmp_path, *versions):
return loader.main([str(tmp_path / "json"), "--source", str(tmp_path / "test.db"),
"--out", str(tmp_path / "out.db"), "--workers", "2"])
- @pytest.mark.parametrize("major", [3, 4])
- def test_a_source_declaring_3_or_later_is_refused_before_anything_is_written(self, db, tmp_path, capsys, major):
+ def test_a_source_declaring_a_later_major_is_refused_before_anything_is_written(self, db, tmp_path, capsys):
with pytest.raises(SystemExit) as exited:
- self.run(db, tmp_path, (2, "2026-01-01 00:00:00"), (major, "2026-10-19 00:00:00"))
+ self.run(db, tmp_path, (3, "2026-01-01 00:00:00"), (4, "2026-10-19 00:00:00"))
assert exited.value.code == 2
- assert f"declares documentation version {major}" in capsys.readouterr().err
+ assert "declares documentation version 4" in capsys.readouterr().err
assert not (tmp_path / "out.db").exists()
+ def test_a_source_declaring_3_loads_compressed_templates_and_stays_3_0_0(self, db, tmp_path):
+ assert self.run(db, tmp_path, (3, "2026-10-19 00:00:00")) == 0
+ out = sqlite3.connect(tmp_path / "out.db")
+ try:
+ assert out.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall() == [(3, 0, 0)]
+ with DictionaryCompressor(load_dictionary(out)) as compressor:
+ for name, stored in out.execute("SELECT name, content FROM Templates").fetchall():
+ assert name.startswith("android-")
+ assert "{%" in decode_template(stored, compressor) or "{{" in decode_template(stored, compressor)
+ finally:
+ out.close()
+
def test_a_source_declaring_2_still_loads(self, db, tmp_path):
# The newest row by changeTime is 2, though a 3 was inserted after it.
assert self.run(db, tmp_path, (2, "2026-10-19 00:00:00"), (3, "2026-01-01 00:00:00")) == 0
@@ -291,6 +304,8 @@ def test_a_source_declaring_2_still_loads(self, db, tmp_path):
try:
assert out.execute("SELECT major, minor FROM DocumentationDatabaseVersion "
"ORDER BY rowid DESC LIMIT 1").fetchone() == (2, 1)
- assert out.execute("SELECT COUNT(*) FROM Templates").fetchone() == (3,)
+ templates = out.execute("SELECT content FROM Templates").fetchall()
+ assert len(templates) == 3
+ assert all(decode_template(content, None) for (content,) in templates)
finally:
out.close()
diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py
index fca72836..f12b56a6 100755
--- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py
+++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/verify_android_json_db.py
@@ -27,7 +27,7 @@
/ "ProcessKotlinDocs" / "ProcessKotlinWebsiteJSON")
sys.path.insert(0, str(_KOTLIN_PIPELINE))
-from populate_db import DictionaryCompressor, load_dictionary # noqa: E402
+from populate_db import DictionaryCompressor, decode_template, load_dictionary, templates_compressed # noqa: E402
from migrate_content_to_dictionary_brotli import read_item # noqa: E402
RENDERER_CLASSPATH = (Path(__file__).resolve().parent
@@ -81,7 +81,10 @@ def main(argv: list[str] | None = None) -> int:
conn = sqlite3.connect(args.db)
failures = 0
- templates = {row[0]: row[1].decode("utf-8")
+ compressor = DictionaryCompressor(load_dictionary(conn))
+ # From 3.0.0 every template is compressed against the dictionary (ADFA-6740).
+ template_compressor = compressor if templates_compressed(conn) else None
+ templates = {row[0]: decode_template(row[1], template_compressor)
for row in conn.execute("SELECT id, content FROM Templates").fetchall()}
rows = android_rows(conn)
print(f"==> {len(rows):,} Android pages")
@@ -94,7 +97,6 @@ def main(argv: list[str] | None = None) -> int:
print(f" ! {path}")
kinds: dict[str, int] = {}
- compressor = DictionaryCompressor(load_dictionary(conn))
known = {path for path, _t, _c in rows}
known |= {path for (path,) in conn.execute(
"SELECT path FROM Content WHERE path NOT LIKE '%-1'").fetchall()}
diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py
index e7e58921..6ce6b339 100644
--- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py
+++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py
@@ -267,7 +267,13 @@ def declare_database_version(
lower than the file already declares: rebuilding from an older pipeline
genuinely produces an older format, and the row has to say what the file
contains now rather than the highest it ever contained.
+
+ Except from COMPRESSED_TEMPLATES_MAJOR on: this step writes that format too
+ (upsert_template compresses, and it writes no tooltips), so a 3.x file keeps
+ the version it declares (ADFA-6740).
"""
+ if (declared_major(conn) or 0) >= COMPRESSED_TEMPLATES_MAJOR:
+ return False
conn.execute(VERSION_TABLE_SQL)
declared = conn.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall()
if len(declared) == 1 and tuple(declared[0]) == DATABASE_FORMAT_VERSION:
@@ -321,6 +327,16 @@ def train_dictionary(samples: list, dict_size: int = DEFAULT_DICT_SIZE) -> bytes
shutil.rmtree(work_dir, ignore_errors=True)
+# Brotli's largest RFC 7932 window (see large_window).
+WINDOW_BITS = 24
+
+
+def large_window(stream: bytes) -> bool:
+ """Whether a Brotli stream uses the large-window extension (window over 24 bits), which the CLI decodes and
+ the app's brotli4j refuses. Same test as migrate_tooltip_schema.large_window."""
+ return bool(stream) and stream[0] & 0x7F == 0x11
+
+
class DictionaryCompressor:
"""Compresses/decompresses bytes against a fixed raw Brotli dictionary,
shelling out to the `brotli` CLI (the installed Python `brotli` package
@@ -375,7 +391,9 @@ def _run(self, *extra_args: str, data: bytes) -> bytes:
return result.stdout
def compress(self, data: bytes) -> bytes:
- return self._run(data=data)
+ # -w 24 is the CLI's default for stdin, made explicit as migrate_tooltip_schema.py does: never a
+ # large-window stream, which the app's brotli4j refuses.
+ return self._run("-w", str(WINDOW_BITS), data=data)
def decompress(self, data: bytes) -> bytes:
return self._run("-d", data=data)
@@ -569,9 +587,8 @@ def insert_file(conn, data: bytes, name: str, db_path: str, language_id: int, co
return True
-# From this documentation major on, migrate_tooltip_schema.py (ADFA-3305) has stored every template compressed
-# against the dictionary, and the app decodes every one: a plain template written into such a database would not
-# render, and the file would still declare 3.x.
+# From this documentation major on, every template is stored compressed against the dictionary, with no per-row
+# marker (migrate_tooltip_schema.py, ADFA-3305), and the app decodes every one; below it, plain UTF-8.
COMPRESSED_TEMPLATES_MAJOR = 3
@@ -585,25 +602,55 @@ def declared_major(conn) -> int | None:
return None if row is None else int(row[0])
-def compressed_templates_refusal(conn) -> str | None:
- """Why no plain template may be written into this database, or None."""
+# The newest documentation major this step knows how to write. A later one may store rows in a way it would
+# misread, so it refuses that file before changing anything.
+NEWEST_KNOWN_MAJOR = 3
+
+
+def unknown_format_refusal(conn) -> str | None:
+ """Why this step must not write into this database, or None."""
major = declared_major(conn)
- if major is None or major < COMPRESSED_TEMPLATES_MAJOR:
+ if major is None or major <= NEWEST_KNOWN_MAJOR:
return None
- return (f"the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 every "
- "template is stored compressed, and this step writes plain ones. Run it before migrate_tooltip_schema.py")
+ return (f"the database declares documentation version {major}, and this step writes versions up to "
+ f"{NEWEST_KNOWN_MAJOR}.x")
+
+
+def templates_compressed(conn) -> bool:
+ return (declared_major(conn) or 0) >= COMPRESSED_TEMPLATES_MAJOR
+
+
+def encode_template(conn, content: str) -> bytes:
+ """What Templates stores for `content`: plain UTF-8, or from COMPRESSED_TEMPLATES_MAJOR on, Brotli against
+ the database's dictionary, checked by decoding it back. Raises RuntimeError rather than store a template the
+ app cannot read."""
+ data = content.encode("utf-8")
+ if not templates_compressed(conn):
+ return data
+ with DictionaryCompressor(load_dictionary(conn)) as compressor:
+ stream = compressor.compress(data)
+ if large_window(stream) or compressor.decompress(stream) != data:
+ raise RuntimeError("a compressed template did not decode back to its text")
+ return stream
+
+
+def decode_template(stored: bytes, compressor: "DictionaryCompressor | None") -> str:
+ """A Templates row's text: `compressor` is the database's DictionaryCompressor if its templates are
+ compressed (templates_compressed), else None."""
+ stored = bytes(stored)
+ if compressor is not None:
+ if large_window(stored):
+ raise RuntimeError("a template is a large-window Brotli stream, which the app cannot decode")
+ stored = compressor.decompress(stored)
+ return stored.decode("utf-8")
def upsert_template(conn, name: str, content: str) -> int:
- """Writes one plain template. Raises RuntimeError, writing nothing, if the database's templates are stored
- compressed."""
- refusal = compressed_templates_refusal(conn)
- if refusal:
- raise RuntimeError(refusal)
+ """Writes one template, compressed if the database's templates are (encode_template)."""
conn.execute(
"INSERT INTO Templates (name, content) VALUES (?, ?) "
"ON CONFLICT(name) DO UPDATE SET content = excluded.content",
- (name, content.encode("utf-8")),
+ (name, encode_template(conn, content)),
)
return conn.execute("SELECT id FROM Templates WHERE name = ?", (name,)).fetchone()[0]
@@ -751,7 +798,7 @@ def main():
# Checked before the conversion and the backup, so a refusal costs nothing and writes nothing.
conn = sqlite3.connect(f"file:{args.db_path}?mode=ro", uri=True)
try:
- refusal = compressed_templates_refusal(conn)
+ refusal = unknown_format_refusal(conn)
finally:
conn.close()
if refusal:
@@ -981,11 +1028,11 @@ def main():
# Declared next to the dictionary because the dictionary is what the
# version means (ADFA-5220): the two have to land in the same
# transaction, or a reader can see one without the other.
- version = ".".join(str(part) for part in DATABASE_FORMAT_VERSION)
if declare_database_version(conn):
+ version = ".".join(str(part) for part in DATABASE_FORMAT_VERSION)
print(f"Declared documentation database version {version}", file=sys.stderr)
else:
- print(f"Documentation database already declares version {version}", file=sys.stderr)
+ print(f"Documentation database keeps the version it declares, {declared_major(conn)}.x", file=sys.stderr)
with DictionaryCompressor(dictionary_data) as compressor:
for page, json_bytes in zip(pages, page_json_bytes):
path = f"{page['id']}.html"
diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py
index 774c69d8..ecc1cc65 100644
--- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py
+++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_database_version.py
@@ -53,24 +53,23 @@ def test_an_older_declaration_is_replaced_not_appended(self):
self.assertTrue(declare_database_version(self.conn))
self.assertEqual([row[:3] for row in self.declared()], [DATABASE_FORMAT_VERSION])
- def test_a_downgrade_replaces_a_higher_declaration(self):
- """Rebuilding from an older pipeline really does produce an older format.
- The row has to say what the file contains now, not the highest version it
- ever contained."""
+ def test_a_3x_declaration_is_kept(self):
+ """This step writes schema 3 too (compressed templates, no tooltips), so a
+ 3.0.0 file stays 3.0.0 rather than being declared 2.0.0 (ADFA-6740)."""
self.conn.execute(VERSION_TABLE_SQL)
self.conn.execute(
"INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) "
- "VALUES (9, 0, 0, 'future', 'a format this pipeline does not produce')"
+ "VALUES (3, 0, 0, 'ADFA-3305', 'keyed tooltips')"
)
- self.assertTrue(declare_database_version(self.conn))
- self.assertEqual([row[:3] for row in self.declared()], [DATABASE_FORMAT_VERSION])
+ self.assertFalse(declare_database_version(self.conn))
+ self.assertEqual(self.declared(), [(3, 0, 0, "ADFA-3305")])
def test_several_rows_are_collapsed_to_one(self):
"""A database that picked up extra rows -- an older tool that appended, a
hand-edit -- is repaired rather than read around, so "the version" can
never be ambiguous."""
self.conn.execute(VERSION_TABLE_SQL)
- for major in (1, 2, 3):
+ for major in (1, 2, 1):
self.conn.execute(
"INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) "
"VALUES (?, 0, 0, 'older tool', 'appended')",
diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py
index 8ee4ec21..d8454867 100644
--- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py
+++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py
@@ -1,8 +1,9 @@
#!/usr/bin/env python3
-"""Tests that populate_db.py writes no plain template into a database whose templates are compressed (ADFA-3305).
+"""Tests that populate_db.py writes templates the way the database stores them (ADFA-6740).
-From documentation 3.0.0 migrate_tooltip_schema.py stores every template compressed, and the app decodes every one,
-so a plain page.peb or nav.peb written afterwards would not render while the file still declared 3.x.
+From documentation 3.0.0 every template is stored Brotli-compressed against CompressionDictionary, with no per-row
+marker, and the app decodes every one; below it they are plain. A 3.0.0 file stays 3.0.0, and a later major, a
+format this step does not know, is refused before anything is written.
Run directly: python3 test_populate_db_templates.py
"""
@@ -15,7 +16,8 @@
import zipfile
from pathlib import Path
-from populate_db import compressed_templates_refusal, declared_major, upsert_template
+from populate_db import (DictionaryCompressor, declared_major, decode_template, large_window, templates_compressed,
+ unknown_format_refusal, upsert_template)
HERE = Path(__file__).resolve().parent
@@ -36,7 +38,7 @@
"""
-class CompressedTemplatesRefusalTest(unittest.TestCase):
+class TemplatesTest(unittest.TestCase):
def setUp(self):
directory = tempfile.TemporaryDirectory()
self.addCleanup(directory.cleanup)
@@ -62,23 +64,28 @@ def connect(self):
def templates(self) -> list:
return self.connect().execute("SELECT name, content FROM Templates ORDER BY id").fetchall()
- def test_upsert_template_refuses_major_3(self):
+ def compressor(self):
+ dictionary = self.connect().execute("SELECT data FROM CompressionDictionary").fetchone()[0]
+ compressor = DictionaryCompressor(dictionary)
+ self.addCleanup(compressor.close)
+ return compressor
+
+ def test_upsert_template_compresses_at_major_3(self):
self.declare((2, "2026-01-01 00:00:00"), (3, "2026-10-19 00:00:00"))
- before = self.templates()
conn = self.connect()
- with self.assertRaises(RuntimeError) as raised:
- upsert_template(conn, "page.peb", "new")
- self.assertIn("declares documentation version 3", str(raised.exception))
- self.assertIn("Run it before migrate_tooltip_schema.py", str(raised.exception))
+ upsert_template(conn, "page.peb", "new \u00e9t\u00e9")
conn.commit()
- self.assertEqual(self.templates(), before)
+ (name, stored), = self.templates()
+ self.assertEqual(name, "page.peb")
+ self.assertFalse(large_window(stored))
+ self.assertEqual(self.compressor().decompress(stored), "new \u00e9t\u00e9".encode())
+ self.assertEqual(decode_template(stored, self.compressor()), "new \u00e9t\u00e9")
- def test_upsert_template_refuses_a_later_major(self):
- self.declare((4, "2026-10-19 00:00:00"))
+ def test_decode_template_refuses_a_large_window_stream(self):
with self.assertRaises(RuntimeError):
- upsert_template(self.connect(), "nav.peb", "")
+ decode_template(b"\x11\x00", self.compressor())
- def test_upsert_template_writes_at_major_2_and_without_a_version(self):
+ def test_upsert_template_writes_plain_at_major_2_and_without_a_version(self):
conn = self.connect()
upsert_template(conn, "nav.peb", "")
conn.commit()
@@ -86,35 +93,59 @@ def test_upsert_template_writes_at_major_2_and_without_a_version(self):
upsert_template(conn, "page.peb", "new")
conn.commit()
self.assertEqual(self.templates(), [("page.peb", b"new"), ("nav.peb", b"")])
+ self.assertEqual(decode_template(b"", None), "")
def test_reads_the_newest_version_row_as_the_app_does(self):
# Newest changeTime first, whatever the insert order; then the newest rowid.
self.declare((3, "2026-10-19 00:00:00"), (2, "2026-01-01 00:00:00"))
self.assertEqual(declared_major(self.connect()), 3)
+ self.assertTrue(templates_compressed(self.connect()))
self.declare((2, "2026-10-19 00:00:00"))
self.assertEqual(declared_major(self.connect()), 2)
- self.assertIsNone(compressed_templates_refusal(self.connect()))
+ self.assertFalse(templates_compressed(self.connect()))
- def test_command_line_refuses_before_converting_or_backing_up(self):
+ def test_only_a_later_major_is_refused(self):
+ self.assertIsNone(unknown_format_refusal(self.connect()))
self.declare((3, "2026-10-19 00:00:00"))
+ self.assertIsNone(unknown_format_refusal(self.connect()))
+ self.declare((4, "2026-10-20 00:00:00"))
+ self.assertIn("declares documentation version 4", unknown_format_refusal(self.connect()))
+
+ def populate(self) -> subprocess.CompletedProcess:
docs = self.dir / "docs"
(docs / "topics").mkdir(parents=True)
- (docs / "topics" / "home.md").write_text("# Home\n\nText.\n", encoding="utf-8")
+ (docs / "topics" / "basics.md").write_text("# Basics\n\nText.\n", encoding="utf-8")
(docs / "kr.tree").write_text(''
- '', encoding="utf-8")
+ '', encoding="utf-8")
config = self.dir / "config.json"
config.write_text('{"broken-ext-link-color": "#c00", "menu-no-link-color": "#888"}', encoding="utf-8")
images = self.dir / "images.zip"
zipfile.ZipFile(images, "w").close()
+ return subprocess.run([sys.executable, str(HERE / "populate_db.py"), str(docs), str(config), str(images),
+ str(self.db_path)], capture_output=True, text=True, cwd=self.dir, check=False)
+
+ def test_command_line_refuses_a_later_major_before_converting_or_backing_up(self):
+ self.declare((4, "2026-10-19 00:00:00"))
sha = hashlib.sha256(self.db_path.read_bytes()).hexdigest()
- result = subprocess.run([sys.executable, str(HERE / "populate_db.py"), str(docs), str(config), str(images),
- str(self.db_path)], capture_output=True, text=True, cwd=self.dir, check=False)
+ result = self.populate()
self.assertEqual(result.returncode, 1, result.stderr)
- self.assertIn("Run it before migrate_tooltip_schema.py; nothing was changed", result.stderr)
+ self.assertIn("writes versions up to 3.x; nothing was changed", result.stderr)
self.assertNotIn("Backing up", result.stderr)
self.assertEqual(hashlib.sha256(self.db_path.read_bytes()).hexdigest(), sha)
self.assertEqual(sorted(p.name for p in self.dir.glob("documentation.db*")), ["documentation.db"])
+ def test_command_line_on_a_3_0_0_database_keeps_it_3_0_0(self):
+ self.declare((3, "2026-10-19 00:00:00"))
+ result = self.populate()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ conn = self.connect()
+ self.assertEqual(conn.execute("SELECT major, minor, patch FROM DocumentationDatabaseVersion").fetchall(),
+ [(3, 0, 0)])
+ compressor = self.compressor()
+ texts = {name: decode_template(stored, compressor) for name, stored in self.templates()}
+ self.assertEqual(sorted(texts), ["nav.peb", "page.peb"])
+ self.assertIn("{%", texts["nav.peb"])
+
if __name__ == "__main__":
unittest.main()
diff --git a/docdb-studio/CLAUDE.md b/docdb-studio/CLAUDE.md
index 84f85617..d688e5df 100644
--- a/docdb-studio/CLAUDE.md
+++ b/docdb-studio/CLAUDE.md
@@ -18,7 +18,7 @@ Dependencies are managed with `uv`. Always use `uv run` rather than activating a
This is a Flet (Flutter-based) GUI tool over a SQLite database (`documentation.db`, ~380 MB) that holds an IDE's tooltip / help-content data. There are two apps:
-- `docdb_studio.py` — full CRUD over the `Tooltips` / `TooltipCategories` / `TooltipButtons` tables, with paginated browse, search, view, edit, single-insert, and CSV bulk-import flows. ~1600 lines, single file by design.
+- `docdb_studio.py` — full CRUD over the tooltip tables, with paginated browse, search, view, edit, single-insert, and CSV bulk-import flows, plus a Templates editor. Single file by design.
- `db-summary.py` — read-only window listing every user table and its row count.
### Schema is locked
@@ -28,6 +28,7 @@ Per `AGENTS.md`: **never change the schema.** The full schema is in `SCHEMA.md`.
- `Tooltips.categoryId` → `TooltipCategories.id`
- `TooltipButtons.tooltipId` → `Tooltips.id`, `TooltipButtons.buttonNumberId` → `TooltipButtonNumbers.id` (button-number IDs are manually assigned to control display order)
- Tooltip uniqueness is `(categoryId, tag)`
+- Two shapes, chosen by the declared major (`_is_keyed`, ADFA-6740). Schema 2: the tables above, plain templates. Schema 3: `Tooltips(id, categoryId, tag)` keyed on `(categoryId, tag)`, text in `TooltipTexts` (a new tooltip inserts its text first and takes that id), button strings in `TooltipButtonDescriptions` / `TooltipButtonUris` (deduped; unused ones are deleted after every button change), and every `Templates` row Brotli-compressed against `CompressionDictionary`. Read SQL names `{tooltips}` / `{buttons}` and goes through `_tooltip_sql`, which substitutes 2.x-shaped subqueries on schema 3; writes go through `_insert_tooltip_row` / `_update_tooltip_text` / `_insert_button_row`.
- Button URIs are validated against `Content.path` after stripping `?query` and `#fragment` (see `_uri_path_for_content_lookup` / `uris_exist_in_content`)
- Every mutation should call `update_last_change(db, documentation_set, user)` so the `LastChange` table reflects who edited what and when. The `--user` CLI arg feeds this.
diff --git a/docdb-studio/README.md b/docdb-studio/README.md
index 0061b988..6e65b810 100644
--- a/docdb-studio/README.md
+++ b/docdb-studio/README.md
@@ -83,11 +83,11 @@ These same commands work on macOS, Linux, and Windows (in PowerShell, Command Pr
## Database versions it will open
-Databases declare a format version in `DocumentationDatabaseVersion` (ADFA-5220). docdb-studio understands MAJOR **2** and **refuses to open anything higher**, showing what it found and what it expected instead of the browser.
+Databases declare a format version in `DocumentationDatabaseVersion` (ADFA-5220). docdb-studio understands MAJOR **2** and **3** and **refuses to open anything higher**, showing what it found and what it expected instead of the browser.
That refusal is not caution for its own sake: a higher MAJOR means the format changed in a way this build would read incorrectly, and because this tool *writes*, a misreading gets saved back into the file. Update docdb-studio when you meet one.
-An older version, or none at all, opens normally. Those formats are strictly simpler -- no shared compression dictionary -- and are read correctly by the same fallback that handles plain-Brotli rows.
+From MAJOR 3 (ADFA-6740) tooltip text lives in `TooltipTexts`, button strings in `TooltipButtonDescriptions` / `TooltipButtonUris`, and every `Templates` row is Brotli-compressed against the shared dictionary; docdb-studio reads and writes that shape. MAJOR 2, an older version, or none at all opens with the original tables and plain templates.
One dependency `uv sync` cannot install for you. Databases built since ADFA-5153 compress their `Content` rows against a shared dictionary stored inside the database itself, and no Python library exposes a custom dictionary, so docdb-studio runs the `brotli` **command-line program** to read and write those rows. If it is missing, content rows come up blank and docdb-studio prints an explanatory error in the terminal you launched it from — so a blank preview plus that message means "install this tool", not "this page is empty". A database with no `CompressionDictionary` table needs nothing extra.
@@ -248,11 +248,14 @@ The toolbar's orange **Add** button opens a chooser:
- **Import content folder** — pick a local folder; every file is brotli-compressed (when its `ContentTypes.compression` says so), split into 1 MiB chunks if needed, and inserted into `Content`. Files with unknown extensions are skipped and reported. The folder's basename is recorded in `LastChange` as `content-`.
### Validate URIs
-The toolbar's blue **Validate URIs** button audits every `TooltipButtons.uri` against `Content.path` (with `?query` and `#fragment` stripped). Broken URIs are listed with one-click **Edit** access. After saving or cancelling, you return to the same audit page — same pagination position included.
+The toolbar's blue **Validate URIs** button audits every tooltip button URI against `Content.path` (with `?query` and `#fragment` stripped). Broken URIs are listed with one-click **Edit** access. After saving or cancelling, you return to the same audit page — same pagination position included.
### Browse content
The toolbar's blue **Browse content** button opens a Finder-style horizontally-scrolling column view of every `Content.path`. Folders show item counts and aggregate byte sizes; HTML files show inbound tooltip-button reference counts. Selecting an HTML file shows the list of referencing tooltips with per-row View / Edit buttons. Toggle between alphabetical and size-descending sort at any time.
+### Templates
+The toolbar's blue **Templates** button lists the `Templates` rows (the Pebble templates the app renders pages through), filterable by name. Click one to edit its source and **Save**; in a MAJOR 3 database it is decoded and re-compressed against the shared dictionary, so it needs the `brotli` command-line tool.
+
### Edit page niceties
- Live ✓ / ✗ / ? icon next to each URI input.
- **Browse…** button next to each URI input opens a searchable picker over all `Content.path` values.
@@ -270,4 +273,4 @@ docdb-studio/
└── pyproject.toml # project metadata + dependencies
```
-The database schema is documented in `SCHEMA.md` and is treated as immutable — the tool performs no migrations.
+The database schema is documented in `SCHEMA.md`. The tool performs no migrations; it reads and writes whichever shape the database declares.
diff --git a/docdb-studio/SCHEMA.md b/docdb-studio/SCHEMA.md
index d26a682e..845eef78 100644
--- a/docdb-studio/SCHEMA.md
+++ b/docdb-studio/SCHEMA.md
@@ -167,3 +167,40 @@ CREATE TRIGGER DeleteBook AFTER DELETE ON Content WHEN OLD.path LIKE '%.pdf'
BEGIN
DELETE FROM Bookshelf WHERE contentID = OLD.id;
END;
+
+-- Schema 3.0.0 (ADFA-3305, scripts/migrate_tooltip_schema.py; read and written since ADFA-6740).
+-- These replace Tooltips and TooltipButtons above; every Templates.content is then
+-- Brotli-compressed against CompressionDictionary, with no per-row marker. Dumped from
+-- the 2026-10-23 3.0.0 database the same way as above.
+CREATE TABLE TooltipButtonDescriptions (
+ id INTEGER PRIMARY KEY,
+ description TEXT NOT NULL
+);
+CREATE TABLE TooltipButtonUris (
+ id INTEGER PRIMARY KEY,
+ uri TEXT NOT NULL
+);
+CREATE TABLE TooltipButtons (
+ tooltipId INTEGER NOT NULL,
+ buttonNumberId INTEGER NOT NULL,
+ descriptionId INTEGER NOT NULL,
+ uriId INTEGER NOT NULL,
+ PRIMARY KEY (tooltipId, buttonNumberId),
+ FOREIGN KEY(tooltipId) REFERENCES TooltipTexts(id),
+ FOREIGN KEY(buttonNumberId) REFERENCES TooltipButtonNumbers(id),
+ FOREIGN KEY(descriptionId) REFERENCES TooltipButtonDescriptions(id),
+ FOREIGN KEY(uriId) REFERENCES TooltipButtonUris(id)
+) WITHOUT ROWID;
+CREATE TABLE TooltipTexts (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ summary TEXT NOT NULL,
+ detail TEXT NOT NULL
+);
+CREATE TABLE Tooltips (
+ id INTEGER NOT NULL,
+ categoryId INTEGER NOT NULL,
+ tag TEXT NOT NULL,
+ PRIMARY KEY (categoryId, tag),
+ FOREIGN KEY(categoryId) REFERENCES TooltipCategories(id),
+ FOREIGN KEY(id) REFERENCES TooltipTexts(id)
+) WITHOUT ROWID;
diff --git a/docdb-studio/docdb_studio.py b/docdb-studio/docdb_studio.py
index 5be44109..331f22f2 100644
--- a/docdb-studio/docdb_studio.py
+++ b/docdb-studio/docdb_studio.py
@@ -61,16 +61,31 @@
ZEBRA_EVEN = ft.Colors.SURFACE_CONTAINER_LOWEST
ZEBRA_ODD = ft.Colors.SURFACE_CONTAINER_LOW
+# Tooltip SQL names its tables {tooltips} and {buttons}; _tooltip_sql fills them in
+# for the database's shape. In schema 2 they are the tables themselves. In schema 3
+# (ADFA-6740, scripts/migrate_tooltip_schema.py) the text lives in TooltipTexts and the
+# button strings in two lookup tables, so each becomes a subquery with the 2.x columns.
+V3_TOOLTIPS = (
+ "(SELECT T.id AS id, T.categoryId AS categoryId, T.tag AS tag, X.summary AS summary,"
+ " X.detail AS detail FROM Tooltips T JOIN TooltipTexts X ON X.id = T.id)"
+)
+V3_BUTTONS = (
+ "(SELECT B.tooltipId AS tooltipId, B.buttonNumberId AS buttonNumberId,"
+ " D.description AS description, U.uri AS uri FROM TooltipButtons B"
+ " JOIN TooltipButtonDescriptions D ON D.id = B.descriptionId"
+ " JOIN TooltipButtonUris U ON U.id = B.uriId)"
+)
+
JOIN_BASE = """
SELECT t.id, tc.category, t.tag, t.summary, t.detail
-FROM Tooltips t
+FROM {tooltips} t
JOIN TooltipCategories tc ON t.categoryId = tc.id
"""
JOIN_ORDER = " ORDER BY tc.category, t.tag"
COUNT_BASE = """
SELECT COUNT(*)
-FROM Tooltips t
+FROM {tooltips} t
JOIN TooltipCategories tc ON t.categoryId = tc.id
"""
@@ -82,7 +97,7 @@
# Non-correlated subquery: SQLite evaluates it once (not per Tooltips row),
# so URI/description search stays a single TooltipButtons scan instead of a
# per-row nested loop. IN also keeps each tooltip to one result row.
- " OR t.id IN (SELECT tb.tooltipId FROM TooltipButtons tb"
+ " OR t.id IN (SELECT tb.tooltipId FROM {buttons} tb"
" WHERE tb.uri LIKE ? ESCAPE '\\' OR tb.description LIKE ? ESCAPE '\\'))"
)
@@ -169,10 +184,12 @@ def _search_params(term: str) -> tuple[str, ...]:
# in a way this code would misread -- and this tool *writes*, so misreading means
# writing damage back. Refuse it instead.
#
-# A lower or absent version is fine and is not refused: those formats are
-# strictly simpler (no shared dictionary, plain Brotli) and decompress_brotli's
-# fallback already reads them.
-SUPPORTED_DATABASE_MAJOR_VERSION = 2
+# A lower or absent version is not refused. From KEYED_SCHEMA_MAJOR_VERSION on the
+# tooltips are in the keyed tables and every Templates row is Brotli-compressed against
+# CompressionDictionary, with no per-row marker (ADFA-6740); below it, the 2.x tables
+# and plain templates. Both are read and written.
+SUPPORTED_DATABASE_MAJOR_VERSION = 3
+KEYED_SCHEMA_MAJOR_VERSION = 3
def database_major_version(db_path: Path) -> int | None:
@@ -186,15 +203,36 @@ def database_major_version(db_path: Path) -> int | None:
DatabaseVersionResolver, and populate_db repairs the file when it sees this.
"""
with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:
- table = conn.execute(
- "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'DocumentationDatabaseVersion'"
- ).fetchone()
- if table is None:
- return None
- row = conn.execute(
- "SELECT major FROM DocumentationDatabaseVersion ORDER BY rowid DESC LIMIT 1"
- ).fetchone()
- return row[0] if row is not None and row[0] is not None else None
+ return _declared_major(conn)
+
+
+def _declared_major(conn: sqlite3.Connection) -> int | None:
+ table = conn.execute(
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'DocumentationDatabaseVersion'"
+ ).fetchone()
+ if table is None:
+ return None
+ # Newest by changeTime, then rowid: the order the app, migrate_tooltip_schema.py
+ # and remint_dictionary.py read it in.
+ row = conn.execute(
+ "SELECT major FROM DocumentationDatabaseVersion ORDER BY changeTime DESC, rowid DESC LIMIT 1"
+ ).fetchone()
+ return row[0] if row is not None and row[0] is not None else None
+
+
+def _is_keyed(conn: sqlite3.Connection) -> bool:
+ """Whether the database declares schema 3 or later: keyed tooltip tables and
+ compressed templates (ADFA-6740)."""
+ major = _declared_major(conn)
+ return major is not None and major >= KEYED_SCHEMA_MAJOR_VERSION
+
+
+def _tooltip_sql(conn: sqlite3.Connection, sql: str) -> str:
+ """`sql` with {tooltips} and {buttons} filled in for this database's shape."""
+ keyed = _is_keyed(conn)
+ return sql.replace("{tooltips}", V3_TOOLTIPS if keyed else "Tooltips").replace(
+ "{buttons}", V3_BUTTONS if keyed else "TooltipButtons"
+ )
def unsupported_version_error(db_path: Path, major: int) -> str:
@@ -212,7 +250,7 @@ def get_total_count(db_path: Path, search_term: str | None = None) -> int:
sql = COUNT_BASE.rstrip() + (SEARCH_WHERE if search_term else "")
params = _search_params(search_term) if search_term else ()
with sqlite3.connect(db_path) as conn:
- cur = conn.execute(sql, params)
+ cur = conn.execute(_tooltip_sql(conn, sql), params)
return cur.fetchone()[0]
@@ -227,7 +265,7 @@ def get_page(
)
params = (_search_params(search_term) + (limit, offset)) if search_term else (limit, offset)
with sqlite3.connect(db_path) as conn:
- cur = conn.execute(sql, params)
+ cur = conn.execute(_tooltip_sql(conn, sql), params)
return cur.fetchall()
@@ -237,12 +275,12 @@ def get_tooltip_by_id(
"""Return (category, tag, summary, detail) for the tooltip id, or None."""
sql = """
SELECT tc.category, t.tag, t.summary, t.detail
- FROM Tooltips t
+ FROM {tooltips} t
JOIN TooltipCategories tc ON t.categoryId = tc.id
WHERE t.id = ?
"""
with sqlite3.connect(db_path) as conn:
- cur = conn.execute(sql, (tooltip_id,))
+ cur = conn.execute(_tooltip_sql(conn, sql), (tooltip_id,))
row = cur.fetchone()
return tuple(row) if row else None
@@ -253,12 +291,12 @@ def get_tooltip_buttons(
"""Return rows (buttonNumberId, description, uri) for the tooltip, sorted by buttonNumberId."""
sql = """
SELECT buttonNumberId, description, uri
- FROM TooltipButtons
+ FROM {buttons} tb
WHERE tooltipId = ?
ORDER BY buttonNumberId
"""
with sqlite3.connect(db_path) as conn:
- cur = conn.execute(sql, (tooltip_id,))
+ cur = conn.execute(_tooltip_sql(conn, sql), (tooltip_id,))
return cur.fetchall()
@@ -476,14 +514,14 @@ def get_all_button_rows(
"""Return (tooltip_id, category, tag, button_number_id, description, uri) for every non-empty TooltipButton."""
sql = """
SELECT t.id, tc.category, t.tag, tb.buttonNumberId, tb.description, tb.uri
- FROM TooltipButtons tb
- JOIN Tooltips t ON tb.tooltipId = t.id
+ FROM {buttons} tb
+ JOIN {tooltips} t ON tb.tooltipId = t.id
JOIN TooltipCategories tc ON t.categoryId = tc.id
WHERE tb.uri IS NOT NULL AND tb.uri != ''
ORDER BY tc.category, t.tag, tb.buttonNumberId
"""
with sqlite3.connect(db_path) as conn:
- cur = conn.execute(sql)
+ cur = conn.execute(_tooltip_sql(conn, sql))
return cur.fetchall()
@@ -511,11 +549,11 @@ def get_tooltip_export_rows(
)
params = _search_params(search_term) if search_term else ()
with sqlite3.connect(db_path) as conn:
- tooltips = conn.execute(tooltip_sql, params).fetchall()
+ tooltips = conn.execute(_tooltip_sql(conn, tooltip_sql), params).fetchall()
# One pass over TooltipButtons, grouped by tooltip -> {buttonNumberId: (label, uri)}.
buttons_by_tooltip: dict[int, dict[int, tuple[str, str]]] = {}
for tid, bnum, desc, uri in conn.execute(
- "SELECT tooltipId, buttonNumberId, description, uri FROM TooltipButtons"
+ _tooltip_sql(conn, "SELECT tooltipId, buttonNumberId, description, uri FROM {buttons} tb")
):
buttons_by_tooltip.setdefault(tid, {})[bnum] = (desc or "", uri or "")
@@ -555,7 +593,7 @@ def find_empty_summary_tooltips(db_path: Path) -> list[tuple[int, str]]:
CSV export skips them; the validate page surfaces them for fixing."""
with sqlite3.connect(db_path) as conn:
cur = conn.execute(
- "SELECT id, tag FROM Tooltips WHERE TRIM(summary) = '' ORDER BY tag"
+ _tooltip_sql(conn, "SELECT id, tag FROM {tooltips} t WHERE TRIM(summary) = '' ORDER BY tag")
)
return [(row[0], row[1] or "") for row in cur.fetchall()]
@@ -647,7 +685,7 @@ def get_button_reference_counts(db_path: Path) -> dict[str, int]:
counts: dict[str, int] = {}
with sqlite3.connect(db_path) as conn:
cur = conn.execute(
- "SELECT uri FROM TooltipButtons WHERE uri IS NOT NULL AND uri != ''"
+ _tooltip_sql(conn, "SELECT uri FROM {buttons} tb WHERE uri IS NOT NULL AND uri != ''")
)
for (uri,) in cur.fetchall():
stripped = _uri_path_for_content_lookup(uri or "")
@@ -663,12 +701,12 @@ def get_tooltip_for_edit(
"""Return (category_id, category, tag, summary, detail) for the tooltip id, or None."""
sql = """
SELECT t.categoryId, tc.category, t.tag, t.summary, t.detail
- FROM Tooltips t
+ FROM {tooltips} t
JOIN TooltipCategories tc ON t.categoryId = tc.id
WHERE t.id = ?
"""
with sqlite3.connect(db_path) as conn:
- cur = conn.execute(sql, (tooltip_id,))
+ cur = conn.execute(_tooltip_sql(conn, sql), (tooltip_id,))
row = cur.fetchone()
return tuple(row) if row else None
@@ -771,6 +809,7 @@ def delete_tooltips_bulk(db_path: Path, tooltip_ids: list[int]) -> int:
return 0
deleted = 0
with sqlite3.connect(db_path) as conn:
+ keyed = _is_keyed(conn)
for batch in _chunked(tooltip_ids, SQL_PARAM_BATCH):
placeholders = ",".join("?" * len(batch))
conn.execute(
@@ -782,6 +821,9 @@ def delete_tooltips_bulk(db_path: Path, tooltip_ids: list[int]) -> int:
batch,
)
deleted += cur.rowcount
+ if keyed:
+ conn.execute(f"DELETE FROM TooltipTexts WHERE id IN ({placeholders})", batch)
+ _drop_unused_button_strings(conn, keyed)
conn.commit()
if deleted:
vacuum_database(db_path)
@@ -916,15 +958,77 @@ def insert_tooltip(
summary = sanitize_text(summary, field="summary")
detail = sanitize_text(detail, field="detail", allow_newline=True)
with sqlite3.connect(db_path) as conn:
+ tooltip_id = _insert_tooltip_row(conn, _is_keyed(conn), category_id, tag, summary, detail)
+ conn.commit()
+ return tooltip_id
+
+
+def _insert_tooltip_row(
+ conn: sqlite3.Connection, keyed: bool, category_id: int, tag: str, summary: str, detail: str
+) -> int:
+ """Insert one tooltip and return its id. In schema 3 the text goes in first and
+ Tooltips takes the id TooltipTexts hands out, never one of its own (ADFA-6740)."""
+ if not keyed:
cur = conn.execute(
- """
- INSERT INTO Tooltips (categoryId, tag, summary, detail)
- VALUES (?, ?, ?, ?)
- """,
+ "INSERT INTO Tooltips (categoryId, tag, summary, detail) VALUES (?, ?, ?, ?)",
(category_id, tag, summary, detail),
)
- conn.commit()
return cur.lastrowid or 0
+ tooltip_id = conn.execute(
+ "INSERT INTO TooltipTexts (summary, detail) VALUES (?, ?)", (summary, detail)
+ ).lastrowid
+ conn.execute(
+ "INSERT INTO Tooltips (id, categoryId, tag) VALUES (?, ?, ?)",
+ (tooltip_id, category_id, tag),
+ )
+ return tooltip_id
+
+
+def _update_tooltip_text(
+ conn: sqlite3.Connection, keyed: bool, tooltip_id: int, summary: str, detail: str
+) -> None:
+ table = "TooltipTexts" if keyed else "Tooltips"
+ conn.execute(f"UPDATE {table} SET summary = ?, detail = ? WHERE id = ?", (summary, detail, tooltip_id))
+
+
+def _insert_button_row(
+ conn: sqlite3.Connection, keyed: bool, tooltip_id: int, button_number_id: int, description: str, uri: str
+) -> None:
+ if not keyed:
+ conn.execute(
+ "INSERT INTO TooltipButtons (tooltipId, buttonNumberId, description, uri) VALUES (?, ?, ?, ?)",
+ (tooltip_id, button_number_id, description, uri),
+ )
+ return
+ conn.execute(
+ "INSERT INTO TooltipButtons (tooltipId, buttonNumberId, descriptionId, uriId) VALUES (?, ?, ?, ?)",
+ (
+ tooltip_id,
+ button_number_id,
+ _button_string_id(conn, "TooltipButtonDescriptions", "description", description),
+ _button_string_id(conn, "TooltipButtonUris", "uri", uri),
+ ),
+ )
+
+
+def _button_string_id(conn: sqlite3.Connection, table: str, column: str, value: str) -> int:
+ """The id of `value` in a schema 3 button lookup table, adding it if absent, so
+ each string is stored once."""
+ # ponytail: the text column has no index, so this scans the table (~20k URIs, ~1 ms);
+ # an index would cost the space schema 3 saves.
+ row = conn.execute(f"SELECT id FROM {table} WHERE {column} = ? LIMIT 1", (value,)).fetchone()
+ if row is not None:
+ return row[0]
+ return conn.execute(f"INSERT INTO {table} ({column}) VALUES (?)", (value,)).lastrowid
+
+
+def _drop_unused_button_strings(conn: sqlite3.Connection, keyed: bool) -> None:
+ """Delete schema 3 button descriptions and URIs no button uses any more."""
+ if keyed:
+ conn.execute(
+ "DELETE FROM TooltipButtonDescriptions WHERE id NOT IN (SELECT descriptionId FROM TooltipButtons)"
+ )
+ conn.execute("DELETE FROM TooltipButtonUris WHERE id NOT IN (SELECT uriId FROM TooltipButtons)")
def update_tooltip(
@@ -941,13 +1045,10 @@ def update_tooltip(
detail = sanitize_text(detail, field="detail", allow_newline=True)
with sqlite3.connect(db_path) as conn:
conn.execute(
- """
- UPDATE Tooltips
- SET categoryId = ?, tag = ?, summary = ?, detail = ?
- WHERE id = ?
- """,
- (category_id, tag, summary, detail, tooltip_id),
+ "UPDATE Tooltips SET categoryId = ?, tag = ? WHERE id = ?",
+ (category_id, tag, tooltip_id),
)
+ _update_tooltip_text(conn, _is_keyed(conn), tooltip_id, summary, detail)
conn.commit()
@@ -958,49 +1059,6 @@ def get_button_number_ids(db_path: Path) -> list[int]:
return [row[0] for row in cur.fetchall()]
-def add_tooltip_button(
- db_path: Path,
- tooltip_id: int,
- button_number_id: int,
- description: str,
- uri: str,
-) -> None:
- """Insert a row into TooltipButtons."""
- description = sanitize_text(description.strip(), field="button description")
- uri = sanitize_text(uri.strip(), field="button uri")
- with sqlite3.connect(db_path) as conn:
- conn.execute(
- """
- INSERT INTO TooltipButtons (tooltipId, buttonNumberId, description, uri)
- VALUES (?, ?, ?, ?)
- """,
- (tooltip_id, button_number_id, description, uri),
- )
- conn.commit()
-
-
-def update_tooltip_button(
- db_path: Path,
- tooltip_id: int,
- button_number_id: int,
- description: str,
- uri: str,
-) -> None:
- """Update description and uri for one TooltipButtons row."""
- description = sanitize_text(description.strip(), field="button description")
- uri = sanitize_text(uri.strip(), field="button uri")
- with sqlite3.connect(db_path) as conn:
- conn.execute(
- """
- UPDATE TooltipButtons
- SET description = ?, uri = ?
- WHERE tooltipId = ? AND buttonNumberId = ?
- """,
- (description, uri, tooltip_id, button_number_id),
- )
- conn.commit()
-
-
def delete_tooltip_button(
db_path: Path, tooltip_id: int, button_number_id: int
) -> None:
@@ -1013,6 +1071,7 @@ def delete_tooltip_button(
""",
(tooltip_id, button_number_id),
)
+ _drop_unused_button_strings(conn, _is_keyed(conn))
conn.commit()
vacuum_database(db_path)
@@ -1033,18 +1092,14 @@ def replace_tooltip_buttons(
)
)
with sqlite3.connect(db_path) as conn:
+ keyed = _is_keyed(conn)
conn.execute(
"DELETE FROM TooltipButtons WHERE tooltipId = ?",
(tooltip_id,),
)
for button_number_id, description, uri in cleaned:
- conn.execute(
- """
- INSERT INTO TooltipButtons (tooltipId, buttonNumberId, description, uri)
- VALUES (?, ?, ?, ?)
- """,
- (tooltip_id, button_number_id, description, uri),
- )
+ _insert_button_row(conn, keyed, tooltip_id, button_number_id, description, uri)
+ _drop_unused_button_strings(conn, keyed)
conn.commit()
vacuum_database(db_path)
@@ -1216,6 +1271,7 @@ def import_csv_rows(
inserted = 0
updated = 0
with sqlite3.connect(db_path) as conn:
+ keyed = _is_keyed(conn)
for row in rows:
(
category_name,
@@ -1239,28 +1295,14 @@ def import_csv_rows(
).fetchone()
if existing is not None:
tooltip_id = existing[0]
- conn.execute(
- """
- UPDATE Tooltips
- SET summary = ?, detail = ?
- WHERE id = ?
- """,
- (summary, detail, tooltip_id),
- )
+ _update_tooltip_text(conn, keyed, tooltip_id, summary, detail)
conn.execute(
"DELETE FROM TooltipButtons WHERE tooltipId = ?",
(tooltip_id,),
)
updated += 1
if tooltip_id == 0:
- cur = conn.execute(
- """
- INSERT INTO Tooltips (categoryId, tag, summary, detail)
- VALUES (?, ?, ?, ?)
- """,
- (cid, tag, summary, detail),
- )
- tooltip_id = cur.lastrowid or 0
+ tooltip_id = _insert_tooltip_row(conn, keyed, cid, tag, summary, detail)
inserted += 1
for bid, label, uri in [
(1, b1_label, b1_uri),
@@ -1268,13 +1310,8 @@ def import_csv_rows(
(3, b3_label, b3_uri),
]:
if label and uri:
- conn.execute(
- """
- INSERT INTO TooltipButtons (tooltipId, buttonNumberId, description, uri)
- VALUES (?, ?, ?, ?)
- """,
- (tooltip_id, bid, label, uri),
- )
+ _insert_button_row(conn, keyed, tooltip_id, bid, label, uri)
+ _drop_unused_button_strings(conn, keyed)
conn.commit()
if updated:
vacuum_database(db_path)
@@ -1538,6 +1575,66 @@ def decompress_brotli(data: bytes, db_path: Path) -> bytes:
return brotli.decompress(data)
+def _template_brotli(db_path: Path, data: bytes, *args: str) -> bytes:
+ """Run the brotli CLI against db_path's dictionary, as migrate_tooltip_schema.py's
+ DictionaryCodec does. Templates have no plain fallback: in schema 3 every one is
+ compressed (ADFA-6740)."""
+ dictionary_data = get_compression_dictionary(db_path, strict=True)
+ if dictionary_data is None:
+ raise RuntimeError(f"{db_path} declares schema 3 but has no CompressionDictionary to decode its templates")
+ dict_path = _dictionary_temp_path(db_path, dictionary_data)
+ result = subprocess.run(
+ [_find_brotli_cli(), "-D", str(dict_path), *args, "-c"],
+ input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(f"brotli failed on a template: {result.stderr.decode(errors='replace').strip()}")
+ return result.stdout
+
+
+def _large_window(stream: bytes) -> bool:
+ """A large-window Brotli stream (window over 24 bits), which the brotli CLI decodes
+ and the app's brotli4j refuses. Same test as migrate_tooltip_schema.large_window."""
+ return bool(stream) and stream[0] & 0x7F == 0x11
+
+
+def get_templates(db_path: Path) -> list[tuple[int, str]]:
+ """(id, name) for every Templates row, by name; empty if there is no Templates table."""
+ with sqlite3.connect(db_path) as conn:
+ if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'Templates'").fetchone():
+ return []
+ return conn.execute("SELECT id, name FROM Templates ORDER BY name").fetchall()
+
+
+def get_template_text(db_path: Path, template_id: int) -> str:
+ """The template's Pebble source, decoded from the dictionary in schema 3."""
+ with sqlite3.connect(db_path) as conn:
+ keyed = _is_keyed(conn)
+ stored = bytes(conn.execute("SELECT content FROM Templates WHERE id = ?", (template_id,)).fetchone()[0])
+ if keyed:
+ if _large_window(stored):
+ raise RuntimeError("template is a large-window Brotli stream, which the app cannot decode")
+ stored = _template_brotli(db_path, stored, "-d")
+ return stored.decode("utf-8")
+
+
+def save_template(db_path: Path, template_id: int, text: str) -> None:
+ """Store the template's Pebble source: plain UTF-8 below schema 3, Brotli against
+ the dictionary from it, with a 24-bit window (never large-window, which brotli4j
+ refuses), checked by decoding it back before anything is written."""
+ data = text.encode("utf-8")
+ with sqlite3.connect(db_path) as conn:
+ keyed = _is_keyed(conn)
+ if keyed:
+ plain = data
+ data = _template_brotli(db_path, plain, "-w", "24")
+ if _large_window(data) or _template_brotli(db_path, data, "-d") != plain:
+ raise RuntimeError("compressed template did not decode back to its text; nothing was written")
+ with sqlite3.connect(db_path) as conn:
+ conn.execute("UPDATE Templates SET content = ? WHERE id = ?", (data, template_id))
+ conn.commit()
+
+
def fragment_blob(blob: bytes, chunk_size: int = CONTENT_CHUNK_SIZE) -> list[bytes]:
"""Split blob into chunks of up to chunk_size. Empty input returns one empty chunk."""
if not blob:
@@ -3724,6 +3821,91 @@ def on_file_click(name: str, depth: int, full_path: str) -> None:
)
rerender()
+ def show_templates_page(filter_text: str = "") -> None:
+ """List the Templates rows (the Pebble templates WebServer.kt renders pages
+ through), filterable by name; click one to edit it."""
+ _capture(user_name, "templates_opened")
+ page.controls.clear()
+ try:
+ templates = get_templates(db_path)
+ except sqlite3.Error as e:
+ page.add(ft.Text(f"Could not load templates: {e}", color=ft.Colors.RED, selectable=True))
+ page.update()
+ return
+ template_list = ft.ListView(expand=True)
+
+ def fill(_: object = None) -> None:
+ term = (filter_field.value or "").lower()
+ template_list.controls = [
+ ft.TextButton(name, on_click=lambda _, tid=tid, name=name: show_template_edit_page(tid, name, filter_field.value or ""))
+ for tid, name in templates
+ if term in name.lower()
+ ]
+ page.update()
+
+ filter_field = ft.TextField(label="Filter by name", value=filter_text, on_change=fill, width=400)
+ page.add(
+ ft.Column(
+ [
+ ft.Button(
+ "Back",
+ style=ft.ButtonStyle(bgcolor=ft.Colors.BLUE, color=ft.Colors.BLACK),
+ on_click=lambda _: show_browse_page(get_total_count(db_path)),
+ ),
+ ft.Text(f"{len(templates)} templates", style=ft.TextStyle(weight=ft.FontWeight.BOLD, size=14)),
+ filter_field,
+ template_list,
+ ],
+ expand=True,
+ )
+ )
+ fill()
+
+ def show_template_edit_page(template_id: int, name: str, filter_text: str) -> None:
+ page.controls.clear()
+ try:
+ text = get_template_text(db_path, template_id)
+ except (sqlite3.Error, RuntimeError, brotli.error, UnicodeDecodeError) as e:
+ page.add(ft.Text(f"Could not read template {name}: {e}", color=ft.Colors.RED, selectable=True))
+ page.update()
+ return
+ editor = ft.TextField(value=text, multiline=True, expand=True, text_style=ft.TextStyle(font_family="monospace"))
+
+ def on_save(_: ft.ControlEvent) -> None:
+ try:
+ save_template(db_path, template_id, editor.value or "")
+ update_last_change(db_path, "templates", user_name)
+ except (sqlite3.Error, RuntimeError, brotli.error) as e:
+ page.show_dialog(ft.SnackBar(content=ft.Text(f"Not saved: {e}")))
+ return
+ _capture(user_name, "template_saved")
+ page.show_dialog(ft.SnackBar(content=ft.Text(f"Saved {name}.")))
+
+ page.add(
+ ft.Column(
+ [
+ ft.Row(
+ [
+ ft.Button(
+ "Back",
+ style=ft.ButtonStyle(bgcolor=ft.Colors.BLUE, color=ft.Colors.BLACK),
+ on_click=lambda _: show_templates_page(filter_text),
+ ),
+ ft.Button(
+ "Save",
+ style=ft.ButtonStyle(bgcolor=ft.Colors.GREEN_700, color=ft.Colors.WHITE),
+ on_click=on_save,
+ ),
+ ]
+ ),
+ ft.Text(name, style=ft.TextStyle(weight=ft.FontWeight.BOLD, size=14), selectable=True),
+ editor,
+ ],
+ expand=True,
+ )
+ )
+ page.update()
+
def show_validate_uris_page(restore_page: int | None = None) -> None:
"""Audit tooltips for problems (broken button URIs and empty summaries);
render each problem tooltip with a Problem-type column and Edit link."""
@@ -4792,6 +4974,11 @@ async def choose_import_content(_: ft.ControlEvent) -> None:
style=ft.ButtonStyle(bgcolor=ft.Colors.BLUE, color=ft.Colors.WHITE),
on_click=lambda _: show_content_tree_page(),
)
+ templates_btn = ft.Button(
+ "Templates",
+ style=ft.ButtonStyle(bgcolor=ft.Colors.BLUE, color=ft.Colors.WHITE),
+ on_click=lambda _: show_templates_page(),
+ )
async def export_csv(_: ft.ControlEvent) -> None:
# Export the current view: all tooltips when the search box is empty,
@@ -4978,6 +5165,7 @@ def update_bulk_delete_button() -> None:
add_btn,
validate_btn,
browse_content_btn,
+ templates_btn,
export_btn,
bulk_delete_btn,
search_bar,
diff --git a/docdb-studio/tests/test_schema_v3.py b/docdb-studio/tests/test_schema_v3.py
new file mode 100644
index 00000000..d171a511
--- /dev/null
+++ b/docdb-studio/tests/test_schema_v3.py
@@ -0,0 +1,202 @@
+"""docdb-studio on a schema 3.0.0 database (ADFA-6740): keyed tooltip tables and
+templates Brotli-compressed against CompressionDictionary.
+
+The fixture is a 2.x database converted by the real scripts/migrate_tooltip_schema.py,
+so these tests track whatever that step produces. Needs the brotli CLI."""
+
+import shutil
+import sqlite3
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+import docdb_studio as ds
+
+MIGRATE = Path(__file__).resolve().parents[2] / "scripts" / "migrate_tooltip_schema.py"
+DICTIONARY = b"{{ title }}{% for s in b %}{{ s.r | raw }}{% endfor %}" * 40
+TEMPLATE = b"{{ title | raw }}{% for s in b %}{{ s.r | raw }}{% endfor %}\n"
+
+# What Code On The Go runs on a 3.0.0 database (migrate_tooltip_schema.APP_TOOLTIP_QUERY / APP_BUTTONS_QUERY).
+APP_TOOLTIP = ("SELECT T.id, X.summary, X.detail FROM Tooltips AS T JOIN TooltipTexts AS X ON X.id = T.id "
+ "WHERE T.categoryId IN (SELECT id FROM TooltipCategories WHERE category = ?) AND T.tag = ?")
+APP_BUTTONS = ("SELECT D.description, U.uri FROM TooltipButtons AS B "
+ "JOIN TooltipButtonDescriptions AS D ON D.id = B.descriptionId "
+ "JOIN TooltipButtonUris AS U ON U.id = B.uriId WHERE B.tooltipId = ? ORDER BY B.buttonNumberId")
+
+SCHEMA = """
+CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL);
+CREATE TABLE TooltipButtonNumbers (id INTEGER UNIQUE);
+CREATE TABLE Tooltips (id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL,
+ summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag));
+CREATE TABLE TooltipButtons (tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT);
+CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL,
+ who TEXT NOT NULL, comment TEXT NOT NULL, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
+CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL,
+ content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE(path));
+CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL);
+CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL,
+ UNIQUE(name));
+CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT);
+INSERT INTO TooltipCategories VALUES (1, 'ide'), (2, 'kotlin');
+INSERT INTO TooltipButtonNumbers VALUES (1), (2), (3);
+INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) VALUES (2, 0, 0, 'test', 'test');
+INSERT INTO Content (path, languageID, content, contentTypeID) VALUES ('a/x.html', 1, x'00', 1);
+INSERT INTO Tooltips VALUES (10, 1, 'build', 'Build it', 'Builds it'),
+ (11, 1, 'run', 'Run it', ''),
+ (12, 2, 'fun', '', 'A function');
+INSERT INTO TooltipButtons VALUES (10, 1, 'View full documentation', 'a/x.html'),
+ (10, 2, 'Examples', 'a/build.html#ex'),
+ (11, 1, 'View full documentation', 'a/x.html');
+"""
+
+pytestmark = pytest.mark.skipif(shutil.which("brotli") is None, reason="needs the brotli CLI")
+
+
+def _cli(data: bytes, *args: str) -> bytes:
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(DICTIONARY)
+ f.flush()
+ return subprocess.run(["brotli", "-D", f.name, *args, "-c"], input=data, capture_output=True,
+ check=True).stdout
+
+
+def _v2(tmp_path: Path) -> Path:
+ db = tmp_path / "documentation.db"
+ with sqlite3.connect(db) as conn:
+ conn.executescript(SCHEMA)
+ conn.execute("INSERT INTO CompressionDictionary VALUES (1, ?)", (DICTIONARY,))
+ conn.execute("INSERT INTO Templates (name, content) VALUES ('page.peb', ?)", (TEMPLATE,))
+ return db
+
+
+@pytest.fixture
+def v3(tmp_path: Path) -> Path:
+ db = _v2(tmp_path)
+ subprocess.run([sys.executable, str(MIGRATE), str(db)], check=True, capture_output=True)
+ assert ds.database_major_version(db) == 3
+ return db
+
+
+def _q(db: Path, sql: str, params: tuple = ()) -> list:
+ with sqlite3.connect(db) as conn:
+ return conn.execute(sql, params).fetchall()
+
+
+def _orphans(db: Path) -> list:
+ return [
+ _q(db, "SELECT COUNT(*) FROM TooltipTexts WHERE id NOT IN (SELECT id FROM Tooltips)")[0][0],
+ _q(db, "SELECT COUNT(*) FROM TooltipButtonDescriptions WHERE id NOT IN "
+ "(SELECT descriptionId FROM TooltipButtons)")[0][0],
+ _q(db, "SELECT COUNT(*) FROM TooltipButtonUris WHERE id NOT IN (SELECT uriId FROM TooltipButtons)")[0][0],
+ ]
+
+
+def test_version_3_is_supported() -> None:
+ assert ds.SUPPORTED_DATABASE_MAJOR_VERSION >= 3
+
+
+def test_reads_tooltips_and_buttons(v3: Path) -> None:
+ assert ds.get_total_count(v3) == 3
+ assert ds.get_page(v3, 50, 0) == [
+ (10, "ide", "build", "Build it", "Builds it"),
+ (11, "ide", "run", "Run it", ""),
+ (12, "kotlin", "fun", "", "A function"),
+ ]
+ assert ds.get_tooltip_by_id(v3, 10) == ("ide", "build", "Build it", "Builds it")
+ assert ds.get_tooltip_for_edit(v3, 12) == (2, "kotlin", "fun", "", "A function")
+ assert ds.get_tooltip_buttons(v3, 10) == [(1, "View full documentation", "a/x.html"),
+ (2, "Examples", "a/build.html#ex")]
+ assert ds.get_total_count(v3, "Run") == 1 # summary
+ assert ds.get_total_count(v3, "build.html") == 1 # button uri
+ assert ds.get_button_reference_counts(v3) == {"a/x.html": 2, "a/build.html": 1}
+ assert [r[0] for r in ds.get_all_button_rows(v3)] == [10, 10, 11]
+ assert ds.find_empty_summary_tooltips(v3) == [(12, "fun")]
+ assert ds.get_tooltip_export_rows(v3, "build")[0][:6] == [
+ "ide", "build", "Build it", "Builds it", "View full documentation", "a/x.html"]
+
+
+def test_insert_takes_the_text_id_and_the_app_finds_it(v3: Path) -> None:
+ new_id = ds.insert_tooltip(v3, 2, "val", "A value", "Immutable")
+ ds.replace_tooltip_buttons(v3, new_id, [(1, "View full documentation", "a/x.html"), (2, "New", "k/val.html")])
+ assert _q(v3, "SELECT id FROM TooltipTexts WHERE summary = 'A value'") == [(new_id,)]
+ (row,) = _q(v3, APP_TOOLTIP, ("kotlin", "val"))
+ assert row == (new_id, "A value", "Immutable")
+ assert _q(v3, APP_BUTTONS, (new_id,)) == [("View full documentation", "a/x.html"), ("New", "k/val.html")]
+ # The shared strings are reused, not stored twice.
+ assert _q(v3, "SELECT COUNT(*) FROM TooltipButtonUris WHERE uri = 'a/x.html'") == [(1,)]
+ assert _orphans(v3) == [0, 0, 0]
+
+
+def test_duplicate_insert_leaves_no_text_behind(v3: Path) -> None:
+ with pytest.raises(sqlite3.IntegrityError):
+ ds.insert_tooltip(v3, 1, "build", "dup", "dup")
+ assert _orphans(v3) == [0, 0, 0]
+
+
+def test_update_renames_and_rewrites_the_text(v3: Path) -> None:
+ ds.update_tooltip(v3, 10, 2, "compile", "Compile it", "Compiles")
+ assert _q(v3, APP_TOOLTIP, ("ide", "build")) == []
+ assert _q(v3, APP_TOOLTIP, ("kotlin", "compile")) == [(10, "Compile it", "Compiles")]
+ assert len(_q(v3, APP_BUTTONS, (10,))) == 2
+ with pytest.raises(sqlite3.IntegrityError):
+ ds.update_tooltip(v3, 11, 2, "compile", "x", "y")
+
+
+def test_replacing_buttons_drops_strings_nothing_uses(v3: Path) -> None:
+ ds.replace_tooltip_buttons(v3, 10, [(1, "View full documentation", "a/x.html")])
+ assert _q(v3, "SELECT COUNT(*) FROM TooltipButtonDescriptions WHERE description = 'Examples'") == [(0,)]
+ assert _orphans(v3) == [0, 0, 0]
+
+
+def test_delete_removes_text_and_unused_strings(v3: Path) -> None:
+ assert ds.delete_tooltips_bulk(v3, [10]) == 1
+ assert _q(v3, "SELECT id FROM TooltipTexts ORDER BY id") == [(11,), (12,)]
+ assert _orphans(v3) == [0, 0, 0]
+ assert _q(v3, APP_BUTTONS, (11,)) == [("View full documentation", "a/x.html")]
+
+
+def test_csv_import_updates_and_inserts(v3: Path) -> None:
+ rows = [
+ ["ide", "run", "Run now", "Runs", "Run docs", "a/run.html", "", "", "", ""],
+ ["ide", "stop", "Stop it", "", "View full documentation", "a/x.html", "", "", "", ""],
+ ]
+ assert ds.import_csv_rows(v3, rows, {"ide": 1, "kotlin": 2}, mode="update") == (1, 1)
+ assert _q(v3, APP_TOOLTIP, ("ide", "run")) == [(11, "Run now", "Runs")]
+ assert _q(v3, APP_BUTTONS, (11,)) == [("Run docs", "a/run.html")]
+ (stop_id, _, _), = _q(v3, APP_TOOLTIP, ("ide", "stop"))
+ assert stop_id > 12
+ assert _orphans(v3) == [0, 0, 0]
+
+
+def test_template_round_trips_compressed(v3: Path) -> None:
+ ((tid, name),) = ds.get_templates(v3)
+ assert name == "page.peb"
+ assert ds.get_template_text(v3, tid) == TEMPLATE.decode()
+ ds.save_template(v3, tid, "
{{ edited }} été
")
+ ((stored,),) = _q(v3, "SELECT content FROM Templates WHERE id = ?", (tid,))
+ assert stored[0] & 0x7F != 0x11 # not large-window
+ assert _cli(stored, "-d").decode() == "
")
+ assert _q(db, "SELECT content FROM Templates") == [(b"
plain
",)]
+
+
+def test_version_is_read_newest_changetime_first(tmp_path: Path) -> None:
+ db = _v2(tmp_path)
+ with sqlite3.connect(db) as conn:
+ conn.execute("UPDATE DocumentationDatabaseVersion SET changeTime = '2026-10-01 00:00:00'")
+ conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) "
+ "VALUES (3, 0, 0, 't', 't', '2026-10-02 00:00:00')")
+ conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) "
+ "VALUES (1, 0, 0, 't', 't', '2026-09-01 00:00:00')")
+ assert ds.database_major_version(db) == 3
diff --git a/scripts/migrate_tooltip_schema.py b/scripts/migrate_tooltip_schema.py
index 7e833b26..5ee066c6 100644
--- a/scripts/migrate_tooltip_schema.py
+++ b/scripts/migrate_tooltip_schema.py
@@ -97,16 +97,15 @@
needs the brotli CLI on PATH.
Pipeline position: on the copy that ships, after every step that writes
-tooltips, all of which write the old tables (TooltipManager.py, docdb-studio,
-sync_kdoc_json_to_db.py, the one-off scripts), and after every step that writes
-templates, all of which write them plain. On that copy the order is: the
+tooltips in the old tables (TooltipManager.py, sync_kdoc_json_to_db.py, the
+one-off scripts), and after every step that writes templates plain. On that copy the order is: the
minify, the row drops, the Kotlin trim, the a/ pages-to-templates step, this
step, PR #48's remint_dictionary.py --seeds 4, and rebuild_database.py, which
then packs Content,TooltipTexts: it cannot pack a WITHOUT ROWID table, so not
Tooltips. hm_pages_to_templates.py (PR #43) writes templates too, so it also
goes before this step, after the i/ page steps it lists. populate_db.py,
-sync_javadoc_json_to_db.py and load_android_json_db.py refuse a database
-declaring 3.0.0 or later; the steps in the open PRs do not check.
+sync_javadoc_json_to_db.py, load_android_json_db.py and docdb-studio read and
+write 3.0.0 as well as 2.x (ADFA-6740); the steps in the open PRs do not check.
Never run it on the master database those tools edit: they read or write the
old shape and plain templates (check-tools, which docdb-regression-test.yaml
runs daily, fails on the new tooltip tables), and nothing converts it back.
diff --git a/scripts/sync_java_docs/sync_javadoc_json_to_db.py b/scripts/sync_java_docs/sync_javadoc_json_to_db.py
index 17251fd3..7e9c5a35 100755
--- a/scripts/sync_java_docs/sync_javadoc_json_to_db.py
+++ b/scripts/sync_java_docs/sync_javadoc_json_to_db.py
@@ -95,8 +95,18 @@
REFERENCE_KEYS = frozenset({"specifiedBy", "members", "overrides"})
-# From this documentation major on, every template is stored compressed (migrate_tooltip_schema.py, ADFA-3305).
+# From this documentation major on, every template is stored compressed against CompressionDictionary, with no
+# per-row marker (migrate_tooltip_schema.py, ADFA-3305); below it, plain. A later major than the newest this step
+# knows may store rows in a way it would misread, so it is refused (ADFA-6740).
COMPRESSED_TEMPLATES_MAJOR = 3
+NEWEST_KNOWN_MAJOR = 3
+# Brotli's largest RFC 7932 window; never a large-window stream, which the app's brotli4j refuses.
+WINDOW_BITS = 24
+
+
+def large_window(stream: bytes) -> bool:
+ """Same test as migrate_tooltip_schema.large_window: a window over 24 bits."""
+ return bool(stream) and stream[0] & 0x7F == 0x11
def declared_major(conn) -> int | None:
@@ -166,15 +176,28 @@ def __init__(self, dictionary_data: bytes):
self._dict.write_bytes(dictionary_data)
atexit.register(lambda: shutil.rmtree(self._dir, ignore_errors=True))
- def compress(self, data: bytes) -> bytes:
+ def _run(self, *args: str, data: bytes) -> bytes:
result = subprocess.run(
- [self._brotli, "-D", str(self._dict), "-c"],
+ [self._brotli, "-D", str(self._dict), *args, "-c"],
input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False,
)
if result.returncode != 0:
raise RuntimeError(f"brotli failed: {result.stderr.decode(errors='replace').strip()}")
return result.stdout
+ def compress(self, data: bytes) -> bytes:
+ return self._run("-w", str(WINDOW_BITS), data=data)
+
+ def decompress(self, data: bytes) -> bytes:
+ return self._run("-d", data=data)
+
+ def compress_template(self, data: bytes) -> bytes:
+ """`data` compressed as a schema 3 template, checked by decoding it back."""
+ stream = self.compress(data)
+ if large_window(stream) or self.decompress(stream) != data:
+ raise RuntimeError("the compressed template did not decode back to its text")
+ return stream
+
def rewrite_links(value):
"""Recursively swap `.json` for `.html` in every string of a parsed JSON document.
@@ -249,12 +272,17 @@ def main() -> int:
check = sqlite3.connect(f"file:{args.db}?mode=ro", uri=True)
try:
major = declared_major(check)
+ has_dictionary = load_compression_dictionary(check) is not None
finally:
check.close()
- if major is not None and major >= COMPRESSED_TEMPLATES_MAJOR:
- print(f"Error: the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 "
- f"every template is stored compressed, and {TEMPLATE_NAME} would be written plain. Run this before "
- "migrate_tooltip_schema.py; nothing was changed.", file=sys.stderr)
+ if major is not None and major > NEWEST_KNOWN_MAJOR:
+ print(f"Error: the database declares documentation version {major}, and this step writes versions up to "
+ f"{NEWEST_KNOWN_MAJOR}.x; nothing was changed.", file=sys.stderr)
+ return 1
+ templates_compressed = major is not None and major >= COMPRESSED_TEMPLATES_MAJOR
+ if templates_compressed and not has_dictionary:
+ print(f"Error: the database declares documentation version {major}, whose templates are compressed "
+ "against CompressionDictionary, but it has none; nothing was changed.", file=sys.stderr)
return 1
if args.dry_run:
@@ -271,15 +299,18 @@ def main() -> int:
# --- template --------------------------------------------------------
source = (args.templates / TEMPLATE_NAME).read_text(encoding="utf-8")
+ # Against the dictionary from 3.0.0 even with --plain-brotli: the app decodes every template that way.
+ stored = (DictionaryBrotli(load_compression_dictionary(conn)).compress_template(source.encode())
+ if templates_compressed else source.encode())
existing = cur.execute("SELECT id FROM Templates WHERE name = ?", (TEMPLATE_NAME,)).fetchone()
if args.dry_run:
template_id = existing[0] if existing else -1
print(f" [{'UPDATE' if existing else 'INSERT'} TEMPLATE] {TEMPLATE_NAME} ({len(source)} bytes)")
elif existing:
- cur.execute("UPDATE Templates SET content = ? WHERE id = ?", (source.encode(), existing[0]))
+ cur.execute("UPDATE Templates SET content = ? WHERE id = ?", (stored, existing[0]))
template_id = existing[0]
else:
- cur.execute("INSERT INTO Templates (name, content) VALUES (?, ?)", (TEMPLATE_NAME, source.encode()))
+ cur.execute("INSERT INTO Templates (name, content) VALUES (?, ?)", (TEMPLATE_NAME, stored))
template_id = cur.lastrowid
print(f"Installed template '{TEMPLATE_NAME}' ({len(source)} bytes) for all page kinds.")
diff --git a/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py b/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py
index 4de86431..2a305444 100644
--- a/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py
+++ b/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py
@@ -1,8 +1,9 @@
#!/usr/bin/env python3
-"""Tests that sync_javadoc_json_to_db.py writes no plain template into a database whose templates are compressed.
+"""Tests that sync_javadoc_json_to_db.py writes javadoc.peb the way the database stores templates (ADFA-6740).
-From documentation 3.0.0 migrate_tooltip_schema.py (ADFA-3305) stores every template compressed, and the app decodes
-every one, so a plain javadoc.peb written afterwards would not render while the file still declared 3.x.
+From documentation 3.0.0 every template is stored compressed against CompressionDictionary, with no per-row marker,
+and the app decodes every one; below it they are plain. A later major, a format this step does not know, is refused
+before the backup.
Run directly (python3 scripts/sync_java_docs/test_sync_javadoc_json_to_db.py) or under pytest. Needs the brotli CLI.
"""
@@ -15,6 +16,8 @@
from pathlib import Path
SCRIPT = Path(__file__).resolve().parent / "sync_javadoc_json_to_db.py"
+DICTIONARY = b"{{ page }}{% if page %}{% endif %}" * 20
+TEMPLATE = b"{{ page }}"
SCHEMA = """
CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE);
@@ -25,12 +28,13 @@
CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT);
CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL,
who TEXT NOT NULL, comment TEXT NOT NULL, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
+CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL);
INSERT INTO Languages (value) VALUES ('en-US');
INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli');
"""
-class CompressedTemplatesRefusalTest(unittest.TestCase):
+class TemplateTest(unittest.TestCase):
def setUp(self):
directory = tempfile.TemporaryDirectory()
self.addCleanup(directory.cleanup)
@@ -38,10 +42,13 @@ def setUp(self):
self.db_path = self.dir / "documentation.db"
conn = sqlite3.connect(self.db_path)
conn.executescript(SCHEMA)
+ conn.execute("INSERT INTO CompressionDictionary VALUES (1, ?)", (DICTIONARY,))
+ conn.commit()
conn.close()
(self.dir / "api").mkdir()
(self.dir / "templates").mkdir()
- (self.dir / "templates" / "javadoc.peb").write_text("{{ page }}", encoding="utf-8")
+ (self.dir / "templates" / "javadoc.peb").write_bytes(TEMPLATE)
+ (self.dir / "dictionary.bin").write_bytes(DICTIONARY)
def declare(self, *rows):
"""Appends (major, changeTime) rows to DocumentationDatabaseVersion, in order."""
@@ -51,46 +58,64 @@ def declare(self, *rows):
conn.commit()
conn.close()
- def sync(self) -> subprocess.CompletedProcess:
+ def sync(self, *args: str) -> subprocess.CompletedProcess:
return subprocess.run([sys.executable, str(SCRIPT), str(self.dir / "api"), "--db", str(self.db_path),
- "--templates", str(self.dir / "templates")],
+ "--templates", str(self.dir / "templates"), *args],
capture_output=True, text=True, cwd=self.dir, check=False)
def sha(self) -> str:
return hashlib.sha256(self.db_path.read_bytes()).hexdigest()
- def assert_refused(self, major: int):
+ def stored(self) -> bytes:
+ conn = sqlite3.connect(self.db_path)
+ try:
+ (name, content), = conn.execute("SELECT name, content FROM Templates").fetchall()
+ finally:
+ conn.close()
+ self.assertEqual(name, "javadoc.peb")
+ return content
+
+ def decoded(self, stream: bytes) -> bytes:
+ self.assertNotEqual(stream[0] & 0x7F, 0x11) # never large-window
+ return subprocess.run(["brotli", "-D", str(self.dir / "dictionary.bin"), "-d", "-c"], input=stream,
+ capture_output=True, check=True).stdout
+
+ def test_refuses_a_later_major_before_the_backup(self):
+ self.declare((4, "2026-10-19 00:00:00"))
sha = self.sha()
result = self.sync()
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
- self.assertIn(f"declares documentation version {major}", result.stderr)
- self.assertIn("Run this before migrate_tooltip_schema.py; nothing was changed.", result.stderr)
+ self.assertIn("declares documentation version 4, and this step writes versions up to 3.x", result.stderr)
self.assertEqual(self.sha(), sha)
self.assertEqual(sorted(p.name for p in self.dir.glob("documentation.db*")), ["documentation.db"])
- def test_refuses_major_3_before_the_backup(self):
+ def test_compresses_the_template_at_major_3(self):
self.declare((2, "2026-01-01 00:00:00"), (3, "2026-10-19 00:00:00"))
- self.assert_refused(3)
+ result = self.sync()
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(self.decoded(self.stored()), TEMPLATE)
- def test_refuses_a_later_major(self):
- self.declare((4, "2026-10-19 00:00:00"))
- self.assert_refused(4)
+ def test_compresses_the_template_at_major_3_even_with_plain_brotli(self):
+ self.declare((3, "2026-10-19 00:00:00"))
+ self.assertEqual(self.sync("--plain-brotli").returncode, 0)
+ self.assertEqual(self.decoded(self.stored()), TEMPLATE)
- def test_reads_the_newest_changetime_first(self):
- self.declare((3, "2026-10-19 00:00:00"), (2, "2026-01-01 00:00:00"))
- self.assert_refused(3)
+ def test_major_3_without_a_dictionary_is_refused(self):
+ self.declare((3, "2026-10-19 00:00:00"))
+ conn = sqlite3.connect(self.db_path)
+ conn.execute("DELETE FROM CompressionDictionary")
+ conn.commit()
+ conn.close()
+ result = self.sync()
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("but it has none; nothing was changed", result.stderr)
- def test_writes_the_template_at_major_2(self):
+ def test_writes_the_template_plain_at_major_2(self):
# The newest rowid breaks the changeTime tie.
self.declare((3, "2026-10-19 00:00:00"), (2, "2026-10-19 00:00:00"))
result = self.sync()
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
- conn = sqlite3.connect(self.db_path)
- try:
- self.assertEqual(conn.execute("SELECT name, content FROM Templates").fetchall(),
- [("javadoc.peb", b"{{ page }}")])
- finally:
- conn.close()
+ self.assertEqual(self.stored(), TEMPLATE)
if __name__ == "__main__":