עדכון דלתא: hash תוכן לכל טבלה במניפסט

.github/workflows/contract.yml

@@ -13,6 +13,8 @@ jobs:
     env:
       KOTLIN_FIXTURE: generator/common/src/jvmTest/resources/patch_tables_contract.json
       DART_FIXTURE: seforim-library-updater/test/patch_tables_contract.json
+      KOTLIN_HASH_FIXTURE: generator/common/src/jvmTest/resources/logical_hash_contract.json
+      DART_HASH_FIXTURE: seforim-library-updater/test/logical_hash_contract.json
       KOTLIN_VISIBILITY_FIXTURE: generator/sefariasqlite/src/jvmTest/resources/link_visibility_contract_v1.json
       SEFARIA_VISIBILITY_FIXTURE: sefaria-export/link_visibility_contract_v1.json
     steps:
@@ -63,13 +65,16 @@ jobs:
         run: |
           test -f "$KOTLIN_FIXTURE"
           test -f "$DART_FIXTURE"
+          test -f "$KOTLIN_HASH_FIXTURE"
+          test -f "$DART_HASH_FIXTURE"
           test -f "$KOTLIN_VISIBILITY_FIXTURE"
           test -f "$SEFARIA_VISIBILITY_FIXTURE"
 
       # השוואת בתים ישירה בין שני ה-fixtures — הבדיקה המקבילה בצד ה-Dart רצה ב-CI של ה-updater
       - name: cmp fixtures (byte-identical)
         run: |
           cmp "$KOTLIN_FIXTURE" "$DART_FIXTURE"
+          cmp "$KOTLIN_HASH_FIXTURE" "$DART_HASH_FIXTURE"
           cmp "$KOTLIN_VISIBILITY_FIXTURE" "$SEFARIA_VISIBILITY_FIXTURE"
 
       # cmp לבדו תופס רק סטייה בין שני ה-fixtures. שינוי ברשימת הטבלאות בצד Kotlin
@@ -89,4 +94,4 @@ jobs:
         run: chmod +x gradlew
 
       - name: Run Kotlin patch-tables contract test
-        run: ./gradlew :generator-common:jvmTest --tests '*PatchTablesContractTest*' --no-daemon
+        run: ./gradlew :generator-common:jvmTest --tests '*PatchTablesContractTest*' --tests '*LogicalHashContractTest*' --no-daemon

DELTA_UPDATE_WORKFLOW.md

@@ -194,6 +194,42 @@ copy(v1) → apply(patch) → LogicalContentHasher.compute() == hash(v2)
 If this fails, `producePatchAndVerify` exits non-zero and the
 release is NOT published.
 
+### 1.4 Per-table content hashes
+
+`LogicalContentHasher.computeReport()` returns, in one pass, both the whole-DB
+hash and `tableHashes` — for each table of the hash table order,
+`sha256` of exactly the bytes that table contributed to the whole stream
+(the `" table:<t> "` prefix included; an absent table hashes to its prefix
+alone). Therefore
+`wholeHash == sha256(stream(t1) ‖ stream(t2) ‖ …)`.
+
+The pipeline ships both maps in the manifest:
+
+```json
+"fromTableContentHashes": { "source": "<hex>", "author": "<hex>", … },
+"toTableContentHashes":   { "source": "<hex>", … }
+```
+
+- Keys = every table of the hash table order for `fromSchemaVersion` /
+  `toSchemaVersion` respectively, in that order. Both maps are present or
+  both absent; `fromContentHash` / `toContentHash` stay required, so old
+  clients are unaffected.
+- The verify-apply gate compares the applied per-table hashes with new's, on
+  top of the whole-hash check — the maps a release publishes are gated as
+  hard as the hash the client verifies against.
+
+**Why the client cares:** whole-DB verification streams ~80% of a 6 GB file
+after every applied patch. With these maps the client verifies only the
+tables whose `from` and `to` hashes differ, plus every table the patch has
+upsert/delete rows for, plus `schema_meta`; the rest are deferred to a
+read-only pass after the update is committed and the library is readable
+again. Drift found in a deferred table is reported, not rolled back.
+
+The oracle for both sides is `logical_hash_contract.json` — byte-identical in
+`generator/common/src/jvmTest/resources/` and the updater's `test/`, cmp'd by
+`contract.yml`. Do not regenerate it unless both sides fail identically and
+the change is deliberate.
+
 ---
 
 ## 2. CDN / static-host layout

delta-updater/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/deltaupdater/Manifest.kt

@@ -17,6 +17,12 @@ data class DeltaManifest(
     val toSchemaVersion: Int,
     val fromContentHash: String,
     val toContentHash: String,
+    /**
+     * Per-table logical hashes, keyed by the hash table order of
+     * from/toSchemaVersion. Empty on manifests produced before this field.
+     */
+    val fromTableContentHashes: Map<String, String> = emptyMap(),
+    val toTableContentHashes: Map<String, String> = emptyMap(),
     /**
      * Books touched / added / removed / renamed between fromVersion and toVersion.
      * Empty when the diff has no per-book scope (e.g. lookup-only changes).

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalContentHasher.kt

@@ -24,40 +24,71 @@ class LogicalContentHasher(
     private val logger: Logger = Logger.withTag("LogicalContentHasher"),
 ) {
 
-    fun compute(conn: Connection): String {
-        val md = MessageDigest.getInstance("SHA-256")
-        for (table in tables) {
-            md.update(" table:$table ".toByteArray())
-            val cols = readColumnsCanonical(conn, table) ?: continue // table not present
-            md.update(cols.joinToString(",", prefix = "cols:").toByteArray())
-            md.update(byteArrayOf(0x00))
+    /**
+     * Whole-DB hash plus the per-table digest of exactly the bytes each table
+     * contributed to it, in hash table order.
+     */
+    data class Report(
+        val wholeHash: String,
+        val tableHashes: LinkedHashMap<String, String>,
+    )
+
+    fun compute(conn: Connection): String = computeReport(conn).wholeHash
+
+    /**
+     * One pass, two digests: whole-DB and per-table (reset at each table boundary),
+     * so the client can verify only the tables a patch could have touched.
+     */
+    fun computeReport(conn: Connection): Report {
+        val whole = MessageDigest.getInstance("SHA-256")
+        val table = MessageDigest.getInstance("SHA-256")
+        val sink = DualDigest(whole, table)
+        val tableHashes = LinkedHashMap<String, String>()
+        for (t in tables) {
+            sink.update(" table:$t ".toByteArray())
+            val cols = readColumnsCanonical(conn, t)
+            if (cols == null) { // table not present — its stream is just the prefix
+                tableHashes[t] = hex(table.digest())
+                continue
+            }
+            sink.update(cols.joinToString(",", prefix = "cols:").toByteArray())
+            sink.update(byteArrayOf(0x00))
 
             val colsSql = cols.joinToString(",") { "\"$it\"" }
             val pkOrder = if ("id" in cols) "id" else cols.joinToString(",") { "\"$it\"" }
             conn.createStatement().use { st ->
-                st.executeQuery("SELECT $colsSql FROM \"$table\" ORDER BY $pkOrder").use { rs ->
-                    val meta = rs.metaData
-                    val n = meta.columnCount
+                st.executeQuery("SELECT $colsSql FROM \"$t\" ORDER BY $pkOrder").use { rs ->
+                    val n = rs.metaData.columnCount
                     while (rs.next()) {
-                        for (i in 1..n) encodeCell(md, rs, i)
-                        md.update(byteArrayOf(0xFF.toByte()))
+                        for (i in 1..n) encodeCell(sink, rs, i)
+                        sink.update(byteArrayOf(0xFF.toByte()))
                     }
                 }
             }
+            tableHashes[t] = hex(table.digest()) // digest() also resets for the next table
         }
-        val digest = md.digest()
-        return digest.joinToString("") { "%02x".format(it) }
+        return Report(hex(whole.digest()), tableHashes)
     }
 
-    private fun encodeCell(md: MessageDigest, rs: java.sql.ResultSet, i: Int) {
+    /** Feeds the same bytes to the whole-DB digest and the current table's digest. */
+    private class DualDigest(private val whole: MessageDigest, private val table: MessageDigest) {
+        fun update(bytes: ByteArray) {
+            whole.update(bytes)
+            table.update(bytes)
+        }
+    }
+
+    private fun hex(digest: ByteArray): String = digest.joinToString("") { "%02x".format(it) }
+
+    private fun encodeCell(sink: DualDigest, rs: java.sql.ResultSet, i: Int) {
         val obj = rs.getObject(i)
         when {
-            obj == null || rs.wasNull() -> md.update(byteArrayOf(0))
-            obj is ByteArray -> { md.update(byteArrayOf(1)); md.update(obj) }
-            obj is Number -> { md.update(byteArrayOf(2)); md.update(obj.toString().toByteArray()) }
-            else -> { md.update(byteArrayOf(3)); md.update(obj.toString().toByteArray()) }
+            obj == null || rs.wasNull() -> sink.update(byteArrayOf(0))
+            obj is ByteArray -> { sink.update(byteArrayOf(1)); sink.update(obj) }
+            obj is Number -> { sink.update(byteArrayOf(2)); sink.update(obj.toString().toByteArray()) }
+            else -> { sink.update(byteArrayOf(3)); sink.update(obj.toString().toByteArray()) }
         }
-        md.update(byteArrayOf(0x1F)) // unit separator between cells
+        sink.update(byteArrayOf(0x1F)) // unit separator between cells
     }
 
     private fun readColumnsCanonical(conn: Connection, table: String): List<String>? {

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchPipelineCli.kt

@@ -97,9 +97,10 @@ fun main(args: Array<String>) {
     // Verify apply: copy prev, apply patch, hash, compare with hash(new).
     val target = outPath.resolveSibling("verify-${outPath.fileName}")
     Files.copy(prevPath, target, StandardCopyOption.REPLACE_EXISTING)
-    val newHash = DriverManager.getConnection("jdbc:sqlite:${newPath.toAbsolutePath()}").use {
-        LogicalContentHasher.forSchemaVersion(toSchemaVersion).compute(it)
+    val newReport = DriverManager.getConnection("jdbc:sqlite:${newPath.toAbsolutePath()}").use {
+        LogicalContentHasher.forSchemaVersion(toSchemaVersion).computeReport(it)
     }
+    val newHash = newReport.wholeHash
     DriverManager.getConnection("jdbc:sqlite:${target.toAbsolutePath()}").use { conn ->
         conn.createStatement().use { it.execute("PRAGMA foreign_keys = ON") }
         // The producer ships upserts/deletes for every table in
@@ -114,11 +115,21 @@ fun main(args: Array<String>) {
             expectedToContentHash = newHash,
             expectedToSchemaVersion = toSchemaVersion,
         )
-        val appliedHash = LogicalContentHasher.forSchemaVersion(toSchemaVersion).compute(conn)
+        val appliedReport = LogicalContentHasher.forSchemaVersion(toSchemaVersion).computeReport(conn)
+        val appliedHash = appliedReport.wholeHash
         check(appliedHash == newHash) {
             "Patch verification FAILED: applied=$appliedHash expected=$newHash — " +
                 "inspect with diagnoseHashMismatch; refusing to publish this patch."
         }
+        // The per-table hashes ship in the manifest and drive the client's
+        // partial verification, so they are gated as hard as the whole hash.
+        val divergent = newReport.tableHashes.keys.filter {
+            appliedReport.tableHashes[it] != newReport.tableHashes[it]
+        }
+        check(divergent.isEmpty()) {
+            "Patch verification FAILED: per-table hashes diverge for ${divergent.joinToString(", ")} — " +
+                "refusing to publish this patch."
+        }
         logger.i { "✅ Patch apply verified: target hash matches new ($newHash)" }
     }
     runCatching { Files.deleteIfExists(target) }
@@ -158,17 +169,21 @@ fun main(args: Array<String>) {
             "(${"%.1f".format(compressed.compressedSize * 100.0 / Files.size(outPath))}%)"
     }
 
+    val prevReport = DriverManager.getConnection("jdbc:sqlite:${prevPath.toAbsolutePath()}").use {
+        LogicalContentHasher.forSchemaVersion(fromSchemaVersion).computeReport(it)
+    }
+
     // Emit a per-delta manifest.json next to the .zst.
     ReleaseManifestWriter(logger).writeManifest(
         patchFile = outPath,
         fromVersion = from,
         toVersion = to,
         fromSchemaVersion = fromSchemaVersion,
         toSchemaVersion = toSchemaVersion,
-        fromContentHash = DriverManager.getConnection("jdbc:sqlite:${prevPath.toAbsolutePath()}").use {
-            LogicalContentHasher.forSchemaVersion(fromSchemaVersion).compute(it)
-        },
+        fromContentHash = prevReport.wholeHash,
         toContentHash = newHash,
+        fromTableContentHashes = prevReport.tableHashes,
+        toTableContentHashes = newReport.tableHashes,
         compressed = ReleaseManifestWriter.CompressedPatchSpec(
             file = compressed.compressedFile,
             sha256 = compressed.compressedSha256,

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/ReleaseManifestWriter.kt

@@ -51,8 +51,15 @@ class ReleaseManifestWriter(
         toContentHash: String,
         compressed: CompressedPatchSpec,
         catalogBlobName: String? = null,
+        fromTableContentHashes: Map<String, String> = emptyMap(),
+        toTableContentHashes: Map<String, String> = emptyMap(),
     ): Path {
         require(Files.isRegularFile(patchFile)) { "patch file not found: $patchFile" }
+        // Both maps or neither: a client that sees only one side cannot decide
+        // which tables a patch left untouched, and would silently fall back.
+        require(fromTableContentHashes.isEmpty() == toTableContentHashes.isEmpty()) {
+            "fromTableContentHashes and toTableContentHashes must both be given or both omitted"
+        }
         val uncompressedSha256 = sha256(patchFile)
         val uncompressedSize = Files.size(patchFile)
         val target = compressed.file.resolveSibling("${compressed.file.fileName}.manifest.json")
@@ -65,6 +72,10 @@ class ReleaseManifestWriter(
             append("  \"patchFormatVersion\": ").append(PatchDbSchema.CURRENT_VERSION).append(",\n")
             append("  \"fromContentHash\": ").appendString(fromContentHash).append(",\n")
             append("  \"toContentHash\": ").appendString(toContentHash).append(",\n")
+            if (fromTableContentHashes.isNotEmpty()) {
+                appendHashMap("fromTableContentHashes", fromTableContentHashes)
+                appendHashMap("toTableContentHashes", toTableContentHashes)
+            }
             append("  \"patchFiles\": [\n")
             append("    {\n")
             append("      \"file\": ").appendString(compressed.file.fileName.toString()).append(",\n")
@@ -166,6 +177,18 @@ class ReleaseManifestWriter(
 
     // ─── Minimal JSON helpers (kept dep-free) ──────────────────────────────────
 
+    /** Emits `"<name>": { … },` — keys in map iteration order (= hash table order). */
+    private fun StringBuilder.appendHashMap(name: String, values: Map<String, String>) {
+        append("  ").appendString(name).append(": {\n")
+        var i = 0
+        for ((table, hash) in values) {
+            append("    ").appendString(table).append(": ").appendString(hash)
+            if (++i < values.size) append(",")
+            append("\n")
+        }
+        append("  },\n")
+    }
+
     private fun StringBuilder.appendString(s: String): StringBuilder {
         append('"')
         for (c in s) when (c) {

generator/common/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalHashContractTest.kt

@@ -0,0 +1,91 @@
+package io.github.kdroidfilter.seforimlibrary.common.patch
+
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.int
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import java.sql.Connection
+import java.sql.DriverManager
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+/**
+ * Shared oracle with the Dart client (`contract.yml` cmp's the two fixtures).
+ * A failure here means the two hashers drifted and deltas would be rejected.
+ */
+class LogicalHashContractTest {
+
+    private val fixture = Json.parseToJsonElement(
+        requireNotNull(javaClass.getResourceAsStream("/logical_hash_contract.json")) {
+            "logical_hash_contract.json missing from test resources"
+        }.bufferedReader(Charsets.UTF_8).readText(),
+    ).jsonObject
+
+    private val schemaVersion = fixture.getValue("schemaVersion").jsonPrimitive.int
+    private val hashTableOrder =
+        fixture.getValue("hashTableOrder").jsonArray.map { it.jsonPrimitive.content }
+    private val setupSql = fixture.getValue("setupSql").jsonArray.map { it.jsonPrimitive.content }
+    private val wholeHash = fixture.getValue("wholeHash").jsonPrimitive.content
+    private val tableHashes = fixture.getValue("tableHashes").jsonObject
+        .mapValues { it.value.jsonPrimitive.content }
+
+    private fun <T> withFixtureDb(block: (Connection) -> T): T =
+        DriverManager.getConnection("jdbc:sqlite::memory:").use { conn ->
+            conn.createStatement().use { st -> setupSql.forEach { st.execute(it) } }
+            block(conn)
+        }
+
+    @Test
+    fun `hash table order matches the fixture`() {
+        assertEquals(hashTableOrder, LogicalContentHasher.tablesForSchemaVersion(schemaVersion))
+    }
+
+    @Test
+    fun `whole hash and every per-table hash match the shared oracle`() {
+        val report = withFixtureDb {
+            LogicalContentHasher.forSchemaVersion(schemaVersion).computeReport(it)
+        }
+        assertEquals(wholeHash, report.wholeHash, "whole hash")
+        assertEquals(hashTableOrder, report.tableHashes.keys.toList(), "per-table key order")
+        for (table in hashTableOrder) {
+            assertEquals(tableHashes[table], report.tableHashes[table], "table hash for '$table'")
+        }
+    }
+
+    @Test
+    fun `compute is unchanged by the report API`() {
+        val (whole, report) = withFixtureDb {
+            val hasher = LogicalContentHasher.forSchemaVersion(schemaVersion)
+            hasher.compute(it) to hasher.computeReport(it)
+        }
+        assertEquals(whole, report.wholeHash)
+        assertEquals(wholeHash, whole)
+    }
+
+    @Test
+    fun `absent tables still contribute a hash and distinct tables differ`() {
+        val report = withFixtureDb {
+            LogicalContentHasher.forSchemaVersion(schemaVersion).computeReport(it)
+        }
+        // `topic` is absent from the fixture DB — it hashes its prefix only,
+        // which must still differ from another absent table's prefix hash.
+        assertTrue(report.tableHashes.getValue("topic").isNotEmpty())
+        assertTrue(report.tableHashes.getValue("topic") != report.tableHashes.getValue("pub_place"))
+    }
+
+    @Test
+    fun `a table's hash is the digest of its stream alone`() {
+        // A one-table hasher's whole hash must equal that table's entry in the
+        // full run: wholeHash is the digest of the concatenated table streams.
+        withFixtureDb { conn ->
+            val full = LogicalContentHasher.forSchemaVersion(schemaVersion).computeReport(conn)
+            for (table in hashTableOrder) {
+                val single = LogicalContentHasher(listOf(table)).computeReport(conn)
+                assertEquals(full.tableHashes[table], single.wholeHash, "stream digest for '$table'")
+                assertEquals(single.wholeHash, single.tableHashes.getValue(table))
+            }
+        }
+    }
+}