feat: remove and replace indexes in place (sde-index protocol 2)

README.md

@@ -99,8 +99,9 @@ The Python [local cutover operator](docs/local-cutover.md) adds durable executio
 for signed generation-bearing packets, including native access changes for existing Python and
 TypeScript processes. [Staging](docs/staging.md) creates successive fresh copies while preserving
 the source and local recovery history; controller handoff and workload qualification are separate.
-A design that only adds indexes is [built in place](docs/in-place-index.md), on the live tables and
-without pausing a write, instead of through a copy.
+A design that only changes indexes - adds, removes or replaces them - is carried out
+[in place](docs/in-place-index.md), on the live tables and without pausing a write, instead of
+through a copy.
 The opt-in [workload qualification](docs/cutover-qualification.md) runs installed Python and npm
 artifacts under mixed traffic and checks scheduled latency, native recovery and acknowledged values.
 [Session lifetime](docs/session-lifecycle.md) defines borrowed/owned connections and transaction
@@ -197,7 +198,7 @@ Nothing about that fails at compile time.
 So the encoding is specified at the byte level in [`docs/format-contract.md`](docs/format-contract.md)
 — UTF-8, keys NFC-normalised then sorted by code point, no insignificant whitespace, minimal escaping,
 no float literals, a closed type vocabulary so that `Decimal` and `BigDecimal` land on the same bytes.
-And [`conformance/`](conformance/) holds the vectors — **334 of them, in ten families** — that every
+And [`conformance/`](conformance/) holds the vectors — **342 of them, in ten families** — that every
 library runs in its own test runner, so a divergence is a red test for whoever caused it rather than
 an operation written to the wrong engine in production.
 

conformance/README.md

@@ -314,11 +314,15 @@ Both map signatures, exact unchanged source/routing, portable names and envelope
 validated without constructing engines. `tools/staging_vectors.py` uses the independent fixture
 encoder and OpenSSL. Native creation and crash recovery have separate live tests.
 
-`migration/143`–`179` carry `plan.json` and `index.json` for signed in-place index builds: three
-acceptances (a B-tree under a raised contract, several indexes after the ones in force, ClickHouse
-data-skipping indexes) and one refusal per loader rule, each with the `match` fragment of the
+`migration/143`–`187` carry `plan.json` and `index.json` for signed in-place index builds: three
+acceptances of protocol 1 (a B-tree under a raised contract, several indexes after the ones in
+force, ClickHouse data-skipping indexes), four of protocol 2 (a removal alone, a replacement, the
+other indexes kept in order, an index an earlier build added removed - `index.json` names the
+removed ones in `removed`) and one refusal per loader rule, each with the `match` fragment of the
 message both libraries must give. `tools/index_vectors.py` uses the independent fixture encoder and
-OpenSSL. The native build, its recovery and abandonment have separate live tests.
+OpenSSL; every vector is signed with a key of its own, so `--only` rewrites the named vectors and
+leaves the others byte for byte. The native build, its removals, recovery and abandonment have
+separate live tests.
 
 `migration/122`–`132` carry `bulk.json`: application-batch operations, expected source/copy
 call order, transaction outcomes, capability/bound refusals and exact value-free metric bytes.

conformance/tools/index_vectors.py

@@ -155,6 +155,32 @@ def reused(plan: dict[str, Any]) -> None:
         document["groups"]["Order"]["source"]["layout"]["tables"]["Order"] = name(1)
 
 
+OTHER = {"entity": "Event", "name": "event_at", "columns": ["at"], "method": "brin"}
+BUILT = {"entity": "Event", "name": name(1, "7" * 32), "columns": ["id"]}
+"""An index a previous in-place build added: a bound name of another build id."""
+
+
+def change(*prepared: dict[str, Any], kept: tuple[dict[str, Any], ...] = (KEPT,)) -> Callable[
+    [dict[str, Any]], None
+]:
+    """Protocol 2: the map in force carries ``kept``; the next map carries ``prepared``.
+
+    With nothing left the next map has no ``indexes`` at all - the form a controller writes, for
+    which an empty list would claim that indexing was considered and none chosen.
+    """
+
+    def apply(plan: dict[str, Any]) -> None:
+        plan["protocol"] = 2
+        with_kept(plan, *kept)
+        layout = source(plan["prepared"])["layout"]
+        if prepared:
+            layout["indexes"] = copy.deepcopy(list(prepared))
+        else:
+            layout.pop("indexes", None)
+
+    return apply
+
+
 def cases() -> list[tuple[str, Callable[[dict[str, Any]], None] | None, str | None]]:
     out: list[tuple[str, Callable[[dict[str, Any]], None] | None, str | None]] = []
 
@@ -193,7 +219,7 @@ def change(plan: dict[str, Any]) -> None:
     )
     case(
         "149-index-build-refuses-an-unknown-protocol",
-        at("protocol", 2),
+        at("protocol", 3),
         "unsupported index build authorization kind or protocol",
     )
     case(
@@ -311,13 +337,55 @@ def change(plan: dict[str, Any]) -> None:
         REFUSED,
     )
     case("179-index-build-model-is-bound", at("prepared/model_version", "0" * 16), REFUSED)
+    # Protocol 2: indexes the map in force declares are removed as well as added.
+    new_btree = {"entity": "Event", "name": name(1), "columns": ["at"]}
+    case("180-index-change-removes-an-index-in-place", change(), None)
+    case("181-index-change-replaces-an-index", change(new_btree), None)
+    case(
+        "182-index-change-keeps-the-others-in-order",
+        change(copy.deepcopy(OTHER), new_btree, kept=(KEPT, OTHER, BUILT)),
+        None,
+    )
+    case(
+        "183-index-change-removes-an-index-a-build-added",
+        change(copy.deepcopy(KEPT), kept=(KEPT, BUILT)),
+        None,
+    )
+    case(
+        "184-index-change-removes-at-least-one",
+        change(copy.deepcopy(KEPT), new_btree),
+        "index build protocol 2 removes at least one index in force",
+    )
+    case(
+        "185-index-change-keeps-the-order-of-the-others",
+        change(copy.deepcopy(BUILT), copy.deepcopy(OTHER), kept=(KEPT, OTHER, BUILT)),
+        "an index change keeps the other indexes in force, in order, before the new ones",
+    )
+    case(
+        "186-index-change-names-bind-the-build-id",
+        change({**new_btree, "name": name(1, "9" * 32)}),
+        bound,
+    )
+
+    def moved_key(plan: dict[str, Any]) -> None:
+        change(new_btree)(plan)
+        source(plan["prepared"])["layout"]["key_order"] = {"Event": ["id"]}
+
+    case("187-index-change-keeps-the-key-order", moved_key, only_indexes)
     return out
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--i-am-changing-the-contract", action="store_true", required=True)
     parser.add_argument("--scratch-directory", type=Path, required=True)
+    parser.add_argument(
+        "--only",
+        action="append",
+        default=[],
+        help="write only these vectors (repeatable); every vector has its own key, so the others "
+        "stay byte for byte as they are",
+    )
     args = parser.parse_args()
     scratch = args.scratch_directory.resolve()
     if not scratch.is_dir() or scratch.is_relative_to(ROOT):
@@ -364,7 +432,10 @@ def sign(document: dict[str, Any]) -> None:
                 "value": base64.b64encode((work / "sig").read_bytes()).decode(),
             }
 
-        written = cases()
+        written = [entry for entry in cases() if not args.only or entry[0] in args.only]
+        unknown = set(args.only) - {entry[0] for entry in cases()}
+        if unknown:
+            parser.error(f"no such vector: {sorted(unknown)}")
         for label, change, match in written:
             plan = template()
             if change:
@@ -388,8 +459,10 @@ def sign(document: dict[str, Any]) -> None:
             if match is not None:
                 expected.update(error="MigrationRefused", match=match)
             else:
-                indexes = source(plan["prepared"])["layout"]["indexes"]
-                kept = source(plan["current"])["layout"].get("indexes", [])
+                indexes = source(plan["prepared"])["layout"].get("indexes", [])
+                remaining = {index["name"] for index in indexes}
+                in_force = source(plan["current"])["layout"].get("indexes", [])
+                kept = [index for index in in_force if index["name"] in remaining]
                 expected.update(
                     index_fingerprint=hashlib.sha256(payload(plan)).hexdigest(),
                     verified_with="index",
@@ -400,6 +473,10 @@ def sign(document: dict[str, Any]) -> None:
                     added=[index["name"] for index in indexes[len(kept) :]],
                     build_budget_ms=plan["build_budget_ms"],
                 )
+                if plan["protocol"] == 2:
+                    expected["removed"] = [
+                        index["name"] for index in in_force if index["name"] not in remaining
+                    ]
             directory = VECTORS / "migration" / label
             directory.mkdir(exist_ok=True)
             for filename, value in {

conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/keys.json

@@ -1,3 +1,3 @@
 {
-  "index": "iDVAooL8PTov2s9FYFSnb+9EPl90sp72vut52dGqxIE="
+  "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY="
 }

conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/plan.json

@@ -1,6 +1,6 @@
 {
   "kind": "sde-index",
-  "protocol": 2,
+  "protocol": 3,
   "index_id": "66666666666666666666666666666666",
   "project_id": "11111111111111111111111111111111",
   "group": "Event",
@@ -68,7 +68,7 @@
     "signature": {
       "alg": "ed25519",
       "key_id": "index",
-      "value": "y9+HB9/iWdvlkzt7Z9CMm5p/fJTGbBh/jbGf+Yst7NiV87t9+hl8UWDssdPmgtJl+B5y/kwcx0KYoWU0kec3Dg=="
+      "value": "W1kR3q2Q5a4Ila6u5m3xUjal/TokoekT83uFEeoNrpOBcDaJk9eNUIDsXjGBQPDi2doa3WobndisMPIwiVVKDw=="
     }
   },
   "prepared": {
@@ -144,13 +144,13 @@
     "signature": {
       "alg": "ed25519",
       "key_id": "index",
-      "value": "0k+qiBRWsoe67kAA6G7/XmJXJm67v2syco7ArvEs/hwYz3K7jSCHWEwk9CMDQu7Av1b2Ab1Uo5jgzhlKtMSkDA=="
+      "value": "DlWx0oPKcO22SkbsZyZAY3CmGRCxAz8SL0EUDuiI7L1KmiUGWXPJ9lX4C5jeuAQF87y74ufa8TMFZhqFJf3SBA=="
     }
   },
   "build_budget_ms": 3600000,
   "signature": {
     "alg": "ed25519",
     "key_id": "index",
-    "value": "jCJ8blY4dZOgpNfaqKwJoplDLsGDxfU6eRMU1UzjSxU/KpXCOxGyzd4YsKRFXWYZwlQqhluTz/tYD34DA0jICA=="
+    "value": "ajO5x62pqzEdAdGAnJUmRsS5585bPCdROhBDs/6wn/aZw5kqck82DUqGssRtFPQEdHFaEI9HKrOpnrjxIxUNAA=="
   }
 }

conformance/vectors/migration/180-index-change-removes-an-index-in-place/index.json

@@ -0,0 +1,14 @@
+{
+  "project_id": "11111111111111111111111111111111",
+  "index_fingerprint": "d767bfb6c32f8652b02d4848224a4ae8116a52172b1ab433d9e58a91b831e945",
+  "verified_with": "index",
+  "map_fingerprints": {
+    "current": "8e0a65adc4e36dd53fbda3183cb3650c8731e9511cc0eac2fd3a2439e54af2cd",
+    "prepared": "505f854472806a82b9910d17ed175ccb81993af4a4ec7bf265774307e9cbb2ff"
+  },
+  "added": [],
+  "build_budget_ms": 3600000,
+  "removed": [
+    "event_name"
+  ]
+}

conformance/vectors/migration/180-index-change-removes-an-index-in-place/keys.json

@@ -0,0 +1,3 @@
+{
+  "index": "+loeROgpPde3TDVXd5d2OOqMUf46DKPmrEiHm6Pq+64="
+}

conformance/vectors/migration/180-index-change-removes-an-index-in-place/model.json

@@ -0,0 +1,102 @@
+{
+  "entities": [
+    {
+      "name": "Event",
+      "fields": [
+        {
+          "name": "at",
+          "type": "timestamptz"
+        },
+        {
+          "name": "id",
+          "type": "uuid"
+        },
+        {
+          "name": "name",
+          "type": "string"
+        }
+      ],
+      "key": [
+        "id"
+      ]
+    },
+    {
+      "name": "Order",
+      "fields": [
+        {
+          "name": "id",
+          "type": "uuid"
+        },
+        {
+          "name": "placed_at",
+          "type": "timestamptz"
+        },
+        {
+          "name": "tenant",
+          "type": "uuid"
+        },
+        {
+          "name": "total",
+          "type": "decimal(12,2)"
+        }
+      ],
+      "key": [
+        "tenant",
+        "id"
+      ],
+      "residency": "EU"
+    },
+    {
+      "name": "Payment",
+      "fields": [
+        {
+          "name": "amount",
+          "type": "decimal(12,2)"
+        },
+        {
+          "name": "id",
+          "type": "uuid"
+        }
+      ],
+      "key": [
+        "id"
+      ]
+    },
+    {
+      "name": "User",
+      "fields": [
+        {
+          "name": "email",
+          "type": "string"
+        },
+        {
+          "name": "id",
+          "type": "uuid"
+        }
+      ],
+      "key": [
+        "id"
+      ],
+      "pii": [
+        "email"
+      ]
+    }
+  ],
+  "relations": [
+    {
+      "name": "user",
+      "from": "Order",
+      "to": "User"
+    }
+  ],
+  "atomic": [
+    [
+      "Order",
+      "Payment"
+    ]
+  ],
+  "cost_ceiling": {
+    "amount": "750.00",
+    "currency": "EUR"
+  }
+}