From fd546874667685153be56fb370143896c08c310b Mon Sep 17 00:00:00 2001 From: Krzysztof Macewicz Date: Thu, 24 Sep 2026 23:40:56 +0200 Subject: [PATCH] feat: remove and replace indexes in place (sde-index protocol 2) A design that drops an index in force - one no read uses while every write maintains it, or one replaced by a better one - no longer needs a copy. Protocol 2 of the signed sde-index packet: the prepared indexes are the ones in force without the removed ones, byte for byte and in order, followed by the new ones, if any; at least one is removed. Both loaders read it (IndexPlan.removed, INDEX_CHANGE_PROTOCOL); migration/180-187 pin the acceptances and a refusal per changed rule, and migration/149 now refuses protocol 3. Protocol 1 is unchanged. The Python operator checks every removed index on its table in the declared shape before any DDL, builds and qualifies what is new as before, records its decision and publishes the next map, and only then removes each index, one resumable step each: DROP INDEX CONCURRENTLY IF EXISTS on PostgreSQL; on ClickHouse a pending materialization is killed and the index dropped with alter_sync = 0; the catalogue is read back. Absent, or another object under the name, after the decision means ours is gone; the other object is left alone. The receipt carries protocol 2 and a row per removed index once built, none once abandoned. A state holding such a record uses storage contract 5, which operators knowing contracts 1 to 4 refuse. Measured before the design (docs/qualification/in-place-index-drop): a concurrent drop pauses no write; an open transaction that has read the table holds it, an older snapshot alone does not; a stopped drop leaves the index invalid yet maintained, and a second drop removes it; ClickHouse drops at once with alter_sync = 0. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 7 +- conformance/README.md | 12 +- conformance/tools/index_vectors.py | 85 +++- .../keys.json | 2 +- .../plan.json | 8 +- .../index.json | 14 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 157 +++++++ .../index.json | 16 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 166 ++++++++ .../index.json | 17 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 189 +++++++++ .../index.json | 14 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 174 ++++++++ .../index.json | 5 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 174 ++++++++ .../index.json | 5 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 189 +++++++++ .../index.json | 5 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 166 ++++++++ .../index.json | 5 + .../keys.json | 3 + .../model.json | 102 +++++ .../plan.json | 171 ++++++++ docs/format-contract.md | 15 +- docs/in-place-index.md | 84 +++- docs/physical-design.md | 9 +- .../in-place-index-drop/README.md | 48 +++ .../in-place-index-drop/SHA256SUMS | 2 + .../in-place-index-drop/probe_drop.out.json | 60 +++ .../in-place-index-drop/probe_drop.py | 246 +++++++++++ python/src/sde/__init__.py | 2 + python/src/sde/_cutover_project.py | 29 +- python/src/sde/engines/_index_build.py | 44 ++ python/src/sde/index_build.py | 49 ++- python/src/sde/index_operator.py | 41 +- python/tests/test_conformance.py | 3 + python/tests/test_cutover_project.py | 40 ++ python/tests/test_index_change_live.py | 401 ++++++++++++++++++ python/tests/test_index_operator_live.py | 20 +- typescript/src/in-place-index.ts | 38 +- typescript/src/index.ts | 2 +- typescript/tests/conformance.test.ts | 5 +- 56 files changed, 3495 insertions(+), 64 deletions(-) create mode 100644 conformance/vectors/migration/180-index-change-removes-an-index-in-place/index.json create mode 100644 conformance/vectors/migration/180-index-change-removes-an-index-in-place/keys.json create mode 100644 conformance/vectors/migration/180-index-change-removes-an-index-in-place/model.json create mode 100644 conformance/vectors/migration/180-index-change-removes-an-index-in-place/plan.json create mode 100644 conformance/vectors/migration/181-index-change-replaces-an-index/index.json create mode 100644 conformance/vectors/migration/181-index-change-replaces-an-index/keys.json create mode 100644 conformance/vectors/migration/181-index-change-replaces-an-index/model.json create mode 100644 conformance/vectors/migration/181-index-change-replaces-an-index/plan.json create mode 100644 conformance/vectors/migration/182-index-change-keeps-the-others-in-order/index.json create mode 100644 conformance/vectors/migration/182-index-change-keeps-the-others-in-order/keys.json create mode 100644 conformance/vectors/migration/182-index-change-keeps-the-others-in-order/model.json create mode 100644 conformance/vectors/migration/182-index-change-keeps-the-others-in-order/plan.json create mode 100644 conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/index.json create mode 100644 conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/keys.json create mode 100644 conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/model.json create mode 100644 conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/plan.json create mode 100644 conformance/vectors/migration/184-index-change-removes-at-least-one/index.json create mode 100644 conformance/vectors/migration/184-index-change-removes-at-least-one/keys.json create mode 100644 conformance/vectors/migration/184-index-change-removes-at-least-one/model.json create mode 100644 conformance/vectors/migration/184-index-change-removes-at-least-one/plan.json create mode 100644 conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/index.json create mode 100644 conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/keys.json create mode 100644 conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/model.json create mode 100644 conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/plan.json create mode 100644 conformance/vectors/migration/186-index-change-names-bind-the-build-id/index.json create mode 100644 conformance/vectors/migration/186-index-change-names-bind-the-build-id/keys.json create mode 100644 conformance/vectors/migration/186-index-change-names-bind-the-build-id/model.json create mode 100644 conformance/vectors/migration/186-index-change-names-bind-the-build-id/plan.json create mode 100644 conformance/vectors/migration/187-index-change-keeps-the-key-order/index.json create mode 100644 conformance/vectors/migration/187-index-change-keeps-the-key-order/keys.json create mode 100644 conformance/vectors/migration/187-index-change-keeps-the-key-order/model.json create mode 100644 conformance/vectors/migration/187-index-change-keeps-the-key-order/plan.json create mode 100644 docs/qualification/in-place-index-drop/README.md create mode 100644 docs/qualification/in-place-index-drop/SHA256SUMS create mode 100644 docs/qualification/in-place-index-drop/probe_drop.out.json create mode 100644 docs/qualification/in-place-index-drop/probe_drop.py create mode 100644 python/tests/test_index_change_live.py diff --git a/README.md b/README.md index 9f06492..52946d8 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,9 @@ The Python [local cutover operator](docs/local-cutover.md) adds durable executio for signed generation-bearing packets, including native access changes for existing Python and TypeScript processes. [Staging](docs/staging.md) creates successive fresh copies while preserving the source and local recovery history; controller handoff and workload qualification are separate. -A design that only adds indexes is [built in place](docs/in-place-index.md), on the live tables and -without pausing a write, instead of through a copy. +A design that only changes indexes - adds, removes or replaces them - is carried out +[in place](docs/in-place-index.md), on the live tables and without pausing a write, instead of +through a copy. The opt-in [workload qualification](docs/cutover-qualification.md) runs installed Python and npm artifacts under mixed traffic and checks scheduled latency, native recovery and acknowledged values. [Session lifetime](docs/session-lifecycle.md) defines borrowed/owned connections and transaction @@ -197,7 +198,7 @@ Nothing about that fails at compile time. So the encoding is specified at the byte level in [`docs/format-contract.md`](docs/format-contract.md) — UTF-8, keys NFC-normalised then sorted by code point, no insignificant whitespace, minimal escaping, no float literals, a closed type vocabulary so that `Decimal` and `BigDecimal` land on the same bytes. -And [`conformance/`](conformance/) holds the vectors — **334 of them, in ten families** — that every +And [`conformance/`](conformance/) holds the vectors — **342 of them, in ten families** — that every library runs in its own test runner, so a divergence is a red test for whoever caused it rather than an operation written to the wrong engine in production. diff --git a/conformance/README.md b/conformance/README.md index 235b4f7..708d731 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -314,11 +314,15 @@ Both map signatures, exact unchanged source/routing, portable names and envelope validated without constructing engines. `tools/staging_vectors.py` uses the independent fixture encoder and OpenSSL. Native creation and crash recovery have separate live tests. -`migration/143`–`179` carry `plan.json` and `index.json` for signed in-place index builds: three -acceptances (a B-tree under a raised contract, several indexes after the ones in force, ClickHouse -data-skipping indexes) and one refusal per loader rule, each with the `match` fragment of the +`migration/143`–`187` carry `plan.json` and `index.json` for signed in-place index builds: three +acceptances of protocol 1 (a B-tree under a raised contract, several indexes after the ones in +force, ClickHouse data-skipping indexes), four of protocol 2 (a removal alone, a replacement, the +other indexes kept in order, an index an earlier build added removed - `index.json` names the +removed ones in `removed`) and one refusal per loader rule, each with the `match` fragment of the message both libraries must give. `tools/index_vectors.py` uses the independent fixture encoder and -OpenSSL. The native build, its recovery and abandonment have separate live tests. +OpenSSL; every vector is signed with a key of its own, so `--only` rewrites the named vectors and +leaves the others byte for byte. The native build, its removals, recovery and abandonment have +separate live tests. `migration/122`–`132` carry `bulk.json`: application-batch operations, expected source/copy call order, transaction outcomes, capability/bound refusals and exact value-free metric bytes. diff --git a/conformance/tools/index_vectors.py b/conformance/tools/index_vectors.py index ee58393..effcb72 100644 --- a/conformance/tools/index_vectors.py +++ b/conformance/tools/index_vectors.py @@ -155,6 +155,32 @@ def reused(plan: dict[str, Any]) -> None: document["groups"]["Order"]["source"]["layout"]["tables"]["Order"] = name(1) +OTHER = {"entity": "Event", "name": "event_at", "columns": ["at"], "method": "brin"} +BUILT = {"entity": "Event", "name": name(1, "7" * 32), "columns": ["id"]} +"""An index a previous in-place build added: a bound name of another build id.""" + + +def change(*prepared: dict[str, Any], kept: tuple[dict[str, Any], ...] = (KEPT,)) -> Callable[ + [dict[str, Any]], None +]: + """Protocol 2: the map in force carries ``kept``; the next map carries ``prepared``. + + With nothing left the next map has no ``indexes`` at all - the form a controller writes, for + which an empty list would claim that indexing was considered and none chosen. + """ + + def apply(plan: dict[str, Any]) -> None: + plan["protocol"] = 2 + with_kept(plan, *kept) + layout = source(plan["prepared"])["layout"] + if prepared: + layout["indexes"] = copy.deepcopy(list(prepared)) + else: + layout.pop("indexes", None) + + return apply + + def cases() -> list[tuple[str, Callable[[dict[str, Any]], None] | None, str | None]]: out: list[tuple[str, Callable[[dict[str, Any]], None] | None, str | None]] = [] @@ -193,7 +219,7 @@ def change(plan: dict[str, Any]) -> None: ) case( "149-index-build-refuses-an-unknown-protocol", - at("protocol", 2), + at("protocol", 3), "unsupported index build authorization kind or protocol", ) case( @@ -311,6 +337,41 @@ def change(plan: dict[str, Any]) -> None: REFUSED, ) case("179-index-build-model-is-bound", at("prepared/model_version", "0" * 16), REFUSED) + # Protocol 2: indexes the map in force declares are removed as well as added. + new_btree = {"entity": "Event", "name": name(1), "columns": ["at"]} + case("180-index-change-removes-an-index-in-place", change(), None) + case("181-index-change-replaces-an-index", change(new_btree), None) + case( + "182-index-change-keeps-the-others-in-order", + change(copy.deepcopy(OTHER), new_btree, kept=(KEPT, OTHER, BUILT)), + None, + ) + case( + "183-index-change-removes-an-index-a-build-added", + change(copy.deepcopy(KEPT), kept=(KEPT, BUILT)), + None, + ) + case( + "184-index-change-removes-at-least-one", + change(copy.deepcopy(KEPT), new_btree), + "index build protocol 2 removes at least one index in force", + ) + case( + "185-index-change-keeps-the-order-of-the-others", + change(copy.deepcopy(BUILT), copy.deepcopy(OTHER), kept=(KEPT, OTHER, BUILT)), + "an index change keeps the other indexes in force, in order, before the new ones", + ) + case( + "186-index-change-names-bind-the-build-id", + change({**new_btree, "name": name(1, "9" * 32)}), + bound, + ) + + def moved_key(plan: dict[str, Any]) -> None: + change(new_btree)(plan) + source(plan["prepared"])["layout"]["key_order"] = {"Event": ["id"]} + + case("187-index-change-keeps-the-key-order", moved_key, only_indexes) return out @@ -318,6 +379,13 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--i-am-changing-the-contract", action="store_true", required=True) parser.add_argument("--scratch-directory", type=Path, required=True) + parser.add_argument( + "--only", + action="append", + default=[], + help="write only these vectors (repeatable); every vector has its own key, so the others " + "stay byte for byte as they are", + ) args = parser.parse_args() scratch = args.scratch_directory.resolve() if not scratch.is_dir() or scratch.is_relative_to(ROOT): @@ -364,7 +432,10 @@ def sign(document: dict[str, Any]) -> None: "value": base64.b64encode((work / "sig").read_bytes()).decode(), } - written = cases() + written = [entry for entry in cases() if not args.only or entry[0] in args.only] + unknown = set(args.only) - {entry[0] for entry in cases()} + if unknown: + parser.error(f"no such vector: {sorted(unknown)}") for label, change, match in written: plan = template() if change: @@ -388,8 +459,10 @@ def sign(document: dict[str, Any]) -> None: if match is not None: expected.update(error="MigrationRefused", match=match) else: - indexes = source(plan["prepared"])["layout"]["indexes"] - kept = source(plan["current"])["layout"].get("indexes", []) + indexes = source(plan["prepared"])["layout"].get("indexes", []) + remaining = {index["name"] for index in indexes} + in_force = source(plan["current"])["layout"].get("indexes", []) + kept = [index for index in in_force if index["name"] in remaining] expected.update( index_fingerprint=hashlib.sha256(payload(plan)).hexdigest(), verified_with="index", @@ -400,6 +473,10 @@ def sign(document: dict[str, Any]) -> None: added=[index["name"] for index in indexes[len(kept) :]], build_budget_ms=plan["build_budget_ms"], ) + if plan["protocol"] == 2: + expected["removed"] = [ + index["name"] for index in in_force if index["name"] not in remaining + ] directory = VECTORS / "migration" / label directory.mkdir(exist_ok=True) for filename, value in { diff --git a/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/keys.json b/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/keys.json index cc181e6..5d9efb9 100644 --- a/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/keys.json +++ b/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/keys.json @@ -1,3 +1,3 @@ { - "index": "iDVAooL8PTov2s9FYFSnb+9EPl90sp72vut52dGqxIE=" + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" } diff --git a/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/plan.json b/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/plan.json index f975074..6cf22d2 100644 --- a/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/plan.json +++ b/conformance/vectors/migration/149-index-build-refuses-an-unknown-protocol/plan.json @@ -1,6 +1,6 @@ { "kind": "sde-index", - "protocol": 2, + "protocol": 3, "index_id": "66666666666666666666666666666666", "project_id": "11111111111111111111111111111111", "group": "Event", @@ -68,7 +68,7 @@ "signature": { "alg": "ed25519", "key_id": "index", - "value": "y9+HB9/iWdvlkzt7Z9CMm5p/fJTGbBh/jbGf+Yst7NiV87t9+hl8UWDssdPmgtJl+B5y/kwcx0KYoWU0kec3Dg==" + "value": "W1kR3q2Q5a4Ila6u5m3xUjal/TokoekT83uFEeoNrpOBcDaJk9eNUIDsXjGBQPDi2doa3WobndisMPIwiVVKDw==" } }, "prepared": { @@ -144,13 +144,13 @@ "signature": { "alg": "ed25519", "key_id": "index", - "value": "0k+qiBRWsoe67kAA6G7/XmJXJm67v2syco7ArvEs/hwYz3K7jSCHWEwk9CMDQu7Av1b2Ab1Uo5jgzhlKtMSkDA==" + "value": "DlWx0oPKcO22SkbsZyZAY3CmGRCxAz8SL0EUDuiI7L1KmiUGWXPJ9lX4C5jeuAQF87y74ufa8TMFZhqFJf3SBA==" } }, "build_budget_ms": 3600000, "signature": { "alg": "ed25519", "key_id": "index", - "value": "jCJ8blY4dZOgpNfaqKwJoplDLsGDxfU6eRMU1UzjSxU/KpXCOxGyzd4YsKRFXWYZwlQqhluTz/tYD34DA0jICA==" + "value": "ajO5x62pqzEdAdGAnJUmRsS5585bPCdROhBDs/6wn/aZw5kqck82DUqGssRtFPQEdHFaEI9HKrOpnrjxIxUNAA==" } } diff --git a/conformance/vectors/migration/180-index-change-removes-an-index-in-place/index.json b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/index.json new file mode 100644 index 0000000..1ac359d --- /dev/null +++ b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/index.json @@ -0,0 +1,14 @@ +{ + "project_id": "11111111111111111111111111111111", + "index_fingerprint": "d767bfb6c32f8652b02d4848224a4ae8116a52172b1ab433d9e58a91b831e945", + "verified_with": "index", + "map_fingerprints": { + "current": "8e0a65adc4e36dd53fbda3183cb3650c8731e9511cc0eac2fd3a2439e54af2cd", + "prepared": "505f854472806a82b9910d17ed175ccb81993af4a4ec7bf265774307e9cbb2ff" + }, + "added": [], + "build_budget_ms": 3600000, + "removed": [ + "event_name" + ] +} diff --git a/conformance/vectors/migration/180-index-change-removes-an-index-in-place/keys.json b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/keys.json new file mode 100644 index 0000000..d390386 --- /dev/null +++ b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/keys.json @@ -0,0 +1,3 @@ +{ + "index": "+loeROgpPde3TDVXd5d2OOqMUf46DKPmrEiHm6Pq+64=" +} diff --git a/conformance/vectors/migration/180-index-change-removes-an-index-in-place/model.json b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/180-index-change-removes-an-index-in-place/plan.json b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/plan.json new file mode 100644 index 0000000..536a987 --- /dev/null +++ b/conformance/vectors/migration/180-index-change-removes-an-index-in-place/plan.json @@ -0,0 +1,157 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "b0waP6eJtbSIQqFPR1OviYsphChdNROavG83blZOUwdGujA+GQ+O2qBsBhwvoejoh+bwf3v5O4VF7xuI+WcACQ==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + } + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "Xxne0GV8p13icPkrVxokum5iItrKhrfVMFajaZ1+NhxJ/cs9UJnW3DfzIdCUbOCs4S4AGnRvhQdjFBf3pcXnAQ==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "6Ccrlf3BncWVE8D55M2VIfPxEcx53f9ClCigUlBkYELCELWz92LAK4O7yk2Gfbuk7Fr4Rv0k4kB5VaZOXeUSCQ==" + } +} diff --git a/conformance/vectors/migration/181-index-change-replaces-an-index/index.json b/conformance/vectors/migration/181-index-change-replaces-an-index/index.json new file mode 100644 index 0000000..bd8e6d0 --- /dev/null +++ b/conformance/vectors/migration/181-index-change-replaces-an-index/index.json @@ -0,0 +1,16 @@ +{ + "project_id": "11111111111111111111111111111111", + "index_fingerprint": "1a8abeac6a5c7589e2a984fb075cb134321871a42886fab2272eeb5ea9ec5d14", + "verified_with": "index", + "map_fingerprints": { + "current": "8e0a65adc4e36dd53fbda3183cb3650c8731e9511cc0eac2fd3a2439e54af2cd", + "prepared": "fb7f0d7a12683f2fa8a8a58bad979a7a3c41c5f79431da2443419502193eb8b6" + }, + "added": [ + "sde_i_66666666666666666666666666666666_000001" + ], + "build_budget_ms": 3600000, + "removed": [ + "event_name" + ] +} diff --git a/conformance/vectors/migration/181-index-change-replaces-an-index/keys.json b/conformance/vectors/migration/181-index-change-replaces-an-index/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/181-index-change-replaces-an-index/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/181-index-change-replaces-an-index/model.json b/conformance/vectors/migration/181-index-change-replaces-an-index/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/181-index-change-replaces-an-index/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/181-index-change-replaces-an-index/plan.json b/conformance/vectors/migration/181-index-change-replaces-an-index/plan.json new file mode 100644 index 0000000..087ee4b --- /dev/null +++ b/conformance/vectors/migration/181-index-change-replaces-an-index/plan.json @@ -0,0 +1,166 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "N2EoGzsTIgLLV8UqgOkHGnB9yLHsgSvZm6cp0+dQP9e+iGTUbu832YHlRAyAFwwfXhbYM9ET/q/tRyUP45WkAA==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "sde_i_66666666666666666666666666666666_000001", + "columns": [ + "at" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "DlWx0oPKcO22SkbsZyZAY3CmGRCxAz8SL0EUDuiI7L1KmiUGWXPJ9lX4C5jeuAQF87y74ufa8TMFZhqFJf3SBA==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "cVpe6mzO/mdm1MzwIlnFIOZStB2BfaZmS4tYsMowgNML3CYq2jMa/b/ZOxR6tzFAK8FdxLYxr6bYoXgXWiJvAg==" + } +} diff --git a/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/index.json b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/index.json new file mode 100644 index 0000000..10df48e --- /dev/null +++ b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/index.json @@ -0,0 +1,17 @@ +{ + "project_id": "11111111111111111111111111111111", + "index_fingerprint": "7f54bf627c87bee37a8a1b7e00dd1354673bdf146a81afe476fdf2c2be374383", + "verified_with": "index", + "map_fingerprints": { + "current": "8b1d0368696efde3425bedd1bf63cdebad41067d688e90b395f9e7a5c2419bd8", + "prepared": "9fd04c664e4e280a85be91219a4e0968c7796d79ec6d43b675ea0c3c33f89eb3" + }, + "added": [ + "sde_i_66666666666666666666666666666666_000001" + ], + "build_budget_ms": 3600000, + "removed": [ + "event_name", + "sde_i_77777777777777777777777777777777_000001" + ] +} diff --git a/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/keys.json b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/model.json b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/plan.json b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/plan.json new file mode 100644 index 0000000..e3fff5a --- /dev/null +++ b/conformance/vectors/migration/182-index-change-keeps-the-others-in-order/plan.json @@ -0,0 +1,189 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + }, + { + "entity": "Event", + "name": "event_at", + "columns": [ + "at" + ], + "method": "brin" + }, + { + "entity": "Event", + "name": "sde_i_77777777777777777777777777777777_000001", + "columns": [ + "id" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "xxVivb0g5fpXiJmf3wWYieteQxSfOb560nio0E3bOzNWvdYPryoXOof6cTC8kF1ouq1maOA6Pli1FqFpXPVADQ==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_at", + "columns": [ + "at" + ], + "method": "brin" + }, + { + "entity": "Event", + "name": "sde_i_66666666666666666666666666666666_000001", + "columns": [ + "at" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "MnC2hzhLspq5O0vWN0+DrPKEKhSwoArvFsNL5qbtZfour/mlwht4GcOi9xm8tJELUzRen+dTtmVd3mQb3Me9Ag==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "1hNsqWC8QMDk4RYna3YQ4HXysp0zTh1x0o1cZluing/VoFaF7H6r0euHCbBUKW1Y5yQjxd9zjzm/z29iQxVDDQ==" + } +} diff --git a/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/index.json b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/index.json new file mode 100644 index 0000000..deacdee --- /dev/null +++ b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/index.json @@ -0,0 +1,14 @@ +{ + "project_id": "11111111111111111111111111111111", + "index_fingerprint": "09e7ee60deb768e1dae70a78a1d902cf2f4b08f4c1d8bc3ccc03ec5025e166f6", + "verified_with": "index", + "map_fingerprints": { + "current": "652ed0046750016096525f2981bebbe68a8474c816d7da3c33b878da0f4b4867", + "prepared": "5132e3433ae255bb6e1f5daa432459a7d4ae97c954423ae202409ffbeea19fad" + }, + "added": [], + "build_budget_ms": 3600000, + "removed": [ + "sde_i_77777777777777777777777777777777_000001" + ] +} diff --git a/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/keys.json b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/model.json b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/plan.json b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/plan.json new file mode 100644 index 0000000..2513d63 --- /dev/null +++ b/conformance/vectors/migration/183-index-change-removes-an-index-a-build-added/plan.json @@ -0,0 +1,174 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + }, + { + "entity": "Event", + "name": "sde_i_77777777777777777777777777777777_000001", + "columns": [ + "id" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "KhtxbxF8Q7ll3ri3FOON8ozRvve5XIgp3qTB5s9Qd8PpC0hyZ/AtcQchfHgJOP/Ls2GqW/y/nc0Gd4bz32wmBA==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "PzYNbT1eQV0wV0ZY4D3l+O5meGM7hsWNYomU04s7G9LYx03W4EvOT3AAQGbHqNS+L06R4VZVV/izjtEZha57Dg==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "quwo5TtB8ZB2GpC2/ikFwb/0XuGcS/0ZMFHzBZku6CMfMkMcrhF9QAuMhH8rEgfxO2BqDMs1aEgnSgk0fIP+Dg==" + } +} diff --git a/conformance/vectors/migration/184-index-change-removes-at-least-one/index.json b/conformance/vectors/migration/184-index-change-removes-at-least-one/index.json new file mode 100644 index 0000000..c987b8d --- /dev/null +++ b/conformance/vectors/migration/184-index-change-removes-at-least-one/index.json @@ -0,0 +1,5 @@ +{ + "project_id": "11111111111111111111111111111111", + "error": "MigrationRefused", + "match": "index build protocol 2 removes at least one index in force" +} diff --git a/conformance/vectors/migration/184-index-change-removes-at-least-one/keys.json b/conformance/vectors/migration/184-index-change-removes-at-least-one/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/184-index-change-removes-at-least-one/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/184-index-change-removes-at-least-one/model.json b/conformance/vectors/migration/184-index-change-removes-at-least-one/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/184-index-change-removes-at-least-one/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/184-index-change-removes-at-least-one/plan.json b/conformance/vectors/migration/184-index-change-removes-at-least-one/plan.json new file mode 100644 index 0000000..eb8c030 --- /dev/null +++ b/conformance/vectors/migration/184-index-change-removes-at-least-one/plan.json @@ -0,0 +1,174 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "N2EoGzsTIgLLV8UqgOkHGnB9yLHsgSvZm6cp0+dQP9e+iGTUbu832YHlRAyAFwwfXhbYM9ET/q/tRyUP45WkAA==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + }, + { + "entity": "Event", + "name": "sde_i_66666666666666666666666666666666_000001", + "columns": [ + "at" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "juZ2LZj3LQz2RoHjUPmAFY6/QMxzCJYHlFyf8oqjri9BNgpWqbDzFPNNuOCza57jyLGWV4dIG1+1nV/mJNcpAw==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "+RfOi2Y1WNZrH+PiUZEXI0wHK4xMyMijFexSwIIe+UVRTOpSj4saRC9LJiqdui8TDsPGaRGd8RqWY1iTmxeXDA==" + } +} diff --git a/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/index.json b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/index.json new file mode 100644 index 0000000..965054a --- /dev/null +++ b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/index.json @@ -0,0 +1,5 @@ +{ + "project_id": "11111111111111111111111111111111", + "error": "MigrationRefused", + "match": "an index change keeps the other indexes in force, in order, before the new ones" +} diff --git a/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/keys.json b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/model.json b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/plan.json b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/plan.json new file mode 100644 index 0000000..574863f --- /dev/null +++ b/conformance/vectors/migration/185-index-change-keeps-the-order-of-the-others/plan.json @@ -0,0 +1,189 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + }, + { + "entity": "Event", + "name": "event_at", + "columns": [ + "at" + ], + "method": "brin" + }, + { + "entity": "Event", + "name": "sde_i_77777777777777777777777777777777_000001", + "columns": [ + "id" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "xxVivb0g5fpXiJmf3wWYieteQxSfOb560nio0E3bOzNWvdYPryoXOof6cTC8kF1ouq1maOA6Pli1FqFpXPVADQ==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "sde_i_77777777777777777777777777777777_000001", + "columns": [ + "id" + ] + }, + { + "entity": "Event", + "name": "event_at", + "columns": [ + "at" + ], + "method": "brin" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "hznOXaH2dn2nMYj9OKO0iVULIkQ3gP9+DG4htiGu/sxvsTinxYjFOigmcDbnk9Z3qPVPxgiuCKmj3KovfPaiBA==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "sb4xPMrPQcD0M+P+DJJWSNiodtIt6ekxhaZVt2gyzVAW5IdnmebK7feTOJh6AR0a+W/hlIknI+USwQWzEe9xDw==" + } +} diff --git a/conformance/vectors/migration/186-index-change-names-bind-the-build-id/index.json b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/index.json new file mode 100644 index 0000000..7cd5655 --- /dev/null +++ b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/index.json @@ -0,0 +1,5 @@ +{ + "project_id": "11111111111111111111111111111111", + "error": "MigrationRefused", + "match": "new indexes need fresh names bound to the index build id and their position" +} diff --git a/conformance/vectors/migration/186-index-change-names-bind-the-build-id/keys.json b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/186-index-change-names-bind-the-build-id/model.json b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/186-index-change-names-bind-the-build-id/plan.json b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/plan.json new file mode 100644 index 0000000..6aebba4 --- /dev/null +++ b/conformance/vectors/migration/186-index-change-names-bind-the-build-id/plan.json @@ -0,0 +1,166 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "N2EoGzsTIgLLV8UqgOkHGnB9yLHsgSvZm6cp0+dQP9e+iGTUbu832YHlRAyAFwwfXhbYM9ET/q/tRyUP45WkAA==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "sde_i_99999999999999999999999999999999_000001", + "columns": [ + "at" + ] + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "e05LixrxJQqXeWFdsBRxo4Dse23qo7hu+6UStKvxGr3g3iEZpHsf2yIX2wR8lfR0u3kxVBquM/BOtdnNHvctAw==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "V1KI3Q/ZYJQaF+fbK4XgPtUa+X9o7pJR5zllsVuyBMCHzwAupzCw6UJfg+YeK2LOWYcJq7Ta28FYDFGOLxKJCw==" + } +} diff --git a/conformance/vectors/migration/187-index-change-keeps-the-key-order/index.json b/conformance/vectors/migration/187-index-change-keeps-the-key-order/index.json new file mode 100644 index 0000000..f6645fc --- /dev/null +++ b/conformance/vectors/migration/187-index-change-keeps-the-key-order/index.json @@ -0,0 +1,5 @@ +{ + "project_id": "11111111111111111111111111111111", + "error": "MigrationRefused", + "match": "an index build changes nothing about the source but its indexes" +} diff --git a/conformance/vectors/migration/187-index-change-keeps-the-key-order/keys.json b/conformance/vectors/migration/187-index-change-keeps-the-key-order/keys.json new file mode 100644 index 0000000..5d9efb9 --- /dev/null +++ b/conformance/vectors/migration/187-index-change-keeps-the-key-order/keys.json @@ -0,0 +1,3 @@ +{ + "index": "Uig6u4CnPdRjwBD1yZ/BM6wAvyMfDIZik7f2BAsOjeY=" +} diff --git a/conformance/vectors/migration/187-index-change-keeps-the-key-order/model.json b/conformance/vectors/migration/187-index-change-keeps-the-key-order/model.json new file mode 100644 index 0000000..a389c80 --- /dev/null +++ b/conformance/vectors/migration/187-index-change-keeps-the-key-order/model.json @@ -0,0 +1,102 @@ +{ + "entities": [ + { + "name": "Event", + "fields": [ + { + "name": "at", + "type": "timestamptz" + }, + { + "name": "id", + "type": "uuid" + }, + { + "name": "name", + "type": "string" + } + ], + "key": [ + "id" + ] + }, + { + "name": "Order", + "fields": [ + { + "name": "id", + "type": "uuid" + }, + { + "name": "placed_at", + "type": "timestamptz" + }, + { + "name": "tenant", + "type": "uuid" + }, + { + "name": "total", + "type": "decimal(12,2)" + } + ], + "key": [ + "tenant", + "id" + ], + "residency": "EU" + }, + { + "name": "Payment", + "fields": [ + { + "name": "amount", + "type": "decimal(12,2)" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ] + }, + { + "name": "User", + "fields": [ + { + "name": "email", + "type": "string" + }, + { + "name": "id", + "type": "uuid" + } + ], + "key": [ + "id" + ], + "pii": [ + "email" + ] + } + ], + "relations": [ + { + "name": "user", + "from": "Order", + "to": "User" + } + ], + "atomic": [ + [ + "Order", + "Payment" + ] + ], + "cost_ceiling": { + "amount": "750.00", + "currency": "EUR" + } +} diff --git a/conformance/vectors/migration/187-index-change-keeps-the-key-order/plan.json b/conformance/vectors/migration/187-index-change-keeps-the-key-order/plan.json new file mode 100644 index 0000000..fd591cf --- /dev/null +++ b/conformance/vectors/migration/187-index-change-keeps-the-key-order/plan.json @@ -0,0 +1,171 @@ +{ + "kind": "sde-index", + "protocol": 2, + "index_id": "66666666666666666666666666666666", + "project_id": "11111111111111111111111111111111", + "group": "Event", + "current": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 1, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "event_name", + "columns": [ + "name" + ], + "method": "btree" + } + ] + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "N2EoGzsTIgLLV8UqgOkHGnB9yLHsgSvZm6cp0+dQP9e+iGTUbu832YHlRAyAFwwfXhbYM9ET/q/tRyUP45WkAA==" + } + }, + "prepared": { + "contract": 5, + "project_id": "11111111111111111111111111111111", + "model_version": "59a263d15793fb78", + "map_version": 2, + "groups": { + "Event": { + "write_epoch": 1, + "source": { + "engine": "pg-main", + "id": "Event@pg", + "layout": { + "tables": { + "Event": "event_source" + }, + "columns": { + "Event": { + "at": "timestamptz", + "id": "uuid", + "name": "text" + } + }, + "indexes": [ + { + "entity": "Event", + "name": "sde_i_66666666666666666666666666666666_000001", + "columns": [ + "at" + ] + } + ], + "key_order": { + "Event": [ + "id" + ] + } + } + } + }, + "Order": { + "write_epoch": 7, + "source": { + "engine": "pg-main", + "id": "Order@pg", + "layout": { + "tables": { + "Order": "order", + "Payment": "payment", + "User": "user" + }, + "columns": { + "Order": { + "id": "uuid", + "placed_at": "timestamptz", + "tenant": "uuid", + "total": "numeric(12,2)", + "user_id": "uuid" + }, + "Payment": { + "amount": "numeric(12,2)", + "id": "uuid" + }, + "User": { + "email": "text", + "id": "uuid" + } + } + } + } + } + }, + "routing": { + "2477d087f39d0ae4": "Event@pg", + "12f8b2171bc2bf78": "Order@pg" + }, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "9zKSS/4S0PtduYpr4brFWtppdmeyEkl5o1vmjbk9B55xmn+npOl6POZI/tYz45GrEdoamW8YChfAAhGowS6gAQ==" + } + }, + "build_budget_ms": 3600000, + "signature": { + "alg": "ed25519", + "key_id": "index", + "value": "+uz7nGYLElrVM90V7OW0W2d8B3Kf9BRgmWIcDrZvQtbsk8+bW6vMtVMTYUDBxZsoy6+EDiFo/uuBpi4pJCrjCQ==" + } +} diff --git a/docs/format-contract.md b/docs/format-contract.md index 27575f2..4c97773 100644 --- a/docs/format-contract.md +++ b/docs/format-contract.md @@ -1618,11 +1618,14 @@ contract from 4 to 5 - the fresh copy is where a physical design first appears - ## 7j. Signed in-place index build packet -[In-place index builds, protocol 1](in-place-index.md) authorize indexes added to the tables a +[In-place index builds](in-place-index.md) authorize a change to the indexes of the tables a group's source already uses, with no copy and no cutover: the exact signed map in force and a next -map that differs from it only by new indexes after the ones in force, named -`sde_i__`, under the same write generation. The prepared map may raise the -contract from 4 to 5 and may not lower it. Both SDKs load and validate the packet; -`migration/143`-`179` pin its acceptances and one refusal per rule, each refusal with the message -fragment both libraries give. The Python local operator executes the build, its recovery and its +map that differs from it only by its indexes, under the same write generation. Protocol 1 keeps +every index in force, in order, and adds at least one after them, named +`sde_i__`. Protocol 2 removes at least one index in force - each one whose name +the next map no longer carries - keeps the others byte for byte in their order, and adds new ones +after them under the same naming rule, possibly none. The prepared map may raise the contract from +4 to 5 and may not lower it. Both SDKs load and validate the packet; `migration/143`-`187` pin its +acceptances and one refusal per rule, each refusal with the message fragment both libraries give. +The Python local operator executes the build, the removals after its decision, its recovery and its abandonment. Loading an authorization builds nothing and activates no map. diff --git a/docs/in-place-index.md b/docs/in-place-index.md index 7056921..2a5a20c 100644 --- a/docs/in-place-index.md +++ b/docs/in-place-index.md @@ -1,7 +1,8 @@ # Build an index in place A signed in-place build authorization adds indexes to the tables a group already uses, while the -application keeps writing to them. The Python `LocalCutover.index` executor builds them in the +application keeps writing to them, and from protocol 2 also removes indexes the map in force +declares. The Python `LocalCutover.index` executor builds them in the customer's environment with the engines' own non-blocking forms, confirms them from the engine's catalogue, and publishes the next map, which differs from the map in force only by declaring them. No row is copied, no write generation is raised, and no process is fenced. @@ -16,8 +17,8 @@ on PostgreSQL the copy path's cutover ran into the 30 s watchdog and was rolled in-place build was published in 618 ms. Measurements and scripts: [qualification/in-place-index](qualification/in-place-index/README.md). -Only adding indexes can be done in place. A new key order, a partition, other tables or another -engine still need a copy, and so does dropping or changing an index in force. +Only indexes can be changed in place: added (protocol 1), and removed or replaced (protocol 2). A +new key order, a partition, other tables or another engine still need a copy. ## Signed packet protocol 1 @@ -71,6 +72,28 @@ naming rule; names are 45 ASCII bytes. `migration/143`-`179` are shared fixtures with OpenSSL rather than by either implementation, and every refusal names the fragment of the message both libraries give. +## Signed packet protocol 2: removing and replacing indexes + +A design can also drop an index: one no read uses while every write maintains it, or one replaced +by a better one. Protocol 2 has the same envelope with `"protocol": 2`. Rules 1-5, 7 and 8 hold as +they are, and rule 6 becomes: + +6. the prepared indexes are the ones in force without the removed ones, byte for byte and in their + order in force, followed by the new indexes, if any, named as in protocol 1. An index is removed + when the prepared map no longer carries its name, and at least one is. + +A removed index was named by a design or by an earlier build, so its name follows no rule here; a +new one still binds this build's `index_id`, and rule 7 keeps a removed name from coming back as a +new one. An index kept under its name must be kept byte for byte, so changing one is a removal and +an addition under a new name. Protocol 1 is unchanged - every index in force stays - and a +controller that removes nothing keeps issuing it, which operators without protocol 2 execute. + +`IndexPlan.removed` holds the definitions in force that the next map drops, in their order in force, +and `protocol` the packet's protocol; in protocol 2 `added` may be empty. `INDEX_CHANGE_PROTOCOL` is +exported by both libraries. `migration/180`-`187` pin the acceptances - a removal alone, a +replacement, the other indexes kept in order, an index an earlier build added removed - and a +refusal for each rule protocol 2 changes, with `migration/149` refusing protocol 3. + ## Local execution Use the [operator configuration](local-cutover.md#local-configuration-and-command-handoff) already @@ -88,7 +111,10 @@ source's native endpoint and table identities; each table's write barrier comple generation and **without any hold** - a build does not start beside another operation's barrier; watermarks not newer than the map in force; the runtime logins as staging checks them; the design in force read back from the catalogue without a finding; and nothing foreign under any new -name. A refusal at this point leaves no state behind. +name. A change also checks each index it removes: it must be on the bound table in the shape the map +in force declares - ready or, on PostgreSQL, left invalid by a stopped drop. An absent index or +another object under the name refuses here, because removing an index the table does not hold as +declared would publish a map about another table. A refusal at this point leaves no state behind. **PostgreSQL** builds each index with `CREATE INDEX CONCURRENTLY`, never with `IF NOT EXISTS`. An index of the bound name on the bound table, of the declared method and columns, not unique, without @@ -116,6 +142,27 @@ Without a recorded decision, recovery builds on - unlike a cutover, where the so until a decision and recovery therefore aborts. Here nothing waits, and an index that gets built is harmless. After `built`, recovery only finishes the publication. +**Removal follows the decision.** A change removes nothing before `built` is recorded and the next +map published. Then each removed index is a step of its own: PostgreSQL `DROP INDEX CONCURRENTLY`; +ClickHouse kills an unfinished materialization of that index, if any, and runs `ALTER TABLE ... +DROP INDEX ... SETTINGS alter_sync = 0`; the catalogue is then read again and must no longer list +it. Neither pauses a write - measured on 100 000 rows, 6.3 ms on PostgreSQL with the longest gap +between two writes 7.9 ms (10.2 ms before), 13.2 ms on ClickHouse +([qualification/in-place-index-drop](qualification/in-place-index-drop/README.md)). A process still +on the map in force keeps writing and reading; one that opens that map after the removal reports the +index missing as a [physical finding](physical-design.md) and serves rows all the same. + +A concurrent drop waits for every transaction that holds a lock on the table when it starts: an +open transaction that has read the table, an idle session left in one included. Unlike a build, an +older snapshot alone does not hold it (measured). The build budget bounds that wait as well. When it +ends, the next map is already in force; the drop may run on in the server until the transaction it +waits for ends, or stop and leave the index invalid - no query uses it, writes still maintain it - +and `resume` finishes the removal either way. Recovery repeats a removal the same way: an index of +the declared shape, finished or left invalid, is dropped, `IF EXISTS` so that an earlier drop +finishing meanwhile is not an error; an absent one or another object under the name means this +change's index is gone, and the other object is left alone. The receipt exists only after the last +removal. + The deadline is the signed build budget, not the 30-second watchdog of staging and cutover. Nothing is paused while an index builds, so the budget is not a pause budget; it bounds a build on a server that stopped answering. When it ends a build the operator closes its connections and stops; resume @@ -130,8 +177,9 @@ then removes this build's own indexes: PostgreSQL with `DROP INDEX CONCURRENTLY` killing the materialization if it is still running and dropping the index with `SETTINGS alter_sync = 0`, which leaves the catalogue at once instead of waiting for the merge pool (measured: the default form waits for as long as merges are stopped). Objects under the bound names -that are not this build's are left alone. The map in force, the watermarks and every process stay -as they were, and the prepared map version stays burned - the authorization is spent. Abandonment +that are not this build's are left alone. A change abandoned this way removes no index of the map in +force - it removes those only after its decision. The map in force, the watermarks and every process +stay as they were, and the prepared map version stays burned - the authorization is spent. Abandonment needs only the same native database and the bound tables; it does not need a barrier-free table or unchanged logins, which is what keeps it available when a build cannot finish. @@ -143,20 +191,27 @@ it. The metadata-only receipt contains `protocol: 1`, `index_id`, `index_fingerprint`, `project_id`, `group`, `outcome` (`built` or `abandoned`), `map_version` and `map_fingerprint` of the map active afterwards (the prepared map, or the map in force), `indexes` (engine binding, entity, index name -and the table's native identity for each), `elapsed_ms` and `recovered`. No rows, values or -credentials. The controller validates it against the exact packet it reserved before recording the +and the table's native identity for each), `elapsed_ms` and `recovered`. Protocol 2 carries +`protocol: 2` and adds `removed`: a row per removed index with the same four fields when the outcome +is `built`, and none when it is `abandoned`. No rows, values or credentials. The controller validates it against the exact packet it reserved before recording the outcome. After the first build, `project.json` uses storage contract 3, which adds the `indexes` history; operators that know contracts 1 and 2 refuse it rather than ignore a history they would not keep. +A state holding a protocol-2 record, executing or kept, uses storage contract 5: operators that know +contracts 1 to 4 refuse it rather than resume a change unaware that removals follow its decision, +and an envelope of an earlier contract holding such a record is refused as well. Stage receipts, cutover decisions and retired names are carried unchanged. A retry of a completed build returns the same receipt and reconfirms the directory's durability first; a retry of an abandoned one returns the abandonment. ## What this does not do -- It does not drop, rename or change an index in force, and it does not change a key order, a - partition, a table or an engine. Those are relayouts or moves, through a copy. +- It does not rename an index or change one in place - a change removes it and adds a new one under + a new name - and it does not change a key order, a partition, a table or an engine. Those are + relayouts or moves, through a copy. +- It does not remove an index the map in force does not declare. One created by hand is neither + verified nor touched. - It does not adopt an index somebody created under a bound name, even one of the right shape on the right table: a unique or partial index, or one on another table, is refused, and so is a table or view holding the name. @@ -179,5 +234,10 @@ a leftover that is dropped and rebuilt, ClickHouse's mutation is found again, no abandonment before the decision and its refusal after; an interrupted abandonment finished by resume or by a second abandonment; foreign objects under the bound name refused before any DDL and left alone by abandonment; another operation's barrier refused before and during a build; the -build budget ending a held build; the command line. `python/tests/test_physical_live.py` pins the -PostgreSQL physical check that reports a unique or unfinished index. +build budget ending a held build; the command line. `python/tests/test_index_change_live.py`, on +both engines: an index in force removed while a process on the map in force writes, and one replaced; +recovery after every step after the decision; abandonment before the decision leaving every index +in force; an index in force absent or of another shape refused before any DDL. On PostgreSQL also a +removal held by an open transaction that read the table until the budget ends and then resumed, and +a killed operator mid-removal whose drop is resumed. `python/tests/test_physical_live.py` pins the PostgreSQL physical +check that reports a unique, unfinished or absent index. diff --git a/docs/physical-design.md b/docs/physical-design.md index c90b959..fb6e403 100644 --- a/docs/physical-design.md +++ b/docs/physical-design.md @@ -106,8 +106,8 @@ What happens with a difference depends on who asked: - **`prepare_schema` / `prepareSchema` refuses it**, naming the table, the aspect and both values. That is where a person applying a map can act on it. A new layout belongs in fresh tables - the staging protocol creates them under new names - not in the old ones under a new declaration. The - one exception is adding indexes, which a signed [in-place build](in-place-index.md) does on the - tables in force, without a copy. + one exception is indexes, which a signed [in-place build](in-place-index.md) adds and removes on + the tables in force, without a copy. PostgreSQL indexes are built only after the table's key is confirmed, so a refused provisioning does not first build an index on the old table: `CREATE INDEX` without `CONCURRENTLY` blocks that table's writes while it builds. @@ -136,8 +136,9 @@ a physical design first appears, and may not lower it. `migration/133`-`137` pin - It does not change an existing table's columns, key order or partition. A different design means a new table, created by staging and switched to by cutover. The only object this library creates on - a table in force is an index a signed [in-place build](in-place-index.md) adds - and the only one - it drops is such an index of its own, when that build is abandoned. + a table in force is an index a signed [in-place build](in-place-index.md) adds, and the only ones + it drops are such an index of its own, when that build is abandoned, and an index of the map in + force that a signed change removes after its decision. - It does not partition PostgreSQL, partition by week, partition on a non-key column or on a zoneless timestamp, or accept an expression anywhere. - It does not verify indexes a map does not declare. An extra index added outside SDE is left alone. diff --git a/docs/qualification/in-place-index-drop/README.md b/docs/qualification/in-place-index-drop/README.md new file mode 100644 index 0000000..d9edc3d --- /dev/null +++ b/docs/qualification/in-place-index-drop/README.md @@ -0,0 +1,48 @@ +# Removing an index from a live table: evidence + +What [removing and replacing indexes in place](../../in-place-index.md#signed-packet-protocol-2-removing-and-replacing-indexes) +rests on, measured before the design and kept with the script that measured it. It ran on the SDK's +own test engines in containers on one laptop (i3-7100U, 2 cores / 4 threads, ~16 GB), PostgreSQL +15.19 and ClickHouse 24.8.14.39, on 24 September 2026. The machine was shared with other work, so +the timings are indicative. + +`probe_drop.py` -> `probe_drop.out.json`, with a writer inserting a row every 5 ms throughout: + +| engine | what | result | +|---|---|---| +| PostgreSQL | `DROP INDEX CONCURRENTLY` on 100 000 rows | 6.3 ms; the longest gap between two writes 7.9 ms (10.2 ms before) | +| PostgreSQL | the same drop beside a `REPEATABLE READ` transaction that has taken its snapshot but not touched the table | finished in 6.5 ms; the index is gone | +| PostgreSQL | the same drop beside a `REPEATABLE READ` transaction that has read the table and stays open | still running after 3 s, waiting on `Lock` / `virtualxid`; meanwhile the index is `valid = false`, `ready = true` - no query uses it, writes still maintain it; the longest write gap 14.4 ms | +| PostgreSQL | that drop terminated (`pg_terminate_backend`) | the dropping session ends with `AdminShutdown`; the index is left `valid = false`, `ready = true` | +| PostgreSQL | a second `DROP INDEX CONCURRENTLY` | 10.7 ms; the index is gone | +| ClickHouse | `ALTER TABLE ... DROP INDEX ... SETTINGS alter_sync = 0` on 100 000 rows | 13.2 ms; no longer listed in `system.data_skipping_indices`; a read the index served still answers; the longest write gap 12.1 ms (19.8 ms before) | + +No write failed: 634 inserts on PostgreSQL, 174 on ClickHouse. + +A concurrent drop waits for the transactions that hold a lock on the table, not for older +snapshots as a concurrent build does - PostgreSQL's `index_drop` waits for the table's lockers. +The first run of this probe, without the second row, called the third one "held by an old +snapshot"; its transaction had read the table, and the second row shows that the reading, not the +snapshot, is what holds the drop. The live tests hold removals with a transaction that has read the +table for that reason. All numbers here are from the run with the second row. + +What the operator's rules take from it: + +- A removal waits for no write and holds none, so it needs no barrier - but it cannot be undone. + The operator therefore removes only after its decision and the publication of the next map, one + resumable step per index; a process still on the map in force loses nothing but the index's help. +- An open transaction that has read the table holds a concurrent drop. The signed build budget + bounds the wait; after it the drop may run on in the server until that transaction ends, or stop + and leave the index invalid and maintained, and `resume` finishes the removal. +- A stopped drop leaves an index of the declared shape that is not valid, and a second drop removes + it. Recovery therefore drops an index of the declared shape whether it is valid or not, with + `IF EXISTS` for an earlier drop that finishes meanwhile, and treats an absent one - or another + object under the name, which it leaves alone - as already removed. +- ClickHouse drops with `alter_sync = 0`, as abandonment already does: without it the ALTER waits + for the merge pool ([in-place-index](../in-place-index/README.md), `probe_merge_pool.py`). + +A process on a map that declares an index the table no longer holds reports it as a physical +finding - `absent` on PostgreSQL, `unverified` where a restricted login cannot read the index +catalogue - and keeps serving rows: `python/tests/test_physical_live.py`. + +`SHA256SUMS` covers the script and its output. diff --git a/docs/qualification/in-place-index-drop/SHA256SUMS b/docs/qualification/in-place-index-drop/SHA256SUMS new file mode 100644 index 0000000..eb81c08 --- /dev/null +++ b/docs/qualification/in-place-index-drop/SHA256SUMS @@ -0,0 +1,2 @@ +17868159ee0a1e01f823d6b3f3f80101afdf6cf9a544bfdfd6f30daff39af46e probe_drop.py +10edb67b31425360194354525fed282616e2198fc09217379548f5d9b8ee18e1 probe_drop.out.json diff --git a/docs/qualification/in-place-index-drop/probe_drop.out.json b/docs/qualification/in-place-index-drop/probe_drop.out.json new file mode 100644 index 0000000..cff1e86 --- /dev/null +++ b/docs/qualification/in-place-index-drop/probe_drop.out.json @@ -0,0 +1,60 @@ +{ + "at": "2026-09-24T21:12:18Z", + "postgres": { + "server_version": "15.19", + "plain_drop": { + "rows": 100000, + "drop_ms": 6.3, + "longest_write_gap_ms_before": 10.2, + "longest_write_gap_ms_during": 7.9 + }, + "beside_an_old_snapshot_that_did_not_read_the_table": { + "finished_within_3_s": true, + "drop_ms": 6.5, + "index_after": "gone" + }, + "held_by_a_transaction_that_read_the_table": { + "still_running_after_3_s": true, + "activity": [ + "active", + "Lock", + "virtualxid" + ], + "index_while_waiting": { + "valid": false, + "ready": true + }, + "longest_write_gap_ms_while_waiting": 14.4 + }, + "terminated_drop": { + "dropper_outcome": "AdminShutdown", + "index_left": { + "valid": false, + "ready": true + } + }, + "second_drop": { + "drop_ms": 10.7, + "index_after": "gone" + }, + "writer": { + "inserts": 634, + "failures": 0 + } + }, + "clickhouse": { + "server_version": "24.8.14.39", + "drop": { + "rows": 100000, + "alter_ms": 13.2, + "listed_in_the_catalogue_after": 0, + "a_read_the_index_served_answers": true, + "longest_write_gap_ms_before": 19.8, + "longest_write_gap_ms_during": 12.1 + }, + "writer": { + "inserts": 174, + "failures": 0 + } + } +} diff --git a/docs/qualification/in-place-index-drop/probe_drop.py b/docs/qualification/in-place-index-drop/probe_drop.py new file mode 100644 index 0000000..50c23e4 --- /dev/null +++ b/docs/qualification/in-place-index-drop/probe_drop.py @@ -0,0 +1,246 @@ +"""What removing an index from a live table does, on both engines, before the design relies on it. + +Usage (the SDK's test engines): + SDE_POSTGRES_DSN=... SDE_CLICKHOUSE_DSN=... python probe_drop.py > probe_drop.out.json + +PostgreSQL: ``DROP INDEX CONCURRENTLY`` while a writer inserts a row every 5 ms - the longest gap +between writes; whether an old snapshot that has not touched the table holds it, and whether an +open transaction that has read the table does; what a terminated drop leaves in the catalogue, and +whether a second drop removes that. ClickHouse: ``ALTER TABLE ... DROP INDEX`` with +``alter_sync = 0`` while the writer runs, and whether a read the index served still answers. +Everything is created in a probe namespace of its own and removed at the end. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +import uuid +from typing import Any + +import clickhouse_connect +import psycopg +from psycopg import sql + +ROWS = 100_000 + + +class Writer(threading.Thread): + """Inserts one row every 5 ms and records the longest gap between two successful inserts.""" + + def __init__(self, insert: Any) -> None: + super().__init__(daemon=True) + self.insert, self.stop, self.gaps, self.failures = insert, threading.Event(), [], 0 + + def run(self) -> None: + last, number = time.monotonic(), 10_000_000 + while not self.stop.is_set(): + try: + self.insert(number) + now = time.monotonic() + self.gaps.append(now - last) + last = now + except Exception: + self.failures += 1 + number += 1 + time.sleep(0.005) + + def longest(self, since: int = 0) -> float: + return round(max(self.gaps[since:], default=0.0) * 1000, 1) + + +def postgres(dsn: str) -> dict[str, Any]: + schema = "sde_probe_drop_" + uuid.uuid4().hex[:12] + out: dict[str, Any] = {"server_version": None} + admin = psycopg.connect(dsn, autocommit=True) + try: + out["server_version"] = admin.execute("SHOW server_version").fetchone()[0] + admin.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + admin.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(schema))) + admin.execute("CREATE TABLE t (id bigint PRIMARY KEY, v int NOT NULL)") + admin.execute("INSERT INTO t SELECT g, g %% 1000 FROM generate_series(1, %s) g", (ROWS,)) + options = "-csearch_path=" + schema + + def connection() -> psycopg.Connection[Any]: + return psycopg.connect(dsn, autocommit=True, options=options) + + writer_cx = connection() + writer = Writer(lambda n: writer_cx.execute("INSERT INTO t VALUES (%s, %s)", (n, n % 1000))) + + # 1. A plain concurrent drop, with the writer running. + admin.execute("CREATE INDEX i_plain ON t (v)") + writer.start() + time.sleep(1.0) + before = writer.longest() + mark = len(writer.gaps) + started = time.monotonic() + admin.execute("DROP INDEX CONCURRENTLY i_plain") + out["plain_drop"] = { + "rows": ROWS, + "drop_ms": round((time.monotonic() - started) * 1000, 1), + "longest_write_gap_ms_before": before, + } + time.sleep(0.5) + out["plain_drop"]["longest_write_gap_ms_during"] = writer.longest(mark) + + # 2a. An old snapshot on another connection that has not touched the table. A concurrent + # build waits for every older snapshot; does a concurrent drop? + admin.execute("CREATE INDEX i_snapshot ON t (v)") + snapshot = connection() + snapshot.autocommit = False + snapshot.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ + snapshot.execute("SELECT 1") # takes the snapshot; holds no lock on t + snapshot_dropper = connection() + snapshot_result: dict[str, Any] = {} + + def drop_beside_a_snapshot() -> None: + began = time.monotonic() + snapshot_dropper.execute("DROP INDEX CONCURRENTLY i_snapshot") + snapshot_result["ms"] = round((time.monotonic() - began) * 1000, 1) + + thread = threading.Thread(target=drop_beside_a_snapshot, daemon=True) + thread.start() + thread.join(3.0) + out["beside_an_old_snapshot_that_did_not_read_the_table"] = { + "finished_within_3_s": not thread.is_alive(), + "drop_ms": snapshot_result.get("ms"), + "index_after": "gone" + if admin.execute("SELECT to_regclass('i_snapshot')").fetchone()[0] is None + else "present", + } + snapshot.rollback() + snapshot.close() + thread.join(10) + snapshot_dropper.close() + + # 2b. An open transaction on another connection that has read the table: it holds a lock + # on the table until it ends. Does the drop wait for it? + admin.execute("CREATE INDEX i_held ON t (v)") + holder = connection() + holder.autocommit = False + holder.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ + holder.execute("SELECT count(*) FROM t WHERE v = 7") # AccessShareLock on t, kept open + dropper = connection() + pid = dropper.execute("SELECT pg_backend_pid()").fetchone()[0] + result: dict[str, Any] = {} + + def drop_held() -> None: + began = time.monotonic() + try: + dropper.execute("DROP INDEX CONCURRENTLY i_held") + result["outcome"] = "dropped" + except Exception as exc: # terminated below + result["outcome"] = type(exc).__name__ + result["ms"] = round((time.monotonic() - began) * 1000, 1) + + thread = threading.Thread(target=drop_held, daemon=True) + mark = len(writer.gaps) + thread.start() + time.sleep(3.0) + waiting = admin.execute( + "SELECT state, wait_event_type, wait_event FROM pg_stat_activity WHERE pid = %s", (pid,) + ).fetchone() + catalogue = admin.execute( + "SELECT i.indisvalid, i.indisready FROM pg_index i " + "WHERE i.indexrelid = to_regclass('i_held')" + ).fetchone() + out["held_by_a_transaction_that_read_the_table"] = { + "still_running_after_3_s": thread.is_alive(), + "activity": list(waiting) if waiting else None, + "index_while_waiting": {"valid": catalogue[0], "ready": catalogue[1]} + if catalogue + else "gone", + "longest_write_gap_ms_while_waiting": writer.longest(mark), + } + + # 3. The waiting drop terminated: what does it leave? + admin.execute("SELECT pg_terminate_backend(%s)", (pid,)) + thread.join(10) + left = admin.execute( + "SELECT i.indisvalid, i.indisready FROM pg_index i " + "WHERE i.indexrelid = to_regclass('i_held')" + ).fetchone() + out["terminated_drop"] = { + "dropper_outcome": result.get("outcome"), + "index_left": {"valid": left[0], "ready": left[1]} if left else "gone", + } + holder.rollback() + holder.close() + + # 4. A second concurrent drop over what the terminated one left. + started = time.monotonic() + admin.execute("DROP INDEX CONCURRENTLY i_held") + out["second_drop"] = { + "drop_ms": round((time.monotonic() - started) * 1000, 1), + "index_after": "gone" + if admin.execute("SELECT to_regclass('i_held')").fetchone()[0] is None + else "present", + } + writer.stop.set() + writer.join(5) + out["writer"] = {"inserts": len(writer.gaps), "failures": writer.failures} + writer_cx.close() + dropper.close() + finally: + admin.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(schema))) + admin.close() + return out + + +def clickhouse(dsn: str) -> dict[str, Any]: + database = "sde_probe_drop_" + uuid.uuid4().hex[:12] + root = clickhouse_connect.get_client(dsn=dsn) + out: dict[str, Any] = {"server_version": root.server_version} + try: + root.command(f"CREATE DATABASE {database} ENGINE = Atomic") + client = clickhouse_connect.get_client(dsn=dsn, database=database) + client.command( + "CREATE TABLE t (id Int64, v Int32, INDEX i_v v TYPE minmax GRANULARITY 1) " + "ENGINE = ReplacingMergeTree ORDER BY id" + ) + client.command(f"INSERT INTO t SELECT number, number % 1000 FROM numbers({ROWS})") + writer_client = clickhouse_connect.get_client(dsn=dsn, database=database) + writer = Writer(lambda n: writer_client.command(f"INSERT INTO t VALUES ({n}, {n % 1000})")) + writer.start() + time.sleep(1.0) + before = writer.longest() + mark = len(writer.gaps) + answer = client.query("SELECT count() FROM t FINAL WHERE v >= 990").result_rows[0][0] + started = time.monotonic() + client.command("ALTER TABLE t DROP INDEX i_v SETTINGS alter_sync = 0") + dropped_ms = round((time.monotonic() - started) * 1000, 1) + listed = client.query( + "SELECT count() FROM system.data_skipping_indices " + "WHERE database = currentDatabase() AND table = 't' AND name = 'i_v'" + ).result_rows[0][0] + after = client.query("SELECT count() FROM t FINAL WHERE v >= 990").result_rows[0][0] + time.sleep(0.5) + writer.stop.set() + writer.join(5) + out["drop"] = { + "rows": ROWS, + "alter_ms": dropped_ms, + "listed_in_the_catalogue_after": listed, + "a_read_the_index_served_answers": after >= answer, + "longest_write_gap_ms_before": before, + "longest_write_gap_ms_during": writer.longest(mark), + } + out["writer"] = {"inserts": len(writer.gaps), "failures": writer.failures} + finally: + root.command(f"DROP DATABASE IF EXISTS {database} SYNC") + return out + + +if __name__ == "__main__": + print( + json.dumps( + { + "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "postgres": postgres(os.environ["SDE_POSTGRES_DSN"]), + "clickhouse": clickhouse(os.environ["SDE_CLICKHOUSE_DSN"]), + }, + indent=2, + ) + ) diff --git a/python/src/sde/__init__.py b/python/src/sde/__init__.py index 07c0571..424f0aa 100644 --- a/python/src/sde/__init__.py +++ b/python/src/sde/__init__.py @@ -55,6 +55,7 @@ class Meta: from .groups import Group, colocation_groups, group_of from .hashing import NameMap, hash_identifiers, load_or_create_salt from .index_build import ( + INDEX_CHANGE_PROTOCOL, INDEX_PROTOCOL, IndexPlan, IndexReceipt, @@ -163,6 +164,7 @@ class Meta: "DIALECTS", "DIALECT_PRECISION", "FIXED_SCHEMA", + "INDEX_CHANGE_PROTOCOL", "INDEX_PROTOCOL", "MAP_CONTRACT", "MAP_CONTRACT_FLOOR", diff --git a/python/src/sde/_cutover_project.py b/python/src/sde/_cutover_project.py index a160d8e..26e83fa 100644 --- a/python/src/sde/_cutover_project.py +++ b/python/src/sde/_cutover_project.py @@ -31,6 +31,25 @@ def _abandoned_stages(payload: Any) -> bool: ) +def _index_changes(payload: Any) -> bool: + """Whether an in-place index record - executing or kept - also removes indexes (protocol 2).""" + if not isinstance(payload, dict): + return False + records: list[Any] = [] + history = payload.get("indexes") + if isinstance(history, dict): + records.extend(history.values()) + execution = payload.get("execution") + if isinstance(execution, dict) and execution.get("kind") == "index": + records.append(execution) + return any( + isinstance(record, dict) + and isinstance(record.get("plan"), dict) + and record["plan"].get("protocol") == 2 + for record in records + ) + + class ProjectState: """One POSIX directory and lock, shared by all local executors for this project.""" @@ -55,7 +74,7 @@ def read(self) -> dict[str, Any]: raise ValueError("invalid state envelope") if type(envelope["storage_contract"]) is not int or envelope[ "storage_contract" - ] not in (1, 2, 3, 4): + ] not in (1, 2, 3, 4, 5): raise ValueError("unsupported state storage contract") payload = envelope["payload"] if ( @@ -90,6 +109,10 @@ def read(self) -> dict[str, Any]: # Contract 4 is what makes an older operator refuse an abandoned staging's record # instead of reading a receipt whose tables may name no identity. raise ValueError("an abandoned staging needs state storage contract 4") + if envelope["storage_contract"] < 5 and _index_changes(payload): + # Contract 5 is what makes an older operator refuse a build that also removes + # indexes, instead of resuming it with no idea that removals follow the decision. + raise ValueError("an index change needs state storage contract 5") if set(payload) != fields: raise ValueError("unknown or missing project state fields") return payload @@ -108,7 +131,9 @@ def confirm(self) -> None: def write(self, payload: dict[str, Any]) -> None: body = encode(payload) envelope = { - "storage_contract": 4 + "storage_contract": 5 + if _index_changes(payload) + else 4 if _abandoned_stages(payload) else 3 if "indexes" in payload diff --git a/python/src/sde/engines/_index_build.py b/python/src/sde/engines/_index_build.py index 1ca6c2d..be81e19 100644 --- a/python/src/sde/engines/_index_build.py +++ b/python/src/sde/engines/_index_build.py @@ -78,6 +78,50 @@ def build(self, table: TableIdentity, index: Mapping[str, Any]) -> None: if self.status(table, index) != "ready": raise MigrationRefused("the index build did not leave a ready index") + def declared(self, table: TableIdentity, index: Mapping[str, Any]) -> Status: + """Where an index the map in force declares stands on its table, whatever its name. + + The indexes an index change removes were named by a design or by an earlier build, so the + bound-name rule of :meth:`inspect` does not apply; the table and the shape still do. + """ + if self.native.identity(table.name).physical_key != table.physical_key: + raise MigrationRefused("the table an index change names was replaced") + name = str(index["name"]) + if self.dialect == "postgres": + return self._pg_status(table, index, name)[0] + return self._ch_status(table, index, name)[0] + + def remove(self, table: TableIdentity, index: Mapping[str, Any]) -> None: + """Remove an index the map in force declares; run after the decision, so resumably. + + An index of the declared shape goes, finished or not - a PostgreSQL drop that was stopped + leaves it invalid yet still maintained (measured), and dropping again removes it. Absent, + or another object under the name, means ours is already gone; the other object stays. + """ + status = self.declared(table, index) + if status in ("absent", "foreign"): + return + name = str(index["name"]) + if self.dialect == "postgres": + # IF EXISTS only for an index that goes while this drop waits for its lock: an earlier + # drop whose client the budget closed runs on in the server until the transaction it + # waits for ends. What is left afterwards is read back below, as always. + self.native.command(f"DROP INDEX CONCURRENTLY IF EXISTS {self.quote(name)}") + else: + for mutation_id, is_done, _failure in self._ch_mutations(table, name): + if not is_done: + self.native.command( + "KILL MUTATION WHERE database = currentDatabase() " + f"AND table = '{self._literal(table.name)}' " + f"AND mutation_id = '{self._literal(mutation_id)}'" + ) + self.native.command( + f"ALTER TABLE {self.quote(table.name)} DROP INDEX {self.quote(name)} " + "SETTINGS alter_sync = 0" + ) + if self.declared(table, index) in ("unfinished", "ready"): + raise MigrationRefused("a removed index is still in the catalogue") + def drop(self, table: TableIdentity, index: Mapping[str, Any]) -> None: """Remove this build's own index, finished or not, and anything still materializing it. diff --git a/python/src/sde/index_build.py b/python/src/sde/index_build.py index 119f71e..4cad8eb 100644 --- a/python/src/sde/index_build.py +++ b/python/src/sde/index_build.py @@ -11,6 +11,12 @@ only by indexes added to one group's source. Nothing else may change, because nothing else can change without a copy - and the tables, the write generation and every running process stay as they are, which is what lets the build run without a barrier. + +Protocol 2 also removes indexes the map in force declares, for a design that drops an index nobody +reads or replaces one with another. The removed ones leave the next map, the others keep their +order, and the new ones follow. The operator removes them only after its decision, one resumable +step each: ``DROP INDEX CONCURRENTLY`` pauses no write (measured, like the build), and a process on +the map in force that still declares a removed index reports it missing and keeps serving rows. """ from __future__ import annotations @@ -32,6 +38,11 @@ INDEX_PROTOCOL = 1 """Indexes added to one group's source, built on the tables in force; no copy, no cutover.""" +INDEX_CHANGE_PROTOCOL = 2 +"""Indexes added to and removed from one group's source, in place; at least one removed.""" + +_PROTOCOLS = (INDEX_PROTOCOL, INDEX_CHANGE_PROTOCOL) + MAX_BUILD_BUDGET_MS = 86_400_000 """A day. The budget bounds a build on a server that stopped answering; nothing is paused while it runs, so it is not a pause budget and it is deliberately far longer than a cutover's.""" @@ -71,6 +82,8 @@ class IndexPlan: added: tuple[Mapping[str, Any], ...] """The new index definitions, in position order, exactly as the prepared map carries them.""" verified_with: str | None + removed: tuple[Mapping[str, Any], ...] = () + """Protocol 2: the definitions in force the next map drops, in their order in force.""" fingerprint: str | None = field(default=None, init=False) _document: bytes = field(default=b"", init=False, repr=False) @@ -83,6 +96,10 @@ def as_record(self) -> dict[str, Any]: value: dict[str, Any] = json.loads(self._document) return value + @property + def protocol(self) -> int: + return int(self.as_record()["protocol"]) + def prepared_payload(self) -> bytes: return canonical_bytes(self.as_record()["prepared"]) @@ -133,10 +150,11 @@ def _load( raise MigrationRefused("index build authorization has missing or unknown fields") if ( type(body["protocol"]) is not int - or body["protocol"] != INDEX_PROTOCOL + or body["protocol"] not in _PROTOCOLS or body["kind"] != "sde-index" ): raise MigrationRefused("unsupported index build authorization kind or protocol") + protocol = int(body["protocol"]) identity = _hex(body["index_id"], 32, "index_id", _SUBJECT) local = _hex(body["project_id"], 32, "project_id", _SUBJECT) if local != project_id: @@ -171,7 +189,7 @@ def _load( parsed = load_map(document, model=model, public_key=public_key, require_signature=True) if parsed.contract < GENERATIONS_SINCE: raise MigrationRefused( - f"index build protocol {INDEX_PROTOCOL} requires map contract " + f"index build protocol {protocol} requires map contract " f"{GENERATIONS_SINCE} or later" ) check_map_project(parsed, project_id) @@ -212,12 +230,25 @@ def without_indexes(material: Mapping[str, Any]) -> dict[str, Any]: "an index build changes nothing about the source but its indexes; a new key order, " "partition, table or engine is a relayout or a move" ) - kept = list(old_group["source"]["layout"].get("indexes", []) or []) + in_force = list(old_group["source"]["layout"].get("indexes", []) or []) after = list(new_group["source"]["layout"].get("indexes", []) or []) - if len(after) <= len(kept) or canonical_bytes(after[: len(kept)]) != canonical_bytes(kept): - raise MigrationRefused( - "an index build keeps every index in force, in order, and adds at least one after them" - ) + if protocol == INDEX_PROTOCOL: + kept, removed = in_force, [] + if len(after) <= len(kept) or canonical_bytes(after[: len(kept)]) != canonical_bytes(kept): + raise MigrationRefused( + "an index build keeps every index in force, in order, and adds at least one after " + "them" + ) + else: + remaining = {str(index.get("name")) for index in after} + kept = [index for index in in_force if str(index.get("name")) in remaining] + removed = [index for index in in_force if str(index.get("name")) not in remaining] + if not removed: + raise MigrationRefused("index build protocol 2 removes at least one index in force") + if canonical_bytes(after[: len(kept)]) != canonical_bytes(kept): + raise MigrationRefused( + "an index change keeps the other indexes in force, in order, before the new ones" + ) added = after[len(kept) :] for position, index in enumerate(added, start=1): if index.get("name") != index_build_name(identity, position): @@ -245,6 +276,7 @@ def without_indexes(material: Mapping[str, Any]) -> dict[str, Any]: prepared_raw["groups"][other] ): raise MigrationRefused("an index build cannot change an unaffected group") + gone = {str(index["name"]) for index in removed} plan = IndexPlan( identity, local, @@ -252,9 +284,10 @@ def without_indexes(material: Mapping[str, Any]) -> dict[str, Any]: current, prepared, budget, - # From the loaded map, which freezes nested structures, not from the caller's dictionaries. + # From the loaded maps, which freeze nested structures, not from the caller's dictionaries. tuple(new.source.layout.indexes[len(kept) :]), verified, + tuple(index for index in old.source.layout.indexes if str(index["name"]) in gone), ) object.__setattr__(plan, "_document", canonical_bytes(body)) object.__setattr__( diff --git a/python/src/sde/index_operator.py b/python/src/sde/index_operator.py index a5ac5a4..18c1060 100644 --- a/python/src/sde/index_operator.py +++ b/python/src/sde/index_operator.py @@ -62,6 +62,17 @@ def _snapshot(operator: LocalCutover, plan: IndexPlan, state: dict[str, Any]) -> status, reason = builder.inspect(identity, index) if status == "foreign": raise MigrationRefused(reason) + for index in plan.removed: + # Before any DDL, as for a build beside a table that differs from its map: removing + # an index the table does not hold as declared would publish a map about another table. + if index["entity"] == entity and builder.declared(identity, index) not in ( + "ready", + "unfinished", + ): + raise MigrationRefused( + "an index the map in force declares is not on its table as declared; " + "inspect the table before changing its indexes" + ) allowed: dict[str, set[str]] = {name: {WATERMARK_TABLE} for name in operator.engines} for placed in plan.current.groups.values(): for material in placed.all(): @@ -94,6 +105,10 @@ def _snapshot(operator: LocalCutover, plan: IndexPlan, state: dict[str, Any]) -> {"entity": str(index["entity"]), "name": str(index["name"]), "index": dict(index)} for index in plan.added ], + "removed": [ + {"entity": str(index["entity"]), "name": str(index["name"]), "index": dict(index)} + for index in plan.removed + ], } @@ -187,6 +202,16 @@ def publish() -> None: operator.store.publish(plan.prepared_payload()) operator._step(state, "index_publish", publish) + # Removals follow the decision and the publication: each is a step of its own, resumed + # after a crash, and a process still on the map in force only reports a removed index + # missing. Before this point nothing of the map in force was touched. + for row in execution.get("removed", ()): + table = TableIdentity(**execution["tables"][row["entity"]]) + operator._step( + state, + "index_remove_" + str(row["name"]), + partial(builder.remove, table, row["index"]), + ) outcome, active = "built", plan.prepared else: for table, index in rows: @@ -196,8 +221,8 @@ def publish() -> None: if operator.active_map().fingerprint != plan.current.fingerprint: raise MigrationRefused("an abandoned index build found another active map") outcome, active = "abandoned", plan.current - receipt = { - "protocol": 1, + receipt: dict[str, Any] = { + "protocol": plan.protocol, "index_id": plan.index_id, "index_fingerprint": plan.fingerprint, "project_id": operator.project_id, @@ -217,6 +242,18 @@ def publish() -> None: "elapsed_ms": operator._elapsed(), "recovered": recovered, } + if plan.protocol == 2: + # Abandoned before its decision, a change removed nothing: the rows say what was removed. + receipt["removed"] = [ + { + "engine": execution["engine"], + "entity": row["entity"], + "name": row["name"], + "table": execution["tables"][row["entity"]], + } + for row in execution.get("removed", ()) + if outcome == "built" + ] state["indexes"][plan.index_id] = { "plan_fingerprint": plan.fingerprint, "plan": plan.as_record(), diff --git a/python/tests/test_conformance.py b/python/tests/test_conformance.py index 60e754e..d3b7af3 100644 --- a/python/tests/test_conformance.py +++ b/python/tests/test_conformance.py @@ -1049,6 +1049,9 @@ def _drive_index_vector(case: Path, model: sde.LogicalModel) -> None: sde.index_build_name(plan.index_id, position) for position in range(1, len(wanted["added"]) + 1) ] == wanted["added"] + # Protocol 2 removes indexes in force; a vector that names none removes none. + assert plan.protocol == raw["protocol"] + assert [index["name"] for index in plan.removed] == wanted.get("removed", []) decoded = sde.load_map( json.loads(plan.prepared_payload()), model=model, public_key=keys, require_signature=True ) diff --git a/python/tests/test_cutover_project.py b/python/tests/test_cutover_project.py index 2622ec7..d92c1e3 100644 --- a/python/tests/test_cutover_project.py +++ b/python/tests/test_cutover_project.py @@ -192,3 +192,43 @@ def test_contract_four_still_carries_the_index_history(tmp_path: Path) -> None: _rewrite(store, 4, payload) # no "indexes" field with pytest.raises(MigrationRefused, match="corrupt"): store.read() + + +def _index_record(protocol: int) -> dict[str, Any]: + return {"plan_fingerprint": "f" * 64, "plan": {"protocol": protocol}, "receipt": {}} + + +def test_an_index_change_raises_the_state_contract_to_five(tmp_path: Path) -> None: + store = ProjectState(tmp_path, PROJECT, MODEL) + store.enroll({"map_version": 1}, b"map") + state = store.read() + state["stages"], state["indexes"] = {}, {"a" * 32: _index_record(1)} + store.write(state) + assert json.loads(store.path.read_bytes())["storage_contract"] == 3 + state["indexes"]["b" * 32] = _index_record(2) + store.write(state) + assert json.loads(store.path.read_bytes())["storage_contract"] == 5 + assert store.read()["indexes"]["b" * 32]["plan"]["protocol"] == 2 + # An executing change is one too, before it has any history. + state["indexes"], state["execution"] = {}, {"kind": "index", "plan": {"protocol": 2}} + store.write(state) + assert json.loads(store.path.read_bytes())["storage_contract"] == 5 + + +@pytest.mark.parametrize("where", ["history", "execution"]) +@pytest.mark.parametrize("contract", [3, 4]) +def test_an_index_change_needs_contract_five(tmp_path: Path, contract: int, where: str) -> None: + """What makes an operator that knows contracts 1 to 4 refuse, not resume, an index change.""" + store = ProjectState(tmp_path, PROJECT, MODEL) + store.enroll({"map_version": 1}, b"map") + payload = store.read() + payload["stages"], payload["indexes"] = {}, {} + if where == "history": + payload["indexes"]["b" * 32] = _index_record(2) + else: + payload["execution"] = {"kind": "index", "plan": {"protocol": 2}} + _rewrite(store, contract, payload) + with pytest.raises(MigrationRefused, match="corrupt"): + store.read() + _rewrite(store, 5, payload) + assert store.read()["stages"] == {} diff --git a/python/tests/test_index_change_live.py b/python/tests/test_index_change_live.py new file mode 100644 index 0000000..b5bd312 --- /dev/null +++ b/python/tests/test_index_change_live.py @@ -0,0 +1,401 @@ +"""An index change removes indexes in force - alone or beside new ones - on the live tables. + +Protocol 2 of the ``sde-index`` authorization. The operator builds and qualifies what it adds as +protocol 1 does, records its decision, publishes the next map, and only then removes each index the +next map no longer declares, one resumable step each. Measured before the design (SDK +``docs/qualification/in-place-index-drop/``): ``DROP INDEX CONCURRENTLY`` pauses no write; an open +transaction that has read the table holds it, while an older snapshot alone does not; and a drop +that is stopped leaves its index invalid yet still maintained - which a second drop removes. +ClickHouse drops a data-skipping index at once with ``alter_sync = 0``. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest +from test_index_operator_live import ( + CHECKPOINTS, + Build, + Crash, + admin, + assert_built, + ch_indexes, + ch_materializations, + crash_at, + expect_line, + initial, + kill, + pg_indexes, + quiet, + spawn, + wait_for, +) + +import sde +from sde.local_cutover import CutoverRecoveryRequired + +REMOVED = "sde_i_kept_000001" +AFTER_THE_DECISION = [ + *CHECKPOINTS[5:], + "index_remove:intent", + "index_remove:done", +] + + +def present(build: Build) -> set[str]: + """The table's own indexes, by name, as the engine's catalogue lists them.""" + if build.engine == "postgres": + return set(pg_indexes(build.role)) + return set(ch_indexes(build.role)) + + +def assert_changed(build: Build, receipt: dict[str, Any]) -> None: + assert receipt["protocol"] == 2 + assert receipt["outcome"] == "built" + assert [row["name"] for row in receipt["removed"]] == [REMOVED] + assert receipt["removed"][0]["table"]["name"] == "initial_events" + assert REMOVED not in present(build) + if build.plan.added: + assert_built(build) + source = build.plan.prepared.groups["Event"].source + assert build.role.operator.validate_schema(source.layout, keys={"Event": ["id"]}) == () + assert build.operator.active_map().fingerprint == build.plan.prepared.fingerprint + + +@contextmanager +def open_reader(build: Build) -> Iterator[None]: + """A transaction that has read the table and stays open: what a concurrent drop waits for. + + It holds a lock on the table until it ends. An older snapshot alone would not hold the drop + (measured), unlike a concurrent build. + """ + import psycopg + from psycopg.conninfo import make_conninfo + + holder = psycopg.connect( + make_conninfo(build.role.operator._dsn, options="-csearch_path=" + build.role.namespace) + ) + holder.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ + holder.execute("SELECT count(*) FROM initial_events") + try: + yield + finally: + holder.rollback() + holder.close() + + +@pytest.mark.parametrize("engine", ["postgres", "clickhouse"]) +def test_an_index_in_force_is_removed_while_the_application_writes( + engine: str, tmp_path: Path +) -> None: + with initial(engine, tmp_path, kept=True, added=0, remove=True) as build: + assert REMOVED in present(build) + assert [index["name"] for index in build.plan.removed] == [REMOVED] + assert build.plan.added == () + build.old.save_many("Event", [{"id": n, "value": n % 7} for n in range(2, 202)]) + receipt = build.operator.index(build.plan).as_record() + assert_changed(build, receipt) + assert receipt["indexes"] == [] + # A process still on the map in force keeps writing and reading, and a process on the + # next map finds its table exactly as that map says. + build.old.save("Event", {"id": 5000, "value": 5}) + assert build.old.get("Event", {"id": 5000}) == {"id": 5000, "value": 5} + fresh = build.session() + assert fresh.get("Event", {"id": 200}) == {"id": 200, "value": 200 % 7} + if engine == "postgres": + assert fresh.physical == () + # The state that holds a change is contract 5, which an older operator refuses. + envelope = json.loads((tmp_path / "project.json").read_bytes()) + assert envelope["storage_contract"] == 5 + # Retrying the completed change returns the same receipt. + assert build.operator.index(build.plan).as_record() == receipt + + +@pytest.mark.parametrize("engine", ["postgres", "clickhouse"]) +def test_an_index_is_replaced_in_place(engine: str, tmp_path: Path) -> None: + with initial(engine, tmp_path, kept=True, added=1, remove=True) as build: + receipt = build.operator.index(build.plan).as_record() + assert_changed(build, receipt) + assert [row["name"] for row in receipt["indexes"]] == build.names + assert set(build.names) <= present(build) + + +@pytest.mark.parametrize("checkpoint", AFTER_THE_DECISION) +@pytest.mark.parametrize("engine", ["postgres", "clickhouse"]) +def test_a_change_resumes_after_every_checkpoint_after_its_decision( + engine: str, checkpoint: str, tmp_path: Path +) -> None: + with initial(engine, tmp_path, kept=True, added=1, remove=True) as build: + build.operator._after_step = crash_at(checkpoint) + with pytest.raises(Crash): + build.operator.index(build.plan) + build.operator._after_step = quiet + if checkpoint != "index_remove:done": + # Nothing of the map in force is touched before the next map is published. + assert REMOVED in present(build) + build.old.save("Event", {"id": 2, "value": 22}) # nothing is paused between attempts + with pytest.raises(sde.MigrationRefused, match="cannot be abandoned"): + build.operator.abandon() + receipt = build.operator.resume().as_record() + assert receipt["recovered"] is True + assert_changed(build, receipt) + assert build.operator.index(build.plan).as_record() == receipt + + +@pytest.mark.parametrize("engine", ["postgres", "clickhouse"]) +def test_abandoning_a_change_before_its_decision_removes_nothing_in_force( + engine: str, tmp_path: Path +) -> None: + with initial(engine, tmp_path, kept=True, added=1, remove=True) as build: + build.operator._after_step = crash_at("index_build:done") + with pytest.raises(Crash): + build.operator.index(build.plan) + build.operator._after_step = quiet + receipt = build.operator.abandon().as_record() + assert (receipt["protocol"], receipt["outcome"]) == (2, "abandoned") + assert receipt["removed"] == [] + assert REMOVED in present(build) # the map in force is untouched + assert not set(build.names) & present(build) # and our own addition is gone + assert build.operator.active_map().fingerprint == build.plan.current.fingerprint + current = build.plan.current.groups["Event"].source + assert build.role.operator.validate_schema(current.layout, keys={"Event": ["id"]}) == () + + +@pytest.mark.parametrize("change", ["absent", "another_shape"]) +@pytest.mark.parametrize("engine", ["postgres", "clickhouse"]) +def test_an_index_in_force_that_is_not_as_declared_refuses_before_any_ddl( + engine: str, change: str, tmp_path: Path +) -> None: + """The design in force is read back before any DDL, and an index a change removes is in it.""" + with ( + initial(engine, tmp_path, kept=True, added=1, remove=True) as build, + admin(build.role) as run, + ): + if engine == "postgres": + run(f'DROP INDEX "{REMOVED}"') + if change == "another_shape": + run(f'CREATE INDEX "{REMOVED}" ON initial_events (id)') + else: + run(f"ALTER TABLE initial_events DROP INDEX `{REMOVED}` SETTINGS alter_sync = 0") + if change == "another_shape": + run( + f"ALTER TABLE initial_events ADD INDEX `{REMOVED}` value TYPE minmax " + "GRANULARITY 1" + ) + before = present(build) + with pytest.raises(sde.MigrationRefused, match="differ from the physical design"): + build.operator.index(build.plan) + assert present(build) == before # nothing was built and nothing removed + assert build.operator.store.read()["execution"] is None + + +def test_an_index_in_force_with_another_sort_order_refuses_before_any_ddl( + tmp_path: Path, +) -> None: + """The one shape the physical read-back does not compare: the removal's own check sees it.""" + with ( + initial("postgres", tmp_path, kept=True, added=1, remove=True) as build, + admin(build.role) as run, + ): + run(f'DROP INDEX "{REMOVED}"') + run(f'CREATE INDEX "{REMOVED}" ON initial_events (value DESC, id)') + before = pg_indexes(build.role)[REMOVED] + with pytest.raises(sde.MigrationRefused, match="not on its table as declared"): + build.operator.index(build.plan) + assert pg_indexes(build.role)[REMOVED] == before + assert not set(build.names) & present(build) + assert build.operator.store.read()["execution"] is None + + +def test_a_removal_held_by_an_open_reader_is_bounded_and_resumed(tmp_path: Path) -> None: + with initial("postgres", tmp_path, kept=True, added=0, remove=True, budget_ms=6000) as build: + with open_reader(build): + started = time.monotonic() + with pytest.raises(CutoverRecoveryRequired, match="build budget"): + build.operator.index(build.plan) + assert time.monotonic() - started < 20 + operator = build.reconnect() + # Decided and published: the next map is in force, and the index stays - invalid, so + # no query uses it, yet maintained - while the drop waits for the reader. + assert operator.active_map().fingerprint == build.plan.prepared.fingerprint + assert pg_indexes(build.role)[REMOVED][0] is False + build.old.save("Event", {"id": 7000, "value": 7}) # writes go on meanwhile + with pytest.raises(sde.MigrationRefused, match="cannot be abandoned"): + operator.abandon() + receipt = operator.resume().as_record() + assert receipt["recovered"] is True + assert_changed(build, receipt) + assert build.session().get("Event", {"id": 7000}) == {"id": 7000, "value": 7} + + +def test_a_killed_operator_mid_removal_resumes_it(tmp_path: Path) -> None: + with ( + initial("postgres", tmp_path, kept=True, added=0, remove=True) as build, + admin(build.role) as run, + ): + with open_reader(build): + worker = spawn(build, "native") + try: + expect_line(worker, "STARTED") + rows = wait_for( + lambda: run( + "SELECT pid FROM pg_stat_activity " + "WHERE query LIKE 'DROP INDEX CONCURRENTLY%' AND wait_event_type = 'Lock'" + ), + "the removal to wait for the open reader", + ) + kill(worker) + pid = int(rows[0][0]) + # The server notices a vanished client only when it next talks to it; stand in + # for that moment, which is when the drop stops server-side. + run("SELECT pg_terminate_backend(%s)", [pid]) + wait_for( + lambda: not run("SELECT 1 FROM pg_stat_activity WHERE pid = %s", [pid]), + "the killed removal's backend to exit", + ) + finally: + kill(worker) + assert pg_indexes(build.role)[REMOVED][0] is False # left invalid, still maintained + receipt = build.reconnect().resume().as_record() + assert receipt["recovered"] is True + assert_changed(build, receipt) + + +@pytest.mark.parametrize("engine", ["postgres", "clickhouse"]) +def test_a_foreign_object_under_a_removed_name_after_the_decision_is_left_alone( + engine: str, tmp_path: Path +) -> None: + with ( + initial(engine, tmp_path, kept=True, added=0, remove=True) as build, + admin(build.role) as run, + ): + build.operator._after_step = crash_at("index_publish:done") + with pytest.raises(Crash): + build.operator.index(build.plan) + build.operator._after_step = quiet + # Between the publication and the removal, somebody else's index takes the name. + if engine == "postgres": + run(f'DROP INDEX "{REMOVED}"') + run(f'CREATE INDEX "{REMOVED}" ON initial_events (id)') + foreign: Any = pg_indexes(build.role)[REMOVED] + else: + run(f"ALTER TABLE initial_events DROP INDEX `{REMOVED}` SETTINGS alter_sync = 0") + run(f"ALTER TABLE initial_events ADD INDEX `{REMOVED}` value TYPE minmax GRANULARITY 1") + foreign = ch_indexes(build.role)[REMOVED] + receipt = build.operator.resume().as_record() + assert receipt["outcome"] == "built" + assert [row["name"] for row in receipt["removed"]] == [REMOVED] # ours is gone + found = pg_indexes(build.role) if engine == "postgres" else ch_indexes(build.role) + assert found[REMOVED] == foreign # theirs stays as it was + + +def test_a_removal_that_leaves_its_index_is_not_done( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The catalogue is read back after a drop: a drop that left the index is not a removal.""" + with initial("postgres", tmp_path, kept=True, added=0, remove=True) as build: + native = build.operator.native["postgres"] + command = native.command + + def no_drop(sql: str) -> None: + if not sql.startswith("DROP INDEX"): + command(sql) + + monkeypatch.setattr(native, "command", no_drop) + with pytest.raises(CutoverRecoveryRequired): + build.operator.index(build.plan) + assert REMOVED in present(build) + monkeypatch.setattr(native, "command", command) + receipt = build.operator.resume().as_record() + assert receipt["recovered"] is True + assert_changed(build, receipt) + + +def test_a_pending_materialization_of_a_removed_index_is_killed(tmp_path: Path) -> None: + """Left behind, the mutation would fail on every retry once its index is gone.""" + with ( + initial("clickhouse", tmp_path, kept=True, added=0, remove=True) as build, + admin(build.role) as run, + ): + where = f"`{build.role.namespace}`.`initial_events`" + run(f"SYSTEM STOP MERGES {where}") + try: + run(f"ALTER TABLE initial_events MATERIALIZE INDEX `{REMOVED}`") + wait_for( + lambda: ch_materializations(build.role, REMOVED) == [False], + "the materialization to be pending", + ) + receipt = build.operator.index(build.plan).as_record() + assert_changed(build, receipt) + assert ch_materializations(build.role, REMOVED) == [] + finally: + run(f"SYSTEM START MERGES {where}") + + +def test_a_resumed_removal_waits_for_the_stopped_drop_and_finishes(tmp_path: Path) -> None: + """The budget closed the operator's connection, not its drop: that runs on in the server. + + Resumed while the stopped drop still waits for the reader, the new drop waits behind it for + the table's lock; when the reader ends, the stopped drop removes the index first. The resumed + step must then finish, not fail on an index that is already gone. + """ + import threading + + import psycopg + from psycopg.conninfo import make_conninfo + + with ( + initial("postgres", tmp_path, kept=True, added=0, remove=True, budget_ms=6000) as build, + admin(build.role) as run, + ): + holder = psycopg.connect( + make_conninfo(build.role.operator._dsn, options="-csearch_path=" + build.role.namespace) + ) + holder.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ + holder.execute("SELECT count(*) FROM initial_events") + seen: list[Any] = [] + + def release_once_both_drops_wait() -> None: + try: + seen.append( + wait_for( + lambda: ( + len( + run( + "SELECT pid FROM pg_stat_activity WHERE query LIKE " + "'DROP INDEX CONCURRENTLY%' AND wait_event_type = 'Lock'" + ) + ) + == 2 + ), + "the resumed drop to wait behind the stopped one", + ) + ) + except AssertionError as exc: + seen.append(exc) + finally: + holder.rollback() + holder.close() + + try: + with pytest.raises(CutoverRecoveryRequired, match="build budget"): + build.operator.index(build.plan) + operator = build.reconnect() + except BaseException: + holder.close() + raise + helper = threading.Thread(target=release_once_both_drops_wait) + helper.start() + try: + receipt = operator.resume().as_record() + finally: + helper.join(60) + assert seen == [True] # the race this is about did happen + assert receipt["recovered"] is True + assert_changed(build, receipt) diff --git a/python/tests/test_index_operator_live.py b/python/tests/test_index_operator_live.py index 2c4f664..00f4659 100644 --- a/python/tests/test_index_operator_live.py +++ b/python/tests/test_index_operator_live.py @@ -101,14 +101,22 @@ def reconnect(self) -> LocalCutover: @contextmanager def initial( - engine: str, root: Path, *, kept: bool = False, added: int = 1, budget_ms: int = 60000 + engine: str, + root: Path, + *, + kept: bool = False, + added: int = 1, + budget_ms: int = 60000, + remove: bool = False, ) -> Iterator[Build]: """A source-only map in force, a process writing on it, an operator, and a build authorization. ``kept`` gives the map in force a physical design of its own (contract 5), which the next map must carry unchanged; without it the next map raises the contract from 4 to 5, because the - first index is where a physical design first appears. + first index is where a physical design first appears. ``remove`` makes it an index change + (protocol 2): the next map drops that index in force and adds ``added`` new ones. """ + assert kept or not remove, "only an index in force can be removed" from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey with runtime_roles("postgres") as pg, runtime_roles("clickhouse") as ch: @@ -181,17 +189,19 @@ def signed(raw: dict[str, Any]) -> dict[str, Any]: prepared["contract"], prepared["map_version"] = 5, 2 design = prepared["groups"]["Event"]["source"]["layout"] design["indexes"] = [ - *design.get("indexes", []), + *([] if remove else design.get("indexes", [])), *( {"entity": "Event", "name": sde.index_build_name(identity, position), **shape} for position, shape in enumerate(ADDED[engine][:added], start=1) ), ] + if not design["indexes"]: + del design["indexes"] # absent, never empty, as a controller writes it plan = sde.load_index_plan( signed( { "kind": "sde-index", - "protocol": 1, + "protocol": 2 if remove else 1, "index_id": identity, "project_id": PROJECT, "group": "Event", @@ -416,7 +426,7 @@ class Crash(BaseException): def matches(step: str, checkpoint: str) -> bool: """``index_build:intent`` names the intent of every index's build step, whatever its name.""" phase, _, suffix = checkpoint.partition(":") - if phase in ("index_build", "index_drop"): + if phase in ("index_build", "index_drop", "index_remove"): return step.startswith(phase + "_") and step.endswith(":" + suffix) return step == checkpoint diff --git a/typescript/src/in-place-index.ts b/typescript/src/in-place-index.ts index 8e41ac4..ffaa7c4 100644 --- a/typescript/src/in-place-index.ts +++ b/typescript/src/in-place-index.ts @@ -7,6 +7,9 @@ * pauses no write. This authorization binds that build: the exact map in force and the next map, * which differs from it only by indexes added to one group's source. The Python library executes * it; this loader holds both libraries to one reading of the packet. + * + * Protocol 2 also removes indexes the map in force declares: the removed ones leave the next map, + * the others keep their order, and the new ones follow. The operator removes them after its decision. */ import { createHash } from 'node:crypto' import { CanonicalError, canonicalBytes, compareCodePoints } from './canonical.js' @@ -17,6 +20,8 @@ import { fingerprintOf, loadMap, verifyMapSignature, type LoadOptions, type Plac /** Indexes added to one group's source, built on the tables in force; no copy, no cutover. */ export const INDEX_PROTOCOL = 1 +/** Indexes added to and removed from one group's source, in place; at least one removed. */ +export const INDEX_CHANGE_PROTOCOL = 2 /** A day. The budget bounds a build on a server that stopped answering; nothing is paused while * it runs, so it is not a pause budget and it is deliberately far longer than a cutover's. */ export const MAX_BUILD_BUDGET_MS = 86_400_000 @@ -96,7 +101,9 @@ export class IndexPlan { constructor(readonly indexId: string, readonly projectId: string, readonly group: string, readonly current: PlacementMap, readonly prepared: PlacementMap, readonly buildBudgetMs: number, /** The new index definitions, in position order, exactly as the prepared map carries them. */ - readonly added: readonly Index[], readonly verifiedWith: string | null) { + readonly added: readonly Index[], readonly verifiedWith: string | null, + /** Protocol 2: the definitions in force the next map drops, in their order in force. */ + readonly removed: readonly Index[] = []) { Object.freeze(this) } private loaded() { @@ -105,6 +112,7 @@ export class IndexPlan { return saved } get fingerprint(): string | undefined { return provenance.get(this)?.fingerprint } + get protocol(): number { return this.asRecord()['protocol'] as number } asRecord(): Record { return JSON.parse(this.loaded().document) as Record } preparedPayload(): Uint8Array { return canonicalBytes(this.asRecord()['prepared']) } checkCurrent(current: PlacementMap): void { @@ -119,9 +127,11 @@ export class IndexPlan { function load(raw: unknown, model: LogicalModel, projectId: string, publicKey: PublicKeys): IndexPlan { const body = record(structuredClone(raw), 'authorization') if (!equal(sortedKeys(body), FIELDS)) throw new MigrationRefused('index build authorization has missing or unknown fields') - if (typeof body['protocol'] !== 'number' || body['protocol'] !== INDEX_PROTOCOL || body['kind'] !== 'sde-index') { + if (typeof body['protocol'] !== 'number' || ![INDEX_PROTOCOL, INDEX_CHANGE_PROTOCOL].includes(body['protocol']) || + body['kind'] !== 'sde-index') { throw new MigrationRefused('unsupported index build authorization kind or protocol') } + const protocol = body['protocol'] const identity = hex(body['index_id'], 32, 'index_id'), local = hex(body['project_id'], 32, 'project_id') if (local !== projectId) throw new MigrationRefused('index build authorization belongs to another local project') const group = body['group'] @@ -146,7 +156,7 @@ function load(raw: unknown, model: LogicalModel, projectId: string, publicKey: P } const parsed = loadMap(document, { model, publicKey, requireSignature: true }) if (parsed.contract < GENERATIONS_SINCE) { - throw new MigrationRefused(`index build protocol ${INDEX_PROTOCOL} requires map contract ${GENERATIONS_SINCE} or later`) + throw new MigrationRefused(`index build protocol ${protocol} requires map contract ${GENERATIONS_SINCE} or later`) } checkMapProject(parsed, projectId) for (const placed of Object.values(parsed.groups)) { @@ -184,9 +194,21 @@ function load(raw: unknown, model: LogicalModel, projectId: string, publicKey: P throw new MigrationRefused('an index build changes nothing about the source but its indexes; a new key order, ' + 'partition, table or engine is a relayout or a move') } - const kept = indexesOf(oldSource), after = indexesOf(newSource) - if (after.length <= kept.length || !equal(after.slice(0, kept.length), kept)) { - throw new MigrationRefused('an index build keeps every index in force, in order, and adds at least one after them') + const inForce = indexesOf(oldSource), after = indexesOf(newSource) + const nameOf = (index: unknown) => String(record(index, 'index')['name']) + let kept = inForce, removed: unknown[] = [] + if (protocol === INDEX_PROTOCOL) { + if (after.length <= kept.length || !equal(after.slice(0, kept.length), kept)) { + throw new MigrationRefused('an index build keeps every index in force, in order, and adds at least one after them') + } + } else { + const remaining = new Set(after.map(nameOf)) + kept = inForce.filter(index => remaining.has(nameOf(index))) + removed = inForce.filter(index => !remaining.has(nameOf(index))) + if (removed.length === 0) throw new MigrationRefused('index build protocol 2 removes at least one index in force') + if (!equal(after.slice(0, kept.length), kept)) { + throw new MigrationRefused('an index change keeps the other indexes in force, in order, before the new ones') + } } const added = after.slice(kept.length) added.forEach((index, offset) => { @@ -206,7 +228,9 @@ function load(raw: unknown, model: LogicalModel, projectId: string, publicKey: P if (other !== group && !equal(oldGroups[other], newGroups[other])) throw new MigrationRefused('an index build cannot change an unaffected group') } // From the loaded map, which freezes nested structures, not from the caller's objects. - const plan = new IndexPlan(identity, local, group, current, prepared, budget, next.source.layout.indexes.slice(kept.length), verified) + const gone = new Set(removed.map(nameOf)) + const plan = new IndexPlan(identity, local, group, current, prepared, budget, next.source.layout.indexes.slice(kept.length), + verified, old.source.layout.indexes.filter(index => gone.has(String(index['name'])))) provenance.set(plan, { document: canonicalBytes(body).toString('utf8'), fingerprint: createHash('sha256').update(canonicalBytes(except(body, ['signature']))).digest('hex') }) return plan diff --git a/typescript/src/index.ts b/typescript/src/index.ts index 1e34a92..ff6c89d 100644 --- a/typescript/src/index.ts +++ b/typescript/src/index.ts @@ -190,4 +190,4 @@ export { verifyFrozen, frozenVerifyRecord, type FrozenVerifyReport, type FrozenT export { CUTOVER_PROTOCOL, CUTOVER_RELAYOUT_PROTOCOL, CutoverPlan, loadCutoverPlan } from './cutover.js' export { STAGING_PROTOCOL, STAGING_RELAYOUT_PROTOCOL, StagingPlan, loadStagingPlan, stagingTableName } from './staging.js' -export { INDEX_PROTOCOL, IndexPlan, indexBuildName, loadIndexPlan } from './in-place-index.js' +export { INDEX_CHANGE_PROTOCOL, INDEX_PROTOCOL, IndexPlan, indexBuildName, loadIndexPlan } from './in-place-index.js' diff --git a/typescript/tests/conformance.test.ts b/typescript/tests/conformance.test.ts index ff1aa72..0538cb6 100644 --- a/typescript/tests/conformance.test.ts +++ b/typescript/tests/conformance.test.ts @@ -1277,7 +1277,7 @@ function driveIndexVector(dir: string, model: LogicalModel): void { const raw = readJson>(join(dir, 'plan.json')) const wanted = readJson<{ project_id: string; error?: string; match?: string; index_fingerprint: string; verified_with: string; map_fingerprints: Record<'current' | 'prepared', string>; added: string[]; - build_budget_ms: number }>(join(dir, 'index.json')) + removed?: string[]; build_budget_ms: number }>(join(dir, 'index.json')) const keys = readJson>(join(dir, 'keys.json')) const publicKey = Object.fromEntries(Object.entries(keys).map(([name, value]) => [name, Buffer.from(value, 'base64')])) const options = { model, projectId: wanted.project_id, publicKey } @@ -1297,6 +1297,9 @@ function driveIndexVector(dir: string, model: LogicalModel): void { for (const name of ['current', 'prepared'] as const) expect(plan[name].fingerprint).toBe(wanted.map_fingerprints[name]) expect(plan.added.map(index => index['name'])).toEqual(wanted.added) expect(wanted.added.map((_, offset) => indexBuildName(plan.indexId, offset + 1))).toEqual(wanted.added) + // Protocol 2 removes indexes in force; a vector that names none removes none. + expect(plan.protocol).toBe(raw['protocol']) + expect(plan.removed.map(index => index['name'])).toEqual(wanted.removed ?? []) const decoded = loadMap(JSON.parse(Buffer.from(plan.preparedPayload()).toString('utf8')), { model, publicKey, requireSignature: true }) expect(decoded.fingerprint).toBe(wanted.map_fingerprints.prepared) }