diff --git a/docs/self-host/docker.md b/docs/self-host/docker.md index edeef7af9..892be6121 100644 --- a/docs/self-host/docker.md +++ b/docs/self-host/docker.md @@ -271,7 +271,7 @@ The [configuration reference](configuration.md) describes the file and every set ## Upgrade {#upgrade} -The ledger stays on its volume or in its database, and the new version migrates it when it starts. +The ledger stays on its volume or in its database. A release that keeps the ledger's layout opens it as it is; before launch, one that changes the layout refuses the older ledger at start and says which files to delete from the volume, or to point `DATABASE_URL` at an empty database. 1. Read the [release notes](https://github.com/BeOnAuto/runtime/releases) of the new version. 2. Change the version on the `IMAGE` line of `.env`, such as `IMAGE=docker.on.auto/onauto/runtime:0.9.2`. diff --git a/docs/self-host/postgresql.md b/docs/self-host/postgresql.md index 1671ba61d..d6f6795dd 100644 --- a/docs/self-host/postgresql.md +++ b/docs/self-host/postgresql.md @@ -20,7 +20,7 @@ For a managed service or a server you run, create a database and a user for the ## Create the database and its user -The server creates its tables in the database's `public` schema at every start, and migrates them there after an upgrade. Its user needs `CREATE` on that schema every time, not only the first. A user that owns the database has it: +The server creates its tables in the database's `public` schema at its first start and any it lacks at every later start; before launch it migrates nothing, and refuses a database made in another layout of the ledger. Its user needs `CREATE` on that schema every time, not only the first. A user that owns the database has it: ```sql CREATE ROLE auto_runtime LOGIN PASSWORD ''; diff --git a/docs/self-host/production.md b/docs/self-host/production.md index 13eb5517e..3c63508b5 100644 --- a/docs/self-host/production.md +++ b/docs/self-host/production.md @@ -2,7 +2,7 @@ Auto Runtime is in early development and is not ready for production use; see [Availability](../concepts/functions.md#availability). Your team can still run it today, and you can rely on these behaviors: -- Auto keeps every playbook, definition, run and its history in its ledger, with the workflows that wait on timers and events. On a durable volume or database, all of it survives a restart, and a new version migrates it when it starts. +- Auto keeps every playbook, definition, run and its history in its ledger, with the workflows that wait on timers and events. On a durable volume or database, all of it survives a restart. A release that keeps the ledger's layout opens it as it is; before launch, one that changes the layout refuses the older ledger at start and says what to delete ([Upgrades](#upgrades)). - Backups are yours to run: copy the volume, or rely on your database's own backups. - One server runs the workflows at a time. If it stops or dies, a standby that shares its database takes them over within about three seconds. @@ -182,7 +182,7 @@ Pin a full version, such as `docker.on.auto/onauto/runtime:0.9.1`, and upgrade o 1. Read the [release notes](https://github.com/BeOnAuto/runtime/releases). 2. Pull the new tag. With the files from [Run with Docker](docker.md#upgrade), change the `IMAGE` line of `.env`. -3. Replace the container. The new server migrates the ledger at start. +3. Replace the container. A new server that keeps the ledger's layout opens the ledger as it is. Before launch nothing is carried over: a server that changes the layout refuses the ledger at start, names the layout it found and the one it keeps, and says to delete the files or to point `DATABASE_URL` at an empty database. Servers of different versions never share a database at the same time. With several servers, follow [Upgrade several servers](workflows.md#upgrade-several-servers): stop every old server before you start a new one. diff --git a/packages/ledger/README.md b/packages/ledger/README.md index 2c3dad4f4..b58b3a186 100644 --- a/packages/ledger/README.md +++ b/packages/ledger/README.md @@ -267,7 +267,7 @@ The ledger keeps content whole beside its messages rather than in them, since ev `content.stage(playbook, run, content)`, `content.staged(playbook, run, sha256)` and `content.unstage(playbook, run)` keep, read and remove content staged under a run, by its id, with the rules of a put and a get for the uploads, the heads and the digest, so a server reads what another staged on the same database. `unstage` records the run in `unstaged_runs` and then, in one transaction, removes the run's heads and then its chunks. The record stays, so `unstaged_runs` grows by one small row, the playbook key and the run id, for each run whose staging ended, as many as the runs that stage content; no capability of this repository stages content yet. A stage under a run already unstaged is refused with `StagingEnded` and leaves its stream unread, and a stage looks again once its head is written; if the run was unstaged meanwhile it deletes its own head and chunks and is refused the same way. Since the unstage records the run before it deletes and the stage writes its head before it looks, either the stage sees the record or the unstage's deletes see the head, so a stage that races an unstage leaves nothing behind. A read of a staged content whose run is unstaged while it is read fails at the first chunk it finds gone, as a defect, in words that say so: `The content staged under has no chunk , as when its run is unstaged while it is read`. -A database whose content tables have another shape than these, a chunk of text or a head without a media type, is refused when the ledger opens, as a `LedgerShapeRefused` of `@on.auto/ledger` that names the table, what it holds, the columns found and the columns kept, in words that say to delete the local database, since nothing stored before launch is carried over. The server, which knows where the ledger is kept, answers it with words of its own: where the ledger is, what to do, and where to ask. +These tables and their indexes are part of the layout of the ledger, so a change to them raises `ledgerLayout`, and a database made in another layout is refused when the ledger opens (see [The layout of the ledger](#the-layout-of-the-ledger)). `measure/recorded-content.ts` runs `measuredContent` of `measure/content-measurement.ts` on each store. It puts the answers the tests keep, `answerOfAbout` of `@on.auto/operations/testing`, of 100 bytes to 20 MB of JSON, three times each, the second time as it is and the third with its digest expected, and reads each back and parses it; it puts the 64 MiB of bytes the tests generate, `generatedStream`, as a stream the same three ways and streams them back, their digest checked. The content is made before each put is timed, and SQLite is opened as the server opens it, with a page cache of 8 MiB and no memory map. It then puts and gets the 64 MiB once more for another playbook, made as the stream is pulled, and samples two figures of the process every 5 ms: the heap in use and the array buffers after a full collection, `heapUsed + arrayBuffers` of `process.memoryUsage()` after `gc()`, and the resident set, `rss`, each reported as the most it rose above its figure before the put or the get. The routine takes any `RecordedContent`, and is the subpath `@on.auto/ledger/content-measurement`, so the object store of `@on.auto/object-storage` is measured by the same steps, as its README records: @@ -298,7 +298,7 @@ Measured on 2026-10-12 on an Apple M4 Max with 16 cores, with Node 26.10.0, SQLi ## Kept secrets -The ledger keeps the rows of the secret store of [`@on.auto/secrets`](../secrets/README.md) apart from its facts and from its content: what the runtime must hold to act for someone, such as a key a person pasted or a token minted for them, which no fact of a stream may carry, since every reader of a stream reads its facts. It keeps the rows alone and seals nothing: a value reaches it as its keeper holds it, sealed under a data key of its own that the store's custodian wraps, as that package's README sets out under [Sealing](../secrets/README.md#sealing), or not at all, where a secret manager keeps it and the row holds the name and the version of the value there, as its README sets out under [The keeper](../secrets/README.md#the-keeper), so the ledger knows no key and neither its database nor a backup of it holds a value anyone can read without the custodian's or the manager's. The rows are not part of the `Ledger` port: `sqliteSecretRows({ fileName })` of `@on.auto/ledger/sqlite3` and `postgresqlSecretRows({ pools })` of `@on.auto/ledger/postgresql` each answer an `Effect` of the `SecretRows` port of `@on.auto/secrets`, opened on connections of their own from the settings the ledger is opened from, as the workflow host opens its own, once the ledger is open, as the server opens them. On SQLite the rows migrate the ledger's schema on their own pool first, which finds everything made when the ledger opened and creates nothing, since a pool of its own on `:memory:` is a database of its own; on PostgreSQL they open a pool of two connections, from `pools` or any `(bounds) => SecretConnections`, on the database the ledger migrated, and log a connection they lose as the ledger logs one of its own. One table holds them, `kept_secrets`, created with `CREATE TABLE IF NOT EXISTS` by the same schema step that creates the content tables, on SQLite and on PostgreSQL, through the same dialect, and looked up first in the catalog as they are, so a start that finds it creates nothing: +The ledger keeps the rows of the secret store of [`@on.auto/secrets`](../secrets/README.md) apart from its facts and from its content: what the runtime must hold to act for someone, such as a key a person pasted or a token minted for them, which no fact of a stream may carry, since every reader of a stream reads its facts. It keeps the rows alone and seals nothing: a value reaches it as its keeper holds it, sealed under a data key of its own that the store's custodian wraps, as that package's README sets out under [Sealing](../secrets/README.md#sealing), or not at all, where a secret manager keeps it and the row holds the name and the version of the value there, as its README sets out under [The keeper](../secrets/README.md#the-keeper), so the ledger knows no key and neither its database nor a backup of it holds a value anyone can read without the custodian's or the manager's. The rows are not part of the `Ledger` port: `sqliteSecretRows({ fileName })` of `@on.auto/ledger/sqlite3` and `postgresqlSecretRows({ pools })` of `@on.auto/ledger/postgresql` each answer an `Effect` of the `SecretRows` port of `@on.auto/secrets`, opened on connections of their own from the settings the ledger is opened from, as the workflow host opens its own, once the ledger is open, as the server opens them. On SQLite the rows migrate the ledger's schema on their own pool first, which finds the layout the ledger wrote and everything it made when it opened, and creates nothing, since a pool of its own on `:memory:` is a database of its own; on PostgreSQL they open a pool of two connections, from `pools` or any `(bounds) => SecretConnections`, on the database the ledger migrated, and log a connection they lose as the ledger logs one of its own. One table holds them, `kept_secrets`, created with `CREATE TABLE IF NOT EXISTS` by the same schema step that creates the content tables, on SQLite and on PostgreSQL, through the same dialect, and looked up first in the catalog as they are, so a start that finds it creates nothing: | Column | What it holds | | --------------- | ---------------------------------------------------------------------------------------- | @@ -312,7 +312,7 @@ The ledger keeps the rows of the secret store of [`@on.auto/secrets`](../secrets | `wrapped_key` | the value's data key as the custodian wrapped it, or `NULL` for a keeper that wraps none | | `kid` | the kid of the custodian's key that wrapped it, or `NULL` for a keeper that wraps none | -A database whose table of secrets has another shape than this is refused when the ledger opens, as an older shape of the content tables is, by `requireTheShapeKept` of `src/content/content-tables.ts`, with the same `LedgerShapeRefused`. +The table is part of the layout of the ledger, as the content tables are (see [The layout of the ledger](#the-layout-of-the-ledger)). Each operation of the port is one statement, every value bound: `keep` an `INSERT`; `read` a `SELECT` that answers `claimed_until` only while the claim is open; `replace` and `claim` one conditional `UPDATE` each with `RETURNING`, under the version read, the claim's also under no claim being open; `endingsOf` one `SELECT` of the ids named, and nothing asked for no ids; `forget` a `DELETE` that returns the holding it deleted; `heldBy` a `SELECT` of a keeper's rows, under a kid where one is named and after an id where one is named, `ORDER BY id` with the `LIMIT` asked; and `reheld` a conditional `UPDATE` of the four columns of the holding, only while the row's version, `kept_by` and `held` are the ones read and its `wrapped_key` and `kid` too, compared through `COALESCE(…, '')` so that a holding that has none compares alike on both stores. Every time the rows write or compare, `kept_at`, the claim's end and the moment a claim is judged open, is read from Effect's `Clock`, the clock the times of the facts are read from, so a test drives it with `TestClock`. A failure of the database is a defect, as it is for the ledger. @@ -347,7 +347,7 @@ The table is `run_outcomes_4`, with the index `run_outcomes_4_by_playbook_and_da ### Creating and filling a table -When the ledger opens with a projection, it looks its table up by name in the catalog, as it does its indexes, and does nothing more when it finds it. When it does not, it creates the table and its indexes, fills it by replaying every run stream of the store, drops the tables of earlier versions, such as `run_outcomes_3` and below, and analyses the new one, all in one transaction, so that a fill that is interrupted, by a mapping that throws or a process that stops, leaves nothing behind and is done again at the next open. On SQLite this runs after Emmett's migration and the indexes, on the same connections; on PostgreSQL in `onAfterSchemaCreated`, inside the migration's transaction and under its advisory lock, so that servers that start together fill the table once. Emmett's migrator waits 10 s for that lock, `defaultDatabaseLockOptions` in `@event-driven-io/dumbo` 0.13.0-beta.56, `dist/pg.js` line 673, less than the fill of a large ledger takes, and the typed options of Emmett's store and of its `schema.migrate()` offer no longer wait. So the ledger takes the same lock first, in `onBeforeSchemaCreated`, inside the migration's transaction (`migrationLockTakenWithin` in `src/postgresql/postgresql-run-outcomes.ts`): it tries `pg_try_advisory_xact_lock` every 100 ms for up to 60 s, past the longest fill measured below, 42.7 s, and once it has the lock holds it through the migration and the fill until the transaction commits; the migrator, in the same transaction, takes it again at once. A server that starts while another fills a large ledger waits for it, and stops at start, saying why, only when the lock is still held after 60 s. +When the ledger opens with a projection, it looks its table up by name in the catalog, as it does its indexes, and does nothing more when it finds it. When it does not, it creates the table and its indexes, fills it by replaying every run stream of the store, drops the tables of earlier versions, such as `run_outcomes_3` and below, and analyses the new one, all in one transaction, so that a fill that is interrupted, by a mapping that throws or a process that stops, leaves nothing behind and is done again at the next open. On SQLite this runs after Emmett's migration and the indexes, on the same connections; on PostgreSQL in `onAfterSchemaCreated`, inside the migration's transaction and under its advisory lock, so that servers that start together fill the table once. Emmett's migrator waits 10 s for that lock, `defaultDatabaseLockOptions` in `@event-driven-io/dumbo` 0.13.0-beta.56, `dist/pg.js` line 673, less than the fill of a large ledger takes, and the typed options of Emmett's store and of its `schema.migrate()` offer no longer wait. So the ledger takes the same lock first, in `onBeforeSchemaCreated`, inside the migration's transaction (`migrationLockTakenWithin` in `src/postgresql/postgresql-run-outcomes.ts`): it tries `pg_try_advisory_xact_lock` every 100 ms for up to 60 s, past the longest fill measured below, 42.7 s, and once it has the lock holds it through the migration and the fill until the transaction commits; the migrator, in the same transaction, takes it again at once. With the lock held, the same hook checks the layout of the ledger before anything else (see [The layout of the ledger](#the-layout-of-the-ledger)). A server that starts while another fills a large ledger waits for it, and stops at start, saying why, only when the lock is still held after 60 s. A projection keyed by its stream is filled a stream at a time. No index lists the streams of a kind across playbooks, so the fill scans the store's streams, `emt_streams`, for the names of the streams of the projection's kinds in the order of their names, with the size of their messages of the mapping's types, and takes of them as many as hold at most 16 MiB of those messages, at least one, so that a ledger of records of 1 MiB does not hold hundreds of megabytes at once. It lists as many streams as the batch before showed fit: one at first, then as many as 16 MiB holds at the bytes per stream of the streams the batch before took, at most twice as many as it listed before and at most 100. On PostgreSQL the size is the length of each record's text, which reads the record, so listing 100 streams of records of 1 MiB would read 100 MiB to load 16 MiB; listed this way, the fill sizes about as many bytes as it loads. For each batch it reads, in one statement, those messages in their order, decodes each as the store's reads do, folds the messages of each stream through the mapping, and writes the rows in upserts of as many rows as 100 parameters hold on SQLite, the most the ledger binds in one statement there, 8 for the outcomes of runs, and of 500 on PostgreSQL. A change to what a table keeps is a new version, `run_outcomes_5`, which the next server fills this way and whose fill drops `run_outcomes_4`. Servers of different versions do not share a database at once. @@ -446,6 +446,18 @@ Building the layer connects to the database and creates or migrates Emmett's tab The database's user needs `CREATE` on the `public` schema at every start, not only the first: the ledger migrates at every start, and the migrator runs `CREATE TABLE IF NOT EXISTS` on its own table, which PostgreSQL checks for the privilege before it looks whether the table exists, so a user that may only read and write the tables fails to start with `permission denied for schema public`. It also needs to read and write the ledger's tables, and to create its functions and sequence on an empty database. +## The layout of the ledger + +`ledgerLayout`, in `src/layout/ledger-layout.ts`, numbers the layout of every table and index the ledger makes itself: the playbook's indexes, the content tables and their indexes, `kept_secrets`, the table of each projection with its indexes, the outcomes of runs among them, and on PostgreSQL the table that keeps the number. It is 1, and is raised by hand with any change to one of them. The tables Emmett and its database library make, whose names begin with `emt_` and `dmb_`, are theirs and outside the layout. + +The ledger writes the number once, when it makes a database, before any table: on SQLite in `PRAGMA user_version`, as the first step of the store's `migrate`, before Emmett's migration; on PostgreSQL in the one row of `ledger_layout`, `layout INTEGER NOT NULL` beside a key whose `CHECK` allows a single value, in `onBeforeSchemaCreated`, inside Emmett's migration transaction, right after `migrationLockTakenWithin` has the advisory lock (`beforeTheSchema` of `src/postgresql/postgresql-schema.ts`), so servers that start together agree and a refusal rolls everything back. Every open applies one rule, `requireTheLayoutKept` of `src/layout/layout-kept.ts`, through a dialect of each store, `sqliteLayout` and `postgresqlLayout` of `src/layout/layout-dialects.ts`, which reads the number and counts the tables of the schema, in one statement on SQLite, and on PostgreSQL in one that counts them and finds `ledger_layout` among them and, when it does, a second that reads its row: + +- A database in the layout kept opens, and the tables and indexes it lacks are created, each looked up first. +- A database without a number that holds no table, `sqlite_master` holding none on SQLite and `pg_tables` none in `current_schema()` on PostgreSQL, is new: it is given the number, and the ledger goes on. +- A database in another layout, or one that holds tables but no number, is refused as a `LedgerLayoutRefused` of `@on.auto/ledger`, tagged `ledger_layout_refused`, with `found`, the layout found or none, and `kept`, the layout this server keeps, in words that say to delete the local database, since nothing stored before launch is carried over. The server, which knows where the ledger is kept, answers it with words of its own: where the ledger is, the layout it keeps and the one it found, what to do, and where to ask. + +`src/layout/ledger-layout.test.ts` pins the layout. It opens a new ledger on SQLite with the run outcome mapping and the projections the tests use, reads from `sqlite_master` the name and the statement of every table and index whose name begins with neither `emt_`, `dmb_` nor `sqlite_`, each statement on one line, and compares them, sorted by name, and `ledgerLayout` with the catalog it keeps for layout 1, so a change to anything the ledger makes fails it, in words that say to raise `ledgerLayout` and pin the catalog again. + ## PostgreSQL The ledger on PostgreSQL behaves as it does on SQLite; one suite of tests runs against both (see [Testing](#testing)). What was verified on Emmett's PostgreSQL store, `@event-driven-io/emmett-postgresql` 0.43.0-beta.50 with `pg` 8.23.1 against PostgreSQL 18.6: @@ -470,7 +482,7 @@ What every store must provide, for the ledger to run at all: - The event store's own operations, to read a stream or its tail after a version, append with an expected version, migrate and close. A command makes at most one append and needs no transaction of its own. - The read of what a playbook recorded, with the indexes the ledger creates when it opens; each statement binds at most 15 parameters, and none needs a transaction. - An append on SQLite carries at most eight events: Emmett binds ten parameters for each event it inserts, and the ledger binds at most 100 in one statement there. -- A SQLite database must provide what the reads use: `octet_length` (SQLite 3.43 and later), window functions (3.25 and later), the JSON functions, of which the reads use `json_each` and `json_group_array` (built in since 3.38), and partial and expression indexes (3.8 and 3.9), and what the rows of kept secrets use, an `UPDATE` and a `DELETE` with `RETURNING` (3.35); `sqlite3` 6.0.1 carries SQLite 3.52.0. +- A SQLite database must provide what the reads use: `octet_length` (SQLite 3.43 and later), window functions (3.25 and later), the JSON functions, of which the reads use `json_each` and `json_group_array` (built in since 3.38), and partial and expression indexes (3.8 and 3.9), what the rows of kept secrets use, an `UPDATE` and a `DELETE` with `RETURNING` (3.35), and what the layout of the ledger uses, `PRAGMA user_version` and its table-valued function `pragma_user_version` (3.16); `sqlite3` 6.0.1 carries SQLite 3.52.0. - No adapter or application may nest a stream under `runs/` within a playbook: the key of a stream's kind ends at the fourth `/`, so the list of runs would take any stream named `runs//…` for a run. - Only the entries `src/sqlite3.ts` and `src/postgresql/postgresql-ledger.ts` know which driver is in use; the main entry loads neither `sqlite3` nor `pg`. @@ -484,7 +496,7 @@ Opened without the run outcome mapping and the projections, the ledger neither c ## Testing -The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it), and a database whose content tables have an older shape refused (`older-content-shapes-behaviour.ts`); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first, with a table of secrets of an older shape refused: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing). `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, its fake answering the columns of the content tables and of the table of secrets (`src/secrets/kept-table-columns.ts`). The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. +The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)). Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and counts the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. ```bash docker run --detach --name ledger-pg-test --publish 127.0.0.1:19632:5432 --env POSTGRES_PASSWORD=ledger-test postgres:18.6-alpine diff --git a/packages/ledger/measure-heads.ts b/packages/ledger/measure-heads.ts index 80bc61d5a..585e7f6f7 100644 --- a/packages/ledger/measure-heads.ts +++ b/packages/ledger/measure-heads.ts @@ -5,7 +5,7 @@ import { Client } from 'pg'; import type { StoredPageRequest } from './src/event-store.ts'; import { postgresqlRecordedStore, type Query } from './src/postgresql-reads/postgresql-recorded.ts'; -import { plainPoolsOn, postgresqlEventStore } from './src/postgresql/postgresql-event-store.ts'; +import { plainPoolsOn, postgresqlEventStore } from './src/postgresql/postgresql-ledger.ts'; type Querying = Readonly>; diff --git a/packages/ledger/measure/stores.ts b/packages/ledger/measure/stores.ts index 3ff3a8fbd..3ddb25ac8 100644 --- a/packages/ledger/measure/stores.ts +++ b/packages/ledger/measure/stores.ts @@ -5,7 +5,7 @@ import { Function } from 'effect'; import { Client } from 'pg'; import type { LedgerStore } from '../src/event-store.ts'; -import { plainPoolsOn, postgresqlEventStore } from '../src/postgresql/postgresql-event-store.ts'; +import { plainPoolsOn, postgresqlEventStore } from '../src/postgresql/postgresql-ledger.ts'; import { sqliteEventStore } from '../src/sqlite-event-store.ts'; import type { modestConnections } from '../src/sqlite3.ts'; import { temporaryDatabase } from '../src/testing/temporary-database.ts'; diff --git a/packages/ledger/src/content/content-behaviour.ts b/packages/ledger/src/content/content-behaviour.ts index 77bbb281b..9456a0db9 100644 --- a/packages/ledger/src/content/content-behaviour.ts +++ b/packages/ledger/src/content/content-behaviour.ts @@ -16,7 +16,6 @@ import { Effect, Schema } from 'effect'; import { describe, expect, it } from 'vitest'; import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; -import { contentOfAnOlderShape } from './older-content-shapes-behaviour.ts'; import { unnamedChunksSwept } from './unnamed-chunks-behaviour.ts'; const ChunkRows = Schema.Array(Schema.Struct({ chunk: Schema.Number, bytes: Schema.Number })); @@ -183,5 +182,4 @@ export function contentBehaviour(entry: LedgerEntry): void { readWhileUnstaged(entry); }); unnamedChunksSwept(entry); - contentOfAnOlderShape(entry); } diff --git a/packages/ledger/src/content/content-dialects.ts b/packages/ledger/src/content/content-dialects.ts index db8efeb5a..d17efb69b 100644 --- a/packages/ledger/src/content/content-dialects.ts +++ b/packages/ledger/src/content/content-dialects.ts @@ -6,7 +6,6 @@ export interface ContentDialect { readonly bytesType: string; readonly bytesOfHex: (hex: string) => SQL; readonly relationsAmong: (kind: RelationKind, names: readonly string[]) => SQL; - readonly columnsOf: (table: string) => SQL; } const postgresqlKinds: Readonly> = { @@ -19,7 +18,6 @@ export const sqliteContent: ContentDialect = { bytesOfHex: (hex) => SQL`unhex(${hex})`, relationsAmong: (kind, names) => SQL`SELECT name FROM sqlite_master WHERE type = ${kind} AND name IN (SELECT value FROM json_each(${JSON.stringify(names)}))`, - columnsOf: (table) => SQL`SELECT name FROM pragma_table_info(${table})`, }; export const postgresqlContent: ContentDialect = { @@ -30,6 +28,4 @@ export const postgresqlContent: ContentDialect = { names.map((name) => SQL`${name}`), ', ', )}) AND pg_table_is_visible(oid)`, - columnsOf: (table) => SQL`SELECT attname AS name FROM pg_attribute - WHERE attrelid = to_regclass(${table}) AND attnum > 0 AND NOT attisdropped`, }; diff --git a/packages/ledger/src/content/content-tables.ts b/packages/ledger/src/content/content-tables.ts index 9dc2facaa..e2d1f8338 100644 --- a/packages/ledger/src/content/content-tables.ts +++ b/packages/ledger/src/content/content-tables.ts @@ -2,10 +2,8 @@ import { SQL } from '@event-driven-io/dumbo'; import { Schema } from 'effect'; import type { StatementExecutor } from '../event-store.ts'; -import { inTurn } from '../projections/inline-projection.ts'; import { createMissingIndexes, type PlaybookIndex } from '../recorded/missing-indexes.ts'; import type { ContentDialect } from './content-dialects.ts'; -import { ledgerShapeRefused } from './ledger-shape-refused.ts'; export interface Shelf { readonly chunks: string; @@ -31,9 +29,8 @@ export const stagingShelf: Shelf = { export const unstagedRuns = 'unstaged_runs'; -export interface ContentTable { +interface ContentTable { readonly name: string; - readonly columns: readonly string[]; readonly create: (dialect: ContentDialect) => SQL; } @@ -55,7 +52,6 @@ function chunkTableOf({ chunks, placeColumns }: Shelf): ContentTable { const key = [...placeColumns, 'upload_id', 'chunk']; return { name: chunks, - columns: [...key, 'data'], create: ({ bytesType }) => SQL`CREATE TABLE IF NOT EXISTS ${SQL.identifier(chunks)} ( ${textColumns([...placeColumns, 'upload_id'])}, chunk INTEGER NOT NULL, @@ -69,7 +65,6 @@ function headTableOf({ heads, placeColumns }: Shelf): ContentTable { const key = [...placeColumns, 'sha256']; return { name: heads, - columns: [...key, 'upload_id', 'bytes', 'chunks', 'media_type'], create: () => SQL`CREATE TABLE IF NOT EXISTS ${SQL.identifier(heads)} ( ${textColumns([...key, 'upload_id'])}, bytes BIGINT NOT NULL, @@ -82,7 +77,6 @@ function headTableOf({ heads, placeColumns }: Shelf): ContentTable { const unstagedRunsTable: ContentTable = { name: unstagedRuns, - columns: stagingShelf.placeColumns, create: () => SQL`CREATE TABLE IF NOT EXISTS ${SQL.identifier(unstagedRuns)} ( ${textColumns(stagingShelf.placeColumns)}, PRIMARY KEY (${namesOf(stagingShelf.placeColumns)}) @@ -91,7 +85,7 @@ const unstagedRunsTable: ContentTable = { export const shelves: readonly Shelf[] = [keptShelf, stagingShelf]; -export const contentTables: readonly ContentTable[] = [ +const contentTables: readonly ContentTable[] = [ ...shelves.flatMap((shelf) => [chunkTableOf(shelf), headTableOf(shelf)]), unstagedRunsTable, ]; @@ -116,30 +110,8 @@ export async function namesFound(execute: StatementExecutor, sql: SQL): Promise< return Schema.decodeUnknownSync(NameRows)(rows).map(({ name }) => name); } -export interface KeptTable { - readonly name: string; - readonly columns: readonly string[]; - readonly holds: string; -} - -export async function requireTheShapeKept( - execute: StatementExecutor, - dialect: ContentDialect, - { name, columns, holds }: KeptTable, -): Promise { - const found = (await namesFound(execute, dialect.columnsOf(name))).toSorted(); - const kept = columns.toSorted(); - if (found.join() !== kept.join()) { - throw ledgerShapeRefused({ table: name, holds, found, kept }); - } -} - export async function createdContentTables(execute: StatementExecutor, dialect: ContentDialect): Promise { const existing = new Set(await namesFound(execute, dialect.relationsAmong('table', contentTableNames))); - await inTurn( - contentTables.filter(({ name }) => existing.has(name)), - (table) => requireTheShapeKept(execute, dialect, { ...table, holds: 'recorded content' }), - ); await createMissingIndexes( execute, contentTables.map((table) => ({ name: table.name, create: () => table.create(dialect) })), diff --git a/packages/ledger/src/content/ledger-shape-refused.ts b/packages/ledger/src/content/ledger-shape-refused.ts deleted file mode 100644 index 987d6a1df..000000000 --- a/packages/ledger/src/content/ledger-shape-refused.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Data } from 'effect'; - -export interface RefusedShape { - readonly table: string; - readonly holds: string; - readonly found: readonly string[]; - readonly kept: readonly string[]; -} - -export class LedgerShapeRefused extends Data.TaggedError('ledger_shape_refused')< - RefusedShape & { readonly message: string } -> {} - -export function ledgerShapeRefused({ table, holds, found, kept }: RefusedShape): LedgerShapeRefused { - return new LedgerShapeRefused({ - table, - holds, - found, - kept, - message: `The ledger's database keeps ${holds} in a shape this server no longer reads: ${table} has the columns ${found.join(', ')}, where this server keeps ${kept.join(', ')}. Nothing stored before launch is carried over, so delete the local database and start the server again`, - }); -} diff --git a/packages/ledger/src/content/older-content-shapes-behaviour.ts b/packages/ledger/src/content/older-content-shapes-behaviour.ts deleted file mode 100644 index 8c4b1e52f..000000000 --- a/packages/ledger/src/content/older-content-shapes-behaviour.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; - -const olderShapes: ReadonlyArray = [ - [ - 'a chunk that keeps text', - 'CREATE TABLE recorded_content_chunks (playbook_key TEXT NOT NULL, sha256 TEXT NOT NULL, chunk INTEGER NOT NULL, text TEXT NOT NULL, PRIMARY KEY (playbook_key, sha256, chunk))', - 'recorded_content_chunks has the columns chunk, playbook_key, sha256, text, where this server keeps chunk, data, playbook_key, upload_id', - ], - [ - 'a head without a media type', - 'CREATE TABLE recorded_content_heads (playbook_key TEXT NOT NULL, sha256 TEXT NOT NULL, bytes BIGINT NOT NULL, chunks INTEGER NOT NULL, PRIMARY KEY (playbook_key, sha256))', - 'recorded_content_heads has the columns bytes, chunks, playbook_key, sha256, where this server keeps bytes, chunks, media_type, playbook_key, sha256, upload_id', - ], -]; - -export function contentOfAnOlderShape(entry: LedgerEntry): void { - describe('a database whose recorded content has an older shape', () => { - it.each(olderShapes)( - 'is refused when the ledger opens, for %s, with words that say to delete it', - async (_shape, table, found) => { - const database = await entry.aDatabase(); - await entry.queried(database, table); - - await expect(aLedger(entry, database)).rejects.toThrow( - `The ledger's database keeps recorded content in a shape this server no longer reads: ${found}. Nothing stored before launch is carried over, so delete the local database and start the server again`, - ); - }, - ); - }); -} diff --git a/packages/ledger/src/index.ts b/packages/ledger/src/index.ts index 9975ba15a..4a0f29db5 100644 --- a/packages/ledger/src/index.ts +++ b/packages/ledger/src/index.ts @@ -20,7 +20,7 @@ export type { StoredPlace, StoredRecord, } from './event-store.ts'; -export { LedgerShapeRefused, type RefusedShape } from './content/ledger-shape-refused.ts'; +export { LedgerLayoutRefused } from './layout/ledger-layout-refused.ts'; export { cursorOf as recordedCursorOf } from './recorded/cursor.ts'; export { decisionLoop, diff --git a/packages/ledger/src/layout/layout-dialects.ts b/packages/ledger/src/layout/layout-dialects.ts new file mode 100644 index 000000000..c9b026f98 --- /dev/null +++ b/packages/ledger/src/layout/layout-dialects.ts @@ -0,0 +1,59 @@ +import { SQL } from '@event-driven-io/dumbo'; +import { Schema } from 'effect'; + +import type { StatementExecutor } from '../event-store.ts'; + +export interface FoundLayout { + readonly layout: number | undefined; + readonly tables: number; +} + +export interface LayoutDialect { + readonly found: (execute: StatementExecutor) => Promise; + readonly written: (layout: number) => readonly SQL[]; +} + +const noLayout = 0; + +const NumberedRow = Schema.Tuple([Schema.Struct({ tables: Schema.Number, layout: Schema.Number })]); + +export const sqliteLayout: LayoutDialect = { + found: async (execute) => { + const { rows } = await execute.query( + SQL`SELECT (SELECT count(*) FROM sqlite_master WHERE type = 'table') AS tables, user_version AS layout FROM pragma_user_version`, + ); + const [{ tables, layout }] = Schema.decodeUnknownSync(NumberedRow)(rows); + return { layout: layout === noLayout ? undefined : layout, tables }; + }, + written: (layout) => [SQL`PRAGMA user_version = ${SQL.plain(String(layout))}`], +}; + +const layoutTable = 'ledger_layout'; + +const TablesRow = Schema.Tuple([Schema.Struct({ tables: Schema.Number, numbered: Schema.Boolean })]); + +const LayoutRow = Schema.Tuple([Schema.Struct({ layout: Schema.Number })]); + +async function layoutKeptIn(execute: StatementExecutor): Promise { + const { rows } = await execute.query(SQL`SELECT layout FROM ${SQL.identifier(layoutTable)}`); + const [{ layout }] = Schema.decodeUnknownSync(LayoutRow)(rows); + return layout; +} + +export const postgresqlLayout: LayoutDialect = { + found: async (execute) => { + const { rows } = await execute.query( + SQL`SELECT count(*)::int AS tables, count(*) FILTER (WHERE tablename = ${layoutTable}) > 0 AS numbered + FROM pg_tables WHERE schemaname = current_schema()`, + ); + const [{ tables, numbered }] = Schema.decodeUnknownSync(TablesRow)(rows); + return { layout: numbered ? await layoutKeptIn(execute) : undefined, tables }; + }, + written: (layout) => [ + SQL`CREATE TABLE ${SQL.identifier(layoutTable)} ( + one_row BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (one_row), + layout INTEGER NOT NULL + )`, + SQL`INSERT INTO ${SQL.identifier(layoutTable)} (layout) VALUES (${layout})`, + ], +}; diff --git a/packages/ledger/src/layout/layout-kept.ts b/packages/ledger/src/layout/layout-kept.ts new file mode 100644 index 000000000..d578cc17a --- /dev/null +++ b/packages/ledger/src/layout/layout-kept.ts @@ -0,0 +1,20 @@ +import type { StatementExecutor } from '../event-store.ts'; +import { inTurn } from '../projections/inline-projection.ts'; +import type { FoundLayout, LayoutDialect } from './layout-dialects.ts'; +import { ledgerLayoutRefused } from './ledger-layout-refused.ts'; +import { ledgerLayout } from './ledger-layout.ts'; + +function isNew({ layout, tables }: FoundLayout): boolean { + return layout === undefined && tables === 0; +} + +export async function requireTheLayoutKept(execute: StatementExecutor, dialect: LayoutDialect): Promise { + const found = await dialect.found(execute); + if (found.layout === ledgerLayout) { + return; + } + if (!isNew(found)) { + throw ledgerLayoutRefused({ found: found.layout, kept: ledgerLayout }); + } + await inTurn(dialect.written(ledgerLayout), (statement) => execute.command(statement)); +} diff --git a/packages/ledger/src/layout/ledger-layout-behaviour.ts b/packages/ledger/src/layout/ledger-layout-behaviour.ts new file mode 100644 index 000000000..41ff404b3 --- /dev/null +++ b/packages/ledger/src/layout/ledger-layout-behaviour.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +import { inTurn } from '../projections/inline-projection.ts'; +import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; +import type { FoundLayout } from './layout-dialects.ts'; +import { LedgerLayoutRefused } from './ledger-layout-refused.ts'; +import { ledgerLayout } from './ledger-layout.ts'; + +const anotherLayout = ledgerLayout + 1; + +const whatToDo = + 'Nothing stored before launch is carried over, so delete the local database and start the server again'; + +export function layoutIn(entry: LedgerEntry, database: string): Promise { + return entry.executedOn(database, (execute) => entry.layout.found(execute)); +} + +function stampedWith(entry: LedgerEntry, database: string, layout: number): Promise { + return entry.executedOn(database, (execute) => + inTurn(entry.layout.written(layout), (statement) => execute.command(statement)), + ); +} + +function refusalOf(entry: LedgerEntry, database: string): Promise { + return aLedger(entry, database).catch((error: unknown) => error); +} + +export function ledgerLayoutBehaviour(entry: LedgerEntry): void { + describe('the layout of the ledger', () => { + it('is written into a database that holds no table, which then opens again in that layout', async () => { + const database = await entry.aDatabase(); + await aLedger(entry, database); + + await aLedger(entry, database); + + expect((await layoutIn(entry, database)).layout).toBe(ledgerLayout); + }); + + it('refuses a database in another layout, naming the layout found and the one kept, and leaves it as it was', async () => { + const database = await entry.aDatabase(); + await stampedWith(entry, database, anotherLayout); + const before = await layoutIn(entry, database); + + const refused = await refusalOf(entry, database); + + expect(refused).toBeInstanceOf(LedgerLayoutRefused); + expect(refused).toMatchObject({ + found: anotherLayout, + kept: ledgerLayout, + message: `The ledger's database is in layout ${anotherLayout} of the ledger, where this server keeps layout ${ledgerLayout}. ${whatToDo}`, + }); + expect(await layoutIn(entry, database)).toEqual(before); + }); + + it('refuses a database that holds a table but no layout number, and leaves it as it was', async () => { + const database = await entry.aDatabase(); + await entry.queried(database, 'CREATE TABLE kept_notes (note TEXT)'); + + const refused = await refusalOf(entry, database); + + expect(refused).toBeInstanceOf(LedgerLayoutRefused); + expect(refused).toMatchObject({ + found: undefined, + kept: ledgerLayout, + message: `The ledger's database holds tables but no layout number of the ledger, where this server keeps layout ${ledgerLayout}. ${whatToDo}`, + }); + expect(await layoutIn(entry, database)).toEqual({ layout: undefined, tables: 1 }); + }); + }); +} diff --git a/packages/ledger/src/layout/ledger-layout-refused.ts b/packages/ledger/src/layout/ledger-layout-refused.ts new file mode 100644 index 000000000..e131850bb --- /dev/null +++ b/packages/ledger/src/layout/ledger-layout-refused.ts @@ -0,0 +1,27 @@ +import { Data } from 'effect'; + +interface RefusedLayout { + readonly found: number | undefined; + readonly kept: number; +} + +export class LedgerLayoutRefused extends Data.TaggedError('ledger_layout_refused')< + RefusedLayout & { readonly message: string } +> {} + +const whatToDo = + 'Nothing stored before launch is carried over, so delete the local database and start the server again'; + +function whatWasFound(found: number | undefined): string { + return found === undefined + ? "The ledger's database holds tables but no layout number of the ledger" + : `The ledger's database is in layout ${found} of the ledger`; +} + +export function ledgerLayoutRefused({ found, kept }: RefusedLayout): LedgerLayoutRefused { + return new LedgerLayoutRefused({ + found, + kept, + message: `${whatWasFound(found)}, where this server keeps layout ${kept}. ${whatToDo}`, + }); +} diff --git a/packages/ledger/src/layout/ledger-layout.test.ts b/packages/ledger/src/layout/ledger-layout.test.ts new file mode 100644 index 000000000..b9c3c6906 --- /dev/null +++ b/packages/ledger/src/layout/ledger-layout.test.ts @@ -0,0 +1,129 @@ +import { runTallies, runTallyRows, topicRows } from '@on.auto/operations/testing'; +import { Schema } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { queried } from '../recorded/sqlite-checks.ts'; +import { ledgerLayer } from '../sqlite3.ts'; +import { openLedgerWith } from '../testing/open-ledger.ts'; +import { temporaryDatabase } from '../testing/temporary-database.ts'; +import { ledgerLayout } from './ledger-layout.ts'; + +const pinned = { + layout: 1, + catalog: [ + { + name: 'kept_secrets', + sql: 'CREATE TABLE kept_secrets (id TEXT PRIMARY KEY, version INTEGER NOT NULL, kept_at TEXT NOT NULL, claimed_until TEXT, ends_with TEXT, kept_by TEXT NOT NULL, held TEXT NOT NULL, wrapped_key TEXT, kid TEXT)', + }, + { + name: 'ledger_definition_streams', + sql: "CREATE INDEX ledger_definition_streams ON emt_streams (CASE WHEN substr(stream_id, length(substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/'))) + 1, 12) = 'definitions/' THEN substr(stream_id, length(substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/'))) + 13) END) WHERE CASE WHEN substr(stream_id, length(substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/'))) + 1, 12) = 'definitions/' THEN substr(stream_id, length(substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/'))) + 13) END IS NOT NULL", + }, + { + name: 'ledger_first_messages_by_kind', + sql: "CREATE INDEX ledger_first_messages_by_kind ON emt_messages (substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/') + 1), '/')), global_position) WHERE stream_position = 1", + }, + { name: 'ledger_messages_by_id', sql: 'CREATE INDEX ledger_messages_by_id ON emt_messages (message_id)' }, + { + name: 'ledger_messages_by_playbook', + sql: "CREATE INDEX ledger_messages_by_playbook ON emt_messages (substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/')), global_position)", + }, + { + name: 'ledger_messages_by_playbook_and_correlation', + sql: "CREATE INDEX ledger_messages_by_playbook_and_correlation ON emt_messages (substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/')), json_extract(message_metadata, '$.correlationId'), global_position)", + }, + { + name: 'ledger_messages_by_playbook_and_time', + sql: "CREATE INDEX ledger_messages_by_playbook_and_time ON emt_messages (substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/')), created)", + }, + { + name: 'ledger_messages_by_stream', + sql: 'CREATE INDEX ledger_messages_by_stream ON emt_messages (stream_id, global_position)', + }, + { + name: 'recorded_content_chunks', + sql: 'CREATE TABLE recorded_content_chunks (playbook_key TEXT NOT NULL, upload_id TEXT NOT NULL, chunk INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY (playbook_key, upload_id, chunk))', + }, + { + name: 'recorded_content_heads', + sql: 'CREATE TABLE recorded_content_heads (playbook_key TEXT NOT NULL, sha256 TEXT NOT NULL, upload_id TEXT NOT NULL, bytes BIGINT NOT NULL, chunks INTEGER NOT NULL, media_type TEXT NOT NULL, PRIMARY KEY (playbook_key, sha256))', + }, + { + name: 'recorded_content_heads_by_upload', + sql: 'CREATE INDEX recorded_content_heads_by_upload ON recorded_content_heads (playbook_key, upload_id)', + }, + { + name: 'run_outcomes_4', + sql: 'CREATE TABLE run_outcomes_4 (playbook_key TEXT NOT NULL, row_key TEXT NOT NULL, started_day TEXT, started_at TEXT, last_started_at TEXT, definition_type TEXT, name TEXT, status TEXT, duration_ms INTEGER, input_tokens INTEGER, output_tokens INTEGER, cached_tokens INTEGER, PRIMARY KEY (playbook_key, row_key))', + }, + { + name: 'run_outcomes_4_by_playbook_and_day', + sql: 'CREATE INDEX run_outcomes_4_by_playbook_and_day ON run_outcomes_4 (playbook_key, started_day)', + }, + { + name: 'run_tallies_2', + sql: 'CREATE TABLE run_tallies_2 (playbook_key TEXT NOT NULL, row_key TEXT NOT NULL, fn TEXT, began_at INTEGER, status TEXT, facts INTEGER, open INTEGER, due_at INTEGER, last_message TEXT, PRIMARY KEY (playbook_key, row_key))', + }, + { + name: 'run_tallies_2_by_playbook_and_status', + sql: 'CREATE INDEX run_tallies_2_by_playbook_and_status ON run_tallies_2 (playbook_key, status, began_at)', + }, + { + name: 'run_tallies_2_due', + sql: 'CREATE INDEX run_tallies_2_due ON run_tallies_2 (due_at) WHERE due_at IS NOT NULL', + }, + { + name: 'staged_content', + sql: 'CREATE TABLE staged_content (playbook_key TEXT NOT NULL, run_id TEXT NOT NULL, sha256 TEXT NOT NULL, upload_id TEXT NOT NULL, bytes BIGINT NOT NULL, chunks INTEGER NOT NULL, media_type TEXT NOT NULL, PRIMARY KEY (playbook_key, run_id, sha256))', + }, + { + name: 'staged_content_by_upload', + sql: 'CREATE INDEX staged_content_by_upload ON staged_content (playbook_key, run_id, upload_id)', + }, + { + name: 'staged_content_chunks', + sql: 'CREATE TABLE staged_content_chunks (playbook_key TEXT NOT NULL, run_id TEXT NOT NULL, upload_id TEXT NOT NULL, chunk INTEGER NOT NULL, data BLOB NOT NULL, PRIMARY KEY (playbook_key, run_id, upload_id, chunk))', + }, + { + name: 'topics_2', + sql: 'CREATE TABLE topics_2 (playbook_key TEXT NOT NULL, row_key TEXT NOT NULL, topic TEXT, note TEXT, last_message TEXT, open INTEGER, next_at INTEGER, due_at INTEGER, PRIMARY KEY (playbook_key, row_key))', + }, + { name: 'topics_2_due', sql: 'CREATE INDEX topics_2_due ON topics_2 (due_at) WHERE due_at IS NOT NULL' }, + { + name: 'unstaged_runs', + sql: 'CREATE TABLE unstaged_runs (playbook_key TEXT NOT NULL, run_id TEXT NOT NULL, PRIMARY KEY (playbook_key, run_id))', + }, + ], +}; + +const theLedgersOwnRelations = `SELECT name, sql FROM sqlite_master + WHERE type IN ('table', 'index') AND name NOT GLOB 'emt_*' AND name NOT GLOB 'dmb_*' AND name NOT GLOB 'sqlite_*' + ORDER BY name`; + +function onOneLine(sql: string): string { + return sql.replaceAll(/\s+/gu, ' ').replaceAll('( ', '(').replaceAll(' )', ')'); +} + +const CatalogRows = Schema.Array(Schema.Struct({ name: Schema.String, sql: Schema.String })); + +async function catalogOfANewLedger(): Promise { + const { fileName, remove } = temporaryDatabase(); + onTestFinished(remove); + const opened = await openLedgerWith( + ledgerLayer({ fileName, runOutcomes: runTallies, projections: [runTallyRows, topicRows] }), + ); + await opened.dispose(); + const rows = Schema.decodeUnknownSync(CatalogRows)(await queried(fileName, theLedgersOwnRelations)); + return rows.map(({ name, sql }) => ({ name, sql: onOneLine(sql) })); +} + +describe('the layout of the ledger', () => { + it('is the layout of the tables and indexes the ledger makes, pinned here', async () => { + const catalog = await catalogOfANewLedger(); + + expect( + { layout: ledgerLayout, catalog }, + "The ledger's layout changed: raise ledgerLayout in src/layout/ledger-layout.ts, then pin here the layout and the catalog of every table and index the ledger makes", + ).toEqual(pinned); + }); +}); diff --git a/packages/ledger/src/layout/ledger-layout.ts b/packages/ledger/src/layout/ledger-layout.ts new file mode 100644 index 000000000..202d2fb39 --- /dev/null +++ b/packages/ledger/src/layout/ledger-layout.ts @@ -0,0 +1 @@ +export const ledgerLayout = 1; diff --git a/packages/ledger/src/layout/sqlite-schema.ts b/packages/ledger/src/layout/sqlite-schema.ts new file mode 100644 index 000000000..8ce469fec --- /dev/null +++ b/packages/ledger/src/layout/sqlite-schema.ts @@ -0,0 +1,17 @@ +import { sqliteContent } from '../content/content-dialects.ts'; +import { createdContentTables } from '../content/content-tables.ts'; +import type { StatementExecutor } from '../event-store.ts'; +import { createSQLitePlaybookIndexes } from '../recorded/sqlite-indexes.ts'; +import { createdSecretTable } from '../secrets/kept-secrets.ts'; +import { sqliteLayout } from './layout-dialects.ts'; +import { requireTheLayoutKept } from './layout-kept.ts'; + +export function sqliteLayoutKept(execute: StatementExecutor): Promise { + return requireTheLayoutKept(execute, sqliteLayout); +} + +export async function sqliteSchemaCreated(execute: StatementExecutor): Promise { + await createSQLitePlaybookIndexes(execute); + await createdContentTables(execute, sqliteContent); + await createdSecretTable(execute, sqliteContent); +} diff --git a/packages/ledger/src/ledger.test.ts b/packages/ledger/src/ledger.test.ts index e6be4eb12..66b21e59f 100644 --- a/packages/ledger/src/ledger.test.ts +++ b/packages/ledger/src/ledger.test.ts @@ -2,7 +2,8 @@ import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3' import { describe, onTestFinished } from 'vitest'; import { sqliteEventStore } from './index.ts'; -import { definitionStreamsIndexed, planOf, queried } from './recorded/sqlite-checks.ts'; +import { sqliteLayout } from './layout/layout-dialects.ts'; +import { definitionStreamsIndexed, executedOn, planOf, queried } from './recorded/sqlite-checks.ts'; import { keptSecretsBehaviour } from './secrets/kept-secrets-behaviour.ts'; import { ledgerLayer, sqliteSecretRows } from './sqlite3.ts'; import { ledgerBehaviour } from './testing/ledger-behaviour.ts'; @@ -22,6 +23,8 @@ const onSQLite: LedgerEntry = { ledgerLayer({ fileName, projections, ...(runOutcomes === undefined ? {} : { runOutcomes }) }), storeOn: (fileName) => sqliteEventStore(() => ({ driver: sqlite3EventStoreDriver, fileName })), queried, + executedOn, + layout: sqliteLayout, definitionStreamsIndexed, planOf, throughTheKindIndex: 'USING INDEX ledger_first_messages_by_kind', diff --git a/packages/ledger/src/postgresql/ledger-on-postgresql.test.ts b/packages/ledger/src/postgresql/ledger-on-postgresql.test.ts index 0bd4512aa..298ebe44a 100644 --- a/packages/ledger/src/postgresql/ledger-on-postgresql.test.ts +++ b/packages/ledger/src/postgresql/ledger-on-postgresql.test.ts @@ -1,9 +1,8 @@ -import { randomUUID } from 'node:crypto'; - import { Client } from 'pg'; import { describe, onTestFinished } from 'vitest'; -import type { RecordedStore } from '../event-store.ts'; +import { executorOn } from '../content/postgresql-content.ts'; +import { postgresqlLayout } from '../layout/layout-dialects.ts'; import { definitionStreamsPlan } from '../postgresql-reads/index-checks.ts'; import { postgresqlRecordedStore } from '../postgresql-reads/postgresql-recorded.ts'; import { keptSecretsBehaviour } from '../secrets/kept-secrets-behaviour.ts'; @@ -18,6 +17,16 @@ import { const server = process.env['LEDGER_TEST_POSTGRESQL_URL'] ?? ''; +const executedOn: LedgerEntry['executedOn'] = async (database, work) => { + const client = new Client({ connectionString: database }); + await client.connect(); + try { + return await work(executorOn(client)); + } finally { + await client.end(); + } +}; + async function queried(database: string, statement: string): Promise { const client = new Client({ connectionString: database }); await client.connect(); @@ -42,7 +51,7 @@ async function definitionStreamsIndexed(database: string): Promise { } } -async function planOf(database: string, read: (store: RecordedStore) => Promise): Promise { +const planOf: LedgerEntry['planOf'] = async (database, read) => { const asked: { readonly text: string; readonly values: readonly unknown[] }[] = []; await read( postgresqlRecordedStore((text, values) => { @@ -65,10 +74,10 @@ async function planOf(database: string, read: (store: RecordedStore) => Promise< } finally { await client.end(); } -} +}; async function aDatabase(): Promise { - const name = `ledger_${randomUUID().replaceAll('-', '')}`; + const name = `ledger_${crypto.randomUUID().replaceAll('-', '')}`; await queried(server, `CREATE DATABASE ${name}`); onTestFinished(async () => { await queried(server, `DROP DATABASE ${name} WITH (FORCE)`); @@ -99,6 +108,8 @@ const onPostgreSQL: LedgerEntry = { }, }), queried, + executedOn, + layout: postgresqlLayout, definitionStreamsIndexed, planOf, throughTheKindIndex: `Index Cond: (("substring"(stream_id, '^(?:[^/]*/){4}'::text) = ANY`, diff --git a/packages/ledger/src/postgresql/playbook-indexes-on-postgresql.test.ts b/packages/ledger/src/postgresql/playbook-indexes-on-postgresql.test.ts index c2a519be4..6abb7b829 100644 --- a/packages/ledger/src/postgresql/playbook-indexes-on-postgresql.test.ts +++ b/packages/ledger/src/postgresql/playbook-indexes-on-postgresql.test.ts @@ -84,7 +84,10 @@ describe.skipIf(skipped)(`The playbook's indexes on PostgreSQL${notice}`, () => await open.query('COMMIT'); expect( - await queried(database, "SELECT count(*)::int AS indexes FROM pg_indexes WHERE indexname LIKE 'ledger%'"), + await queried( + database, + "SELECT count(*)::int AS indexes FROM pg_indexes WHERE tablename IN ('emt_messages', 'emt_streams') AND indexname LIKE 'ledger%'", + ), ).toEqual([{ indexes: 7 }]); }); }); diff --git a/packages/ledger/src/postgresql/postgresql-event-store.ts b/packages/ledger/src/postgresql/postgresql-event-store.ts index bcdfb30d8..e5cbff58b 100644 --- a/packages/ledger/src/postgresql/postgresql-event-store.ts +++ b/packages/ledger/src/postgresql/postgresql-event-store.ts @@ -1,31 +1,14 @@ import { getPostgreSQLEventStore } from '@event-driven-io/emmett-postgresql'; import { pgEventStoreDriver } from '@event-driven-io/emmett-postgresql/pg'; -import type { RunOutcomeMapping, KeyedProjection } from '@on.auto/operations'; -import { Pool } from 'pg'; import { emmettEventStore } from '../emmett/emmett-event-store.ts'; -import type { AppendObserver, LedgerStore } from '../event-store.ts'; +import type { LedgerStore } from '../event-store.ts'; import { formattedFor, postgresqlRecordedStoreOn, type Query } from '../postgresql-reads/postgresql-recorded.ts'; import { dataAsJsonText } from './json-text.ts'; +import type { PostgreSQLOptions } from './postgresql-options.ts'; import { postgresqlProjectionsOf } from './postgresql-projections.ts'; -import { migrationLockTakenWithin, postgresqlRunOutcomesOf } from './postgresql-run-outcomes.ts'; - -export interface PoolBounds { - readonly max: number; -} - -export type PostgreSQLPools = (bounds: PoolBounds) => Pool; - -export function plainPoolsOn(connectionString: string): PostgreSQLPools { - return ({ max }) => new Pool({ connectionString, max }); -} - -export interface PostgreSQLOptions { - readonly pools: PostgreSQLPools; - readonly runOutcomes?: RunOutcomeMapping; - readonly projections?: readonly KeyedProjection[]; - readonly observer?: AppendObserver | undefined; -} +import { postgresqlRunOutcomesOf } from './postgresql-run-outcomes.ts'; +import { beforeTheSchema } from './postgresql-schema.ts'; export interface PostgreSQLStoreOptions extends PostgreSQLOptions { readonly reportLostConnection: (error: Readonly) => void; @@ -51,7 +34,7 @@ export function postgresqlEventStore({ connectionOptions: { pool }, schema: { autoMigration: 'None' }, projections: [...kept.registrations], - hooks: { onBeforeSchemaCreated: migrationLockTakenWithin(), onAfterSchemaCreated: kept.afterTheSchema }, + hooks: { onBeforeSchemaCreated: beforeTheSchema, onAfterSchemaCreated: kept.afterTheSchema }, }), { data: dataAsJsonText, mostEventsInOneAppend: eventsInOneBoundedAppend, observer }, ); diff --git a/packages/ledger/src/postgresql/postgresql-ledger.ts b/packages/ledger/src/postgresql/postgresql-ledger.ts index 8224fc09b..a97a9a3dc 100644 --- a/packages/ledger/src/postgresql/postgresql-ledger.ts +++ b/packages/ledger/src/postgresql/postgresql-ledger.ts @@ -4,16 +4,11 @@ import { Effect, Layer } from 'effect'; import type { AppendSignal } from '../appends/append-signal.ts'; import { ledgerLayerOver } from '../ledger-layer.ts'; import { lostConnectionsLoggedWith } from './lost-connections.ts'; -import { postgresqlEventStore, type PostgreSQLOptions } from './postgresql-event-store.ts'; +import { postgresqlEventStore } from './postgresql-event-store.ts'; +import type { PostgreSQLOptions } from './postgresql-options.ts'; -export { - plainPoolsOn, - postgresqlEventStore, - type PoolBounds, - type PostgreSQLOptions, - type PostgreSQLPools, - type PostgreSQLStoreOptions, -} from './postgresql-event-store.ts'; +export { postgresqlEventStore, type PostgreSQLStoreOptions } from './postgresql-event-store.ts'; +export { plainPoolsOn, type PoolBounds, type PostgreSQLOptions, type PostgreSQLPools } from './postgresql-options.ts'; export { postgresqlSecretRows, type PostgreSQLSecretRowsOptions, diff --git a/packages/ledger/src/postgresql/postgresql-options.ts b/packages/ledger/src/postgresql/postgresql-options.ts new file mode 100644 index 000000000..b8d013194 --- /dev/null +++ b/packages/ledger/src/postgresql/postgresql-options.ts @@ -0,0 +1,21 @@ +import type { KeyedProjection, RunOutcomeMapping } from '@on.auto/operations'; +import { Pool } from 'pg'; + +import type { AppendObserver } from '../event-store.ts'; + +export interface PoolBounds { + readonly max: number; +} + +export type PostgreSQLPools = (bounds: PoolBounds) => Pool; + +export function plainPoolsOn(connectionString: string): PostgreSQLPools { + return ({ max }) => new Pool({ connectionString, max }); +} + +export interface PostgreSQLOptions { + readonly pools: PostgreSQLPools; + readonly runOutcomes?: RunOutcomeMapping; + readonly projections?: readonly KeyedProjection[]; + readonly observer?: AppendObserver | undefined; +} diff --git a/packages/ledger/src/postgresql/postgresql-run-outcomes.test.ts b/packages/ledger/src/postgresql/postgresql-run-outcomes.test.ts index e2afca378..11f917366 100644 --- a/packages/ledger/src/postgresql/postgresql-run-outcomes.test.ts +++ b/packages/ledger/src/postgresql/postgresql-run-outcomes.test.ts @@ -3,10 +3,10 @@ import { pgFormatter } from '@event-driven-io/dumbo/pg'; import { runTallies, type RunFact } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; +import { contentTableNames } from '../content/content-tables.ts'; import type { StatementExecutor } from '../event-store.ts'; import { aRecordedFillOf, mebibyte, runIdsOf, type RecordedFill } from '../outcomes/recorded-fill.ts'; import type { Query } from '../postgresql-reads/postgresql-recorded.ts'; -import { columnsOrNames } from '../secrets/kept-table-columns.ts'; import { postgresqlProjectionsOf } from './postgresql-projections.ts'; import { emmettsMigrationLock, @@ -56,22 +56,20 @@ const indexes = [ 'ledger_messages_by_id', ].map((name) => ({ name })); -const contentIndexesAndSecrets = ['recorded_content_heads_by_upload', 'staged_content_by_upload', 'kept_secrets']; - -function contentAnswering(statement: string): readonly { readonly name: string }[] { - return columnsOrNames(statement); -} +const contentAndSecrets = [ + ...contentTableNames, + 'recorded_content_heads_by_upload', + 'staged_content_by_upload', + 'kept_secrets', +].map((name) => ({ name })); function aLedgerWithOneRun(tables: readonly string[]): Answers { return (statement) => { if (statement.includes('ledger_messages_by_playbook')) { return indexes; } - if (statement.includes('recorded_content_chunks') || statement.includes('pg_attribute')) { - return contentAnswering(statement); - } - if (/_by_upload|kept_secrets/u.test(statement)) { - return contentIndexesAndSecrets.map((name) => ({ name })); + if (/recorded_content_chunks|_by_upload|kept_secrets/u.test(statement)) { + return contentAndSecrets; } if (statement.includes('pg_class')) { return tables.map((name) => ({ name })); diff --git a/packages/ledger/src/postgresql/postgresql-schema.test.ts b/packages/ledger/src/postgresql/postgresql-schema.test.ts index 7c151be00..e1d5c2e51 100644 --- a/packages/ledger/src/postgresql/postgresql-schema.test.ts +++ b/packages/ledger/src/postgresql/postgresql-schema.test.ts @@ -2,10 +2,15 @@ import { SQL } from '@event-driven-io/dumbo'; import { pgFormatter } from '@event-driven-io/dumbo/pg'; import { describe, expect, it } from 'vitest'; -import { contentTables } from '../content/content-tables.ts'; +import { contentTableNames } from '../content/content-tables.ts'; import type { StatementExecutor } from '../event-store.ts'; -import { columnsOrNames } from '../secrets/kept-table-columns.ts'; -import { postgresqlSchemaCreated } from './postgresql-schema.ts'; +import { ledgerLayout } from '../layout/ledger-layout.ts'; +import { emmettsMigrationLock } from './postgresql-run-outcomes.ts'; +import { beforeTheSchema, postgresqlSchemaCreated } from './postgresql-schema.ts'; + +function described(sql: SQL): string { + return SQL.describe(sql, pgFormatter).replaceAll(/\s+/gu, ' ').trim(); +} function executorFinding(names: readonly string[]): { readonly execute: StatementExecutor; @@ -15,9 +20,9 @@ function executorFinding(names: readonly string[]): { return { commands, execute: { - query: (sql) => Promise.resolve({ rows: columnsOrNames(SQL.describe(sql, pgFormatter), names) }), + query: () => Promise.resolve({ rows: names.map((name) => ({ name })) }), command: (sql) => { - commands.push(SQL.describe(sql, pgFormatter).replaceAll(/\s+/gu, ' ').split(' (')[0] ?? ''); + commands.push(described(sql).split(' (')[0] ?? ''); return Promise.resolve(); }, }, @@ -43,7 +48,7 @@ describe("the ledger's own schema on PostgreSQL", () => { it("creates the playbook's indexes, the tables of the content it keeps and their indexes, then the table of its kept secrets, each only when missing", async () => { const empty = executorFinding([]); const found = executorFinding([ - ...contentTables.map(({ name }) => name), + ...contentTableNames, 'recorded_content_heads_by_upload', 'staged_content_by_upload', 'kept_secrets', @@ -63,3 +68,61 @@ describe("the ledger's own schema on PostgreSQL", () => { expect(found.commands.filter((command) => /content|kept_secrets/u.test(command))).toEqual([]); }); }); + +interface Held { + readonly tables: number; + readonly layout?: number; +} + +function answerTo(statement: string, { tables, layout }: Held): unknown { + if (statement.includes('pg_try_advisory_xact_lock')) { + return { locked: true }; + } + return statement.includes('pg_tables') ? { tables, numbered: layout !== undefined } : { layout }; +} + +function aDatabaseHolding(held: Held): { readonly execute: StatementExecutor; readonly asked: string[] } { + const asked: string[] = []; + return { + asked, + execute: { + query: (sql) => { + const statement = described(sql); + asked.push(statement); + return Promise.resolve({ rows: [answerTo(statement, held)] }); + }, + command: (sql) => { + asked.push(described(sql)); + return Promise.resolve(); + }, + }, + }; +} + +const lockTaken = `SELECT pg_try_advisory_xact_lock(${emmettsMigrationLock}) AS locked`; + +const tablesCounted = + 'SELECT count(*)::int AS tables, count(*) FILTER (WHERE tablename = "ledger_layout") > 0 AS numbered FROM pg_tables WHERE schemaname = current_schema()'; + +describe('the layout of the ledger on PostgreSQL, as the migration begins', () => { + it('takes the migration lock, then counts the tables of the schema, and writes the layout into a table of one row before any other table of a database that holds none', async () => { + const { execute, asked } = aDatabaseHolding({ tables: 0 }); + + await beforeTheSchema({ execute }); + + expect(asked).toEqual([ + lockTaken, + tablesCounted, + 'CREATE TABLE ledger_layout ( one_row BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (one_row), layout INTEGER NOT NULL )', + `INSERT INTO ledger_layout (layout) VALUES (${ledgerLayout})`, + ]); + }); + + it('reads the layout of a database that holds its table, and writes nothing when it is the layout kept', async () => { + const { execute, asked } = aDatabaseHolding({ tables: 12, layout: ledgerLayout }); + + await beforeTheSchema({ execute }); + + expect(asked).toEqual([lockTaken, tablesCounted, 'SELECT layout FROM ledger_layout']); + }); +}); diff --git a/packages/ledger/src/postgresql/postgresql-schema.ts b/packages/ledger/src/postgresql/postgresql-schema.ts index f24b8a5a5..3143b59b0 100644 --- a/packages/ledger/src/postgresql/postgresql-schema.ts +++ b/packages/ledger/src/postgresql/postgresql-schema.ts @@ -1,8 +1,20 @@ import { postgresqlContent } from '../content/content-dialects.ts'; import { createdContentTables } from '../content/content-tables.ts'; import type { StatementExecutor } from '../event-store.ts'; +import { postgresqlLayout } from '../layout/layout-dialects.ts'; +import { requireTheLayoutKept } from '../layout/layout-kept.ts'; import { createPostgreSQLPlaybookIndexes } from '../postgresql-reads/playbook-indexes.ts'; import { createdSecretTable } from '../secrets/kept-secrets.ts'; +import { migrationLockTakenWithin } from './postgresql-run-outcomes.ts'; + +interface Migration { + readonly execute: StatementExecutor; +} + +export async function beforeTheSchema(migration: Migration): Promise { + await migrationLockTakenWithin()(migration); + await requireTheLayoutKept(migration.execute, postgresqlLayout); +} export async function postgresqlSchemaCreated(execute: StatementExecutor): Promise { await createPostgreSQLPlaybookIndexes({ execute }); diff --git a/packages/ledger/src/projections/projection-table-behaviour.ts b/packages/ledger/src/projections/projection-table-behaviour.ts index 55d2305cd..bb1ab4ecc 100644 --- a/packages/ledger/src/projections/projection-table-behaviour.ts +++ b/packages/ledger/src/projections/projection-table-behaviour.ts @@ -11,6 +11,7 @@ import { import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; +import { runOutcomeTableBehaviour } from '../outcomes/run-outcome-table-behaviour.ts'; import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; import { projectionsBehaviour, topicsOf } from './projections-behaviour.ts'; @@ -158,6 +159,7 @@ function aKeyedTableNotThereYet(entry: LedgerEntry): void { } export function projectionTableBehaviour(entry: LedgerEntry): void { + runOutcomeTableBehaviour(entry); projectionsBehaviour((projections) => aLedger(entry, undefined, undefined, projections)); anAppendThatBreaksDown(entry); aTableNotThereYet(entry); diff --git a/packages/ledger/src/recorded/sqlite-checks.ts b/packages/ledger/src/recorded/sqlite-checks.ts index fa3eb605a..b58a2314d 100644 --- a/packages/ledger/src/recorded/sqlite-checks.ts +++ b/packages/ledger/src/recorded/sqlite-checks.ts @@ -1,46 +1,43 @@ -import { SQL, dumbo } from '@event-driven-io/dumbo'; +import { SQL, dumbo, type SQLExecutor } from '@event-driven-io/dumbo'; import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3'; import { definitionStreamsQuery } from '../definitions/sqlite-definition-streams.ts'; import type { RecordedStore } from '../event-store.ts'; import { sqliteRecordedStore } from './sqlite-recorded.ts'; -export async function queried(fileName: string, statement: string): Promise { +export async function executedOn(fileName: string, work: (execute: SQLExecutor) => Promise): Promise { const pool = dumbo(sqlite3EventStoreDriver.mapToDumboOptions({ fileName })); try { - const { rows } = await pool.execute.query(SQL`${SQL.plain(statement)}`); - return rows; + return await work(pool.execute); } finally { await pool.close(); } } -export async function definitionStreamsIndexed(fileName: string): Promise { - const pool = dumbo(sqlite3EventStoreDriver.mapToDumboOptions({ fileName })); - try { - const { rows } = await pool.execute.query(SQL`EXPLAIN QUERY PLAN ${definitionStreamsQuery('recall')}`); +export function queried(fileName: string, statement: string): Promise { + return executedOn(fileName, async (execute) => (await execute.query(SQL`${SQL.plain(statement)}`)).rows); +} + +export function definitionStreamsIndexed(fileName: string): Promise { + return executedOn(fileName, async (execute) => { + const { rows } = await execute.query(SQL`EXPLAIN QUERY PLAN ${definitionStreamsQuery('recall')}`); return JSON.stringify(rows).includes('USING INDEX ledger_definition_streams'); - } finally { - await pool.close(); - } + }); } -export async function planOf(fileName: string, read: (store: RecordedStore) => Promise): Promise { +export function planOf(fileName: string, read: (store: RecordedStore) => Promise): Promise { const asked: SQL[] = []; - const pool = dumbo(sqlite3EventStoreDriver.mapToDumboOptions({ fileName })); - try { + return executedOn(fileName, async (execute) => { await read( sqliteRecordedStore({ - ...pool.execute, + ...execute, query: (sql) => { asked.push(sql); - return pool.execute.query(sql); + return execute.query(sql); }, }), ); - const plans = await pool.execute.batchQuery(asked.map((sql) => SQL`EXPLAIN QUERY PLAN ${sql}`)); + const plans = await execute.batchQuery(asked.map((sql) => SQL`EXPLAIN QUERY PLAN ${sql}`)); return JSON.stringify(plans); - } finally { - await pool.close(); - } + }); } diff --git a/packages/ledger/src/recorded/sqlite-indexes.ts b/packages/ledger/src/recorded/sqlite-indexes.ts index 300b25dd2..6dd1059ba 100644 --- a/packages/ledger/src/recorded/sqlite-indexes.ts +++ b/packages/ledger/src/recorded/sqlite-indexes.ts @@ -1,9 +1,6 @@ import { SQL } from '@event-driven-io/dumbo'; import { Schema } from 'effect'; -import { sqliteContent } from '../content/content-dialects.ts'; -import { createdContentTables } from '../content/content-tables.ts'; -import { createdSecretTable } from '../secrets/kept-secrets.ts'; import { createMissingIndexes, type PlaybookIndex, type IndexExecutor } from './missing-indexes.ts'; function prefixThroughSlashes(count: number): string { @@ -75,9 +72,3 @@ export async function createSQLitePlaybookIndexes(execute: IndexExecutor): Promi const existing = new Set(Schema.decodeUnknownSync(NameRows)(rows).map(({ name }) => name)); await createMissingIndexes(execute, playbookIndexes, existing); } - -export async function sqliteSchemaCreated(execute: IndexExecutor): Promise { - await createSQLitePlaybookIndexes(execute); - await createdContentTables(execute, sqliteContent); - await createdSecretTable(execute, sqliteContent); -} diff --git a/packages/ledger/src/secrets/kept-secrets-behaviour.ts b/packages/ledger/src/secrets/kept-secrets-behaviour.ts index 14179b592..e1b8e29d0 100644 --- a/packages/ledger/src/secrets/kept-secrets-behaviour.ts +++ b/packages/ledger/src/secrets/kept-secrets-behaviour.ts @@ -1,9 +1,10 @@ import type { SecretRows } from '@on.auto/secrets'; import { rowsBehaviour, storeBehaviour, type SecretsEntry } from '@on.auto/secrets/testing'; -import type { Effect } from 'effect'; -import { describe, expect, it } from 'vitest'; +import { Effect } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; -import { LedgerShapeRefused } from '../content/ledger-shape-refused.ts'; +import { layoutIn } from '../layout/ledger-layout-behaviour.ts'; +import { ledgerLayout } from '../layout/ledger-layout.ts'; import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; export interface KeptSecretsEntry extends LedgerEntry { @@ -25,25 +26,16 @@ function secretsEntryOf(entry: KeptSecretsEntry): SecretsEntry { }; } -function olderShapeBehaviour(entry: KeptSecretsEntry): void { - it('refuses a database whose table of secrets has an older shape when the ledger opens, naming the columns found and kept, with words that say to delete it', async () => { +function openedOnTheLedgersDatabase(entry: KeptSecretsEntry): void { + it('are read on connections of their own, opened on the database the ledger made in its layout, which they leave in it', async () => { const database = await entry.aDatabase(); - await entry.queried( - database, - 'CREATE TABLE kept_secrets (id TEXT PRIMARY KEY, version INTEGER NOT NULL, kept_at TEXT NOT NULL, kept_by TEXT NOT NULL, held TEXT NOT NULL)', - ); - - const refused = await aLedger(entry, database).catch((error: unknown) => error); - - expect(refused).toBeInstanceOf(LedgerShapeRefused); - expect(refused).toMatchObject({ - table: 'kept_secrets', - holds: 'secrets', - found: ['held', 'id', 'kept_at', 'kept_by', 'version'], - kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], - message: - "The ledger's database keeps secrets in a shape this server no longer reads: kept_secrets has the columns held, id, kept_at, kept_by, version, where this server keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Nothing stored before launch is carried over, so delete the local database and start the server again", - }); + await aLedger(entry, database); + + const rows = await Effect.runPromise(entry.secretRowsOn(database)); + onTestFinished(() => rows.close()); + + expect(await Effect.runPromise(rows.read('0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'))).toBeUndefined(); + expect((await layoutIn(entry, database)).layout).toBe(ledgerLayout); }); } @@ -51,6 +43,6 @@ export function keptSecretsBehaviour(entry: KeptSecretsEntry): void { describe('the secrets it keeps', () => { rowsBehaviour(secretsEntryOf(entry)); storeBehaviour(secretsEntryOf(entry)); - olderShapeBehaviour(entry); + openedOnTheLedgersDatabase(entry); }); } diff --git a/packages/ledger/src/secrets/kept-secrets.ts b/packages/ledger/src/secrets/kept-secrets.ts index 927a0d3a9..813bc60a0 100644 --- a/packages/ledger/src/secrets/kept-secrets.ts +++ b/packages/ledger/src/secrets/kept-secrets.ts @@ -1,20 +1,14 @@ import { SQL } from '@event-driven-io/dumbo'; import type { ContentDialect } from '../content/content-dialects.ts'; -import { namesFound, requireTheShapeKept, type KeptTable } from '../content/content-tables.ts'; +import { namesFound } from '../content/content-tables.ts'; import type { StatementExecutor } from '../event-store.ts'; import { createMissingIndexes, type PlaybookIndex } from '../recorded/missing-indexes.ts'; export const keptSecrets = SQL.identifier('kept_secrets'); -export const keptSecretsShape: KeptTable = { - name: 'kept_secrets', - columns: ['id', 'version', 'kept_at', 'claimed_until', 'ends_with', 'kept_by', 'held', 'wrapped_key', 'kid'], - holds: 'secrets', -}; - const keptSecretsTable: PlaybookIndex = { - name: keptSecretsShape.name, + name: 'kept_secrets', create: () => SQL`CREATE TABLE IF NOT EXISTS ${keptSecrets} ( id TEXT PRIMARY KEY, version INTEGER NOT NULL, @@ -29,9 +23,6 @@ const keptSecretsTable: PlaybookIndex = { }; export async function createdSecretTable(execute: StatementExecutor, dialect: ContentDialect): Promise { - const existing = new Set(await namesFound(execute, dialect.relationsAmong('table', [keptSecretsShape.name]))); - if (existing.has(keptSecretsShape.name)) { - await requireTheShapeKept(execute, dialect, keptSecretsShape); - } + const existing = new Set(await namesFound(execute, dialect.relationsAmong('table', [keptSecretsTable.name]))); await createMissingIndexes(execute, [keptSecretsTable], existing); } diff --git a/packages/ledger/src/secrets/kept-table-columns.ts b/packages/ledger/src/secrets/kept-table-columns.ts deleted file mode 100644 index a15d650c8..000000000 --- a/packages/ledger/src/secrets/kept-table-columns.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { contentTableNames, contentTables } from '../content/content-tables.ts'; -import { keptSecretsShape } from './kept-secrets.ts'; - -export function columnsOrNames( - statement: string, - names: readonly string[] = contentTableNames, -): readonly { readonly name: string }[] { - const table = [...contentTables, keptSecretsShape].find(({ name }) => statement.includes(`to_regclass("${name}")`)); - return (table === undefined ? names : table.columns).map((name) => ({ name })); -} diff --git a/packages/ledger/src/secrets/postgresql-secret-rows.ts b/packages/ledger/src/secrets/postgresql-secret-rows.ts index 078f643de..e9fd7fba5 100644 --- a/packages/ledger/src/secrets/postgresql-secret-rows.ts +++ b/packages/ledger/src/secrets/postgresql-secret-rows.ts @@ -3,7 +3,7 @@ import { Effect } from 'effect'; import { executorOn } from '../content/postgresql-content.ts'; import { lostConnectionsLoggedWith } from '../postgresql/lost-connections.ts'; -import type { PoolBounds } from '../postgresql/postgresql-event-store.ts'; +import type { PoolBounds } from '../postgresql/postgresql-options.ts'; import { secretRowsOn } from './secret-rows.ts'; export interface SecretConnections { diff --git a/packages/ledger/src/sqlite-event-store.ts b/packages/ledger/src/sqlite-event-store.ts index 5b5f8af42..d3d39cda6 100644 --- a/packages/ledger/src/sqlite-event-store.ts +++ b/packages/ledger/src/sqlite-event-store.ts @@ -3,11 +3,11 @@ import { getSQLiteEventStore } from '@event-driven-io/emmett-sqlite'; import { dataAsWritten, emmettEventStore } from './emmett/emmett-event-store.ts'; import type { AppendObserver, LedgerStore } from './event-store.ts'; +import { sqliteLayoutKept, sqliteSchemaCreated } from './layout/sqlite-schema.ts'; import { ledgerLayerOver, type StoreLayerOptions } from './ledger-layer.ts'; import { sqliteRunOutcomesOf } from './outcomes/sqlite-run-outcomes.ts'; import type { KeptTables } from './projections/projection-parts.ts'; import { preparedOn, sqliteProjectionsOf } from './projections/sqlite-projections.ts'; -import { sqliteSchemaCreated } from './recorded/sqlite-indexes.ts'; import { sqliteRecordedStoreOn } from './recorded/sqlite-recorded.ts'; export type AnyDriver = Parameters[0]['driver']; @@ -45,6 +45,7 @@ export function sqliteEventStore( }), readRunOutcomes: sqliteRunOutcomesOf(kept.runOutcomes, pool.execute), migrate: async () => { + await sqliteLayoutKept(pool.execute); await streams.migrate(); await sqliteSchemaCreated(pool.execute); await preparedOn(projections.prepare, pool); diff --git a/packages/ledger/src/testing/index.ts b/packages/ledger/src/testing/index.ts index 2af5dfb41..d438b6132 100644 --- a/packages/ledger/src/testing/index.ts +++ b/packages/ledger/src/testing/index.ts @@ -1 +1,2 @@ +export { ledgerLayout } from '../layout/ledger-layout.ts'; export { unreachableDatabase, type UnreachableDatabase } from './unreachable-database.ts'; diff --git a/packages/ledger/src/testing/ledger-behaviour.ts b/packages/ledger/src/testing/ledger-behaviour.ts index 7bdab9148..bbe5b33e6 100644 --- a/packages/ledger/src/testing/ledger-behaviour.ts +++ b/packages/ledger/src/testing/ledger-behaviour.ts @@ -1,7 +1,7 @@ import { contentBehaviour } from '../content/content-behaviour.ts'; import { storedContextBehaviour } from '../context/stored-context-behaviour.ts'; import { definitionStreamsBehaviour } from '../definitions/definition-streams-behaviour.ts'; -import { runOutcomeTableBehaviour } from '../outcomes/run-outcome-table-behaviour.ts'; +import { ledgerLayoutBehaviour } from '../layout/ledger-layout-behaviour.ts'; import { projectionTableBehaviour } from '../projections/projection-table-behaviour.ts'; import { commandsBehaviour } from './commands-behaviour.ts'; import { eventsBehaviour } from './events-behaviour.ts'; @@ -13,9 +13,9 @@ export function ledgerBehaviour(entry: LedgerEntry): void { commandsBehaviour(entry); eventsBehaviour(entry); storeBehaviour(entry); + ledgerLayoutBehaviour(entry); definitionStreamsBehaviour(entry); recordedBehaviour(() => aLedger(entry)); - runOutcomeTableBehaviour(entry); projectionTableBehaviour(entry); storedContextBehaviour(entry); contentBehaviour(entry); diff --git a/packages/ledger/src/testing/ledger-entry.ts b/packages/ledger/src/testing/ledger-entry.ts index d6b85f45d..e8b76dc18 100644 --- a/packages/ledger/src/testing/ledger-entry.ts +++ b/packages/ledger/src/testing/ledger-entry.ts @@ -2,7 +2,8 @@ import type { Ledger, RunOutcomeMapping, KeyedProjection } from '@on.auto/operat import type { Layer } from 'effect'; import { onTestFinished } from 'vitest'; -import type { LedgerStore, RecordedStore } from '../event-store.ts'; +import type { LedgerStore, RecordedStore, StatementExecutor } from '../event-store.ts'; +import type { LayoutDialect } from '../layout/layout-dialects.ts'; import { openLedgerWith } from './open-ledger.ts'; export interface LedgerEntry { @@ -17,6 +18,8 @@ export interface LedgerEntry { ) => Layer.Layer; readonly storeOn: (database: string) => LedgerStore; readonly queried: (database: string, statement: string) => Promise; + readonly executedOn: (database: string, work: (execute: StatementExecutor) => Promise) => Promise; + readonly layout: LayoutDialect; readonly definitionStreamsIndexed: (database: string) => Promise; readonly planOf: (database: string, read: (store: RecordedStore) => Promise) => Promise; readonly throughTheKindIndex: string; diff --git a/packages/server/src/command-line/failure-notice.test.ts b/packages/server/src/command-line/failure-notice.test.ts index 3da5da6c6..9b39f6730 100644 --- a/packages/server/src/command-line/failure-notice.test.ts +++ b/packages/server/src/command-line/failure-notice.test.ts @@ -7,14 +7,9 @@ import { ledgerOfAnotherVersion } from '../database/ledger-of-another-version.ts import type { ColorStream } from '../logging/terminal.ts'; import { startupFailureNoticeFor } from './failure-notice.ts'; -const secretsOfAnOlderShape = { - table: 'kept_secrets', - holds: 'secrets', - found: ['held', 'id', 'kept_at', 'kept_by', 'version'], - kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], -}; +const inAnotherLayout = { found: 2, kept: 1 }; -const ledgerFileOfAnotherVersion = ledgerOfAnotherVersion(secretsOfAnOlderShape, { +const ledgerFileOfAnotherVersion = ledgerOfAnotherVersion(inAnotherLayout, { ledger: { store: 'sqlite', file: '/Users/ada/.auto-runtime/ledger.db' }, distribution: 'npx', }); @@ -26,13 +21,13 @@ const notATerminal: ColorStream = {}; const aTerminalWithColors: ColorStream = { isTTY: true, getColorDepth: () => 24 }; describe('the notice that a server could not start on a ledger file of another version', () => { - it('says where the ledger is and why it is refused, what to do with the command that does it, where to ask, and the columns last', () => { + it('says where the ledger is and why it is refused, what to do with the command that does it, and where to ask, last', () => { expect(startupFailureNoticeFor(notATerminal)(servicesFailed)?.split('\n')).toEqual([ ' ✖ Auto Runtime could not start', '', ' The ledger is kept in the file /Users/ada/.auto-runtime/ledger.db, and was', - ' made by another version of Auto Runtime. This version keeps secrets in a', - ' different shape and cannot read the secrets this ledger holds.', + ' made by another version of Auto Runtime. This version keeps the ledger in', + ' layout 1 and cannot read a ledger in layout 2.', '', ' What you can do', ' Delete the ledger and start Auto Runtime again. Everything in the ledger', @@ -41,14 +36,10 @@ describe('the notice that a server could not start on a ledger file of another v ' rm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm', '', ' Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', - '', - ' kept_secrets has the columns held, id, kept_at, kept_by, version, where', - ' this version keeps claimed_until, ends_with, held, id, kept_at, kept_by,', - ' kid, version, wrapped_key.', ]); }); - it('marks the failure in red, sets the command apart in cyan, links Discord and dims the columns where stderr is a terminal with colors', () => { + it('marks the failure in red, sets the command apart in cyan and links Discord where stderr is a terminal with colors', () => { const notice = String(startupFailureNoticeFor(aTerminalWithColors)(ledgerFileOfAnotherVersion)); expect(notice).toContain('\u001B[31m✖\u001B[39m'); @@ -56,7 +47,6 @@ describe('the notice that a server could not start on a ledger file of another v '\u001B[36mrm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm\u001B[39m', ); expect(notice).toContain('\u001B]8;;https://discord.gg/B8BKcKMRm8\u001B\\'); - expect(notice).toContain('\u001B[2m kid, version, wrapped_key.\u001B[22m'); expect(stripVTControlCharacters(notice)).toBe(startupFailureNoticeFor(notATerminal)(servicesFailed)); }); @@ -70,7 +60,7 @@ describe('the notice that a server could not start on a ledger file of another v describe('the notice that a server in the image could not start on a ledger file of another version', () => { it('says which files to delete from the volume, or to remove the volume, with no command', () => { - const inTheImage = ledgerOfAnotherVersion(secretsOfAnOlderShape, { + const inTheImage = ledgerOfAnotherVersion(inAnotherLayout, { ledger: { store: 'sqlite', file: '/app/data/ledger.db' }, distribution: 'docker', }); @@ -89,7 +79,7 @@ describe('the notice that a server in the image could not start on a ledger file describe('the notice that a server could not start on a ledger in PostgreSQL of another version', () => { it('says to point DATABASE_URL at an empty database, with no command', () => { - const inPostgreSQL = ledgerOfAnotherVersion(secretsOfAnOlderShape, { + const inPostgreSQL = ledgerOfAnotherVersion(inAnotherLayout, { ledger: { store: 'postgresql', url: Redacted.make('postgresql://playbooks:a-secret-password@db.example.com/auto_runtime'), @@ -100,11 +90,10 @@ describe('the notice that a server could not start on a ledger in PostgreSQL of distribution: 'npx', }); - expect(startupFailureNoticeFor(notATerminal)(inPostgreSQL)?.split('\n').slice(2, 15)).toEqual([ + expect(startupFailureNoticeFor(notATerminal)(inPostgreSQL)?.split('\n').slice(2)).toEqual([ ' The ledger is kept in PostgreSQL, in the database auto_runtime on', ' db.example.com:5432, and was made by another version of Auto Runtime. This', - ' version keeps secrets in a different shape and cannot read the secrets', - ' this ledger holds.', + ' version keeps the ledger in layout 1 and cannot read a ledger in layout 2.', '', ' What you can do', ' Point DATABASE_URL at an empty database, or empty this one, and start', @@ -112,8 +101,6 @@ describe('the notice that a server could not start on a ledger in PostgreSQL of ' definitions or runs.', '', ' Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', - '', - ' kept_secrets has the columns held, id, kept_at, kept_by, version, where', ]); }); }); diff --git a/packages/server/src/command-line/failure-notice.ts b/packages/server/src/command-line/failure-notice.ts index 4cf504461..1110a6e8e 100644 --- a/packages/server/src/command-line/failure-notice.ts +++ b/packages/server/src/command-line/failure-notice.ts @@ -1,5 +1,4 @@ import { - columnsCompared, helpOfferedAt, ledgerOfAnotherVersionIn, type LedgerOfAnotherVersion, @@ -25,7 +24,7 @@ function wrapped(text: string, indent: string): readonly string[] { return lines.map((line) => `${indent}${line}`); } -type Refusal = Pick; +type Refusal = Pick; function ledgerNotice(refusal: Refusal, depth: ColorDepth): string { const paint = painterFor(depth); @@ -42,8 +41,6 @@ function ledgerNotice(refusal: Refusal, depth: ColorDepth): string { ...shown, '', ` ${helpOfferedAt(paint('cyan', link(discordInvite, discordInvite)))}`, - '', - ...wrapped(columnsCompared(refusal), ' ').map((line) => paint('dim', line)), ].join('\n'); } diff --git a/packages/server/src/composition/composition-root.ts b/packages/server/src/composition/composition-root.ts index 09282928e..7d13bd6ab 100644 --- a/packages/server/src/composition/composition-root.ts +++ b/packages/server/src/composition/composition-root.ts @@ -1,7 +1,7 @@ import type { DispatcherServices } from '@on.auto/operations'; import { storesAndConnectionsOf } from '../connections/served-connections.ts'; -import { ledgerDatabaseOf, withARefusedShapeExplained } from '../database/ledger-database.ts'; +import { ledgerDatabaseOf, withARefusedLayoutExplained } from '../database/ledger-database.ts'; import { defaultServerOptions, type ServerOptions } from '../lifecycle/lifecycle.ts'; import { serveWorkflows, type WorkflowParts } from '../workflows/workflows.ts'; import { applicationLayer } from './application-layer.ts'; @@ -23,7 +23,7 @@ export function compositionRootWith( const database = ledgerDatabaseOf(settings, telemetry.appended); return { services: applicationLayer( - withARefusedShapeExplained(ledgerLayerFor(settings, database, wiring.projections.appends), settings), + withARefusedLayoutExplained(ledgerLayerFor(settings, database, wiring.projections.appends), settings), ), serve: async (runtime, ownOrigin) => { const { ledger, workflows, identityIssuer } = settings; diff --git a/packages/server/src/database/ledger-database.test.ts b/packages/server/src/database/ledger-database.test.ts index 034b910ca..b7b1ba90a 100644 --- a/packages/server/src/database/ledger-database.test.ts +++ b/packages/server/src/database/ledger-database.test.ts @@ -1,11 +1,11 @@ -import { LedgerShapeRefused } from '@on.auto/ledger'; +import { LedgerLayoutRefused } from '@on.auto/ledger'; import { Ledger } from '@on.auto/operations'; import { Cause, Effect, Exit, Layer } from 'effect'; import { describe, expect, it } from 'vitest'; import { readSettings } from '../settings/settings.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; -import { ledgerDatabaseOf, withARefusedShapeExplained } from './ledger-database.ts'; +import { ledgerDatabaseOf, withARefusedLayoutExplained } from './ledger-database.ts'; import { LedgerOfAnotherVersion } from './ledger-of-another-version.ts'; const somePools: unknown = expect.any(Function); @@ -30,12 +30,10 @@ describe('the database of the ledger and the workflow host', () => { }); }); -const refusedShape = new LedgerShapeRefused({ - table: 'kept_secrets', - holds: 'secrets', - found: ['held', 'id', 'kept_at', 'kept_by', 'version'], - kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], - message: "The ledger's database keeps secrets in a shape this server no longer reads", +const refusedLayout = new LedgerLayoutRefused({ + found: 2, + kept: 1, + message: "The ledger's database is in layout 2 of the ledger, where this server keeps layout 1", }); async function builtFrom(ledgerLayer: Layer.Layer): Promise { @@ -43,28 +41,27 @@ async function builtFrom(ledgerLayer: Layer.Layer): Promise { return Exit.isFailure(exit) ? Cause.squash(exit.cause) : 'built'; } -describe('a ledger that refuses the shape of its database as it opens', () => { +describe('a ledger that refuses the layout of its database as it opens', () => { it('stops the start with words that say where the ledger is kept, from the settings', async () => { const refused = await builtFrom( - withARefusedShapeExplained( - Layer.effect(Ledger, Effect.die(refusedShape)), + withARefusedLayoutExplained( + Layer.effect(Ledger, Effect.die(refusedLayout)), readSettings({ LEDGER_FILE: '/data/ledger.db' }), ), ); expect(refused).toBeInstanceOf(LedgerOfAnotherVersion); expect(refused).toMatchObject({ - table: 'kept_secrets', - found: refusedShape.found, - kept: refusedShape.kept, + found: 2, + kept: 1, remedy: { command: 'rm -f /data/ledger.db /data/ledger.db-wal /data/ledger.db-shm' }, }); }); it('says in the image which files to delete from the volume, from the distribution the settings name', async () => { const refused = await builtFrom( - withARefusedShapeExplained( - Layer.effect(Ledger, Effect.die(refusedShape)), + withARefusedLayoutExplained( + Layer.effect(Ledger, Effect.die(refusedLayout)), readSettings({ LEDGER_FILE: '/app/data/ledger.db', RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' }), ), ); @@ -82,7 +79,7 @@ describe('a ledger that refuses the shape of its database as it opens', () => { expect( await builtFrom( - withARefusedShapeExplained( + withARefusedLayoutExplained( Layer.effect(Ledger, Effect.die(unreachable)), readSettings({ LEDGER_FILE: '/data/ledger.db' }), ), diff --git a/packages/server/src/database/ledger-database.ts b/packages/server/src/database/ledger-database.ts index e9d3021ba..4a6675144 100644 --- a/packages/server/src/database/ledger-database.ts +++ b/packages/server/src/database/ledger-database.ts @@ -1,4 +1,4 @@ -import { LedgerShapeRefused, type AppendObserver } from '@on.auto/ledger'; +import { LedgerLayoutRefused, type AppendObserver } from '@on.auto/ledger'; import { Ledger } from '@on.auto/operations'; import type { DatabaseSettings } from '@on.auto/workflow-host'; import { Cause, Effect, Layer } from 'effect'; @@ -16,7 +16,7 @@ export function ledgerDatabaseOf( : { store: 'postgresql', pools: databasePoolsOf(ledger, proxy), observer }; } -export function withARefusedShapeExplained( +export function withARefusedLayoutExplained( ledgerLayer: Layer.Layer, settings: Pick, ): Layer.Layer { @@ -24,7 +24,7 @@ export function withARefusedShapeExplained( const failure = Cause.squash(cause); return Layer.effect( Ledger, - failure instanceof LedgerShapeRefused + failure instanceof LedgerLayoutRefused ? Effect.die(ledgerOfAnotherVersion(failure, settings)) : Effect.failCause(cause), ); diff --git a/packages/server/src/database/ledger-of-another-version.test.ts b/packages/server/src/database/ledger-of-another-version.test.ts index 8b892187f..419d83116 100644 --- a/packages/server/src/database/ledger-of-another-version.test.ts +++ b/packages/server/src/database/ledger-of-another-version.test.ts @@ -8,12 +8,7 @@ import { describe, expect, it, onTestFinished } from 'vitest'; import { ledgerOfAnotherVersion } from './ledger-of-another-version.ts'; -const secretsOfAnOlderShape = { - table: 'kept_secrets', - holds: 'secrets', - found: ['held', 'id', 'kept_at', 'kept_by', 'version'], - kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], -}; +const inAnotherLayout = { found: 2, kept: 1 }; const inPostgreSQL = { store: 'postgresql', @@ -24,7 +19,7 @@ const inPostgreSQL = { } as const; function refusedFileAt(file: string, distribution: 'npx' | 'docker' = 'npx') { - return ledgerOfAnotherVersion(secretsOfAnOlderShape, { ledger: { store: 'sqlite', file }, distribution }); + return ledgerOfAnotherVersion(inAnotherLayout, { ledger: { store: 'sqlite', file }, distribution }); } const awkwardFolder = "Application Support/ada's $HOME"; @@ -34,9 +29,10 @@ describe('a ledger made by another version of Auto Runtime', () => { const refused = refusedFileAt('/Users/ada/.auto-runtime/ledger.db'); expect(refused.message).toBe( - 'The ledger is kept in the file /Users/ada/.auto-runtime/ledger.db, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Delete the ledger and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8 To delete the ledger, run: rm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm', + 'The ledger is kept in the file /Users/ada/.auto-runtime/ledger.db, and was made by another version of Auto Runtime. This version keeps the ledger in layout 1 and cannot read a ledger in layout 2. Delete the ledger and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8 To delete the ledger, run: rm -f /Users/ada/.auto-runtime/ledger.db /Users/ada/.auto-runtime/ledger.db-wal /Users/ada/.auto-runtime/ledger.db-shm', ); expect(String(refused)).toMatch(/^LedgerOfAnotherVersion: The ledger is kept in the file /u); + expect(refused).toMatchObject(inAnotherLayout); }); it('places a ledger file given as a relative path in the directory the server started in', () => { @@ -49,10 +45,10 @@ describe('a ledger made by another version of Auto Runtime', () => { }); it('names the PostgreSQL database and its host, never the URL, and says to point DATABASE_URL at an empty one', () => { - const refused = ledgerOfAnotherVersion(secretsOfAnOlderShape, { ledger: inPostgreSQL, distribution: 'npx' }); + const refused = ledgerOfAnotherVersion(inAnotherLayout, { ledger: inPostgreSQL, distribution: 'npx' }); expect(refused.message).toBe( - 'The ledger is kept in PostgreSQL, in the database auto_runtime on db.example.com:5432, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Point DATABASE_URL at an empty database, or empty this one, and start Auto Runtime again. The ledger then starts with no playbooks, definitions or runs. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', + 'The ledger is kept in PostgreSQL, in the database auto_runtime on db.example.com:5432, and was made by another version of Auto Runtime. This version keeps the ledger in layout 1 and cannot read a ledger in layout 2. Point DATABASE_URL at an empty database, or empty this one, and start Auto Runtime again. The ledger then starts with no playbooks, definitions or runs. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8', ); expect(refused.remedy.command).toBeUndefined(); expect(refused.message).not.toContain('a-secret-password'); @@ -66,7 +62,20 @@ describe('a ledger made by another version of Auto Runtime', () => { command: undefined, }); expect(refused.message).toBe( - "The ledger is kept in the file /app/data/ledger.db, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key. Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume mounted at /data, the ledger's folder /app/data in the image, or remove the volume, and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8", + "The ledger is kept in the file /app/data/ledger.db, and was made by another version of Auto Runtime. This version keeps the ledger in layout 1 and cannot read a ledger in layout 2. Delete ledger.db, ledger.db-wal and ledger.db-shm from the volume mounted at /data, the ledger's folder /app/data in the image, or remove the volume, and start Auto Runtime again. Everything in the ledger goes with it, every playbook, definition, run and connection among it. Need a hand? Ask us on Discord: https://discord.gg/B8BKcKMRm8", + ); + }); +}); + +describe('a ledger made before the ledger carried a layout number', () => { + it('says this version cannot read a ledger that carries no layout number', () => { + const refused = ledgerOfAnotherVersion( + { found: undefined, kept: 1 }, + { ledger: { store: 'sqlite', file: '/data/ledger.db' }, distribution: 'npx' }, + ); + + expect(refused.happened).toBe( + 'The ledger is kept in the file /data/ledger.db, and was made by another version of Auto Runtime. This version keeps the ledger in layout 1 and cannot read a ledger that carries no layout number.', ); }); }); diff --git a/packages/server/src/database/ledger-of-another-version.ts b/packages/server/src/database/ledger-of-another-version.ts index 4c5e63a75..f3477a5a4 100644 --- a/packages/server/src/database/ledger-of-another-version.ts +++ b/packages/server/src/database/ledger-of-another-version.ts @@ -1,6 +1,6 @@ import { basename, dirname } from 'node:path'; -import type { RefusedShape } from '@on.auto/ledger'; +import type { LedgerLayoutRefused } from '@on.auto/ledger'; import { Data } from 'effect'; import { discordInvite } from '../command-line/local-banner.ts'; @@ -12,8 +12,10 @@ interface LedgerRemedy { readonly command: string | undefined; } +type RefusedLayout = Pick; + export class LedgerOfAnotherVersion extends Data.TaggedError('LedgerOfAnotherVersion')< - RefusedShape & { readonly happened: string; readonly remedy: LedgerRemedy; readonly message: string } + RefusedLayout & { readonly happened: string; readonly remedy: LedgerRemedy; readonly message: string } > {} export function helpOfferedAt(invite: string): string { @@ -26,12 +28,12 @@ function shellWord(text: string): string { return unquotedInAShell.test(text) ? text : `'${text.replaceAll("'", String.raw`'\''`)}'`; } -function whatHappened(ledger: LedgerSettings, holds: string): string { - return `${whereTheLedgerIsKept(ledger)}, and was made by another version of Auto Runtime. This version keeps ${holds} in a different shape and cannot read the ${holds} this ledger holds.`; +function aLedgerIn(found: number | undefined): string { + return found === undefined ? 'a ledger that carries no layout number' : `a ledger in layout ${found}`; } -export function columnsCompared({ table, found, kept }: RefusedShape): string { - return `${table} has the columns ${found.join(', ')}, where this version keeps ${kept.join(', ')}.`; +function whatHappened(ledger: LedgerSettings, { found, kept }: RefusedLayout): string { + return `${whereTheLedgerIsKept(ledger)}, and was made by another version of Auto Runtime. This version keeps the ledger in layout ${kept} and cannot read ${aLedgerIn(found)}.`; } const everythingInItGoes = @@ -74,17 +76,18 @@ function remedyFor({ ledger, distribution }: Pick, ): LedgerOfAnotherVersion { - const happened = whatHappened(settings.ledger, shape.holds); + const happened = whatHappened(settings.ledger, refused); const remedy = remedyFor(settings); const deleting = remedy.command === undefined ? [] : [`To delete the ledger, run: ${remedy.command}`]; return new LedgerOfAnotherVersion({ - ...shape, + found: refused.found, + kept: refused.kept, happened, remedy, - message: [happened, columnsCompared(shape), remedy.says, helpOfferedAt(discordInvite), ...deleting].join(' '), + message: [happened, remedy.says, helpOfferedAt(discordInvite), ...deleting].join(' '), }); } diff --git a/packages/server/src/lifecycle/ledger-startup.test.ts b/packages/server/src/lifecycle/ledger-startup.test.ts index 9de17dd27..5cc2372f7 100644 --- a/packages/server/src/lifecycle/ledger-startup.test.ts +++ b/packages/server/src/lifecycle/ledger-startup.test.ts @@ -1,12 +1,12 @@ import { fileURLToPath } from 'node:url'; -import { unreachableDatabase } from '@on.auto/ledger/testing'; +import { ledgerLayout, unreachableDatabase } from '@on.auto/ledger/testing'; import { describe, expect, it, onTestFinished } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; import { spawnServer } from '../testing/processes/spawned-server.ts'; -import { aLedgerOfAnotherVersionAt } from '../testing/records/a-ledger-of-another-version.ts'; +import { aLedgerOfAnotherVersionAt, anotherLayout } from '../testing/records/a-ledger-of-another-version.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { startServer } from './lifecycle.ts'; @@ -65,7 +65,7 @@ describe('a server given both DATABASE_URL and LEDGER_FILE', () => { }); function refusedFor(file: string): string { - return `Auto Runtime could not start: The ledger is kept in the file ${file}, and was made by another version of Auto Runtime. This version keeps secrets in a different shape and cannot read the secrets this ledger holds. kept_secrets has the columns held, id, kept_at, kept_by, version, where this version keeps claimed_until, ends_with, held, id, kept_at, kept_by, kid, version, wrapped_key.`; + return `Auto Runtime could not start: The ledger is kept in the file ${file}, and was made by another version of Auto Runtime. This version keeps the ledger in layout ${ledgerLayout} and cannot read a ledger in layout ${anotherLayout}.`; } const goesWithIt = 'Everything in the ledger goes with it, every playbook, definition, run and connection among it.'; diff --git a/packages/server/src/lifecycle/run-server.test.ts b/packages/server/src/lifecycle/run-server.test.ts index e5ed3e826..8044f725b 100644 --- a/packages/server/src/lifecycle/run-server.test.ts +++ b/packages/server/src/lifecycle/run-server.test.ts @@ -198,12 +198,7 @@ describe('a server that cannot start', () => { describe('a server that cannot start on a ledger of another version', () => { const refused = ledgerOfAnotherVersion( - { - table: 'kept_secrets', - holds: 'secrets', - found: ['held', 'id', 'kept_at', 'kept_by', 'version'], - kept: ['claimed_until', 'ends_with', 'held', 'id', 'kept_at', 'kept_by', 'kid', 'version', 'wrapped_key'], - }, + { found: 2, kept: 1 }, { ledger: { store: 'sqlite', file: '/data/ledger.db' }, distribution: 'source' }, ); diff --git a/packages/server/src/testing/records/a-ledger-of-another-version.ts b/packages/server/src/testing/records/a-ledger-of-another-version.ts index cde6dcda5..f291ce19d 100644 --- a/packages/server/src/testing/records/a-ledger-of-another-version.ts +++ b/packages/server/src/testing/records/a-ledger-of-another-version.ts @@ -1,10 +1,11 @@ import { DatabaseSync } from 'node:sqlite'; -const secretsOfAnOlderShape = - 'CREATE TABLE kept_secrets (id TEXT PRIMARY KEY, version INTEGER NOT NULL, kept_at TEXT NOT NULL, kept_by TEXT NOT NULL, held TEXT NOT NULL)'; +import { ledgerLayout } from '@on.auto/ledger/testing'; + +export const anotherLayout = ledgerLayout + 1; export function aLedgerOfAnotherVersionAt(file: string): void { const database = new DatabaseSync(file); - database.exec(secretsOfAnOlderShape); + database.exec(`PRAGMA user_version = ${anotherLayout}`); database.close(); }