feat(ledger): a layout number, written at creation and checked at open, replaces the shape checks

docs/self-host/docker.md

@@ -271,7 +271,7 @@ The [configuration reference](configuration.md) describes the file and every set
 
 ## Upgrade {#upgrade}
 
-The ledger stays on its volume or in its database, and the new version migrates it when it starts.
+The ledger stays on its volume or in its database. A release that keeps the ledger's layout opens it as it is; before launch, one that changes the layout refuses the older ledger at start and says which files to delete from the volume, or to point `DATABASE_URL` at an empty database.
 
 1. Read the [release notes](https://github.com/BeOnAuto/runtime/releases) of the new version.
 2. Change the version on the `IMAGE` line of `.env`, such as `IMAGE=docker.on.auto/onauto/runtime:0.9.2`.

docs/self-host/postgresql.md

@@ -20,7 +20,7 @@ For a managed service or a server you run, create a database and a user for the
 
 ## Create the database and its user
 
-The server creates its tables in the database's `public` schema at every start, and migrates them there after an upgrade. Its user needs `CREATE` on that schema every time, not only the first. A user that owns the database has it:
+The server creates its tables in the database's `public` schema at its first start and any it lacks at every later start; before launch it migrates nothing, and refuses a database made in another layout of the ledger. Its user needs `CREATE` on that schema every time, not only the first. A user that owns the database has it:
 
 ```sql
 CREATE ROLE auto_runtime LOGIN PASSWORD '<a long random password>';

docs/self-host/production.md

@@ -2,7 +2,7 @@
 
 Auto Runtime is in early development and is not ready for production use; see [Availability](../concepts/functions.md#availability). Your team can still run it today, and you can rely on these behaviors:
 
-- Auto keeps every playbook, definition, run and its history in its ledger, with the workflows that wait on timers and events. On a durable volume or database, all of it survives a restart, and a new version migrates it when it starts.
+- Auto keeps every playbook, definition, run and its history in its ledger, with the workflows that wait on timers and events. On a durable volume or database, all of it survives a restart. A release that keeps the ledger's layout opens it as it is; before launch, one that changes the layout refuses the older ledger at start and says what to delete ([Upgrades](#upgrades)).
 - Backups are yours to run: copy the volume, or rely on your database's own backups.
 - One server runs the workflows at a time. If it stops or dies, a standby that shares its database takes them over within about three seconds.
 
@@ -182,7 +182,7 @@ Pin a full version, such as `docker.on.auto/onauto/runtime:0.9.1`, and upgrade o
 
 1. Read the [release notes](https://github.com/BeOnAuto/runtime/releases).
 2. Pull the new tag. With the files from [Run with Docker](docker.md#upgrade), change the `IMAGE` line of `.env`.
-3. Replace the container. The new server migrates the ledger at start.
+3. Replace the container. A new server that keeps the ledger's layout opens the ledger as it is. Before launch nothing is carried over: a server that changes the layout refuses the ledger at start, names the layout it found and the one it keeps, and says to delete the files or to point `DATABASE_URL` at an empty database.
 
 Servers of different versions never share a database at the same time. With several servers, follow [Upgrade several servers](workflows.md#upgrade-several-servers): stop every old server before you start a new one.
 

packages/ledger/measure-heads.ts

@@ -5,7 +5,7 @@ import { Client } from 'pg';
 
 import type { StoredPageRequest } from './src/event-store.ts';
 import { postgresqlRecordedStore, type Query } from './src/postgresql-reads/postgresql-recorded.ts';
-import { plainPoolsOn, postgresqlEventStore } from './src/postgresql/postgresql-event-store.ts';
+import { plainPoolsOn, postgresqlEventStore } from './src/postgresql/postgresql-ledger.ts';
 
 type Querying = Readonly<Pick<Client, 'query'>>;
 

packages/ledger/measure/stores.ts

@@ -5,7 +5,7 @@ import { Function } from 'effect';
 import { Client } from 'pg';
 
 import type { LedgerStore } from '../src/event-store.ts';
-import { plainPoolsOn, postgresqlEventStore } from '../src/postgresql/postgresql-event-store.ts';
+import { plainPoolsOn, postgresqlEventStore } from '../src/postgresql/postgresql-ledger.ts';
 import { sqliteEventStore } from '../src/sqlite-event-store.ts';
 import type { modestConnections } from '../src/sqlite3.ts';
 import { temporaryDatabase } from '../src/testing/temporary-database.ts';

packages/ledger/src/content/content-behaviour.ts

@@ -16,7 +16,6 @@ import { Effect, Schema } from 'effect';
 import { describe, expect, it } from 'vitest';
 
 import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts';
-import { contentOfAnOlderShape } from './older-content-shapes-behaviour.ts';
 import { unnamedChunksSwept } from './unnamed-chunks-behaviour.ts';
 
 const ChunkRows = Schema.Array(Schema.Struct({ chunk: Schema.Number, bytes: Schema.Number }));
@@ -183,5 +182,4 @@ export function contentBehaviour(entry: LedgerEntry): void {
     readWhileUnstaged(entry);
   });
   unnamedChunksSwept(entry);
-  contentOfAnOlderShape(entry);
 }

packages/ledger/src/content/content-dialects.ts

@@ -6,7 +6,6 @@ export interface ContentDialect {
   readonly bytesType: string;
   readonly bytesOfHex: (hex: string) => SQL;
   readonly relationsAmong: (kind: RelationKind, names: readonly string[]) => SQL;
-  readonly columnsOf: (table: string) => SQL;
 }
 
 const postgresqlKinds: Readonly<Record<RelationKind, SQL>> = {
@@ -19,7 +18,6 @@ export const sqliteContent: ContentDialect = {
   bytesOfHex: (hex) => SQL`unhex(${hex})`,
   relationsAmong: (kind, names) =>
     SQL`SELECT name FROM sqlite_master WHERE type = ${kind} AND name IN (SELECT value FROM json_each(${JSON.stringify(names)}))`,
-  columnsOf: (table) => SQL`SELECT name FROM pragma_table_info(${table})`,
 };
 
 export const postgresqlContent: ContentDialect = {
@@ -30,6 +28,4 @@ export const postgresqlContent: ContentDialect = {
       names.map((name) => SQL`${name}`),
       ', ',
     )}) AND pg_table_is_visible(oid)`,
-  columnsOf: (table) => SQL`SELECT attname AS name FROM pg_attribute
-    WHERE attrelid = to_regclass(${table}) AND attnum > 0 AND NOT attisdropped`,
 };

packages/ledger/src/content/content-tables.ts

@@ -2,10 +2,8 @@ import { SQL } from '@event-driven-io/dumbo';
 import { Schema } from 'effect';
 
 import type { StatementExecutor } from '../event-store.ts';
-import { inTurn } from '../projections/inline-projection.ts';
 import { createMissingIndexes, type PlaybookIndex } from '../recorded/missing-indexes.ts';
 import type { ContentDialect } from './content-dialects.ts';
-import { ledgerShapeRefused } from './ledger-shape-refused.ts';
 
 export interface Shelf {
   readonly chunks: string;
@@ -31,9 +29,8 @@ export const stagingShelf: Shelf = {
 
 export const unstagedRuns = 'unstaged_runs';
 
-export interface ContentTable {
+interface ContentTable {
   readonly name: string;
-  readonly columns: readonly string[];
   readonly create: (dialect: ContentDialect) => SQL;
 }
 
@@ -55,7 +52,6 @@ function chunkTableOf({ chunks, placeColumns }: Shelf): ContentTable {
   const key = [...placeColumns, 'upload_id', 'chunk'];
   return {
     name: chunks,
-    columns: [...key, 'data'],
     create: ({ bytesType }) => SQL`CREATE TABLE IF NOT EXISTS ${SQL.identifier(chunks)} (
       ${textColumns([...placeColumns, 'upload_id'])},
       chunk INTEGER NOT NULL,
@@ -69,7 +65,6 @@ function headTableOf({ heads, placeColumns }: Shelf): ContentTable {
   const key = [...placeColumns, 'sha256'];
   return {
     name: heads,
-    columns: [...key, 'upload_id', 'bytes', 'chunks', 'media_type'],
     create: () => SQL`CREATE TABLE IF NOT EXISTS ${SQL.identifier(heads)} (
       ${textColumns([...key, 'upload_id'])},
       bytes BIGINT NOT NULL,
@@ -82,7 +77,6 @@ function headTableOf({ heads, placeColumns }: Shelf): ContentTable {
 
 const unstagedRunsTable: ContentTable = {
   name: unstagedRuns,
-  columns: stagingShelf.placeColumns,
   create: () => SQL`CREATE TABLE IF NOT EXISTS ${SQL.identifier(unstagedRuns)} (
     ${textColumns(stagingShelf.placeColumns)},
     PRIMARY KEY (${namesOf(stagingShelf.placeColumns)})
@@ -91,7 +85,7 @@ const unstagedRunsTable: ContentTable = {
 
 export const shelves: readonly Shelf[] = [keptShelf, stagingShelf];
 
-export const contentTables: readonly ContentTable[] = [
+const contentTables: readonly ContentTable[] = [
   ...shelves.flatMap((shelf) => [chunkTableOf(shelf), headTableOf(shelf)]),
   unstagedRunsTable,
 ];
@@ -116,30 +110,8 @@ export async function namesFound(execute: StatementExecutor, sql: SQL): Promise<
   return Schema.decodeUnknownSync(NameRows)(rows).map(({ name }) => name);
 }
 
-export interface KeptTable {
-  readonly name: string;
-  readonly columns: readonly string[];
-  readonly holds: string;
-}
-
-export async function requireTheShapeKept(
-  execute: StatementExecutor,
-  dialect: ContentDialect,
-  { name, columns, holds }: KeptTable,
-): Promise<void> {
-  const found = (await namesFound(execute, dialect.columnsOf(name))).toSorted();
-  const kept = columns.toSorted();
-  if (found.join() !== kept.join()) {
-    throw ledgerShapeRefused({ table: name, holds, found, kept });
-  }
-}
-
 export async function createdContentTables(execute: StatementExecutor, dialect: ContentDialect): Promise<void> {
   const existing = new Set(await namesFound(execute, dialect.relationsAmong('table', contentTableNames)));
-  await inTurn(
-    contentTables.filter(({ name }) => existing.has(name)),
-    (table) => requireTheShapeKept(execute, dialect, { ...table, holds: 'recorded content' }),
-  );
   await createMissingIndexes(
     execute,
     contentTables.map((table) => ({ name: table.name, create: () => table.create(dialect) })),

packages/ledger/src/index.ts

@@ -20,7 +20,7 @@ export type {
   StoredPlace,
   StoredRecord,
 } from './event-store.ts';
-export { LedgerShapeRefused, type RefusedShape } from './content/ledger-shape-refused.ts';
+export { LedgerLayoutRefused } from './layout/ledger-layout-refused.ts';
 export { cursorOf as recordedCursorOf } from './recorded/cursor.ts';
 export {
   decisionLoop,

packages/ledger/src/layout/layout-dialects.ts

@@ -0,0 +1,59 @@
+import { SQL } from '@event-driven-io/dumbo';
+import { Schema } from 'effect';
+
+import type { StatementExecutor } from '../event-store.ts';
+
+export interface FoundLayout {
+  readonly layout: number | undefined;
+  readonly tables: number;
+}
+
+export interface LayoutDialect {
+  readonly found: (execute: StatementExecutor) => Promise<FoundLayout>;
+  readonly written: (layout: number) => readonly SQL[];
+}
+
+const noLayout = 0;
+
+const NumberedRow = Schema.Tuple([Schema.Struct({ tables: Schema.Number, layout: Schema.Number })]);
+
+export const sqliteLayout: LayoutDialect = {
+  found: async (execute) => {
+    const { rows } = await execute.query(
+      SQL`SELECT (SELECT count(*) FROM sqlite_master WHERE type = 'table') AS tables, user_version AS layout FROM pragma_user_version`,
+    );
+    const [{ tables, layout }] = Schema.decodeUnknownSync(NumberedRow)(rows);
+    return { layout: layout === noLayout ? undefined : layout, tables };
+  },
+  written: (layout) => [SQL`PRAGMA user_version = ${SQL.plain(String(layout))}`],
+};
+
+const layoutTable = 'ledger_layout';
+
+const TablesRow = Schema.Tuple([Schema.Struct({ tables: Schema.Number, numbered: Schema.Boolean })]);
+
+const LayoutRow = Schema.Tuple([Schema.Struct({ layout: Schema.Number })]);
+
+async function layoutKeptIn(execute: StatementExecutor): Promise<number> {
+  const { rows } = await execute.query(SQL`SELECT layout FROM ${SQL.identifier(layoutTable)}`);
+  const [{ layout }] = Schema.decodeUnknownSync(LayoutRow)(rows);
+  return layout;
+}
+
+export const postgresqlLayout: LayoutDialect = {
+  found: async (execute) => {
+    const { rows } = await execute.query(
+      SQL`SELECT count(*)::int AS tables, count(*) FILTER (WHERE tablename = ${layoutTable}) > 0 AS numbered
+        FROM pg_tables WHERE schemaname = current_schema()`,
+    );
+    const [{ tables, numbered }] = Schema.decodeUnknownSync(TablesRow)(rows);
+    return { layout: numbered ? await layoutKeptIn(execute) : undefined, tables };
+  },
+  written: (layout) => [
+    SQL`CREATE TABLE ${SQL.identifier(layoutTable)} (
+      one_row BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (one_row),
+      layout INTEGER NOT NULL
+    )`,
+    SQL`INSERT INTO ${SQL.identifier(layoutTable)} (layout) VALUES (${layout})`,
+  ],
+};

packages/ledger/src/layout/layout-kept.ts

@@ -0,0 +1,20 @@
+import type { StatementExecutor } from '../event-store.ts';
+import { inTurn } from '../projections/inline-projection.ts';
+import type { FoundLayout, LayoutDialect } from './layout-dialects.ts';
+import { ledgerLayoutRefused } from './ledger-layout-refused.ts';
+import { ledgerLayout } from './ledger-layout.ts';
+
+function isNew({ layout, tables }: FoundLayout): boolean {
+  return layout === undefined && tables === 0;
+}
+
+export async function requireTheLayoutKept(execute: StatementExecutor, dialect: LayoutDialect): Promise<void> {
+  const found = await dialect.found(execute);
+  if (found.layout === ledgerLayout) {
+    return;
+  }
+  if (!isNew(found)) {
+    throw ledgerLayoutRefused({ found: found.layout, kept: ledgerLayout });
+  }
+  await inTurn(dialect.written(ledgerLayout), (statement) => execute.command(statement));
+}

packages/ledger/src/layout/ledger-layout-behaviour.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest';
+
+import { inTurn } from '../projections/inline-projection.ts';
+import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts';
+import type { FoundLayout } from './layout-dialects.ts';
+import { LedgerLayoutRefused } from './ledger-layout-refused.ts';
+import { ledgerLayout } from './ledger-layout.ts';
+
+const anotherLayout = ledgerLayout + 1;
+
+const whatToDo =
+  'Nothing stored before launch is carried over, so delete the local database and start the server again';
+
+export function layoutIn(entry: LedgerEntry, database: string): Promise<FoundLayout> {
+  return entry.executedOn(database, (execute) => entry.layout.found(execute));
+}
+
+function stampedWith(entry: LedgerEntry, database: string, layout: number): Promise<void> {
+  return entry.executedOn(database, (execute) =>
+    inTurn(entry.layout.written(layout), (statement) => execute.command(statement)),
+  );
+}
+
+function refusalOf(entry: LedgerEntry, database: string): Promise<unknown> {
+  return aLedger(entry, database).catch((error: unknown) => error);
+}
+
+export function ledgerLayoutBehaviour(entry: LedgerEntry): void {
+  describe('the layout of the ledger', () => {
+    it('is written into a database that holds no table, which then opens again in that layout', async () => {
+      const database = await entry.aDatabase();
+      await aLedger(entry, database);
+
+      await aLedger(entry, database);
+
+      expect((await layoutIn(entry, database)).layout).toBe(ledgerLayout);
+    });
+
+    it('refuses a database in another layout, naming the layout found and the one kept, and leaves it as it was', async () => {
+      const database = await entry.aDatabase();
+      await stampedWith(entry, database, anotherLayout);
+      const before = await layoutIn(entry, database);
+
+      const refused = await refusalOf(entry, database);
+
+      expect(refused).toBeInstanceOf(LedgerLayoutRefused);
+      expect(refused).toMatchObject({
+        found: anotherLayout,
+        kept: ledgerLayout,
+        message: `The ledger's database is in layout ${anotherLayout} of the ledger, where this server keeps layout ${ledgerLayout}. ${whatToDo}`,
+      });
+      expect(await layoutIn(entry, database)).toEqual(before);
+    });
+
+    it('refuses a database that holds a table but no layout number, and leaves it as it was', async () => {
+      const database = await entry.aDatabase();
+      await entry.queried(database, 'CREATE TABLE kept_notes (note TEXT)');
+
+      const refused = await refusalOf(entry, database);
+
+      expect(refused).toBeInstanceOf(LedgerLayoutRefused);
+      expect(refused).toMatchObject({
+        found: undefined,
+        kept: ledgerLayout,
+        message: `The ledger's database holds tables but no layout number of the ledger, where this server keeps layout ${ledgerLayout}. ${whatToDo}`,
+      });
+      expect(await layoutIn(entry, database)).toEqual({ layout: undefined, tables: 1 });
+    });
+  });
+}