diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 41a7a41..43aeab9 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -51,3 +51,52 @@ jobs: echo "::error::a commit in this range has no Signed-off-by line"; exit 1; } - run: mix gate + + # Slice 010 AC1 and AC2: the same migrations and the same suite on Postgres. A second job + # rather than a matrix entry so the required check keeps its context name, `gate`. This job + # is not required by the ruleset until it has run green on main once; then it is added. + # Tests tagged :sqlite read SQLite pragmas and are excluded here by tag, never by skip. + postgres: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: trinity + POSTGRES_PASSWORD: trinity + POSTGRES_DB: trinity_test + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U trinity" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + MIX_ENV: test + TRINITY_DB: postgres + DATABASE_URL: postgres://trinity:trinity@localhost:5432/trinity_test + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + with: + version-file: .tool-versions + version-type: strict + + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-postgres-${{ hashFiles('mix.lock') }} + restore-keys: ${{ runner.os }}-mix-postgres- + + - run: mix deps.get + + # The adapter is chosen at compile time (config/config.exs); this proves the build + # under TRINITY_DB=postgres links the Postgres adapter and nothing SQLite. + - name: The compiled adapter is Postgres + run: mix run -e 'Ecto.Adapters.Postgres = Trinity.Repo.__adapter__()' + + - run: mix ecto.reset + - run: mix test --exclude sqlite diff --git a/.sobelow-skips b/.sobelow-skips index fb94cfb..71b14d2 100644 --- a/.sobelow-skips +++ b/.sobelow-skips @@ -1 +1,5 @@ Config.CSP: Missing Content-Security-Policy,lib/trinity_web/router.ex:12,2DBC9F9 +Traversal.FileModule: Directory Traversal in `File.mkdir_p`,lib/trinity/data_dir/lock.ex:57,35AB9EC +Traversal.FileModule: Directory Traversal in `File.read`,lib/trinity/data_dir/lock.ex:143,55D5ADB +Traversal.FileModule: Directory Traversal in `File.rm`,lib/trinity/data_dir/lock.ex:82,21D88E2 +Traversal.FileModule: Directory Traversal in `File.write`,lib/trinity/data_dir/lock.ex:132,1C42814 diff --git a/.sobelow-skips.reasons b/.sobelow-skips.reasons index dd0bfb7..38747f7 100644 --- a/.sobelow-skips.reasons +++ b/.sobelow-skips.reasons @@ -10,3 +10,7 @@ # # format: 2DBC9F9 Missing CSP on the browser pipeline. The scaffold ships no Content-Security-Policy, and setting one needs LiveView nonce handling, which belongs with the UI rather than the toolchain. Slice 013 owns it. Skipped rather than fixed blind: a CSP written without the UI it protects is a guess. +35AB9EC Slice 010, Trinity.DataDir.Lock.acquire/2: the directory is the application's own data directory from config or Trinity.Paths, never a request or a model output; the lock has to create it before the Repo does. Low confidence traversal by the tool's own label. +55D5ADB Slice 010, Trinity.DataDir.Lock read/1: reads /LOCK, a path this module composes from the data directory and a constant file name; no caller supplies it. +21D88E2 Slice 010, Trinity.DataDir.Lock.release/2: removes /LOCK only when the file carries this process's own token; same composed path as the read. +1C42814 Slice 010, Trinity.DataDir.Lock take_over/2: rewrites /LOCK after the recorded holder's pid is found dead; same composed path, same constant name. diff --git a/ROADMAP.md b/ROADMAP.md index 6b691dc..6a1c681 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -33,7 +33,7 @@ standards register names the rows that ask for them. | 001 | Packaging spike: Burrito + ex_tauri smoke build | 0 Foundation | M | 000 | approved | | 002 | Supply chain, early: SBOM, build provenance, the TLS floor | 0 Foundation | S | 000 | planned | | 003 | FIPS build leg in CI, from source | 0 Foundation | M | 000 | planned | -| 010 | Core domain + persistence (Ecto/SQLite, schemas, Repo owner) | 1 Core loop | M | 000 | planned | +| 010 | Core domain + persistence (Ecto/SQLite, schemas, Repo owner) | 1 Core loop | M | 000 | done | | 011 | LLM provider layer (req_llm behind `Trinity.LLM` behaviour) | 1 Core loop | M | 010 | planned | | 012 | Session process + agent loop (gen_statem, DynamicSupervisor, rehydration) | 1 Core loop | L | 010, 011 | planned | | 013 | LiveView chat UI with streaming | 1 Core loop | M | 012 | planned | diff --git a/VERSIONS.md b/VERSIONS.md index 93e9fff..d73f1a9 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -85,7 +85,8 @@ never pin a version hex marks as retired or vulnerable. | `bandit` | ~> 1.5 | โœ… in `mix.lock` | HTTP server. | | `ecto_sql` | ~> 3.13 | โœ… in `mix.lock` | | | `ecto_sqlite3` | >= 0.0.0 | โœ… in `mix.lock` | Primary DB. FTS5 available. | -| `postgrex + pgvector` | optional, ~> 0.3 | ๐Ÿ” not a single package | Secondary DB path. Not in default deps; behind `TRINITY_DB=postgres`. Two packages, so no single lock key. | +| `postgrex` | >= 0.0.0 (optional) | โœ… in `mix.lock` | Secondary DB driver, `optional: true` so the desktop build carries none of it; compiled in only under `TRINITY_DB=postgres`, which the CI job proves. Added at Slice 010. Was one row with pgvector; pgvector keeps its own row below. | +| `pgvector` | optional, ~> 0.3 | ๐Ÿ” not yet a dependency | Vectors on the Postgres path. Not yet a dependency; Slice 032 decides. Split from the postgrex row at Slice 010. | | `oban` | ~> 2.24 | ๐Ÿ” not yet a dependency | Uses `Oban.Engines.Lite` on SQLite. โš ๏ธ Oban Pro Workflows/Smart engine are Postgres-only. Added at Slice 050. | | `req` | ~> 0.5 | โœ… in `mix.lock` | HTTP client. | | `req_llm` | ~> 1.22 | ๐Ÿ” not yet a dependency | Provider layer (streaming, tools, structured output, usage). โš ๏ธ The pin was `~> 1.10` against a recorded latest of 1.10.0; the real latest was twelve minors ahead. Check event shapes against the current version at Slice 011, not against this file's prose. Added at Slice 011. | diff --git a/config/config.exs b/config/config.exs index e59028d..3322e21 100644 --- a/config/config.exs +++ b/config/config.exs @@ -24,6 +24,37 @@ config :trinity, ecto_repos: [Trinity.Repo], generators: [timestamp_type: :utc_datetime] +# Slice 010. The database adapter is chosen at compile time: SQLite is primary and the default, +# Postgres is the CI-tested alternative behind TRINITY_DB=postgres (docs/adr/0002). An Ecto +# adapter is fixed in `use Ecto.Repo`, so switching means recompiling, and this file says so +# rather than pretending a runtime variable could do it. +config :trinity, + :db_adapter, + (case System.get_env("TRINITY_DB", "sqlite") do + "sqlite" -> Ecto.Adapters.SQLite3 + "postgres" -> Ecto.Adapters.Postgres + other -> raise "TRINITY_DB must be sqlite or postgres, got #{inspect(other)}" + end) + +# Slice 010, every environment, SQLite only (the Postgres adapter ignores keys it does not +# know, and the CI matrix proves that). One writer: the pool has exactly one connection, so the +# single-writer rule SQLite imposes is the pool's shape rather than a hope. Each pragma is named +# here rather than inherited from the adapter's default, so a default change upstream is a diff +# here and not a silent behaviour change. `synchronous: :normal` under WAL can lose the last +# transaction on power loss and cannot corrupt; the receipts file (slice 024, its own Repo +# below) decides its own setting. `wal_auto_check_point` is in pages; the 010 stress test +# reports the -wal size after its run so the value can be set from a measurement. +if System.get_env("TRINITY_DB", "sqlite") == "sqlite" do + config :trinity, Trinity.Repo, + pool_size: 1, + journal_mode: :wal, + synchronous: :normal, + foreign_keys: :on, + busy_timeout: 5_000, + cache_size: -64_000, + wal_auto_check_point: 1_000 +end + # Configure the endpoint config :trinity, TrinityWeb.Endpoint, url: [host: "localhost"], diff --git a/config/dev.exs b/config/dev.exs index e54c55b..4062e19 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -5,7 +5,6 @@ import Config # Configure your database config :trinity, Trinity.Repo, database: Path.expand("../trinity_dev.db", __DIR__), - pool_size: 5, stacktrace: true, show_sensitive_data_on_connection_error: true diff --git a/config/runtime.exs b/config/runtime.exs index 17c9c1e..6ab086a 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -66,9 +66,16 @@ if config_env() == :prod do # any business writing to. database_path = System.get_env("DATABASE_PATH") || Trinity.Paths.database_path() - config :trinity, Trinity.Repo, - database: database_path, - pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5") + # Slice 010: the pool size is 1 by config/config.exs and is not read from the environment + # here, because a larger pool on SQLite is a second writer waiting on busy_timeout, not + # capacity. A Postgres build sets its own size below. + config :trinity, Trinity.Repo, database: database_path + + if System.get_env("TRINITY_DB") == "postgres" do + config :trinity, Trinity.Repo, + url: System.get_env("DATABASE_URL") || raise("TRINITY_DB=postgres needs DATABASE_URL"), + pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10") + end # The secret key base is used to sign/encrypt cookies and other secrets. # A default value is used in config/dev.exs and config/test.exs but you diff --git a/config/test.exs b/config/test.exs index 0a568ae..bb2f3cc 100644 --- a/config/test.exs +++ b/config/test.exs @@ -7,10 +7,26 @@ import Config # The MIX_TEST_PARTITION environment variable can be used # to provide built-in test partitioning in CI environment. # Run `mix help test` for more information. -config :trinity, Trinity.Repo, - database: Path.expand("../trinity_test.db", __DIR__), - pool_size: 5, - pool: Ecto.Adapters.SQL.Sandbox +# Slice 010: the data-dir lock takes a temporary directory in tests, so a test run never +# contends with a running Trinity on the same machine, and two test runs at once do contend, +# which is the property under test. +config :trinity, Trinity.DataDir.Lock, + dir: + Path.join(System.tmp_dir!(), "trinity-test-lock-#{System.get_env("MIX_TEST_PARTITION", "0")}") + +# Slice 010: the pool has one connection (config/config.exs), so the sandbox hands every test +# the same connection and concurrent writers in a test queue on it exactly as they do in +# production. Tests that touch the Repo are not `async: true` for that reason. +if System.get_env("TRINITY_DB") == "postgres" do + config :trinity, Trinity.Repo, + url: System.get_env("DATABASE_URL") || raise("TRINITY_DB=postgres needs DATABASE_URL"), + pool_size: 10, + pool: Ecto.Adapters.SQL.Sandbox +else + config :trinity, Trinity.Repo, + database: Path.expand("../trinity_test.db", __DIR__), + pool: Ecto.Adapters.SQL.Sandbox +end # We don't run a server during test. If one is required, # you can enable the server option below. diff --git a/coverage.tsv b/coverage.tsv index 47bf0f9..5eb6566 100644 --- a/coverage.tsv +++ b/coverage.tsv @@ -1,3 +1,4 @@ slice_id percent sha date 000 27.01 e935c7b 2026-09-06 001 30.37 5a9c8f7 2026-09-06 +010 44.88 45ba4f0 2026-09-20 diff --git a/docs/05-data-model.md b/docs/05-data-model.md index afffa1d..2af372e 100644 --- a/docs/05-data-model.md +++ b/docs/05-data-model.md @@ -1,6 +1,6 @@ # 05: Data model -All tables have `id` (UUIDv7 as binary_id, sortable), `inserted_at`, `updated_at` (utc_datetime_usec). +All tables have `id` (UUIDv7 as binary_id, sortable, minted by `Trinity.UUID` since slice 010), `inserted_at`, `updated_at` (utc_datetime_usec). SQLite is primary; every migration must also run on Postgres in the CI matrix. Use Ecto types that map on both (`:binary_id`, `:map` โ†’ JSON text on SQLite, `:utc_datetime_usec`). Vector columns and FTS tables are created with adapter-specific `execute/1` guarded by `repo().__adapter__()`. diff --git a/lib/trinity.ex b/lib/trinity.ex index 50df656..da5d8dc 100644 --- a/lib/trinity.ex +++ b/lib/trinity.ex @@ -3,10 +3,15 @@ defmodule Trinity do # DataCase and NetworkGuard live in test/support, which is compiled only in :test, so the # export list is environment-dependent. TrinityWeb.ConnCase crosses the boundary to reach - # DataCase, and the live-tagged tests reach NetworkGuard. + # DataCase, and the live-tagged tests reach NetworkGuard. Slice 010 exports Paths (the + # application locks the data directory before the Repo starts), Repo and UUID (the contexts + # use them), and the Sessions sub-boundary: a context TrinityWeb may call (docs/01), which + # exports only its API module and keeps Store and the schemas inside. use Boundary, deps: [], - exports: if(Mix.env() == :test, do: [DataCase, NetworkGuard], else: []) + exports: + [Paths, Repo, UUID, Sessions] ++ + if(Mix.env() == :test, do: [DataCase, NetworkGuard, Factory], else: []) @moduledoc """ Trinity keeps the contexts that define your domain diff --git a/lib/trinity/application.ex b/lib/trinity/application.ex index 96bf06d..9efa129 100644 --- a/lib/trinity/application.ex +++ b/lib/trinity/application.ex @@ -25,6 +25,9 @@ defmodule Trinity.Application do desktop_children() ++ [ TrinityWeb.Telemetry, + # Slice 010: one node per data directory. Before the Repo, so a refused boot has + # opened no database file; the reason names the holder's OS pid and mode. + {Trinity.DataDir.Lock, dir: lock_dir(), mode: mode()}, Trinity.Repo, {Ecto.Migrator, repos: Application.fetch_env!(:trinity, :ecto_repos), skip: skip_migrations?()}, @@ -63,6 +66,22 @@ defmodule Trinity.Application do @spec desktop_children() :: [module()] def desktop_children, do: @desktop_children + # The lock lives in the data directory the database defaults to. A deployment that points + # DATABASE_PATH elsewhere still locks the data directory, which is the thing two instances + # would otherwise share; the test environment points it at a temporary directory. + defp lock_dir do + Application.get_env(:trinity, Trinity.DataDir.Lock, [])[:dir] || + Trinity.Paths.ensure_data_dir() + end + + # `desktop` unless the process says otherwise; slice 061's headless release sets it. + defp mode do + case System.get_env("TRINITY_MODE", "desktop") do + "headless" -> :headless + _ -> :desktop + end + end + defp skip_migrations? do # By default, sqlite migrations are run when using a release System.get_env("RELEASE_NAME") == nil diff --git a/lib/trinity/data_dir/lock.ex b/lib/trinity/data_dir/lock.ex new file mode 100644 index 0000000..fa6ea89 --- /dev/null +++ b/lib/trinity/data_dir/lock.ex @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.DataDir.Lock do + @moduledoc """ + One node per data directory, enforced rather than assumed. Slice 010. + + The single-connection pool makes concurrent writes safe within one BEAM node and does + nothing about two OS processes opening the same SQLite file, which is the corruption the + vision names. Slice 061 ships a headless profile and slice 100 a desktop app, and nothing + else stops them sharing a data directory. This process takes `/LOCK` at boot, + before the Repo starts, and refuses to start when the file is held by a live process, + naming the holder's OS pid and mode. The application then fails to start with that reason, + and no database file has been opened. + + The file is created with `:exclusive`, so two boots racing for it cannot both win. It + carries the holder's OS pid, mode (`desktop` or `headless`), a per-boot token and the time. + A file whose pid is no longer alive is stale and is taken over; on platforms where + liveness cannot be read this module treats the file as held, so the only way past a + dead holder there is removing the file by hand, which is the safe direction. + + The lock file is not the database and is never inside a transaction; it says nothing + about the integrity of the data, only about who may open it. + """ + + use GenServer + + @type mode :: :desktop | :headless + @type holder :: %{pid: pos_integer(), mode: mode(), token: String.t(), at: String.t()} + + @file_name "LOCK" + + ## Client + + @doc "Starts the lock as a supervised child. `dir:` and `mode:` are required." + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: Keyword.get(opts, :name, __MODULE__)) + end + + @doc """ + Takes the lock file in `dir` for `mode`. Returns `{:ok, holder}` with what was written, or + `{:error, {:held, holder}}` naming the live holder, or `{:error, {:unwritable, reason}}`. + Pure with respect to processes: no GenServer involved, so a test can call it directly. + """ + @spec acquire(Path.t(), mode()) :: + {:ok, holder()} | {:error, {:held, holder()} | {:unwritable, term()}} + def acquire(dir, mode) when mode in [:desktop, :headless] do + path = Path.join(dir, @file_name) + + holder = %{ + pid: os_pid(), + mode: mode, + token: token(), + at: DateTime.to_iso8601(DateTime.utc_now()) + } + + with :ok <- File.mkdir_p(dir), + {:error, :eexist} <- write_exclusive(path, holder) do + contend(path, holder, read(path)) + else + {:ok, holder} -> {:ok, holder} + {:error, reason} -> {:error, {:unwritable, reason}} + end + end + + # The file exists. A live holder is refused by name; a dead one is taken over; a file that + # cannot be read is a holder that cannot be named, and is refused. + defp contend(path, holder, {:ok, %{pid: pid} = existing}) do + if alive?(pid), do: {:error, {:held, existing}}, else: take_over(path, holder) + end + + defp contend(_path, _holder, {:error, reason}) do + {:error, {:held, %{pid: 0, mode: :unknown, token: "unreadable: #{inspect(reason)}", at: ""}}} + end + + @doc "Releases the lock file if this process's token wrote it. Idempotent." + @spec release(Path.t(), holder()) :: :ok + def release(dir, %{token: token}) do + path = Path.join(dir, @file_name) + + case read(path) do + {:ok, %{token: ^token}} -> File.rm(path) |> then(fn _ -> :ok end) + _ -> :ok + end + end + + @doc "Reads the holder recorded in `dir`, if any." + @spec holder(Path.t()) :: {:ok, holder()} | {:error, term()} + def holder(dir), do: read(Path.join(dir, @file_name)) + + ## Server + + @impl true + def init(opts) do + dir = Keyword.fetch!(opts, :dir) + mode = Keyword.fetch!(opts, :mode) + + case acquire(dir, mode) do + {:ok, holder} -> + Process.flag(:trap_exit, true) + {:ok, %{dir: dir, holder: holder}} + + {:error, {:held, %{pid: pid, mode: held_mode}}} -> + {:stop, + {:data_dir_held, + "#{dir} is held by OS pid #{pid} in #{held_mode} mode; refusing to start and touching no database file"}} + + {:error, {:unwritable, reason}} -> + {:stop, + {:data_dir_unwritable, "cannot write #{Path.join(dir, @file_name)}: #{inspect(reason)}"}} + end + end + + @impl true + def terminate(_reason, %{dir: dir, holder: holder}), do: release(dir, holder) + + ## Internals + + defp write_exclusive(path, holder) do + case :file.open(String.to_charlist(path), [:write, :exclusive, :binary]) do + {:ok, io} -> + :ok = :file.write(io, encode(holder)) + :ok = :file.close(io) + {:ok, holder} + + {:error, reason} -> + {:error, reason} + end + end + + defp take_over(path, holder) do + case File.write(path, encode(holder)) do + :ok -> {:ok, holder} + {:error, reason} -> {:error, {:unwritable, reason}} + end + end + + # One line per field, no JSON library in the boot path. + defp encode(%{pid: pid, mode: mode, token: token, at: at}), + do: "pid #{pid}\nmode #{mode}\ntoken #{token}\nat #{at}\n" + + defp read(path) do + with {:ok, body} <- File.read(path), + %{"pid" => pid, "mode" => mode, "token" => token, "at" => at} <- parse(body), + {pid_int, ""} <- Integer.parse(pid), + true <- mode in ["desktop", "headless"] do + {:ok, %{pid: pid_int, mode: String.to_existing_atom(mode), token: token, at: at}} + else + {:error, reason} -> {:error, reason} + _ -> {:error, :malformed} + end + end + + defp parse(body) do + body + |> String.split("\n", trim: true) + |> Enum.map(&String.split(&1, " ", parts: 2)) + |> Enum.filter(&match?([_, _], &1)) + |> Map.new(fn [k, v] -> {k, v} end) + end + + defp os_pid, do: String.to_integer(System.pid()) + + defp token, do: Base.url_encode64(:crypto.strong_rand_bytes(9), padding: false) + + # Liveness of an OS pid. Linux answers through /proc; anywhere else this says "alive", so a + # stale file from a crashed holder is refused rather than silently taken over. That is + # the safe direction and the message names the pid to remove it by hand. + defp alive?(pid) do + case :os.type() do + {:unix, :linux} -> File.dir?("/proc/#{pid}") + _ -> true + end + end +end diff --git a/lib/trinity/repo.ex b/lib/trinity/repo.ex index dc5ef0f..63ca914 100644 --- a/lib/trinity/repo.ex +++ b/lib/trinity/repo.ex @@ -1,7 +1,17 @@ # SPDX-FileCopyrightText: Sudo Apt Holdings LLC # SPDX-License-Identifier: Apache-2.0 defmodule Trinity.Repo do + @moduledoc """ + The one writer. Slice 010. + + The adapter is fixed at compile time from `config :trinity, :db_adapter` (SQLite by default, + Postgres behind `TRINITY_DB=postgres`), because `use Ecto.Repo` takes it as a literal and no + runtime variable can change it afterwards. On SQLite the pool holds one connection + (`config/config.exs`), so every write in the node serialises here rather than in + `busy_timeout`. + """ + use Ecto.Repo, otp_app: :trinity, - adapter: Ecto.Adapters.SQLite3 + adapter: Application.compile_env(:trinity, :db_adapter, Ecto.Adapters.SQLite3) end diff --git a/lib/trinity/repo/receipts.ex b/lib/trinity/repo/receipts.ex new file mode 100644 index 0000000..84c3fd2 --- /dev/null +++ b/lib/trinity/repo/receipts.ex @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Repo.Receipts do + @moduledoc """ + The slot for a second database file, reserved at slice 010 and unused until slice 024. + + Declared so that the receipts chain can live in its own SQLite file with its own + `synchronous` setting (`:full`, if an auditor wants the last committed receipt durable + across power loss) without moving the primary database later. Not started by the + application, not in `:ecto_repos`, and no migration targets it at this slice. Slice 024 + configures and starts it; until then any call here fails because the repo is not running, + which is the intended state. + """ + + use Ecto.Repo, + otp_app: :trinity, + adapter: Application.compile_env(:trinity, :db_adapter, Ecto.Adapters.SQLite3) +end diff --git a/lib/trinity/sessions.ex b/lib/trinity/sessions.ex new file mode 100644 index 0000000..eacc332 --- /dev/null +++ b/lib/trinity/sessions.ex @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Sessions do + @moduledoc """ + The persistence API for sessions and messages. Slice 010. + + The only public surface over the `personas`, `sessions` and `messages` tables. `boundary` + exports this module alone; `Trinity.Sessions.Store` and the schemas stay inside. Slice 012 + adds the session process on top of this API and changes nothing here. + """ + use Boundary, deps: [Trinity], exports: [] + + alias Trinity.Sessions.{Message, Persona, Session, Store} + + @type session_id :: String.t() + + @doc "Creates a persona. `name` is required and unique." + @spec create_persona(map()) :: {:ok, Persona.t()} | {:error, Ecto.Changeset.t()} + def create_persona(attrs), do: Store.insert_persona(attrs) + + @doc "The persona with this name, or nil." + @spec get_persona_by_name(String.t()) :: Persona.t() | nil + def get_persona_by_name(name), do: Store.get_persona_by_name(name) + + @doc "Creates a session. `persona_id` is required; `origin` and `status` come from a closed vocabulary." + @spec create_session(map()) :: {:ok, Session.t()} | {:error, Ecto.Changeset.t()} + def create_session(attrs), do: Store.insert_session(attrs) + + @doc "The session with this id, or nil." + @spec get_session(session_id()) :: Session.t() | nil + def get_session(id), do: Store.get_session(id) + + @doc "Sessions, most recently active first. Options: `status:`, `limit:` (default 50)." + @spec list_sessions(keyword()) :: [Session.t()] + def list_sessions(opts \\ []), do: Store.list_sessions(opts) + + @doc """ + Appends a message to a session, assigning the next gapless `seq` atomically. Rejects an + unknown role or empty content with `{:error, %Ecto.Changeset{}}` and a missing session with + `{:error, :no_session}`. Touches the session's `last_activity_at`. + """ + @spec append_message(session_id(), map()) :: + {:ok, Message.t()} | {:error, Ecto.Changeset.t() | :no_session} + def append_message(session_id, attrs) do + changeset = Message.changeset(%Message{}, attrs) + + if changeset.valid? do + Store.append_message(session_id, changeset) + else + {:error, %{changeset | action: :insert}} + end + end + + @doc "Messages in `seq` order. Options: `limit:` (default 200), `offset:` (default 0)." + @spec history(session_id(), keyword()) :: [Message.t()] + def history(session_id, opts \\ []), do: Store.history(session_id, opts) + + @doc "Marks a session archived." + @spec archive(Session.t()) :: {:ok, Session.t()} | {:error, Ecto.Changeset.t()} + def archive(%Session{} = session), do: Store.update_session(session, %{status: "archived"}) + + @doc "The number of messages in a session." + @spec message_count(session_id()) :: non_neg_integer() + def message_count(session_id), do: Store.message_count(session_id) + + @doc "Every `seq` in a session, ascending. The stress test's population." + @spec seqs(session_id()) :: [pos_integer()] + def seqs(session_id), do: Store.seqs(session_id) +end diff --git a/lib/trinity/sessions/message.ex b/lib/trinity/sessions/message.ex new file mode 100644 index 0000000..0bd4ad3 --- /dev/null +++ b/lib/trinity/sessions/message.ex @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Sessions.Message do + @moduledoc """ + One append-only row per message. `seq` is assigned by `Trinity.Sessions.append_message/2` + inside a transaction and is never taken from the caller; the unique index on + `(session_id, seq)` is the last line of defence on both adapters. + """ + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, Trinity.UUID, autogenerate: true} + @foreign_key_type Trinity.UUID + @timestamps_opts [type: :utc_datetime_usec] + + @roles ~w(system user assistant tool) + + @type t :: %__MODULE__{} + + schema "messages" do + field :seq, :integer + field :role, :string + field :content, :string + field :parts, :map, default: %{} + field :tool_call_id, :string + field :usage, :map + field :provider_meta, :map, default: %{} + belongs_to :session, Trinity.Sessions.Session + timestamps() + end + + @doc "The closed vocabulary of roles, from docs/05." + @spec roles() :: [String.t()] + def roles, do: @roles + + @doc """ + The caller's half of a message: role and content are required, the role is one of four, + and empty content is refused. `seq` and `session_id` are set by the context, not cast. + """ + @spec changeset(t(), map()) :: Ecto.Changeset.t() + def changeset(message, attrs) do + message + |> cast(attrs, [:role, :content, :parts, :tool_call_id, :usage, :provider_meta]) + |> validate_required([:role, :content]) + |> validate_inclusion(:role, @roles) + |> validate_change(:content, fn :content, content -> + if String.trim(content) == "", do: [content: "cannot be blank"], else: [] + end) + end + + @doc false + @spec place(Ecto.Changeset.t(), String.t(), pos_integer()) :: Ecto.Changeset.t() + def place(changeset, session_id, seq) do + changeset + |> put_change(:session_id, session_id) + |> put_change(:seq, seq) + |> unique_constraint([:session_id, :seq], name: :messages_session_id_seq_index) + end +end diff --git a/lib/trinity/sessions/persona.ex b/lib/trinity/sessions/persona.ex new file mode 100644 index 0000000..e61b11f --- /dev/null +++ b/lib/trinity/sessions/persona.ex @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Sessions.Persona do + @moduledoc """ + The minimal persona row (slice 010): `name`, `soul`, `model`, `settings`. Slice 030 gives it + its behaviour; here it exists so a session has an owner to reference. + """ + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, Trinity.UUID, autogenerate: true} + @foreign_key_type Trinity.UUID + @timestamps_opts [type: :utc_datetime_usec] + + @type t :: %__MODULE__{} + + schema "personas" do + field :name, :string + field :soul, :string + field :model, :string + field :settings, :map, default: %{} + has_many :sessions, Trinity.Sessions.Session + timestamps() + end + + @doc false + @spec changeset(t(), map()) :: Ecto.Changeset.t() + def changeset(persona, attrs) do + persona + |> cast(attrs, [:name, :soul, :model, :settings]) + |> validate_required([:name]) + |> validate_length(:name, min: 1, max: 100) + |> unique_constraint(:name) + end +end diff --git a/lib/trinity/sessions/session.ex b/lib/trinity/sessions/session.ex new file mode 100644 index 0000000..cb05150 --- /dev/null +++ b/lib/trinity/sessions/session.ex @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Sessions.Session do + @moduledoc """ + A conversation. Slice 010 owns the row; slice 012 owns the process that runs it. `origin` + and `status` are strings with a closed vocabulary from docs/05, validated here. + """ + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, Trinity.UUID, autogenerate: true} + @foreign_key_type Trinity.UUID + @timestamps_opts [type: :utc_datetime_usec] + + @origins ~w(desktop telegram discord console cron subagent mcp a2a) + @statuses ~w(active archived compacted) + + @type t :: %__MODULE__{} + + schema "sessions" do + field :title, :string + field :origin, :string, default: "desktop" + field :origin_ref, :map, default: %{} + field :status, :string, default: "active" + field :model, :string + field :token_usage, :map, default: %{} + field :last_activity_at, :utc_datetime_usec + belongs_to :persona, Trinity.Sessions.Persona + belongs_to :parent, __MODULE__ + has_many :messages, Trinity.Sessions.Message + timestamps() + end + + @doc "The closed vocabulary of origins, from docs/05." + @spec origins() :: [String.t()] + def origins, do: @origins + + @doc "The closed vocabulary of statuses, from docs/05." + @spec statuses() :: [String.t()] + def statuses, do: @statuses + + @doc false + @spec changeset(t(), map()) :: Ecto.Changeset.t() + def changeset(session, attrs) do + session + |> cast(attrs, [ + :title, + :persona_id, + :parent_id, + :origin, + :origin_ref, + :status, + :model, + :token_usage + ]) + |> validate_required([:persona_id, :origin, :status]) + |> validate_inclusion(:origin, @origins) + |> validate_inclusion(:status, @statuses) + |> foreign_key_constraint(:persona_id) + |> foreign_key_constraint(:parent_id) + end +end diff --git a/lib/trinity/sessions/store.ex b/lib/trinity/sessions/store.ex new file mode 100644 index 0000000..5ea082e --- /dev/null +++ b/lib/trinity/sessions/store.ex @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Sessions.Store do + @moduledoc """ + Every query the Sessions context runs. Internal to `Trinity.Sessions`: the boundary exports + the context and not this module, so `TrinityWeb` cannot reach the tables except through the + context's API (slice 010 AC5). + """ + import Ecto.Query + + alias Trinity.Repo + alias Trinity.Sessions.{Message, Persona, Session} + + @spec insert_persona(map()) :: {:ok, Persona.t()} | {:error, Ecto.Changeset.t()} + def insert_persona(attrs), do: %Persona{} |> Persona.changeset(attrs) |> Repo.insert() + + @spec get_persona_by_name(String.t()) :: Persona.t() | nil + def get_persona_by_name(name), do: Repo.get_by(Persona, name: name) + + @spec insert_session(map()) :: {:ok, Session.t()} | {:error, Ecto.Changeset.t()} + def insert_session(attrs), do: %Session{} |> Session.changeset(attrs) |> Repo.insert() + + @spec get_session(String.t()) :: Session.t() | nil + def get_session(id), do: Repo.get(Session, id) + + @spec list_sessions(keyword()) :: [Session.t()] + def list_sessions(opts) do + status = Keyword.get(opts, :status) + limit = Keyword.get(opts, :limit, 50) + + Session + |> maybe_status(status) + |> order_by([s], desc: s.last_activity_at, desc: s.inserted_at) + |> limit(^limit) + |> Repo.all() + end + + defp maybe_status(query, nil), do: query + defp maybe_status(query, status), do: where(query, [s], s.status == ^status) + + @spec update_session(Session.t(), map()) :: {:ok, Session.t()} | {:error, Ecto.Changeset.t()} + def update_session(session, attrs), do: session |> Session.changeset(attrs) |> Repo.update() + + @doc """ + Appends one message with the next `seq` for its session, in one transaction. On Postgres the + session row is locked `FOR UPDATE` so two transactions cannot read the same `max(seq)`; on + SQLite the single connection serialises the whole transaction, and the unique index on + `(session_id, seq)` refuses a duplicate on either. + """ + @spec append_message(String.t(), Ecto.Changeset.t()) :: + {:ok, Message.t()} | {:error, Ecto.Changeset.t() | :no_session} + def append_message(session_id, changeset) do + Repo.transaction(fn -> + with %Session{} = session <- lock_session(session_id), + next = next_seq(session_id), + {:ok, message} <- changeset |> Message.place(session_id, next) |> Repo.insert(), + {:ok, _} <- touch(session) do + message + else + nil -> Repo.rollback(:no_session) + {:error, %Ecto.Changeset{} = cs} -> Repo.rollback(cs) + end + end) + end + + # The adapter is a compile-time fact (config/config.exs), so the row lock is compiled in or + # out rather than branched at runtime: a Postgres build locks the session row FOR UPDATE; a + # SQLite build has one connection and needs no row lock. The type checker refuses a + # runtime branch on a constant, which is how this shape was arrived at. Each build carries + # exactly one of the two definitions, and the CI matrix compiles both. + if Application.compile_env(:trinity, :db_adapter, Ecto.Adapters.SQLite3) == + Ecto.Adapters.Postgres do + defp lock_session(session_id) do + Repo.one(from(s in Session, where: s.id == ^session_id, lock: "FOR UPDATE")) + end + else + defp lock_session(session_id) do + Repo.one(from(s in Session, where: s.id == ^session_id)) + end + end + + defp next_seq(session_id) do + (Repo.one(from(m in Message, where: m.session_id == ^session_id, select: max(m.seq))) || 0) + + 1 + end + + defp touch(session) do + session + |> Ecto.Changeset.change(last_activity_at: DateTime.utc_now()) + |> Repo.update() + end + + @spec history(String.t(), keyword()) :: [Message.t()] + def history(session_id, opts) do + limit = Keyword.get(opts, :limit, 200) + offset = Keyword.get(opts, :offset, 0) + + Message + |> where([m], m.session_id == ^session_id) + |> order_by([m], asc: m.seq) + |> limit(^limit) + |> offset(^offset) + |> Repo.all() + end + + @spec message_count(String.t()) :: non_neg_integer() + def message_count(session_id) do + Repo.one(from(m in Message, where: m.session_id == ^session_id, select: count(m.id))) + end + + @spec seqs(String.t()) :: [pos_integer()] + def seqs(session_id) do + Repo.all( + from(m in Message, where: m.session_id == ^session_id, order_by: m.seq, select: m.seq) + ) + end +end diff --git a/lib/trinity/uuid.ex b/lib/trinity/uuid.ex new file mode 100644 index 0000000..481a412 --- /dev/null +++ b/lib/trinity/uuid.ex @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.UUID do + @moduledoc """ + UUIDv7 (RFC 9562 section 5.7), generated in-tree. Slice 010. + + Layout, most significant first: 48 bits of Unix milliseconds; 4 bits version `0111`; 12 bits + `rand_a`; 2 bits variant `10`; 62 bits `rand_b`. `rand_a` carries the low twelve bits of a + VM-wide monotonic counter (RFC 9562 section 6.2, method 1), so ids minted within one + millisecond sort in the order they were minted; `rand_b` is random on every call, so two + ids can never be equal even if the counter wrapped inside a millisecond, which takes more + than 4096 mints in that millisecond. Beyond that rate the order within the millisecond is + no longer guaranteed and uniqueness still is. + + Every primary key in the tree uses this so rows sort by creation time without a second + column. The module is also an `Ecto.Type` of underlying type `:uuid`, delegating cast, + dump and load to `Ecto.UUID` so each adapter keeps its own storage rule (text on SQLite, + `uuid` on Postgres, with `:binary_id` migration columns), and supplying `autogenerate/0` + so a schema declares `@primary_key {:id, Trinity.UUID, autogenerate: true}`. + """ + + use Ecto.Type + + @impl Ecto.Type + def type, do: :uuid + + @impl Ecto.Type + def cast(value), do: Ecto.UUID.cast(value) + + @impl Ecto.Type + def dump(value), do: Ecto.UUID.dump(value) + + @impl Ecto.Type + def load(value), do: Ecto.UUID.load(value) + + @impl Ecto.Type + def autogenerate, do: generate() + + @doc "A new UUIDv7 in the canonical 36-character string form." + @spec generate() :: String.t() + def generate do + ms = System.system_time(:millisecond) + seq = :erlang.unique_integer([:monotonic, :positive]) + <> = :crypto.strong_rand_bytes(8) + + encode(<>) + end + + @doc "The Unix millisecond timestamp an id carries." + @spec timestamp_ms(String.t()) :: {:ok, non_neg_integer()} | :error + def timestamp_ms(<<_::binary-size(36)>> = uuid) do + case Base.decode16(String.replace(uuid, "-", ""), case: :mixed) do + {:ok, <>} -> {:ok, ms} + _ -> :error + end + end + + def timestamp_ms(_), do: :error + + defp encode(<>) do + [<>, <>, <>, <>, <>] + |> Enum.map_join("-", &Base.encode16(&1, case: :lower)) + end +end diff --git a/lib/trinity/versions.ex b/lib/trinity/versions.ex index e4e48df..3171174 100644 --- a/lib/trinity/versions.ex +++ b/lib/trinity/versions.ex @@ -123,11 +123,18 @@ defmodule Trinity.Versions do note: "Primary DB. FTS5 available." }, %{ - name: "postgrex + pgvector", + name: "postgrex", + pin: ">= 0.0.0 (optional)", + lock: "postgrex", + note: + "Secondary DB driver, `optional: true` so the desktop build carries none of it; compiled in only under `TRINITY_DB=postgres`, which the CI job proves. Added at Slice 010. Was one row with pgvector; pgvector keeps its own row below." + }, + %{ + name: "pgvector", pin: "optional, ~> 0.3", - lock: nil, + lock: "pgvector", note: - "Secondary DB path. Not in default deps; behind `TRINITY_DB=postgres`. Two packages, so no single lock key." + "Vectors on the Postgres path. Not yet a dependency; Slice 032 decides. Split from the postgrex row at Slice 010." }, %{ name: "oban", diff --git a/mix.exs b/mix.exs index af9b671..833c88b 100644 --- a/mix.exs +++ b/mix.exs @@ -93,6 +93,10 @@ defmodule Trinity.MixProject do {:phoenix_ecto, "~> 4.5"}, {:ecto_sql, "~> 3.13"}, {:ecto_sqlite3, ">= 0.0.0"}, + # Slice 010: the CI-tested alternative behind TRINITY_DB=postgres (docs/adr/0002). + # Optional so the standalone desktop build carries no Postgres driver; the CI matrix + # job compiles with the variable set and proves the migrations on both. + {:postgrex, ">= 0.0.0", optional: true}, {:phoenix_html, "~> 4.1"}, {:phoenix_live_reload, "~> 1.2", only: :dev}, {:phoenix_live_view, "~> 1.2.0"}, diff --git a/mix.lock b/mix.lock index fa93325..c1494a6 100644 --- a/mix.lock +++ b/mix.lock @@ -52,6 +52,7 @@ "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "plug": {:hex, :plug, "1.20.3", "56c480c633ec2ce10140e236e15233bf576e1d323887d7c96711bd02ab5160db", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "be266aee1b8536ef6409d58cf39a3121319f0ec47cfa1b24024485aa0e76ad76"}, "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, + "postgrex": {:hex, :postgrex, "0.22.4", "d271f595dfd25230b6398354e19d17bb5e2d20130fd2d9bdca7e15f125d43552", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "4aae45a2d60e35b04eea2602440be152fae332901f1fc7a60fc7cb7f0f9a9c5a"}, "req": {:hex, :req, "0.7.4", "23e9ffec17de032a46a4b15ed65c09793893bf4a7c680f4bbf6227fce6bdf74d", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b192d63253e8dcc6221ef992ea9ebef7d3555166e8423aa5b553e86bc3c69a2"}, "rewrite": {:hex, :rewrite, "1.3.0", "67448ba7975690b35ba7e7f35717efcce317dbd5963cb0577aa7325c1923121a", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "d111ac7ff3a58a802ef4f193bbd1831e00a9c57b33276e5068e8390a212714a5"}, "sobelow": {:hex, :sobelow, "0.15.0", "b067d7f8522a9d758fa89cb2bfcbab7ad72c45a0993cb958c989c6fd956fdd56", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "24a800e2d7fa8c3bd21561b6ad8ad4745ed726a09fd606598981d9048708da98"}, diff --git a/priv/repo/migrations/20260920120000_create_personas.exs b/priv/repo/migrations/20260920120000_create_personas.exs new file mode 100644 index 0000000..44c90a7 --- /dev/null +++ b/priv/repo/migrations/20260920120000_create_personas.exs @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Repo.Migrations.CreatePersonas do + @moduledoc """ + Slice 010. The minimal persona row docs/05 names: name, soul, model, settings. Slice 030 fills + it in. Ids are UUIDv7 strings minted by `Trinity.UUID` (`:binary_id` on both adapters). + """ + use Ecto.Migration + + def change do + create table(:personas, primary_key: false) do + add :id, :binary_id, primary_key: true + add :name, :string, null: false + add :soul, :text + add :model, :string + add :settings, :map, null: false, default: %{} + timestamps(type: :utc_datetime_usec) + end + + create unique_index(:personas, [:name]) + end +end diff --git a/priv/repo/migrations/20260920120100_create_sessions.exs b/priv/repo/migrations/20260920120100_create_sessions.exs new file mode 100644 index 0000000..b4c6ab4 --- /dev/null +++ b/priv/repo/migrations/20260920120100_create_sessions.exs @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Repo.Migrations.CreateSessions do + @moduledoc "Slice 010. The sessions table per docs/05; `parent_id` is the lineage slice 023 uses." + use Ecto.Migration + + def change do + create table(:sessions, primary_key: false) do + add :id, :binary_id, primary_key: true + add :title, :string + add :persona_id, references(:personas, type: :binary_id, on_delete: :restrict), null: false + add :parent_id, references(:sessions, type: :binary_id, on_delete: :nilify_all) + add :origin, :string, null: false, default: "desktop" + add :origin_ref, :map, null: false, default: %{} + add :status, :string, null: false, default: "active" + add :model, :string + add :token_usage, :map, null: false, default: %{} + add :last_activity_at, :utc_datetime_usec + timestamps(type: :utc_datetime_usec) + end + + create index(:sessions, [:persona_id]) + create index(:sessions, [:parent_id]) + create index(:sessions, [:status, :last_activity_at]) + end +end diff --git a/priv/repo/migrations/20260920120200_create_messages.exs b/priv/repo/migrations/20260920120200_create_messages.exs new file mode 100644 index 0000000..c94b2a2 --- /dev/null +++ b/priv/repo/migrations/20260920120200_create_messages.exs @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Repo.Migrations.CreateMessages do + @moduledoc """ + Slice 010. Append-only messages with a gapless `seq` per session. The unique index on + `(session_id, seq)` is the property the stress test proves; `append_message/2` assigns + `seq` inside one transaction and this index refuses a duplicate on either adapter. + """ + use Ecto.Migration + + def change do + create table(:messages, primary_key: false) do + add :id, :binary_id, primary_key: true + + add :session_id, references(:sessions, type: :binary_id, on_delete: :delete_all), + null: false + + add :seq, :integer, null: false + add :role, :string, null: false + add :content, :text, null: false + add :parts, :map, null: false, default: %{} + add :tool_call_id, :string + add :usage, :map + add :provider_meta, :map, null: false, default: %{} + timestamps(type: :utc_datetime_usec) + end + + create unique_index(:messages, [:session_id, :seq]) + end +end diff --git a/scripts/plan_check.sh b/scripts/plan_check.sh index b0a15d7..a38e841 100755 --- a/scripts/plan_check.sh +++ b/scripts/plan_check.sh @@ -202,6 +202,21 @@ case "$branch" in ;; esac +section "12. Every workflow and Dependabot file parses as YAML" +# Found at slice 010 G1, from PR #7: an em dash replaced by a bare colon inside a step name made +# .github/workflows/package.yml unparseable. GitHub reported a run named after the file path with +# zero jobs, Dependabot's updater failed on the same file, and the gate stayed green because +# nothing in it reads those files. The ubuntu runner and the developer machine both carry the +# Python YAML module; if it is absent the rule says so and fails rather than passing quietly. +if ! python3 -c 'import yaml' >/dev/null 2>&1; then + report "FAIL rule 12: python3 with the yaml module is required to parse the workflow files, and it is absent" +else + for f in $(git ls-files '.github/*.yml' '.github/*.yaml' '.github/**/*.yml' '.github/**/*.yaml' | sort -u); do + err=$(python3 -c 'import sys, yaml; yaml.safe_load(open(sys.argv[1]))' "$f" 2>&1 >/dev/null | tail -1) + [ -n "$err" ] && report "FAIL $f: not parseable as YAML: $err" + done +fi + printf '\n' if [ "$fail" -eq 0 ]; then echo "plan_check: PASS"; else echo "plan_check: FAIL"; fi exit "$fail" diff --git a/scripts/stress_010.exs b/scripts/stress_010.exs new file mode 100644 index 0000000..444b84a --- /dev/null +++ b/scripts/stress_010.exs @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Slice 010 AC2, outside the test sandbox, so the writes reach the WAL and the -wal size and +# the throughput are real numbers. Run against the dev database (it is reset first): +# +# MIX_ENV=dev mix ecto.reset && MIX_ENV=dev mix run scripts/stress_010.exs +# +# Prints: appends, wall time, appends per second, integrity_check, -wal size, per-session +# gapless verdict. Nothing here is a test assertion; the numbers go in PROOF.md by hand. +alias Trinity.Sessions + +writers = 20 +per_writer = 200 +session_count = 5 + +{:ok, persona} = Sessions.create_persona(%{name: "stress-#{System.unique_integer([:positive])}"}) +ids = for _ <- 1..session_count, do: elem(Sessions.create_session(%{persona_id: persona.id}), 1).id + +{micros, results} = + :timer.tc(fn -> + 1..writers + |> Task.async_stream( + fn w -> + for i <- 1..per_writer do + Sessions.append_message(Enum.at(ids, rem(w + i, session_count)), %{role: "user", content: "w#{w} i#{i}"}) + end + end, + max_concurrency: writers, + timeout: 600_000, + ordered: false + ) + |> Enum.flat_map(fn {:ok, list} -> list end) + end) + +errors = Enum.reject(results, &match?({:ok, _}, &1)) +gapless = + Enum.all?(ids, fn id -> + seqs = Sessions.seqs(id) + seqs == Enum.to_list(1..length(seqs)//1) + end) +%{rows: [[integrity]]} = Trinity.Repo.query!("PRAGMA integrity_check") +db = Trinity.Repo.config()[:database] +wal = if File.exists?(db <> "-wal"), do: File.stat!(db <> "-wal").size, else: 0 + +%{rows: [[sqlite_version]]} = Trinity.Repo.query!("select sqlite_version()") +per_s = Float.round(length(results) / (micros / 1_000_000), 1) + +IO.puts( + "stress_010: appends=#{length(results)} errors=#{length(errors)} wall_ms=#{div(micros, 1000)} " <> + "appends_per_s=#{per_s} integrity=#{integrity} wal_bytes=#{wal} gapless=#{gapless} sqlite=#{sqlite_version}" +) diff --git a/slices/010-core-domain-persistence/NOTES.md b/slices/010-core-domain-persistence/NOTES.md new file mode 100644 index 0000000..620ee80 --- /dev/null +++ b/slices/010-core-domain-persistence/NOTES.md @@ -0,0 +1,184 @@ +# Slice 010: NOTES + +## G1 plan, 2026-09-20 + +Tree at `68000bc` on `main`; branch `slice/010-core-domain-persistence`; ROADMAP row set to `in_progress` in +this commit (docs/04 lifecycle, plan_check rule 11). Machine: Ryzen AI Max+ 395, OTP 28.5.0.5, Elixir 1.20.4, +ecto_sqlite3 and exqlite as locked. Every line below names its test or command; the order is the build order. + +1. **Gate follow-up first, its own commit:** `plan_check` rule 12 parses every `.github/**/*.yml` and + `.github/dependabot.yml` with the Python YAML module the ubuntu runner and this machine both carry + (pyyaml 6.0.1 here); red demonstrated on a planted bare colon (the defect PR #7 fixed), then green. +2. Repo config: `journal_mode: :wal`, `synchronous: :normal`, `busy_timeout`, `wal_auto_check_point` named in + config; write pool of size 1 (`Trinity.Repo`) and a read pool; `Trinity.Repo.Receipts` declared with the same + adapter and not started (the slot the 2026-09-20 amendment reserves). Test: the pragmas read back from a + connection (`PRAGMA journal_mode`, `PRAGMA synchronous`). +3. `TRINITY_DB=postgres` branch in `config/runtime.exs` with `postgrex` as an optional dependency; a CI matrix + job with a Postgres service running `mix ecto.reset` and the suite. Test on both: the migrations apply. +4. Data-dir lock: `Trinity.DataDir.Lock` writes `/LOCK` carrying pid, mode (`desktop | headless`) and + a monotonic token, taken with `:file.open` exclusive create plus a liveness check on the recorded pid; a + second boot against a held dir refuses to start naming holder pid and mode and touches no database file. + Test: two applications in one VM cannot both own the dir; the refusal message names the holder (AC6). +5. Migrations: `personas` (name unique, soul, model, settings), `sessions`, `messages` per docs/05, UUIDv7 ids + from a small generator in `Trinity.UUID` (no new dependency; tested against RFC 9562's version and variant + bits and for monotonic ordering within a millisecond). +6. `Trinity.Sessions.Store` (queries) behind `Trinity.Sessions` (`create_session/1`, `get_session/1`, + `list_sessions/1`, `append_message/2`, `history/2`, `archive/1`); `boundary` on `Trinity.Sessions` with + `exports: [Trinity.Sessions]` so `Store` is internal. Test for AC5: a module under `test/support` that calls + `Store` from `TrinityWeb` fails `mix compile --warnings-as-errors` (output pasted). +7. `append_message/2`: one transaction reading `max(seq)` and inserting; on Postgres `SELECT ... FOR UPDATE` on + the session row. Changeset refuses unknown roles and empty content (AC3). `history/2` ordered by `seq` with + `limit` and `offset` (AC4). +8. Factories in `test/support/factory.ex` (no `ex_machina`; plain functions). +9. Stress test (AC2): 20 processes, 200 appends each, 5 sessions; gapless `seq` per session; no `SQLITE_BUSY` + surfaced; `PRAGMA integrity_check` returns `ok`; run on both adapters; the `-wal` file size reported after + the run (the 2026-09-20 risk line). +10. `mix ecto.reset` on SQLite and on Postgres in CI (AC1), log excerpts in PROOF.md. +11. Coverage line and `mix gate` (AC7); docs/05 synced with any column the migrations add. +12. PROOF.md from the template; ROADMAP row to `done`; final commit and tag per CLAUDE.md section 4, through a + pull request as the ruleset requires. + +Manual verification queue: none. Every criterion is `[auto]`. + +Deviations from SLICE.md, stated before building: line 3 adds `postgrex` as `optional: true` rather than a +default dependency, so the standalone desktop build carries no Postgres driver; line 5 writes the UUIDv7 +generator in-tree rather than adding `uniq`, and proposes nothing new for VERSIONS.md. + +## Line 1, 2026-09-20: plan_check rule 12, red then green + +Planted the exact defect PR #7 fixed (the quotes removed from the step name at +`.github/workflows/package.yml:173`, working tree only, reverted after): + +``` +$ ./scripts/plan_check.sh | grep -E '== 12|FAIL .github|plan_check:' +== 12. Every workflow and Dependabot file parses as YAML == +FAIL .github/workflows/package.yml: not parseable as YAML: in ".github/workflows/package.yml", line 173, column 25 +plan_check: FAIL +exit=1 + +$ git checkout -- .github/workflows/package.yml && ./scripts/plan_check.sh | grep -E '== 12|FAIL|plan_check:' +== 12. Every workflow and Dependabot file parses as YAML == +plan_check: PASS +exit=0 +``` + +The population is `git ls-files '.github/*.yml' '.github/*.yaml' '.github/**/*.yml' '.github/**/*.yaml'`, three +files today. The parser is Python's yaml module (6.0.1 here, present on the ubuntu runner); its absence is a +FAIL, not a skip. + +## Line 2, 2026-09-20: the one-connection pool, named pragmas, the receipts slot + +`config/config.exs` names every pragma (`journal_mode :wal`, `synchronous :normal`, `foreign_keys :on`, +`busy_timeout 5000`, `cache_size -64000`, `wal_auto_check_point 1000`) and sets `pool_size: 1` for SQLite in +every environment; dev and test no longer override the pool. The adapter is chosen at compile time from +`TRINITY_DB` (default sqlite), because `use Ecto.Repo` takes the adapter as a literal; the config says so. +`Trinity.Repo.Receipts` is declared with the same adapter, not started, not in `:ecto_repos`. + +`test/trinity/repo_config_test.exs` reads the pragmas back from a live connection: `wal`, `1` (normal), `1` +(foreign keys), `1000` (autocheckpoint pages) all read back as configured. **One does not:** `PRAGMA +busy_timeout` reads `0`, because exqlite installs its own busy handler and applies the timeout on its side of +it (`deps/exqlite/lib/exqlite/connection.ex`, the comment above `set_busy_timeout/2`, at 0.40.0); the pragma +would destroy that handler, so the driver never sets it. The test asserts the configured value and the `0` +together, with the reason. Contention itself is the stress test's job (line 9). + +A read pool is not added at this slice: WAL readers do not block the writer, but a one-connection pool +serialises reads behind writes in the same process queue. Measured need arrives with 012 and 013, and the +Repo layout admits a read-only replica repo then without moving anything. Recorded under Follow-ups. + +``` +$ mix test test/trinity/repo_config_test.exs โ†’ 7 passed +$ mix compile --warnings-as-errors --force โ†’ Generated trinity app +$ mix test โ†’ 79 passed +$ mix credo --strict โ†’ found no issues +``` + +## Follow-ups +- A read-only replica repo over the same file, when 012 or 013 measures read latency behind the single writer. + +## Line 3, 2026-09-20: the Postgres branch, proven in CI on this branch + +Run 35509095238 on `5a9ec9b`, job `postgres` (postgres:17 service, `TRINITY_DB=postgres`, `DATABASE_URL` from +the job env), lines from its log: + +``` +mix run -e 'Ecto.Adapters.Postgres = Trinity.Repo.__adapter__()' (exit 0: the compiled adapter is Postgres) +The database for Trinity.Repo has been dropped +The database for Trinity.Repo has been created +Excluding tags: [:sqlite] +Result: 72 passed, 7 excluded +``` + +The seven excluded are the SQLite pragma read-backs (`@moduletag :sqlite`), excluded by tag on that job and +never skipped. Job `gate` on the same run: success. The job is not yet a required check on the ruleset; it is +added after its first green run on `main`, which this slice's merge will be. + +No local Postgres was used: the machine has a server on 5432 whose password this seat does not have and did +not guess. AC1 names CI as the proof for this half. + +## Line 4, 2026-09-20: the data-dir lock + +`Trinity.DataDir.Lock`: `/LOCK` created with `:exclusive`, carrying OS pid, mode, a per-boot token and +the time; a supervised child placed before `Trinity.Repo`; refusal names the holder's pid and mode and touches +no database file. Liveness through `/proc/` on Linux; elsewhere a held file is treated as held (the safe +direction) and the message names the pid to remove by hand. A malformed file is held, never taken over. +Test env points the lock at a temporary directory keyed by `MIX_TEST_PARTITION`. + +``` +$ mix test test/trinity/data_dir โ†’ 10 passed +``` + +One thing seen and left as it is: `mix test` halts the VM without running `terminate/2`, so the test lock +file survives a run and is taken over as stale at the next (its pid is dead). Correct behaviour, and the +reason the stale path has a test. + +## Lines 5 to 9, 2026-09-20: migrations, UUIDv7, the Sessions context, factories, the stress test + +**UUIDv7** (`Trinity.UUID`): an `Ecto.Type` of underlying type `:uuid` delegating cast, dump and load to +`Ecto.UUID`, with `autogenerate/0` minting v7. Ecto refuses `autogenerate: {m, f, a}` on id types, which is how +the type shape was arrived at; the schemas declare `@primary_key {:id, Trinity.UUID, autogenerate: true}` and +`@foreign_key_type Trinity.UUID`. `rand_a` carries the low twelve bits of `:erlang.unique_integer([:monotonic])`, +so ids minted within one millisecond sort in mint order up to 4096 per millisecond; `rand_b` is random on every +call so uniqueness never depends on the counter. Tests: form, version, variant, timestamp, 1000 in order, 5000 +from 50 processes unique. + +**Migrations**: `personas` (name unique), `sessions`, `messages` with the unique index on `(session_id, seq)`, +`:binary_id` columns (text on SQLite, uuid on Postgres), `utc_datetime_usec` timestamps. `MIX_ENV=test mix +ecto.reset` and `MIX_ENV=dev mix ecto.reset` both apply them; the Postgres half is CI's (line 3's job runs +`ecto.reset` on every push). + +**`append_message/2`** is one transaction: the session row is read (and on a Postgres build locked `FOR +UPDATE`), `max(seq) + 1` is assigned, the row inserted, the session touched. The lock is compiled in or out +from the adapter at build time rather than branched at runtime: the type checker refuses a runtime branch on a +compile-time constant (`Repo.__adapter__()` is a literal per build), so each build carries one definition and +the CI matrix compiles both. `seq` is never cast from the caller. + +**Boundary (AC5)**: `Trinity.Sessions` is a sub-boundary exporting only its API module; `Trinity` exports the +sub-boundary so `TrinityWeb` may call the context and nothing inside it. Test: a `TrinityWeb` module calling +`Store` is written to `lib/trinity_web/`, `mix compile --warnings-as-errors` fails naming the reference, the same +module calling `Sessions` compiles, the probe is removed. The first version of that test failed on the lawful +half: a top-level boundary cannot list a nested one as a dep, and the fix was exporting the sub-boundary from +its parent, which is the documented shape. + +**Stress (AC2)**, inside the sandbox: 20 writers, 200 appends each, 5 sessions, gapless, integrity `ok`, 0.6 s. +The `-wal` size printed there is 0 and is not a measurement: the sandbox rolls the test back and nothing reaches +the WAL. Outside the sandbox (`scripts/stress_010.exs` on the dev database, this machine, SQLite 3.53.4): + +``` +$ MIX_ENV=dev mix ecto.reset && MIX_ENV=dev mix run scripts/stress_010.exs +stress_010: appends=4000 errors=0 wall_ms=1327 appends_per_s=3012.1 integrity=ok wal_bytes=4152992 gapless=true sqlite=3.53.4 +$ MIX_ENV=dev mix run scripts/stress_010.exs (second run, same database) +stress_010: appends=4000 errors=0 wall_ms=1169 appends_per_s=3420.3 integrity=ok wal_bytes=4152992 gapless=true sqlite=3.53.4 +``` + +4,152,992 bytes is 1,014 pages of 4 KiB: the WAL sits at the 1,000-page `wal_auto_check_point` and the +checkpoint keeps it there. No stall was seen. The threshold stays at 1000 pages; a reason to raise it would be +a measured stall, and there is none. About 3,000 appends per second with one transaction per append is the +number 024's checkpoint window starts from. + +``` +$ mix compile --warnings-as-errors --force โ†’ Generated trinity app +$ mix test โ†’ 105 passed +$ mix credo --strict โ†’ found no issues +$ ./scripts/plan_check.sh โ†’ PASS +$ mix trinity.names โ†’ OK over 199 tracked files +``` diff --git a/slices/010-core-domain-persistence/PROOF.md b/slices/010-core-domain-persistence/PROOF.md new file mode 100644 index 0000000..7d99aca --- /dev/null +++ b/slices/010-core-domain-persistence/PROOF.md @@ -0,0 +1,145 @@ +# Proof for slice 010: Core domain + persistence + +Agent: Trinity ยท Coding Agent ยท Date: 2026-09-20 ยท Branch: slice/010-core-domain-persistence ยท Final commit: `332e881` (the PROOF sha line was filled in by the commit after it, which is the only way a file can name the commit that carries it) + +## Summary +The Repo is one writer with every pragma named; `Trinity.Repo.Receipts` is a declared, unstarted slot for slice +024's own file. The adapter is chosen at compile time (`TRINITY_DB`), and a Postgres job in CI runs the +migrations and the suite on postgres:17. A data-dir lock before the Repo makes one node per data directory a +refusal with a named holder rather than a hope. UUIDv7 ids are minted in-tree by an `Ecto.Type`. Three +migrations, the `Trinity.Sessions` context over an internal `Store`, gapless `seq` inside one transaction, +factories, and a stress test run inside the sandbox and again outside it for the real WAL number. Hard parts, +all in NOTES.md: the type checker refusing a runtime branch on the compile-time adapter; a sub-boundary that a +top-level boundary cannot list as a dep; mix's one-second mtime resolution defeating the boundary probe twice. +Deferred: a read-only replica repo (NOTES.md Follow-ups). + +## Gate +``` +$ mix gate (this machine, OTP 28.5.0.5, Elixir 1.20.4, under a 32 GiB cgroup) +Result: 105 passed +trinity.coverage: 010 44.88% vs 001 30.37%: OK +plan_check: PASS +exit=0 +``` + +## Tests +``` +$ mix test --cover +Result: 105 passed +| 44.88% | Total | +``` +`coverage.tsv` row: `010 44.88 45ba4f0 2026-09-20`. Up 14.51 points from 001's 30.37%; `trinity.coverage: 010 +44.88% vs 001 30.37%: OK`. + +## Acceptance criteria evidence + +### AC1: `mix ecto.reset` works on SQLite; `TRINITY_DB=postgres mix ecto.reset` works in CI (log excerpt) +SQLite, this machine: +``` +$ MIX_ENV=test mix ecto.reset +The database for Trinity.Repo has been dropped +The database for Trinity.Repo has been created +== Running 20260920120000 Trinity.Repo.Migrations.CreatePersonas.change/0 forward +== Migrated 20260920120000 in 0.0s +== Running 20260920120100 Trinity.Repo.Migrations.CreateSessions.change/0 forward +== Migrated 20260920120100 in 0.0s +== Running 20260920120200 Trinity.Repo.Migrations.CreateMessages.change/0 forward +== Migrated 20260920120200 in 0.0s +``` +Postgres, CI job `postgres` on run 35509095238 (branch head `5a9ec9b`, postgres:17 service): +``` +mix run -e 'Ecto.Adapters.Postgres = Trinity.Repo.__adapter__()' (exit 0) +The database for Trinity.Repo has been dropped +The database for Trinity.Repo has been created +Excluding tags: [:sqlite] +Result: 72 passed, 7 excluded +``` +That run predates the migrations; the job runs `mix ecto.reset` on every push, and the run on the closing +commit of this branch carries the three migrations (its id is in the pull request's checks). + +### AC2: stress test passes on both adapters: gapless `seq` per session under concurrency; `integrity_check` ok +Inside the sandbox (`test/trinity/sessions_stress_test.exs`, 20 writers, 200 appends each, 5 sessions): +``` +$ mix test test/trinity/sessions_stress_test.exs +stress: -wal size after 4000 appends: 0 bytes (the sandbox rolls back; not a measurement) +Result: 1 passed +``` +Outside the sandbox, so the WAL is real (`scripts/stress_010.exs`, dev database, SQLite 3.53.4): +``` +$ MIX_ENV=dev mix ecto.reset && MIX_ENV=dev mix run scripts/stress_010.exs +stress_010: appends=4000 errors=0 wall_ms=1327 appends_per_s=3012.1 integrity=ok wal_bytes=4152992 gapless=true sqlite=3.53.4 +$ MIX_ENV=dev mix run scripts/stress_010.exs +stress_010: appends=4000 errors=0 wall_ms=1169 appends_per_s=3420.3 integrity=ok wal_bytes=4152992 gapless=true sqlite=3.53.4 +``` +4,152,992 bytes is 1,014 pages of 4 KiB: the WAL sits at the 1,000-page autocheckpoint. On Postgres the same +test runs in the CI job with the pragma assertion excluded by adapter; gapless is the whole property there. + +### AC3: `append_message/2` rejects unknown roles and empty content with `{:error, %Ecto.Changeset{}}` +``` +$ mix test test/trinity/sessions_test.exs --trace +* test append_message/2 (AC3) rejects an unknown role with a changeset +* test append_message/2 (AC3) rejects empty and blank content with a changeset +* test append_message/2 (AC3) rejects a missing session by name +* test append_message/2 (AC3) assigns seq from 1 and never takes it from the caller +Result: 9 passed +``` + +### AC4: `history/2` returns messages in `seq` order and respects `limit` +``` +* test history/2 (AC4) returns messages in seq order and respects limit and offset (same run, passed) +``` + +### AC5: `boundary` prevents `TrinityWeb` from calling `Trinity.Sessions.Store` directly +`test/trinity/sessions_boundary_test.exs` writes a `TrinityWeb` module calling `Store` into `lib/trinity_web/`, +runs `mix compile --warnings-as-errors --force`, asserts a non-zero exit and this text in the output: +``` +warning: forbidden reference to Trinity.Sessions.Store + (module Trinity.Sessions.Store is not exported by its owner boundary Trinity.Sessions) + lib/trinity_web/zz_boundary_violation.ex:2 +``` +then writes the same module calling `Trinity.Sessions`, asserts exit 0 and no "forbidden reference", and +removes both probes. Four consecutive runs pass (`Result: 1 passed`, 2.7 s each). + +### AC6: a second instance against a held data dir refuses to start, names the holder's pid and mode, leaves the database untouched +``` +$ mix test test/trinity/data_dir --trace +* test the supervised child refuses to start against a held dir with pid and mode in the reason +* test a held dir has no database file created by the refused instance +* test a second acquire in the same VM is refused, naming the live holder +* test the application holds the configured directory +* test the lock child starts before the Repo +Result: 10 passed +``` +The refusal reads: ` is held by OS pid in desktop mode; refusing to start and touching no database +file`. The child sits before `Trinity.Repo` in the supervisor, asserted by position. + +### AC7: gate green; coverage line reported +Above: `mix gate` exit 0; `trinity.coverage: 010 44.88% vs 001 30.37%: OK`. + +## Manual verification for the reviewer +None. Every criterion is `[auto]`. The Postgres half of AC1 and AC2 is CI's; the reviewer reads the `postgres` +job on the pull request. + +## Deviations from SLICE.md +See NOTES.md G1 plan: `postgrex` optional rather than default; an in-tree UUIDv7 generator instead of a new +dependency. One more, found while building: the Postgres row lock is compiled in or out from the adapter rather +than branched at runtime, because the type checker refuses a runtime branch on a compile-time constant. + +## Versions touched +`VERSIONS.md` updated: yes. The `postgrex + pgvector` row split into `postgrex` (in `mix.lock`, optional) and +`pgvector` (not yet a dependency). `mix hex.outdated` not run; no pin moved. + +## Git +``` +$ git log --oneline main..HEAD +32d745d test(s010): the gate is green: coverage row, four sobelow skips with reasons, the boundary probe compiles forced +45ba4f0 feat(s010): migrations, UUIDv7 ids, the Sessions context with gapless seq, the stress test +a45db66 fix(s010): Trinity exports Paths; the boundary refused the application's data-dir call +34842c0 refactor(s010): the lock's contention branch is its own function; credo --strict was red on nesting depth +22937bf feat(s010): the data-dir lock, before the Repo, refusing a held directory by pid and mode +5a9ec9b feat(s010): the Postgres branch: postgrex optional, test config, a CI job with a Postgres service +0b38a68 feat(s010): one-connection write pool, every pragma named, the receipts repo slot +17850ce feat(s010): plan_check rule 12 parses every workflow and Dependabot file +7b2fb39 docs(s010): G1 plan, and the slice opens +332e881 feat(s010): complete slice 010 (core domain and persistence) +``` diff --git a/test/support/factory.ex b/test/support/factory.ex new file mode 100644 index 0000000..b3ce728 --- /dev/null +++ b/test/support/factory.ex @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Factory do + @moduledoc """ + Plain functions that insert rows for tests. Slice 010. No factory library: three functions + are cheaper to read than a DSL, and every attribute they set is visible here. + """ + alias Trinity.Sessions + + @spec persona!(map()) :: Sessions.Persona.t() + def persona!(attrs \\ %{}) do + name = Map.get(attrs, :name, "persona-#{System.unique_integer([:positive])}") + + {:ok, persona} = + Sessions.create_persona( + Map.merge(%{name: name, soul: "test soul", model: "fake:model"}, attrs) + ) + + persona + end + + @spec session!(map()) :: Sessions.Session.t() + def session!(attrs \\ %{}) do + attrs = Map.put_new_lazy(attrs, :persona_id, fn -> persona!().id end) + {:ok, session} = Sessions.create_session(attrs) + session + end + + @spec message!(String.t(), map()) :: Sessions.Message.t() + def message!(session_id, attrs \\ %{}) do + {:ok, message} = + Sessions.append_message(session_id, Map.merge(%{role: "user", content: "hello"}, attrs)) + + message + end +end diff --git a/test/trinity/data_dir/lock_boot_test.exs b/test/trinity/data_dir/lock_boot_test.exs new file mode 100644 index 0000000..4d6fdb2 --- /dev/null +++ b/test/trinity/data_dir/lock_boot_test.exs @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.DataDir.LockBootTest do + @moduledoc """ + Slice 010 AC6, the boot half: the running application holds its lock, the lock child sits + before the Repo in the supervision tree, and a second acquire of the application's own + directory is refused naming this VM's OS pid. + """ + use ExUnit.Case, async: false + + alias Trinity.DataDir.Lock + + test "the application holds the configured directory" do + dir = Application.fetch_env!(:trinity, Lock)[:dir] + assert {:ok, holder} = Lock.holder(dir) + assert holder.pid == String.to_integer(System.pid()) + assert holder.mode == :desktop + assert {:error, {:held, ^holder}} = Lock.acquire(dir, :headless) + end + + test "the lock child starts before the Repo" do + ids = + Supervisor.which_children(Trinity.Supervisor) |> Enum.map(&elem(&1, 0)) |> Enum.reverse() + + assert Enum.find_index(ids, &(&1 == Lock)) < Enum.find_index(ids, &(&1 == Trinity.Repo)) + end +end diff --git a/test/trinity/data_dir/lock_test.exs b/test/trinity/data_dir/lock_test.exs new file mode 100644 index 0000000..a8a15ac --- /dev/null +++ b/test/trinity/data_dir/lock_test.exs @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.DataDir.LockTest do + @moduledoc """ + Slice 010 AC6: a second instance against a held data dir refuses to start, names the + holder's pid and mode, and leaves the database untouched. + """ + use ExUnit.Case, async: true + + alias Trinity.DataDir.Lock + + setup do + dir = Path.join(System.tmp_dir!(), "trinity-lock-test-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf(dir) end) + {:ok, dir: dir} + end + + test "the first acquire wins and the file names this OS pid and the mode", %{dir: dir} do + assert {:ok, holder} = Lock.acquire(dir, :desktop) + assert holder.pid == String.to_integer(System.pid()) + assert holder.mode == :desktop + assert {:ok, ^holder} = Lock.holder(dir) + end + + test "a second acquire in the same VM is refused, naming the live holder", %{dir: dir} do + assert {:ok, holder} = Lock.acquire(dir, :desktop) + assert {:error, {:held, ^holder}} = Lock.acquire(dir, :headless) + end + + test "the supervised child refuses to start against a held dir with pid and mode in the reason", + %{dir: dir} do + assert {:ok, holder} = Lock.acquire(dir, :desktop) + Process.flag(:trap_exit, true) + + assert {:error, {:data_dir_held, message}} = + Lock.start_link( + dir: dir, + mode: :headless, + name: :"lock-test-#{System.unique_integer()}" + ) + + assert message =~ "OS pid #{holder.pid}" + assert message =~ "desktop mode" + assert message =~ "touching no database file" + end + + test "a held dir has no database file created by the refused instance", %{dir: dir} do + assert {:ok, _} = Lock.acquire(dir, :desktop) + Process.flag(:trap_exit, true) + + assert {:error, _} = + Lock.start_link( + dir: dir, + mode: :headless, + name: :"lock-test-#{System.unique_integer()}" + ) + + assert File.ls!(dir) == ["LOCK"] + end + + test "a stale file from a dead pid is taken over (linux liveness through /proc)", %{dir: dir} do + File.mkdir_p!(dir) + # 4194304 is above Linux's default pid_max, so no live process carries it. + File.write!( + Path.join(dir, "LOCK"), + "pid 4194304\nmode headless\ntoken stale\nat 2026-09-20T00:00:00Z\n" + ) + + case :os.type() do + {:unix, :linux} -> + assert {:ok, holder} = Lock.acquire(dir, :desktop) + assert holder.mode == :desktop + assert {:ok, ^holder} = Lock.holder(dir) + + _ -> + assert {:error, {:held, %{pid: 4_194_304}}} = Lock.acquire(dir, :desktop) + end + end + + test "a malformed file is treated as held, never taken over silently", %{dir: dir} do + File.mkdir_p!(dir) + File.write!(Path.join(dir, "LOCK"), "garbage") + assert {:error, {:held, %{mode: :unknown}}} = Lock.acquire(dir, :desktop) + assert File.read!(Path.join(dir, "LOCK")) == "garbage" + end + + test "release removes the file only for the token that wrote it", %{dir: dir} do + assert {:ok, holder} = Lock.acquire(dir, :desktop) + assert :ok = Lock.release(dir, %{holder | token: "someone-else"}) + assert File.exists?(Path.join(dir, "LOCK")) + assert :ok = Lock.release(dir, holder) + refute File.exists?(Path.join(dir, "LOCK")) + end + + test "the child releases on terminate", %{dir: dir} do + {:ok, pid} = + Lock.start_link(dir: dir, mode: :desktop, name: :"lock-test-#{System.unique_integer()}") + + assert File.exists?(Path.join(dir, "LOCK")) + :ok = GenServer.stop(pid) + refute File.exists?(Path.join(dir, "LOCK")) + end +end diff --git a/test/trinity/repo_config_test.exs b/test/trinity/repo_config_test.exs new file mode 100644 index 0000000..d5ba04b --- /dev/null +++ b/test/trinity/repo_config_test.exs @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.RepoConfigTest do + @moduledoc """ + Slice 010 line 2. The pragmas config/config.exs names are read back from a live connection, + so a default change upstream or a typo in the config key is a red here and not a silent + behaviour change. SQLite only; the Postgres matrix job excludes this module by tag. + """ + use Trinity.DataCase, async: false + + @moduletag :sqlite + + describe "the write pool" do + test "has exactly one connection" do + assert Trinity.Repo.config()[:pool_size] == 1 + end + end + + describe "pragmas read back from the connection" do + test "journal_mode is wal" do + assert %{rows: [["wal"]]} = Trinity.Repo.query!("PRAGMA journal_mode") + end + + test "synchronous is normal (1)" do + assert %{rows: [[1]]} = Trinity.Repo.query!("PRAGMA synchronous") + end + + test "foreign_keys are on (1)" do + assert %{rows: [[1]]} = Trinity.Repo.query!("PRAGMA foreign_keys") + end + + test "busy_timeout is the configured 5000 ms, and the pragma cannot show it" do + # exqlite installs its own busy handler through sqlite3_busy_handler and applies the + # timeout with sqlite3_busy_timeout on its side of that handler, so `PRAGMA busy_timeout` + # reads 0 on every connection it opens (deps/exqlite/lib/exqlite/connection.ex, the + # comment above set_busy_timeout/2, at the locked 0.40.0). The config value is the one + # the driver applies; contention itself is exercised by the slice 010 stress test. + assert Trinity.Repo.config()[:busy_timeout] == 5000 + assert %{rows: [[0]]} = Trinity.Repo.query!("PRAGMA busy_timeout") + end + + test "wal_autocheckpoint is the configured 1000 pages" do + assert %{rows: [[1000]]} = Trinity.Repo.query!("PRAGMA wal_autocheckpoint") + end + end + + describe "the receipts repo slot" do + test "is declared, uses the same adapter, and is not running" do + assert Trinity.Repo.Receipts.__adapter__() == Trinity.Repo.__adapter__() + assert Process.whereis(Trinity.Repo.Receipts) == nil + end + end +end diff --git a/test/trinity/sessions_boundary_test.exs b/test/trinity/sessions_boundary_test.exs new file mode 100644 index 0000000..922d860 --- /dev/null +++ b/test/trinity/sessions_boundary_test.exs @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.SessionsBoundaryTest do + @moduledoc """ + Slice 010 AC5: `boundary` prevents `TrinityWeb` from calling `Trinity.Sessions.Store` + directly. Demonstrated by compiling a violating module in a scratch copy of the project + with `--warnings-as-errors` and asserting the compile fails naming the reference; then + the same module with the call routed through `Trinity.Sessions` compiles. Both outputs + are captured. Slow (two full compiles), so it is tagged and runs in the gate only. + """ + use ExUnit.Case, async: false + + @moduletag :boundary_compile + @moduletag timeout: 300_000 + + @violation """ + defmodule TrinityWeb.Violation do + def history(id), do: Trinity.Sessions.Store.history(id, []) + end + """ + + @lawful """ + defmodule TrinityWeb.Lawful do + def history(id), do: Trinity.Sessions.history(id, []) + end + """ + + # Every compile is --force: mix decides what to recompile by mtime at one-second + # resolution, so a probe written, replaced or removed within a second of the last compile + # is invisible to an incremental build. Found by the gate, twice, in both directions (a + # lawful half compiled against the stale violation; a red half that never compiled the + # violation). Three forced compiles cost seconds and remove the class. + @red_path "lib/trinity_web/zz_boundary_violation.ex" + @green_path "lib/trinity_web/zz_boundary_lawful.ex" + + test "a TrinityWeb module calling Store fails the compile; the same call through Sessions passes" do + on_exit(fn -> Enum.each([@red_path, @green_path], &File.rm/1) end) + + File.write!(@red_path, @violation) + {out_red, code_red} = compile() + assert code_red != 0 + assert out_red =~ "forbidden reference to Trinity.Sessions.Store" + assert out_red =~ "zz_boundary_violation.ex" + + File.rm!(@red_path) + File.write!(@green_path, @lawful) + {out_green, code_green} = compile() + assert code_green == 0, out_green + refute out_green =~ "forbidden reference" + + File.rm!(@green_path) + {_, 0} = compile() + end + + defp compile do + System.cmd("mix", ["compile", "--warnings-as-errors", "--force"], + env: [{"MIX_ENV", "test"}], + stderr_to_stdout: true + ) + end +end diff --git a/test/trinity/sessions_stress_test.exs b/test/trinity/sessions_stress_test.exs new file mode 100644 index 0000000..61f2151 --- /dev/null +++ b/test/trinity/sessions_stress_test.exs @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.SessionsStressTest do + @moduledoc """ + Slice 010 AC2: 20 concurrent processes each append 200 messages across 5 sessions; every + session's `seq` is gapless; no SQLITE_BUSY surfaces; `PRAGMA integrity_check` is `ok`. On + the Postgres job the pragma assertion does not apply and the gapless property is the whole + test. The -wal size is printed after the run, for the checkpoint threshold in config. + """ + use Trinity.DataCase, async: false + + alias Trinity.Factory + alias Trinity.Sessions + + @writers 20 + @per_writer 200 + @sessions 5 + + @tag timeout: 300_000 + test "gapless seq per session under 20 concurrent writers" do + sessions = for _ <- 1..@sessions, do: Factory.session!() + ids = Enum.map(sessions, & &1.id) + + results = + 1..@writers + |> Task.async_stream( + fn w -> + for i <- 1..@per_writer do + id = Enum.at(ids, rem(w + i, @sessions)) + Sessions.append_message(id, %{role: "user", content: "w#{w} i#{i}"}) + end + end, + max_concurrency: @writers, + timeout: 240_000, + ordered: false + ) + |> Enum.flat_map(fn {:ok, list} -> list end) + + case Enum.reject(results, &match?({:ok, _}, &1)) do + [] -> :ok + [first | _] = errors -> flunk("#{length(errors)} appends failed; first: #{inspect(first)}") + end + + assert length(results) == @writers * @per_writer + + for id <- ids do + seqs = Sessions.seqs(id) + assert seqs == Enum.to_list(1..length(seqs)), "session #{id} has a gap or a duplicate" + end + + assert Enum.sum(Enum.map(ids, &Sessions.message_count/1)) == @writers * @per_writer + + if Trinity.Repo.__adapter__() == Ecto.Adapters.SQLite3 do + assert %{rows: [["ok"]]} = Trinity.Repo.query!("PRAGMA integrity_check") + wal = Trinity.Repo.config()[:database] <> "-wal" + size = if File.exists?(wal), do: File.stat!(wal).size, else: 0 + IO.puts("\nstress: -wal size after #{@writers * @per_writer} appends: #{size} bytes") + end + end +end diff --git a/test/trinity/sessions_test.exs b/test/trinity/sessions_test.exs new file mode 100644 index 0000000..2d80b32 --- /dev/null +++ b/test/trinity/sessions_test.exs @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.SessionsTest do + @moduledoc "Slice 010 AC3 and AC4, and the plain API around them." + use Trinity.DataCase, async: false + + alias Trinity.Factory + alias Trinity.Sessions + + describe "create_session/1" do + test "needs a persona and a known origin and status" do + persona = Factory.persona!() + + assert {:ok, session} = + Sessions.create_session(%{persona_id: persona.id, origin: "telegram"}) + + assert session.status == "active" + + assert {:error, cs} = + Sessions.create_session(%{persona_id: persona.id, origin: "carrier-pigeon"}) + + assert %{origin: ["is invalid"]} = errors_on(cs) + assert {:error, cs} = Sessions.create_session(%{origin: "desktop"}) + assert %{persona_id: ["can't be blank"]} = errors_on(cs) + end + + test "a persona name is unique" do + Factory.persona!(%{name: "twin"}) + assert {:error, cs} = Sessions.create_persona(%{name: "twin"}) + assert %{name: ["has already been taken"]} = errors_on(cs) + end + end + + describe "append_message/2 (AC3)" do + setup do + {:ok, session: Factory.session!()} + end + + test "rejects an unknown role with a changeset", %{session: s} do + assert {:error, %Ecto.Changeset{} = cs} = + Sessions.append_message(s.id, %{role: "oracle", content: "x"}) + + assert %{role: ["is invalid"]} = errors_on(cs) + assert Sessions.message_count(s.id) == 0 + end + + test "rejects empty and blank content with a changeset", %{session: s} do + assert {:error, %Ecto.Changeset{} = cs} = + Sessions.append_message(s.id, %{role: "user", content: ""}) + + assert %{content: [_ | _]} = errors_on(cs) + # Ecto's validate_required already treats whitespace-only strings as blank; the + # custom check exists for the case a later change relaxes that. + assert {:error, %Ecto.Changeset{} = cs} = + Sessions.append_message(s.id, %{role: "user", content: " "}) + + assert %{content: [message]} = errors_on(cs) + assert message =~ "blank" + assert Sessions.message_count(s.id) == 0 + end + + test "rejects a missing session by name", %{session: _} do + assert {:error, :no_session} = + Sessions.append_message(Trinity.UUID.generate(), %{role: "user", content: "x"}) + end + + test "assigns seq from 1 and never takes it from the caller", %{session: s} do + assert {:ok, m1} = Sessions.append_message(s.id, %{role: "user", content: "one", seq: 99}) + assert {:ok, m2} = Sessions.append_message(s.id, %{role: "assistant", content: "two"}) + assert {m1.seq, m2.seq} == {1, 2} + end + + test "touches the session's last_activity_at", %{session: s} do + assert s.last_activity_at == nil + Factory.message!(s.id) + assert %DateTime{} = Sessions.get_session(s.id).last_activity_at + end + end + + describe "history/2 (AC4)" do + test "returns messages in seq order and respects limit and offset" do + s = Factory.session!() + for i <- 1..5, do: Factory.message!(s.id, %{content: "m#{i}"}) + assert Enum.map(Sessions.history(s.id), & &1.seq) == [1, 2, 3, 4, 5] + assert Enum.map(Sessions.history(s.id, limit: 2), & &1.content) == ["m1", "m2"] + assert Enum.map(Sessions.history(s.id, limit: 2, offset: 3), & &1.content) == ["m4", "m5"] + end + end + + describe "list_sessions/1 and archive/1" do + test "lists most recently active first and filters by status" do + older = Factory.session!() + newer = Factory.session!() + Factory.message!(older.id) + Factory.message!(newer.id) + assert Enum.map(Sessions.list_sessions(), & &1.id) == [newer.id, older.id] + {:ok, archived} = Sessions.archive(older) + assert archived.status == "archived" + assert Enum.map(Sessions.list_sessions(status: "active"), & &1.id) == [newer.id] + end + end +end diff --git a/test/trinity/uuid_test.exs b/test/trinity/uuid_test.exs new file mode 100644 index 0000000..bedacff --- /dev/null +++ b/test/trinity/uuid_test.exs @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.UUIDTest do + use ExUnit.Case, async: true + + alias Trinity.UUID + + test "is 36 characters in the canonical form with version 7 and the RFC variant" do + uuid = UUID.generate() + assert String.length(uuid) == 36 + assert String.at(uuid, 14) == "7" + assert String.at(uuid, 19) in ["8", "9", "a", "b"] + + assert Regex.match?( + ~r/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + uuid + ) + end + + test "carries the current time to the millisecond" do + before = System.system_time(:millisecond) + {:ok, ms} = UUID.timestamp_ms(UUID.generate()) + after_ = System.system_time(:millisecond) + assert before <= ms and ms <= after_ + end + + test "1000 ids minted in a tight loop are unique and sort in mint order" do + ids = for _ <- 1..1000, do: UUID.generate() + assert length(Enum.uniq(ids)) == 1000 + assert ids == Enum.sort(ids) + end + + test "ids minted from many processes at once are unique" do + ids = + 1..50 + |> Task.async_stream(fn _ -> for _ <- 1..100, do: UUID.generate() end, max_concurrency: 50) + |> Enum.flat_map(fn {:ok, list} -> list end) + + assert length(Enum.uniq(ids)) == 5000 + end + + test "timestamp_ms refuses anything that is not a v7 uuid" do + assert :error = UUID.timestamp_ms("not-a-uuid") + assert :error = UUID.timestamp_ms("123e4567-e89b-12d3-a456-426614174000") + end +end