Slice 031: Session search (FTS5)

ROADMAP.md

@@ -45,7 +45,7 @@ standards register names the rows that ask for them.
 | 025 | Encryption at rest, and the key-custody seam | 2 Tools | M | 010, 024 | planned |
 | 026 | Store-and-forward receipts for disconnected operation | 2 Tools | L | 024 | planned |
 | 030 | Persona (SOUL) + always-on memory tier | 3 Memory | M | 012 | planned |
-| 031 | Session search (SQLite FTS5) | 3 Memory | S | 010 | planned |
+| 031 | Session search (SQLite FTS5) | 3 Memory | S | 010 | done |
 | 032 | Embeddings + semantic memory + hybrid retrieval | 3 Memory | L | 031 | planned |
 | 033 | Project context: AGENTS.md | 3 Memory | S | 030, 022 | planned |
 | 034 | Export, import, restore | 3 Memory | S | 030, 031 | planned |

config/config.exs

@@ -71,12 +71,16 @@ config :trinity, :tools,
     Trinity.Tools.FS.Grep,
     Trinity.Tools.Web.Fetch,
     Trinity.Tools.Web.Search,
+    # Slice 031: full-text search over past messages.
+    Trinity.Tools.SessionSearch,
     Trinity.Tools.Shell.Run
   ],
   toolsets: %{
     fs: ["fs_read", "fs_write", "fs_edit", "fs_list", "fs_glob", "fs_grep"],
     web: ["web_fetch", "web_search"],
-    shell: ["shell"]
+    shell: ["shell"],
+    # Slice 031: search over past conversations.
+    memory: ["session_search"]
   }
 
 # Slice 022: the filesystem roots beside the data directory (always a root) and the session's

config/test.exs

@@ -25,13 +25,17 @@ config :trinity, :tools,
     Trinity.Tools.FS.Grep,
     Trinity.Tools.Web.Fetch,
     Trinity.Tools.Web.Search,
+    # Slice 031: full-text search over past messages.
+    Trinity.Tools.SessionSearch,
     Trinity.Tools.Shell.Run
   ],
   toolsets: %{
     core: ["echo", "sleep", "crash", "big", "write_note"],
     fs: ["fs_read", "fs_write", "fs_edit", "fs_list", "fs_glob", "fs_grep"],
     web: ["web_fetch", "web_search"],
-    shell: ["shell"]
+    shell: ["shell"],
+    # Slice 031: search over past conversations.
+    memory: ["session_search"]
   },
   timeout_ms: 2_000
 

coverage.tsv

@@ -11,3 +11,4 @@ slice_id	percent	sha	date
 023	75.39	5989b73	2026-09-20
 003	75.39	39518c2	2026-09-20
 024	76.55	f977b84	2026-09-20
+031	76.96	866dc3f	2026-09-20

docs/01-architecture.md

@@ -74,7 +74,7 @@ without anything failing.
 |---|---|---|
 | `Trinity.Sessions` | Session process, turn loop, message log | LLM, Tools, **Effects**, Permissions, Memory, Skills, Repo, PubSub |
 | `Trinity.LLM` | Provider behaviour, req_llm adapter, model registry, streaming, usage | Repo (usage), Telemetry |
-| `Trinity.Tools` | Tool behaviour, registry, execution runtime, core tools, and (as built at 024) the compile-time effect catalog `Trinity.Tools.Catalog`, because the registry reads it and Effects depends on Tools | Permissions, Sandbox, Repo |
+| `Trinity.Tools` | Tool behaviour, registry, execution runtime, core tools, and (as built at 024) the compile-time effect catalog `Trinity.Tools.Catalog`, because the registry reads it and Effects depends on Tools | Permissions, Sandbox, Repo, **Memory** (as built at 031: `session_search` reads the index; Memory never depends on Tools) |
 | `Trinity.Permissions` | Policy, tier/1 (name-only), fingerprint-bound approvals, override adjudication | Repo, PubSub |
 | `Trinity.Effects` | The membrane; the runner in force (`Effects.Runner`, the executor `Tools.Runner` takes as a function); decision and query receipts; the boot receipt | **Tools**, Permissions, Authority, Receipts, Repo |
 | `Trinity.Authority` | Behaviour; `Local` implementation (the one caller of `execute/2` for effectful tools); selection at boot; `Staged` | Receipts, Repo |

docs/05-data-model.md

@@ -48,6 +48,13 @@ turn ended that way); the role, seq and session never change.
 ### messages_fts (Slice 031): SQLite `fts5(content, session_id UNINDEXED, message_id UNINDEXED)`; on Postgres a
 `tsvector` generated column on `messages`.
 
+As built: SQLite `messages_fts` with `tokenize = 'porter unicode61'`, `rowid` equal to the message row's, kept
+by three triggers (`messages_fts_ai`, `_ad`, `_au` on `content`) and backfilled by the migration; rebuildable by
+`mix trinity.search.reindex`. Postgres: `messages.content_tsv` generated as `to_tsvector('english', content)` with
+the GIN index `messages_content_tsv_idx`; nothing to rebuild. Stemming is suffix-based on both: "running" meets
+"runs" at `run` and never "ran". `Trinity.Memory.Search.messages/2` binds the query as a parameter and quotes
+every term for FTS5, so operators are text.
+
 ### memories (Slice 030/032)
 | column | type | notes |
 |---|---|---|

lib/mix/tasks/trinity.search.reindex.ex

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+defmodule Mix.Tasks.Trinity.Search.Reindex do
+  @shortdoc "Rebuilds the message search index from the messages table"
+
+  @moduledoc """
+  Slice 031. On SQLite, empties `messages_fts` and refills it from `messages`, then optimises;
+  on Postgres the index is a generated column and there is nothing to rebuild, which the task
+  says. Prints the time.
+
+      mix trinity.search.reindex
+  """
+  use Boundary, classify_to: Trinity
+  use Mix.Task
+
+  @impl Mix.Task
+  def run(_argv) do
+    Mix.Task.run("app.start")
+    {us, {:ok, what}} = :timer.tc(fn -> Trinity.Memory.Search.reindex() end)
+
+    case what do
+      :rebuilt ->
+        Mix.shell().info("reindexed messages_fts in #{div(us, 1000)} ms")
+
+      :generated_column ->
+        Mix.shell().info("nothing to rebuild: content_tsv is a generated column")
+    end
+  end
+end

lib/trinity.ex

@@ -28,6 +28,7 @@ defmodule Trinity do
         LLM,
         Memory,
         Memory.Tokens,
+        Memory.Search,
         Tools,
         Permissions,
         Permissions.Approval,

lib/trinity/memory.ex

@@ -6,5 +6,5 @@ defmodule Trinity.Memory do
   to 032 add the tiers, the search and the semantic recall. It depends on the LLM and on the
   core, never on Sessions: the Session calls it and writes what it returns.
   """
-  use Boundary, deps: [Trinity, Trinity.LLM], exports: [Tokens, Compactor]
+  use Boundary, deps: [Trinity, Trinity.LLM], exports: [Tokens, Compactor, Search]
 end

lib/trinity/memory/search.ex

@@ -0,0 +1,165 @@
+# SPDX-FileCopyrightText: Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+defmodule Trinity.Memory.Search do
+  @moduledoc """
+  Full-text search over every message (slice 031). SQLite: the `messages_fts` FTS5 table,
+  porter stemming, `bm25()` order, `snippet()`; Postgres: the generated `content_tsv` column,
+  `plainto_tsquery('english')`, `ts_rank`, `ts_headline`. Same shape of hit either way, so the
+  tool and the page do not know which database they run on.
+
+  The query text is bound as a parameter and never spliced into SQL. On SQLite every term is
+  additionally quoted for FTS5, so a user's `"`, `*`, `-` or `OR` is text to find, not an
+  operator; the search is "all these words", stemmed, in any order. Stemming is a suffix
+  operation: "running" and "runs" meet at "run", and "ran" does not (NOTES.md, fact 2).
+
+  Schemaless queries on purpose: Memory depends on the core and LLM, never on Sessions
+  (docs/01), so the tables are named here and the ids are cast through `Trinity.UUID`.
+  """
+
+  import Ecto.Query
+
+  alias Trinity.Repo
+
+  @default_limit 20
+  @max_limit 100
+
+  # The adapter is fixed at compile time (config/config.exs, slice 010), so the two databases'
+  # query shapes are compiled in, not chosen at run time; the postgres job proves the other.
+  @adapter Application.compile_env(:trinity, :db_adapter, Ecto.Adapters.SQLite3)
+
+  @type hit :: %{
+          message_id: String.t(),
+          session_id: String.t(),
+          session_title: String.t() | nil,
+          seq: non_neg_integer(),
+          role: String.t(),
+          snippet: String.t(),
+          inserted_at: DateTime.t(),
+          rank: float()
+        }
+
+  @doc """
+  Ranked hits for `query`. Options: `limit:` (#{@default_limit}, at most #{@max_limit}), `role:`
+  (a message role), `persona_id:`, `since:` and `until:` (`DateTime`, on the message's
+  `inserted_at`). An empty or all-punctuation query is no hits.
+  """
+  @spec messages(String.t(), keyword()) :: [hit()]
+  def messages(query, opts \\ []) when is_binary(query) do
+    case terms(query) do
+      [] -> []
+      terms -> terms |> build(opts) |> Repo.all() |> Enum.map(&to_hit/1)
+    end
+  end
+
+  @doc "The words of a query, punctuation dropped; the population FTS5 and tsquery both receive."
+  @spec terms(String.t()) :: [String.t()]
+  def terms(query) do
+    query
+    |> String.split(~r/[^\p{L}\p{N}_']+/u, trim: true)
+    |> Enum.map(&String.replace(&1, "'", ""))
+    |> Enum.reject(&(&1 == ""))
+    |> Enum.take(32)
+  end
+
+  @doc "Rebuilds the index from `messages`; on Postgres the column is generated and this reports so."
+  @spec reindex() :: {:ok, :rebuilt | :generated_column}
+  if @adapter == Ecto.Adapters.SQLite3 do
+    def reindex do
+      Repo.transaction(fn ->
+        Repo.query!("DELETE FROM messages_fts")
+
+        Repo.query!(
+          "INSERT INTO messages_fts(rowid, content, session_id, message_id) SELECT rowid, content, session_id, id FROM messages"
+        )
+
+        Repo.query!("INSERT INTO messages_fts(messages_fts) VALUES('optimize')")
+      end)
+
+      {:ok, :rebuilt}
+    end
+  else
+    def reindex, do: {:ok, :generated_column}
+  end
+
+  defp build(terms, opts) do
+    limit = opts |> Keyword.get(:limit, @default_limit) |> min(@max_limit) |> max(1)
+
+    base()
+    |> match(terms)
+    |> filter(:role, opts[:role])
+    |> filter(:persona_id, opts[:persona_id])
+    |> filter(:since, opts[:since])
+    |> filter(:until, opts[:until])
+    |> limit(^limit)
+  end
+
+  defp base do
+    from(m in "messages",
+      join: s in "sessions",
+      on: s.id == m.session_id,
+      select: %{
+        message_id: type(m.id, Trinity.UUID),
+        session_id: type(m.session_id, Trinity.UUID),
+        session_title: s.title,
+        seq: m.seq,
+        role: m.role,
+        inserted_at: type(m.inserted_at, :utc_datetime_usec),
+        persona_id: type(s.persona_id, Trinity.UUID)
+      }
+    )
+  end
+
+  # SQLite: every term quoted for FTS5 (a double quote inside is doubled), joined by spaces,
+  # which FTS5 reads as AND; the whole string is one bound parameter.
+  if @adapter == Ecto.Adapters.SQLite3 do
+    defp match(query, terms) do
+      needle = Enum.map_join(terms, " ", &("\"" <> String.replace(&1, "\"", "\"\"") <> "\""))
+
+      from([m, s] in query,
+        join: f in "messages_fts",
+        on: f.rowid == m.rowid,
+        where: fragment("messages_fts MATCH ?", ^needle),
+        order_by: fragment("bm25(messages_fts)"),
+        select_merge: %{
+          snippet: fragment("snippet(messages_fts, 0, '[', ']', '…', 12)"),
+          rank: fragment("bm25(messages_fts)")
+        }
+      )
+    end
+  else
+    defp match(query, terms) do
+      needle = Enum.join(terms, " ")
+
+      from([m, s] in query,
+        where: fragment("? @@ plainto_tsquery('english', ?)", m.content_tsv, ^needle),
+        order_by: [
+          desc: fragment("ts_rank(?, plainto_tsquery('english', ?))", m.content_tsv, ^needle)
+        ],
+        select_merge: %{
+          snippet:
+            fragment(
+              "ts_headline('english', ?, plainto_tsquery('english', ?), 'StartSel=[, StopSel=], MaxWords=12, MinWords=6')",
+              m.content,
+              ^needle
+            ),
+          rank: fragment("ts_rank(?, plainto_tsquery('english', ?))", m.content_tsv, ^needle)
+        }
+      )
+    end
+  end
+
+  defp filter(query, _key, nil), do: query
+  defp filter(query, :role, role), do: from([m, s] in query, where: m.role == ^role)
+
+  defp filter(query, :persona_id, id),
+    do: from([m, s] in query, where: s.persona_id == type(^id, Trinity.UUID))
+
+  defp filter(query, :since, at), do: from([m, s] in query, where: m.inserted_at >= ^at)
+  defp filter(query, :until, at), do: from([m, s] in query, where: m.inserted_at <= ^at)
+
+  defp to_hit(row) do
+    row
+    |> Map.delete(:persona_id)
+    |> Map.update!(:rank, &(&1 * 1.0))
+  end
+end

lib/trinity/tools.ex

@@ -10,8 +10,9 @@ defmodule Trinity.Tools do
   comparison of a session's declared surfaces with the calls its turns made
   (`surface_diff/1`, docs/07: a non-empty diff is a finding).
   """
+  # Slice 031: Memory, for the core tools that read it (session_search; 032's recall follows).
   use Boundary,
-    deps: [Trinity, Trinity.Permissions],
+    deps: [Trinity, Trinity.Permissions, Trinity.Memory],
     exports: [Tool, Context, Result, Registry, Runner, Schema, Catalog]
 
   alias Trinity.Sessions.Message

lib/trinity/tools/session_search.ex

@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+defmodule Trinity.Tools.SessionSearch do
+  @moduledoc """
+  `session_search`: full-text search over every past message (slice 031, `Trinity.Memory.Search`).
+  A read: risk `:read`, effect `:none`, so it runs without asking and leaves a query receipt.
+  The hits are the user's own history and come back as text the model reads; they are
+  wrapped as untrusted all the same, because a past message may itself have carried
+  untrusted content (docs/07: a tool result re-entering the prompt is data, not command).
+  """
+  @behaviour Trinity.Tools.Tool
+
+  alias Trinity.Memory.Search
+  alias Trinity.Tools.{Context, Untrusted}
+
+  @default_limit 10
+  @max_limit 50
+
+  @impl true
+  def name, do: "session_search"
+
+  @impl true
+  def description,
+    do:
+      "Searches every past conversation for the words given (stemmed, any order). Returns at most `limit` hits, newest-ranked first, each as `session · when · role: …snippet…` with the session id, so a decision or a fact from an earlier session can be found and quoted."
+
+  @impl true
+  def schema,
+    do: %{
+      "type" => "object",
+      "properties" => %{
+        "query" => %{"type" => "string", "description" => "The words to find"},
+        "limit" => %{
+          "type" => "integer",
+          "minimum" => 1,
+          "maximum" => @max_limit,
+          "description" => "At most this many hits (default #{@default_limit})"
+        }
+      },
+      "required" => ["query"],
+      "additionalProperties" => false
+    }
+
+  @impl true
+  def risk, do: :read
+
+  @impl true
+  def effect, do: :none
+
+  @impl true
+  def execute(%{"query" => query} = args, %Context{}) do
+    limit = args |> Map.get("limit", @default_limit) |> min(@max_limit)
+    hits = Search.messages(query, limit: limit)
+
+    text =
+      case hits do
+        [] -> "No message matches #{inspect(query)}."
+        hits -> Enum.map_join(hits, "\n", &line/1)
+      end
+
+    meta = %{"query" => query, "hits" => length(hits), "limit" => limit}
+    {:ok, Untrusted.result(text, tool: name(), source_ref: "search:" <> query, meta: meta)}
+  end
+
+  defp line(hit) do
+    title = hit.session_title || "untitled"
+    when_ = Calendar.strftime(hit.inserted_at, "%Y-%m-%d %H:%M")
+    "#{title} (#{hit.session_id}) · #{when_} · #{hit.role}: #{hit.snippet}"
+  end
+end