diff --git a/ROADMAP.md b/ROADMAP.md index bab2523..67caa57 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -34,7 +34,7 @@ standards register names the rows that ask for them. | 002 | Supply chain, early: SBOM, build provenance, the TLS floor | 0 Foundation | S | 000 | planned | | 003 | FIPS build leg in CI, from source | 0 Foundation | M | 000 | planned | | 010 | Core domain + persistence (Ecto/SQLite, schemas, Repo owner) | 1 Core loop | M | 000 | approved | -| 011 | LLM provider layer (req_llm behind `Trinity.LLM` behaviour) | 1 Core loop | M | 010 | planned | +| 011 | LLM provider layer (req_llm behind `Trinity.LLM` behaviour) | 1 Core loop | M | 010 | done | | 012 | Session process + agent loop (gen_statem, DynamicSupervisor, rehydration) | 1 Core loop | L | 010, 011 | planned | | 013 | LiveView chat UI with streaming | 1 Core loop | M | 012 | planned | | 020 | Tool protocol + registry | 2 Tools | M | 012 | planned | diff --git a/VERSIONS.md b/VERSIONS.md index d73f1a9..8082e19 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -89,7 +89,7 @@ never pin a version hex marks as retired or vulnerable. | `pgvector` | optional, ~> 0.3 | ๐Ÿ” not yet a dependency | Vectors on the Postgres path. Not yet a dependency; Slice 032 decides. Split from the postgrex row at Slice 010. | | `oban` | ~> 2.24 | ๐Ÿ” not yet a dependency | Uses `Oban.Engines.Lite` on SQLite. โš ๏ธ Oban Pro Workflows/Smart engine are Postgres-only. Added at Slice 050. | | `req` | ~> 0.5 | โœ… in `mix.lock` | HTTP client. | -| `req_llm` | ~> 1.22 | ๐Ÿ” not yet a dependency | Provider layer (streaming, tools, structured output, usage). โš ๏ธ The pin was `~> 1.10` against a recorded latest of 1.10.0; the real latest was twelve minors ahead. Check event shapes against the current version at Slice 011, not against this file's prose. Added at Slice 011. | +| `req_llm` | ~> 1.22 | โœ… in `mix.lock` | Provider layer (streaming, tools, structured output, usage). โš ๏ธ The pin was `~> 1.10` against a recorded latest of 1.10.0; the real latest was twelve minors ahead. Check event shapes against the current version at Slice 011, not against this file's prose. Added at Slice 011. | | `beam_mcp` | ~> 0.8 | ๐Ÿ” not yet a dependency | MCP server core, Apache-2.0, ADR-0007 decision 5 (owner decision 2026-09-08, recorded 2026-09-20). 0.8.0 on hex.pm, standing before 1.0.0. Server side only: the client, MRTR and OAuth are Trinity's, above it. Added at Slice 059. The earlier candidate list (anubis_mcp, fastest_mcp, gen_mcp) is history. | | `jido` | ~> 2.3 (pending ADR-0009) | ๐Ÿ” not yet a dependency | Actions, directives and the effect boundary, if the Slice 012 checkpoint adopts it. | | `jason` | ~> 1.2 | โœ… in `mix.lock` | | diff --git a/config/config.exs b/config/config.exs index 3322e21..b25afce 100644 --- a/config/config.exs +++ b/config/config.exs @@ -36,6 +36,10 @@ config :trinity, other -> raise "TRINITY_DB must be sqlite or postgres, got #{inspect(other)}" end) +# Slice 011: the model registry lives in its own file so the live test suite can read it +# without evaluating the environment-specific imports below. +import_config "llm.exs" + # Slice 010, every environment, SQLite only (the Postgres adapter ignores keys it does not # know, and the CI matrix proves that). One writer: the pool has exactly one connection, so the # single-writer rule SQLite imposes is the pool's shape rather than a hope. Each pragma is named diff --git a/config/llm.exs b/config/llm.exs new file mode 100644 index 0000000..b4d8026 --- /dev/null +++ b/config/llm.exs @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +import Config + +# Slice 011: the model registry. Ids are Trinity's; `model` is req_llm's "provider:model"; +# keys are read through Trinity.Config.secret/1 from the named variable at call time. Prices +# are US dollars per million tokens and are the source of every recorded cost. Free-tier +# models carry 0.0. The default model is the OpenRouter one the owner picked; the NVIDIA +# endpoint is reached as an OpenAI-compatible base_url. Test config replaces all of this. +config :trinity, :llm, + default_model: "openrouter:ling", + providers: %{req_llm: Trinity.LLM.Providers.ReqLLM}, + retry: [attempts: 3, base_ms: 200], + models: [ + %{ + id: "openrouter:ling", + provider: :req_llm, + model: + "openrouter:" <> + System.get_env("TRINITY_LIVE_MODEL", "inclusionai/ling-3.0-flash-vl:free"), + api_key_env: "OPENROUTER_API_KEY", + caps: [:stream, :tools, :json], + price: %{input: 0.0, output: 0.0} + }, + %{ + id: "nvidia:nemotron", + provider: :req_llm, + model: + "openai:" <> System.get_env("NEMOTRON_MODEL", "nvidia/nemotron-3.5-lightning-30b-a3b"), + base_url: System.get_env("NEMOTRON_BASE_URL", "https://integrate.api.nvidia.com/v1"), + api_key_env: "NEMOTRON_API_KEY", + caps: [:stream, :tools, :json], + price: %{input: 0.0, output: 0.0} + }, + %{ + id: "nvidia:embed", + provider: :req_llm, + model: "openai:nvidia/nemotron-3-embed-1b", + base_url: System.get_env("NEMOTRON_BASE_URL", "https://integrate.api.nvidia.com/v1"), + api_key_env: "NEMOTRON_API_KEY", + caps: [:embed], + price: %{input: 0.0, output: 0.0} + } + ] diff --git a/config/test.exs b/config/test.exs index bb2f3cc..b235187 100644 --- a/config/test.exs +++ b/config/test.exs @@ -7,6 +7,36 @@ import Config # The MIX_TEST_PARTITION environment variable can be used # to provide built-in test partitioning in CI environment. # Run `mix help test` for more information. +# Slice 011: the registry in tests is the scripted fake plus a Mox mock; the live tests set +# their own entries from the environment at runtime. +config :trinity, :llm, + default_model: "fake:chat", + providers: %{fake: Trinity.LLM.Providers.Fake, mock: Trinity.LLM.ProviderMock}, + retry: [attempts: 3, base_ms: 1], + models: [ + %{ + id: "fake:chat", + provider: :fake, + model: "chat", + caps: [:stream, :tools, :json], + price: %{input: 1.0, output: 2.0} + }, + %{ + id: "fake:embed", + provider: :fake, + model: "embed", + caps: [:embed, {:embed_dim, 8}], + price: %{input: 0.5, output: 0.0} + }, + %{ + id: "mock:chat", + provider: :mock, + model: "chat", + caps: [:stream, :tools], + price: %{input: 0.0, output: 0.0} + } + ] + # Slice 010: the data-dir lock takes a temporary directory in tests, so a test run never # contends with a running Trinity on the same machine, and two test runs at once do contend, # which is the property under test. diff --git a/coverage.tsv b/coverage.tsv index 5eb6566..1a04a6d 100644 --- a/coverage.tsv +++ b/coverage.tsv @@ -2,3 +2,4 @@ slice_id percent sha date 000 27.01 e935c7b 2026-09-06 001 30.37 5a9c8f7 2026-09-06 010 44.88 45ba4f0 2026-09-20 +011 51.57 ec5334a 2026-09-20 diff --git a/docs/05-data-model.md b/docs/05-data-model.md index 2af372e..6d27e8d 100644 --- a/docs/05-data-model.md +++ b/docs/05-data-model.md @@ -101,8 +101,20 @@ Pending/decided approval requests: `session_id`, `tool`, `args`, `risk`, `status Execution history is in `oban_jobs` + a `task_runs` table (status, session_id, summary). ### usage_events (Slice 011; the ledger and budgets that read it are Slice 090) -Per LLM call: `session_id`, `provider`, `model`, `prompt_tokens`, `completion_tokens`, `cached_tokens`, -`cost_usd`, `latency_ms`. Cost ledger and budgets derive from this. +One row per completed call, as built at slice 011: + +| column | type | notes | +|---|---|---| +| model_id | string | the registry id (`"openrouter:ling"`), not the provider's model name | +| provider | string | the registry entry's provider atom as text | +| kind | string | "chat" \| "object" \| "embed" | +| input_tokens, output_tokens, cached_tokens, reasoning_tokens | integer | the names `Trinity.LLM.Event`'s usage map uses; `prompt_tokens` and `completion_tokens` in the first draft of this table are these two | +| cost_usd | float | computed from the registry's price in dollars per million tokens; the only cost Trinity reports | +| session_id | fk sessions, nullable | set when the call belongs to a session | +| provider_meta | map | `provider_cost`: the provider's own figure when it reports one, kept for comparison and never used | + +Append-only; `inserted_at` only. Latency is not a column: it is a Telemetry measurement at slice 090, where the +call is timed at the one place every call passes. ### gateway_identities (Slice 070) `adapter`, `external_user_id`, `display`, `paired_at`, `allowed`: DM pairing and allowlists. diff --git a/lib/trinity.ex b/lib/trinity.ex index da5d8dc..c34e237 100644 --- a/lib/trinity.ex +++ b/lib/trinity.ex @@ -10,7 +10,7 @@ defmodule Trinity do use Boundary, deps: [], exports: - [Paths, Repo, UUID, Sessions] ++ + [Paths, Repo, UUID, Config, Sessions, LLM] ++ if(Mix.env() == :test, do: [DataCase, NetworkGuard, Factory], else: []) @moduledoc """ diff --git a/lib/trinity/application.ex b/lib/trinity/application.ex index 9efa129..a5314e2 100644 --- a/lib/trinity/application.ex +++ b/lib/trinity/application.ex @@ -33,6 +33,8 @@ defmodule Trinity.Application do repos: Application.fetch_env!(:trinity, :ecto_repos), skip: skip_migrations?()}, {DNSCluster, query: Application.get_env(:trinity, :dns_cluster_query) || :ignore}, {Phoenix.PubSub, name: Trinity.PubSub}, + # Slice 011: streams to a pid run under this supervisor, never as bare tasks. + {Task.Supervisor, name: Trinity.LLM.TaskSupervisor}, # Start to serve requests, typically the last entry TrinityWeb.Endpoint ] ++ Trinity.Smoke.children(Trinity.Smoke.argv()) diff --git a/lib/trinity/config.ex b/lib/trinity/config.ex new file mode 100644 index 0000000..284ff89 --- /dev/null +++ b/lib/trinity/config.ex @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Config do + @moduledoc """ + Secrets, read from one place. Slice 011: the environment. Slice 100 adds the OS keychain + behind the same function, and nothing else in the tree reads a key any other way. + """ + + @doc "The secret named by `env_var`, or a named error; never nil handed to a provider." + @spec secret(String.t()) :: {:ok, String.t()} | {:error, {:missing_secret, String.t()}} + def secret(env_var) when is_binary(env_var) do + case System.get_env(env_var) do + value when is_binary(value) and value != "" -> {:ok, value} + _ -> {:error, {:missing_secret, env_var}} + end + end +end diff --git a/lib/trinity/llm.ex b/lib/trinity/llm.ex new file mode 100644 index 0000000..d17ae42 --- /dev/null +++ b/lib/trinity/llm.ex @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM do + @moduledoc """ + The one door to a language model. Slice 011. + + Sessions call this and never a provider. A request names a registry model id (or nothing, for + the default); the registry names the provider module; the call is retried on transient errors; + a completed call writes a `usage_events` row with cost from the registry price. Events reach + the caller either through a function (`stream/3`) or as messages to a pid (`stream_to/3`), + which is what slice 012's Session wants. + """ + use Boundary, deps: [Trinity], exports: [Error, Event, Request, Provider, Registry] + + alias Trinity.LLM.{Error, Registry, Request, Retry, Usage} + + @type opts :: keyword() + + @doc "Streams events through `emit`, in the calling process. Returns the usage." + @spec stream(Request.t(), opts(), Trinity.LLM.Provider.emit()) :: + {:ok, map()} | {:error, Error.t() | term()} + def stream(%Request{} = request, opts \\ [], emit) when is_function(emit, 1) do + with {:ok, entry, module} <- resolve(request) do + call(entry, "chat", opts, fn -> + module.stream(request, provider_opts(entry, opts), emit) + end) + end + end + + @doc """ + Streams events as messages `{:llm_event, ref, event}` to `pid`, then `{:llm_done, ref, result}`. + Runs in a supervised task so the caller is never blocked; `ref` is returned at once. + """ + @spec stream_to(Request.t(), opts(), pid()) :: {:ok, reference()} | {:error, term()} + def stream_to(%Request{} = request, opts \\ [], pid) when is_pid(pid) do + ref = make_ref() + + with {:ok, _entry, _module} <- resolve(request), + {:ok, _task} <- + Task.Supervisor.start_child(Trinity.LLM.TaskSupervisor, fn -> + result = stream(request, opts, &send(pid, {:llm_event, ref, &1})) + send(pid, {:llm_done, ref, result}) + end) do + {:ok, ref} + end + end + + @doc "One complete response." + @spec generate(Request.t(), opts()) :: {:ok, Trinity.LLM.Provider.result()} | {:error, term()} + def generate(%Request{} = request, opts \\ []) do + with {:ok, entry, module} <- resolve(request) do + call(entry, "chat", opts, fn -> module.generate(request, provider_opts(entry, opts)) end) + end + end + + @doc "A map validated against a JSON Schema." + @spec generate_object(Request.t(), map(), opts()) :: {:ok, map()} | {:error, term()} + def generate_object(%Request{} = request, schema, opts \\ []) when is_map(schema) do + with {:ok, entry, module} <- resolve(request), + {:ok, object, _usage} <- + call(entry, "object", opts, fn -> + module.generate_object(request, schema, provider_opts(entry, opts)) + end) do + {:ok, object} + end + end + + @doc "One vector per text. `opts[:model]` names the embedding model's registry id." + @spec embed([String.t()], opts()) :: {:ok, [[float()]]} | {:error, term()} + def embed(texts, opts \\ []) when is_list(texts) do + with {:ok, entry} <- Registry.lookup(Keyword.get(opts, :model)), + {:ok, module} <- Registry.provider_module(entry), + {:ok, vectors, _usage} <- + call(entry, "embed", opts, fn -> module.embed(texts, provider_opts(entry, opts)) end) do + {:ok, vectors} + end + end + + @doc "The registry's models." + @spec models() :: [Registry.entry()] + def models, do: Registry.models() + + @doc "The registry's default model id." + @spec default_model() :: String.t() | nil + def default_model, do: Registry.default_model() + + @doc "A model's capabilities, from its registry entry." + @spec capabilities(String.t()) :: {:ok, [atom() | {atom(), term()}]} | {:error, term()} + def capabilities(model_id) do + with {:ok, entry} <- Registry.lookup(model_id), do: {:ok, entry.caps} + end + + defp resolve(%Request{model: model}) do + with {:ok, entry} <- Registry.lookup(model), + {:ok, module} <- Registry.provider_module(entry) do + {:ok, entry, module} + end + end + + # The entry's keys win: a caller's `:model` is a registry id, the entry's is the provider's + # own name, and the provider must see the latter. Found by the live suite, where an embed + # call asked req_llm for a provider named after the registry id. + defp provider_opts(entry, opts) do + entry_opts = + entry |> Map.take([:model, :base_url, :api_key_env, :price, :caps]) |> Map.to_list() + + Keyword.merge(opts, entry_opts) + end + + # Retry around the provider; on success the usage is recorded. The usage row is written for + # the call that completed, once, whatever the number of attempts it took. + defp call(entry, kind, opts, fun) do + case Retry.run(fun, opts) do + {:ok, %{usage: usage}} = ok -> + record(entry, kind, usage, opts) + ok + + {:ok, usage} = ok when is_map(usage) -> + record(entry, kind, usage, opts) + ok + + {:ok, _value, usage} = ok -> + record(entry, kind, usage, opts) + ok + + other -> + other + end + end + + defp record(entry, kind, usage, opts) do + {:ok, _} = Usage.record(entry, kind, usage, Keyword.take(opts, [:session_id])) + end +end diff --git a/lib/trinity/llm/error.ex b/lib/trinity/llm/error.ex new file mode 100644 index 0000000..17a87dc --- /dev/null +++ b/lib/trinity/llm/error.ex @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Error do + @moduledoc """ + A provider failure with one bit that matters to the caller: whether trying again could help. + Timeouts, rate limits, server errors and refused connections are transient; everything else + (a bad key, an unknown model, a malformed request) is not and is returned at once. + """ + + @type t :: %__MODULE__{transient?: boolean(), reason: term(), status: non_neg_integer() | nil} + defexception [:reason, :status, transient?: false] + + @impl true + def message(%__MODULE__{transient?: t, status: status, reason: reason}) do + kind = if t, do: "transient", else: "permanent" + "#{kind} LLM error#{if status, do: " (HTTP #{status})", else: ""}: #{inspect(reason)}" + end + + @doc "Classifies an HTTP status: 408, 425, 429 and 5xx are transient." + @spec from_status(non_neg_integer(), term()) :: t() + def from_status(status, reason) do + %__MODULE__{ + status: status, + reason: reason, + transient?: status in [408, 425, 429] or status >= 500 + } + end + + @doc "A transient error with no status: a timeout, a closed or refused connection." + @spec transient(term()) :: t() + def transient(reason), do: %__MODULE__{reason: reason, transient?: true} + + @doc "A permanent error with no status." + @spec permanent(term()) :: t() + def permanent(reason), do: %__MODULE__{reason: reason, transient?: false} +end diff --git a/lib/trinity/llm/event.ex b/lib/trinity/llm/event.ex new file mode 100644 index 0000000..1e2fab8 --- /dev/null +++ b/lib/trinity/llm/event.ex @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Event do + @moduledoc """ + The event stream shape every provider emits and every consumer reads. Slice 011. + + Seven shapes, from SLICE.md, and nothing else. Slice 012's Session and slice 013's UI consume + these and never a provider's own chunks; `valid?/1` is the guard a provider's output is + held to in tests. + """ + + @type tool_call_id :: String.t() + @type t :: + {:text_delta, String.t()} + | {:tool_call_start, tool_call_id(), name :: String.t()} + | {:tool_call_delta, tool_call_id(), json_chunk :: String.t()} + | {:tool_call_end, tool_call_id(), args :: map()} + | {:usage, map()} + | {:done, reason :: :stop | :length | :tool_calls | :content_filter | atom()} + | {:error, term()} + + @doc "True for exactly the seven shapes above." + @spec valid?(term()) :: boolean() + def valid?({:text_delta, s}) when is_binary(s), do: true + def valid?({:tool_call_start, id, name}) when is_binary(id) and is_binary(name), do: true + def valid?({:tool_call_delta, id, chunk}) when is_binary(id) and is_binary(chunk), do: true + def valid?({:tool_call_end, id, args}) when is_binary(id) and is_map(args), do: true + def valid?({:usage, usage}) when is_map(usage), do: true + def valid?({:done, reason}) when is_atom(reason), do: true + def valid?({:error, _}), do: true + def valid?(_), do: false +end diff --git a/lib/trinity/llm/provider.ex b/lib/trinity/llm/provider.ex new file mode 100644 index 0000000..d487fc4 --- /dev/null +++ b/lib/trinity/llm/provider.ex @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Provider do + @moduledoc """ + The behaviour every provider implements and the only thing `Trinity.LLM` calls. Slice 011. + + `opts` carries what the registry entry says about the model (`:model`, the provider's own + model name; `:base_url` and `:api_key` for OpenAI-compatible endpoints; `:price`) plus what + the caller passed. A provider emits `Trinity.LLM.Event` shapes through `emit` in the calling + process and returns when the stream ends. Errors are `Trinity.LLM.Error` structs. + """ + + alias Trinity.LLM.{Error, Request} + + @type opts :: keyword() + @type emit :: (Trinity.LLM.Event.t() -> any()) + @type usage :: %{ + optional(:input_tokens) => non_neg_integer(), + optional(:output_tokens) => non_neg_integer(), + optional(:cached_tokens) => non_neg_integer(), + optional(:reasoning_tokens) => non_neg_integer(), + optional(:provider_cost) => number() | nil + } + @type result :: %{ + text: String.t(), + tool_calls: [%{id: String.t(), name: String.t(), args: map()}], + usage: usage(), + finish: atom() + } + + @doc "Streams events through `emit`; returns the usage when the stream ends." + @callback stream(Request.t(), opts(), emit()) :: {:ok, usage()} | {:error, Error.t()} + + @doc "One complete response." + @callback generate(Request.t(), opts()) :: {:ok, result()} | {:error, Error.t()} + + @doc "A map validated against `schema` (a JSON Schema), with the usage beside it." + @callback generate_object(Request.t(), schema :: map(), opts()) :: + {:ok, map(), usage()} | {:error, Error.t()} + + @doc "One vector per text, of the dimension `capabilities/1` declares." + @callback embed([String.t()], opts()) :: {:ok, [[float()]], usage()} | {:error, Error.t()} + + @doc "The provider's own model names it can serve, for diagnostics." + @callback models() :: [String.t()] + + @doc "Capabilities of a model: `:stream`, `:tools`, `:json`, `:embed`, `{:embed_dim, n}`." + @callback capabilities(model :: String.t()) :: [atom() | {atom(), term()}] +end diff --git a/lib/trinity/llm/providers/req_llm.ex b/lib/trinity/llm/providers/req_llm.ex new file mode 100644 index 0000000..7ab10c2 --- /dev/null +++ b/lib/trinity/llm/providers/req_llm.ex @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Providers.ReqLLM do + @moduledoc """ + `Trinity.LLM.Provider` over req_llm (ADR-0003). Slice 011. + + A registry entry names the req_llm provider and model as `":"` in `:model` + (the first colon splits them, so a model name may itself contain colons), the environment + variable holding the key in `:api_key_env`, and, for OpenAI-compatible endpoints, the + `:base_url`. Keys reach req_llm only as a per-request `:api_key` read through + `Trinity.Config.secret/1`; a missing key is a permanent error before any request is made. + + The half that talks lives here; the pure half (chunks into events, responses into results, + errors into `Trinity.LLM.Error`) is `Trinity.LLM.Providers.ReqLLM.Mapping`, tested without a + network. + """ + @behaviour Trinity.LLM.Provider + + alias Trinity.Config + alias Trinity.LLM.{Error, Request} + alias Trinity.LLM.Providers.ReqLLM.Mapping + + @impl true + def stream(%Request{} = request, opts, emit) do + with {:ok, spec, call_opts} <- prepare(request, opts) do + case ReqLLM.stream_text(spec, context(request), call_opts) do + {:ok, response} -> + state = Mapping.reduce(response.stream, emit) + usage = Mapping.normalise_usage(ReqLLM.StreamResponse.usage(response)) + emit.({:usage, usage}) + emit.({:done, state.finish || :stop}) + {:ok, usage} + + {:error, reason} -> + {:error, Mapping.classify(reason)} + end + end + rescue + e -> {:error, Mapping.classify(e)} + end + + @impl true + def generate(%Request{} = request, opts) do + with {:ok, spec, call_opts} <- prepare(request, opts), + {:ok, response} <- Mapping.wrap(ReqLLM.generate_text(spec, context(request), call_opts)) do + {:ok, + %{ + text: ReqLLM.Response.text(response) || "", + tool_calls: Enum.map(ReqLLM.Response.tool_calls(response), &Mapping.tool_call/1), + usage: Mapping.normalise_usage(ReqLLM.Response.usage(response)), + finish: Mapping.finish(ReqLLM.Response.finish_reason(response)) + }} + end + rescue + e -> {:error, Mapping.classify(e)} + end + + @impl true + def generate_object(%Request{} = request, schema, opts) do + with {:ok, spec, call_opts} <- prepare(request, opts), + {:ok, response} <- + Mapping.wrap(ReqLLM.generate_object(spec, context(request), schema, call_opts)) do + {:ok, ReqLLM.Response.object(response), + Mapping.normalise_usage(ReqLLM.Response.usage(response))} + end + rescue + e -> {:error, Mapping.classify(e)} + end + + @impl true + def embed(texts, opts) do + # The embedding call validates its own option set: no receive_timeout there, so the + # budget travels as total_timeout instead. + with {:ok, spec, call_opts} <- prepare(%Request{}, opts), + embed_opts = + call_opts + |> Keyword.delete(:receive_timeout) + |> Keyword.put(:total_timeout, receive_timeout()) + |> Keyword.put(:return_usage, true), + {:ok, %{embedding: vectors, usage: usage}} <- + Mapping.wrap(ReqLLM.embed(spec, texts, embed_opts)) do + vectors = + if texts |> length() == 1 and is_list(hd(vectors)) == false, do: [vectors], else: vectors + + {:ok, vectors, Mapping.normalise_usage(usage)} + end + rescue + e -> {:error, Mapping.classify(e)} + end + + @impl true + def models, do: [] + + @impl true + def capabilities(_model), do: [:stream, :tools, :json] + + ## Request mapping + + defp prepare(%Request{} = request, opts) do + with {:ok, provider, model} <- split_model(Keyword.fetch!(opts, :model)), + {:ok, key} <- Config.secret(Keyword.get(opts, :api_key_env, default_env(provider))) do + call_opts = + [api_key: key, receive_timeout: receive_timeout()] + |> maybe_put(:base_url, Keyword.get(opts, :base_url)) + |> Keyword.merge(params(request, provider)) + |> maybe_put(:tools, tools(request)) + + # An inline spec, not a catalog lookup: Trinity's registry is the catalog, and model ids + # here are configured, so req_llm's "unverified model" warning does not apply. An + # embedding model says so in its capabilities, or req_llm refuses the operation. + spec = %{ + provider: provider, + id: model, + capabilities: capabilities_of(Keyword.get(opts, :caps, [])) + } + + {:ok, spec, call_opts} + else + {:error, {:missing_secret, var}} -> {:error, Error.permanent({:missing_secret, var})} + {:error, reason} -> {:error, Error.permanent(reason)} + end + end + + defp capabilities_of(caps) do + if :embed in caps, do: %{embeddings: true}, else: %{} + end + + # The five providers SLICE.md names, and nothing else: a name from config never mints an + # atom. `openai_compatible` is req_llm's openai provider with a base_url, which is how the + # NVIDIA endpoint, Ollama and LM Studio are reached. + @providers %{ + "anthropic" => :anthropic, + "openai" => :openai, + "openai_compatible" => :openai, + "openrouter" => :openrouter, + "google" => :google + } + + defp split_model(spec) when is_binary(spec) do + with [name, model] when model != "" <- String.split(spec, ":", parts: 2), + {:ok, provider} <- Map.fetch(@providers, name) do + {:ok, provider, model} + else + :error -> {:error, {:unknown_provider, spec}} + _ -> {:error, {:bad_model_spec, spec}} + end + end + + defp default_env(provider), + do: provider |> Atom.to_string() |> String.upcase() |> Kernel.<>("_API_KEY") + + # Reasoning models can sit for longer than req_llm's 30-second default before the first + # byte; measured on the NVIDIA endpoint under load. Configurable, generous by default. + defp receive_timeout do + Keyword.get(Application.get_env(:trinity, :llm, []), :receive_timeout_ms, 120_000) + end + + defp params(%Request{params: params}, provider) do + params + |> Enum.flat_map(fn + {:max_tokens, n} -> + [max_tokens: n] + + {:temperature, t} -> + [temperature: t] + + {:cache, true} when provider == :anthropic -> + [provider_options: [cache_control: %{type: "ephemeral"}]] + + {:cache, _} -> + [] + + {k, v} when k in [:top_p, :stop, :seed] -> + [{k, v}] + + _ -> + [] + end) + end + + defp tools(%Request{tools: []}), do: nil + + defp tools(%Request{tools: tools}) do + Enum.map(tools, fn tool -> + ReqLLM.Tool.new!( + name: tool.name, + description: Map.get(tool, :description, ""), + parameter_schema: tool.parameters, + callback: fn _ -> {:ok, nil} end + ) + end) + end + + defp context(%Request{system: system, messages: messages}) do + base = if system, do: [ReqLLM.Context.system(system)], else: [] + ReqLLM.Context.new(base ++ Enum.map(messages, &message/1)) + end + + defp message(%{role: "system", content: c}), do: ReqLLM.Context.system(c) + defp message(%{role: "user", content: c}), do: ReqLLM.Context.user(c) + + defp message(%{role: "tool", content: c} = m), + do: ReqLLM.Context.tool_result(Map.fetch!(m, :tool_call_id), c) + + defp message(%{role: "assistant", content: c} = m) do + case Map.get(m, :tool_calls, []) do + [] -> + ReqLLM.Context.assistant(c) + + calls -> + ReqLLM.Context.assistant(c, tool_calls: Enum.map(calls, &{&1.id, &1.name, &1.args})) + end + end + + defp maybe_put(opts, _key, nil), do: opts + defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value) +end diff --git a/lib/trinity/llm/providers/req_llm/mapping.ex b/lib/trinity/llm/providers/req_llm/mapping.ex new file mode 100644 index 0000000..d6c007d --- /dev/null +++ b/lib/trinity/llm/providers/req_llm/mapping.ex @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Providers.ReqLLM.Mapping do + @moduledoc """ + The pure half of the req_llm adapter: streamed chunks into `Trinity.LLM.Event` shapes, a + provider response into a result, req_llm errors into `Trinity.LLM.Error`. No network, so + the default test suite covers it with recorded chunk sequences; the live suite covers the + half that talks. + + Streaming, as req_llm 1.24.0 emits it: `:content` is a text delta; `:thinking` is dropped; + a `:tool_call` chunk opens a call (its metadata carries `id`, `index` and, when arguments + follow in fragments, `expects_arg_fragments`), or opens and closes one when it arrives with + complete arguments; a `:meta` chunk carries `tool_call_args` fragments keyed by index, or the + `finish_reason` that closes every open call. + """ + + alias Trinity.LLM.Error + + @type emit :: (Trinity.LLM.Event.t() -> any()) + @type state :: %{calls: map(), order: [String.t()], finish: atom() | nil} + + @doc "The empty assembly state." + @spec new_state() :: state() + def new_state, do: %{calls: %{}, order: [], finish: nil} + + @doc "Folds every chunk of a stream through `handle_chunk/3`, then closes any call still open." + @spec reduce(Enumerable.t(), emit()) :: state() + def reduce(chunks, emit) do + chunks + |> Enum.reduce(new_state(), &handle_chunk(&1, &2, emit)) + |> close_open_calls(emit) + end + + @doc "One chunk into zero or more events, returning the new state." + @spec handle_chunk(map(), state(), emit()) :: state() + def handle_chunk(%{type: :content, text: text}, state, emit) + when is_binary(text) and text != "" do + emit.({:text_delta, text}) + state + end + + def handle_chunk(%{type: :tool_call, name: name, arguments: args, metadata: meta}, state, emit) do + id = call_id(meta, state) + + state = + if Map.has_key?(state.calls, id), do: state, else: open_call(state, id, name, meta, emit) + + if Map.get(meta, :expects_arg_fragments, false) or args == %{} do + state + else + close_call(state, id, args, emit) + end + end + + def handle_chunk( + %{type: :meta, metadata: %{tool_call_args: %{index: index, fragment: fragment}}}, + state, + emit + ) do + case Enum.find(state.calls, fn {_id, call} -> call.index == index and call.open end) do + {id, call} -> + emit.({:tool_call_delta, id, fragment}) + put_in(state.calls[id], %{call | fragments: [fragment | call.fragments]}) + + nil -> + state + end + end + + def handle_chunk(%{type: :meta, metadata: meta}, state, emit) do + state = + case Map.get(meta, :finish_reason) do + nil -> state + reason -> %{state | finish: finish(reason)} + end + + if state.finish in [:tool_calls, :stop, :length], + do: close_open_calls(state, emit), + else: state + end + + def handle_chunk(_other, state, _emit), do: state + + @doc "Closes every call still open, decoding its fragments; called at the end of a stream." + @spec close_open_calls(state(), emit()) :: state() + def close_open_calls(state, emit) do + Enum.reduce(state.order, state, fn id, acc -> + case acc.calls[id] do + %{open: true, fragments: fragments} -> + close_call(acc, id, decode_fragments(fragments), emit) + + _ -> + acc + end + end) + end + + defp call_id(meta, state) do + case Map.get(meta, :id) do + id when is_binary(id) -> id + _ -> "call_#{map_size(state.calls) + 1}" + end + end + + defp open_call(state, id, name, meta, emit) do + emit.({:tool_call_start, id, name}) + call = %{name: name, index: Map.get(meta, :index), fragments: [], open: true} + %{state | calls: Map.put(state.calls, id, call), order: state.order ++ [id]} + end + + defp close_call(state, id, args, emit) do + emit.({:tool_call_end, id, args}) + put_in(state.calls[id].open, false) + end + + # Fragments arrive in order and are kept reversed; an unparseable body is an empty map, + # which the consumer sees as a tool called with no arguments rather than a crash mid-stream. + defp decode_fragments(fragments) do + case Jason.decode(fragments |> Enum.reverse() |> IO.iodata_to_binary()) do + {:ok, map} when is_map(map) -> map + _ -> %{} + end + end + + ## Results + + @doc "A tool call from a complete response, in Trinity's shape." + @spec tool_call(map()) :: %{id: String.t(), name: String.t(), args: map()} + def tool_call(%{id: id, name: name, arguments: args}), + do: %{id: id, name: name, args: args || %{}} + + def tool_call(%{id: id, function: %{name: name, arguments: args}}), + do: %{id: id, name: name, args: args || %{}} + + def tool_call(other), + do: %{id: Map.get(other, :id, ""), name: Map.get(other, :name, ""), args: %{}} + + @doc "A finish reason from the closed vocabulary; anything else is `:other`, never a new atom." + @spec finish(term()) :: atom() + def finish(nil), do: :stop + def finish(reason) when is_atom(reason), do: reason + def finish("stop"), do: :stop + def finish("length"), do: :length + def finish("tool_calls"), do: :tool_calls + def finish("content_filter"), do: :content_filter + def finish(other) when is_binary(other), do: :other + + @doc "Usage in Trinity's keys; the provider's own cost figure kept aside as `provider_cost`." + @spec normalise_usage(map() | nil) :: map() + def normalise_usage(nil), do: %{} + + def normalise_usage(usage) when is_map(usage) do + %{ + input_tokens: Map.get(usage, :input_tokens, 0) || 0, + output_tokens: Map.get(usage, :output_tokens, 0) || 0, + cached_tokens: Map.get(usage, :cached_tokens, 0) || 0, + reasoning_tokens: Map.get(usage, :reasoning_tokens, 0) || 0, + provider_cost: Map.get(usage, :total_cost) + } + end + + ## Errors + + @doc "Passes an ok through and classifies an error." + @spec wrap({:ok, term()} | {:error, term()}) :: {:ok, term()} | {:error, Error.t()} + def wrap({:ok, _} = ok), do: ok + def wrap({:error, reason}), do: {:error, classify(reason)} + + @doc """ + A failure as a `Trinity.LLM.Error`. req_llm wraps failures: a stream error carries its + cause, an API error its status and a retryable flag, a class error a list of errors. The + verdict comes from the innermost thing that has one. Found by the live suite: an upstream + 429 arrived inside a wrapper whose own status was nil and was called permanent. + """ + @spec classify(term()) :: Error.t() + def classify(%Error{} = e), do: e + def classify(%{status: status} = e) when is_integer(status), do: Error.from_status(status, e) + def classify(%{retryable: true} = e), do: Error.transient(e) + def classify(%Req.TransportError{} = e), do: Error.transient(e) + def classify(%Mint.TransportError{} = e), do: Error.transient(e) + + def classify(%{__exception__: true} = e) do + case inner(e) do + nil -> if timeout?(e), do: Error.transient(e), else: Error.permanent(e) + inner -> %{classify(inner) | reason: e} + end + end + + def classify(other), do: Error.permanent(other) + + defp inner(%{cause: %{__exception__: true} = c}), do: c + defp inner(%{errors: [%{__exception__: true} = c | _]}), do: c + defp inner(%{reason: %{__exception__: true} = c}), do: c + defp inner(_), do: nil + + defp timeout?(%{cause: :timeout}), do: true + defp timeout?(%{reason: reason}) when reason in [:timeout, :econnrefused, :closed], do: true + + defp timeout?(%{reason: reason}) when is_binary(reason), + do: reason =~ ~r/timeout|closed|refused/i + + defp timeout?(_), do: false +end diff --git a/lib/trinity/llm/registry.ex b/lib/trinity/llm/registry.ex new file mode 100644 index 0000000..d20b3cd --- /dev/null +++ b/lib/trinity/llm/registry.ex @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Registry do + @moduledoc """ + The models Trinity may use, from `config :trinity, :llm`. Slice 011. + + config :trinity, :llm, + default_model: "openrouter:ling", + providers: %{req_llm: Trinity.LLM.Providers.ReqLLM}, + models: [ + %{id: "openrouter:ling", provider: :req_llm, model: "openrouter:inclusionai/ling-3.0-flash-vl:free", + caps: [:stream, :tools, :json], price: %{input: 0.0, output: 0.0}} + ] + + `price` is US dollars per million tokens, input and output, and is the source of every cost + Trinity records; a provider's own figure is kept beside it for comparison and never used. + Read at call time, not cached, so a test or a settings page can change the default without a + restart (AC7). + """ + + @type entry :: %{ + required(:id) => String.t(), + required(:provider) => atom(), + required(:model) => String.t(), + required(:caps) => [atom() | {atom(), term()}], + required(:price) => %{input: number(), output: number()}, + optional(:base_url) => String.t(), + optional(:api_key_env) => String.t() + } + + @doc "Every registry entry." + @spec models() :: [entry()] + def models, do: Keyword.get(config(), :models, []) + + @doc "The default model id, or nil when the registry is empty." + @spec default_model() :: String.t() | nil + def default_model, do: Keyword.get(config(), :default_model) + + @doc "The entry for an id, or the default's when nil, refusing an unknown id by name." + @spec lookup(String.t() | nil) :: {:ok, entry()} | {:error, {:unknown_model, String.t() | nil}} + def lookup(nil), do: lookup(default_model()) + + def lookup(id) do + case Enum.find(models(), &(&1.id == id)) do + nil -> {:error, {:unknown_model, id}} + entry -> {:ok, entry} + end + end + + @doc "The module implementing `Trinity.LLM.Provider` for an entry's provider atom." + @spec provider_module(entry()) :: {:ok, module()} | {:error, {:unknown_provider, atom()}} + def provider_module(%{provider: provider}) do + case Map.fetch(Keyword.get(config(), :providers, %{}), provider) do + {:ok, module} -> {:ok, module} + :error -> {:error, {:unknown_provider, provider}} + end + end + + defp config, do: Application.get_env(:trinity, :llm, []) +end diff --git a/lib/trinity/llm/request.ex b/lib/trinity/llm/request.ex new file mode 100644 index 0000000..9591976 --- /dev/null +++ b/lib/trinity/llm/request.ex @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Request do + @moduledoc """ + What a caller asks a provider for. Slice 011. + + `messages` are maps with `role` and `content` (and `tool_call_id` for tool results, `tool_calls` + for assistant turns that made them), in the shape the `messages` table stores; the adapter maps + them to the provider's format. `tools` are maps with `name`, `description` and a JSON Schema + under `parameters`. `params` carries `max_tokens`, `temperature` and provider hints such as + `cache: true`. `model` is a registry id (`"provider:model"`); nil means the registry default. + """ + + @roles ~w(system user assistant tool) + + @type message :: %{ + required(:role) => String.t(), + required(:content) => String.t(), + optional(:tool_call_id) => String.t(), + optional(:tool_calls) => [map()] + } + @type tool :: %{ + required(:name) => String.t(), + required(:description) => String.t(), + required(:parameters) => map() + } + @type t :: %__MODULE__{ + system: String.t() | nil, + messages: [message()], + tools: [tool()], + model: String.t() | nil, + params: map() + } + + defstruct system: nil, messages: [], tools: [], model: nil, params: %{} + + @doc "Builds a request, refusing an unknown role or a tool without a name and parameters." + @spec new(map() | keyword()) :: {:ok, t()} | {:error, {:invalid_request, term()}} + def new(attrs) do + attrs = Map.new(attrs) + request = struct(__MODULE__, attrs) + + with :ok <- check_messages(request.messages), + :ok <- check_tools(request.tools) do + {:ok, request} + end + end + + @doc "Like `new/1` but raises on an invalid request." + @spec new!(map() | keyword()) :: t() + def new!(attrs) do + case new(attrs) do + {:ok, request} -> request + {:error, reason} -> raise ArgumentError, "invalid LLM request: #{inspect(reason)}" + end + end + + defp check_messages(messages) when is_list(messages) do + Enum.reduce_while(messages, :ok, fn + %{role: role, content: content}, :ok when role in @roles and is_binary(content) -> + {:cont, :ok} + + other, :ok -> + {:halt, {:error, {:invalid_request, {:message, other}}}} + end) + end + + defp check_messages(other), do: {:error, {:invalid_request, {:messages, other}}} + + defp check_tools(tools) when is_list(tools) do + Enum.reduce_while(tools, :ok, fn + %{name: name, parameters: %{} = _schema}, :ok when is_binary(name) -> {:cont, :ok} + other, :ok -> {:halt, {:error, {:invalid_request, {:tool, other}}}} + end) + end + + defp check_tools(other), do: {:error, {:invalid_request, {:tools, other}}} +end diff --git a/lib/trinity/llm/retry.ex b/lib/trinity/llm/retry.ex new file mode 100644 index 0000000..b55a6a4 --- /dev/null +++ b/lib/trinity/llm/retry.ex @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Retry do + @moduledoc """ + Retries a provider call on transient errors with exponential backoff; returns a permanent + error at once. Slice 011 AC4. `attempts` and `base_ms` come from `config :trinity, :llm, + retry: [attempts: 3, base_ms: 200]`; an opt overrides either for one call. The sleep is + `base_ms * 2^n` with no jitter, which is enough for one desktop's rate limits and is written + down so nobody expects more. + """ + + alias Trinity.LLM.Error + + @default_attempts 3 + @default_base_ms 200 + + @doc "Runs `fun` up to `attempts` times while it returns a transient error." + @spec run((-> {:ok, term()} | {:error, Error.t()}), keyword()) :: + {:ok, term()} | {:error, Error.t()} + def run(fun, opts \\ []) do + retry = Keyword.get(Application.get_env(:trinity, :llm, []), :retry, []) + attempts = Keyword.get(opts, :attempts, Keyword.get(retry, :attempts, @default_attempts)) + base_ms = Keyword.get(opts, :base_ms, Keyword.get(retry, :base_ms, @default_base_ms)) + sleep = Keyword.get(opts, :sleep, &Process.sleep/1) + attempt(fun, 1, attempts, base_ms, sleep) + end + + defp attempt(fun, n, attempts, base_ms, sleep) do + case fun.() do + {:error, %Error{transient?: true}} when n < attempts -> + sleep.(base_ms * Integer.pow(2, n - 1)) + attempt(fun, n + 1, attempts, base_ms, sleep) + + {:error, %Error{transient?: true} = error} -> + {:error, %{error | reason: {:exhausted, attempts, error.reason}}} + + other -> + other + end + end +end diff --git a/lib/trinity/llm/usage.ex b/lib/trinity/llm/usage.ex new file mode 100644 index 0000000..e2d0bea --- /dev/null +++ b/lib/trinity/llm/usage.ex @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Usage do + @moduledoc """ + One `usage_events` row per completed call, with cost from the registry price. Slice 011 AC5. + The provider's own cost figure, when it reports one, is kept in `provider_meta` for comparison + and is never the recorded cost: the registry is the source Trinity can explain. + """ + use Ecto.Schema + import Ecto.Changeset + + alias Trinity.Repo + + @primary_key {:id, Trinity.UUID, autogenerate: true} + @timestamps_opts [type: :utc_datetime_usec] + + @type t :: %__MODULE__{} + + schema "usage_events" do + field :model_id, :string + field :provider, :string + field :kind, :string + field :input_tokens, :integer, default: 0 + field :output_tokens, :integer, default: 0 + field :cached_tokens, :integer, default: 0 + field :reasoning_tokens, :integer, default: 0 + field :cost_usd, :float, default: 0.0 + field :session_id, Trinity.UUID + field :provider_meta, :map, default: %{} + timestamps(updated_at: false) + end + + @doc "Cost in US dollars from a price in dollars per million tokens." + @spec cost(map(), %{input: number(), output: number()}) :: float() + def cost(usage, %{input: in_price, output: out_price}) do + input = Map.get(usage, :input_tokens, 0) + output = Map.get(usage, :output_tokens, 0) + Float.round(input / 1_000_000 * in_price + output / 1_000_000 * out_price, 8) + end + + @doc "Records a completed call. `kind` is `chat`, `object` or `embed`." + @spec record(map(), String.t(), map(), keyword()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def record(entry, kind, usage, opts \\ []) do + %__MODULE__{} + |> cast( + %{ + model_id: entry.id, + provider: Atom.to_string(entry.provider), + kind: kind, + input_tokens: Map.get(usage, :input_tokens, 0), + output_tokens: Map.get(usage, :output_tokens, 0), + cached_tokens: Map.get(usage, :cached_tokens, 0), + reasoning_tokens: Map.get(usage, :reasoning_tokens, 0), + cost_usd: cost(usage, entry.price), + session_id: Keyword.get(opts, :session_id), + provider_meta: %{"provider_cost" => Map.get(usage, :provider_cost)} + }, + [ + :model_id, + :provider, + :kind, + :input_tokens, + :output_tokens, + :cached_tokens, + :reasoning_tokens, + :cost_usd, + :session_id, + :provider_meta + ] + ) + |> validate_required([:model_id, :provider, :kind]) + |> validate_inclusion(:kind, ~w(chat object embed)) + |> Repo.insert() + end +end diff --git a/mix.exs b/mix.exs index 833c88b..6705a33 100644 --- a/mix.exs +++ b/mix.exs @@ -97,6 +97,9 @@ defmodule Trinity.MixProject do # Optional so the standalone desktop build carries no Postgres driver; the CI matrix # job compiles with the variable set and proves the migrations on both. {:postgrex, ">= 0.0.0", optional: true}, + # Slice 011: the provider layer behind Trinity.LLM (docs/adr/0003). What it brings into + # mix.lock is counted in the slice's NOTES.md, because the desktop binary carries it. + {:req_llm, "~> 1.22"}, {:phoenix_html, "~> 4.1"}, {:phoenix_live_reload, "~> 1.2", only: :dev}, {:phoenix_live_view, "~> 1.2.0"}, diff --git a/mix.lock b/mix.lock index c1494a6..e4c9615 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "abnf_parsec": {:hex, :abnf_parsec, "2.1.0", "c4e88d5d089f1698297c0daced12be1fb404e6e577ecf261313ebba5477941f9", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e0ed6290c7cc7e5020c006d1003520390c9bdd20f7c3f776bd49bfe3c5cd362a"}, "bandit": {:hex, :bandit, "1.12.5", "af205a8e550f304caae09a97d29fd3c79a7f337526ea7cd772d2ff11d2f7c800", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.5", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "c5684ca062fa407cac115aec3256383f3e2ec9fdced7904d59cf5a7bb7ed6181"}, "boundary": {:hex, :boundary, "0.10.4", "5fec5d2736c12f9bfe1720c3a2bd8c48c3547c24d6002ebf8e087570afd5bd2f", [:mix], [], "hexpm", "8baf6f23987afdb1483033ed0bde75c9c703613c22ed58d5f23bf948f203247c"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, @@ -9,6 +10,7 @@ "db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"}, "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, "dns_cluster": {:hex, :dns_cluster, "0.2.0", "aa8eb46e3bd0326bd67b84790c561733b25c5ba2fe3c7e36f28e88f384ebcb33", [:mix], [], "hexpm", "ba6f1893411c69c01b9e8e8f772062535a4cf70f3f35bcc964a324078d8c8240"}, + "dotenvy": {:hex, :dotenvy, "1.2.1", "43b28d17c996d70ac09f6a6dfd2245378b346ed7290c36452b46cdca77cd96bc", [:mix], [], "hexpm", "20cf780119be89a7cae7808543ebc8b7e56f0993b6aa058260fab686afe9073d"}, "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "ecto": {:hex, :ecto, "3.14.2", "99db28a864293a789c970651de711e3cae184291e0e7ea1166c54055ac41c1f3", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25d60b8c816a07d19d85b80bdf60978bd8b102209dda198d768cd7c6745339a6"}, "ecto_sql": {:hex, :ecto_sql, "3.14.0", "06446ab8410d2f85bfbb80857ee224ab3b693700cbb38f6535d507449a627b2e", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.14.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.8", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f4d8d36faf294c9417b5a37ec7ac8217ee2abdef5fcf197ba690f361548d3949"}, @@ -27,9 +29,12 @@ "glob_ex": {:hex, :glob_ex, "0.1.12", "7b2d9369c20e2697efcfd185d13d6e84c94cd3bfd2730fbde613141c2e015c00", [:mix], [], "hexpm", "2e2fac83f113514434c7eaf267b4c38af2f91766f1cab2c5db7053b7fc1ee0bb"}, "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "0435d4ca364a608cc75e2f8683d374e55abbae26", [tag: "v2.2.0", sparse: "optimized", depth: 1]}, "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, + "idna": {:hex, :idna, "7.1.0", "1067a13043538129602d2f2ce6899d8713125c7d19734aa557ce2e3ea55bd4f1", [:rebar3], [], "hexpm", "6ae959a025bf36df61a8cab8508d9654891b5426a84c44d82deaffd6ddf8c71f"}, "igniter": {:hex, :igniter, "0.8.4", "f79f1bbdc2fb7b9ca030a22d12a585b060cbf5b94b9d3f23b1148578a9e05d11", [:mix], [{:ex_ast, "~> 0.5", [hex: :ex_ast, repo: "hexpm", optional: false]}, {:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4.5", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:phx_new, "~> 1.7", [hex: :phx_new, repo: "hexpm", optional: true]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:rewrite, ">= 1.1.1 and < 2.0.0-0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "a9b1cbec996ccb100b4f7d8130129b2dd3f18eb4224ac9a0e907e428ca90dbd7"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "jsv": {:hex, :jsv, "0.23.0", "db238039fc2e437bca3e226b09eacd72ebd588d6e014d0d4150330f6ca6762fa", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:idna, "~> 6.0 or ~> 7.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:texture, ">= 1.2.1", [hex: :texture, repo: "hexpm", optional: false]}], "hexpm", "3876f6ada437b3a7ec6214c9b942bd218f25d245b92a7970a034a7cf06dbe925"}, "lazy_html": {:hex, :lazy_html, "0.1.12", "31a55ee622918fce988c94b06232227b42daa64e4eab14ac32081d0f3fd8db6f", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "8a0da594776caee58782c6f93b2abaa5bdb809daf8d43351a561f7de9dc2e2a8"}, + "llm_db": {:hex, :llm_db, "2026.9.4", "f56fdc8012477a0fa33247841f980c9a71e80408a5b69f63692ea238f46f8636", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "196a162cfc8826746ee84eeb395dd3a17268bfff2154ba378799bc1511e722ce"}, "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, @@ -54,19 +59,26 @@ "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, "postgrex": {:hex, :postgrex, "0.22.4", "d271f595dfd25230b6398354e19d17bb5e2d20130fd2d9bdca7e15f125d43552", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "4aae45a2d60e35b04eea2602440be152fae332901f1fc7a60fc7cb7f0f9a9c5a"}, "req": {:hex, :req, "0.7.4", "23e9ffec17de032a46a4b15ed65c09793893bf4a7c680f4bbf6227fce6bdf74d", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b192d63253e8dcc6221ef992ea9ebef7d3555166e8423aa5b553e86bc3c69a2"}, + "req_llm": {:hex, :req_llm, "1.24.0", "cdc5c5cd7f38c0e17cf7a949ced99b6760dcba0d860cd4d066fea1aeba422929", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, ">= 2026.9.3 and < 2027.0.0", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "8ab6bda68e28afc3adf070cc6616d51831b42b8ff144116a9f104999af4c763f"}, "rewrite": {:hex, :rewrite, "1.3.0", "67448ba7975690b35ba7e7f35717efcce317dbd5963cb0577aa7325c1923121a", [:mix], [{:glob_ex, "~> 0.1", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.0", [hex: :sourceror, repo: "hexpm", optional: false]}, {:text_diff, "~> 0.1", [hex: :text_diff, repo: "hexpm", optional: false]}], "hexpm", "d111ac7ff3a58a802ef4f193bbd1831e00a9c57b33276e5068e8390a212714a5"}, + "server_sent_events": {:hex, :server_sent_events, "1.1.0", "54606238b9182ba673a10ce90bd95f81a0939c1baa91cdf86b66acc04edd1832", [:mix], [], "hexpm", "8e164db8e295a2d869a8faafbf4a1eeaa749b62fe93f66271adff6394d93ce15"}, "sobelow": {:hex, :sobelow, "0.15.0", "b067d7f8522a9d758fa89cb2bfcbab7ad72c45a0993cb958c989c6fd956fdd56", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "24a800e2d7fa8c3bd21561b6ad8ad4745ed726a09fd606598981d9048708da98"}, "sourceror": {:hex, :sourceror, "1.12.3", "f58eebef0765c7a369a49a755ab2a5ee88d92777f403ea5a08b722cefcc37f51", [:mix], [], "hexpm", "d5f2f37099de794840f08c54ae546d7f6e4ea015e397be64aebe4996fa9f7da7"}, "spitfire": {:hex, :spitfire, "0.4.2", "5c719208d4eeb810e5b2a2aa1024d1e4b2974b7ca422c274b2486666e5159735", [:mix], [], "hexpm", "9bbbbffe93e6f88ccf193487ef56b83c2646a6dc3875bb0975d4654bfe96c5bb"}, + "splode": {:hex, :splode, "0.3.2", "7716b6b2260a98a6f018c65cc0393da2cdf17314202eb351a0956e8762190dff", [:mix], [], "hexpm", "08fd658f80da7f1cd254b149164dcff8acd44b22f032001d5416b97223d32bc9"}, "tailwind": {:hex, :tailwind, "0.5.1", "35435b13158c90d37da11e1cfc808755fca1d7b6c5ab87b1b19c5de87e2f0a10", [:mix], [], "hexpm", "c4e26302a59fec72abc5610ecb6ad2116d9aa31f31aab2d4b8eb6e95d25a689c"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, "telemetry_metrics": {:hex, :telemetry_metrics, "1.2.0", "7632c19c01d88d8aaca5da1a0e8912f5af39b79e7c08a2c253aeb3c14c2c957e", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "71dde12fc29b58b9c77ec17ec319109e5ca848d010fc1965ed4463bba1837c07"}, "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"}, "text_diff": {:hex, :text_diff, "0.1.0", "1caf3175e11a53a9a139bc9339bd607c47b9e376b073d4571c031913317fecaa", [:mix], [], "hexpm", "d1ffaaecab338e49357b6daa82e435f877e0649041ace7755583a0ea3362dbd7"}, + "texture": {:hex, :texture, "1.2.1", "ae8d875fb099bcc1ccb0269296bf6c4590116eff9a44895049379359a14d5fca", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}], "hexpm", "925b1938891ce5c1d589df408faa7ed57dd872e4aac9521b32495a29b265cb17"}, "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, + "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.6.0", "73db5ab8aaefd1a876a97ce3e6afc96562625de69ef17a4e04426e034849d0b8", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "50021a85bce8f203b086705d9e0c5415e2c7eb05d319111b0428fe71f9934617"}, + "websockex": {:hex, :websockex, "0.5.1", "9de28d37bbe34f371eb46e29b79c94c94fff79f93c960d842fbf447253558eb4", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8ef39576ed56bc3804c9cd8626f8b5d6b5721848d2726c0ccd4f05385a3c9f14"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, "yaml_elixir": {:hex, :yaml_elixir, "2.12.2", "9dd1330fb4cd9a36a7b0f502e5b12486eff632792ee4a5f0eba52a4d4ec32c9c", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "e7c1b10122f973e6558462d51c39026ba0e14afbc6745318e990ea82cfe9e159"}, + "zoi": {:hex, :zoi, "0.18.7", "0d6b09d19fd1feff4340b7c5660bab04fbc80c1642ee1e5c75f06d527ac326db", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "5fedddd755dec84a5e78b3671070a5e595026aa3479f7fa566a42b8c4e4e5ff2"}, } diff --git a/priv/repo/migrations/20260920150000_create_usage_events.exs b/priv/repo/migrations/20260920150000_create_usage_events.exs new file mode 100644 index 0000000..e45a4dd --- /dev/null +++ b/priv/repo/migrations/20260920150000_create_usage_events.exs @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.Repo.Migrations.CreateUsageEvents do + @moduledoc "Slice 011. One row per completed LLM call; the cost ledger of slice 090 reads it." + use Ecto.Migration + + def change do + create table(:usage_events, primary_key: false) do + add :id, :binary_id, primary_key: true + add :model_id, :string, null: false + add :provider, :string, null: false + add :kind, :string, null: false + add :input_tokens, :integer, null: false, default: 0 + add :output_tokens, :integer, null: false, default: 0 + add :cached_tokens, :integer, null: false, default: 0 + add :reasoning_tokens, :integer, null: false, default: 0 + add :cost_usd, :float, null: false, default: 0.0 + add :session_id, references(:sessions, type: :binary_id, on_delete: :nilify_all) + add :provider_meta, :map, null: false, default: %{} + add :inserted_at, :utc_datetime_usec, null: false + end + + create index(:usage_events, [:session_id]) + create index(:usage_events, [:inserted_at]) + create index(:usage_events, [:model_id, :inserted_at]) + end +end diff --git a/slices/011-llm-provider-layer/NOTES.md b/slices/011-llm-provider-layer/NOTES.md new file mode 100644 index 0000000..12463bc --- /dev/null +++ b/slices/011-llm-provider-layer/NOTES.md @@ -0,0 +1,187 @@ +# Slice 011: NOTES + +## G1 plan, 2026-09-20 + +Tree at `6af7ee6` on `main` (010 approved); branch `slice/011-llm-provider-layer`; ROADMAP row 011 set to +`in_progress` in this commit. req_llm 1.24.0 was read from its Hex tarball before this plan was written (public +entry points `generate_text/3`, `stream_text/3`, `generate_object/4`, `embed/3`; stream chunk types `:content`, +`:thinking`, `:tool_call`, `:meta`; usage keys `input_tokens`, `output_tokens`, `total_tokens`, `cached_tokens`, +`reasoning_tokens`, `total_cost`; model spec `"provider:model"`; keys from a per-request `:api_key` option, the +`:req_llm` application env, or the provider's env var). Each line names its test; the order is the build order. + +1. Dependencies: `req_llm` at 1.24.0 inside the `~> 1.22` pin, `mox` for tests. What enters `mix.lock` with + req_llm is counted and its licences listed in NOTES.md, because the desktop binary carries it. VERSIONS rows + flipped by `mix versions.gen`. Test: `mix hex.audit` and `mix deps.audit` green; `versions.verify` OK. +2. `Trinity.LLM.Event`: the seven event shapes from SLICE.md as a typespec plus `valid?/1`, the single source of + truth 012 and 013 consume. `Trinity.LLM.Request` struct (system, messages, tools, model, params) with a + changeset-free `new/1` that validates roles and tool schemas. Tests: every shape accepted, a foreign shape + refused. +3. `Trinity.LLM.Provider` behaviour: `stream/3`, `generate/2`, `generate_object/3`, `embed/2`, `models/0`, + `capabilities/1`. `Trinity.LLM.Registry` from `config :trinity, :llm` (models as `{id, provider, model, caps, + price}`, `default_model`), `models/0`, `default_model/0`, `lookup/1`. Test: a registry entry resolves to its + provider module; an unknown id is refused by name. +4. `Trinity.LLM` public API, resolving the provider from the registry entry for the request's model; `stream/3` + (function) and `stream_to/3` (pid, events as messages); `Trinity.LLM.ProviderMock` (Mox) as the test provider + for the registry's `fake` provider. Boundary `Trinity.LLM` with `deps: [Trinity]`, exporting the API, + `Request` and `Event`. Test (AC7): switching `default_model` in config changes the provider used, no code + change. +5. `Trinity.LLM.Providers.Fake` in `test/support`: streams a scripted response (deltas, one tool call in three + chunks, usage, done), scriptable to raise transient or non-transient errors N times. Test (AC1): the full + event sequence, in order. +6. `Trinity.LLM.Providers.ReqLLM`: `Request` to req_llm messages and `ReqLLM.Tool`; `stream_text` chunks to + events with tool calls assembled (start on the first chunk naming a call, deltas while arguments grow, end + when the call is complete or the stream ends); `:meta` to `{:usage, _}` and `{:done, reason}`; `generate`, + `generate_object` (req_llm's JSON schema path), `embed`. Provider mapping for `anthropic`, `openai`, + `openrouter`, `google`, `openai_compatible` (the openai provider with `base_url` and a per-request key). + Errors classified: timeouts, 429, 5xx, connection refused are transient; 4xx otherwise are not. Unit tests + over recorded chunk sequences; live tests in line 9. +7. `Trinity.LLM.Retry`: attempts and base backoff from config; transient errors retried, non-transient returned + at once; optional fallback model list tried in order after the attempts. Test (AC4): the fake raises a + transient error twice then succeeds; raises N+1 times then `{:error, _}`; a non-transient error returns + immediately with one attempt. +8. `usage_events` migration and `Trinity.LLM.Usage`: one row per completed call with model, provider, tokens, + and cost computed from the registry price; req_llm's own `total_cost` kept in `provider_meta` and never + used. Test (AC5): a row exists after a fake call with the expected cost; the Postgres job applies the + migration. +9. Live tests, `@tag :live`, excluded by default (NetworkGuard's opt-in path): OpenRouter with + `TRINITY_LIVE_MODEL` and NVIDIA through `openai_compatible` with `NEMOTRON_BASE_URL`, `NEMOTRON_MODEL` and the + key as `:api_key`; a preflight that the model id is on `GET /models` and names a 410 as end of life. Cases: + a streamed completion with deltas and done; a tool call; `generate_object` against a schema (AC2); + `embed` against `nvidia/nemotron-3-embed-1b` with the declared dimension (AC3). Output pasted, keys + redacted (AC6). +10. `Trinity.Config.secret/1`: environment now, keychain at slice 100; the adapter reads keys only through it. + Test: a missing key is a named error, not a nil passed to the provider. +11. Anthropic prompt-cache hint: a `cache: true` param maps to req_llm's cache-control option for the anthropic + provider and is dropped for others. Unit test on the mapping; not measured live (no Anthropic key here). +12. Gate, coverage row, PROOF.md, ROADMAP to `done`, through a pull request; tag `slice/011` after the merge. + +Manual verification queue, as SLICE.md tags them: AC2 (live `generate_object`) and AC6 (`mix test --only live`, +output pasted with keys redacted). Both run on this machine with the owner's keys in `.env`; the owner reads +the pasted output at G4. The developer machine, the model ids and the probe outputs behind the choice: + +- OpenRouter, `inclusionai/ling-3.0-flash-vl:free` (owner's pick): tool call `get_weather` with + `{"city": "Paris"}`, 298 tokens, served by Novita; streamed completion 59 chunks. +- NVIDIA endpoint, `nvidia/nemotron-3.5-lightning-30b-a3b`: tool call, 317 tokens; streamed completion 20 + chunks. The id first set, `nvidia/llama-3.3-nemotron-super-49b-v1.5`, answered `410 Gone` (end of life + 2026-08-26); that is why line 9 carries a preflight. + +Deviations from SLICE.md, stated before building: the five provider mappings are written; only OpenRouter and +the NVIDIA endpoint are measured live, and PROOF.md says so per provider. `Trinity.LLM.Supervisor` from docs/01 +(rate limiters) is not built here: nothing in this slice needs a process, and a supervisor with no children +would be a claim; 012 adds it when the Session needs one. Recorded as a follow-up. + +## Line 1, 2026-09-20: the dependencies, counted + +`req_llm ~> 1.22` resolved to 1.24.0 and `mox ~> 1.2` to 1.3.1. `mix.lock` went from 68 to 80 packages. The +twelve that entered, with licence from hex.pm metadata (`curl -s https://hex.pm/api/packages/ | jq +.meta.licenses`): + +| package | version | licence | why req_llm needs it | +|---|---|---|---| +| req_llm | 1.24.0 | Apache-2.0 | the provider layer | +| llm_db | 2026.9.4 | Apache-2.0 | its model and price database, dated | +| dotenvy | 1.2.1 | Apache-2.0 | reads `.env` for provider keys | +| jsv | 0.23.0 | Apache-2.0 | JSON schema validation for structured output | +| zoi | 0.18.7 | Apache-2.0 | its struct schemas | +| splode | 0.3.2 | MIT | its error classes | +| server_sent_events | 1.1.0 | MIT | SSE parsing for streaming | +| websockex | 0.5.1 | MIT | a realtime transport Trinity does not use | +| texture | 1.2.1 | Apache-2.0 | transitive | +| toml | 0.7.0 | Apache-2.0 | transitive | +| abnf_parsec | 2.1.0 | MIT | transitive (idna) | +| idna | 7.1.0 | MIT | transitive | + +Plus `mox` 1.3.1 (Apache-2.0), test only. Every licence is Apache-2.0 or MIT. `mix hex.audit`: no retired or +advisory packages. `mix deps.audit`: no vulnerabilities. `versions.verify`: OK, 80 locked, 46 pins. Binary size +delta is measured at the next package run, not estimated here. + +One thing to know about `dotenvy`: req_llm's key lookup reads `.env` through it at startup. Trinity's `.env` is +gitignored and holds the owner's keys, so in the default test run the keys may be present in the environment +while the network guard still refuses every connection; the live tag is what opens the network, not the +presence of a key. Line 10 routes every key through `Trinity.Config.secret/1` anyway. + +## Lines 2 to 11, 2026-09-20: what was built, and what the live suite found + +**Built.** `Trinity.LLM.Event` (seven shapes, `valid?/1`); `Trinity.LLM.Request` (`new/1`, `new!/1`, roles and +tools validated); `Trinity.LLM.Error` (`transient?`, `status`); `Trinity.LLM.Provider` behaviour; +`Trinity.LLM.Registry` over `config :trinity, :llm` (read at call time, which is what makes AC7 a test); +`Trinity.LLM` (`stream/3`, `stream_to/3` under `Trinity.LLM.TaskSupervisor`, `generate/2`, `generate_object/3`, +`embed/2`, `models/0`, `default_model/0`, `capabilities/1`); `Trinity.LLM.Retry` (attempts, base backoff, no +jitter, stated); `Trinity.LLM.Usage` and the `usage_events` migration; `Trinity.Config.secret/1`; +`Trinity.LLM.Providers.ReqLLM` (the half that talks) and `Trinity.LLM.Providers.ReqLLM.Mapping` (the pure half: +chunks to events, responses to results, errors to `Error`); `Trinity.LLM.Providers.Fake` in test/support; the +Mox mock; `config/llm.exs` as the registry's own file. The registry's model strings are `":"` +split on the first colon; the provider name is one of the five the spec names, mapped to req_llm's atom +through a fixed map, so a name from config never mints an atom (sobelow found the first version doing so). + +**The live suite found four defects, each fixed with its reason left in the code.** + +1. `Trinity.LLM.provider_opts/2` merged the caller's opts over the entry's, so `embed(texts, model: + "nvidia:embed")` handed req_llm a provider named `nvidia`. Entry keys now win. +2. req_llm's streaming `receive_timeout` defaults to 30 s; the NVIDIA reasoning model exceeded it before its + first byte under load. The adapter passes `receive_timeout_ms` from config, 120 000 by default; the embedding + call validates a different option set (no `receive_timeout`), so the budget travels as `total_timeout` there. +3. An upstream `429` from OpenRouter's free pool arrived inside a `ReqLLM.Error.API.Stream` whose own `status` + was nil, and the first classifier called it permanent. `classify/1` now reads the innermost error that + carries a status, a `retryable` flag or a timeout cause. Pinned by `mapping_test.exs`. +4. An inline model spec without capabilities is refused for embeddings by req_llm ("does not support embedding + operations"); the spec now carries `capabilities: %{embeddings: true}` for a registry entry with `:embed`. + +**Model ids.** req_llm's catalog does not know the free OpenRouter id or the NVIDIA ids and warned about +"unverified" models; the adapter uses an inline spec (`%{provider:, id:, capabilities:}`) because Trinity's +registry is the catalog. The NVIDIA embedding model answers with dimension **2048**. + +**The tool-call assembly** follows req_llm 1.24.0's default decoder: a `:tool_call` chunk with `id`, `index` and +`expects_arg_fragments` opens a call; `:meta` chunks carry `tool_call_args` fragments by index; the finish +reason closes every open call. Reproduced with the library's own `ReqLLM.StreamChunk` constructors in +`mapping_test.exs` (13 tests), so a change in that shape is a red here first. + +**Live suite, this machine, 2026-09-20** (`TRINITY_LIVE=1 mix test --only live`, keys from `.env`, redacted; +OpenRouter `inclusionai/ling-3.0-flash-vl:free` served by Novita; NVIDIA `nvidia/nemotron-3.5-lightning-30b-a3b` +and `nvidia/nemotron-3-embed-1b` at `integrate.api.nvidia.com/v1`): + +``` +* test nvidia:nemotron preflight: the configured model is on the provider's list [L#56] * test nvidia:nemotron preflight: the configured m +* test nvidia:nemotron a streamed completion yields text deltas, usage and done [L#62] * test nvidia:nemotron a streamed completion yields +* test openrouter:ling preflight: the configured model is on the provider's list [L#56] * test openrouter:ling preflight: the configured m +* test nvidia:nemotron generate_object returns a map that validates against the schema (AC2) [L#121] * test nvidia:nemotron generate_objec +* test openrouter:ling a tool call arrives as start, end, and a done of tool_calls [L#85] * test openrouter:ling a tool call arrives as st +live embed: dimension 2048 +* test embed returns vectors of the declared dimension against the NVIDIA embedding model (AC3) (467.4ms) [L#148] +* test openrouter:ling a streamed completion yields text deltas, usage and done [L#62] * test openrouter:ling a streamed completion yields +* test nvidia:nemotron a tool call arrives as start, end, and a done of tool_calls [L#85] * test nvidia:nemotron a tool call arrives as st +* test openrouter:ling generate_object returns a map that validates against the schema (AC2) [L#121] * test openrouter:ling generate_objec +live usage row: 25 in, 14 out, provider_cost nil +* test a live call writes one usage_events row with the tokens the provider reported (740.8ms) [L#160] +Finished in 150.6 seconds (0.1s async, 150.5s sync) +Result: 10 passed, 138 excluded +``` + +An earlier run took 219.8 s and passed 10 of 10 as well; a run before the fixes above passed 6 of 10 and then +8 of 10, which is the record of finding them. OpenRouter's free pool returned `429` several times during these +runs; req_llm retries a 429 three times on its own and Trinity's retry sits above that. + +**Coverage** 51.57% (up 6.69 from 010). `Trinity.LLM.Providers.ReqLLM` itself reads 0% in the default run: it +is the half that talks, and only the live suite reaches it; `Mapping` reads 86%. + +``` +$ mix gate โ†’ exit 0; 138 passed, 10 excluded; plan_check: PASS +$ mix test --only live โ†’ 10 passed (above) +$ mix trinity.coverage โ†’ 011 51.57% vs 010 44.88%: OK +``` + +**Deviations from SLICE.md, in addition to the two stated at G1.** The `usage_events` columns are +`input_tokens`/`output_tokens` (the Event usage keys) rather than `prompt_tokens`/`completion_tokens`, and there is +no `latency_ms` column: latency is a Telemetry measurement at 090; docs/05 now says both. `models/0` on the +req_llm provider returns `[]`: req_llm's catalog is not Trinity's registry and listing it would be a claim +about models Trinity has not configured. `capabilities/1` on the adapter is the fixed `[:stream, :tools, +:json]`; the registry entry is the source callers use. + +## Follow-ups +- `Trinity.LLM.Supervisor` (docs/01, rate limiters): when 012's Session needs a process. Only the task + supervisor for `stream_to/3` exists. +- Prompt-cache hint for Anthropic: mapped (`cache: true` becomes `provider_options: [cache_control: ...]` for + the anthropic provider, dropped for others), not measured: no Anthropic key here. +- The `:google` and `:anthropic` and plain `:openai` mappings are written and not measured live. +- OpenRouter's free pool rate-limits under repeated runs; a paid key or a second free id in the registry would + make the live suite steadier. `nvidia/nemotron-3.5-lightning:free` passed the same probe on 2026-09-20. diff --git a/slices/011-llm-provider-layer/PROOF.md b/slices/011-llm-provider-layer/PROOF.md new file mode 100644 index 0000000..4326d56 --- /dev/null +++ b/slices/011-llm-provider-layer/PROOF.md @@ -0,0 +1,155 @@ +# Proof for slice 011: LLM provider layer + +Agent: Trinity ยท Coding Agent ยท Date: 2026-09-20 ยท Branch: slice/011-llm-provider-layer ยท Final commit: `49005dc` (filled by the commit after it) + +## Summary +`Trinity.LLM` is the one door to a model: a registry id names the provider module, transient errors retry with +backoff, and a completed call writes one `usage_events` row with cost from the registry price. The seven event +shapes 012 and 013 consume are fixed in `Trinity.LLM.Event`. The req_llm adapter is split into the half that +talks and a pure `Mapping` half tested over recorded chunk sequences. The live suite against OpenRouter and the +NVIDIA endpoint found four defects on its first runs (a merge-order bug, a 30-second stream timeout, a wrapped +429 called permanent, an embedding capability the inline spec had to declare); each is fixed with its reason in +the code and pinned by a test. Deferred: `Trinity.LLM.Supervisor` until a Session needs a process (NOTES.md +Follow-ups). Keys reach a provider only through `Trinity.Config.secret/1`; none appears in this file. + +## Gate +``` +$ mix gate (this machine, OTP 28.5.0.5, Elixir 1.20.4, under a 32 GiB cgroup) +385 mods/funs, found no issues. +No vulnerabilities found. +Result: 138 passed, 10 excluded +trinity.coverage: 010 44.88% vs 001 30.37%: OK +plan_check: PASS +exit=0 +``` + +## Tests +``` +$ mix test --cover +Result: 138 passed, 10 excluded +| 51.57% | Total | +| 86.11% | Trinity.LLM.Providers.ReqLLM.Mapping | +| 0.00% | Trinity.LLM.Providers.ReqLLM | (the half that talks; the live suite covers it) +``` +`coverage.tsv` row: `011 51.57 ec5334a 2026-09-20`. `trinity.coverage: 011 51.57% vs 010 44.88%: OK`. + +## Acceptance criteria evidence + +### AC1: FakeProvider-driven test shows a full event sequence: text deltas, tool_call events, usage, done +``` +$ mix test test/trinity/llm/llm_test.exs --trace +* test default_model (AC7) an unknown model id is refused by name +* test default_model (AC7) an unknown model id is refused by name (0.06ms) +* test default_model (AC7) switching default_model in config changes the provider used, with no code change +* test default_model (AC7) switching default_model in config changes the provider used, with no code change (0.4ms) +* test embed/2 (AC3) returns one vector per text of the declared dimension +* test embed/2 (AC3) returns one vector per text of the declared dimension (0.1ms) +* test retry (AC4) a permanent error returns at once with one attempt +* test retry (AC4) a permanent error returns at once with one attempt (0.05ms) +* test retry (AC4) a transient error is retried and then succeeds +* test retry (AC4) a transient error is retried and then succeeds (4.8ms) +* test retry (AC4) attempts exhausted returns the error, named as exhausted +* test retry (AC4) attempts exhausted returns the error, named as exhausted (4.3ms) +* test retry (AC4) no usage row is written for a failed call +* test retry (AC4) no usage row is written for a failed call (0.1ms) +* test stream/3 (AC1) emits the full sequence: text deltas, tool call start, deltas, end, usage, done +* test stream/3 (AC1) emits the full sequence: text deltas, tool call start, deltas, end, usage, done (0.6ms) +* test stream/3 (AC1) stream_to/3 delivers the same events as messages and then llm_done +* test stream/3 (AC1) stream_to/3 delivers the same events as messages and then llm_done (0.1ms) +* test usage_events (AC5) embed and object calls record their own kinds +* test usage_events (AC5) embed and object calls record their own kinds (23.7ms) +* test usage_events (AC5) one row per completed call, cost from the registry price +* test usage_events (AC5) one row per completed call, cost from the registry price (0.2ms) +* test usage_events (AC5) the session id is recorded when given +* test usage_events (AC5) the session id is recorded when given (6.3ms) +Result: 20 passed +``` +The AC1 test asserts the exact list: two `text_delta`, `tool_call_start`, two `tool_call_delta`, +`tool_call_end` with `%{"city" => "Paris"}`, `usage`, `done :tool_calls`; every event passes `Event.valid?/1`. +`stream_to/3` delivers the same as messages with a `{:llm_done, ref, result}` last. + +### AC2: [manual] generate_object/3 returns a validated map for a JSON schema (fake) and (live) for one real provider +Fake: `generate_object/3 returns a map shaped by the schema` (above). Live, both providers, from the live trace +below: `openrouter:ling generate_object returns a map that validates against the schema (AC2)` and +`nvidia:nemotron generate_object ...` both pass; the assertion is a map with a string `city` matching Paris and a +numeric `population_millions`, both required by the schema. + +### AC3: embed/2 returns vectors of the declared dimension (fake + live) +Fake: `embed/2 (AC3) returns one vector per text of the declared dimension` (8, from the registry entry). +Live: `live embed: dimension 2048` against `nvidia/nemotron-3-embed-1b`, two texts, two vectors, every element a +float. + +### AC4: transient error retried N times then {:error, _}; non-transient immediate +``` +* test retry (AC4) a transient error is retried and then succeeds (fails twice on 429, third call succeeds; 3 calls) +* test retry (AC4) attempts exhausted returns the error, named as exhausted ({:exhausted, 3, :down}; 3 calls) +* test retry (AC4) a permanent error returns at once with one attempt (401; 1 call) +* test retry (AC4) no usage row is written for a failed call +* test Retry.run/2 backs off exponentially and stops at the attempt count (sleeps 10, 20, 40 ms for 4 attempts) +``` + +### AC5: a usage_events row per completed call with tokens and cost from the registry price +``` +* test usage_events (AC5) one row per completed call, cost from the registry price + model_id "fake:chat", provider "fake", kind "chat", 10 in, 5 out, + cost_usd 0.00002 (10 tokens at $1.00 per million + 5 at $2.00 per million), provider_meta %{"provider_cost" => nil} +* test usage_events (AC5) the session id is recorded when given +* test usage_events (AC5) embed and object calls record their own kinds +``` +Live: `live usage row: 25 in, 14 out, provider_cost nil` from a real OpenRouter call, `cost_usd 0.0` because the +registry prices the free tier at 0. + +### AC6: [manual] mix test --only live passes against at least one configured provider (output pasted; keys redacted) +Two providers, this machine, 2026-09-20. Keys loaded from `.env` (gitignored) and redacted by pattern in this +transcript; none appears below. +``` +$ set -a; . ./.env; set +a; TRINITY_LIVE=1 mix test --only live --trace +* test nvidia:nemotron preflight: the configured model is on the provider's list [L#56] * test nvidia:nemotron preflight: the configured m +* test nvidia:nemotron a streamed completion yields text deltas, usage and done [L#62] * test nvidia:nemotron a streamed completion yields +* test openrouter:ling preflight: the configured model is on the provider's list [L#56] * test openrouter:ling preflight: the configured m +* test nvidia:nemotron generate_object returns a map that validates against the schema (AC2) [L#121] * test nvidia:nemotron generate_objec +* test openrouter:ling a tool call arrives as start, end, and a done of tool_calls [L#85] * test openrouter:ling a tool call arrives as st +live embed: dimension 2048 +* test embed returns vectors of the declared dimension against the NVIDIA embedding model (AC3) (467.4ms) [L#148] +* test openrouter:ling a streamed completion yields text deltas, usage and done [L#62] * test openrouter:ling a streamed completion yields +* test nvidia:nemotron a tool call arrives as start, end, and a done of tool_calls [L#85] * test nvidia:nemotron a tool call arrives as st +* test openrouter:ling generate_object returns a map that validates against the schema (AC2) [L#121] * test openrouter:ling generate_objec +live usage row: 25 in, 14 out, provider_cost nil +* test a live call writes one usage_events row with the tokens the provider reported (740.8ms) [L#160] +Finished in 150.6 seconds (0.1s async, 150.5s sync) +Result: 10 passed, 138 excluded +``` +Each provider ran a preflight against its `GET /models` (an id that reached end of life is a named refusal), +a streamed completion, a tool call and a structured object; the embedding model ran once; one real call wrote +a usage row. The earlier runs that found the four defects are recorded in NOTES.md as 6/10 then 8/10. + +### AC7: switching default_model in config changes the provider used with no code change +``` +* test default_model (AC7) switching default_model in config changes the provider used, with no code change + default "fake:chat" streams from the fake; Application.put_env(... default_model: "mock:chat") and the same + request streams from the Mox mock, which asserts it received the entry's model name "chat". +* test default_model (AC7) an unknown model id is refused by name +``` + +## Manual verification for the reviewer +AC2 and AC6 are the live runs above; the reviewer reads the transcript and, with keys in `.env`, may re-run +`TRINITY_LIVE=1 mix test --only live`. OpenRouter's free pool rate-limits under repeated runs (429, retried). + +## Deviations from SLICE.md +See NOTES.md: five provider mappings written, two measured live; no `Trinity.LLM.Supervisor` until a process +needs one; `usage_events` columns are the Event usage keys and carry no latency (090's telemetry measures it); +the adapter's `models/0` returns `[]` and `capabilities/1` is fixed, the registry being the source. + +## Versions touched +`VERSIONS.md` updated: yes. `req_llm` 1.24.0 and `mox` 1.3.1 read in `mix.lock`. Twelve packages entered the +lock with req_llm, listed with licences in NOTES.md line 1; all Apache-2.0 or MIT. `mix hex.outdated` not run. + +## Git +``` +$ git log --oneline main..HEAD +d53bfec feat(s011): the adapter's pure half is a module with recorded-chunk tests; coverage row; docs/05 synced +ec5334a feat(s011): the provider layer: behaviour, events, registry, req_llm adapter, retry, usage rows, fake and live suites +73caeb5 feat(s011): req_llm 1.24.0 and mox, with what they bring counted and licensed +b9cbcf2 docs(s011): G1 plan, and the slice opens +49005dc feat(s011): complete slice 011 (LLM provider layer) +``` diff --git a/test/support/fake_provider.ex b/test/support/fake_provider.ex new file mode 100644 index 0000000..b5df87a --- /dev/null +++ b/test/support/fake_provider.ex @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Providers.Fake do + @moduledoc """ + A scripted provider for tests. Slice 011. The default script streams two text deltas, one + tool call in three chunks, usage and done. A test overrides the script through the process + dictionary of the calling process (`script/1`), or asks for `n` failures before success + (`fail/2`), so the retry policy is exercised without a network. + """ + @behaviour Trinity.LLM.Provider + + alias Trinity.LLM.Error + + @default_script [ + {:text_delta, "Hello, "}, + {:text_delta, "world."}, + {:tool_call_start, "call_1", "get_weather"}, + {:tool_call_delta, "call_1", ~s({"city":)}, + {:tool_call_delta, "call_1", ~s("Paris"})}, + {:tool_call_end, "call_1", %{"city" => "Paris"}}, + {:usage, %{input_tokens: 10, output_tokens: 5}}, + {:done, :tool_calls} + ] + + @doc "Sets the events the next stream emits, for the calling process." + @spec script([Trinity.LLM.Event.t()]) :: :ok + def script(events) do + Process.put({__MODULE__, :script}, events) + :ok + end + + @doc "Makes the next `n` calls fail with `error` before succeeding." + @spec fail(non_neg_integer(), Error.t()) :: :ok + def fail(n, %Error{} = error) do + Process.put({__MODULE__, :fail}, {n, error}) + :ok + end + + @doc "How many calls the provider has served in this process." + @spec calls() :: non_neg_integer() + def calls, do: Process.get({__MODULE__, :calls}, 0) + + @impl true + def stream(_request, _opts, emit) do + with :ok <- maybe_fail() do + events = Process.get({__MODULE__, :script}, @default_script) + Enum.each(events, emit) + {:ok, usage_of(events)} + end + end + + @impl true + def generate(_request, _opts) do + with :ok <- maybe_fail() do + {:ok, + %{ + text: "Hello, world.", + tool_calls: [%{id: "call_1", name: "get_weather", args: %{"city" => "Paris"}}], + usage: %{input_tokens: 10, output_tokens: 5}, + finish: :tool_calls + }} + end + end + + @impl true + def generate_object(_request, schema, _opts) do + with :ok <- maybe_fail() do + object = + schema + |> Map.get("properties", %{}) + |> Map.new(fn + {k, %{"type" => "integer"}} -> {k, 42} + {k, %{"type" => "number"}} -> {k, 4.2} + {k, %{"type" => "boolean"}} -> {k, true} + {k, _} -> {k, "fake"} + end) + + {:ok, object, %{input_tokens: 8, output_tokens: 4}} + end + end + + @impl true + def embed(texts, opts) do + with :ok <- maybe_fail() do + dim = Keyword.get(opts, :dim, 8) + {:ok, Enum.map(texts, fn _ -> List.duplicate(0.5, dim) end), %{input_tokens: length(texts)}} + end + end + + @impl true + def models, do: ["chat", "embed"] + + @impl true + def capabilities("embed"), do: [:embed, {:embed_dim, 8}] + def capabilities(_), do: [:stream, :tools, :json] + + defp maybe_fail do + Process.put({__MODULE__, :calls}, calls() + 1) + + case Process.get({__MODULE__, :fail}) do + {n, error} when n > 0 -> + Process.put({__MODULE__, :fail}, {n - 1, error}) + {:error, error} + + _ -> + :ok + end + end + + defp usage_of(events) do + Enum.find_value(events, %{}, fn + {:usage, usage} -> usage + _ -> nil + end) + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 9e61a86..b97368f 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,4 +1,10 @@ # SPDX-FileCopyrightText: Sudo Apt Holdings LLC # SPDX-License-Identifier: Apache-2.0 -ExUnit.start() +# docs/03: the default run excludes :live (real providers, opt-in with `mix test --only live` +# and TRINITY_LIVE=1) and :desktop (needs the Tauri shell). Slice 011 made this explicit; until +# then a :live test would have run in the default suite and been refused by the network guard. +ExUnit.start(exclude: [:live, :desktop]) Ecto.Adapters.SQL.Sandbox.mode(Trinity.Repo, :manual) + +# Slice 011: the Mox mock the registry's :mock provider points at. +Mox.defmock(Trinity.LLM.ProviderMock, for: Trinity.LLM.Provider) diff --git a/test/trinity/llm/live_test.exs b/test/trinity/llm/live_test.exs new file mode 100644 index 0000000..2f98867 --- /dev/null +++ b/test/trinity/llm/live_test.exs @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.LiveTest do + @moduledoc """ + Slice 011 AC2, AC3, AC6: the req_llm adapter against real providers. Opt-in: + + set -a; . ./.env; set +a; TRINITY_LIVE=1 mix test --only live + + Reads the registry the way dev and prod do (config/config.exs), not the test fake. Each + provider's model id is checked against its `GET /models` listing first, so an id that reached + end of life is a named refusal (410) and not a mysterious failure downstream. Nothing here runs + in the default suite; keys are never printed. + """ + use Trinity.DataCase, async: false + + alias Trinity.LLM + alias Trinity.LLM.{Event, Request} + + @moduletag :live + @moduletag timeout: 180_000 + + @chat_models ["openrouter:ling", "nvidia:nemotron"] + + setup_all do + if System.get_env("TRINITY_LIVE") != "1", + do: raise("TRINITY_LIVE=1 is required for the live suite") + + original = Application.get_env(:trinity, :llm) + Application.put_env(:trinity, :llm, live_registry()) + on_exit(fn -> Application.put_env(:trinity, :llm, original) end) + :ok + end + + # The dev/prod registry (config/llm.exs), read now so the environment loaded from .env is + # what it sees. + defp live_registry, do: Config.Reader.read!("config/llm.exs")[:trinity][:llm] + + defp preflight(entry) do + ["openai", model] = + entry.model + |> String.split(":", parts: 2) + |> then(fn [p, m] -> [if(p == "openrouter", do: "openai", else: p), m] end) + + base = Map.get(entry, :base_url, "https://openrouter.ai/api/v1") + {:ok, key} = Trinity.Config.secret(entry.api_key_env) + %{status: 200, body: %{"data" => data}} = Req.get!(base <> "/models", auth: {:bearer, key}) + ids = Enum.map(data, & &1["id"]) + + assert model in ids, + "#{entry.id}: model #{model} is not on #{base}/models (#{length(ids)} listed); it may have reached end of life" + end + + for id <- @chat_models do + describe "#{id}" do + @tag model: id + test "preflight: the configured model is on the provider's list", %{model: id} do + {:ok, entry} = LLM.Registry.lookup(id) + preflight(entry) + end + + @tag model: id + test "a streamed completion yields text deltas, usage and done", %{model: id} do + request = + Request.new!(%{ + model: id, + messages: [%{role: "user", content: "Reply with exactly: ready"}], + params: %{max_tokens: 64} + }) + + {:ok, agent} = Agent.start_link(fn -> [] end) + assert {:ok, usage} = LLM.stream(request, [], fn e -> Agent.update(agent, &[e | &1]) end) + events = agent |> Agent.get(& &1) |> Enum.reverse() + assert Enum.all?(events, &Event.valid?/1) + + text = + events |> Enum.filter(&match?({:text_delta, _}, &1)) |> Enum.map_join("", &elem(&1, 1)) + + assert text =~ ~r/ready/i, "text was: #{inspect(text)}" + assert {:usage, _} = Enum.find(events, &match?({:usage, _}, &1)) + assert {:done, _} = List.last(events) + assert usage.input_tokens > 0 and usage.output_tokens > 0 + end + + @tag model: id + test "a tool call arrives as start, end, and a done of tool_calls", %{model: id} do + request = + Request.new!(%{ + model: id, + messages: [ + %{role: "user", content: "What is the weather in Paris? Use the get_weather tool."} + ], + tools: [ + %{ + name: "get_weather", + description: "Weather for a city", + parameters: %{ + "type" => "object", + "properties" => %{"city" => %{"type" => "string"}}, + "required" => ["city"] + } + } + ], + params: %{max_tokens: 512} + }) + + {:ok, agent} = Agent.start_link(fn -> [] end) + assert {:ok, _} = LLM.stream(request, [], fn e -> Agent.update(agent, &[e | &1]) end) + events = agent |> Agent.get(& &1) |> Enum.reverse() + + assert {:tool_call_start, call_id, "get_weather"} = + Enum.find(events, &match?({:tool_call_start, _, _}, &1)) + + assert {:tool_call_end, ^call_id, %{"city" => city}} = + Enum.find(events, &match?({:tool_call_end, _, _}, &1)) + + assert city =~ ~r/paris/i + assert {:done, :tool_calls} = List.last(events) + end + + @tag model: id + test "generate_object returns a map that validates against the schema (AC2)", %{model: id} do + schema = %{ + "type" => "object", + "properties" => %{ + "city" => %{"type" => "string"}, + "population_millions" => %{"type" => "number"} + }, + "required" => ["city", "population_millions"] + } + + request = + Request.new!(%{ + model: id, + messages: [ + %{role: "user", content: "Give the city of Paris and its population in millions."} + ], + params: %{max_tokens: 256} + }) + + assert {:ok, %{"city" => city, "population_millions" => pop}} = + LLM.generate_object(request, schema) + + assert city =~ ~r/paris/i and is_number(pop) + end + end + end + + test "embed returns vectors of the declared dimension against the NVIDIA embedding model (AC3)" do + {:ok, entry} = LLM.Registry.lookup("nvidia:embed") + preflight(entry) + + assert {:ok, [v1, v2]} = + LLM.embed(["the cat sat", "a cat was sitting"], model: "nvidia:embed") + + assert length(v1) == length(v2) and v1 != [] + assert Enum.all?(v1, &is_float/1) + IO.puts("\nlive embed: dimension #{length(v1)}") + end + + test "a live call writes one usage_events row with the tokens the provider reported" do + request = + Request.new!(%{ + model: "openrouter:ling", + messages: [%{role: "user", content: "Reply with exactly: ok"}], + params: %{max_tokens: 16} + }) + + assert {:ok, %{usage: usage}} = LLM.generate(request) + assert [row] = Repo.all(Trinity.LLM.Usage) + assert row.model_id == "openrouter:ling" and row.kind == "chat" + assert row.input_tokens == usage.input_tokens and row.input_tokens > 0 + assert row.cost_usd == 0.0 + + IO.puts( + "\nlive usage row: #{row.input_tokens} in, #{row.output_tokens} out, provider_cost #{inspect(row.provider_meta["provider_cost"])}" + ) + end +end diff --git a/test/trinity/llm/llm_test.exs b/test/trinity/llm/llm_test.exs new file mode 100644 index 0000000..8fb8739 --- /dev/null +++ b/test/trinity/llm/llm_test.exs @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLMTest do + @moduledoc "Slice 011 AC1, AC3, AC4, AC5, AC7 through the scripted fake and the Mox mock." + use Trinity.DataCase, async: false + import Mox + + alias Trinity.LLM + alias Trinity.LLM.{Error, Event, Providers.Fake, Request, Usage} + + setup :verify_on_exit! + + setup do + Process.delete({Fake, :script}) + Process.delete({Fake, :fail}) + Process.delete({Fake, :calls}) + :ok + end + + defp request(attrs \\ %{}) do + Request.new!(Map.merge(%{messages: [%{role: "user", content: "hi"}]}, attrs)) + end + + defp collect(request, opts \\ []) do + {:ok, agent} = Agent.start_link(fn -> [] end) + result = LLM.stream(request, opts, fn event -> Agent.update(agent, &[event | &1]) end) + events = agent |> Agent.get(& &1) |> Enum.reverse() + Agent.stop(agent) + {result, events} + end + + describe "stream/3 (AC1)" do + test "emits the full sequence: text deltas, tool call start, deltas, end, usage, done" do + {{:ok, usage}, events} = collect(request()) + + assert Enum.all?(events, &Event.valid?/1) + + assert [ + {:text_delta, "Hello, "}, + {:text_delta, "world."}, + {:tool_call_start, "call_1", "get_weather"}, + {:tool_call_delta, "call_1", _}, + {:tool_call_delta, "call_1", _}, + {:tool_call_end, "call_1", %{"city" => "Paris"}}, + {:usage, %{input_tokens: 10, output_tokens: 5}}, + {:done, :tool_calls} + ] = events + + assert usage == %{input_tokens: 10, output_tokens: 5} + end + + test "stream_to/3 delivers the same events as messages and then llm_done" do + {:ok, ref} = LLM.stream_to(request(), [], self()) + assert_receive {:llm_event, ^ref, {:text_delta, "Hello, "}}, 1_000 + assert_receive {:llm_event, ^ref, {:done, :tool_calls}}, 1_000 + assert_receive {:llm_done, ^ref, {:ok, %{input_tokens: 10}}}, 1_000 + end + end + + describe "embed/2 (AC3)" do + test "returns one vector per text of the declared dimension" do + {:ok, [{:embed_dim, dim}]} = + then(LLM.capabilities("fake:embed"), fn {:ok, caps} -> + {:ok, Enum.filter(caps, &match?({:embed_dim, _}, &1))} + end) + + assert {:ok, vectors} = LLM.embed(["a", "b", "c"], model: "fake:embed") + assert length(vectors) == 3 + assert Enum.all?(vectors, &(length(&1) == dim)) + end + end + + describe "retry (AC4)" do + test "a transient error is retried and then succeeds" do + Fake.fail(2, Error.from_status(429, :rate_limited)) + assert {:ok, %{text: "Hello, world."}} = LLM.generate(request()) + assert Fake.calls() == 3 + end + + test "attempts exhausted returns the error, named as exhausted" do + Fake.fail(10, Error.from_status(503, :down)) + + assert {:error, %Error{transient?: true, reason: {:exhausted, 3, :down}}} = + LLM.generate(request()) + + assert Fake.calls() == 3 + end + + test "a permanent error returns at once with one attempt" do + Fake.fail(10, Error.from_status(401, :bad_key)) + assert {:error, %Error{transient?: false, status: 401}} = LLM.generate(request()) + assert Fake.calls() == 1 + end + + test "no usage row is written for a failed call" do + Fake.fail(10, Error.from_status(401, :bad_key)) + {:error, _} = LLM.generate(request()) + assert Repo.aggregate(Usage, :count) == 0 + end + end + + describe "usage_events (AC5)" do + test "one row per completed call, cost from the registry price" do + {{:ok, _}, _} = collect(request()) + assert [row] = Repo.all(Usage) + assert row.model_id == "fake:chat" + assert row.provider == "fake" + assert row.kind == "chat" + assert {row.input_tokens, row.output_tokens} == {10, 5} + # 10 input tokens at $1.00 per million plus 5 output tokens at $2.00 per million. + assert row.cost_usd == 0.00002 + assert row.provider_meta == %{"provider_cost" => nil} + end + + test "the session id is recorded when given" do + session = Trinity.Factory.session!() + {{:ok, _}, _} = collect(request(), session_id: session.id) + assert [%{session_id: sid}] = Repo.all(Usage) + assert sid == session.id + end + + test "embed and object calls record their own kinds" do + {:ok, _} = LLM.embed(["x"], model: "fake:embed") + + {:ok, _} = + LLM.generate_object(request(), %{"properties" => %{"n" => %{"type" => "integer"}}}) + + assert Repo.all(Usage) |> Enum.map(& &1.kind) |> Enum.sort() == ["embed", "object"] + end + end + + describe "default_model (AC7)" do + test "switching default_model in config changes the provider used, with no code change" do + {{:ok, _}, events} = collect(request()) + assert {:text_delta, "Hello, "} in events + + original = Application.get_env(:trinity, :llm) + on_exit(fn -> Application.put_env(:trinity, :llm, original) end) + Application.put_env(:trinity, :llm, Keyword.put(original, :default_model, "mock:chat")) + + expect(Trinity.LLM.ProviderMock, :stream, fn _request, opts, emit -> + assert opts[:model] == "chat" + emit.({:text_delta, "from the mock"}) + emit.({:done, :stop}) + {:ok, %{input_tokens: 1, output_tokens: 1}} + end) + + {{:ok, _}, events} = collect(request()) + assert events == [{:text_delta, "from the mock"}, {:done, :stop}] + assert LLM.default_model() == "mock:chat" + end + + test "an unknown model id is refused by name" do + assert {:error, {:unknown_model, "nope:x"}} = LLM.generate(request(%{model: "nope:x"})) + end + end + + describe "generate_object/3" do + test "returns a map shaped by the schema (fake)" do + schema = %{ + "type" => "object", + "properties" => %{"age" => %{"type" => "integer"}, "name" => %{"type" => "string"}} + } + + assert {:ok, %{"age" => 42, "name" => "fake"}} = LLM.generate_object(request(), schema) + end + end +end diff --git a/test/trinity/llm/mapping_test.exs b/test/trinity/llm/mapping_test.exs new file mode 100644 index 0000000..7b892c3 --- /dev/null +++ b/test/trinity/llm/mapping_test.exs @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.Providers.ReqLLM.MappingTest do + @moduledoc """ + Slice 011: the req_llm adapter's pure half over recorded chunk sequences. The chunk shapes + are the ones req_llm 1.24.0's default decoder builds (`lib/req_llm/provider/defaults.ex`, + `decode_openai_tool_call_delta/1`), reproduced here with the library's own constructors so a + change in the library's shape is a red here before it is a surprise in production. + """ + use ExUnit.Case, async: true + + alias ReqLLM.StreamChunk, as: C + alias Trinity.LLM.{Error, Event} + alias Trinity.LLM.Providers.ReqLLM.Mapping + + defp run(chunks) do + {:ok, agent} = Agent.start_link(fn -> [] end) + state = Mapping.reduce(chunks, fn e -> Agent.update(agent, &[e | &1]) end) + events = agent |> Agent.get(& &1) |> Enum.reverse() + assert Enum.all?(events, &Event.valid?/1), inspect(events) + {events, state} + end + + test "text chunks become text deltas; thinking and empty content are dropped" do + {events, _} = run([C.text("Hel"), C.thinking("hmm"), C.text(""), C.text("lo")]) + assert events == [{:text_delta, "Hel"}, {:text_delta, "lo"}] + end + + test "a streamed tool call: open with fragments to follow, two fragments, closed by the finish reason" do + chunks = [ + C.tool_call("get_weather", %{}, %{id: "call_9", index: 0, expects_arg_fragments: true}), + C.meta(%{tool_call_args: %{index: 0, fragment: ~s({"city":)}}), + C.meta(%{tool_call_args: %{index: 0, fragment: ~s("Paris"})}}), + C.meta(%{finish_reason: "tool_calls"}) + ] + + {events, state} = run(chunks) + + assert events == [ + {:tool_call_start, "call_9", "get_weather"}, + {:tool_call_delta, "call_9", ~s({"city":)}, + {:tool_call_delta, "call_9", ~s("Paris"})}, + {:tool_call_end, "call_9", %{"city" => "Paris"}} + ] + + assert state.finish == :tool_calls + end + + test "a complete tool call in one chunk opens and closes at once" do + {events, _} = run([C.tool_call("get_weather", %{"city" => "Paris"}, %{id: "c1", index: 0})]) + + assert events == [ + {:tool_call_start, "c1", "get_weather"}, + {:tool_call_end, "c1", %{"city" => "Paris"}} + ] + end + + test "two interleaved calls are assembled by index and closed in order" do + chunks = [ + C.tool_call("a", %{}, %{id: "ca", index: 0, expects_arg_fragments: true}), + C.tool_call("b", %{}, %{id: "cb", index: 1, expects_arg_fragments: true}), + C.meta(%{tool_call_args: %{index: 1, fragment: ~s({"y":2})}}), + C.meta(%{tool_call_args: %{index: 0, fragment: ~s({"x":1})}}) + ] + + {events, _} = run(chunks) + + assert Enum.filter(events, &match?({:tool_call_end, _, _}, &1)) == [ + {:tool_call_end, "ca", %{"x" => 1}}, + {:tool_call_end, "cb", %{"y" => 2}} + ] + end + + test "a call without an id gets a stable synthetic one; unparseable fragments close as no arguments" do + chunks = [ + C.tool_call("t", %{}, %{index: 0, expects_arg_fragments: true}), + C.meta(%{tool_call_args: %{index: 0, fragment: "{not json"}}) + ] + + {events, _} = run(chunks) + + assert [ + {:tool_call_start, "call_1", "t"}, + {:tool_call_delta, "call_1", _}, + {:tool_call_end, "call_1", %{}} + ] = events + end + + test "a fragment for an unknown index is ignored, and a finish of stop is recorded" do + {events, state} = + run([ + C.meta(%{tool_call_args: %{index: 7, fragment: "x"}}), + C.meta(%{finish_reason: "stop"}) + ]) + + assert events == [] + assert state.finish == :stop + end + + test "finish reasons: the closed vocabulary, and :other for anything else, never a new atom" do + assert Mapping.finish("stop") == :stop + assert Mapping.finish("length") == :length + assert Mapping.finish("tool_calls") == :tool_calls + assert Mapping.finish("content_filter") == :content_filter + assert Mapping.finish(:max_output_tokens) == :max_output_tokens + + assert Mapping.finish("some_new_provider_reason_#{System.unique_integer([:positive])}") == + :other + + assert Mapping.finish(nil) == :stop + end + + test "usage is normalised to Trinity's keys with the provider's cost kept aside" do + assert Mapping.normalise_usage(%{input_tokens: 3, output_tokens: nil, total_cost: 0.5}) == + %{ + input_tokens: 3, + output_tokens: 0, + cached_tokens: 0, + reasoning_tokens: 0, + provider_cost: 0.5 + } + + assert Mapping.normalise_usage(nil) == %{} + end + + test "tool calls from a complete response take both shapes req_llm returns" do + assert %{id: "1", name: "t", args: %{"a" => 1}} = + Mapping.tool_call(%{id: "1", name: "t", arguments: %{"a" => 1}}) + + assert %{id: "2", name: "u", args: %{}} = + Mapping.tool_call(%{id: "2", function: %{name: "u", arguments: nil}}) + end + + describe "classify/1" do + test "an API error with a status classifies by status" do + assert %Error{transient?: true, status: 429} = + Mapping.classify(%ReqLLM.Error.API.Request{reason: "rl", status: 429}) + + assert %Error{transient?: false, status: 401} = + Mapping.classify(%ReqLLM.Error.API.Request{reason: "key", status: 401}) + end + + test "a stream error wrapping a 429 with no status of its own is transient (the live-suite defect)" do + inner = %ReqLLM.Error.API.Request{ + reason: "Provider returned error", + status: 429, + retryable: true + } + + outer = %ReqLLM.Error.API.Stream{reason: "Stream failed", cause: inner} + assert %Error{transient?: true, status: 429, reason: ^outer} = Mapping.classify(outer) + end + + test "a stream error whose cause is a timeout is transient" do + assert %Error{transient?: true} = + Mapping.classify(%ReqLLM.Error.API.Stream{ + reason: "Stream failed: :timeout", + cause: :timeout + }) + end + + test "transport errors are transient; an invalid parameter is permanent; a bare term is permanent" do + assert %Error{transient?: true} = + Mapping.classify(%Req.TransportError{reason: :econnrefused}) + + assert %Error{transient?: false} = + Mapping.classify(%ReqLLM.Error.Invalid.Parameter{parameter: "x"}) + + assert %Error{transient?: false, reason: :whatever} = Mapping.classify(:whatever) + assert {:error, %Error{}} = Mapping.wrap({:error, :x}) + assert {:ok, 1} = Mapping.wrap({:ok, 1}) + end + end +end diff --git a/test/trinity/llm/units_test.exs b/test/trinity/llm/units_test.exs new file mode 100644 index 0000000..2e386a7 --- /dev/null +++ b/test/trinity/llm/units_test.exs @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +defmodule Trinity.LLM.UnitsTest do + @moduledoc "Slice 011: Request, Event, Registry, Retry, Config.secret and the error classes." + use ExUnit.Case, async: true + + alias Trinity.Config + alias Trinity.LLM.{Error, Event, Registry, Request, Retry} + + describe "Request.new/1" do + test "accepts the four roles and a tool with a name and parameters" do + assert {:ok, %Request{}} = + Request.new(%{ + messages: [ + %{role: "system", content: "s"}, + %{role: "user", content: "u"}, + %{ + role: "assistant", + content: "", + tool_calls: [%{id: "1", name: "t", args: %{}}] + }, + %{role: "tool", content: "r", tool_call_id: "1"} + ], + tools: [%{name: "t", description: "d", parameters: %{"type" => "object"}}] + }) + end + + test "refuses an unknown role and a tool without parameters, by name" do + assert {:error, {:invalid_request, {:message, %{role: "oracle"}}}} = + Request.new(%{messages: [%{role: "oracle", content: "x"}]}) + + assert {:error, {:invalid_request, {:tool, %{name: "t"}}}} = + Request.new(%{tools: [%{name: "t"}]}) + end + end + + describe "Event.valid?/1" do + test "the seven shapes and nothing else" do + for e <- [ + {:text_delta, "x"}, + {:tool_call_start, "1", "t"}, + {:tool_call_delta, "1", "{"}, + {:tool_call_end, "1", %{}}, + {:usage, %{}}, + {:done, :stop}, + {:error, :any} + ], + do: assert(Event.valid?(e), inspect(e)) + + refute Event.valid?({:text_delta, 1}) + refute Event.valid?({:chunk, "x"}) + refute Event.valid?({:done, "stop"}) + end + end + + describe "Registry" do + test "resolves the default and refuses an unknown id" do + assert {:ok, %{id: "fake:chat", provider: :fake}} = Registry.lookup(nil) + + assert {:ok, Trinity.LLM.Providers.Fake} = + Registry.lookup("fake:chat") + |> then(fn {:ok, e} -> Registry.provider_module(e) end) + + assert {:error, {:unknown_model, "x:y"}} = Registry.lookup("x:y") + assert {:error, {:unknown_provider, :nope}} = Registry.provider_module(%{provider: :nope}) + end + end + + describe "Retry.run/2" do + test "backs off exponentially and stops at the attempt count" do + {:ok, sleeps} = Agent.start_link(fn -> [] end) + sleep = fn ms -> Agent.update(sleeps, &[ms | &1]) end + err = {:error, Error.transient(:t)} + + assert {:error, %Error{reason: {:exhausted, 4, :t}}} = + Retry.run(fn -> err end, attempts: 4, base_ms: 10, sleep: sleep) + + assert Enum.reverse(Agent.get(sleeps, & &1)) == [10, 20, 40] + end + end + + describe "Error" do + test "statuses classify: 408, 425, 429 and 5xx transient; other 4xx permanent" do + for s <- [408, 425, 429, 500, 502, 503], + do: assert(Error.from_status(s, :x).transient?, "#{s}") + + for s <- [400, 401, 403, 404, 422], do: refute(Error.from_status(s, :x).transient?, "#{s}") + assert Exception.message(Error.from_status(429, :rl)) =~ "transient LLM error (HTTP 429)" + end + end + + describe "Config.secret/1" do + test "a missing or empty variable is a named error, never nil" do + System.delete_env("TRINITY_TEST_SECRET") + + assert {:error, {:missing_secret, "TRINITY_TEST_SECRET"}} = + Config.secret("TRINITY_TEST_SECRET") + + System.put_env("TRINITY_TEST_SECRET", "") + assert {:error, {:missing_secret, _}} = Config.secret("TRINITY_TEST_SECRET") + System.put_env("TRINITY_TEST_SECRET", "v") + assert {:ok, "v"} = Config.secret("TRINITY_TEST_SECRET") + System.delete_env("TRINITY_TEST_SECRET") + end + end +end