From f4159ed130be0ee097cc519ff5971dd7e0565c3b Mon Sep 17 00:00:00 2001
From: JNK234 <jwalapuramnarasimha@gmail.com>
Date: Tue, 8 Sep 2026 19:55:47 -0400
Subject: [PATCH] feat: record token usage, cost and latency per agent and per
 run

Every provider reports token usage and the extension discarded it. This
reads it into a Usage record on ChatResponse, normalised across providers
after the OpenTelemetry GenAI conventions: input counts every prompt token
including cached ones, output counts every generated token including
reasoning, and provider-specific extras (reasoning, cache read/write,
OpenRouter's reported cost) are kept where reported. The HTTP layer stamps
wall-clock latency on each call, retries and throttle waits included.

Two reporters expose it in the same [key value] shape as structured output:
llm:usage for the calling agent and llm:usage-total for the run. Both reset
on clear-all. A reply the extension then rejects still counts, since the
provider billed it; a call with no reply records nothing.

Tests: UsageParsingSpec covers each provider's field mapping and the
latency stamp; eight tests.txt cases cover accumulation, per-agent
isolation, async, every primitive, rejected replies and failed calls via a
__TEST_USAGE marker in the test provider. T9 in the live harness verifies
against Groq: 34/34 pass.

Closes #49
---
 demos/e2e-tests/e2e-tests.nlogox              |  28 +++
 docs/API-REFERENCE.md                         |  71 +++++++
 src/main/LLMExtension.scala                   | 113 ++++++++++-
 src/main/models/ChatResponse.scala            |  77 +++++++-
 src/main/providers/BaseHttpProvider.scala     |  13 +-
 src/main/providers/ClaudeProvider.scala       |  30 ++-
 src/main/providers/GeminiProvider.scala       |  27 ++-
 src/main/providers/OllamaProvider.scala       |  11 +-
 .../providers/OpenAICompatibleProvider.scala  |  26 ++-
 src/test/DeterministicTestProvider.scala      |  37 +++-
 src/test/UsageParsingSpec.scala               | 186 ++++++++++++++++++
 tests.txt                                     | 102 ++++++++++
 12 files changed, 699 insertions(+), 22 deletions(-)
 create mode 100644 src/test/UsageParsingSpec.scala

diff --git a/demos/e2e-tests/e2e-tests.nlogox b/demos/e2e-tests/e2e-tests.nlogox
index 5a0b238..2473c7c 100644
--- a/demos/e2e-tests/e2e-tests.nlogox
+++ b/demos/e2e-tests/e2e-tests.nlogox
@@ -276,6 +276,33 @@ to write-throttled-config [src dst cap interval-ms]
   file-close
 end
 
+;; ---------------------------------------------------------------------------
+;; T9 — token usage (#49)
+;;
+;; Every provider reports how many tokens a call consumed; the extension used
+;; to discard it. One real call must show up in the calling agent's usage
+;; and in the run total, with non-zero counts and a measured latency.
+;; ---------------------------------------------------------------------------
+
+to test-usage
+  log-line "T9 token usage"
+  let calls-before llm:get llm:usage "calls"
+  let run-before llm:get llm:usage-total "total-tokens"
+  carefully
+    [ let ignored llm:chat "Reply with exactly the word: OK" ]
+    [ log-line (word "  chat failed: " error-message) ]
+  let u llm:usage
+  log-line (word "  -> " u)
+  assert "one more call was recorded" (llm:get u "calls" = calls-before + 1)
+  assert "input tokens were reported" (llm:get u "input-tokens" > 0)
+  assert "output tokens were reported" (llm:get u "output-tokens" > 0)
+  assert "total equals input plus output"
+         (llm:get u "total-tokens" = (llm:get u "input-tokens") + (llm:get u "output-tokens"))
+  assert "latency was measured" (llm:get u "latency-ms" > 0)
+  assert "run total grew by this call" (llm:get llm:usage-total "total-tokens" > run-before)
+  report-totals
+end
+
 ;; Run everything in sequence.
 to run-headless
   carefully [ file-delete "e2e-results.txt" ] [ ]
@@ -298,6 +325,7 @@ to test-all
   test-structured-output
   test-choose
   test-throttling
+  test-usage
   output-print ""
   log-line (word "TOTAL  passed " pass-count "   failed " fail-count)
 end
diff --git a/docs/API-REFERENCE.md b/docs/API-REFERENCE.md
index 64b1399..c8c3b6f 100644
--- a/docs/API-REFERENCE.md
+++ b/docs/API-REFERENCE.md
@@ -20,6 +20,8 @@ The NetLogo Multi-LLM Extension provides a unified interface for multiple Large
 | `llm:set-thinking bool`            | Reasoning     | Enable/disable reasoning mode for current provider         |
 | `llm:set-reasoning-effort level`   | Reasoning     | Set effort: `"low"`, `"medium"`, `"high"`                  |
 | `llm:set-thinking-budget n`        | Reasoning     | Token budget for thinking (min 1024; Anthropic + Gemini)   |
+| `llm:usage`                        | Usage         | Token counts, cost, latency and call count for this agent  |
+| `llm:usage-total`                  | Usage         | The same totals across every agent in the run              |
 | `llm:history`                      | History       | Get current agent's conversation history                   |
 | `llm:set-history list`             | History       | Set conversation history for current agent                 |
 | `llm:clear-history`                | History       | Clear conversation history for current agent               |
@@ -858,6 +860,75 @@ print llm:list-models  ; Shows all providers, with Anthropic marked as ACTIVE
 - Custom models added via `models-override.yaml` are marked with `[custom]`
 - The currently active provider and model are marked with `[ACTIVE]`
 
+## Token Usage
+
+Every provider reports how many tokens a call consumed. The extension records
+that for each call and sums it per agent and for the whole run. Nothing is
+estimated: counts come from the provider's reply, cost only appears when a
+provider reports one, and latency is measured by the extension.
+
+### llm:usage
+
+**Syntax**: `llm:usage`
+
+**Description**: Token accounting for the calling agent since the last `clear-all`
+
+**Returns**: List of `[key value]` pairs, readable with `llm:get`
+
+| Key                  | Meaning                                                                 |
+| -------------------- | ----------------------------------------------------------------------- |
+| `input-tokens`       | Every prompt token the provider processed, cached tokens included       |
+| `output-tokens`      | Every generated token, reasoning tokens included                        |
+| `total-tokens`       | The provider's total, or input plus output when it reports none         |
+| `reasoning-tokens`   | Tokens spent on reasoning where reported separately, else 0             |
+| `cache-read-tokens`  | Prompt tokens served from a provider cache where reported, else 0       |
+| `cache-write-tokens` | Prompt tokens written to a provider cache where reported, else 0        |
+| `cost`               | Dollars, only when the provider reports it (OpenRouter); otherwise `""` |
+| `latency-ms`         | Summed wall time of the calls, including throttle waits and retries     |
+| `calls`              | How many provider replies contributed to these totals                   |
+
+**Example**:
+
+```netlogo
+ask turtles [
+  let reply llm:chat "Should I forage or rest?"
+  if llm:get llm:usage "total-tokens" > 5000 [
+    set color red   ;; this agent is expensive
+  ]
+]
+print (word "Run so far: " llm:get llm:usage-total "total-tokens" " tokens over "
+            llm:get llm:usage-total "calls" " calls")
+```
+
+**Notes**:
+
+- A reply the extension then rejects, such as a schema reply that does not
+  parse or an `llm:choose` answer that matches no option, still counts. The
+  provider billed those tokens. A call that never produced a reply records
+  nothing.
+- `cost` is `""` rather than `0` when unknown so an unknown cost is never
+  mistaken for a free call. Guard with `is-number?` before arithmetic.
+- Counters are cumulative. To measure one call, read `llm:usage` before and
+  after and subtract.
+- Providers define their fields differently; the mapping is normalised so the
+  headline numbers compare across providers:
+
+| Provider          | input-tokens                                             | output-tokens                          |
+| ----------------- | -------------------------------------------------------- | -------------------------------------- |
+| OpenAI-compatible | `prompt_tokens` (cached tokens are a subset)             | `completion_tokens` (reasoning subset) |
+| Anthropic         | `input_tokens` + cache creation + cache read             | `output_tokens` (thinking included)    |
+| Gemini            | `promptTokenCount` (cached content included)             | `candidatesTokenCount` + `thoughtsTokenCount` |
+| Ollama            | `prompt_eval_count`                                      | `eval_count`                           |
+
+### llm:usage-total
+
+**Syntax**: `llm:usage-total`
+
+**Description**: The same accounting summed over every agent in the run since
+the last `clear-all`, including agents that have since died
+
+**Returns**: List of `[key value]` pairs with the keys listed under `llm:usage`
+
 ## Structured Output Details
 
 ### Why JSON becomes nested lists
diff --git a/src/main/LLMExtension.scala b/src/main/LLMExtension.scala
index 149d4dd..bac7de5 100644
--- a/src/main/LLMExtension.scala
+++ b/src/main/LLMExtension.scala
@@ -4,7 +4,7 @@ import org.nlogo.api._
 import org.nlogo.core.{LogoList, Syntax}
 import org.nlogo.extensions.llm.config.{ConfigLoader, ConfigStore}
 import org.nlogo.extensions.llm.providers.{LLMProvider, ProviderDescriptor, ProviderFactory, ProviderRegistry, ProviderRegistrations, ModelRegistry, OllamaProvider, ReadinessCheck, RetryPolicy}
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse, EnumFormat, JsonObjectFormat, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatResponse, EnumFormat, JsonObjectFormat, ResponseFormat, Usage}
 import org.nlogo.extensions.llm.utils.JsonToNetLogo
 import scala.collection.mutable.{ArrayBuffer, WeakHashMap}
 import scala.concurrent.{Await, ExecutionContext, Future}
@@ -61,6 +61,13 @@ class LLMExtension extends DefaultClassManager {
   // or provider calls; critical sections are small snapshots/appends only.
   private val historyLock = new Object
 
+  // Token accounting. Per-agent totals live in a WeakHashMap like history so a
+  // dead turtle's counters are collected with it; the run total is kept
+  // separately so it survives agent death. Both reset on clear-all.
+  private val agentUsage: WeakHashMap[Agent, UsageTotals] = WeakHashMap()
+  private var runUsage: UsageTotals = UsageTotals.empty
+  private val usageLock = new Object
+
   // Execution context for async operations
   implicit private val ec: ExecutionContext = ExecutionContext.global
   
@@ -129,6 +136,10 @@ class LLMExtension extends DefaultClassManager {
     manager.addPrimitive("list-models", ListModelsReporter)
     manager.addPrimitive("active", ActiveReporter)
     manager.addPrimitive("config", ConfigReporter)
+
+    // Token accounting primitives
+    manager.addPrimitive("usage", UsageReporter)
+    manager.addPrimitive("usage-total", UsageTotalReporter)
   }
   
   /**
@@ -136,6 +147,10 @@ class LLMExtension extends DefaultClassManager {
    */
   override def clearAll(): Unit = {
     historyLock.synchronized { messageHistory.clear() }
+    usageLock.synchronized {
+      agentUsage.clear()
+      runUsage = UsageTotals.empty
+    }
   }
   
   /**
@@ -182,6 +197,34 @@ class LLMExtension extends DefaultClassManager {
       h += assistant
     }
 
+  /**
+   * Credit a provider reply to the calling agent and to the run.
+   *
+   * Called as soon as a response arrives, before the caller decides whether
+   * the reply is acceptable: tokens were billed either way, so a reply the
+   * extension then rejects (schema parse failure, unmatched choice) still
+   * counts. A call that never produced a response records nothing.
+   */
+  private def recordUsage(agent: Agent, response: ChatResponse): Unit = {
+    val delta = UsageTotals(response.usage.getOrElse(Usage.empty), 1L)
+    usageLock.synchronized {
+      agentUsage.update(agent, agentUsage.getOrElse(agent, UsageTotals.empty).plus(delta))
+      runUsage = runUsage.plus(delta)
+    }
+  }
+
+  private def usageFor(agent: Agent): UsageTotals =
+    usageLock.synchronized { agentUsage.getOrElse(agent, UsageTotals.empty) }
+
+  private def usageForRun: UsageTotals =
+    usageLock.synchronized { runUsage }
+
+  /** The assistant message to store in history, mirroring what chat(messages) returned before. */
+  private def replyMessage(response: ChatResponse): ChatMessage =
+    response.firstMessage.getOrElse(
+      throw new RuntimeException("No response message received from provider")
+    )
+
   /**
    * Get timeout from config, falling back to 30 seconds
    */
@@ -501,8 +544,10 @@ class LLMExtension extends DefaultClassManager {
         val userMessage = ChatMessage.user(inputText)
 
         // Send chat request with user message included, but don't mutate history yet
-        val responseFuture = provider.chat(snapshotHistory(agent) :+ userMessage)
-        val responseMessage = Await.result(responseFuture, getAwaitTimeout)
+        val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage)
+        val response = Await.result(responseFuture, getAwaitTimeout)
+        recordUsage(agent, response)
+        val responseMessage = replyMessage(response)
 
         // Only commit both messages after success
         commitExchange(agent, userMessage, responseMessage)
@@ -533,7 +578,9 @@ class LLMExtension extends DefaultClassManager {
 
         // Snapshot on the NetLogo thread; commit the pair atomically on success
         // from the completion thread so overlapping async calls can't interleave.
-        val responseFuture = provider.chat(snapshotHistory(agent) :+ userMessage).map { responseMessage =>
+        val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage).map { response =>
+          recordUsage(agent, response)
+          val responseMessage = replyMessage(response)
           commitExchange(agent, userMessage, responseMessage)
           responseMessage.content
         }
@@ -590,8 +637,10 @@ class LLMExtension extends DefaultClassManager {
         tempHistory += userMessage
 
         // Send chat request
-        val responseFuture = provider.chat(tempHistory.toSeq)
-        val responseMessage = Await.result(responseFuture, getAwaitTimeout)
+        val responseFuture = provider.chatWithFullResponse(tempHistory.toSeq)
+        val response = Await.result(responseFuture, getAwaitTimeout)
+        recordUsage(agent, response)
+        val responseMessage = replyMessage(response)
 
         // Commit both template message and response to permanent history on success
         commitExchange(agent, userMessage, responseMessage)
@@ -652,6 +701,7 @@ class LLMExtension extends DefaultClassManager {
         // the constraint behaves exactly as it did before.
         val responseFuture = provider.chatWithFormat(tempHistory.toSeq, EnumFormat(choices))
         val response = Await.result(responseFuture, getAwaitTimeout)
+        recordUsage(agent, response)
 
         // Extract text: prefer content, fall back to thinking field
         val text = response.firstContent.filter(_.nonEmpty)
@@ -751,6 +801,7 @@ class LLMExtension extends DefaultClassManager {
 
         val responseFuture = provider.chatWithFormat(snapshotHistory(agent) :+ userMessage, format)
         val response = Await.result(responseFuture, getAwaitTimeout)
+        recordUsage(agent, response)
 
         val content = response.firstContent.getOrElse("")
 
@@ -811,6 +862,7 @@ class LLMExtension extends DefaultClassManager {
 
         val responseFuture = provider.chatWithFormat(tempHistory.toSeq, JsonObjectFormat)
         val response = Await.result(responseFuture, getAwaitTimeout)
+        recordUsage(agent, response)
 
         val content = response.firstContent.getOrElse("")
 
@@ -893,6 +945,7 @@ class LLMExtension extends DefaultClassManager {
         // Send with user message included, but don't mutate history yet
         val responseFuture = provider.chatWithFullResponse(snapshotHistory(agent) :+ userMessage)
         val response = Await.result(responseFuture, getAwaitTimeout)
+        recordUsage(agent, response)
 
         val answerText = response.firstContent.getOrElse("")
         val thinkingText = response.thinking.getOrElse("")
@@ -1198,6 +1251,44 @@ class LLMExtension extends DefaultClassManager {
     }
   }
   
+  /**
+   * Token accounting as a `[[key value] ...]` list, the same shape structured
+   * output uses so `llm:get` reads it. Counters are numbers; `cost` is a
+   * number only when a provider reported one and `""` otherwise, matching how
+   * JSON null is reported, so an unknown cost is never mistaken for free.
+   */
+  private def usageToLogo(totals: UsageTotals): LogoList = {
+    val u = totals.usage
+    def num(n: Long): AnyRef = Double.box(n.toDouble)
+    LogoList(
+      LogoList("input-tokens", num(u.inputTokens)),
+      LogoList("output-tokens", num(u.outputTokens)),
+      LogoList("total-tokens", num(u.totalTokens)),
+      LogoList("reasoning-tokens", num(u.reasoningTokens.getOrElse(0L))),
+      LogoList("cache-read-tokens", num(u.cacheReadTokens.getOrElse(0L))),
+      LogoList("cache-write-tokens", num(u.cacheWriteTokens.getOrElse(0L))),
+      LogoList("cost", u.cost.map(c => Double.box(c): AnyRef).getOrElse("")),
+      LogoList("latency-ms", num(u.latencyMs.getOrElse(0L))),
+      LogoList("calls", num(totals.calls))
+    )
+  }
+
+  /** llm:usage — token accounting for the calling agent since clear-all. */
+  object UsageReporter extends Reporter {
+    override def getSyntax: Syntax = Syntax.reporterSyntax(ret = Syntax.ListType)
+
+    override def report(args: Array[Argument], context: Context): AnyRef =
+      usageToLogo(usageFor(context.getAgent))
+  }
+
+  /** llm:usage-total — token accounting for every agent in the run since clear-all. */
+  object UsageTotalReporter extends Reporter {
+    override def getSyntax: Syntax = Syntax.reporterSyntax(ret = Syntax.ListType)
+
+    override def report(args: Array[Argument], context: Context): AnyRef =
+      usageToLogo(usageForRun)
+  }
+
   object ActiveReporter extends Reporter {
     override def getSyntax: Syntax = Syntax.reporterSyntax(ret = Syntax.ListType)
     
@@ -1249,3 +1340,13 @@ class LLMExtension extends DefaultClassManager {
     }
   }
 }
+
+/** Running totals: the summed Usage plus how many calls contributed to it. */
+private[llm] case class UsageTotals(usage: Usage, calls: Long) {
+  def plus(other: UsageTotals): UsageTotals =
+    UsageTotals(usage.plus(other.usage), calls + other.calls)
+}
+
+private[llm] object UsageTotals {
+  val empty: UsageTotals = UsageTotals(Usage.empty, 0L)
+}
diff --git a/src/main/models/ChatResponse.scala b/src/main/models/ChatResponse.scala
index 1994cab..63c0bb8 100644
--- a/src/main/models/ChatResponse.scala
+++ b/src/main/models/ChatResponse.scala
@@ -19,6 +19,78 @@ object Choice {
   implicit val rw: RW[Choice] = macroRW
 }
 
+/**
+ * Token accounting for one provider call, normalised across providers.
+ *
+ * Field names follow the OpenTelemetry GenAI conventions (gen_ai.usage.*):
+ *  - inputTokens counts every prompt token the provider processed, including
+ *    cached ones, so the number is comparable across providers that report
+ *    cached tokens as a subset (OpenAI, Gemini) and one that reports them
+ *    separately (Anthropic).
+ *  - outputTokens counts every generated token including reasoning, for the
+ *    same reason: OpenAI folds reasoning into completion tokens, Gemini does
+ *    not.
+ *  - totalTokens is taken from the provider when reported, else computed.
+ *  - reasoning / cache counts are absent when the provider does not report them.
+ *  - cost is only present when a provider reports it directly (OpenRouter);
+ *    nothing is ever estimated.
+ *  - latencyMs is the wall time of the whole call as the modeler experiences
+ *    it, including throttle queueing and rate-limit retries.
+ */
+case class Usage(
+  inputTokens: Long,
+  outputTokens: Long,
+  totalTokens: Long,
+  reasoningTokens: Option[Long] = None,
+  cacheReadTokens: Option[Long] = None,
+  cacheWriteTokens: Option[Long] = None,
+  cost: Option[Double] = None,
+  latencyMs: Option[Long] = None
+) {
+  /** Field-wise sum. An optional field stays absent only when neither side has it. */
+  def plus(other: Usage): Usage = {
+    def sumOpt[A](a: Option[A], b: Option[A])(add: (A, A) => A): Option[A] =
+      (a, b) match {
+        case (Some(x), Some(y)) => Some(add(x, y))
+        case (Some(x), None)    => Some(x)
+        case (None, Some(y))    => Some(y)
+        case (None, None)       => None
+      }
+    Usage(
+      inputTokens = inputTokens + other.inputTokens,
+      outputTokens = outputTokens + other.outputTokens,
+      totalTokens = totalTokens + other.totalTokens,
+      reasoningTokens = sumOpt(reasoningTokens, other.reasoningTokens)(_ + _),
+      cacheReadTokens = sumOpt(cacheReadTokens, other.cacheReadTokens)(_ + _),
+      cacheWriteTokens = sumOpt(cacheWriteTokens, other.cacheWriteTokens)(_ + _),
+      cost = sumOpt(cost, other.cost)(_ + _),
+      latencyMs = sumOpt(latencyMs, other.latencyMs)(_ + _)
+    )
+  }
+}
+
+object Usage {
+  implicit val rw: RW[Usage] = macroRW
+
+  val empty: Usage = Usage(0L, 0L, 0L)
+
+  /** Read an optional integer field from a JSON object; absence and null read alike. */
+  def longField(obj: ujson.Value, key: String): Option[Long] =
+    obj match {
+      case o: ujson.Obj => o.value.get(key).collect { case ujson.Num(n) => n.toLong }
+      case _            => None
+    }
+
+  /** Read a nested optional integer, e.g. usage.prompt_tokens_details.cached_tokens. */
+  def nestedLongField(obj: ujson.Value, path: String*): Option[Long] =
+    path.dropRight(1).foldLeft(Option(obj)) { (cur, key) =>
+      cur.flatMap {
+        case o: ujson.Obj => o.value.get(key)
+        case _            => None
+      }
+    }.flatMap(longField(_, path.last))
+}
+
 /**
  * Represents a complete chat response from an LLM provider
  *
@@ -26,13 +98,16 @@ object Choice {
  * @param created Timestamp when the response was created
  * @param model The model that generated the response
  * @param choices Array of response choices (usually contains one choice)
+ * @param thinking Reasoning text returned separately from the answer, if any
+ * @param usage Token accounting for this call, when the provider reported it
  */
 case class ChatResponse(
   id: String,
   created: Long,
   model: String,
   choices: Array[Choice],
-  thinking: Option[String] = None
+  thinking: Option[String] = None,
+  usage: Option[Usage] = None
 ) {
   /**
    * Get the first (and usually only) response message
diff --git a/src/main/providers/BaseHttpProvider.scala b/src/main/providers/BaseHttpProvider.scala
index f4b0829..10b2714 100644
--- a/src/main/providers/BaseHttpProvider.scala
+++ b/src/main/providers/BaseHttpProvider.scala
@@ -2,7 +2,7 @@
 // ABOUTME: Reduces boilerplate by providing shared implementation of config, validation, and HTTP request handling
 package org.nlogo.extensions.llm.providers
 
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, ResponseFormat, Usage}
 import org.nlogo.extensions.llm.config.ConfigStore
 import sttp.client4._
 import sttp.client4.httpclient.HttpClientFutureBackend
@@ -290,6 +290,7 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid
 
     val policy = retryPolicy
     val rng = retryRandom
+    val startedAt = System.currentTimeMillis()
 
     def isRateLimited(code: StatusCode, error: String): Boolean =
       code.code == 429 || error.toLowerCase.contains("rate_limit")
@@ -319,11 +320,19 @@ abstract class BaseHttpProvider(implicit ec: ExecutionContext) extends LLMProvid
         }
       }
 
-    requestThrottle match {
+    val completed = requestThrottle match {
       // One permit covers the complete logical request, including all retries.
       case Some(throttle) => throttle.withPermit(attempt(0, 0L))
       case None           => attempt(0, 0L)
     }
+
+    // Latency is the whole logical call as the modeler waits for it: throttle
+    // queueing and rate-limit backoff included. It is stamped even when the
+    // provider reported no token counts, so every call still has a record.
+    completed.map { response =>
+      val elapsedMs = System.currentTimeMillis() - startedAt
+      response.copy(usage = Some(response.usage.getOrElse(Usage.empty).copy(latencyMs = Some(elapsedMs))))
+    }
   }
 
   override def setConfig(key: String, value: String): Unit = {
diff --git a/src/main/providers/ClaudeProvider.scala b/src/main/providers/ClaudeProvider.scala
index 1302d1b..dffd2d3 100644
--- a/src/main/providers/ClaudeProvider.scala
+++ b/src/main/providers/ClaudeProvider.scala
@@ -3,7 +3,7 @@
 
 package org.nlogo.extensions.llm.providers
 
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage}
 import org.nlogo.extensions.llm.config.ConfigStore
 import sttp.client4._
 import sttp.model.Uri
@@ -170,6 +170,32 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider {
       case JsonObjectFormat => ()
     }
 
+  /**
+   * Anthropic reports input_tokens EXCLUDING cached tokens, with cache reads
+   * and writes as separate counts. The cross-provider inputTokens is every
+   * prompt token processed, so all three are summed; the cache counts are
+   * kept as well. Thinking tokens are already inside output_tokens, and no
+   * total is reported, so it is computed.
+   */
+  private def parseUsage(parsed: ujson.Value): Option[Usage] =
+    parsed.obj.get("usage").collect { case u: ujson.Obj => u }.flatMap { u =>
+      for {
+        uncached <- Usage.longField(u, "input_tokens")
+        output   <- Usage.longField(u, "output_tokens")
+      } yield {
+        val cacheWrite = Usage.longField(u, "cache_creation_input_tokens")
+        val cacheRead = Usage.longField(u, "cache_read_input_tokens")
+        val input = uncached + cacheWrite.getOrElse(0L) + cacheRead.getOrElse(0L)
+        Usage(
+          inputTokens = input,
+          outputTokens = output,
+          totalTokens = input + output,
+          cacheReadTokens = cacheRead,
+          cacheWriteTokens = cacheWrite
+        )
+      }
+    }
+
   override protected def parseProviderResponse(responseBody: String, model: String): ChatResponse = {
     try {
       val parsed = ujson.read(responseBody)
@@ -221,7 +247,7 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider {
         )
       )
 
-      ChatResponse(id, created, model, choices, thinking = thinking)
+      ChatResponse(id, created, model, choices, thinking = thinking, usage = parseUsage(parsed))
     } catch {
       case e: Exception =>
         throw new RuntimeException(s"Failed to parse Claude response: ${e.getMessage}\nResponse: $responseBody", e)
diff --git a/src/main/providers/GeminiProvider.scala b/src/main/providers/GeminiProvider.scala
index 3ff489d..50100b3 100644
--- a/src/main/providers/GeminiProvider.scala
+++ b/src/main/providers/GeminiProvider.scala
@@ -3,7 +3,7 @@
 
 package org.nlogo.extensions.llm.providers
 
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage}
 import org.nlogo.extensions.llm.config.ConfigStore
 import sttp.client4._
 import sttp.model.Uri
@@ -157,6 +157,29 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider {
   /**
    * Parse Gemini's response format into ChatResponse
    */
+  /**
+   * Gemini reports thinking tokens SEPARATELY from candidate tokens, and its
+   * promptTokenCount already includes cached content. The cross-provider
+   * outputTokens is every generated token, so thoughts are added to
+   * candidates; totalTokenCount is used when present because Gemini defines
+   * it the same way.
+   */
+  private def parseUsage(parsed: ujson.Value): Option[Usage] =
+    parsed.obj.get("usageMetadata").collect { case u: ujson.Obj => u }.flatMap { u =>
+      Usage.longField(u, "promptTokenCount").map { input =>
+        val candidates = Usage.longField(u, "candidatesTokenCount").getOrElse(0L)
+        val thoughts = Usage.longField(u, "thoughtsTokenCount")
+        val output = candidates + thoughts.getOrElse(0L)
+        Usage(
+          inputTokens = input,
+          outputTokens = output,
+          totalTokens = Usage.longField(u, "totalTokenCount").getOrElse(input + output),
+          reasoningTokens = thoughts,
+          cacheReadTokens = Usage.longField(u, "cachedContentTokenCount")
+        )
+      }
+    }
+
   override protected def parseProviderResponse(responseBody: String, model: String): ChatResponse = {
     try {
       val parsed = ujson.read(responseBody)
@@ -220,7 +243,7 @@ class GeminiProvider(implicit ec: ExecutionContext) extends BaseHttpProvider {
         )
       )
 
-      ChatResponse(s"gemini-${System.currentTimeMillis()}", System.currentTimeMillis() / 1000, model, choices, thinking = thinking)
+      ChatResponse(s"gemini-${System.currentTimeMillis()}", System.currentTimeMillis() / 1000, model, choices, thinking = thinking, usage = parseUsage(parsed))
     } catch {
       case e: RuntimeException => throw new RuntimeException(e.getMessage, e)
       case e: Exception =>
diff --git a/src/main/providers/OllamaProvider.scala b/src/main/providers/OllamaProvider.scala
index 06a995f..1a024a4 100644
--- a/src/main/providers/OllamaProvider.scala
+++ b/src/main/providers/OllamaProvider.scala
@@ -3,7 +3,7 @@
 
 package org.nlogo.extensions.llm.providers
 
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage}
 import org.nlogo.extensions.llm.config.ConfigStore
 import sttp.client4._
 import sttp.model.Uri
@@ -85,6 +85,13 @@ class OllamaProvider(implicit ec: ExecutionContext) extends BaseHttpProvider {
     baseRequest
   }
 
+  /** Ollama reports prompt_eval_count / eval_count at the top level and no total. */
+  private def parseUsage(parsed: ujson.Value): Option[Usage] =
+    for {
+      input  <- Usage.longField(parsed, "prompt_eval_count")
+      output <- Usage.longField(parsed, "eval_count")
+    } yield Usage(inputTokens = input, outputTokens = output, totalTokens = input + output)
+
   override protected def parseProviderResponse(responseBody: String, model: String): ChatResponse = {
     try {
       val parsed = ujson.read(responseBody)
@@ -113,7 +120,7 @@ class OllamaProvider(implicit ec: ExecutionContext) extends BaseHttpProvider {
         )
       )
 
-      ChatResponse(id, created, model, choices, thinking = thinking)
+      ChatResponse(id, created, model, choices, thinking = thinking, usage = parseUsage(parsed))
     } catch {
       case e: Exception =>
         throw new RuntimeException(s"Failed to parse Ollama response: ${e.getMessage}\nResponse: $responseBody", e)
diff --git a/src/main/providers/OpenAICompatibleProvider.scala b/src/main/providers/OpenAICompatibleProvider.scala
index 113fadd..dfe9cb2 100644
--- a/src/main/providers/OpenAICompatibleProvider.scala
+++ b/src/main/providers/OpenAICompatibleProvider.scala
@@ -2,7 +2,7 @@
 // ABOUTME: Shared by OpenAI, OpenRouter, and Together AI — subclasses override hooks for headers, reasoning, and thinking
 package org.nlogo.extensions.llm.providers
 
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage}
 import org.nlogo.extensions.llm.config.ConfigStore
 import sttp.client4._
 import sttp.model.Uri
@@ -123,6 +123,28 @@ abstract class OpenAICompatibleProvider(implicit ec: ExecutionContext) extends B
       )
     )
 
+  /**
+   * Token usage in the OpenAI chat-completions shape. Reasoning tokens are a
+   * subset of completion_tokens and cached tokens a subset of prompt_tokens,
+   * so the headline counts already match the cross-provider definition.
+   * OpenRouter adds a dollar `cost`; it is passed through untouched.
+   */
+  protected def parseUsage(parsed: ujson.Value): Option[Usage] =
+    parsed.obj.get("usage").collect { case u: ujson.Obj => u }.flatMap { u =>
+      for {
+        input  <- Usage.longField(u, "prompt_tokens")
+        output <- Usage.longField(u, "completion_tokens")
+      } yield Usage(
+        inputTokens = input,
+        outputTokens = output,
+        totalTokens = Usage.longField(u, "total_tokens").getOrElse(input + output),
+        reasoningTokens = Usage.nestedLongField(u, "completion_tokens_details", "reasoning_tokens"),
+        cacheReadTokens = Usage.nestedLongField(u, "prompt_tokens_details", "cached_tokens"),
+        cacheWriteTokens = Usage.nestedLongField(u, "prompt_tokens_details", "cache_write_tokens"),
+        cost = u.value.get("cost").collect { case ujson.Num(n) => n }
+      )
+    }
+
   override protected def parseProviderResponse(responseBody: String, model: String): ChatResponse = {
     try {
       val parsed = ujson.read(responseBody)
@@ -147,7 +169,7 @@ abstract class OpenAICompatibleProvider(implicit ec: ExecutionContext) extends B
         .map(_("message"))
         .flatMap(extractThinking)
 
-      ChatResponse(id, created, model, choices, thinking)
+      ChatResponse(id, created, model, choices, thinking, usage = parseUsage(parsed))
     } catch {
       case e: Exception =>
         throw new RuntimeException(s"Failed to parse ${providerName} response: ${e.getMessage}\nResponse: $responseBody", e)
diff --git a/src/test/DeterministicTestProvider.scala b/src/test/DeterministicTestProvider.scala
index cb118e5..cc70b32 100644
--- a/src/test/DeterministicTestProvider.scala
+++ b/src/test/DeterministicTestProvider.scala
@@ -1,7 +1,7 @@
 package org.nlogo.extensions.llm.providers
 
 import org.nlogo.extensions.llm.config.ConfigStore
-import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat}
+import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ChatResponse, Choice, EnumFormat, JsonObjectFormat, JsonSchemaFormat, ResponseFormat, Usage}
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Success, Try}
 
@@ -20,18 +20,42 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid
   private val testDelayRegex = """__TEST_DELAY:(\d+):(.*)""".r.unanchored
   // Captures to end of line so a raw choice may contain spaces, braces, or quotes.
   private val testRawChoiceRegex = """__TEST_RAW_CHOICE:(.*)""".r
+  // __TEST_USAGE:<in>,<out>[,<reasoning>] attaches a usage record to whatever
+  // response the rest of the stub produces, so accounting can be asserted
+  // independently of which primitive made the call.
+  private val testUsageRegex = """__TEST_USAGE:(\d+),(\d+)(?:,(\d+))?""".r.unanchored
+
+  private def lastUserContent(messages: Seq[ChatMessage]): String =
+    messages.reverseIterator.find(_.role == "user").map(_.content).getOrElse("")
+
+  private def withTestUsage(lastUserMessage: String, response: ChatResponse): ChatResponse =
+    lastUserMessage match {
+      case testUsageRegex(in, out, reasoning) =>
+        val i = in.toLong
+        val o = out.toLong
+        response.copy(usage = Some(Usage(
+          inputTokens = i,
+          outputTokens = o,
+          totalTokens = i + o,
+          reasoningTokens = Option(reasoning).map(_.toLong)
+        )))
+      case _ => response
+    }
 
   override def chat(request: ChatRequest): Future[ChatResponse] = {
     chat(request.messages).map { message =>
-      ChatResponse.simple(
+      withTestUsage(lastUserContent(request.messages), ChatResponse.simple(
         id = "deterministic-test-response",
         model = request.model,
         message = message
-      )
+      ))
     }
   }
 
-  override def chatWithFullResponse(messages: Seq[ChatMessage]): Future[ChatResponse] = {
+  override def chatWithFullResponse(messages: Seq[ChatMessage]): Future[ChatResponse] =
+    fullResponse(messages).map(withTestUsage(lastUserContent(messages), _))
+
+  private def fullResponse(messages: Seq[ChatMessage]): Future[ChatResponse] = {
     val lastUserMessage = messages.reverseIterator
       .find(_.role == "user")
       .map(_.content)
@@ -68,7 +92,10 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid
    * request answers in the constrained `{"choice": ...}` shape, which is what
    * a real provider enforcing the constraint returns.
    */
-  override def chatWithFormat(messages: Seq[ChatMessage], format: ResponseFormat): Future[ChatResponse] = {
+  override def chatWithFormat(messages: Seq[ChatMessage], format: ResponseFormat): Future[ChatResponse] =
+    formatResponse(messages, format).map(withTestUsage(lastUserContent(messages), _))
+
+  private def formatResponse(messages: Seq[ChatMessage], format: ResponseFormat): Future[ChatResponse] = {
     val lastUserMessage = messages.reverseIterator
       .find(_.role == "user")
       .map(_.content)
diff --git a/src/test/UsageParsingSpec.scala b/src/test/UsageParsingSpec.scala
new file mode 100644
index 0000000..d13f229
--- /dev/null
+++ b/src/test/UsageParsingSpec.scala
@@ -0,0 +1,186 @@
+// ABOUTME: Deterministic tests that each provider parser reads token usage from a response body
+// ABOUTME: Asserts the cross-provider Usage record is filled consistently, or absent when not reported
+package org.nlogo.extensions.llm.providers
+
+import org.nlogo.extensions.llm.models.{ChatResponse, Usage}
+import org.scalatest.funsuite.AnyFunSuite
+import scala.concurrent.ExecutionContext.Implicits.global
+
+class InspectableOpenAIProvider extends OpenAIProvider()(using global) {
+  def parse(body: String): ChatResponse = parseProviderResponse(body, "test-model")
+}
+
+class InspectableOpenRouterProvider extends OpenRouterProvider()(using global) {
+  def parse(body: String): ChatResponse = parseProviderResponse(body, "test-model")
+}
+
+class InspectableClaudeParser extends ClaudeProvider()(using global) {
+  def parse(body: String): ChatResponse = parseProviderResponse(body, "test-model")
+}
+
+class InspectableGeminiProvider extends GeminiProvider()(using global) {
+  def parse(body: String): ChatResponse = parseProviderResponse(body, "test-model")
+}
+
+class InspectableOllamaProvider extends OllamaProvider()(using global) {
+  def parse(body: String): ChatResponse = parseProviderResponse(body, "test-model")
+}
+
+class UsageParsingSpec extends AnyFunSuite {
+
+  ProviderRegistrations.registerAll()
+
+  private val openAiBody =
+    """{"id":"chatcmpl-1","created":1700000000,"model":"gpt-4o-mini",
+      | "choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],
+      | "usage":{"prompt_tokens":12,"completion_tokens":7,"total_tokens":19,
+      |   "prompt_tokens_details":{"cached_tokens":4},
+      |   "completion_tokens_details":{"reasoning_tokens":3}}}""".stripMargin
+
+  test("OpenAI-compatible parser reads prompt, completion, total, reasoning and cached tokens") {
+    val usage = new InspectableOpenAIProvider().parse(openAiBody).usage
+    assert(usage.contains(Usage(
+      inputTokens = 12,
+      outputTokens = 7,
+      totalTokens = 19,
+      reasoningTokens = Some(3),
+      cacheReadTokens = Some(4)
+    )))
+  }
+
+  test("OpenAI-compatible parser leaves usage absent when the provider omits it") {
+    val body =
+      """{"id":"chatcmpl-2","created":1700000000,"model":"m",
+        | "choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}""".stripMargin
+    assert(new InspectableOpenAIProvider().parse(body).usage.isEmpty)
+  }
+
+  test("OpenAI-compatible parser tolerates missing detail sub-objects") {
+    val body =
+      """{"id":"chatcmpl-3","created":1700000000,"model":"m",
+        | "choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],
+        | "usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}""".stripMargin
+    val usage = new InspectableOpenAIProvider().parse(body).usage
+    assert(usage.contains(Usage(inputTokens = 5, outputTokens = 2, totalTokens = 7)))
+  }
+
+  test("OpenRouter parser exposes the reported dollar cost") {
+    val body =
+      """{"id":"gen-1","created":1700000000,"model":"openai/gpt-4o-mini",
+        | "choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],
+        | "usage":{"prompt_tokens":10,"completion_tokens":4,"total_tokens":14,"cost":0.0000123}}""".stripMargin
+    val usage = new InspectableOpenRouterProvider().parse(body).usage
+    assert(usage.flatMap(_.cost).contains(0.0000123))
+    assert(usage.map(_.inputTokens).contains(10))
+  }
+
+  test("Claude parser counts cached prompt tokens as input and computes the total") {
+    val body =
+      """{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5",
+        | "content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn",
+        | "usage":{"input_tokens":20,"output_tokens":6,
+        |   "cache_creation_input_tokens":8,"cache_read_input_tokens":30}}""".stripMargin
+    val usage = new InspectableClaudeParser().parse(body).usage
+    assert(usage.contains(Usage(
+      inputTokens = 58,
+      outputTokens = 6,
+      totalTokens = 64,
+      cacheReadTokens = Some(30),
+      cacheWriteTokens = Some(8)
+    )))
+  }
+
+  test("Claude parser handles a usage block with only the two required fields") {
+    val body =
+      """{"id":"msg_2","type":"message","role":"assistant","model":"m",
+        | "content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn",
+        | "usage":{"input_tokens":3,"output_tokens":1}}""".stripMargin
+    val usage = new InspectableClaudeParser().parse(body).usage
+    assert(usage.contains(Usage(inputTokens = 3, outputTokens = 1, totalTokens = 4)))
+  }
+
+  test("Gemini parser counts thinking tokens as output and reads cached content tokens") {
+    val body =
+      """{"candidates":[{"content":{"parts":[{"text":"hi"}],"role":"model"},"finishReason":"STOP"}],
+        | "usageMetadata":{"promptTokenCount":15,"candidatesTokenCount":5,"thoughtsTokenCount":9,
+        |   "cachedContentTokenCount":2,"totalTokenCount":29}}""".stripMargin
+    val usage = new InspectableGeminiProvider().parse(body).usage
+    assert(usage.contains(Usage(
+      inputTokens = 15,
+      outputTokens = 14,
+      totalTokens = 29,
+      reasoningTokens = Some(9),
+      cacheReadTokens = Some(2)
+    )))
+  }
+
+  test("Gemini parser leaves usage absent when usageMetadata is missing") {
+    val body =
+      """{"candidates":[{"content":{"parts":[{"text":"hi"}],"role":"model"},"finishReason":"STOP"}]}"""
+    assert(new InspectableGeminiProvider().parse(body).usage.isEmpty)
+  }
+
+  test("Ollama parser maps eval counts to input and output and computes the total") {
+    val body =
+      """{"model":"llama3","message":{"role":"assistant","content":"hi"},"done":true,
+        | "total_duration":5000000000,"prompt_eval_count":11,"eval_count":4}""".stripMargin
+    val usage = new InspectableOllamaProvider().parse(body).usage
+    assert(usage.contains(Usage(inputTokens = 11, outputTokens = 4, totalTokens = 15)))
+  }
+
+  test("Ollama parser leaves usage absent when counts are missing") {
+    val body = """{"model":"llama3","message":{"role":"assistant","content":"hi"},"done":true}"""
+    assert(new InspectableOllamaProvider().parse(body).usage.isEmpty)
+  }
+
+  test("HTTP provider stamps wall-clock latency on every response, even without token usage") {
+    val counter = new java.util.concurrent.atomic.AtomicInteger(0)
+    val stub = sttp.client4.testing.BackendStub.asynchronousFuture.whenAnyRequest.thenRespond {
+      counter.incrementAndGet()
+      sttp.client4.testing.ResponseStub.adjust("hello", sttp.model.StatusCode.Ok)
+    }
+    val provider = new StubbedProvider(stub, counter)
+    val request = org.nlogo.extensions.llm.models.ChatRequest(
+      model = "stub-model",
+      messages = Seq(org.nlogo.extensions.llm.models.ChatMessage.user("hi"))
+    )
+    val response = scala.concurrent.Await.result(provider.chat(request), scala.concurrent.duration.Duration(5, "s"))
+    val latency = response.usage.flatMap(_.latencyMs)
+    assert(latency.isDefined, "latency should be stamped even when the parser reports no usage")
+    assert(latency.get >= 0L)
+    assert(response.usage.map(_.totalTokens).contains(0L))
+  }
+
+  test("latency covers rate-limit retries, not just the final attempt") {
+    val counter = new java.util.concurrent.atomic.AtomicInteger(0)
+    val stub = sttp.client4.testing.BackendStub.asynchronousFuture.whenAnyRequest.thenRespond {
+      if (counter.getAndIncrement() == 0)
+        sttp.client4.testing.ResponseStub.adjust("rate limit exceeded", sttp.model.StatusCode.TooManyRequests)
+      else
+        sttp.client4.testing.ResponseStub.adjust("hello", sttp.model.StatusCode.Ok)
+    }
+    val provider = new StubbedProvider(stub, counter)
+    val request = org.nlogo.extensions.llm.models.ChatRequest(
+      model = "stub-model",
+      messages = Seq(org.nlogo.extensions.llm.models.ChatMessage.user("hi"))
+    )
+    val response = scala.concurrent.Await.result(provider.chat(request), scala.concurrent.duration.Duration(5, "s"))
+    val latency = response.usage.flatMap(_.latencyMs).getOrElse(-1L)
+    assert(provider.delays.nonEmpty, "the stub should have slept once for the 429")
+    assert(latency >= provider.delays.sum, s"latency $latency ms should include the ${provider.delays.sum} ms backoff")
+  }
+
+  test("Usage.plus sums every counter and keeps optional fields optional when both sides lack them") {
+    val a = Usage(inputTokens = 1, outputTokens = 2, totalTokens = 3, reasoningTokens = Some(1), latencyMs = Some(100))
+    val b = Usage(inputTokens = 10, outputTokens = 20, totalTokens = 30, cost = Some(0.5), latencyMs = Some(50))
+    assert(a.plus(b) == Usage(
+      inputTokens = 11,
+      outputTokens = 22,
+      totalTokens = 33,
+      reasoningTokens = Some(1),
+      cost = Some(0.5),
+      latencyMs = Some(150)
+    ))
+    assert(Usage.empty.plus(Usage.empty) == Usage.empty)
+  }
+}
diff --git a/tests.txt b/tests.txt
index b4a1503..4faf5de 100644
--- a/tests.txt
+++ b/tests.txt
@@ -734,3 +734,105 @@ LLMChooseUnmatchedAfterDegradationLeavesHistoryClean
   llm:choose "__TEST_IGNORE_FORMAT __TEST_RAW_CHOICE:up" ["north" "south"] => ERROR Extension exception: llm:choose: response 'up' did not match any choice. Choices: north, south
   llm:history => []
   O> llm:clear-history
+
+LLMUsageStartsAtZero
+  extensions [llm]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  llm:get llm:usage "input-tokens" => 0
+  llm:get llm:usage "output-tokens" => 0
+  llm:get llm:usage "total-tokens" => 0
+  llm:get llm:usage "calls" => 0
+  llm:get llm:usage "cost" => ""
+  llm:get llm:usage-total "calls" => 0
+
+LLMUsageAccumulatesAcrossCalls
+  extensions [llm]
+  globals [r]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> set r llm:chat "__TEST_USAGE:10,5 hello"
+  O> set r llm:chat "__TEST_USAGE:20,7,3 again"
+  llm:get llm:usage "input-tokens" => 30
+  llm:get llm:usage "output-tokens" => 12
+  llm:get llm:usage "total-tokens" => 42
+  llm:get llm:usage "reasoning-tokens" => 3
+  llm:get llm:usage "calls" => 2
+  O> clear-all
+
+LLMUsageIsPerAgent
+  extensions [llm]
+  globals [r]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> crt 2
+  O> ask turtles [ set r llm:chat (word "__TEST_USAGE:" (10 * (who + 1)) ",1 hi") ]
+  [llm:get llm:usage "input-tokens"] of turtle 0 => 10
+  [llm:get llm:usage "input-tokens"] of turtle 1 => 20
+  llm:get llm:usage "input-tokens" => 0
+  llm:get llm:usage-total "input-tokens" => 30
+  llm:get llm:usage-total "output-tokens" => 2
+  llm:get llm:usage-total "calls" => 2
+  O> clear-all
+
+LLMUsageResetsOnClearAll
+  extensions [llm]
+  globals [r]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> set r llm:chat "__TEST_USAGE:10,5 hi"
+  llm:get llm:usage-total "calls" => 1
+  O> clear-all
+  llm:get llm:usage "calls" => 0
+  llm:get llm:usage-total "calls" => 0
+
+LLMUsageRecordsAsyncCalls
+  extensions [llm]
+  globals [p r]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> set p llm:chat-async "__TEST_USAGE:4,3 hi"
+  O> set r (runresult p)
+  llm:get llm:usage "total-tokens" => 7
+  llm:get llm:usage "calls" => 1
+  O> clear-all
+
+LLMUsageRecordsEveryPrimitive
+  extensions [llm]
+  globals [r]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> set r llm:choose "__TEST_USAGE:1,1 pick" ["north" "south"]
+  O> set r llm:chat-with-schema "__TEST_USAGE:2,1" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}"
+  O> set r llm:chat-json "__TEST_USAGE:3,1"
+  O> set r llm:chat-with-thinking "__TEST_USAGE:4,1 think"
+  llm:get llm:usage "input-tokens" => 10
+  llm:get llm:usage "calls" => 4
+  O> clear-all
+
+LLMUsageCountsRejectedReply
+  extensions [llm]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> carefully [ let r llm:chat-with-schema "__TEST_USAGE:9,2 __TEST_RESPOND:not-json" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" ] [ ]
+  llm:history => []
+  llm:get llm:usage "input-tokens" => 9
+  llm:get llm:usage "calls" => 1
+  O> clear-all
+
+LLMUsageUnchangedWhenCallFails
+  extensions [llm]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> clear-all
+  O> carefully [ let r llm:chat "__TEST_USAGE:9,2 __TEST_FAIL" ] [ ]
+  llm:get llm:usage "calls" => 0
+  llm:get llm:usage "input-tokens" => 0
+  O> clear-all