From bb4684fac138c9cbf8d7276df9076f0efd127014 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:35:46 +0000 Subject: [PATCH 01/11] Serialize explicit null Item, event and reasoning fields Official responses always carry these fields (EVT-09, SES-23, SES-25, VA-11): Item events carry output_index, null for input Items; messages carry phase, null for user messages and when the harness reports none; function results carry output and error, null when not submitted; Agent and Session responses carry reasoning.effort and reasoning.summary, null when unset. No model-derived default effort is filled in. These are wire-only. Stored Item payloads keep their original encoding through Item.MarshalStored, so replayed child Items still compare equal, and stored configuration keeps omitting unset reasoning keys, so creation retry identity is unchanged. The OpenAPI marks phase and output_index nullable. The TypeScript client accepts output_index and phase null and still accepts older Cores that omit them; Core Web accepts a null message phase. The pinned SDK scripts now expect the explicit nulls. --- .../sessions/trace/trace-model.test.ts | 18 ++ .../features/sessions/trace/trace-model.ts | 2 +- contracts/agents-api/openapi.yaml | 2 + contracts/agents-api/v1/agents.go | 10 ++ contracts/agents-api/v1/events.go | 29 +++- contracts/agents-api/v1/items.go | 2 +- contracts/agents-api/v1/items_test.go | 157 ++++++++++++++++++ contracts/agents-api/v1/sessions.go | 27 +++ contracts/agents-api/v1/subagent_items.go | 36 +++- packages/agents-client/src/client.test.ts | 53 ++++++ packages/agents-client/src/client.ts | 3 +- .../agents-client/src/history-projection.ts | 3 +- packages/agents-client/src/types.ts | 6 +- .../agents-api/internal/api/handler_test.go | 28 ++++ .../agents-api/internal/api/stream_test.go | 53 ++++++ .../store/function_item_events_test.go | 19 ++- .../internal/store/item_projection.go | 2 +- .../internal/store/subagent_coordination.go | 2 +- .../store/subagent_item_projection.go | 2 +- services/agents-api/tests/official_agents.py | 2 + .../tests/official_function_images.py | 3 +- .../tests/official_function_stream.py | 5 +- .../agents-api/tests/official_functions.py | 5 +- .../tests/official_hosted_functions_native.py | 2 +- 24 files changed, 443 insertions(+), 28 deletions(-) create mode 100644 contracts/agents-api/v1/items_test.go diff --git a/apps/web/src/features/sessions/trace/trace-model.test.ts b/apps/web/src/features/sessions/trace/trace-model.test.ts index 1da1357d1..36e2d06a0 100644 --- a/apps/web/src/features/sessions/trace/trace-model.test.ts +++ b/apps/web/src/features/sessions/trace/trace-model.test.ts @@ -301,6 +301,24 @@ describe("trace summary and search", () => { }); }); + it("accepts the explicit null phase, output and error fields of current Cores", () => { + const items: SessionItem[] = [ + { id: "user", turn_id: "one", type: "message", status: "completed", role: "user", phase: null, content: [{ type: "input_text", text: "question" }] }, + { id: "answer", turn_id: "one", type: "message", status: "completed", role: "assistant", phase: null, content: [{ type: "output_text", text: "answer" }] }, + { id: "call", turn_id: "one", type: "function_call", status: "completed", call_id: "call-1", name: "lookup", arguments: {} }, + { id: "result", turn_id: "one", type: "function_call_output", status: "failed", call_id: "call-1", output: null, error: "lookup failed" }, + ]; + + const rows = buildTraceModel({ turns: [turn("one")], items, agent }).groups[1]?.rows; + + expect(rows?.map((row) => [row.kind, row.title])).toEqual([ + ["user_message", "User message"], + ["assistant_message", "Assistant message"], + ["tool_call", "lookup"], + ]); + expect(rows?.[2]?.tool?.result).toEqual({ state: "available", value: { output: null, error: "lookup failed" } }); + }); + it("degrades malformed known variants without throwing or deriving a title from arguments", () => { const malformedSearch = { id: "search", diff --git a/apps/web/src/features/sessions/trace/trace-model.ts b/apps/web/src/features/sessions/trace/trace-model.ts index 7e2ba79a7..f4de4046c 100644 --- a/apps/web/src/features/sessions/trace/trace-model.ts +++ b/apps/web/src/features/sessions/trace/trace-model.ts @@ -200,7 +200,7 @@ function validatedItemType(item: SessionItem): SessionItem["type"] | null { if (raw.type === "message") { return MESSAGE_ROLES.has(raw.role as string) && validContent(raw.content) - && (!hasOwn(raw, "phase") || raw.phase === undefined || MESSAGE_PHASES.has(raw.phase as string)) + && (!hasOwn(raw, "phase") || raw.phase === undefined || raw.phase === null || MESSAGE_PHASES.has(raw.phase as string)) ? raw.type : null; } diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index f5119a583..ff61f8869 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -982,6 +982,7 @@ definitions: - commentary - final_answer type: string + x-nullable: true reasoning_effort: type: string recipient_agent_id: @@ -2148,6 +2149,7 @@ definitions: type: string output_index: type: integer + x-nullable: true part: $ref: '#/definitions/v1.ItemContent' session: diff --git a/contracts/agents-api/v1/agents.go b/contracts/agents-api/v1/agents.go index 5eafba9d2..17cda82d6 100644 --- a/contracts/agents-api/v1/agents.go +++ b/contracts/agents-api/v1/agents.go @@ -69,6 +69,16 @@ type SavedAgent struct { UpdatedAt int64 `json:"updated_at" binding:"required"` } +// MarshalJSON renders reasoning with explicit null keys. The stored +// SavedAgentConfiguration keeps its original encoding. +func (a SavedAgent) MarshalJSON() ([]byte, error) { + type wire SavedAgent + return json.Marshal(struct { + wire + Reasoning reasoningResponse `json:"reasoning"` + }{wire(a), reasoningResponse(a.Reasoning)}) +} + type SavedAgentList struct { Object string `json:"object" binding:"required" enums:"list"` Data []SavedAgent `json:"data" binding:"required"` diff --git a/contracts/agents-api/v1/events.go b/contracts/agents-api/v1/events.go index 9ddc64483..476557c49 100644 --- a/contracts/agents-api/v1/events.go +++ b/contracts/agents-api/v1/events.go @@ -13,7 +13,7 @@ type SessionEvent struct { Turn *Turn `json:"turn,omitempty"` Item *Item `json:"item,omitempty"` ItemID string `json:"item_id,omitempty"` - OutputIndex *int32 `json:"output_index,omitempty"` + OutputIndex *int32 `json:"output_index,omitempty" extensions:"x-nullable"` ContentIndex *int `json:"content_index,omitempty"` Part *ItemContent `json:"part,omitempty"` Delta *string `json:"delta,omitempty"` @@ -40,15 +40,28 @@ func TerminalTurnEvent(eventType string) bool { return false } -// MarshalJSON keeps the nullable top-level usage on terminal Turn events only. +// itemEvent reports whether an event type adds or completes an Item. +func itemEvent(eventType string) bool { + return eventType == "agent.session.turn.item.added" || eventType == "agent.session.turn.item.done" +} + +// MarshalJSON keeps the nullable top-level usage on terminal Turn events only, +// and a nullable output_index on every Item event (EVT-09). func (e SessionEvent) MarshalJSON() ([]byte, error) { type wire SessionEvent - if !TerminalTurnEvent(e.Type) { + switch { + case TerminalTurnEvent(e.Type): + return json.Marshal(struct { + wire + Usage *TokenUsage `json:"usage"` + }{wire(e), e.Usage}) + case itemEvent(e.Type): e.Usage = nil - return json.Marshal(wire(e)) + return json.Marshal(struct { + wire + OutputIndex *int32 `json:"output_index"` + }{wire(e), e.OutputIndex}) } - return json.Marshal(struct { - wire - Usage *TokenUsage `json:"usage"` - }{wire(e), e.Usage}) + e.Usage = nil + return json.Marshal(wire(e)) } diff --git a/contracts/agents-api/v1/items.go b/contracts/agents-api/v1/items.go index e1a17c9c3..c004bcf4c 100644 --- a/contracts/agents-api/v1/items.go +++ b/contracts/agents-api/v1/items.go @@ -12,7 +12,7 @@ type Item struct { Type string `json:"type" binding:"required" enums:"message,command_execution,mcp_call,function_call,function_call_output,web_search_call,reasoning,agent_message,create_subagent_call,send_subagent_input_call,resume_subagent_call,wait_for_subagents_call,interrupt_subagent_call,close_subagent_call"` Status string `json:"status" enums:"in_progress,completed,failed,incomplete"` Role string `json:"role,omitempty" enums:"user,assistant"` - Phase string `json:"phase,omitempty" enums:"commentary,final_answer"` + Phase string `json:"phase,omitempty" enums:"commentary,final_answer" extensions:"x-nullable"` Content []ItemContent `json:"content,omitempty"` Command string `json:"command,omitempty"` Cwd *string `json:"cwd,omitempty"` diff --git a/contracts/agents-api/v1/items_test.go b/contracts/agents-api/v1/items_test.go new file mode 100644 index 000000000..5de654118 --- /dev/null +++ b/contracts/agents-api/v1/items_test.go @@ -0,0 +1,157 @@ +package v1 + +import ( + "encoding/json" + "testing" +) + +func fields(t *testing.T, value any) map[string]json.RawMessage { + t.Helper() + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + var result map[string]json.RawMessage + if err := json.Unmarshal(raw, &result); err != nil { + t.Fatal(err) + } + return result +} + +func expectField(t *testing.T, got map[string]json.RawMessage, key, want string) { + t.Helper() + value, present := got[key] + if want == "" { + if present { + t.Fatalf("unexpected %s: %s", key, value) + } + return + } + if !present || string(value) != want { + t.Fatalf("%s = %s (present %v), want %s", key, value, present, want) + } +} + +func TestItemWireFieldsAreExplicitlyNull(t *testing.T) { + text, phase := "question", "final_answer" + user := Item{ID: "user", TurnID: "turn", Type: "message", Status: "completed", Role: "user", Content: []ItemContent{{Type: "input_text", Text: &text}}} + got := fields(t, user) + expectField(t, got, "phase", "null") + expectField(t, got, "content", `[{"type":"input_text","text":"question"}]`) + expectField(t, got, "error", "") + expectField(t, got, "output", "") + + assistant := Item{ID: "answer", TurnID: "turn", Type: "message", Status: "in_progress", Role: "assistant", Phase: phase} + got = fields(t, assistant) + expectField(t, got, "phase", `"final_answer"`) + expectField(t, got, "content", "[]") + + for _, test := range []struct { + name string + raw string + output, error string + }{ + {"omitted", `{"id":"r","turn_id":"turn","type":"function_call_output","status":"completed","call_id":"c"}`, "null", "null"}, + {"null", `{"id":"r","turn_id":"turn","type":"function_call_output","status":"failed","call_id":"c","output":null,"error":null}`, "null", "null"}, + {"values", `{"id":"r","turn_id":"turn","type":"function_call_output","status":"failed","call_id":"c","output":[{"type":"input_text","text":"code=9007199254740993"}],"error":"failed"}`, `[{"text":"code=9007199254740993","type":"input_text"}]`, `"failed"`}, + } { + t.Run(test.name, func(t *testing.T) { + var item Item + if err := json.Unmarshal([]byte(test.raw), &item); err != nil { + t.Fatal(err) + } + got := fields(t, item) + expectField(t, got, "output", test.output) + expectField(t, got, "error", test.error) + expectField(t, got, "phase", "") + // Stored payloads keep the original field presence. + stored, err := item.MarshalStored() + if err != nil { + t.Fatal(err) + } + var original, persisted map[string]json.RawMessage + if json.Unmarshal([]byte(test.raw), &original) != nil || json.Unmarshal(stored, &persisted) != nil || len(original) != len(persisted) { + t.Fatalf("stored payload changed: %s", stored) + } + }) + } + + // Other variants keep omitting unset fields, including the new wire nulls. + call := fields(t, Item{ID: "call", TurnID: "turn", Type: "function_call", Status: "completed", Name: "lookup", CallID: "call", Arguments: json.RawMessage(`{}`)}) + for _, key := range []string{"phase", "error", "output", "content"} { + expectField(t, call, key, "") + } + reasoning := fields(t, Item{ID: "rs", TurnID: "turn", Type: "reasoning"}) + expectField(t, reasoning, "status", "null") + expectField(t, reasoning, "summary", "[]") + + stored, err := user.MarshalStored() + if err != nil { + t.Fatal(err) + } + if string(stored) != `{"id":"user","turn_id":"turn","type":"message","status":"completed","role":"user","content":[{"type":"input_text","text":"question"}]}` { + t.Fatalf("stored message encoding changed: %s", stored) + } +} + +func TestItemEventsCarryNullableOutputIndex(t *testing.T) { + text := "question" + item := &Item{ID: "user", TurnID: "turn", Type: "message", Status: "completed", Role: "user", Content: []ItemContent{{Type: "input_text", Text: &text}}} + index, zero := int32(2), 0 + for _, test := range []struct { + event SessionEvent + want string + }{ + {SessionEvent{Type: "agent.session.turn.item.added", Item: item}, "null"}, + {SessionEvent{Type: "agent.session.turn.item.done", Item: item, OutputIndex: &index}, "2"}, + {SessionEvent{Type: "agent.session.turn.content_part.added", ItemID: "answer", OutputIndex: &index, ContentIndex: &zero}, "2"}, + {SessionEvent{Type: "agent.session.turn.created", Turn: &Turn{ID: "turn"}}, ""}, + } { + test.event.EventID, test.event.TurnID = "event", "turn" + got := fields(t, test.event) + expectField(t, got, "output_index", test.want) + expectField(t, got, "usage", "") + if test.event.Item != nil { + expectField(t, fields(t, got["item"]), "phase", "null") + } + } +} + +func TestReasoningResponsesCarryBothKeys(t *testing.T) { + low := "low" + auto := "auto" + for _, test := range []struct { + reasoning Reasoning + want string + }{ + {Reasoning{}, `{"effort":null,"summary":null}`}, + {Reasoning{Effort: &low}, `{"effort":"low","summary":null}`}, + {Reasoning{Summary: &auto}, `{"effort":null,"summary":"auto"}`}, + } { + agent := Agent{ID: "agent", Model: "model", Reasoning: test.reasoning, ServiceTier: "auto", Tools: []json.RawMessage{}} + session := fields(t, Session{ID: "session", Agent: agent, Object: "agent.session", Metadata: map[string]string{}, RequiredActions: []RequiredAction{}, VaultIDs: []string{}}) + expectField(t, fields(t, session["agent"]), "reasoning", test.want) + expectField(t, fields(t, session["agent"]), "model", `"model"`) + expectField(t, session, "usage", "null") + event := fields(t, SessionEvent{Type: "agent.session.idle", EventID: "event", Session: &Session{Agent: agent}}) + expectField(t, fields(t, fields(t, event["session"])["agent"]), "reasoning", test.want) + + saved := SavedAgent{SavedAgentConfiguration: SavedAgentConfiguration{Model: "model", Reasoning: test.reasoning, ServiceTier: "auto", Tools: []json.RawMessage{}}, ID: "agent", Object: "agent", Metadata: map[string]string{}} + got := fields(t, saved) + expectField(t, got, "reasoning", test.want) + expectField(t, got, "id", `"agent"`) + expectField(t, got, "model", `"model"`) + list := fields(t, SavedAgentList{Object: "list", Data: []SavedAgent{saved}}) + var data []map[string]json.RawMessage + if json.Unmarshal(list["data"], &data) != nil || len(data) != 1 || string(data[0]["reasoning"]) != test.want { + t.Fatalf("list reasoning: %s", list["data"]) + } + } + // Requests and stored configuration keep the omitting encoding. + stored := fields(t, SavedAgentConfiguration{Model: "model"}) + expectField(t, stored, "reasoning", "{}") + stored = fields(t, struct { + Agent Agent `json:"agent"` + }{Agent{ID: "agent", Reasoning: Reasoning{Effort: &low}}}) + expectField(t, fields(t, stored["agent"]), "reasoning", `{"effort":"low"}`) +} diff --git a/contracts/agents-api/v1/sessions.go b/contracts/agents-api/v1/sessions.go index 8f9631da9..892316dfb 100644 --- a/contracts/agents-api/v1/sessions.go +++ b/contracts/agents-api/v1/sessions.go @@ -74,6 +74,15 @@ type Reasoning struct { Summary *string `json:"summary,omitempty" extensions:"x-nullable"` } +// reasoningResponse renders both reasoning keys, null when unset, as pinned +// Agent and Session responses do (SES-23, VA-11). Requests and stored +// configuration keep the omitting Reasoning encoding. It does not resolve a +// model-derived default effort. +type reasoningResponse struct { + Effort *string `json:"effort"` + Summary *string `json:"summary"` +} + type TextConfigInput struct { Format *TextFormat `json:"format,omitempty" extensions:"x-nullable"` Verbosity *string `json:"verbosity,omitempty" enums:"low,medium,high" extensions:"x-nullable"` @@ -104,6 +113,24 @@ type Session struct { VaultIDs []string `json:"vault_ids" binding:"required"` } +type agentWire Agent + +// agentResponse renders a Session Agent with explicit reasoning keys. +type agentResponse struct { + agentWire + Reasoning reasoningResponse `json:"reasoning"` +} + +// MarshalJSON renders the Session Agent's reasoning with explicit null keys. +// The stored Session configuration keeps its original encoding. +func (s Session) MarshalJSON() ([]byte, error) { + type wire Session + return json.Marshal(struct { + wire + Agent agentResponse `json:"agent"` + }{wire(s), agentResponse{agentWire(s.Agent), reasoningResponse(s.Agent.Reasoning)}}) +} + type SessionList struct { Object string `json:"object" enums:"list" binding:"required"` FirstID *string `json:"first_id" extensions:"x-nullable"` diff --git a/contracts/agents-api/v1/subagent_items.go b/contracts/agents-api/v1/subagent_items.go index 205806d15..82a40d6a9 100644 --- a/contracts/agents-api/v1/subagent_items.go +++ b/contracts/agents-api/v1/subagent_items.go @@ -67,9 +67,41 @@ type ReasoningItem struct { Summary []SummaryText `json:"summary" binding:"required"` } -// MarshalJSON preserves each coordination variant's required fields and nulls. -// Existing execution Item variants retain their established serialization. +// MarshalJSON renders the wire shape. Messages always carry content and a +// nullable phase, and function results a nullable output and error, as in the +// pinned responses (EVT-09, SES-25). Other variants use the stored encoding. func (i Item) MarshalJSON() ([]byte, error) { + type wire Item + switch i.Type { + case "message": + var phase *string + if i.Phase != "" { + phase = &i.Phase + } + content := i.Content + if content == nil { + content = []ItemContent{} + } + return json.Marshal(struct { + wire + Phase *string `json:"phase"` + Content []ItemContent `json:"content"` + }{wire(i), phase, content}) + case "function_call_output": + return json.Marshal(struct { + wire + Output any `json:"output"` + Error any `json:"error"` + }{wire(i), i.Output, i.Error}) + } + return i.MarshalStored() +} + +// MarshalStored encodes a persisted Item payload. It keeps the encoding used +// before the wire nulls above, so stored payloads and the byte comparison of +// replayed child Items do not change. Coordination variants keep their +// required fields and nulls in both forms. +func (i Item) MarshalStored() ([]byte, error) { switch i.Type { case "create_subagent_call", "send_subagent_input_call", "agent_message": content, err := coordinationContent(i.Content) diff --git a/packages/agents-client/src/client.test.ts b/packages/agents-client/src/client.test.ts index 0abb9d7e2..e3a96fe3a 100644 --- a/packages/agents-client/src/client.test.ts +++ b/packages/agents-client/src/client.test.ts @@ -1047,6 +1047,59 @@ describe("OpenAIAgentsClient", () => { expect(onEvent.mock.calls[2]?.[0]).toMatchObject({ item_id: "item_1", item: { id: "item_1", turn_id: "turn_1" } }); }); + it("projects explicit wire nulls and accepts older Cores that omit them", async () => { + const user = { + id: "user_1", turn_id: "turn_1", type: "message", status: "completed", role: "user", + phase: null, content: [{ type: "input_text", text: "question" }], + }; + const result = { + id: "result_1", turn_id: "turn_1", type: "function_call_output", status: "completed", + call_id: "call_1", output: null, error: null, + }; + const session = { ...sessionResource(), agent: { ...agentSnapshot(), reasoning: { effort: null, summary: null } } }; + const { phase: _omitted, ...legacyUser } = user; + const events = [ + { type: "agent.session.idle", event_id: "idle", session_id: "session", session }, + { type: "agent.session.turn.item.added", event_id: "user", session_id: "session", turn_id: "turn_1", output_index: null, item: user }, + { type: "agent.session.turn.item.added", event_id: "result", session_id: "session", turn_id: "turn_1", output_index: null, item: result }, + { type: "agent.session.turn.item.added", event_id: "legacy", session_id: "session", turn_id: "turn_1", item: { ...legacyUser, id: "user_2" } }, + { + type: "agent.session.turn.item.added", event_id: "answer", session_id: "session", turn_id: "turn_1", + output_index: 0, item: messageItem({ content: [], phase: "final_answer" }), + }, + ]; + const onEvent = vi.fn(); + const client = new OpenAIAgentsClient({ + fetch: recordingFetch(streamResponse(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`)), []), + }); + + await client.streamEvents("session", { onEvent }); + + const projected = onEvent.mock.calls.map((call) => call[0] as SessionEvent); + expect(projected).toHaveLength(events.length); + expect(projected[0]?.session?.agent.reasoning).toEqual({ effort: null, summary: null }); + expect(projected[1]).toMatchObject({ output_index: null, item: { role: "user", phase: null } }); + expect(projected[2]).toMatchObject({ output_index: null, item: { output: null, error: null } }); + expect(Object.prototype.hasOwnProperty.call(projected[2]?.item, "output")).toBe(true); + expect(Object.prototype.hasOwnProperty.call(projected[3], "output_index")).toBe(false); + expect(Object.prototype.hasOwnProperty.call(projected[3]?.item, "phase")).toBe(false); + expect(projected[4]).toMatchObject({ output_index: 0, item: { status: "in_progress", content: [], phase: "final_answer" } }); + + for (const invalid of [{ ...events[1], output_index: "0" }, { ...events[1], output_index: -1 }, { ...events[1], item: { ...user, phase: "draft" } }]) { + const rejected = new OpenAIAgentsClient({ + fetch: recordingFetch(streamResponse([`data: ${JSON.stringify(invalid)}\n\n`]), []), + }); + await expect(rejected.streamEvents("session", { onEvent: vi.fn() })) + .rejects.toMatchObject({ status: 502, code: "invalid_stream_event" }); + } + + const calls: FetchCall[] = []; + const listed = await new OpenAIAgentsClient({ + fetch: recordingFetch(jsonResponse({ object: "list", data: [user, result], first_id: "user_1", last_id: "result_1", has_more: false }), calls), + }).listItems("session"); + expect(listed.data).toEqual([user, result]); + }); + const measuredUsage = { input_tokens: 7, input_tokens_details: { cached_tokens: 2 }, diff --git a/packages/agents-client/src/client.ts b/packages/agents-client/src/client.ts index f25b5f508..841acb436 100644 --- a/packages/agents-client/src/client.ts +++ b/packages/agents-client/src/client.ts @@ -1375,7 +1375,8 @@ function projectStreamEventSession( if (!sameResourceId(item.turn_id, turnId)) return invalidStreamEvent(); const itemId = value.item_id === undefined ? undefined : requiredEventString(value, "item_id"); if (itemId !== undefined && !sameResourceId(item.id, itemId)) return invalidStreamEvent(); - const outputIndex = optionalEventIndex(value, "output_index"); + // Input Items carry a null output index; older Cores omit it. + const outputIndex = value.output_index === null ? null : optionalEventIndex(value, "output_index"); return { ...base, session_id: sessionId, diff --git a/packages/agents-client/src/history-projection.ts b/packages/agents-client/src/history-projection.ts index b58961971..159e8887a 100644 --- a/packages/agents-client/src/history-projection.ts +++ b/packages/agents-client/src/history-projection.ts @@ -97,10 +97,11 @@ export function projectSessionItem(value: unknown, invalid: () => never): Sessio case "message": if ( (value.role !== "user" && value.role !== "assistant") || !Array.isArray(value.content) || - !(value.phase === undefined || value.phase === "commentary" || value.phase === "final_answer") + !(value.phase === undefined || value.phase === null || value.phase === "commentary" || value.phase === "final_answer") ) return invalid(); projected.role = value.role; projected.content = Array.from(value.content, (part) => projectItemContent(part, invalid)); + // Current Cores send null when there is no phase; older ones omit it. if (value.phase !== undefined) projected.phase = value.phase; break; case "command_execution": diff --git a/packages/agents-client/src/types.ts b/packages/agents-client/src/types.ts index 0c6677234..9ad01ee13 100644 --- a/packages/agents-client/src/types.ts +++ b/packages/agents-client/src/types.ts @@ -561,7 +561,8 @@ export interface SessionItemBase { /** Inter-agent messages have no status; reasoning may have an unknown status. */ status?: ItemStatus | null; role?: "user" | "assistant"; - phase?: "commentary" | "final_answer"; + /** Null on user messages and when the harness reports none; older Cores omit it. */ + phase?: "commentary" | "final_answer" | null; content?: ItemContent[]; command?: string; cwd?: string | null; @@ -646,7 +647,8 @@ export interface SessionEventBase { turn?: AgentTurn; item?: SessionItem; item_id?: string; - output_index?: number; + /** Null on Item events for input Items; older Cores omit it. */ + output_index?: number | null; content_index?: number; part?: ItemContent; delta?: string; diff --git a/services/agents-api/internal/api/handler_test.go b/services/agents-api/internal/api/handler_test.go index b8ae095ed..8588bca4a 100644 --- a/services/agents-api/internal/api/handler_test.go +++ b/services/agents-api/internal/api/handler_test.go @@ -90,6 +90,34 @@ func TestHTTPConfigurationAndTenantIdentity(t *testing.T) { } } +// Session responses carry both reasoning keys (SES-23); the stored +// configuration and creation retry identity keep their original encoding. +func TestSessionResponseReasoningKeysAreExplicit(t *testing.T) { + s := &recordingStore{} + h, _, _ := testHandler(t, WithExecution(&inputRecorder{ResourceStore: s})) + request := httptest.NewRequest(http.MethodPost, "/v1/agents/sessions", strings.NewReader(`{"agent":{"model":"requested-model"},"environment":{"type":"none"},"input":"hello"}`)) + request.Header.Set("Authorization", "Bearer test-api-key") + request.Header.Set("OpenAI-Beta", "agents=v1") + w := httptest.NewRecorder() + h.ServeHTTP(w, request) + var body struct { + Agent struct { + Reasoning json.RawMessage `json:"reasoning"` + } `json:"agent"` + } + if w.Code != http.StatusCreated || json.Unmarshal(w.Body.Bytes(), &body) != nil || string(body.Agent.Reasoning) != `{"effort":null,"summary":null}` { + t.Fatalf("response = %d %s", w.Code, w.Body) + } + var stored struct { + Agent struct { + Reasoning json.RawMessage `json:"reasoning"` + } `json:"agent"` + } + if json.Unmarshal(s.input.Configuration, &stored) != nil || string(stored.Agent.Reasoning) != `{}` { + t.Fatalf("stored configuration changed: %s", s.input.Configuration) + } +} + func TestHTTPRejectsUntrustedOrUnsupportedRequests(t *testing.T) { valid := `{"agent":{"model":"example"},"environment":{"type":"none"},"input":"Run the configured request."}` for _, test := range []struct { diff --git a/services/agents-api/internal/api/stream_test.go b/services/agents-api/internal/api/stream_test.go index 75b4c2020..643037c3f 100644 --- a/services/agents-api/internal/api/stream_test.go +++ b/services/agents-api/internal/api/stream_test.go @@ -184,3 +184,56 @@ func TestTerminalTurnEventsMirrorTurnUsage(t *testing.T) { } } } + +// Item events carry output_index, null for input Items, and Session snapshots +// carry both reasoning keys (EVT-09, SES-23). +func TestStreamEventsCarryExplicitNullFields(t *testing.T) { + session := store.Session{ID: "session", Configuration: json.RawMessage(`{"agent":{"id":"agent_test","model":"model","tools":[],"reasoning":{}},"environment":{"type":"none"}}`)} + text := "question" + user := &v1.Item{ID: "item", TurnID: "turn", Type: "message", Status: "completed", Role: "user", Content: []v1.ItemContent{{Type: "input_text", Text: &text}}} + result := &v1.Item{ID: "result", TurnID: "turn", Type: "function_call_output", Status: "completed", CallID: "call", Output: "value"} + index := int32(0) + for _, test := range []struct { + change store.SessionChange + want map[string]string + }{ + {store.SessionChange{Event: v1.SessionEvent{Type: "agent.session.turn.item.added", TurnID: "turn", Item: user}}, map[string]string{"output_index": "null"}}, + {store.SessionChange{Event: v1.SessionEvent{Type: "agent.session.turn.item.added", TurnID: "turn", Item: result}}, map[string]string{"output_index": "null"}}, + {store.SessionChange{Event: v1.SessionEvent{Type: "agent.session.turn.item.done", TurnID: "turn", OutputIndex: &index, Item: &v1.Item{ID: "answer", TurnID: "turn", Type: "message", Status: "completed", Role: "assistant", Content: []v1.ItemContent{{Type: "output_text", Text: &text}}}}}, map[string]string{"output_index": "0"}}, + {store.SessionChange{Event: v1.SessionEvent{Type: "agent.session.idle"}, Turn: &store.Turn{ID: "turn", Status: store.TurnCompleted}}, map[string]string{"output_index": ""}}, + } { + event, err := streamResponse(session, test.change, "") + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + for key, want := range test.want { + if value, present := fields[key]; present != (want != "") || (present && string(value) != want) { + t.Fatalf("%s: %s", key, raw) + } + } + var item map[string]json.RawMessage + _ = json.Unmarshal(fields["item"], &item) + switch { + case test.change.Event.Item == result: + if string(item["output"]) != `"value"` || string(item["error"]) != "null" { + t.Fatalf("function result fields: %s", raw) + } + case test.change.Event.Item != nil: + if phase, present := item["phase"]; !present || string(phase) != "null" { + t.Fatalf("message phase: %s", raw) + } + default: + if !strings.Contains(string(fields["session"]), `"reasoning":{"effort":null,"summary":null}`) { + t.Fatalf("session reasoning: %s", raw) + } + } + } +} diff --git a/services/agents-api/internal/store/function_item_events_test.go b/services/agents-api/internal/store/function_item_events_test.go index 307288616..4566d655a 100644 --- a/services/agents-api/internal/store/function_item_events_test.go +++ b/services/agents-api/internal/store/function_item_events_test.go @@ -62,7 +62,7 @@ func TestFunctionResultItemsRetainSubmittedFields(t *testing.T) { `{"success":false,"output":[{"type":"input_text","text":"before"},{"type":"input_image","image_url":"data:image/png;base64,AA=="}],"error":"failure"}`, } { t.Run(raw, func(t *testing.T) { - s, _ := testStore(t) + s, pool := testStore(t) tenant, session := newTurnSession(t, s) turn := submitMessage(t, s, tenant, session.ID, "start").TurnID transition(t, s, tenant, session.ID, turn, TurnQueued, TurnInProgress) @@ -87,10 +87,10 @@ func TestFunctionResultItemsRetainSubmittedFields(t *testing.T) { if json.Unmarshal([]byte(raw), &expected) != nil || json.Unmarshal(encoded, &actual) != nil { t.Fatal(string(encoded)) } + // The wire always carries both fields, null when not submitted (EVT-09). for _, field := range []string{"output", "error"} { - wanted, present := expected[field] got, exists := actual[field] - if present != exists || !reflect.DeepEqual(wanted, got) { + if !exists || !reflect.DeepEqual(expected[field], got) { t.Fatalf("%s changed: %s", field, encoded) } } @@ -116,6 +116,19 @@ func TestFunctionResultItemsRetainSubmittedFields(t *testing.T) { results++ } } + // The stored payload keeps the submitted field presence. + var stored map[string]any + if err := pool.QueryRow(t.Context(), `SELECT payload FROM session_items WHERE turn_id = $1 AND payload->>'type' = 'function_call_output'`, turn).Scan(&stored); err != nil { + t.Fatal(err) + } + var submitted map[string]any + _ = json.Unmarshal([]byte(raw), &submitted) + for _, field := range []string{"output", "error"} { + _, present := submitted[field] + if _, exists := stored[field]; present != exists { + t.Fatalf("stored %s presence changed: %v", field, stored) + } + } if results != 2 { t.Fatal("missing saved or streamed result", results) } diff --git a/services/agents-api/internal/store/item_projection.go b/services/agents-api/internal/store/item_projection.go index 27f6808bd..a366b92ca 100644 --- a/services/agents-api/internal/store/item_projection.go +++ b/services/agents-api/internal/store/item_projection.go @@ -47,7 +47,7 @@ func projectItemSource(ctx context.Context, q *sqlc.Queries, session, turn pgtyp if err := restoreFunctionItemResult(ctx, q, session, turn, &item); err != nil { return err } - payload, err := json.Marshal(item) + payload, err := item.MarshalStored() if err != nil { return err } diff --git a/services/agents-api/internal/store/subagent_coordination.go b/services/agents-api/internal/store/subagent_coordination.go index 46b8db105..d845c5a1e 100644 --- a/services/agents-api/internal/store/subagent_coordination.go +++ b/services/agents-api/internal/store/subagent_coordination.go @@ -98,7 +98,7 @@ func projectRootCoordination(ctx context.Context, q *sqlc.Queries, session, turn if err != nil { return err } - payload, err := json.Marshal(merged) + payload, err := merged.MarshalStored() if err != nil { return err } diff --git a/services/agents-api/internal/store/subagent_item_projection.go b/services/agents-api/internal/store/subagent_item_projection.go index 11e4154a9..4f70c94a8 100644 --- a/services/agents-api/internal/store/subagent_item_projection.go +++ b/services/agents-api/internal/store/subagent_item_projection.go @@ -51,7 +51,7 @@ func projectSubagentItem(ctx context.Context, q *sqlc.Queries, session pgtype.UU } func putChildItem(ctx context.Context, q *sqlc.Queries, session, childID pgtype.UUID, turn sqlc.SubagentTurn, position int32, item v1.Item) error { - payload, err := json.Marshal(item) + payload, err := item.MarshalStored() if err != nil { return err } diff --git a/services/agents-api/tests/official_agents.py b/services/agents-api/tests/official_agents.py index 0097583ca..e1ebfd890 100644 --- a/services/agents-api/tests/official_agents.py +++ b/services/agents-api/tests/official_agents.py @@ -27,6 +27,8 @@ def verify_agents(client, other, invalid, expect_error): assert body["name"] is None and body["instructions"] is None and body["metadata"] == {} assert body["multi_agent"] == {"enabled": False, "max_concurrent_subagents": None} assert body["text"] == {"format": {"type": "text"}, "verbosity": "medium"} + # Both reasoning keys are present; the model-derived effort stays unresolved. + assert body["reasoning"] == {"effort": None, "summary": None} assert body["tools"] == [] assert agent.created_at == agent.updated_at and abs(agent.created_at - time.time()) < 10 assert agents.retrieve(agent.id) == agent diff --git a/services/agents-api/tests/official_function_images.py b/services/agents-api/tests/official_function_images.py index a6fda8b49..7fc67c80a 100644 --- a/services/agents-api/tests/official_function_images.py +++ b/services/agents-api/tests/official_function_images.py @@ -111,7 +111,8 @@ def run(sid, output=None, expected=None, cancel=False, failed_text=False, valida recovered = {i["call_id"]: i for i in items(sid) if i["type"] == "function_call_output"} for call in proof["calls"]: assert recovered[call["call"]]["output"] == call["output"] - assert ("error" not in recovered[call["call"]]) == call["success"] + assert "error" in recovered[call["call"]] + assert (recovered[call["call"]]["error"] is None) == call["success"] try: diff --git a/services/agents-api/tests/official_function_stream.py b/services/agents-api/tests/official_function_stream.py index d8a32eed3..5a6a0e494 100644 --- a/services/agents-api/tests/official_function_stream.py +++ b/services/agents-api/tests/official_function_stream.py @@ -18,7 +18,8 @@ "parameters": {"type": "object", "properties": {"ticket": {"type": "string"}}, "required": ["ticket"], "additionalProperties": False}, }]} -expected = [{"output": '{"ticket":"42","status":"open"}'}, {"error": "Tool handler failed."}] +# Function result Items always carry output and error, null when not submitted. +expected = [{"output": '{"ticket":"42","status":"open"}', "error": None}, {"output": None, "error": "Tool handler failed."}] with OpenAI(base_url=base + "/v1", api_key=token, max_retries=0, _strict_response_validation=True, http_client=httpx2.Client(trust_env=False, timeout=30)) as client: sessions = client.beta.agents.sessions @@ -70,7 +71,7 @@ def lookup_ticket(arguments): calls.append(call_id) results = [event for event in added if event["item"]["type"] == "function_call_output"] assert len(results) == 1 and results[0]["turn_id"] == turn_id, events - assert results[0].get("output_index") is None + assert "output_index" in results[0] and results[0]["output_index"] is None result = results[0]["item"] assert result["call_id"] == call_id assert {key: result[key] for key in ("output", "error") if key in result} == expected[index], result diff --git a/services/agents-api/tests/official_functions.py b/services/agents-api/tests/official_functions.py index beb9ecf77..7eafd7c27 100644 --- a/services/agents-api/tests/official_functions.py +++ b/services/agents-api/tests/official_functions.py @@ -41,12 +41,13 @@ for event in stream: if event.type in ("agent.session.turn.item.added", "agent.session.turn.item.done") and event.item.type == "function_call_output": assert event.type == "agent.session.turn.item.added" and event.output_index is None + assert "output_index" in event.to_dict() submitted = event.item.to_dict() assert submitted["output"] == output if index == 1: assert submitted["error"] == "synthetic failure" else: - assert "error" not in submitted + assert "error" in submitted and submitted["error"] is None if event.type == "agent.session.requires_action" and not handled: assert event.session.agent.tools[0].to_dict() == expected action = event.session.required_actions[0] @@ -76,7 +77,7 @@ if index == 1: assert results[call]["error"] == "synthetic failure" else: - assert "error" not in results[call] + assert "error" in results[call] and results[call]["error"] is None answers = [item for item in items if item.type=="message" and item.role=="assistant"] assert len(answers)==2 and all(item.content[0].text=="FUNCTION-EXECUTION-OK" for item in answers) assert sessions.retrieve(session.id).agent.tools[0].to_dict() == expected diff --git a/services/agents-api/tests/official_hosted_functions_native.py b/services/agents-api/tests/official_hosted_functions_native.py index 3048d4025..9f57c183f 100644 --- a/services/agents-api/tests/official_hosted_functions_native.py +++ b/services/agents-api/tests/official_hosted_functions_native.py @@ -64,7 +64,7 @@ def success(arguments): return marker turn, result = invoke("Call lookup once with key success. Remember its returned string in conversation. Then use the native shell to create /workspace/outputs/function.txt containing exactly that string, with no newline. Do not call any other function.", "function-success", success) - assert result["output"] == marker and "error" not in result + assert result["output"] == marker and "error" in result and result["error"] is None artifacts = list(sessions.artifacts.list(session.id, limit=100)) output = [a for a in artifacts if a.turn_id == turn and a.path == "/workspace/outputs/function.txt"] assert len(output) == 1 From e356a7ee5f6d824ce38d9a58b04814711da8f573 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:35:59 +0000 Subject: [PATCH 02/11] Frame assistant messages with the official event sequence Official streams add an assistant message in progress with empty content, then an empty content part, deltas, output_text.done, content_part.done and item.done, and stream structured output the same way (EVT-10). Core no longer pre-fills the part in item.added. A message first observed complete, such as Claude structured output or a legacy final answer, now follows the same sequence with its unchanged text in exactly one delta; this frames the text and never alters it. Stored Items and reads are unchanged. --- .../agents-api/internal/store/item_events.go | 17 +- .../internal/store/item_serialization_test.go | 163 ++++++++++++++++++ .../agents-api/tests/official_execution.py | 17 ++ .../official_hosted_structured_native.py | 4 + .../tests/official_structured_output.py | 4 + 5 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 services/agents-api/internal/store/item_serialization_test.go diff --git a/services/agents-api/internal/store/item_events.go b/services/agents-api/internal/store/item_events.go index 0029572a0..ff2cb17fb 100644 --- a/services/agents-api/internal/store/item_events.go +++ b/services/agents-api/internal/store/item_events.go @@ -28,9 +28,10 @@ func recordItemChange(ctx context.Context, q *sqlc.Queries, session pgtype.UUID, textMessage := item.Type == "message" && item.Role == "assistant" && len(item.Content) == 1 && item.Content[0].Text != nil if previous.ID == "" { initial := item - if textMessage && delta != nil { - empty := "" - initial.Content = []v1.ItemContent{{Type: "output_text", Text: &empty}} + if textMessage { + // Assistant text is added empty and in progress; its text arrives + // only through deltas, as in official streams (EVT-10). + initial.Status, initial.Content = "in_progress", []v1.ItemContent{} } event := base event.Item = &initial @@ -38,12 +39,18 @@ func recordItemChange(ctx context.Context, q *sqlc.Queries, session pgtype.UUID, return err } if textMessage { - zero := 0 + zero, empty := 0, "" event = base - event.ItemID, event.ContentIndex, event.Part = item.ID, &zero, &initial.Content[0] + event.ItemID, event.ContentIndex, event.Part = item.ID, &zero, &v1.ItemContent{Type: item.Content[0].Type, Text: &empty} if err := emit("content_part.added", event); err != nil { return err } + if delta == nil { + // A first observation without its own fragment, such as a + // non-streamed native final, carries its unchanged text in one + // delta. This frames the text; it never alters it. + delta = item.Content[0].Text + } } } if !index.Valid { diff --git a/services/agents-api/internal/store/item_serialization_test.go b/services/agents-api/internal/store/item_serialization_test.go new file mode 100644 index 000000000..43004d8ce --- /dev/null +++ b/services/agents-api/internal/store/item_serialization_test.go @@ -0,0 +1,163 @@ +package store + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +type wireEvent struct { + kind string + fields map[string]json.RawMessage + item map[string]json.RawMessage +} + +// wireEvents renders recorded Session events as clients receive them. +func wireEvents(t *testing.T, s *Store, tenant, session string, after int64) []wireEvent { + t.Helper() + changes, err := s.ListSessionEvents(t.Context(), tenant, session, after) + if err != nil { + t.Fatal(err) + } + var result []wireEvent + for _, change := range changes { + raw, err := json.Marshal(change.Event) + if err != nil { + t.Fatal(err) + } + event := wireEvent{kind: strings.TrimPrefix(change.Event.Type, "agent.session.")} + if err := json.Unmarshal(raw, &event.fields); err != nil { + t.Fatal(err) + } + if item := event.fields["item"]; item != nil { + if err := json.Unmarshal(item, &event.item); err != nil { + t.Fatal(err) + } + } + result = append(result, event) + } + return result +} + +func TestAssistantMessageEventsFollowOfficialSequence(t *testing.T) { + // Non-ASCII text, escape sequences and surrounding whitespace stay unchanged. + answer := " {\"text\":\"red é \\u00e9\\n\"}\n" + for _, test := range []struct { + name string + events []ExecutionEvent + phase string + deltas []string + final string + }{ + {"streamed deltas", []ExecutionEvent{ + {Kind: "delta", Payload: json.RawMessage(`{"item_id":"a","delta":"Hel"}`)}, + {Kind: "delta", Payload: json.RawMessage(`{"item_id":"a","delta":"lo"}`)}, + {Kind: "output_message", Payload: json.RawMessage(`{"id":"a","status":"completed","text":"Hello"}`)}, + }, "null", []string{"Hel", "lo"}, "Hello"}, + {"native start", []ExecutionEvent{ + {Kind: "output_message", Payload: json.RawMessage(`{"id":"a","status":"in_progress","phase":"final_answer"}`)}, + {Kind: "delta", Payload: json.RawMessage(`{"item_id":"a","delta":"Hi"}`)}, + {Kind: "output_message", Payload: json.RawMessage(`{"id":"a","status":"completed","phase":"final_answer","text":"Hi"}`)}, + }, `"final_answer"`, []string{"Hi"}, "Hi"}, + // A non-streamed native final carries its exact text in one delta. + {"non-streamed final", []ExecutionEvent{ + {Kind: "output_message", Payload: json.RawMessage(`{"id":"a","status":"completed","phase":"final_answer","text":` + mustJSON(t, answer) + `}`)}, + }, `"final_answer"`, []string{answer}, answer}, + } { + t.Run(test.name, func(t *testing.T) { + s, _ := testStore(t) + tenant, session := newTurnSession(t, s) + turn := submitMessage(t, s, tenant, session.ID, "start").TurnID + transition(t, s, tenant, session.ID, turn, TurnQueued, TurnInProgress) + cursor, err := s.SessionEventCursor(t.Context(), tenant, session.ID) + if err != nil { + t.Fatal(err) + } + if err := s.AppendTurnEvents(t.Context(), tenant, session.ID, turn, 1, test.events); err != nil { + t.Fatal(err) + } + var kinds, deltas []string + var done string + for _, event := range wireEvents(t, s, tenant, session.ID, cursor) { + kinds = append(kinds, event.kind) + if string(event.fields["output_index"]) != "0" { + t.Fatalf("%s output_index = %s", event.kind, event.fields["output_index"]) + } + switch event.kind { + case "turn.item.added": + if string(event.item["status"]) != `"in_progress"` || string(event.item["content"]) != "[]" || string(event.item["phase"]) != test.phase { + t.Fatalf("item.added: %s", event.fields["item"]) + } + case "turn.content_part.added": + if string(event.fields["part"]) != `{"type":"output_text","text":""}` { + t.Fatalf("content_part.added: %s", event.fields["part"]) + } + case "turn.output_text.delta": + var delta string + _ = json.Unmarshal(event.fields["delta"], &delta) + deltas = append(deltas, delta) + case "turn.output_text.done": + _ = json.Unmarshal(event.fields["text"], &done) + case "turn.item.done": + var content []map[string]string + if json.Unmarshal(event.item["content"], &content) != nil || len(content) != 1 || content[0]["text"] != test.final || string(event.item["status"]) != `"completed"` || string(event.item["phase"]) != test.phase { + t.Fatalf("item.done: %s", event.fields["item"]) + } + } + } + want := []string{"turn.item.added", "turn.content_part.added"} + for range test.deltas { + want = append(want, "turn.output_text.delta") + } + want = append(want, "turn.output_text.done", "turn.content_part.done", "turn.item.done") + if !reflect.DeepEqual(kinds, want) || !reflect.DeepEqual(deltas, test.deltas) || done != test.final { + t.Fatalf("sequence %v deltas %q done %q", kinds, deltas, done) + } + page, err := s.ListItems(t.Context(), tenant, session.ID, "", 100, true) + if err != nil || len(page.Items) != 2 || *page.Items[1].Content[0].Text != test.final { + t.Fatalf("stored answer changed: %+v %v", page, err) + } + }) + } +} + +func TestInputItemEventsCarryNullOutputIndexAndPhase(t *testing.T) { + s, _ := testStore(t) + tenant, session := newTurnSession(t, s) + cursor, err := s.SessionEventCursor(t.Context(), tenant, session.ID) + if err != nil { + t.Fatal(err) + } + submitMessage(t, s, tenant, session.ID, "start") + added := 0 + for _, event := range wireEvents(t, s, tenant, session.ID, cursor) { + if event.kind != "turn.item.added" { + if _, present := event.fields["output_index"]; present { + t.Fatalf("%s carries output_index", event.kind) + } + continue + } + added++ + if index, present := event.fields["output_index"]; !present || string(index) != "null" || string(event.item["phase"]) != "null" || string(event.item["role"]) != `"user"` { + t.Fatalf("user item.added: %v %s", event.fields, event.fields["item"]) + } + } + page, err := s.ListItems(t.Context(), tenant, session.ID, "", 100, true) + if err != nil || added != 1 || len(page.Items) != 1 { + t.Fatal(page, added, err) + } + listed, err := json.Marshal(page.Items[0]) + if err != nil || !strings.Contains(string(listed), `"phase":null`) { + t.Fatalf("listed user message: %s %v", listed, err) + } +} + +func mustJSON(t *testing.T, value any) string { + t.Helper() + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return string(raw) +} diff --git a/services/agents-api/tests/official_execution.py b/services/agents-api/tests/official_execution.py index 6f88e109b..61f2c1963 100644 --- a/services/agents-api/tests/official_execution.py +++ b/services/agents-api/tests/official_execution.py @@ -86,6 +86,23 @@ def until_idle(stream): assert text_events[-1].text == answers[0].content[0].text deltas = [value.delta for value in text_events if value.type.endswith(".delta")] assert len(deltas) >= 2 and "".join(deltas) == answers[0].content[0].text, deltas + # Official sequence: the answer is added empty and in progress, then an + # empty part, deltas and completion (EVT-10). Input Items carry explicit + # null output_index and phase (EVT-09). + def about_answer(value): + item = getattr(value, "item", None) + return getattr(value, "item_id", None) == answers[0].id or (item is not None and item.id == answers[0].id) + answer_events = [value for value in first_events if about_answer(value)] + answer_types = [value.type.removeprefix("agent.session.turn.") for value in answer_events] + assert answer_types[:2] == ["item.added", "content_part.added"], answer_types + assert answer_types[-3:] == ["output_text.done", "content_part.done", "item.done"], answer_types + assert set(answer_types[2:-3]) == {"output_text.delta"}, answer_types + assert answer_events[0].item.status == "in_progress" and answer_events[0].item.content == [] + assert answer_events[1].part.text == "" + user_added = [value for value in first_events if value.type == "agent.session.turn.item.added" and value.item.role == "user"] + assert len(user_added) == 2 and all(value.output_index is None and "output_index" in value.to_dict() and + value.item.to_dict().get("phase", "missing") is None for value in user_added) + assert sessions.retrieve(session.id).to_dict()["agent"]["reasoning"] == {"effort": None, "summary": None} try: create(message("changed")["input"], extra_headers=creation_key) raise AssertionError("changed retry accepted") diff --git a/services/agents-api/tests/official_hosted_structured_native.py b/services/agents-api/tests/official_hosted_structured_native.py index 8d07ba5ae..e4d23fb19 100644 --- a/services/agents-api/tests/official_hosted_structured_native.py +++ b/services/agents-api/tests/official_hosted_structured_native.py @@ -90,6 +90,10 @@ def final(sid, eid, turn, expected, events=None): complete = next(i for i, e in enumerate(events) if e["type"] == "agent.session.turn.completed") assert added < done < complete assert next(e["text"] for e in events if e["type"] == "agent.session.turn.output_text.done" and e["item_id"] == answer["id"]) == raw + # The native final is framed like a streamed message; deltas carry its exact text (EVT-10). + assert events[added]["item"]["status"] == "in_progress" and events[added]["item"]["content"] == [] + deltas = [e["delta"] for e in events if e["type"] == "agent.session.turn.output_text.delta" and e["item_id"] == answer["id"]] + assert deltas and "".join(deltas) == raw, deltas artifacts = [a for a in sessions.artifacts.list(sid, limit=100) if a.path == expected["path"] and a.turn_id == turn.id] assert len(artifacts) == 1 with sessions.artifacts.with_streaming_response.content(artifacts[0].id, session_id=sid) as response: diff --git a/services/agents-api/tests/official_structured_output.py b/services/agents-api/tests/official_structured_output.py index c06498631..0f83a55ee 100644 --- a/services/agents-api/tests/official_structured_output.py +++ b/services/agents-api/tests/official_structured_output.py @@ -78,6 +78,10 @@ def final(session, events=None): done = next(i for i, e in enumerate(events) if e["type"] == "agent.session.turn.item.done" and e["item"]["id"] == item_id) terminal = next(i for i, e in enumerate(events) if e["type"] == "agent.session.turn.completed") assert added < done < terminal + # The native final is framed like a streamed message; deltas carry its exact text (EVT-10). + assert events[added]["item"]["status"] == "in_progress" and events[added]["item"]["content"] == [] + deltas = [e["delta"] for e in events if e["type"] == "agent.session.turn.output_text.delta" and e["item_id"] == item_id] + assert deltas and "".join(deltas) == text, deltas assert next(e["text"] for e in events if e["type"] == "agent.session.turn.output_text.done" and e["item_id"] == item_id) == text return answers[-1].id From 3e8460c27a9766cc702675f952365e69550ffa23 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:35:59 +0000 Subject: [PATCH 03/11] Report Session usage only when every root Turn ended measured Official Session usage stayed null after a root Turn ended with unknown usage (EVT-13), and in three more samples while a root Turn was in progress or waiting, even after earlier Turns were measured (ST-03). Session usage is now the sum of recorded root Turn usage only when every root Turn has ended with known usage, and null otherwise. Queued Turns count as not ended; no official read sampled one. A snapshot an active Turn records stays on that Turn. --- .../internal/db/queries/token_usage.sql | 8 +- .../internal/db/sqlc/token_usage.sql.go | 8 +- .../internal/store/public_execution_test.go | 13 ++- .../store/token_usage_integration_test.go | 108 ++++++++++++++++++ 4 files changed, 131 insertions(+), 6 deletions(-) diff --git a/services/agents-api/internal/db/queries/token_usage.sql b/services/agents-api/internal/db/queries/token_usage.sql index 0751d834a..1083454d6 100644 --- a/services/agents-api/internal/db/queries/token_usage.sql +++ b/services/agents-api/internal/db/queries/token_usage.sql @@ -2,7 +2,13 @@ UPDATE turns SET token_usage = $3 WHERE session_id = $1 AND id = $2; -- name: SessionTokenUsage :one -SELECT CASE WHEN count(token_usage) = 0 THEN NULL ELSE jsonb_build_object( +-- Session usage is the sum of its root Turns only when every one has ended +-- with recorded usage. Officially it stayed null while a root Turn was in +-- progress or waiting (ST-03), even after earlier Turns were measured, and after +-- a Turn ended with unknown usage (EVT-13). Queued Turns count as not ended, and +-- a snapshot recorded by an active Turn does not count yet. +SELECT CASE WHEN count(*) = 0 + OR bool_or(token_usage IS NULL OR status NOT IN ('completed', 'failed', 'cancelled')) THEN NULL ELSE jsonb_build_object( 'input_tokens', sum((token_usage->>'input_tokens')::numeric), 'input_tokens_details', jsonb_build_object('cached_tokens', sum((token_usage->'input_tokens_details'->>'cached_tokens')::numeric)), 'output_tokens', sum((token_usage->>'output_tokens')::numeric), diff --git a/services/agents-api/internal/db/sqlc/token_usage.sql.go b/services/agents-api/internal/db/sqlc/token_usage.sql.go index 9787fc434..2571f36cb 100644 --- a/services/agents-api/internal/db/sqlc/token_usage.sql.go +++ b/services/agents-api/internal/db/sqlc/token_usage.sql.go @@ -27,7 +27,8 @@ func (q *Queries) PutTurnUsage(ctx context.Context, arg PutTurnUsageParams) erro } const sessionTokenUsage = `-- name: SessionTokenUsage :one -SELECT CASE WHEN count(token_usage) = 0 THEN NULL ELSE jsonb_build_object( +SELECT CASE WHEN count(*) = 0 + OR bool_or(token_usage IS NULL OR status NOT IN ('completed', 'failed', 'cancelled')) THEN NULL ELSE jsonb_build_object( 'input_tokens', sum((token_usage->>'input_tokens')::numeric), 'input_tokens_details', jsonb_build_object('cached_tokens', sum((token_usage->'input_tokens_details'->>'cached_tokens')::numeric)), 'output_tokens', sum((token_usage->>'output_tokens')::numeric), @@ -37,6 +38,11 @@ SELECT CASE WHEN count(token_usage) = 0 THEN NULL ELSE jsonb_build_object( FROM turns WHERE session_id = $1 ` +// Session usage is the sum of its root Turns only when every one has ended +// with recorded usage. Officially it stayed null while a root Turn was in +// progress or waiting (ST-03), even after earlier Turns were measured, and after +// a Turn ended with unknown usage (EVT-13). Queued Turns count as not ended, and +// a snapshot recorded by an active Turn does not count yet. func (q *Queries) SessionTokenUsage(ctx context.Context, sessionID pgtype.UUID) ([]byte, error) { row := q.db.QueryRow(ctx, sessionTokenUsage, sessionID) var usage []byte diff --git a/services/agents-api/internal/store/public_execution_test.go b/services/agents-api/internal/store/public_execution_test.go index 14557e3e6..5f7adea85 100644 --- a/services/agents-api/internal/store/public_execution_test.go +++ b/services/agents-api/internal/store/public_execution_test.go @@ -123,7 +123,7 @@ func TestWorkerRestartReconcilesClaimedButPreservesQueuedWork(t *testing.T) { if err := h.s.AppendTurnEvents(ctx, h.tenant, h.session.ID, first.TurnID, 1, []store.ExecutionEvent{{Kind: proto.TypeUsage, Payload: usage}}); err != nil { t.Fatal(err) } - checkMeasurement := func() { + checkMeasurement := func(ended bool) { t.Helper() turn, err := h.s.GetTurn(ctx, h.tenant, h.session.ID, first.TurnID) if err != nil { @@ -135,12 +135,17 @@ func TestWorkerRestartReconcilesClaimedButPreservesQueuedWork(t *testing.T) { if json.Unmarshal(turn.Usage, &got) != nil || got.TotalTokens != 13 { t.Fatalf("lost observed usage: %s", turn.Usage) } + // Session usage stays unknown until every root Turn has ended (ST-03). + want := turn.Usage + if !ended { + want = nil + } session, err := h.s.GetSession(ctx, h.tenant, h.session.ID) - if err != nil || string(session.Usage) != string(turn.Usage) { + if err != nil || string(session.Usage) != string(want) { t.Fatalf("Session and Turn measurement differ: %+v %v", session, err) } } - checkMeasurement() + checkMeasurement(false) queued := publicSession(t, h, "queued") if _, err := h.s.SubmitMessage(ctx, h.tenant, queued.ID, "first", json.RawMessage(`{"text":"Not sent"}`)); err != nil { t.Fatal(err) @@ -176,5 +181,5 @@ func TestWorkerRestartReconcilesClaimedButPreservesQueuedWork(t *testing.T) { if err := restarted.Run(stopped); err != context.Canceled { t.Fatal(err) } - checkMeasurement() + checkMeasurement(true) } diff --git a/services/agents-api/internal/store/token_usage_integration_test.go b/services/agents-api/internal/store/token_usage_integration_test.go index af140a30a..a724f6cd6 100644 --- a/services/agents-api/internal/store/token_usage_integration_test.go +++ b/services/agents-api/internal/store/token_usage_integration_test.go @@ -132,3 +132,111 @@ func TestCancellationReceiptUsageSurvivesRecovery(t *testing.T) { t.Fatalf("recovery lost receipt usage: %s", recovered.Usage) } } + +// Official Session usage is the sum only when every root Turn has ended with +// known usage: it stays null while a Turn is queued, active or waiting (ST-03) +// and after a Turn ends with unknown usage (EVT-13). +func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { + ctx := context.Background() + s, _ := store.NewTestStore(t) + tenant := uuid.NewString() + session, err := s.CreateSession(ctx, tenant, store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: "unknown-usage"}) + if err != nil { + t.Fatal(err) + } + usage := func(input int) []store.ExecutionEvent { + return []store.ExecutionEvent{{Kind: "usage", Payload: json.RawMessage(fmt.Sprintf(`{"tokens":{"input_tokens":%d,"cached_input_tokens":4,"output_tokens":3,"reasoning_output_tokens":2,"total_tokens":%d}}`, input, input+3))}} + } + total := func(want int64) { + t.Helper() + got, err := s.GetSession(ctx, tenant, session.ID) + if err != nil { + t.Fatal(err) + } + page, err := s.ListSessions(ctx, tenant, "", 100, true, nil) + if err != nil || len(page.Sessions) != 1 || string(page.Sessions[0].Usage) != string(got.Usage) { + t.Fatalf("list usage: %+v %v", page, err) + } + if want < 0 { + if got.Usage != nil { + t.Fatalf("usage = %s, want null", got.Usage) + } + return + } + var value v1.TokenUsage + if json.Unmarshal(got.Usage, &value) != nil || value.TotalTokens != want { + t.Fatalf("usage = %s, want total %d", got.Usage, want) + } + } + submit := func(key string) store.InputReceipt { + t.Helper() + admission, err := s.SubmitMessage(ctx, tenant, session.ID, key, json.RawMessage(`{"text":"measure"}`)) + if err != nil { + t.Fatal(err) + } + return admission + } + move := func(turn, from, to string) { + t.Helper() + if _, err := s.TransitionTurn(ctx, tenant, session.ID, turn, store.TurnTransition{ExpectedStatus: from, Status: to}); err != nil { + t.Fatal(err) + } + } + finish := func(admission store.InputReceipt, status string) { + t.Helper() + if _, err := s.CompleteExecution(ctx, tenant, session.ID, admission.TurnID, status, json.RawMessage(`{"done":{}}`), "", admission.Sequence); err != nil { + t.Fatal(err) + } + } + lastIdleUsage := func() json.RawMessage { + t.Helper() + changes, err := s.ListSessionEvents(ctx, tenant, session.ID, 0) + if err != nil || len(changes) == 0 || changes[len(changes)-1].Event.Type != "agent.session.idle" { + t.Fatal(changes, err) + } + return changes[len(changes)-1].SessionUsage + } + total(-1) + first := submit("first") + total(-1) + move(first.TurnID, store.TurnQueued, store.TurnInProgress) + if err = s.AppendTurnEvents(ctx, tenant, session.ID, first.TurnID, 1, usage(10)); err != nil { + t.Fatal(err) + } + // An active Turn's recorded snapshot does not count yet. + total(-1) + finish(first, store.TurnCompleted) + total(13) + if idle := lastIdleUsage(); idle == nil { + t.Fatal("settled Session snapshot lost the known total") + } + // A queued, active or waiting Turn hides the known terminal totals. + second := submit("second") + total(-1) + move(second.TurnID, store.TurnQueued, store.TurnInProgress) + total(-1) + if err = s.AppendTurnEvents(ctx, tenant, session.ID, second.TurnID, 1, usage(20)); err != nil { + t.Fatal(err) + } + total(-1) + move(second.TurnID, store.TurnInProgress, store.TurnWaiting) + total(-1) + move(second.TurnID, store.TurnWaiting, store.TurnInProgress) + finish(second, store.TurnCancelled) + total(36) + // A Turn that ends without usage makes the total unknown for good. + third := submit("third") + move(third.TurnID, store.TurnQueued, store.TurnInProgress) + finish(third, store.TurnCancelled) + total(-1) + if idle := lastIdleUsage(); idle != nil && string(idle) != "null" { + t.Fatalf("settled Session snapshot usage: %s", idle) + } + fourth := submit("fourth") + move(fourth.TurnID, store.TurnQueued, store.TurnInProgress) + if err = s.AppendTurnEvents(ctx, tenant, session.ID, fourth.TurnID, 1, usage(30)); err != nil { + t.Fatal(err) + } + finish(fourth, store.TurnCompleted) + total(-1) +} From c82204d1d58d84b4f8b33c1405a96c7037bf778e Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:35:59 +0000 Subject: [PATCH 04/11] Keep unmeasured Codex Turn usage unknown A cancelled Codex Turn read all-zero usage where official keeps it null (EVT-24). Native reports a cumulative thread total and the adapter publishes its difference from the Turn's baseline. After an interrupt before any response reported usage, native sent a total that had not advanced, and the adapter published the all-zero difference as a complete measurement, which the cancellation outcome repeated. Core and storage only stored it. The adapter now ignores a total that has not advanced past the Turn's baseline. A later advanced total is still published, and an explicit native zero in a per-Turn usage payload is still kept. --- .../internal/agent/codex/session_usage.go | 7 ++++ .../agent/codex/session_usage_test.go | 38 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/apps/parsar-daemon/internal/agent/codex/session_usage.go b/apps/parsar-daemon/internal/agent/codex/session_usage.go index 9f9b310ff..309d05b99 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_usage.go +++ b/apps/parsar-daemon/internal/agent/codex/session_usage.go @@ -55,6 +55,13 @@ func (s *Session) onUsageUpdated(raw json.RawMessage) { } s.usageTotal = *p.TokenUsage.Total u := subtractUsage(s.usageTotal, s.usageBaseline) + if u.InputTokens == 0 && u.OutputTokens == 0 && u.TotalTokens == 0 { + // The thread total has not advanced past this Turn's baseline. It + // reports no usage for this Turn, e.g. when native sends it after an + // interrupt before any response reported usage; publishing the + // difference would turn missing usage into measured zeros. + return + } observed = &u } else if p.Usage != nil && s.usageTurnID != "" && (p.TurnID == "" || p.TurnID == s.usageTurnID) { u := *p.Usage diff --git a/apps/parsar-daemon/internal/agent/codex/session_usage_test.go b/apps/parsar-daemon/internal/agent/codex/session_usage_test.go index d0de87f6b..eb7f93c1f 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_usage_test.go +++ b/apps/parsar-daemon/internal/agent/codex/session_usage_test.go @@ -152,3 +152,41 @@ func TestAbnormalTerminationTransmitsKnownUsage(t *testing.T) { t.Fatalf("known usage missing from Done: %+v", done.Usage) } } + +// Native sends the unchanged thread total when a Turn is interrupted before any +// response reported usage (EVT-24). That measures nothing for this Turn, so the +// cancellation outcome keeps usage unknown instead of reporting zeros. +func TestUnadvancedThreadTotalIsNotTurnUsage(t *testing.T) { + previous := `{"inputTokens":13444,"cachedInputTokens":6656,"outputTokens":120,"reasoningOutputTokens":43,"totalTokens":13564}` + for name, replay := range map[string]string{ + "resumed thread": `{"threadId":"thread","turnId":"previous","tokenUsage":{"total":` + previous + `}}`, + "fresh thread": "", + } { + t.Run(name, func(t *testing.T) { + out := make(chan proto.Envelope, 8) + s := &Session{runID: "run", out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s.setThreadID("thread") + total := `{"inputTokens":0,"cachedInputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}` + if replay != "" { + s.onUsageUpdated(json.RawMessage(replay)) + total = previous + } + s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"current"}}`)) + s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"current","tokenUsage":{"total":` + total + `}}`)) + if len(out) != 0 || s.latestUsage != nil { + t.Fatalf("unadvanced total published as usage: %+v", s.latestUsage) + } + outcome := s.CancellationOutcome() + if outcome.Usage.Tokens != nil || outcome.Usage.Provider != "" { + t.Fatalf("cancellation reported unknown usage: %+v", outcome.Usage) + } + // A later advanced total is still this Turn's measurement. + advanced := `{"inputTokens":13500,"cachedInputTokens":6656,"outputTokens":130,"reasoningOutputTokens":43,"totalTokens":13630}` + s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"current","tokenUsage":{"total":` + advanced + `}}`)) + got := s.CancellationOutcome().Usage.Tokens + if got == nil || got.TotalTokens == 0 || len(out) != 1 { + t.Fatalf("advanced usage lost: %+v", got) + } + }) + } +} From 4ddc2dd06751aa2357635ddd6f18e9ce46928ee0 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:36:05 +0000 Subject: [PATCH 05/11] Record the item serialization batch in the contracts Add the dated item serialization section (S1-S8) with its official evidence (EVT-09, EVT-10, EVT-13, SES-23, SES-25, VA-11, ST-03) and the EVT-24 finding, register SR in the operation evidence with updated rows 7, 12 and 15, and update the execution tools matrix, the deferral notes, the Session usage and runtime history token descriptions and the contributor rules for wire nulls, message framing and usage. --- CONTRIBUTING.md | 26 +++-- contracts/agents-api/README.md | 5 +- contracts/agents-api/execution-tools.md | 4 +- contracts/agents-api/history-events-usage.md | 113 ++++++++++++++++++- contracts/agents-api/operation-evidence.md | 9 +- contracts/agents-api/runtime-history-api.md | 5 +- 6 files changed, 143 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1daad049e..c4e69e796 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2177,8 +2177,10 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti Core replaces complete valid token breakdowns and preserves the last committed measurement on interruption. Missing measurements remain unknown. Do not infer token consumption from context occupancy or estimated costs, or parse native - Raw payloads in Core. Session totals cover recorded root Turns; Subagent Turn - listings are not a summable accounting ledger. Native measurement coverage + Raw payloads in Core. Session totals cover recorded root Turns and are null + while any root Turn has not ended or once one ends with unknown usage; Subagent + Turn listings are not a summable accounting ledger. A cumulative native total that has not advanced + past the Turn's baseline is not a measurement of that Turn. Native measurement coverage and exact provider/model attribution remain explicit qualification boundaries. No separate public usage event or historical SSE replay is introduced. - The dispatcher is an internal entry point used by the standalone service worker. @@ -2304,17 +2306,18 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti missing or foreign Sessions keep one 404. Identical saved results remain retryable after termination without applying them again. The execution input cursor skips function results; their separate native receipts still determine application. Public result events - validate variant-specific fields and required values before admission; retain + validate variant-specific fields and required values before admission; store omitted versus null error/output and ordered text/image parts. Inline function tools resolve into the immutable configuration with explicit `defer_loading=false`. Validate required strings and parameter objects before persistence; reject unsupported deferred discovery. Omitted/null/empty tool lists resolve to no tools. The public worker selects or waits for a same-tenant device advertising `function_tools` when the Session has functions. - Function results are Session input Items: emit `item.added` without an output + Function results are Session input Items: emit `item.added` with a null output index, and never emit `item.done`, whose upstream union only allows agent output. - Project their public output/error from the saved submission, including missing - versus null fields; native content normalization must not change public history. + Project their public output/error from the saved submission; the wire always + carries both, null when not submitted, while stored payloads keep the submitted + presence. Native content normalization must not change public history. - Codex function application requires a matching live native dynamic-tool completion, including root thread/Turn/call identity, function, success and ordered content. Writing its JSON-RPC response is not application. The adapter owns pending @@ -2473,7 +2476,16 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti Completion text replaces accumulated deltas. Item merging must not mutate the incoming observation or the previous snapshot: public text delta events read the original fragment after merging, while Items retain the accumulated text. - Copy the content slice before replacing its text pointer. Keep partial output on termination; + Copy the content slice before replacing its text pointer. Assistant text + Items follow the official event sequence: `item.added` in progress with empty + content, an empty `content_part.added`, deltas, then the done events. A first + observation without its own fragment (a non-streamed native final) carries its + unchanged text in one delta; this frames the text and never alters it. Wire-only + explicit nulls (`phase`, function result `output`/`error`, Item event + `output_index`, Agent `reasoning` keys) come from response marshalling. Stored + Item payloads keep their original encoding through `Item.MarshalStored`, so + replayed child Items still compare equal, and stored configuration keeps + omitting unset reasoning keys. Keep partial output on termination; do not turn an unfinished call into a successful result. Thinking fragments are internal observations, not a claim of upstream reasoning-item support. diff --git a/contracts/agents-api/README.md b/contracts/agents-api/README.md index 5c147d0cd..9b1561eba 100644 --- a/contracts/agents-api/README.md +++ b/contracts/agents-api/README.md @@ -541,13 +541,14 @@ support `after`, `limit` (1..100, default 20), and `order` (default `desc`). The cursor is a Turn ID in the same tenant and Session. Failed turns expose a generic `internal_error`, never raw engine diagnostics. `usage` exposes the latest persisted complete token breakdown, including cached input and reasoning output. Missing measurements remain null; Session usage sums recorded -Turn measurements as best-effort usage, without estimating missing history. Session runtime state derives from the latest Turn. +Turn measurements as best-effort usage, without estimating missing history; it is +null while any root Turn has not ended or once one ends with unknown usage. Session runtime state derives from the latest Turn. ### Item recovery reads `GET /v1/agents/sessions/{session_id}/items` supports the same list controls, with a stable Item ID cursor and first-observation ordering. Messages preserve -text, phase and completion snapshots. Commands preserve reported output, exit +text, phase (null when none) and completion snapshots. Commands preserve reported output, exit code, duration and working directory. MCP calls preserve server/tool identity, arguments and structured results/errors. Dynamic functions have linked call and result Items. Native file changes appear as `apply_patch` function calls with diff --git a/contracts/agents-api/execution-tools.md b/contracts/agents-api/execution-tools.md index c88cded0e..13604208f 100644 --- a/contracts/agents-api/execution-tools.md +++ b/contracts/agents-api/execution-tools.md @@ -21,9 +21,9 @@ and Environment Plugin MCP have separate inventories and qualification. | --- | --- | --- | | Initial message input, `sessions.create` | String input and ordered user-message arrays share atomic admission. Codex/Claude text and inline image execution: M1/M2; ordinary MiniMax text: M1/P1. [Input contract](message-input.md), [initial parser](../../services/agents-api/internal/api/session_initial_input.go). | Whitespace-only text is admitted verbatim for Codex and rejected at admission for Claude SDK and MiniMax Code ([text content](message-input.md#text-content)). Empty parts beside text (SES-08) and local size-limit parity need upstream evidence. Images are only qualified for inline PNG/JPEG on `none` and Core-managed Docker `openai_hosted`; MiniMax, self-hosted images and remote URLs remain gaps. | | Prepared and active messages, `sessions.events.create` | Ordered `input_text`/`input_image` arrays retain original content and distinct public user Items. HTTP 202 confirms persistence; native receipts establish application. Initial/prepared/active Docker PNG/JPEG: M2; active PNG on `none`: M1. [Shared admission](../../services/agents-api/internal/api/inputs.go). | Codex flattens native messages with blank-line separators. Claude can fold or queue native turns; it does not promise Codex's same-native-turn behavior. Public durability does not prove native consumption. Claude SDK and MiniMax Code reject messages without an image or non-whitespace text at admission (`unsupported_or_invalid_configuration`); Codex delivers them unchanged. | -| Structured output, `agent.text.format` | Save/inherit/freeze `{type:json_schema,schema:...}`. Claude SDK object-root, single Agent, medium verbosity, ordinary functions returning text: S1 (`none`) and S2 (Docker, including prepared/active input and Files/Artifacts). Native final text is retained unchanged. [Contract](structured-output.md), [profile](../../services/agents-api/internal/engine/claude.go). | An explicit non-object root type is a protocol error for every harness ([validation](official-semantics-alignment.md#agent-configuration-validation--september-23)). Codex/MiniMax, schemas without an object root, schema numbers changed by binary64, self-hosted, Skills/Plugins, MCP, Subagent and discovery combinations reject execution. No output repair, coercion or extra model loop. Arbitrary schema dialects are unverified. | +| Structured output, `agent.text.format` | Save/inherit/freeze `{type:json_schema,schema:...}`. Claude SDK object-root, single Agent, medium verbosity, ordinary functions returning text: S1 (`none`) and S2 (Docker, including prepared/active input and Files/Artifacts). Native final text is retained unchanged; its stream follows the official message sequence with the whole text in one `output_text.delta` ([item serialization](history-events-usage.md#item-serialization-2026-09-23)). [Contract](structured-output.md), [profile](../../services/agents-api/internal/engine/claude.go). | An explicit non-object root type is a protocol error for every harness ([validation](official-semantics-alignment.md#agent-configuration-validation--september-23)). Codex/MiniMax, schemas without an object root, schema numbers changed by binary64, self-hosted, Skills/Plugins, MCP, Subagent and discovery combinations reject execution. No output repair, coercion or extra model loop. Arbitrary schema dialects are unverified. | | Function configuration, saved/inline Agents | Required name/description/schema; `defer_loading` defaults false. Protocol errors, repeated names and explicit non-object root types reject with the official fields ([validation](official-semantics-alignment.md#agent-configuration-validation--september-23)). Saved references resolve into an immutable Session snapshot. Codex/Claude real calls: F1/F2/M2/S1/S2. [Parser](../../services/agents-api/internal/api/function_configuration.go), [saved tools](../../services/agents-api/internal/api/saved_tools.go). | MiniMax public functions reject. Claude requires an explicit object root. Local nonblank/512-byte name and 64-definition Session bounds are compatibility gaps. Saving configuration alone does not qualify execution. | -| Function-result admission, `events.create` | Required `turn_id`, `call_id`, `success`; optional nullable `error` and `output`. Output is string or ordered text/image content. Scoped atomic batches retain field presence, original content and retry identity. Same result retries are accepted; changed results and results after cancellation return 409 `conflict_error`, including after terminal state. In the caller's Session, an unknown call or a call of another Turn returns 400 `invalid_request_error` without changing the pending action; missing and foreign Sessions stay 404 ([error rows](official-semantics-alignment.md#session-input-conflicts-and-result-targets--september-23)). F1/F2/M2 plus [controlled SDK/raw checks](../../services/agents-api/tests/official_function_inputs.py). [Parser](../../services/agents-api/internal/api/function_inputs.go), [Store](../../services/agents-api/internal/store/function_inputs.go). | Admission is separate from application and public Item publication. Invalid or unqualified content cannot consume a pending call. Core messages omit the call and executor IDs that official messages name; hosted defaults and publication timing remain unverified. | +| Function-result admission, `events.create` | Required `turn_id`, `call_id`, `success`; optional nullable `error` and `output`. Output is string or ordered text/image content. Scoped atomic batches retain field presence, original content and retry identity in storage; public result Items and events always carry `output` and `error`, null when not submitted ([item serialization](history-events-usage.md#item-serialization-2026-09-23)). Same result retries are accepted; changed results and results after cancellation return 409 `conflict_error`, including after terminal state. In the caller's Session, an unknown call or a call of another Turn returns 400 `invalid_request_error` without changing the pending action; missing and foreign Sessions stay 404 ([error rows](official-semantics-alignment.md#session-input-conflicts-and-result-targets--september-23)). F1/F2/M2 plus [controlled SDK/raw checks](../../services/agents-api/tests/official_function_inputs.py). [Parser](../../services/agents-api/internal/api/function_inputs.go), [Store](../../services/agents-api/internal/store/function_inputs.go). | Admission is separate from application and public Item publication. Invalid or unqualified content cannot consume a pending call. Core messages omit the call and executor IDs that official messages name; hosted defaults and publication timing remain unverified. | | Function text results and native application | Codex waits for a matching live root dynamic-tool completion; Claude waits for a matching live root native tool result. Text/error results, retry/conflict, cancellation and cold continuation: F1/F2. [Receipt contract](function-result-images.md). | Transport writes alone do not confirm application. Confirmation is not provider consumption, crash recovery or exactly-once external effects. No automatic result replay. | | Function image results | Successful ordered inline PNG/JPEG with text, large PNG and image-only JPEG: Codex/Claude `none` F1/F2; Docker M2. Public Items retain submitted bytes. Codex receipt regression: F2. | Claude rejects failed images and remote references before persistence; native resizing may change its image bytes. MiniMax functions and self-hosted image workflows are unqualified. Other Codex image/error/reference combinations cannot be inferred from successful-inline evidence. | | Deferred function discovery | Type-only `tool_search` plus mixed eager/deferred functions: Claude SDK 0.3.269/native 2.1.269, Kimi K3, single Agent, medium, `none`, text results; text and PNG input D1. Native provider observations establish lazy schema loading. [Contract](tool-search.md). | A repeated `tool_search` is a protocol error. Codex/MiniMax discovery, search-only/missing-search, workspace, MCP, structured-output and Subagent combinations remain gaps. Opaque native policy changes lack a reliable pre-input deferral signal. Saved tools include `tool_search`; the pinned Session response union excludes it. Exact hosted projection is unverified. | diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index 527cb493c..58dd64494 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -38,12 +38,17 @@ Tenant ownership is enforced by Core for both queries and streams. Adapters publish cumulative measurements for the current execution through the existing neutral Usage contract. Core replaces a Turn's snapshot atomically; repeated snapshots, including terminal repeats, do not add consumption. The -Session total sums recorded root-Turn measurements, not Subagent Turn lists. +Session total sums recorded root-Turn measurements, not Subagent Turn lists. It +is null while any root Turn has not ended and once any root Turn ends with +unknown usage ([item serialization](#item-serialization-2026-09-23)). It is best-effort accounting, not an invoice or an estimate of missing work. - Codex publishes observed snapshots while the Turn is active. Exact native resume excludes the previous Session total from the new Turn. A complete public breakdown requires valid input, cached, output, reasoning and total counters. + A thread total that has not advanced past the Turn's baseline measures nothing + for that Turn, so a Turn interrupted before any response reported usage stays + null rather than zero. - Claude retains native result evidence internally. Its per-turn and cumulative query/model counters have different scopes, and reasoning attribution can be incomplete. Complete public TokenUsage remains unqualified and null. @@ -201,11 +206,14 @@ Deferred, with evidence retained in `findings.json`: - EVT-08: official Items omit in-progress and incomplete output Items; Core keeps them under native history ownership. - EVT-09 and EVT-10: null-valued `output_index`, `phase` and `error` fields and - the initial assistant content belong to a separate serialization batch. + the initial assistant content were since aligned by the + [item serialization batch](#item-serialization-2026-09-23). - EVT-11 and EVT-12: unknown call/Turn result and conflict error codes were since aligned by the [input conflict batch](official-semantics-alignment.md#session-input-conflicts-and-result-targets--september-23). - EVT-13: official Session usage became null when any root Turn usage was - unknown; Core sums the known Turns. The in-progress case is unverified. + unknown; Core summed the known Turns. The + [item serialization batch](#item-serialization-2026-09-23) adopted the official + rule, including the non-terminal case later observed as ST-03. - EVT-19: the Core terminal sequence for a Turn cancelled mid-text is recorded by this batch's live acceptance, not changed by it. @@ -215,3 +223,102 @@ initial-input, self-hosted and initial-failure scripts, plus the TypeScript client and Core Web unit tests. Resource-level replay, live model acceptance and the full gate are recorded separately; retry, self-hosted, hosted and no-input creation stream lifetimes have no official observation. + +## Item serialization, 2026-09-23 + +Evidence: the second campaign scan's owned official `environment:none` Sessions +(private `~/.parsar/remediation/20260923/campaign-scan-2/events-tools/`, +`findings.json` EVT-09, EVT-10 and EVT-13, raw frames `official/streams-s1..s4.json` +and Items pages `official/calls-s2.json` `s2-items-after-t1`, `calls-s4.json` +`s4-items`), the first scan's SES-23 and SES-25 (`campaign-scan-1/sessions/`) and +VA-11 (`campaign-scan-1/vaults-agents/`), the fifth scan's ST-03 +(`campaign-scan-5/sessions-turns/findings.json`, raw `official/calls.json`), and +the live-kit observation EVT-24 +(`creation-stream-settlement/acceptance/candidate-evidence/codex-kimi/attempt-1/`, +`r1-events.json`, `r1-reads.json`, `daemon.log`). The plan is +`~/.parsar/remediation/20260923/item-serialization/PLAN.md`. This batch changes +field presence, event framing and the Session usage rule only; it never alters +model output text and never fills a model-derived default or counter. + +- **Null output index (S1, EVT-09).** Official `item.added` events for input Items + (user messages, function results) carried `output_index: null`. Core Item events + (`item.added`, `item.done`) now always carry `output_index`, null for input + Items; Session and Turn events keep omitting it. +- **Message phase (S2, SES-25, EVT-09).** Official message Items always carried + `phase`, null for user messages. Core messages in Items pages and events now + carry `phase`: the native phase when the adapter reports one (Codex message + observations, Claude structured output), otherwise null. Native phase values are + unchanged. +- **Function result fields (S3, EVT-09).** Official `function_call_output` Items + always carried both `output` and `error` (`error` null when not submitted; every + sampled submission had an output). Core Items and events + now always carry both, null when the submission omitted them. The stored payload + and the saved submission keep the submitted presence; the wire no longer + distinguishes an omitted field from null. The official Items page reported a + failed result's `output` as null although its event carried the submitted array; + Core keeps the submitted content in both. +- **Assistant message sequence (S4, EVT-10).** Official streams added an assistant + message in progress with `content: []`, then `content_part.added` with empty + text, deltas, `output_text.done`, `content_part.done` and `item.done`. Core no + longer pre-fills the part in `item.added`: it sends the same sequence, with the + Item in progress and empty content. The stored Item and later reads are + unchanged. Clients that append a part on `content_part.added` no longer see a + duplicate. +- **Non-streamed finals (S5, EVT-10).** Official structured output streamed like + any message. A Core message first observed complete, such as Claude structured + output or a legacy final answer, now follows the same sequence with its full + text in exactly one `output_text.delta`. The delta is the native text byte for + byte; this is event framing only. +- **Reasoning keys (S6, SES-23, VA-11).** Official Agent and Session responses + always carried `reasoning.effort` and `reasoning.summary`. Core saved Agent, + Session, Session list and Session event responses now serialize both keys, null + when unset, instead of `{}`. Official responses fill the model-derived default + effort (for example `medium`); Core does not, which remains a documented native + difference. Requests and stored configuration keep their encoding, so creation + retry identity is unchanged. +- **Session usage (S7, EVT-13, ST-03).** Official Session usage stayed null after + a root Turn ended with unknown usage, and was the exact sum when every Turn was + known (EVT-13). In three more samples it was null in every read while a root + Turn was in progress or waiting for a function result, even after earlier + Turns were measured, and returned to the sum once every Turn had settled with + known usage (ST-03: retrieve, list and the metadata update response). Core + Session usage (retrieve, list, update and Session event snapshots) is now the + sum of recorded root Turn usage only when every root Turn has ended (completed, + failed or cancelled) with known usage; otherwise it is null. A later measured + Turn does not restore the sum after an unmeasured one. The official samples did + not read a queued Turn; Core treats queued Turns like active ones, since their + consumption is not yet known. A usage snapshot that an active Codex Turn + records stays readable on that Turn but does not count in the Session until the + Turn ends. Claude and MiniMax Turns remain unmeasured, so their Sessions stay + null. Official reads also lagged settlement by seconds; Core does not copy that + timing. Runtime history token points sample this canonical Session usage, so + they now have gaps while a root Turn runs and after a Turn ends unmeasured. +- **Cancelled Codex usage (S8, EVT-24).** The all-zero counters on a cancelled + Codex Turn came from the Codex adapter, not from Core or storage. The Turn was + the second of its Session, on a resumed native thread, and was cancelled + mid-text. Native reports a cumulative thread total, and the adapter publishes + its difference from the Turn's baseline. The daemon forwarded no usage frame + for the Turn until one right after the cancel, whose total had not advanced + past the baseline. The adapter published the difference, all zeros, as a + complete measurement, the cancellation outcome repeated it, and Core stored + what it received. Native did not report zero usage for the Turn, it reported no + new usage. The adapter now ignores a total that has not advanced past the + Turn's baseline, so the Turn stays null, as official terminal usage does. A + total that later advances is still published. An explicit native zero in a + per-Turn usage payload would still be kept. + +Unchanged: the resume/cancel event sets (EVT-05/06), result publication timing +(INTERACTION-PUBLICATION-001), reconnect catch-up (EVT-07), Items list contents +(EVT-08), native phase values, Core-only extension fields and the stored +payloads. The TypeScript client accepts `output_index: null`, `phase: null` and +the explicit function result nulls, and still accepts older Cores that omit them; +Core Web accepts a null message phase. + +Batch validation used Go contract tests for the wire shapes, API tests for the +Session and stream rendering, real-PostgreSQL store tests for the event sequences, +stored-payload presence and the Session usage rule, the Codex adapter usage tests +(with `-race`), the pinned-SDK official client suite against a local server and +the TypeScript client and Web unit tests. `make openapi` adds only `x-nullable` to +Item `phase` and event `output_index`; `make sqlc-generate` changes only +`SessionTokenUsage`. The native pinned-SDK scripts updated for these shapes run +only with a native daemon, and live model acceptance is recorded separately. diff --git a/contracts/agents-api/operation-evidence.md b/contracts/agents-api/operation-evidence.md index eb19363df..2474ba43c 100644 --- a/contracts/agents-api/operation-evidence.md +++ b/contracts/agents-api/operation-evidence.md @@ -1,6 +1,6 @@ # Pinned operation evidence inventory — 2026-09-23 -Baseline inventory of main `b5715912f09333e2b4449ec6f0eecaabce44c9b7`. The Session admission batch below updates creation and metadata validation, the list query tolerance batch (L) updates list and resource query handling, the validation error batch (X) updates field error codes/params, malformed path IDs and U+0000 handling, the Environment Files wire batch (G) updates Files.create/list status, envelope, query, path and empty-page behavior, and the creation stream settlement batch (J) updates creation SSE lifetime/snapshot, terminal Turn usage and Turn start order, and the Session deletion batch (Z) updates the deletion lifecycle, and the Agent configuration validation batch (M) updates saved and inline Agent configuration errors, the whitespace input batch (P) admits whitespace-only message text, the list cursor error batch (CE) updates unresolved `after` cursor errors on every list, the input conflict batch (CF) gives every 409 type `conflict_error` and aligns Session input conflicts and tool result target errors, and the saved web_search batch (SW) saves every pinned `web_search` mode while Session admission keeps rejecting enabled search, and the workspace file write batch (FW) aligns Files.create parent creation, no-replacement and the inline size bound; historical evidence retains its original revision and scope. This inventory guides repeated qualification and does not assert complete compatibility. +Baseline inventory of main `b5715912f09333e2b4449ec6f0eecaabce44c9b7`. The Session admission batch below updates creation and metadata validation, the list query tolerance batch (L) updates list and resource query handling, the validation error batch (X) updates field error codes/params, malformed path IDs and U+0000 handling, the Environment Files wire batch (G) updates Files.create/list status, envelope, query, path and empty-page behavior, and the creation stream settlement batch (J) updates creation SSE lifetime/snapshot, terminal Turn usage and Turn start order, and the Session deletion batch (Z) updates the deletion lifecycle, and the Agent configuration validation batch (M) updates saved and inline Agent configuration errors, the whitespace input batch (P) admits whitespace-only message text, the list cursor error batch (CE) updates unresolved `after` cursor errors on every list, the input conflict batch (CF) gives every 409 type `conflict_error` and aligns Session input conflicts and tool result target errors, and the saved web_search batch (SW) saves every pinned `web_search` mode while Session admission keeps rejecting enabled search, and the workspace file write batch (FW) aligns Files.create parent creation, no-replacement and the inline size bound, and the item serialization batch (SR) aligns Item/event null fields, assistant message event framing, reasoning keys and the Session usage rule; historical evidence retains its original revision and scope. This inventory guides repeated qualification and does not assert complete compatibility. Baseline: `contracts/agents-api/upstream.json`, SDK **3.13.0**, upstream commit **d7c41efee1b0802b79f3f88a678ef2052b06e9ce**, `OpenAI-Beta: agents=v1`. AGENTS.md and relevant CONTRIBUTING.md compatibility, ownership and evidence rules govern this inventory. @@ -49,6 +49,7 @@ Repository paths below are relative to the inspected worktree; private evidence | FW | [Environment Files write semantics](environment-files.md#write-semantics--september-23-2026); private `~/.parsar/remediation/20260923/campaign-scan-2/hosted-env/findings.json` HE-11, 12, 13 and 15 with raw records under `official/` (`fc02-nested-missing-parents`, `fc03-overwrite`, `fc06-5mib-plus-1`, `fc15-onto-directory`, `fc20-overwrite-untracked`, `fc21-through-symlink-dir`, `fc22-symlink-outside`) and the saved guide file limits. Rows FW1–FW11 of that section. Rust installer and race tests, daemon writer tests (the native-helper case is opt-in and was run by hand and by live Docker acceptance), gateway/dispatch, Go handler and real-PostgreSQL HTTP/Worker tests with tenant B, TypeScript client and Web tests, without a model; live acceptance is recorded with the batch. | | CF | [Session input conflicts and result targets](official-semantics-alignment.md#session-input-conflicts-and-result-targets--september-23); private `~/.parsar/remediation/20260923/campaign-scan-4/errors/findings.json` ERR-22/27 (raw `official/results.json`: `sessB-message-while-running`, `sessA-delete-while-waiting`) and `~/.parsar/remediation/20260923/campaign-scan-2/events-tools/findings.json` EVT-11/12/14 (raw `official/calls-s2.json`: `s2-result-unknown-call`, `s2-result-unknown-turn`, `s2-result-duplicate-after-terminal`, `s2-result-changed-after-terminal`, `s2-result-after-cancel`). Rows CF1–CF9 of that section. Go API bodies, real-PostgreSQL HTTP replay with tenant B and a no-write digest, pinned-SDK scripts; live acceptance is recorded with the batch. | | SW | [Saved web_search modes](official-semantics-alignment.md#saved-web_search-modes--september-23); private `~/.parsar/remediation/20260923/campaign-scan-3/subagents-tools/findings.json` TV-05 (W01/W02) and `~/.parsar/remediation/20260923/saved-web-search/official/{results.json,ledger.jsonl}`: four owned Agents (all deleted) with create records `type-only`, `mode-null`, `mode-cached`, `mode-cached-full`, update records `update-disabled`, `update-omitted-low`, `update-live-domains-empty` and `retrieve`, plus a second probe of two Agents (deleted, 404 confirmed) with `location-partial-omitted` and `location-empty`, without a Session or model. Rows W1–W8 of that section. Go handler, real-PostgreSQL HTTP exact-bytes/no-write/tenant B, pinned-SDK, TypeScript client and Web tests; independent acceptance is recorded with the batch. | +| SR | [Item serialization](history-events-usage.md#item-serialization-2026-09-23); private `~/.parsar/remediation/20260923/campaign-scan-2/events-tools/findings.json` EVT-09, EVT-10, EVT-13 with raw frames `official/streams-s1..s4.json` and Items pages `official/calls-s2.json` `s2-items-after-t1`, `calls-s4.json` `s4-items`; `campaign-scan-1/sessions/findings.json` SES-23/25 and `campaign-scan-1/vaults-agents/findings.json` VA-11; `campaign-scan-5/sessions-turns/findings.json` ST-03 with raw `official/calls.json`; live-kit EVT-24 `creation-stream-settlement/acceptance/candidate-evidence/codex-kimi/attempt-1/r1-*.json`. Rows S1–S8 of that section. Go contract, API and real-PostgreSQL store tests, Codex adapter usage tests, the pinned-SDK official client suite and TypeScript client/Web tests; live acceptance is recorded with the batch. | ## Per-operation evidence matrix @@ -62,15 +63,15 @@ Paths in the appendix include `/v1`. SDK names here omit `client.`. `P` means pa | 4 | beta.agents.list | P: scoped cursor list; unknown keys ignored, limit 0/above 100 clamp; any unresolved cursor, malformed included, is the missing 404 | R `agent-list-empty-scoped`, `agent-list-limit101`; L VA-01/02/03/04/18; CE ERR-13 `cur-agents-random`, `cur-agents-othertype-session` | L DB `official_list_query.py` tenant A/B; CE DB cursor matrix tenant A/B | Core page capacity 100; no inferred official cap. Overflowing limits unsampled | | 5 | beta.agents.delete | P: resource deletion | R `cleanup-agent`, subsequent 404 | C DB delete/post-delete | Referenced/in-flight/repeated-delete exact parity | | 6 | beta.agents.sessions.create | P: JSON/live SSE 201, saved/inline frozen config, initial messages, native profiles; inline agent configuration errors use official fields with `agent.` params before the input requirement, and saved records with repeated tools or non-object schema roots reject admission; fresh creation SSE sends the JSON 201 projection, then ends right after the first idle recorded when a Turn ends or an input reservation stops being pending, or any failed, never sending later events; nothing admitted ends after `created`; a silent settlement ends after events up to the cursor read with a settled projection in one snapshot. A same-key stream retry returns 201, sends no events and ends at once; whitespace-only text is admitted and stored verbatim, while empty text/content/input keep the local 400 | S `create-1/2.json`, `omitted-input.json`, `null-input.json`, `empty-array-input.json`, `retry-status-original/repeat.json`; H stream; J creation streams closed after idle, open through requires_action; M TV-01..04 `SC01`–`SC11`; P SES-01..03 whitespace-only string and message input 201, verbatim Item | N Live none admission and retry; C Live three hosted profiles; D/T/K/I recorded additional workflows; J DB creation-stream lifetime/snapshot/retry; M DB inline and saved-override configuration errors without writes; P DB verbatim whitespace Items and unchanged empty-input 400 without writes | Session admission batch removes idle `none` creation; local idempotent create still differs from two official IDs. Stream retry, self-hosted, hosted and no-input stream lifetimes are local; work drained before a silent-settlement read can still be sent. Many input/tool/environment combinations restricted; harness admission limits keep the local code (TV-06). Empty-input 400s keep the local code and message; `["", text]` parts are accepted but unobserved officially (SES-08); Claude SDK and MiniMax Code reject whitespace-only messages at admission as a declared native limitation (W6) | -| 7 | beta.agents.sessions.retrieve | P: persisted state, required actions, usage; malformed ID equals missing | S `retrieve-1.json`, `session-after-1.json`; H recovered state; X SES-28 | C Live history; T pending actions; H Core acceptance recorded | Complete statuses/actions/lifecycle timing; Claude/MiniMax public usage remains null | +| 7 | beta.agents.sessions.retrieve | P: persisted state, required actions, usage; malformed ID equals missing; `agent.reasoning` carries both keys, null when unset; usage is the recorded root Turn sum only when every root Turn has ended with known usage, otherwise null | S `retrieve-1.json`, `session-after-1.json`; H recovered state; X SES-28; SR SES-23, EVT-13, ST-03 | C Live history; T pending actions; H Core acceptance recorded; SR API/DB reasoning and usage rule | Complete statuses/actions/lifecycle timing; Claude/MiniMax public usage remains null; model-derived default effort not resolved (VA-11); queued-Turn usage unobserved (treated as not ended) | | 8 | beta.agents.sessions.update | P: metadata-only replacement/clear; metadata errors use official code and `metadata`/`metadata.` param | S `metadata-replace/null/empty/omit/invalid-value.json`; `update-agent.json` uses newer unpinned field | N Live completed Session metadata rejection/clear/isolation; recorded active controlled metadata coverage | Session admission batch changes empty update to observed official 400; Session agent update belongs to baseline upgrade, not fixed-pin operation gap | | 9 | beta.agents.sessions.list | P: Agent filter, full envelope, cursor paging; unknown keys ignored, limit 0/above 100 clamp; any unresolved cursor, malformed included, is the missing 404 | S `list-filter.json`, `list-empty-after.json`, `list-owned-cross-filter-cursor.json`, limit/order/unknown-query samples; L SES-10/11/15/16/17; CE ERR-01 `cur-sessions-malformed`, ERR-13/14 | C Live order/cursors/empty/tenant checks; L DB tenant A/B; CE DB cursor matrix tenant A/B | Core page capacity 100; official cap unknown. Eventual visibility sample is not a required delay | | 10 | beta.agents.sessions.delete | P: deletion only of a durably idle or failed Session without required actions or pending input; a busy root Turn or pending reservation gives 409 `conflict_error` with no change (subagent child Turns and pending Environment file writes are not checked); owner repeat returns the same 200; owned managed cleanup, user compute retained | S cleanup files 200/deleted; retry-session active cleanup initially 409; Z SES-29 `q5-delete-repeat` 200, SES-30 `q5b-delete-while-in-progress` 409, `q5-delete-never-existed` 404, `q5-delete-while-running` 200 right after an `events.create` 202 | D recorded real cleanup; C cleanup separately recorded; Z DB matrix D1–D4 with no-write digest, admission lock race and pinned-SDK script | Core returns 409 right after an `events.create` 202 because it admits Turns synchronously (official 200); input awaiting its Environment cannot be cancelled publicly and is unobserved officially; physical purge/retention may end repeat idempotency; caller compute ownership preserved | | 11 | beta.agents.sessions.events.create | P: 202/empty body, empty-array authenticated no-op, text/cancel/function admission; whitespace-only text is admitted and stored verbatim, empty text/content/input keep the local 400; input the Session cannot accept (result after cancellation, batch while input is pending) and changed results give 409 `conflict_error`/`conflict_error`; in an owned Session an unknown call or a call of another Turn gives 400 `invalid_request_error` without writes; missing/foreign Sessions stay 404; key reuse keeps local 409 `idempotency_conflict` with type `conflict_error` | S `second-turn-create.json`, `events-empty/null.json`; H second-input; P SES-04 two whitespace-only messages 202, SES-06/07 empty text/content/input 400; CF EVT-11 `s2-result-unknown-call`/`s2-result-unknown-turn` 400, EVT-12 `s2-result-changed-after-terminal`/`s2-result-after-cancel` 409, EVT-14 duplicate 202, ERR-22 `sessB-message-while-running` 409 | C Live real continuation/no-op; T qualified message/result/cancel workflows; P DB verbatim whitespace Items and no-write empty-input rejection; CF DB rows CF2–CF4 and CF6–CF9 with tenant B, no-write digest and unchanged pending action, pinned-SDK result script | Mixed prepared-environment batches, native receipt vs durable acceptance, cancel-before-result-publication timing; unqualified content/tools; empty-input error code/message differ; `["", text]` unobserved officially (SES-08); Claude SDK and MiniMax Code reject whitespace-only messages at admission as a declared native limitation (W6); the official asynchronous pending-input window (ERR-22) is not emulated; Core messages omit call/executor IDs; official order between pending-input and target errors unobserved | -| 12 | beta.agents.sessions.events.stream | P: live-only SSE, typed persisted projections; terminal Turn events carry top-level Turn usage (null when unknown); new Turns start `turn.created`, user `item.added`, `session.in_progress`, `turn.in_progress` | H create/reconnect frames; no historical frames in sampled idle interval; J GET streams never ended, terminal `usage` present | C Live; H recorded three-harness disconnect/recovery; T pending actions; J DB order/usage | Full SSE/Item variants/order (EVT-05..10 deferred); child Turns and Items are not streamed on the Session (U, SAT-09) and are read through the Subagent routes; no replay guarantee or observer-disconnect proof for every state | +| 12 | beta.agents.sessions.events.stream | P: live-only SSE, typed persisted projections; terminal Turn events carry top-level Turn usage (null when unknown); new Turns start `turn.created`, user `item.added`, `session.in_progress`, `turn.in_progress`; Item events carry `output_index`, null for input Items; assistant text is added in progress with empty content, then an empty part, deltas (a non-streamed final in one delta) and the done events | H create/reconnect frames; no historical frames in sampled idle interval; J GET streams never ended, terminal `usage` present; SR EVT-09/10 `streams-s1..s4` | C Live; H recorded three-harness disconnect/recovery; T pending actions; J DB order/usage; SR DB sequences and API rendering | Full SSE/Item variants/order (EVT-05..08 deferred); child Turns and Items are not streamed on the Session (U, SAT-09) and are read through the Subagent routes; no replay guarantee or observer-disconnect proof for every state | | 13 | beta.agents.sessions.turns.retrieve | P: persisted root Turn identity; malformed and child Turn IDs equal missing | H/S contain Turn list payloads; no isolated positive retrieve raw request identified in this set; X SES-28 official `turn_` 404; U SAT-07 `C04` child Turn ID 404 | Recorded H/B scoped Turn recovery (B under the earlier mixed root/child contract); C history uses list; U DB child-ID 404 equal to missing, tenant B | Distinguish list-shape evidence from retrieve wire qualification; full lifecycle/usage; official 404 message text differs | | 14 | beta.agents.sessions.turns.list | P: full envelope, ordered root Turns only; a child Turn, malformed or other unresolved cursor equals a missing one; limit outside 1–100 rejects with the Beta code | S `turns-final/empty-page/limit-high/order-empty.json`; H both directions; L SES-14/15/16/17; X SES-28; U SAT-07 `R06` root-only list beside a child Turn; CE ERR-01 `cur-turns-malformed`, ERR-13 | C Live paging; H/B real child/root identity recorded under the earlier mixed contract; L DB tenant A/B; U DB root-only list and cursor; CE DB cursor matrix tenant A/B | All interleavings, same-timestamp paging, interim/failed usage | -| 15 | beta.agents.sessions.items.list | P: scoped root Items, full envelope; limit 0/above 100 clamp within the pinned 1–100 page; a cursor that is not an Item of the Session is 400 ``Invalid session item ID in `after` `` | S `items-final/empty-page/limit-high.json`; H both directions; L SES-12/13/15/16/17; CE ERR-02 `cur-items-*` | C Live; H/T recorded content/coordination/result variants; L DB tenant A/B; CE DB cursor matrix tenant A/B | Full Item union. Newer turn_id filter excluded from pin | +| 15 | beta.agents.sessions.items.list | P: scoped root Items, full envelope; limit 0/above 100 clamp within the pinned 1–100 page; a cursor that is not an Item of the Session is 400 ``Invalid session item ID in `after` ``; messages carry `phase`, null for user messages and when no native phase is reported; function results carry `output` and `error`, null when not submitted | S `items-final/empty-page/limit-high.json`; H both directions; L SES-12/13/15/16/17; CE ERR-02 `cur-items-*`; SR SES-25, EVT-09 `s2-items-after-t1`, `s4-items` | C Live; H/T recorded content/coordination/result variants; L DB tenant A/B; CE DB cursor matrix tenant A/B; SR DB/SDK null fields | Full Item union. Newer turn_id filter excluded from pin. Official pages showed a failed result's submitted array output as null; Core keeps the submitted content | | 16 | beta.agents.sessions.artifacts.retrieve | P: immutable captured metadata; unchanged outputs keep their Artifact ID across Turns | Y HE-58/59 `ar01-retrieve`, missing and `ar03-cross-session` 404 (fields match; official `artifact_` IDs vs Core UUIDs) | D recorded live Docker/user-managed workspace output; Y DB unchanged-path ID and metadata stability | Error messages differ; hard-link/special-file capture edges unobserved | | 17 | beta.agents.sessions.artifacts.list | P: scoped stored list, common list envelope; later Turns publish only new, changed or no-remaining-Artifact paths; symlinks skipped; malformed `environment_id` gives an empty page; a cursor that is not an Artifact of the Session is 400 `after is not a valid artifact ID` | Y HE-50..56 `al01`–`al09`: Turn 1 capture with a skipped link, Turn 2 republication, envelope, order/cursor/limits, other and malformed filters; CE HE-57, ERR-05 `cur-artifacts-*` | D recorded live output enumeration; Y Rust link tests, DB republication, HTTP envelope/filter/tenant and pinned-SDK checks; CE DB cursor matrix tenant A/B | Changed-bytes republication inferred; deleted-newest edge and paging during capture/delete unobserved; another Session's Artifact as a cursor is inferred | | 18 | beta.agents.sessions.artifacts.delete | P: stored deletion; the next Turn republishes a deleted path | Y HE-61 `ad01-delete`, `ad02-delete-repeat` 404, reads after delete 404; HE-52 deleted path republished by Turn 2 | D recorded workflow summary; Y DB deleted-then-unchanged and deleted-during-capture republication | In-flight deletion and physical retention parity | diff --git a/contracts/agents-api/runtime-history-api.md b/contracts/agents-api/runtime-history-api.md index 968a7b66e..985c780bd 100644 --- a/contracts/agents-api/runtime-history-api.md +++ b/contracts/agents-api/runtime-history-api.md @@ -116,7 +116,10 @@ Numeric zero is retained as an observed value. last cumulative canonical Session Usage snapshot sampled in that bucket and contains `input_tokens`, `output_tokens`, and `sampled_at`. Web derives throughput only from adjacent nondecreasing cumulative points. A missing measurement or a -counter regression produces a gap; it is never filled with zero. These counters +counter regression produces a gap; it is never filled with zero. Session Usage is +null while a root Turn has not ended and after one ends unmeasured +([item serialization](history-events-usage.md#item-serialization-2026-09-23)), so +those samples are gaps too. These counters are measured model usage, not price, cost, or billing records. Core Web queries each current managed Session through this boundary with bounded From 90e62461ffd9100838f9b03035199afc7cec908a Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:50:51 +0000 Subject: [PATCH 06/11] Keep Runtime telemetry on continuously measured Session usage Public Session usage now stays null while a root Turn runs and after an unmeasured Turn. Runtime observation telemetry, history token points and their OTLP export are Core extensions and should not lose that signal. Add a separate tenant-scoped internal query with the previous rule: the sum of every recorded root Turn snapshot, active Turns included, null only when nothing is recorded. The Runtime observation resolver reads it instead of the public value. Session retrieve, list, update and every Session event snapshot keep the public rule. --- contracts/agents-api/v1/runtime_history.go | 5 +- .../internal/db/queries/token_usage.sql | 14 ++++++ .../internal/db/sqlc/token_usage.sql.go | 27 +++++++++++ .../internal/runtimehistory/types.go | 4 +- .../internal/runtimeobs/identity.go | 6 ++- .../runtimeobs/storeresolver/resolver.go | 9 +++- .../runtimeobs/storeresolver/resolver_test.go | 46 ++++++++++++++++--- .../internal/store/runtime_history.go | 6 +-- .../agents-api/internal/store/token_usage.go | 22 +++++++++ .../store/token_usage_integration_test.go | 18 ++++++++ 10 files changed, 141 insertions(+), 16 deletions(-) diff --git a/contracts/agents-api/v1/runtime_history.go b/contracts/agents-api/v1/runtime_history.go index 49cd5ee5f..9b80df950 100644 --- a/contracts/agents-api/v1/runtime_history.go +++ b/contracts/agents-api/v1/runtime_history.go @@ -25,8 +25,9 @@ type RuntimeHistory struct { TokenUsage []RuntimeHistoryTokenUsagePoint `json:"token_usage" binding:"required" validate:"max=10000"` } -// RuntimeHistoryTokenUsagePoint is the final cumulative canonical Session -// Usage snapshot observed in one server-selected bucket. +// RuntimeHistoryTokenUsagePoint is the final cumulative measured Session usage +// observed in one server-selected bucket. It counts every recorded root Turn +// snapshot, active Turns included, unlike public Session usage. type RuntimeHistoryTokenUsagePoint struct { Start int64 `json:"start" binding:"required" minimum:"0" maximum:"9007199254740991"` End int64 `json:"end" binding:"required" minimum:"1" maximum:"9007199254740991"` diff --git a/services/agents-api/internal/db/queries/token_usage.sql b/services/agents-api/internal/db/queries/token_usage.sql index 1083454d6..1cfb621ed 100644 --- a/services/agents-api/internal/db/queries/token_usage.sql +++ b/services/agents-api/internal/db/queries/token_usage.sql @@ -16,3 +16,17 @@ SELECT CASE WHEN count(*) = 0 'total_tokens', sum((token_usage->>'total_tokens')::numeric) ) END::jsonb AS usage FROM turns WHERE session_id = $1; + +-- name: SessionMeasuredTokenUsage :one +-- Core-internal measured usage for Runtime telemetry, not public Session usage: +-- the sum of every recorded root Turn snapshot, including active Turns, null +-- only when nothing is recorded. The tenant join keeps the read scoped. +SELECT CASE WHEN count(t.token_usage) = 0 THEN NULL ELSE jsonb_build_object( + 'input_tokens', sum((t.token_usage->>'input_tokens')::numeric), + 'input_tokens_details', jsonb_build_object('cached_tokens', sum((t.token_usage->'input_tokens_details'->>'cached_tokens')::numeric)), + 'output_tokens', sum((t.token_usage->>'output_tokens')::numeric), + 'output_tokens_details', jsonb_build_object('reasoning_tokens', sum((t.token_usage->'output_tokens_details'->>'reasoning_tokens')::numeric)), + 'total_tokens', sum((t.token_usage->>'total_tokens')::numeric) +) END::jsonb AS usage +FROM sessions s JOIN turns t ON t.session_id = s.id +WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL; diff --git a/services/agents-api/internal/db/sqlc/token_usage.sql.go b/services/agents-api/internal/db/sqlc/token_usage.sql.go index 2571f36cb..52423a7e4 100644 --- a/services/agents-api/internal/db/sqlc/token_usage.sql.go +++ b/services/agents-api/internal/db/sqlc/token_usage.sql.go @@ -26,6 +26,33 @@ func (q *Queries) PutTurnUsage(ctx context.Context, arg PutTurnUsageParams) erro return err } +const sessionMeasuredTokenUsage = `-- name: SessionMeasuredTokenUsage :one +SELECT CASE WHEN count(t.token_usage) = 0 THEN NULL ELSE jsonb_build_object( + 'input_tokens', sum((t.token_usage->>'input_tokens')::numeric), + 'input_tokens_details', jsonb_build_object('cached_tokens', sum((t.token_usage->'input_tokens_details'->>'cached_tokens')::numeric)), + 'output_tokens', sum((t.token_usage->>'output_tokens')::numeric), + 'output_tokens_details', jsonb_build_object('reasoning_tokens', sum((t.token_usage->'output_tokens_details'->>'reasoning_tokens')::numeric)), + 'total_tokens', sum((t.token_usage->>'total_tokens')::numeric) +) END::jsonb AS usage +FROM sessions s JOIN turns t ON t.session_id = s.id +WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL +` + +type SessionMeasuredTokenUsageParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + ID pgtype.UUID `json:"id"` +} + +// Core-internal measured usage for Runtime telemetry, not public Session usage: +// the sum of every recorded root Turn snapshot, including active Turns, null +// only when nothing is recorded. The tenant join keeps the read scoped. +func (q *Queries) SessionMeasuredTokenUsage(ctx context.Context, arg SessionMeasuredTokenUsageParams) ([]byte, error) { + row := q.db.QueryRow(ctx, sessionMeasuredTokenUsage, arg.TenantID, arg.ID) + var usage []byte + err := row.Scan(&usage) + return usage, err +} + const sessionTokenUsage = `-- name: SessionTokenUsage :one SELECT CASE WHEN count(*) = 0 OR bool_or(token_usage IS NULL OR status NOT IN ('completed', 'failed', 'cancelled')) THEN NULL ELSE jsonb_build_object( diff --git a/services/agents-api/internal/runtimehistory/types.go b/services/agents-api/internal/runtimehistory/types.go index 224710f20..d739eb285 100644 --- a/services/agents-api/internal/runtimehistory/types.go +++ b/services/agents-api/internal/runtimehistory/types.go @@ -147,8 +147,8 @@ type Result struct { TokenUsage []TokenUsagePoint } -// TokenUsagePoint is the final cumulative canonical Session Usage snapshot in -// one bucket. Throughput is derived from ordered adjacent points by clients. +// TokenUsagePoint is the final cumulative measured Session usage, active Turns +// included, in one bucket. Throughput is derived from ordered adjacent points by clients. type TokenUsagePoint struct { Start, End time.Time SampledAt time.Time diff --git a/services/agents-api/internal/runtimeobs/identity.go b/services/agents-api/internal/runtimeobs/identity.go index 5cb149040..f317c1d14 100644 --- a/services/agents-api/internal/runtimeobs/identity.go +++ b/services/agents-api/internal/runtimeobs/identity.go @@ -28,8 +28,10 @@ type Target struct { TenantID, SessionID, EnvironmentID string Mode Mode Instance Instance - // TokenUsage is the latest canonical Core Session Usage snapshot. It is - // Session-scoped and deliberately independent of the provider Runtime. + // TokenUsage is the latest measured Core Session usage: every recorded + // root Turn snapshot, active Turns included. Unlike public Session usage it + // is not withheld until every Turn ends. It is Session-scoped and + // deliberately independent of the provider Runtime. TokenUsage *TokenUsage } diff --git a/services/agents-api/internal/runtimeobs/storeresolver/resolver.go b/services/agents-api/internal/runtimeobs/storeresolver/resolver.go index d956532ae..dd6433bf0 100644 --- a/services/agents-api/internal/runtimeobs/storeresolver/resolver.go +++ b/services/agents-api/internal/runtimeobs/storeresolver/resolver.go @@ -14,6 +14,7 @@ import ( type sessionStore interface { GetSession(context.Context, string, string) (store.Session, error) + MeasuredSessionUsage(context.Context, string, string) (json.RawMessage, error) GetRuntimeAllocation(context.Context, string, string) (store.RuntimeAllocation, error) ListRuntimeObservationSessions(context.Context, string, int) (store.RuntimeObservationSessionPage, error) } @@ -41,7 +42,13 @@ func (r *Resolver) Resolve(ctx context.Context, tenantID, sessionID string) (run return runtimeobs.Target{}, errors.New("invalid stored Runtime environment configuration") } target := runtimeobs.Target{TenantID: session.TenantID, SessionID: session.ID, Mode: runtimeobs.Mode(configuration.Environment.Type)} - usage, err := decodeTokenUsage(session.Usage) + // Telemetry counts measured usage continuously, including active Turns. + // Public Session usage stays null until every root Turn ends measured. + measured, err := r.store.MeasuredSessionUsage(ctx, session.TenantID, session.ID) + if err != nil { + return runtimeobs.Target{}, fmt.Errorf("resolve Runtime Session usage: %w", err) + } + usage, err := decodeTokenUsage(measured) if err != nil { return runtimeobs.Target{}, err } diff --git a/services/agents-api/internal/runtimeobs/storeresolver/resolver_test.go b/services/agents-api/internal/runtimeobs/storeresolver/resolver_test.go index 855749093..c04a93a76 100644 --- a/services/agents-api/internal/runtimeobs/storeresolver/resolver_test.go +++ b/services/agents-api/internal/runtimeobs/storeresolver/resolver_test.go @@ -2,6 +2,7 @@ package storeresolver import ( "context" + "encoding/json" "errors" "testing" @@ -11,6 +12,7 @@ import ( type resolverStore struct { session store.Session + measured json.RawMessage allocation store.RuntimeAllocation allocationErr error page store.RuntimeObservationSessionPage @@ -20,6 +22,13 @@ func (s resolverStore) GetSession(context.Context, string, string) (store.Sessio return s.session, nil } +func (s resolverStore) MeasuredSessionUsage(_ context.Context, tenant, session string) (json.RawMessage, error) { + if tenant != s.session.TenantID || session != s.session.ID { + return nil, errors.New("measured usage read for another Session") + } + return s.measured, nil +} + func (s resolverStore) GetRuntimeAllocation(context.Context, string, string) (store.RuntimeAllocation, error) { return s.allocation, s.allocationErr } @@ -44,7 +53,8 @@ func TestResolverListsOnlyProviderNeutralSessionIdentity(t *testing.T) { func TestResolverBindsManagedSessionEnvironmentAndAllocation(t *testing.T) { r, err := NewResolver(resolverStore{ - session: store.Session{ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"openai_hosted"}}`), Usage: []byte(`{"input_tokens":120,"input_tokens_details":{"cached_tokens":20},"output_tokens":30,"output_tokens_details":{"reasoning_tokens":10},"total_tokens":150}`), Environment: &store.Environment{ID: "environment", TenantID: "tenant", SessionID: "session"}}, + session: store.Session{ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"openai_hosted"}}`), Environment: &store.Environment{ID: "environment", TenantID: "tenant", SessionID: "session"}}, + measured: []byte(`{"input_tokens":120,"input_tokens_details":{"cached_tokens":20},"output_tokens":30,"output_tokens_details":{"reasoning_tokens":10},"total_tokens":150}`), allocation: store.RuntimeAllocation{ ID: "allocation", TenantID: "tenant", SessionID: "session", EnvironmentID: "environment", ProviderKey: "provider", DeviceID: "device", ComputePhase: "running", ComputeState: []byte(`{"current":{"name":"sandbox"}}`), @@ -67,7 +77,7 @@ func TestResolverBindsManagedSessionEnvironmentAndAllocation(t *testing.T) { t.Fatalf("compute phase was not retained: %s", target.Instance.ComputePhase) } if target.TokenUsage == nil || target.TokenUsage.InputTokens != 120 || target.TokenUsage.OutputTokens != 30 { - t.Fatalf("canonical Session usage was not retained: %+v", target.TokenUsage) + t.Fatalf("measured Session usage was not retained: %+v", target.TokenUsage) } } @@ -80,8 +90,8 @@ func TestResolverRejectsInvalidCanonicalSessionUsage(t *testing.T) { `{"input_tokens":0,"input_tokens_details":{"cached_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":0,"unknown":0}`, } { resolver, err := NewResolver(resolverStore{session: store.Session{ - ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"none"}}`), Usage: []byte(usage), - }}) + ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"none"}}`), + }, measured: []byte(usage)}) if err != nil { t.Fatal(err) } @@ -93,8 +103,8 @@ func TestResolverRejectsInvalidCanonicalSessionUsage(t *testing.T) { func TestResolverKeepsNullCanonicalSessionUsageAbsent(t *testing.T) { resolver, err := NewResolver(resolverStore{session: store.Session{ - ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"none"}}`), Usage: []byte(" \n null \t"), - }}) + ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"none"}}`), + }, measured: []byte(" \n null \t")}) if err != nil { t.Fatal(err) } @@ -104,6 +114,30 @@ func TestResolverKeepsNullCanonicalSessionUsageAbsent(t *testing.T) { } } +// Telemetry reads measured usage, not the public Session value, which stays +// null while a root Turn runs. +func TestResolverUsesMeasuredRatherThanPublicSessionUsage(t *testing.T) { + measured := `{"input_tokens":7,"input_tokens_details":{"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":10}` + for _, test := range []struct { + public, measured string + want *runtimeobs.TokenUsage + }{ + {"null", measured, &runtimeobs.TokenUsage{InputTokens: 7, OutputTokens: 3}}, + {measured, "null", nil}, + } { + resolver, err := NewResolver(resolverStore{session: store.Session{ + ID: "session", TenantID: "tenant", Configuration: []byte(`{"environment":{"type":"none"}}`), Usage: []byte(test.public), + }, measured: []byte(test.measured)}) + if err != nil { + t.Fatal(err) + } + target, err := resolver.Resolve(t.Context(), "tenant", "session") + if err != nil || (target.TokenUsage == nil) != (test.want == nil) || (test.want != nil && *target.TokenUsage != *test.want) { + t.Fatalf("usage source: %+v %v", target.TokenUsage, err) + } + } +} + func TestResolverKeepsUnsupportedModesDistinct(t *testing.T) { for _, tc := range []struct { mode string diff --git a/services/agents-api/internal/store/runtime_history.go b/services/agents-api/internal/store/runtime_history.go index 36c21fc7e..57d646b12 100644 --- a/services/agents-api/internal/store/runtime_history.go +++ b/services/agents-api/internal/store/runtime_history.go @@ -10,9 +10,9 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -// InsertRuntimeHistorySample copies a sanitized periodic observation. Canonical -// Session/Turn Usage remains the accounting authority; these counters are only -// sampled chart values. Deleted or mismatched owners cannot create orphan rows. +// InsertRuntimeHistorySample copies a sanitized periodic observation. Public +// Session/Turn Usage remains the accounting authority; these measured counters +// are only sampled chart values. Deleted or mismatched owners cannot create orphan rows. func (s *Store) InsertRuntimeHistorySample(ctx context.Context, record runtimeobs.ExportRecord) error { tenant, err := parseID(record.TenantID) if err != nil { diff --git a/services/agents-api/internal/store/token_usage.go b/services/agents-api/internal/store/token_usage.go index 8107b75b4..9c4414011 100644 --- a/services/agents-api/internal/store/token_usage.go +++ b/services/agents-api/internal/store/token_usage.go @@ -3,6 +3,7 @@ package store import ( "context" "encoding/json" + "fmt" v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" @@ -11,6 +12,27 @@ import ( "strings" ) +// MeasuredSessionUsage returns Core-internal measured usage for Runtime +// telemetry: the sum of every recorded root Turn snapshot, active Turns +// included, and null only when nothing is recorded. Public Session usage keeps +// the official rule of SessionTokenUsage. A missing Session reads as null, so +// callers resolve the Session first. +func (s *Store) MeasuredSessionUsage(ctx context.Context, tenantID, sessionID string) (json.RawMessage, error) { + tenant, err := parseID(tenantID) + if err != nil { + return nil, err + } + id, err := parseID(sessionID) + if err != nil { + return nil, err + } + usage, err := s.queries.SessionMeasuredTokenUsage(ctx, sqlc.SessionMeasuredTokenUsageParams{TenantID: tenant, ID: id}) + if err != nil { + return nil, fmt.Errorf("read measured session usage: %w", err) + } + return usage, nil +} + func projectSource(ctx context.Context, q *sqlc.Queries, session, turn pgtype.UUID, kind string, sequence int64, raw json.RawMessage, created pgtype.Timestamptz) error { if kind == proto.TypeSubagentIdentity { return projectSubagentIdentity(ctx, q, session, turn, int32(sequence), raw) diff --git a/services/agents-api/internal/store/token_usage_integration_test.go b/services/agents-api/internal/store/token_usage_integration_test.go index a724f6cd6..3304a3e8f 100644 --- a/services/agents-api/internal/store/token_usage_integration_test.go +++ b/services/agents-api/internal/store/token_usage_integration_test.go @@ -147,6 +147,19 @@ func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { usage := func(input int) []store.ExecutionEvent { return []store.ExecutionEvent{{Kind: "usage", Payload: json.RawMessage(fmt.Sprintf(`{"tokens":{"input_tokens":%d,"cached_input_tokens":4,"output_tokens":3,"reasoning_output_tokens":2,"total_tokens":%d}}`, input, input+3))}} } + // Runtime telemetry keeps counting every recorded snapshot, active Turns + // included, and is scoped to the tenant. + measured := func(want int64) { + t.Helper() + got, err := s.MeasuredSessionUsage(ctx, tenant, session.ID) + var value v1.TokenUsage + if err != nil || (want < 0) != (got == nil) || (want >= 0 && (json.Unmarshal(got, &value) != nil || value.TotalTokens != want)) { + t.Fatalf("measured usage = %s %v, want total %d", got, err, want) + } + if foreign, err := s.MeasuredSessionUsage(ctx, uuid.NewString(), session.ID); err != nil || foreign != nil { + t.Fatalf("foreign measured usage: %s %v", foreign, err) + } + } total := func(want int64) { t.Helper() got, err := s.GetSession(ctx, tenant, session.ID) @@ -197,6 +210,7 @@ func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { return changes[len(changes)-1].SessionUsage } total(-1) + measured(-1) first := submit("first") total(-1) move(first.TurnID, store.TurnQueued, store.TurnInProgress) @@ -205,6 +219,7 @@ func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { } // An active Turn's recorded snapshot does not count yet. total(-1) + measured(13) finish(first, store.TurnCompleted) total(13) if idle := lastIdleUsage(); idle == nil { @@ -221,6 +236,7 @@ func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { total(-1) move(second.TurnID, store.TurnInProgress, store.TurnWaiting) total(-1) + measured(36) move(second.TurnID, store.TurnWaiting, store.TurnInProgress) finish(second, store.TurnCancelled) total(36) @@ -229,6 +245,7 @@ func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { move(third.TurnID, store.TurnQueued, store.TurnInProgress) finish(third, store.TurnCancelled) total(-1) + measured(36) if idle := lastIdleUsage(); idle != nil && string(idle) != "null" { t.Fatalf("settled Session snapshot usage: %s", idle) } @@ -239,4 +256,5 @@ func TestSessionUsageRequiresEveryRootTurnEndedAndMeasured(t *testing.T) { } finish(fourth, store.TurnCompleted) total(-1) + measured(69) } From 2430113efa34aaea0e881c97eb70b199bedb743d Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:50:51 +0000 Subject: [PATCH 07/11] Keep live token trends continuous while Session usage is withheld The live trend reads public Session usage, which is now null while a root Turn runs. Keep a listed Session's last reported totals instead of dropping it, so the series neither falls to zero nor breaks; usage measured meanwhile appears in the interval where it is reported again. Durable history is limited to hosted Sessions, so the live trend keeps its source. --- .../features/dashboard/runtime-trends.test.ts | 21 +++++++++++++++++++ .../src/features/dashboard/runtime-trends.ts | 14 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/apps/web/src/features/dashboard/runtime-trends.test.ts b/apps/web/src/features/dashboard/runtime-trends.test.ts index 4554c7364..d844f1d76 100644 --- a/apps/web/src/features/dashboard/runtime-trends.test.ts +++ b/apps/web/src/features/dashboard/runtime-trends.test.ts @@ -235,6 +235,27 @@ describe("Runtime live-window trends", () => { expect(samples.at(-1)?.targets[0]?.cpuRatio ?? null).toBeNull(); }); + it("keeps a Session's last reported tokens while its public usage is null", () => { + const pending = (at: number) => { + const value = snapshot(at); + value.sessions[0] = { ...value.sessions[0]!, status: "in_progress", usage: null }; + return value; + }; + let samples = appendRuntimeTrendSample([], snapshot(60_000, { input: 100, output: 20 })); + samples = appendRuntimeTrendSample(samples, pending(120_000)); + expect(samples.at(-1)?.tokenTotals).toEqual([{ + sessionId: "11111111-1111-4111-8111-111111111111", sampledAt: 120_000, inputTokens: 100, outputTokens: 20, + }]); + samples = appendRuntimeTrendSample(samples, pending(180_000)); + samples = appendRuntimeTrendSample(samples, snapshot(240_000, { input: 400, output: 80 })); + expect(tokenThroughput(samples)).toEqual([ + { sampledAt: 60_000, inputPerMinute: null, outputPerMinute: null }, + { sampledAt: 120_000, inputPerMinute: 0, outputPerMinute: 0 }, + { sampledAt: 180_000, inputPerMinute: 0, outputPerMinute: 0 }, + { sampledAt: 240_000, inputPerMinute: 300, outputPerMinute: 60 }, + ]); + }); + it("keeps Session-set churn as a gap instead of publishing partial throughput", () => { const first = snapshot(60_000, { input: 100, output: 20 }); const second = snapshot(120_000, { input: 220, output: 50 }); diff --git a/apps/web/src/features/dashboard/runtime-trends.ts b/apps/web/src/features/dashboard/runtime-trends.ts index 8f9c9ce85..7179303f8 100644 --- a/apps/web/src/features/dashboard/runtime-trends.ts +++ b/apps/web/src/features/dashboard/runtime-trends.ts @@ -104,6 +104,19 @@ function tokenTotals(sessions: readonly AgentSession[], sampledAt: number): Runt }); } +// Public Session usage is null while a root Turn runs and after a Turn ends +// unmeasured. Keep a listed Session's last reported totals so its series neither +// drops to zero nor breaks; usage measured meanwhile appears once reported. +function carryTokenTotals(previous: RuntimeTrendSample, next: RuntimeTrendSample, snapshot: RuntimeDashboardSnapshot): void { + const listed = new Set(snapshot.sessions.map((session) => session.id)); + const reported = new Set(next.tokenTotals.map((total) => total.sessionId)); + for (const prior of previous.tokenTotals) { + if (listed.has(prior.sessionId) && !reported.has(prior.sessionId)) { + next.tokenTotals.push({ ...prior, sampledAt: next.sampledAt }); + } + } +} + export function runtimeTrendSample(snapshot: RuntimeDashboardSnapshot): RuntimeTrendSample { const sessions = new Map(snapshot.sessions.map((session) => [session.id, session])); const observed = snapshot.observations.flatMap((observation) => { @@ -289,6 +302,7 @@ export function appendRuntimeTrendSample( .slice(-(maximum - 1)); const previous = retained.at(-1); if (previous) { + carryTokenTotals(previous, next, snapshot); Object.assign(next, tokenRate(previous, next)); applyCPURatios(next, cpuRatios(previous, next)); } From a94ab348867fc58e82dd91dc0dc80edf4e091a08 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 18:50:51 +0000 Subject: [PATCH 08/11] Document measured telemetry usage as a Core extension Runtime history token points and telemetry count measured usage continuously and differ from public Session usage by design. Record the Core Web live trend behavior and which consumers use which rule. --- CONTRIBUTING.md | 10 ++++++---- contracts/agents-api/history-events-usage.md | 15 +++++++++++++-- contracts/agents-api/runtime-history-api.md | 11 ++++++----- .../agents-api/runtime-observability-design.md | 3 ++- contracts/agents-api/runtime-observability.md | 5 +++-- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4e69e796..d00198d2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -332,8 +332,9 @@ common field. Runtime history uses the existing Core PostgreSQL database: one sanitized row per periodic observation, seven-day retention and bounded reads. It is best-effort operational evidence, not execution or Usage authority. The execution owner samples -by default every 30 seconds. Existing canonical Session Usage supplies token -snapshots; never aggregate provider counters as model tokens. Preserve missing data +by default every 30 seconds. Core's internal measured Session usage (every +recorded root Turn snapshot, active Turns included) supplies token snapshots, not +the public Session usage rule; never aggregate provider counters as model tokens. Preserve missing data and reset CPU derivation across compute incarnations or counter regressions. The bounded asynchronous database writer and optional OTLP exporter have independent queues; external telemetry outages must not stall local history or execution. @@ -2177,8 +2178,9 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti Core replaces complete valid token breakdowns and preserves the last committed measurement on interruption. Missing measurements remain unknown. Do not infer token consumption from context occupancy or estimated costs, or parse native - Raw payloads in Core. Session totals cover recorded root Turns and are null - while any root Turn has not ended or once one ends with unknown usage; Subagent + Raw payloads in Core. Public Session totals cover recorded root Turns and are + null while any root Turn has not ended or once one ends with unknown usage; + Runtime telemetry uses the separate measured sum of recorded snapshots. Subagent Turn listings are not a summable accounting ledger. A cumulative native total that has not advanced past the Turn's baseline is not a measurement of that Turn. Native measurement coverage and exact provider/model attribution remain explicit qualification boundaries. diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index 58dd64494..6b3a93fbb 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -291,8 +291,19 @@ model output text and never fills a model-derived default or counter. records stays readable on that Turn but does not count in the Session until the Turn ends. Claude and MiniMax Turns remain unmeasured, so their Sessions stay null. Official reads also lagged settlement by seconds; Core does not copy that - timing. Runtime history token points sample this canonical Session usage, so - they now have gaps while a root Turn runs and after a Turn ends unmeasured. + timing. +- **Measured telemetry usage (Core extension).** Runtime observation telemetry, + runtime history token points and their OTLP export are Core extensions with no + official counterpart. They read a separate internal measured usage: the sum of + every recorded root Turn snapshot, active Turns included, null only when + nothing is recorded. It differs from public Session usage by design, so the + token series stays continuous while Turns run and after an unmeasured Turn. + Public Session usage (retrieve, list, update and every Session event snapshot, + including the function-action and Environment-input snapshots) keeps the rule + above. The Core Web live token trend reads public Session usage and keeps a + listed Session's last reported totals while its usage is null, so the series + neither drops to zero nor breaks; usage measured meanwhile appears in the + interval where it is reported again. - **Cancelled Codex usage (S8, EVT-24).** The all-zero counters on a cancelled Codex Turn came from the Codex adapter, not from Core or storage. The Turn was the second of its Session, on a resumed native thread, and was cancelled diff --git a/contracts/agents-api/runtime-history-api.md b/contracts/agents-api/runtime-history-api.md index 985c780bd..199a27afc 100644 --- a/contracts/agents-api/runtime-history-api.md +++ b/contracts/agents-api/runtime-history-api.md @@ -113,13 +113,14 @@ observation and contributor counts. Missing values are null and gaps remain gaps Numeric zero is retained as an observed value. `token_usage` is Session-scoped rather than allocation-scoped. Each point is the -last cumulative canonical Session Usage snapshot sampled in that bucket and +last cumulative measured Session usage sampled in that bucket and contains `input_tokens`, `output_tokens`, and `sampled_at`. Web derives throughput only from adjacent nondecreasing cumulative points. A missing measurement or a -counter regression produces a gap; it is never filled with zero. Session Usage is -null while a root Turn has not ended and after one ends unmeasured -([item serialization](history-events-usage.md#item-serialization-2026-09-23)), so -those samples are gaps too. These counters +counter regression produces a gap; it is never filled with zero. The sampled +value is Core's measured Session usage, a Core extension that sums every +recorded root Turn snapshot, active Turns included. It differs by design from +public Session usage, which is null while a root Turn runs or after one ends +unmeasured ([item serialization](history-events-usage.md#item-serialization-2026-09-23)). These counters are measured model usage, not price, cost, or billing records. Core Web queries each current managed Session through this boundary with bounded diff --git a/contracts/agents-api/runtime-observability-design.md b/contracts/agents-api/runtime-observability-design.md index d55958231..9d4239f0c 100644 --- a/contracts/agents-api/runtime-observability-design.md +++ b/contracts/agents-api/runtime-observability-design.md @@ -251,7 +251,8 @@ they are not global Prometheus labels by default. Core reuses its PostgreSQL database for bounded recent Runtime history. The provider-neutral observation service hands each sanitized periodic result to a bounded asynchronous writer. One typed row contains the observation and optional -canonical Session Usage snapshot. The public API resolves caller ownership before +measured Session usage snapshot (recorded root Turn snapshots, active Turns +included, not the public Session usage rule). The public API resolves caller ownership before issuing bounded queries; Web never queries storage directly. External OTLP export remains optional. Each destination has an independent queue, diff --git a/contracts/agents-api/runtime-observability.md b/contracts/agents-api/runtime-observability.md index 305424b7c..5a1f07ad6 100644 --- a/contracts/agents-api/runtime-observability.md +++ b/contracts/agents-api/runtime-observability.md @@ -97,8 +97,9 @@ credentials are excluded from observations. The internal `runtimehistory` boundary validates Core scope, bucket coverage, nullability, time bounds and point limits. One chart series represents an allocation; -CPU deltas reset across compute incarnations or counter regressions. Canonical -Session Usage supplies independently sampled token counters. History queries survive +CPU deltas reset across compute incarnations or counter regressions. Core's +measured Session usage (recorded root Turn snapshots, active Turns included) +supplies independently sampled token counters. History queries survive Core restart and browser reload without replaying execution. See the [design](runtime-observability-design.md), From 1a7d0a0bc5a328e00b05162fe8c5948cb7919b32 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 19:03:05 +0000 Subject: [PATCH 09/11] Hold last reported tokens in the Runtime summary total The Runtime summary's reported token total summed public Session usage, so it dropped while a Turn withheld usage. Share the live trend's hold as holdLastReported: the summary keeps each listed Session's last reported total across snapshots, a newly reported value replaces it, and unlisted Sessions drop out. Rows still show the current public value. --- .../dashboard/RuntimeObservabilityContent.tsx | 18 ++++++++- .../dashboard/dashboard-model.test.ts | 40 +++++++++++++++++++ .../src/features/dashboard/dashboard-model.ts | 15 ++++++- apps/web/src/features/dashboard/held-usage.ts | 17 ++++++++ .../src/features/dashboard/runtime-trends.ts | 17 ++++---- contracts/agents-api/history-events-usage.md | 9 +++-- 6 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 apps/web/src/features/dashboard/held-usage.ts diff --git a/apps/web/src/features/dashboard/RuntimeObservabilityContent.tsx b/apps/web/src/features/dashboard/RuntimeObservabilityContent.tsx index f52cf5c47..c0ae14c69 100644 --- a/apps/web/src/features/dashboard/RuntimeObservabilityContent.tsx +++ b/apps/web/src/features/dashboard/RuntimeObservabilityContent.tsx @@ -31,9 +31,11 @@ import { formatDashboardDuration, formatDashboardTimestamp, formatDashboardTokens, + reportedSessionTokens, runtimeObservationStatusLabel, type RuntimeDashboardRow, } from "./dashboard-model"; +import { holdLastReported } from "./held-usage"; import type { RuntimeDashboardSnapshot } from "./runtime-snapshot"; import { RuntimeTrendPanel, type RuntimeHistoryLoader } from "./RuntimeTrendPanel"; @@ -287,7 +289,21 @@ export function RuntimeObservabilityContent({ loadRuntimeHistory: RuntimeHistoryLoader; onOpenSession: (sessionId: string) => void; }) { - const model = useMemo(() => buildRuntimeDashboardModel(snapshot.sessions, snapshot.observations), [snapshot]); + // Hold each Session's last reported tokens across snapshots, so the summary + // total does not drop while a Turn withholds public Session usage. + const [heldTokens, setHeldTokens] = useState(() => ({ + snapshot, + totals: holdLastReported(new Map(), reportedSessionTokens(snapshot.sessions)), + })); + let tokenTotals = heldTokens.totals; + if (heldTokens.snapshot !== snapshot) { + tokenTotals = holdLastReported(heldTokens.totals, reportedSessionTokens(snapshot.sessions)); + setHeldTokens({ snapshot, totals: tokenTotals }); + } + const model = useMemo( + () => buildRuntimeDashboardModel(snapshot.sessions, snapshot.observations, tokenTotals), + [snapshot, tokenTotals], + ); const summary = model.summary; return ( <> diff --git a/apps/web/src/features/dashboard/dashboard-model.test.ts b/apps/web/src/features/dashboard/dashboard-model.test.ts index 03a167088..384d8619b 100644 --- a/apps/web/src/features/dashboard/dashboard-model.test.ts +++ b/apps/web/src/features/dashboard/dashboard-model.test.ts @@ -11,8 +11,10 @@ import { formatDashboardTimestamp, formatDashboardBytes, formatDashboardDuration, + reportedSessionTokens, runtimeObservationStatusLabel, } from "./dashboard-model"; +import { holdLastReported } from "./held-usage"; function agent(id: string, overrides: Partial = {}): SavedAgent { return { @@ -295,6 +297,44 @@ describe("Dashboard loaded-snapshot model", () => { expect(formatDashboardDuration(90)).toBe("1m 30s"); }); + it("holds each Session's last reported tokens in the summary while public usage is null", () => { + const running = session("11111111-1111-4111-8111-111111111111", { usage: usage(21) }); + const other = session("22222222-2222-4222-8222-222222222222", { usage: usage(5) }); + const observation = (id: string): RuntimeObservation => ({ + id, + object: "agent.runtime_observation", + session_id: id, + environment_id: null, + mode: "none", + provider_type: null, + instance: { kind: "none", allocation_id: null, device_id: null, connection_generation: null }, + status: "unsupported", + reason: "runtime_mode_not_observable", + allocation_created_at: null, + resolved_at: 225, + observed_at: null, + started_at: null, + cpu: null, + memory: null, + }); + const observations = [observation(running.id), observation(other.id)]; + let held = holdLastReported(new Map(), reportedSessionTokens([running, other])); + expect(buildRuntimeDashboardModel([running, other], observations, held).summary) + .toMatchObject({ totalTokens: 26, tokenCoverageCount: 2 }); + + // A Turn starts: public usage is withheld, the summary keeps the last total. + const withheld = { ...running, status: "in_progress", usage: null } as AgentSession; + held = holdLastReported(held, reportedSessionTokens([withheld, { ...other, usage: usage(9) }])); + const model = buildRuntimeDashboardModel([withheld, { ...other, usage: usage(9) }], observations, held); + expect(model.summary).toMatchObject({ totalTokens: 30, tokenCoverageCount: 2 }); + expect(model.rows.find((row) => row.session.id === withheld.id)?.session.totalTokens).toBeNull(); + expect(buildRuntimeDashboardModel([withheld, other], observations).summary.totalTokens).toBe(5); + + // A newly reported value replaces the held one; unlisted Sessions are dropped. + held = holdLastReported(held, reportedSessionTokens([{ ...running, usage: usage(40) }])); + expect([...held]).toEqual([[running.id, 40]]); + }); + it("does not infer a released allocation lifetime from the current resolution time", () => { const stopped = session("11111111-1111-4111-8111-111111111111"); const observation: RuntimeObservation = { diff --git a/apps/web/src/features/dashboard/dashboard-model.ts b/apps/web/src/features/dashboard/dashboard-model.ts index 5c31b55f1..16da4aacd 100644 --- a/apps/web/src/features/dashboard/dashboard-model.ts +++ b/apps/web/src/features/dashboard/dashboard-model.ts @@ -275,9 +275,19 @@ function elapsedSeconds(start: number | null, end: number | null): number | null return end - start; } +/** Each Session's reported total tokens, null when its public usage is null. */ +export function reportedSessionTokens(sessions: readonly AgentSession[]): Map { + return new Map(sessions.map((session) => [session.id, canonicalUsage(session.usage)?.total_tokens ?? null])); +} + +/** + * Rows show current public usage. The summary token total adds each Session's + * held last reported total (see holdLastReported) while its usage is null. + */ export function buildRuntimeDashboardModel( sessions: readonly AgentSession[], observations: readonly RuntimeObservation[], + heldTokens: ReadonlyMap = new Map(), ): RuntimeDashboardModel { const sessionsById = new Map(sessions.map((session) => [session.id, session])); const rows: RuntimeDashboardRow[] = []; @@ -359,8 +369,9 @@ export function buildRuntimeDashboardModel( unsupportedRuntimeCount += 1; } - if (sessionRow.totalTokens !== null) { - const next = safeAdd(totalTokens, sessionRow.totalTokens); + const sessionTokens = sessionRow.totalTokens ?? heldTokens.get(session.id) ?? null; + if (sessionTokens !== null) { + const next = safeAdd(totalTokens, sessionTokens); if (next !== null) { totalTokens = next; tokensKnown = true; diff --git a/apps/web/src/features/dashboard/held-usage.ts b/apps/web/src/features/dashboard/held-usage.ts new file mode 100644 index 000000000..b250c9da0 --- /dev/null +++ b/apps/web/src/features/dashboard/held-usage.ts @@ -0,0 +1,17 @@ +/** + * Public Session usage is null while a root Turn runs and after a Turn ends + * unmeasured. Dashboard token totals keep each listed Session's last reported + * value instead, so they neither drop to zero nor break. A newly reported value + * replaces the held one, and Sessions no longer listed are dropped. + */ +export function holdLastReported( + previous: ReadonlyMap, + current: ReadonlyMap, +): Map { + const held = new Map(); + for (const [sessionId, value] of current) { + const kept = value ?? previous.get(sessionId); + if (kept !== undefined) held.set(sessionId, kept); + } + return held; +} diff --git a/apps/web/src/features/dashboard/runtime-trends.ts b/apps/web/src/features/dashboard/runtime-trends.ts index 7179303f8..cd276d62c 100644 --- a/apps/web/src/features/dashboard/runtime-trends.ts +++ b/apps/web/src/features/dashboard/runtime-trends.ts @@ -1,5 +1,6 @@ import type { AgentSession, RuntimeObservation } from "@agents-core-web/agents-client"; +import { holdLastReported } from "./held-usage"; import type { RuntimeDashboardSnapshot } from "./runtime-snapshot"; export const RUNTIME_TREND_WINDOW_MS = 60 * 60 * 1_000; @@ -104,17 +105,13 @@ function tokenTotals(sessions: readonly AgentSession[], sampledAt: number): Runt }); } -// Public Session usage is null while a root Turn runs and after a Turn ends -// unmeasured. Keep a listed Session's last reported totals so its series neither -// drops to zero nor breaks; usage measured meanwhile appears once reported. +// Keep a listed Session's last reported totals while its public usage is null; +// usage measured meanwhile appears in the interval where it is reported again. function carryTokenTotals(previous: RuntimeTrendSample, next: RuntimeTrendSample, snapshot: RuntimeDashboardSnapshot): void { - const listed = new Set(snapshot.sessions.map((session) => session.id)); - const reported = new Set(next.tokenTotals.map((total) => total.sessionId)); - for (const prior of previous.tokenTotals) { - if (listed.has(prior.sessionId) && !reported.has(prior.sessionId)) { - next.tokenTotals.push({ ...prior, sampledAt: next.sampledAt }); - } - } + const reported = new Map(snapshot.sessions.map((session) => [session.id, null])); + for (const total of next.tokenTotals) reported.set(total.sessionId, total); + const prior = new Map(previous.tokenTotals.map((total) => [total.sessionId, { ...total, sampledAt: next.sampledAt }])); + next.tokenTotals = [...holdLastReported(prior, reported).values()]; } export function runtimeTrendSample(snapshot: RuntimeDashboardSnapshot): RuntimeTrendSample { diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index 6b3a93fbb..d98b85dd6 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -300,10 +300,11 @@ model output text and never fills a model-derived default or counter. token series stays continuous while Turns run and after an unmeasured Turn. Public Session usage (retrieve, list, update and every Session event snapshot, including the function-action and Environment-input snapshots) keeps the rule - above. The Core Web live token trend reads public Session usage and keeps a - listed Session's last reported totals while its usage is null, so the series - neither drops to zero nor breaks; usage measured meanwhile appears in the - interval where it is reported again. + above. The Core Web live token trend and the Runtime summary's reported token + total read public Session usage and keep a listed Session's last reported + totals while its usage is null, so they neither drop to zero nor break; usage + measured meanwhile appears once it is reported again. Rows still show the + current public value. - **Cancelled Codex usage (S8, EVT-24).** The all-zero counters on a cancelled Codex Turn came from the Codex adapter, not from Core or storage. The Turn was the second of its Session, on a resumed native thread, and was cancelled From d10e5b7404f3d820af1ce7816a590453b7ddb8b3 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 19:27:25 +0000 Subject: [PATCH 10/11] Leave live token rates as gaps while Session usage is held The live trend re-stamped a held Session total with each sample time, so a running Turn read as 0/min and its whole consumption landed as one spike. Held totals now keep their report time and are marked held; tokenRate treats them as unknown, so those intervals are gaps, never zero. The next reported total is spread over the time since the last report. The Runtime summary keeps its held total. --- .../features/dashboard/runtime-trends.test.ts | 14 +++++++++----- .../web/src/features/dashboard/runtime-trends.ts | 16 +++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/apps/web/src/features/dashboard/runtime-trends.test.ts b/apps/web/src/features/dashboard/runtime-trends.test.ts index d844f1d76..40af9e83a 100644 --- a/apps/web/src/features/dashboard/runtime-trends.test.ts +++ b/apps/web/src/features/dashboard/runtime-trends.test.ts @@ -235,7 +235,7 @@ describe("Runtime live-window trends", () => { expect(samples.at(-1)?.targets[0]?.cpuRatio ?? null).toBeNull(); }); - it("keeps a Session's last reported tokens while its public usage is null", () => { + it("leaves a gap while a Session's public usage is null and spreads the next report", () => { const pending = (at: number) => { const value = snapshot(at); value.sessions[0] = { ...value.sessions[0]!, status: "in_progress", usage: null }; @@ -243,16 +243,20 @@ describe("Runtime live-window trends", () => { }; let samples = appendRuntimeTrendSample([], snapshot(60_000, { input: 100, output: 20 })); samples = appendRuntimeTrendSample(samples, pending(120_000)); + // The held total keeps its report time and is never a zero-rate measurement. expect(samples.at(-1)?.tokenTotals).toEqual([{ - sessionId: "11111111-1111-4111-8111-111111111111", sampledAt: 120_000, inputTokens: 100, outputTokens: 20, + sessionId: "11111111-1111-4111-8111-111111111111", sampledAt: 60_000, inputTokens: 100, outputTokens: 20, held: true, }]); samples = appendRuntimeTrendSample(samples, pending(180_000)); samples = appendRuntimeTrendSample(samples, snapshot(240_000, { input: 400, output: 80 })); + samples = appendRuntimeTrendSample(samples, snapshot(300_000, { input: 460, output: 90 })); expect(tokenThroughput(samples)).toEqual([ { sampledAt: 60_000, inputPerMinute: null, outputPerMinute: null }, - { sampledAt: 120_000, inputPerMinute: 0, outputPerMinute: 0 }, - { sampledAt: 180_000, inputPerMinute: 0, outputPerMinute: 0 }, - { sampledAt: 240_000, inputPerMinute: 300, outputPerMinute: 60 }, + { sampledAt: 120_000, inputPerMinute: null, outputPerMinute: null }, + { sampledAt: 180_000, inputPerMinute: null, outputPerMinute: null }, + // 300 input and 60 output tokens over the three minutes since the last report. + { sampledAt: 240_000, inputPerMinute: 100, outputPerMinute: 20 }, + { sampledAt: 300_000, inputPerMinute: 60, outputPerMinute: 10 }, ]); }); diff --git a/apps/web/src/features/dashboard/runtime-trends.ts b/apps/web/src/features/dashboard/runtime-trends.ts index cd276d62c..433151439 100644 --- a/apps/web/src/features/dashboard/runtime-trends.ts +++ b/apps/web/src/features/dashboard/runtime-trends.ts @@ -45,6 +45,8 @@ export interface RuntimeTrendTokenTotal { sampledAt: number; inputTokens: number; outputTokens: number; + /** Kept from the report at sampledAt while public Session usage is null. */ + held?: true; } export interface TokenThroughputSample { @@ -105,12 +107,14 @@ function tokenTotals(sessions: readonly AgentSession[], sampledAt: number): Runt }); } -// Keep a listed Session's last reported totals while its public usage is null; -// usage measured meanwhile appears in the interval where it is reported again. +// Keep a listed Session's last reported totals, with their report time, while +// its public usage is null. tokenRate treats a held total as unknown, so those +// intervals are gaps; the next reported total is spread over the time since the +// last report. function carryTokenTotals(previous: RuntimeTrendSample, next: RuntimeTrendSample, snapshot: RuntimeDashboardSnapshot): void { const reported = new Map(snapshot.sessions.map((session) => [session.id, null])); for (const total of next.tokenTotals) reported.set(total.sessionId, total); - const prior = new Map(previous.tokenTotals.map((total) => [total.sessionId, { ...total, sampledAt: next.sampledAt }])); + const prior = new Map(previous.tokenTotals.map((total) => [total.sessionId, { ...total, held: true as const }])); next.tokenTotals = [...holdLastReported(prior, reported).values()]; } @@ -254,15 +258,17 @@ function tokenRate( const prior = previousTotals.get(current.sessionId); return prior ? [[prior, current]] : []; }); + // A total held from an earlier report is not a measurement at this sample. + const measured = (current: RuntimeTrendTokenTotal) => !current.held; const inputRates = pairs.map(([prior, current]) => { const elapsedMinutes = (current.sampledAt - prior.sampledAt) / 60_000; const delta = current.inputTokens - prior.inputTokens; - return elapsedMinutes > 0 && delta >= 0 ? delta / elapsedMinutes : null; + return measured(current) && elapsedMinutes > 0 && delta >= 0 ? delta / elapsedMinutes : null; }); const outputRates = pairs.map(([prior, current]) => { const elapsedMinutes = (current.sampledAt - prior.sampledAt) / 60_000; const delta = current.outputTokens - prior.outputTokens; - return elapsedMinutes > 0 && delta >= 0 ? delta / elapsedMinutes : null; + return measured(current) && elapsedMinutes > 0 && delta >= 0 ? delta / elapsedMinutes : null; }); const inputRate = inputRates.length > 0 && inputRates.every((rate) => rate !== null) ? inputRates.reduce((total, rate) => total + (rate ?? 0), 0) From 3a7e3d08aee09c3509e581782bba3808142e8a60 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 19:27:25 +0000 Subject: [PATCH 11/11] Correct measured usage and live trend documentation The resolver and the OTLP token gauges read MeasuredSessionUsage, sqlc also adds SessionMeasuredTokenUsage, and held live-trend intervals are gaps. Note that a root Turn cancelled while queued leaves public Session usage null, an inference with no official sample in ST-03. --- contracts/agents-api/history-events-usage.md | 20 +++++++++++-------- .../runtime-observability-design.md | 16 +++++++++------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index d98b85dd6..53cf9a24f 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -287,7 +287,10 @@ model output text and never fills a model-derived default or counter. failed or cancelled) with known usage; otherwise it is null. A later measured Turn does not restore the sum after an unmeasured one. The official samples did not read a queued Turn; Core treats queued Turns like active ones, since their - consumption is not yet known. A usage snapshot that an active Codex Turn + consumption is not yet known. A root Turn cancelled while still queued ends + without usage, so public Session usage stays null afterwards. That follows + from the terminal rule; ST-03 has no official sample of the case, so it is an + inference. A usage snapshot that an active Codex Turn records stays readable on that Turn but does not count in the Session until the Turn ends. Claude and MiniMax Turns remain unmeasured, so their Sessions stay null. Official reads also lagged settlement by seconds; Core does not copy that @@ -300,11 +303,12 @@ model output text and never fills a model-derived default or counter. token series stays continuous while Turns run and after an unmeasured Turn. Public Session usage (retrieve, list, update and every Session event snapshot, including the function-action and Environment-input snapshots) keeps the rule - above. The Core Web live token trend and the Runtime summary's reported token - total read public Session usage and keep a listed Session's last reported - totals while its usage is null, so they neither drop to zero nor break; usage - measured meanwhile appears once it is reported again. Rows still show the - current public value. + above. Core Web reads public Session usage. Its Runtime summary's reported + token total keeps a listed Session's last reported total while the usage is + null, so it does not drop; rows still show the current public value. Its live + token trend keeps that Session in the series but treats the held total as + unknown: those intervals are gaps, never zero, and the rate once usage is + reported again is spread over the time since its last report. - **Cancelled Codex usage (S8, EVT-24).** The all-zero counters on a cancelled Codex Turn came from the Codex adapter, not from Core or storage. The Turn was the second of its Session, on a resumed native thread, and was cancelled @@ -331,6 +335,6 @@ Session and stream rendering, real-PostgreSQL store tests for the event sequence stored-payload presence and the Session usage rule, the Codex adapter usage tests (with `-race`), the pinned-SDK official client suite against a local server and the TypeScript client and Web unit tests. `make openapi` adds only `x-nullable` to -Item `phase` and event `output_index`; `make sqlc-generate` changes only -`SessionTokenUsage`. The native pinned-SDK scripts updated for these shapes run +Item `phase` and event `output_index`; `make sqlc-generate` changes +`SessionTokenUsage` and adds the internal `SessionMeasuredTokenUsage`. The native pinned-SDK scripts updated for these shapes run only with a native daemon, and live model acceptance is recorded separately. diff --git a/contracts/agents-api/runtime-observability-design.md b/contracts/agents-api/runtime-observability-design.md index 9d4239f0c..228162154 100644 --- a/contracts/agents-api/runtime-observability-design.md +++ b/contracts/agents-api/runtime-observability-design.md @@ -294,8 +294,8 @@ The OTLP request uses standard protobuf metrics and these instruments: | `agents.runtime.cpu.capacity` | gauge, cores | configured provider capacity | | `agents.runtime.memory.usage` | gauge, bytes | provider memory usage | | `agents.runtime.memory.limit` | gauge, bytes | configured provider limit | -| `agents.session.tokens.input` | gauge, tokens | canonical cumulative Session Usage | -| `agents.session.tokens.output` | gauge, tokens | canonical cumulative Session Usage | +| `agents.session.tokens.input` | gauge, tokens | measured cumulative Session usage (`MeasuredSessionUsage`) | +| `agents.session.tokens.output` | gauge, tokens | measured cumulative Session usage (`MeasuredSessionUsage`) | | `agents.runtime.sample` | monotonic delta sum | one validated result, including unavailable/unsupported | | `agents.runtime.sample.duration` | delta histogram, seconds | bounded provider read duration | @@ -452,13 +452,17 @@ persisted by Core. ## 12. Token usage boundary Provider Runtime sources do not own or report token usage. During a periodic -history sweep, the Core resolver reads the existing canonical cumulative Session -Usage snapshot from the execution store alongside Runtime identity. The exporter +history sweep, the Core resolver reads the cumulative measured Session usage +(`MeasuredSessionUsage`: every recorded root Turn snapshot, active Turns +included) from the execution store alongside Runtime identity. Public Session +usage follows the stricter official rule and can be null meanwhile +([item serialization](history-events-usage.md#item-serialization-2026-09-23)). The exporter emits Session-scoped input/output token gauges with the same Session and sampling time, independently of Docker, microsandbox, Kubernetes, or another provider. PostgreSQL retains those cumulative snapshots alongside the sample; query results keep Session token points separate from allocation series. Web derives throughput from adjacent nondecreasing points. Missing or -incomplete native usage and counter regressions remain gaps, never zero. +incomplete native usage, counter regressions and intervals where the live trend +holds a Session's last public total remain gaps, never zero. The current snapshot API still does not duplicate Usage fields: Web joins its existing Session collection by exact `session_id`. The Dashboard reports measured @@ -532,7 +536,7 @@ Implemented for the browser-local current-snapshot live window. - Execution-owner sampling with bounded pages, concurrency and source deadlines. - Tenant-scoped history queries with input/output limits, explicit gaps and CPU fences. - Seven-day retention and bounded cleanup independent of active Runtime count. -- Core Web history restoration after refresh, including canonical token snapshots. +- Core Web history restoration after refresh, including measured token snapshots. - Real Docker/microsandbox and PostgreSQL acceptance remains mandatory for delivery. - Exporter queue/drop/error coverage counters remain outside this batch.