diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4f801703a..bec66792d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -3816,3 +3816,13 @@ bounds/catalog details. Preserve Project/key rune limits and node byte limits
separately. Keep sandbox validation metadata through its existing store wrapper
without changing transaction or provider authority. Public Session provider
validation remains byte-compatible; cover it with handler-level golden responses.
+
+Root Session/Turn diagnostics are Core-key reads from one committed database
+snapshot, reusing public status projection and precedence. Keep their finite
+failure catalog separate from private outcome text. Persist safe provisioning
+details in the existing failure transaction; never reconstruct historical details
+from reason strings. Item settlement belongs to the existing Session lock and
+terminal transaction: event/input receipt for normal terminal projections, one
+post-lock database wall-clock sample for forced incomplete Items. Preserve
+historical terminal nulls and public native completion times. See the
+[diagnostics contract](contracts/agents-api/session-diagnostics.md).
diff --git a/apps/docs/content/docs/api-reference/core/sessions.mdx b/apps/docs/content/docs/api-reference/core/sessions.mdx
index 174d997f9..a266de5f0 100644
--- a/apps/docs/content/docs/api-reference/core/sessions.mdx
+++ b/apps/docs/content/docs/api-reference/core/sessions.mdx
@@ -22,8 +22,11 @@ _openapi:
Core key only. Reuses the public resource projection and operation
rules; the Project ID selects the target space and does not
authenticate.
+ - content: >-
+ Core key only. Safe failure categories from one committed snapshot; no
+ native text or historical inference.
---
{/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */}
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/apps/docs/content/docs/api-reference/core/turns.mdx b/apps/docs/content/docs/api-reference/core/turns.mdx
index d29b6ce94..d21b08bb2 100644
--- a/apps/docs/content/docs/api-reference/core/turns.mdx
+++ b/apps/docs/content/docs/api-reference/core/turns.mdx
@@ -18,8 +18,11 @@ _openapi:
Core key only. Reuses the public resource projection and operation
rules; the Project ID selects the target space and does not
authenticate.
+ - content: >-
+ Core key only. At most 1000 root Item receipt timings in public Item
+ order. Receipt intervals are not native execution durations.
---
{/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */}
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/apps/docs/content/docs/public-api.mdx b/apps/docs/content/docs/public-api.mdx
index 8e0ed8da4..4c9bea5e2 100644
--- a/apps/docs/content/docs/public-api.mdx
+++ b/apps/docs/content/docs/public-api.mdx
@@ -48,6 +48,7 @@ with the Core key.
| `projects`, `projects/{project_id}[/archive]`, `projects/{project_id}/keys[/{key_id}]` | Projects and their API keys | [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) |
| `projects/{project_id}/{agents,environment-templates,skills,files,vaults,sessions}/**` | Resource reads and deletion, Session history, artifacts and archive | [Administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) |
| `projects/{project_id}/sessions/{session_id}/execution-configuration` | Committed harness and model selection | [Execution configuration](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/execution-configuration.md) |
+| `projects/{project_id}/sessions/{session_id}/diagnostics`, `projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics` | Root failure categories and Item receipt timing | [Session diagnostics](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/session-diagnostics.md) |
| `projects/{project_id}/sessions/{session_id}/runtime-observation`, `sandbox/runtime-observations` | Current Runtime observations | [Runtime observations](/observability) |
| `projects/{project_id}/sessions/{session_id}/runtime-history` | Stored Runtime history | [Runtime history](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/runtime-history-api.md) |
| `projects/{project_id}/{resource-owners,write-operations}`, `audit-log`, `summary` | Provenance, write history, administrator audit and usage summary | [Write audit](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/write-audit.md), [administrator contract](https://github.com/MiniMax-AI/parsar-core/blob/f36ac3592bfe9f77a2172680aa52b14c6f525062/contracts/agents-api/admin-api.md) |
diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json
index 3e6366f09..135587dfd 100644
--- a/apps/docs/content/guide-sources.json
+++ b/apps/docs/content/guide-sources.json
@@ -8,7 +8,7 @@
"docs/configuration.md": "e911757a03fbce4b803a284c8bb40bdb7000ca2f70e27938706dcdcd74bc7f9a",
"docs/web/core-connection.md": "46f86520e70bf41079708e2c398655aa9087ad9488c8e661d9a73035d11098c4",
"docs/getting-started/quickstart.md": "5effeffcfb343f7939910f0c1758e1492d49e7bd66c409b1819cefe67af592d7",
- "docs/api/README.md": "3c590ca05a07b2bea5ab88ab67719741f8a73a28056ec7d943a5912eb5f9052f",
+ "docs/api/README.md": "592fc2aef474e36eb760be0dd1b2269b6766c0d984f826eaf6f47711ea139387",
"contracts/agents-api/execution-tools.md": "e0b61f6c0c236c362186c5f6d0ad69a16dd8a8d9afabe1329a1fe22e5f4a71e9",
"docs/api/public-agent-api.md": "18829b366bd2e49fe5beb76dc8a709bc1aa8e35d4fbfd2484a0c52b46cf75417",
"contracts/agents-api/environments.md": "89ffc903ee44d00ee87c16950202a3ca4c8345160780dcdd4ed2a71b25b1c2dc",
@@ -36,7 +36,7 @@
"content/docs/bootstrap-projects-keys.zh.mdx": "509b4dcea4c865d6834bcaef78baa681180d48274931230f958a39e46011c88f",
"content/docs/quickstart.mdx": "4ebc8ac16df66042e816ff51cf6672d8a85f44e119211efba584025666dce845",
"content/docs/quickstart.zh.mdx": "c5f119b100b102918a45763834d0ed7fd436479fb411e16a62a408411dbcb3e1",
- "content/docs/public-api.mdx": "da6aa71753cc84081fba9ae03491717042369a12ea94037fc532f5f1f69d2319",
+ "content/docs/public-api.mdx": "12d4fdbe6291c7d74af591369ac1edd2eca2940b93d156612497cc52b933740d",
"content/docs/public-api.zh.mdx": "fb0efb9be2fba3f4a3dc71bf053d2c7eddffbfe010f59fa5a1ae60a3f068545b",
"content/docs/agents-and-tools.mdx": "b8cc50792e579cbf66bcc597685b745d8721266970b75e928d8edee2ce11b08b",
"content/docs/agents-and-tools.zh.mdx": "cc8d4e0ef0ea894f068f3bbd143d5ee70c6346bf98f425b1d5fe07ec3931f128",
diff --git a/apps/docs/openapi/core-api.yaml b/apps/docs/openapi/core-api.yaml
index 0832b7e56..b6d4983be 100644
--- a/apps/docs/openapi/core-api.yaml
+++ b/apps/docs/openapi/core-api.yaml
@@ -2046,6 +2046,64 @@ paths:
summary: Download immutable artifact bytes in a Project
tags:
- Artifacts
+ /core/v1/projects/{project_id}/sessions/{session_id}/diagnostics:
+ get:
+ description: Core key only. Safe failure categories from one committed snapshot; no native text or historical inference.
+ parameters:
+ - schema:
+ type: string
+ description: Project ID
+ in: path
+ name: project_id
+ required: true
+ - schema:
+ type: string
+ description: Session ID
+ in: path
+ name: session_id
+ required: true
+ responses:
+ '200':
+ description: OK
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.SessionDiagnostics'
+ '400':
+ description: Bad Request
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '401':
+ description: Unauthorized
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '404':
+ description: Not Found
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '500':
+ description: Internal Server Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '503':
+ description: Service Unavailable
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ security:
+ - DeploymentAdminAuth: []
+ summary: Retrieve root Session diagnostics
+ tags:
+ - Sessions
/core/v1/projects/{project_id}/sessions/{session_id}/execution-configuration:
get:
description: Core key only; the Project ID selects the target space and does not authenticate. Returns the committed model, harness and safe provider selection with recorded sources. This read never decrypts credentials, resolves current defaults or probes execution health. Deployment defaults frozen after they moved into Core show their safe view; older deployment selections remain redacted. Historical provenance and missing provider projections are explicitly unknown/unavailable.
@@ -2444,6 +2502,70 @@ paths:
summary: Retrieve an execution Turn in a Project
tags:
- Turns
+ /core/v1/projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics:
+ get:
+ description: Core key only. At most 1000 root Item receipt timings in public Item order. Receipt intervals are not native execution durations.
+ parameters:
+ - schema:
+ type: string
+ description: Project ID
+ in: path
+ name: project_id
+ required: true
+ - schema:
+ type: string
+ description: Session ID
+ in: path
+ name: session_id
+ required: true
+ - schema:
+ type: string
+ description: Root Turn ID
+ in: path
+ name: turn_id
+ required: true
+ responses:
+ '200':
+ description: OK
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.TurnDiagnostics'
+ '400':
+ description: Bad Request
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '401':
+ description: Unauthorized
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '404':
+ description: Not Found
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '500':
+ description: Internal Server Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ '503':
+ description: Service Unavailable
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/api.CoreErrorResponse'
+ security:
+ - DeploymentAdminAuth: []
+ summary: Retrieve root Turn diagnostics
+ tags:
+ - Turns
/core/v1/projects/{project_id}/skills:
get:
description: Core key only. Reuses the public resource projection and operation rules; the Project ID selects the target space and does not authenticate.
@@ -4232,6 +4354,17 @@ components:
- data
- object
type: object
+ api.DiagnosticFailure:
+ properties:
+ code:
+ type: string
+ failed_at:
+ anyOf:
+ - type: string
+ - type: 'null'
+ params:
+ type: object
+ type: object
api.EnvironmentExecutorCredentialRequest:
properties:
key_id:
@@ -4397,6 +4530,21 @@ components:
- type: 'null'
description: Applied value; always null for a sensitive setting.
type: object
+ api.ItemDiagnosticTiming:
+ properties:
+ completed_at:
+ anyOf:
+ - type: string
+ - type: 'null'
+ item_id:
+ type: string
+ observed_duration_ms:
+ anyOf:
+ - type: integer
+ - type: 'null'
+ started_at:
+ type: string
+ type: object
api.ProjectAPIKeyRequest:
properties:
name:
@@ -4516,6 +4664,70 @@ components:
$ref: '#/components/schemas/store.RuntimeNode'
type: array
type: object
+ api.SessionDiagnosticFailure:
+ properties:
+ code:
+ type: string
+ failed_at:
+ anyOf:
+ - type: string
+ - type: 'null'
+ params:
+ type: object
+ source:
+ enum:
+ - turn
+ - environment
+ - environment_input
+ type: string
+ turn_id:
+ type: string
+ type: object
+ api.SessionDiagnostics:
+ properties:
+ failure:
+ anyOf:
+ - allOf:
+ - $ref: '#/components/schemas/api.SessionDiagnosticFailure'
+ - type: 'null'
+ object:
+ enum:
+ - core.session_diagnostics
+ type: string
+ session_id:
+ type: string
+ status:
+ enum:
+ - idle
+ - in_progress
+ - requires_action
+ - failed
+ type: string
+ type: object
+ api.TurnDiagnostics:
+ properties:
+ failure:
+ anyOf:
+ - allOf:
+ - $ref: '#/components/schemas/api.DiagnosticFailure'
+ - type: 'null'
+ items:
+ items:
+ $ref: '#/components/schemas/api.ItemDiagnosticTiming'
+ type: array
+ items_truncated:
+ type: boolean
+ object:
+ enum:
+ - core.turn_diagnostics
+ type: string
+ session_id:
+ type: string
+ status:
+ type: string
+ turn_id:
+ type: string
+ type: object
coremetrics.Database:
properties:
ping_ms:
diff --git a/apps/docs/openapi/sources.json b/apps/docs/openapi/sources.json
index f6d797ee4..6e84dee2a 100644
--- a/apps/docs/openapi/sources.json
+++ b/apps/docs/openapi/sources.json
@@ -1,7 +1,7 @@
{
"sources": {
"contracts/agents-api/openapi.yaml": "a8896e8b67d78ed0608c7cc5636077ec33f4b4895a04bd6cfb8b038afe941e01",
- "contracts/agents-api/core.openapi.yaml": "e0bcc1b017c358b543a80cf1d50d33304ea92857015ed15e06f6646fb9213c64",
+ "contracts/agents-api/core.openapi.yaml": "ff8526ba681f7561e49cd8e1818ddb7c6e57ce044021b92d2d69d5b087356900",
"contracts/agents-api/runtime.openapi.yaml": "23852205d3b45892f2de9ecf3d52518c0dd0808962c0c512f219b565aa7ccef2"
},
"outputs": {
@@ -29,18 +29,18 @@
"content/docs/api-reference/core/executor-credentials.mdx": "352dc9cb02cceab6a7fa4fe13356e9532c329277224e7e01d18ae0c02cad1a3e",
"content/docs/api-reference/core/files.mdx": "05ce1d2296dbb3fc76cf4cf98d24dff249114c73529335214cee97363229f384",
"content/docs/api-reference/core/write-audit.mdx": "1298590ab7037077e56f6ab8adb020277763f7a670be796d24142323b2f95411",
- "content/docs/api-reference/core/sessions.mdx": "e8519b5d3b21296b0ea1edede3160766ad6a49197b92b201ee1a845542d99619",
+ "content/docs/api-reference/core/sessions.mdx": "52db14e7c88cb0521a12a9b3d19b14dd38583b97e716b261a44b2bd6bf05140a",
"content/docs/api-reference/core/artifacts.mdx": "620fe72cd41d19b9694fe7080eb8935ef44292852400c7be1c03a893613ca89c",
"content/docs/api-reference/core/execution-configuration.mdx": "f6a9ed4cdb73e87f5f4ce16b4f6fa14b756d07c97f2741bd5cade34a3472c548",
"content/docs/api-reference/core/items.mdx": "7c7b006536d155d003097a504e9931e3dc4d9fd0426852573244453634729250",
"content/docs/api-reference/core/runtime-history.mdx": "eec6384c6fd5a2bf223ec7dc7ee19332440936e18fc67f782fe81b96e299f634",
"content/docs/api-reference/core/runtime-observations.mdx": "db3a238ba77d1fcb887835d51b6f3f8f2b6bdf9cc5e14ac6dd11074a1c75ed48",
- "content/docs/api-reference/core/turns.mdx": "e6f7a154b3c70e1555417e44a7888295afb8f8bd475621300e1b2bbd2fc94933",
+ "content/docs/api-reference/core/turns.mdx": "53f9472de3e0958c06db7e69d7e8f10119e6540b7f05683915248e08d5284b16",
"content/docs/api-reference/core/skills.mdx": "ed899ade7b032b1732b3097d81419a8fdbb9ead0594fca9f2107da644b97031f",
"content/docs/api-reference/core/vaults.mdx": "12d0aac96cd4a3472ef4eaa287f7945f78862017956e79bfc3ed0ef8e772e78a",
"content/docs/api-reference/core/credentials.mdx": "27d586b9a8fd738b9947802f32c80458195370252886bc26abedccf2125ee981",
"content/docs/api-reference/core/sandbox-manager.mdx": "6be7c52e057365949f0247c18c710561d680847a955f78f1bbe57825d4d5f549",
- "openapi/core-api.yaml": "5f40de88d0c39e6083f5c002e2772a03971a266a841726d76c594570f6b9bbcd",
+ "openapi/core-api.yaml": "18e100de2375835350d172e8a472db6966e36f1dd3aaae5e5d8132ba3039fa9c",
"content/docs/api-reference/core/meta.json": "588b163ed2de2100b0f8376688274f80698a6bf6ac037eef56d17c93fa8a5062",
"content/docs/api-reference/core/index.mdx": "ddf0a8d87a6c21761beac703a32f2a40486a1c74abb341bd726f40c44a866405",
"content/docs/api-reference/machine/sandbox-node.mdx": "5b367a08d43cea82686345d2ba1c75cc90627a7b2970fe42a0df9d77fb3e1e80",
diff --git a/contracts/agents-api/core-errors.md b/contracts/agents-api/core-errors.md
index 3081d305e..04f9a2932 100644
--- a/contracts/agents-api/core-errors.md
+++ b/contracts/agents-api/core-errors.md
@@ -111,3 +111,31 @@ Only the marked Core router maps their typed field metadata to this catalog;
public `/v1` and machine routes retain their previous complete error bodies.
Unknown sandbox providers retain the existing untyped error. E2B provider errors
above retain their fixed redaction contract with no echoed template or key.
+
+## Diagnostic failure categories
+
+The [root diagnostics reads](session-diagnostics.md) return these categories
+inside a successful HTTP 200 snapshot, not the operation error envelope. Public
+`/v1` Turn errors remain unchanged. `params` is `{}` unless specified.
+
+| Code | Stored cause or safe meaning |
+| --- | --- |
+| `harness_error` | `engine_failed` without a native classification |
+| `model_provider_required` | Missing frozen model provider |
+| `runtime_unavailable` | `execution_device_unavailable`, `execution_unavailable` |
+| `runtime_disconnected` | `device_disconnected`, `event_stream_incomplete` |
+| `runtime_preparation_failed` | `preparation_start_failed`, `preparation_interrupted` |
+| `execution_interrupted` | Core execution interrupted |
+| `delivery_unconfirmed` | `delivery_unknown`, `input_outcome_unknown`, `cancel_unconfirmed`, `cancel_outcome_unavailable`, `function_result_unconfirmed` |
+| `input_rejected` | `invalid_input`, `input_not_applied`, `message_input_unsupported`, and the exact steering outcomes `input_invalid_input`, `input_run_inactive`, `input_input_conflict`, `input_input_limit`, `input_unsupported`, `input_rejected`, `input_not_ready`, `input_busy` |
+| `executor_protocol_error` | `invalid_executor_result`, `interaction_not_supported`, `execution_state_unavailable`, `execution_state_changed`, `function_call_invalid`, `function_result_invalid` |
+| `core_storage_failed` | `event_persistence_failed`, `artifact_capture_failed` |
+| `internal_error` | Unknown or malformed outcome; no raw value is returned |
+| `environment_connection_timeout` | Initial input connection deadline expired |
+| `environment_unavailable` | Environment unavailable for initial input |
+| `environment_provisioning_failed` | Hosted provisioning failure; params contain nullable `step`, `index`, `exit_code` from a sanitized receipt |
+
+`diagnostics_unavailable` is the HTTP 503 operation error when the diagnostic
+reader is not configured; it has no details. A database failure remains an error,
+never a healthy or empty diagnostic snapshot. Historical provisioning reasons and
+private native messages are not parsed for categories or parameters.
diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml
index d2e8fde9a..40879eceb 100644
--- a/contracts/agents-api/core.openapi.yaml
+++ b/contracts/agents-api/core.openapi.yaml
@@ -250,6 +250,16 @@ definitions:
- data
- object
type: object
+ api.DiagnosticFailure:
+ properties:
+ code:
+ type: string
+ failed_at:
+ type: string
+ x-nullable: true
+ params:
+ type: object
+ type: object
api.EnvironmentExecutorCredentialRequest:
properties:
key_id:
@@ -403,6 +413,19 @@ definitions:
description: Applied value; always null for a sensitive setting.
x-nullable: true
type: object
+ api.ItemDiagnosticTiming:
+ properties:
+ completed_at:
+ type: string
+ x-nullable: true
+ item_id:
+ type: string
+ observed_duration_ms:
+ type: integer
+ x-nullable: true
+ started_at:
+ type: string
+ type: object
api.ProjectAPIKeyRequest:
properties:
name:
@@ -520,6 +543,67 @@ definitions:
$ref: '#/definitions/store.RuntimeNode'
type: array
type: object
+ api.SessionDiagnosticFailure:
+ properties:
+ code:
+ type: string
+ failed_at:
+ type: string
+ x-nullable: true
+ params:
+ type: object
+ source:
+ enum:
+ - turn
+ - environment
+ - environment_input
+ type: string
+ turn_id:
+ type: string
+ type: object
+ api.SessionDiagnostics:
+ properties:
+ failure:
+ allOf:
+ - $ref: '#/definitions/api.SessionDiagnosticFailure'
+ x-nullable: true
+ object:
+ enum:
+ - core.session_diagnostics
+ type: string
+ session_id:
+ type: string
+ status:
+ enum:
+ - idle
+ - in_progress
+ - requires_action
+ - failed
+ type: string
+ type: object
+ api.TurnDiagnostics:
+ properties:
+ failure:
+ allOf:
+ - $ref: '#/definitions/api.DiagnosticFailure'
+ x-nullable: true
+ items:
+ items:
+ $ref: '#/definitions/api.ItemDiagnosticTiming'
+ type: array
+ items_truncated:
+ type: boolean
+ object:
+ enum:
+ - core.turn_diagnostics
+ type: string
+ session_id:
+ type: string
+ status:
+ type: string
+ turn_id:
+ type: string
+ type: object
coremetrics.Database:
properties:
ping_ms:
@@ -4835,6 +4919,52 @@ paths:
summary: Download immutable artifact bytes in a Project
tags:
- Artifacts
+ /core/v1/projects/{project_id}/sessions/{session_id}/diagnostics:
+ get:
+ description: Core key only. Safe failure categories from one committed snapshot; no native text or historical inference.
+ parameters:
+ - description: Project ID
+ in: path
+ name: project_id
+ required: true
+ type: string
+ - description: Session ID
+ in: path
+ name: session_id
+ required: true
+ type: string
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
+ schema:
+ $ref: '#/definitions/api.SessionDiagnostics'
+ "400":
+ description: Bad Request
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "401":
+ description: Unauthorized
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "404":
+ description: Not Found
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "500":
+ description: Internal Server Error
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "503":
+ description: Service Unavailable
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ security:
+ - DeploymentAdminAuth: []
+ summary: Retrieve root Session diagnostics
+ tags:
+ - Sessions
/core/v1/projects/{project_id}/sessions/{session_id}/execution-configuration:
get:
description: Core key only; the Project ID selects the target space and does not authenticate. Returns the committed model, harness and safe provider selection with recorded sources. This read never decrypts credentials, resolves current defaults or probes execution health. Deployment defaults frozen after they moved into Core show their safe view; older deployment selections remain redacted. Historical provenance and missing provider projections are explicitly unknown/unavailable.
@@ -5161,6 +5291,57 @@ paths:
summary: Retrieve an execution Turn in a Project
tags:
- Turns
+ /core/v1/projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics:
+ get:
+ description: Core key only. At most 1000 root Item receipt timings in public Item order. Receipt intervals are not native execution durations.
+ parameters:
+ - description: Project ID
+ in: path
+ name: project_id
+ required: true
+ type: string
+ - description: Session ID
+ in: path
+ name: session_id
+ required: true
+ type: string
+ - description: Root Turn ID
+ in: path
+ name: turn_id
+ required: true
+ type: string
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
+ schema:
+ $ref: '#/definitions/api.TurnDiagnostics'
+ "400":
+ description: Bad Request
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "401":
+ description: Unauthorized
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "404":
+ description: Not Found
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "500":
+ description: Internal Server Error
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ "503":
+ description: Service Unavailable
+ schema:
+ $ref: '#/definitions/api.CoreErrorResponse'
+ security:
+ - DeploymentAdminAuth: []
+ summary: Retrieve root Turn diagnostics
+ tags:
+ - Turns
/core/v1/projects/{project_id}/skills:
get:
description: Core key only. Reuses the public resource projection and operation rules; the Project ID selects the target space and does not authenticate.
diff --git a/contracts/agents-api/session-diagnostics.md b/contracts/agents-api/session-diagnostics.md
new file mode 100644
index 000000000..e8b29a5c9
--- /dev/null
+++ b/contracts/agents-api/session-diagnostics.md
@@ -0,0 +1,78 @@
+# Root Session and Turn diagnostics
+
+These read-only routes require a Core key. The Project ID selects the target
+space; it does not authenticate. Both return `Cache-Control: no-store`:
+
+- `GET /core/v1/projects/{project_id}/sessions/{session_id}/diagnostics`
+- `GET /core/v1/projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics`
+
+Missing, malformed, foreign and deleted resources use the existing Session/Turn
+not-found rules. A Subagent Turn is not a root Turn and returns 404. Reads never
+contact an executor, provision an Environment, repair history or change execution.
+
+## Session snapshot
+
+The object is `core.session_diagnostics`, with `session_id`, official Session
+`status`, and nullable `failure`. Each read uses one repeatable-read database
+snapshot and the existing public status projection. Each Session or Turn snapshot
+read has a five-second budget, shortened by any earlier caller deadline.
+The transaction is closed after the read, including cancellation. Failure is null
+unless that projection says `failed`. Its fields are:
+
+| Field | Value |
+| --- | --- |
+| `source` | `turn`, `environment` or `environment_input` |
+| `turn_id` | Present only for `source: turn` |
+| `code` | Fixed category from [the diagnostics catalog](core-errors.md#diagnostic-failure-categories) |
+| `params` | An object of fixed safe values; `{}` for categories without parameters |
+| `failed_at` | RFC3339 timestamp, or null when historical time is unknown |
+
+Hosted provisioning failure overrides input activity; input activity overrides
+the latest root Turn. The same precedence drives the public Session response.
+Private outcome text, native messages, provider bodies, command text, paths and
+credentials are never included. Unknown outcome codes become `internal_error`;
+no prefix matching or historical reason-text parsing is used.
+
+Provisioning parameters come from the confirmed structured receipt, persisted in
+the existing failure transaction with Environment/input settlement and events.
+`step` is `setup`, `python`, `npm`, `system`, `file`, `skill` or null. `index` is a
+nonnegative JSON-safe integer for setup only, otherwise null. `exit_code` is 1–255
+for script steps, otherwise null. Unknown or historical details remain null.
+Changing these private fields does not change the public failure reason or SSE.
+
+## Turn snapshot and Item receipt timing
+
+The object is `core.turn_diagnostics`, with `session_id`, `turn_id`, the official
+root Turn `status`, nullable `failure`, `items` and `items_truncated`. Turn failure
+has `code`, `params` and nullable `failed_at`, without a source or Turn ID. Only
+failed Turns have failure details; cancelled/completed Turns never inherit an
+error classification from a private outcome.
+
+`items` contains at most 1000 root Items, ordered by `(created_at, position, id)`
+ascending, matching public Items order. Storage reads at most 1001 rows to detect
+truncation. Each entry has `item_id`, `started_at`, nullable `completed_at`, and
+nullable integer `observed_duration_ms`.
+
+- `started_at` is the first persisted Core input/event receipt.
+- `completed_at` is the first terminal input/event receipt. An Item first observed
+ terminal settles at that same receipt (zero observed duration).
+- An Item still in progress when its root Turn terminates settles with one database
+ `clock_timestamp()` sampled after the existing Session lock and terminal
+ projection. It does not use PostgreSQL transaction-start `now()` or the native
+ Turn completion timestamp. All such Items in that transaction share that time.
+- Repeated terminal projections preserve settlement. Already-terminal historical
+ Items with no settlement retain null; no backfill or duration estimate is made.
+- `observed_duration_ms` is the integer millisecond difference when both receipts
+ are known, otherwise null. It is not native execution duration. Journal batching,
+ transport, persistence and database wall-clock behavior affect this interval;
+ the event journal normally flushes about every 100 ms. Values are not clamped.
+
+The public successful Turn completion time may originate at the native executor;
+that public timestamp remains unchanged. Public Session, Turn, Item, SSE and
+`duration_ms` contracts are unchanged. Subagent diagnostics and native failure
+classification are separate work.
+
+The typed client exposes `retrieveSessionDiagnostics` and
+`retrieveTurnDiagnostics` on `AdminClient`, retaining request cancellation and
+validating scope, categories, nullability and safe parameter values. No browser
+credential or Core Web implementation is added by this contract.
diff --git a/docs/api/README.md b/docs/api/README.md
index 0059c859a..2fd197893 100644
--- a/docs/api/README.md
+++ b/docs/api/README.md
@@ -45,6 +45,7 @@ with the Core key.
| `projects`, `projects/{project_id}[/archive]`, `projects/{project_id}/keys[/{key_id}]` | Projects and their API keys | [Administrator contract](../../contracts/agents-api/admin-api.md) |
| `projects/{project_id}/{agents,environment-templates,skills,files,vaults,sessions}/**` | Resource reads and deletion, Session history, artifacts and archive | [Administrator contract](../../contracts/agents-api/admin-api.md) |
| `projects/{project_id}/sessions/{session_id}/execution-configuration` | Committed harness and model selection | [Execution configuration](../../contracts/agents-api/execution-configuration.md) |
+| `projects/{project_id}/sessions/{session_id}/diagnostics`, `projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics` | Root failure categories and Item receipt timing | [Session diagnostics](../../contracts/agents-api/session-diagnostics.md) |
| `projects/{project_id}/sessions/{session_id}/runtime-observation`, `sandbox/runtime-observations` | Current Runtime observations | [Runtime observations](../../contracts/agents-api/runtime-observability-api.md) |
| `projects/{project_id}/sessions/{session_id}/runtime-history` | Stored Runtime history | [Runtime history](../../contracts/agents-api/runtime-history-api.md) |
| `projects/{project_id}/{resource-owners,write-operations}`, `audit-log`, `summary` | Provenance, write history, administrator audit and usage summary | [Write audit](../../contracts/agents-api/write-audit.md), [administrator contract](../../contracts/agents-api/admin-api.md) |
diff --git a/packages/agents-client/src/admin-client.ts b/packages/agents-client/src/admin-client.ts
index 4ab23613d..94c2e85bd 100644
--- a/packages/agents-client/src/admin-client.ts
+++ b/packages/agents-client/src/admin-client.ts
@@ -1,3 +1,4 @@
+import { projectSessionDiagnostics, projectTurnDiagnostics } from "./session-diagnostics";
import {
addVaultPageOptions, projectAgentSession, projectRuntimeObservation,
projectEnvironmentTemplate, projectEnvironmentTemplateList, projectVault, projectVaultList,
@@ -225,6 +226,12 @@ export class AdminClient {
downloadArtifact(projectId: string, sessionId: string, artifactId: string, options?: ReadOptions) {
return this.#content(`${scope(projectId)}/sessions/${segment(sessionId)}/artifacts/${segment(artifactId)}/content`, options);
}
+ async retrieveSessionDiagnostics(projectId: string, sessionId: string, options?: ReadOptions) {
+ return projectSessionDiagnostics(await this.#json(`${scope(projectId)}/sessions/${segment(sessionId)}/diagnostics`, options), sessionId);
+ }
+ async retrieveTurnDiagnostics(projectId: string, sessionId: string, turnId: string, options?: ReadOptions) {
+ return projectTurnDiagnostics(await this.#json(`${scope(projectId)}/sessions/${segment(sessionId)}/turns/${segment(turnId)}/diagnostics`, options), sessionId, turnId);
+ }
async retrieveSessionExecutionConfiguration(projectId: string, sessionId: string, options?: ReadOptions) {
return projectExecutionConfiguration(await this.#json(`${scope(projectId)}/sessions/${segment(sessionId)}/execution-configuration`, options), sessionId, invalidAdminResponse);
}
diff --git a/packages/agents-client/src/index.ts b/packages/agents-client/src/index.ts
index eca855087..4831faa2e 100644
--- a/packages/agents-client/src/index.ts
+++ b/packages/agents-client/src/index.ts
@@ -12,3 +12,5 @@ export { compareSkillVersionNumbers, isSkillId, isSkillUploadPath, isSkillVersio
export { AdminClient } from "./admin-client";
// Skill and SkillVersion come from ./types; the admin projections use the same shapes.
export type { AdminClientOptions, AdminProject, CreateAdminProjectInput, RenameAdminProjectInput, AdminAPIKey, AdminIssuedAPIKey, IssueAdminAPIKeyInput, AdminPage, AdminDeleted, SessionArtifact, AdminContent, AdminResourceType, AdminKeyProvenance, AdminResourceOwner, AdminWriteOperation, AdminWriteOperationOptions, AdminWriteOperationPage, AdminSummaryOptions, AdminSummaryEntry, AdminSummary, AdminRuntimeObservation, AdminAuditOptions, AdminAuditResultID, AdminAuditEntry, AdminAuditPage, ExecutorCredential, ExecutorConnection, ExecutorCredentialList, IssueExecutorCredentialInput, IssuedExecutorCredential, CoreProjectReader, CoreInstallation, CoreInstallationConfiguration, CoreInstallationSetting, CoreAddressBindings } from "./admin-types";
+
+export type { DiagnosticFailureCode, ProvisioningFailureParams, DiagnosticFailure, SessionDiagnosticFailure, SessionDiagnostics, ItemDiagnosticTiming, TurnDiagnostics } from "./session-diagnostics";
diff --git a/packages/agents-client/src/session-diagnostics.test.ts b/packages/agents-client/src/session-diagnostics.test.ts
new file mode 100644
index 000000000..01b99fd88
--- /dev/null
+++ b/packages/agents-client/src/session-diagnostics.test.ts
@@ -0,0 +1,70 @@
+import { expect, it } from "vitest";
+import { AdminClient } from "./admin-client";
+import { projectSessionDiagnostics, projectTurnDiagnostics } from "./session-diagnostics";
+
+const id = "a113b7db-c689-4dfc-af9a-04bd0e466f11";
+const turn = "729dab72-a9ea-4d0a-a38c-1b45e46233a9";
+const item = "bf8d4f3a-d8bd-4d59-ad37-15f186f90832";
+const time = "2026-09-28T03:00:00.123456Z";
+const failure = { code: "runtime_disconnected", params: {}, failed_at: time };
+const session = { object: "core.session_diagnostics", session_id: id, status: "failed", failure: { ...failure, source: "turn", turn_id: turn } };
+const snapshot = { object: "core.turn_diagnostics", session_id: id, turn_id: turn, status: "failed", failure, items: [{ item_id: item, started_at: time, completed_at: null, observed_duration_ms: null }], items_truncated: false };
+
+it("retrieves Core diagnostics with credential, scope and cancellation", async () => {
+ const abort = new AbortController();
+ const paths: string[] = [];
+ const client = new AdminClient({ adminToken: "admin", fetch: (async (url, init) => {
+ paths.push(String(url));
+ expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer admin");
+ expect(init?.signal).toBe(abort.signal);
+ return new Response(JSON.stringify(String(url).includes("/turns/") ? snapshot : session), { headers: { "Content-Type": "application/json" } });
+ }) as typeof fetch });
+ expect(await client.retrieveSessionDiagnostics("project", id, { signal: abort.signal })).toEqual(session);
+ expect(await client.retrieveTurnDiagnostics("project", id, turn, { signal: abort.signal })).toEqual(snapshot);
+ expect(paths).toEqual([`/core/v1/projects/project/sessions/${id}/diagnostics`, `/core/v1/projects/project/sessions/${id}/turns/${turn}/diagnostics`]);
+});
+
+it("preserves historical nulls and finite safe provisioning parameters", () => {
+ const value = { ...session, failure: { source: "environment", code: "environment_provisioning_failed", params: { step: "setup", index: 2, exit_code: 7 }, failed_at: null } };
+ expect(projectSessionDiagnostics(value, id)).toEqual(value);
+ expect(projectTurnDiagnostics(snapshot, id.toUpperCase(), turn.toUpperCase())).toEqual(snapshot);
+});
+
+it.each([
+ (v: any) => { v.failure.error = "secret-canary"; },
+ (v: any) => { v.failure.code = "input_secret-canary"; },
+ (v: any) => { v.failure.params.message = "secret-canary"; },
+ (v: any) => { v.session_id = turn; },
+ (v: any) => { v.turn_id = id; },
+ (v: any) => { v.items[0].observed_duration_ms = 0; },
+ (v: any) => { v.items[0].completed_at = "bad"; },
+ (v: any) => { v.items.push(v.items[0]); },
+ (v: any) => { v.items_truncated = true; },
+ (v: any) => { v.items = Array.from({ length: 1001 }, () => v.items[0]); },
+ (v: any) => { v.status = "completed"; },
+])("rejects malformed, unsafe or mis-scoped Turn snapshots", (mutate) => {
+ const value = structuredClone(snapshot); mutate(value);
+ expect(() => projectTurnDiagnostics(value, id, turn)).toThrow();
+ expect(() => projectTurnDiagnostics(value, id, turn)).not.toThrow(/secret-canary/u);
+});
+
+it.each([
+ { step: "secret-canary", index: null, exit_code: null },
+ { step: "python", index: 3, exit_code: 2 },
+ { step: "setup", index: Number.MAX_SAFE_INTEGER + 1, exit_code: 1 },
+ { step: "file", index: null, exit_code: 1 },
+ { step: "setup", index: 0, exit_code: 256 },
+])("rejects unsafe provisioning parameters", (params) => {
+ const value = { ...session, failure: { source: "environment", code: "environment_provisioning_failed", params, failed_at: time } };
+ expect(() => projectSessionDiagnostics(value, id)).toThrow();
+});
+
+const provisioning = { source: "environment", code: "environment_provisioning_failed", params: { step: "setup", index: null, exit_code: null }, failed_at: null };
+it.each([
+ ["Session status", () => projectSessionDiagnostics({ ...session, status: ["failed"], failure: null }, id)],
+ ["Turn status", () => projectTurnDiagnostics({ ...snapshot, status: ["failed"], failure: null }, id, turn)],
+ ["failure source", () => projectSessionDiagnostics({ ...session, failure: { ...provisioning, source: ["environment"] } }, id)],
+ ["provisioning step", () => projectSessionDiagnostics({ ...session, failure: { ...provisioning, params: { ...provisioning.params, step: ["setup"] } } }, id)],
+] as const)("rejects arrays in %s", (_field, project) => {
+ expect(project).toThrow();
+});
diff --git a/packages/agents-client/src/session-diagnostics.ts b/packages/agents-client/src/session-diagnostics.ts
new file mode 100644
index 000000000..3b3845c9f
--- /dev/null
+++ b/packages/agents-client/src/session-diagnostics.ts
@@ -0,0 +1,67 @@
+import { canonicalUuid, exactFields, isRecord, sameResourceId } from "./response-projection";
+import { invalidAdminResponse } from "./admin-projection";
+
+export type DiagnosticFailureCode = "harness_error" | "model_provider_required" | "runtime_unavailable" | "runtime_disconnected" | "runtime_preparation_failed" | "execution_interrupted" | "delivery_unconfirmed" | "input_rejected" | "executor_protocol_error" | "core_storage_failed" | "internal_error" | "environment_connection_timeout" | "environment_unavailable" | "environment_provisioning_failed";
+export type ProvisioningFailureParams = { step: "setup" | "python" | "npm" | "system" | "file" | "skill" | null; index: number | null; exit_code: number | null };
+export type DiagnosticFailure = { code: DiagnosticFailureCode; params: Record | ProvisioningFailureParams; failed_at: string | null };
+export type SessionDiagnosticFailure = DiagnosticFailure & { source: "turn" | "environment" | "environment_input"; turn_id?: string };
+export type SessionDiagnostics = { object: "core.session_diagnostics"; session_id: string; status: "idle" | "in_progress" | "requires_action" | "failed"; failure: SessionDiagnosticFailure | null };
+/** Core receipt intervals, distinct from the public tool-reported duration_ms. */
+export type ItemDiagnosticTiming = { item_id: string; started_at: string; completed_at: string | null; observed_duration_ms: number | null };
+export type TurnDiagnostics = { object: "core.turn_diagnostics"; session_id: string; turn_id: string; status: string; failure: DiagnosticFailure | null; items: ItemDiagnosticTiming[]; items_truncated: boolean };
+
+const turnCodes = new Set(["harness_error", "model_provider_required", "runtime_unavailable", "runtime_disconnected", "runtime_preparation_failed", "execution_interrupted", "delivery_unconfirmed", "input_rejected", "executor_protocol_error", "core_storage_failed", "internal_error"]);
+const inputCodes = new Set(["environment_connection_timeout", "environment_unavailable", "model_provider_required", "internal_error"]);
+const turnStatuses = new Set(["queued", "in_progress", "waiting", "completed", "failed", "cancelled"]);
+const fields = (...values: string[]) => new Set(values);
+function timestamp(value: unknown): value is string {
+ return typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/u.test(value) && Number.isFinite(Date.parse(value));
+}
+function failure(value: unknown, codes: Set, session = false): DiagnosticFailure {
+ if (!isRecord(value) || !exactFields(value, fields("code", "params", "failed_at", ...(session ? ["source", ...(value.source === "turn" ? ["turn_id"] : [])] : []))) ||
+ typeof value.code !== "string" || !codes.has(value.code) || !isRecord(value.params) || (value.failed_at !== null && !timestamp(value.failed_at))) return invalidAdminResponse();
+ let params: DiagnosticFailure["params"] = {};
+ if (value.code === "environment_provisioning_failed") {
+ const p = value.params;
+ if (!exactFields(p, fields("step", "index", "exit_code")) || (p.step !== null && (typeof p.step !== "string" || !["setup", "python", "npm", "system", "file", "skill"].includes(p.step))) ||
+ (p.index !== null && (!Number.isSafeInteger(p.index) || (p.index as number) < 0 || p.step !== "setup")) ||
+ (p.exit_code !== null && (!Number.isSafeInteger(p.exit_code) || (p.exit_code as number) < 1 || (p.exit_code as number) > 255 || !["setup", "python", "npm", "system"].includes(p.step ?? "")))) return invalidAdminResponse();
+ params = { step: p.step as ProvisioningFailureParams["step"], index: p.index as number | null, exit_code: p.exit_code as number | null };
+ } else if (Object.keys(value.params).length !== 0) return invalidAdminResponse();
+ return { code: value.code as DiagnosticFailureCode, params, failed_at: value.failed_at as string | null };
+}
+
+export function projectSessionDiagnostics(value: unknown, sessionId: string): SessionDiagnostics {
+ if (!isRecord(value) || !exactFields(value, fields("object", "session_id", "status", "failure")) || value.object !== "core.session_diagnostics" ||
+ canonicalUuid(value.session_id) === null || !sameResourceId(value.session_id as string, sessionId) || typeof value.status !== "string" || !["idle", "in_progress", "requires_action", "failed"].includes(value.status)) return invalidAdminResponse();
+ let projected: SessionDiagnosticFailure | null = null;
+ if (value.status === "failed") {
+ if (!isRecord(value.failure) || typeof value.failure.source !== "string" || !["turn", "environment", "environment_input"].includes(value.failure.source)) return invalidAdminResponse();
+ const f = value.failure;
+ const source = f.source as SessionDiagnosticFailure["source"];
+ projected = { ...failure(f, source === "turn" ? turnCodes : source === "environment_input" ? inputCodes : new Set(["environment_provisioning_failed"]), true), source };
+ if (source === "turn") {
+ if (canonicalUuid(f.turn_id) === null) return invalidAdminResponse();
+ projected.turn_id = f.turn_id as string;
+ }
+ } else if (value.failure !== null) return invalidAdminResponse();
+ return { object: "core.session_diagnostics", session_id: value.session_id as string, status: value.status as SessionDiagnostics["status"], failure: projected };
+}
+
+export function projectTurnDiagnostics(value: unknown, sessionId: string, turnId: string): TurnDiagnostics {
+ if (!isRecord(value) || !exactFields(value, fields("object", "session_id", "turn_id", "status", "failure", "items", "items_truncated")) || value.object !== "core.turn_diagnostics" ||
+ canonicalUuid(value.session_id) === null || !sameResourceId(value.session_id as string, sessionId) || canonicalUuid(value.turn_id) === null || !sameResourceId(value.turn_id as string, turnId) ||
+ typeof value.status !== "string" || !turnStatuses.has(value.status) || !Array.isArray(value.items) || value.items.length > 1000 || typeof value.items_truncated !== "boolean" || (value.items_truncated && value.items.length !== 1000)) return invalidAdminResponse();
+ const projected = value.status === "failed" ? failure(value.failure, turnCodes) : null;
+ if (value.status !== "failed" && value.failure !== null) return invalidAdminResponse();
+ const seen = new Set();
+ const items: ItemDiagnosticTiming[] = value.items.map((item: unknown) => {
+ if (!isRecord(item) || !exactFields(item, fields("item_id", "started_at", "completed_at", "observed_duration_ms")) || canonicalUuid(item.item_id) === null || !timestamp(item.started_at) ||
+ (item.completed_at !== null && !timestamp(item.completed_at)) || (item.completed_at === null ? item.observed_duration_ms !== null : !Number.isSafeInteger(item.observed_duration_ms))) return invalidAdminResponse();
+ const id = canonicalUuid(item.item_id)!;
+ if (seen.has(id)) return invalidAdminResponse();
+ seen.add(id);
+ return { item_id: item.item_id as string, started_at: item.started_at, completed_at: item.completed_at as string | null, observed_duration_ms: item.observed_duration_ms as number | null };
+ });
+ return { object: "core.turn_diagnostics", session_id: value.session_id as string, turn_id: value.turn_id as string, status: value.status as string, failure: projected, items, items_truncated: value.items_truncated };
+}
diff --git a/services/agents-api/internal/api/admin_resources.go b/services/agents-api/internal/api/admin_resources.go
index 1d6ac5f55..3519b89ce 100644
--- a/services/agents-api/internal/api/admin_resources.go
+++ b/services/agents-api/internal/api/admin_resources.go
@@ -74,6 +74,8 @@ func (h *Handler) registerAdminResourceRoutes(router chi.Router) {
r.Delete("/projects/{project_id}/vaults/{vault_id}/credentials/{credential_id}", h.adminDeleteCredential)
r.Get("/projects/{project_id}/sessions", h.adminListSessions)
r.Get("/projects/{project_id}/sessions/{session_id}", h.adminGetSession)
+ r.Get("/projects/{project_id}/sessions/{session_id}/diagnostics", h.getSessionDiagnostics)
+ r.Get("/projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics", h.getTurnDiagnostics)
r.Delete("/projects/{project_id}/sessions/{session_id}", h.adminDeleteSession)
if h.adminManagement != nil {
r.Post("/projects/{project_id}/sessions/{session_id}/archive", h.adminArchiveSession)
diff --git a/services/agents-api/internal/api/session_diagnostics.go b/services/agents-api/internal/api/session_diagnostics.go
new file mode 100644
index 000000000..16d71a245
--- /dev/null
+++ b/services/agents-api/internal/api/session_diagnostics.go
@@ -0,0 +1,175 @@
+package api
+
+import (
+ "context"
+ "net/http"
+ "time"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store"
+ "github.com/go-chi/chi/v5"
+)
+
+type DiagnosticFailure struct {
+ Code string `json:"code"`
+ Params CoreErrorDetails `json:"params" swaggertype:"object"`
+ FailedAt *time.Time `json:"failed_at" extensions:"x-nullable"`
+}
+
+type SessionDiagnosticFailure struct {
+ DiagnosticFailure
+ Source string `json:"source" enums:"turn,environment,environment_input"`
+ TurnID string `json:"turn_id,omitempty"`
+}
+
+type SessionDiagnostics struct {
+ Object string `json:"object" enums:"core.session_diagnostics"`
+ SessionID string `json:"session_id"`
+ Status string `json:"status" enums:"idle,in_progress,requires_action,failed"`
+ Failure *SessionDiagnosticFailure `json:"failure" extensions:"x-nullable"`
+}
+
+type ItemDiagnosticTiming struct {
+ ItemID string `json:"item_id"`
+ StartedAt time.Time `json:"started_at"`
+ CompletedAt *time.Time `json:"completed_at" extensions:"x-nullable"`
+ ObservedDurationMS *int64 `json:"observed_duration_ms" extensions:"x-nullable"`
+}
+
+type TurnDiagnostics struct {
+ Object string `json:"object" enums:"core.turn_diagnostics"`
+ SessionID string `json:"session_id"`
+ TurnID string `json:"turn_id"`
+ Status string `json:"status"`
+ Failure *DiagnosticFailure `json:"failure" extensions:"x-nullable"`
+ Items []ItemDiagnosticTiming `json:"items"`
+ ItemsTruncated bool `json:"items_truncated"`
+}
+
+type sessionDiagnosticsStore interface {
+ GetSessionDiagnosticsSnapshot(context.Context, string, string) (store.Session, error)
+ GetTurnDiagnosticsSnapshot(context.Context, string, string, string) (store.TurnDiagnosticsSnapshot, error)
+}
+
+// @Summary Retrieve root Session diagnostics
+// @Description Core key only. Safe failure categories from one committed snapshot; no native text or historical inference.
+// @Tags Sessions
+// @Produce json
+// @Security DeploymentAdminAuth
+// @Param project_id path string true "Project ID"
+// @Param session_id path string true "Session ID"
+// @Success 200 {object} SessionDiagnostics
+// @Failure 400,401,404,500,503 {object} CoreErrorResponse
+// @Router /core/v1/projects/{project_id}/sessions/{session_id}/diagnostics [get]
+func (h *Handler) getSessionDiagnostics(w http.ResponseWriter, r *http.Request) {
+ source, ok := h.store.(sessionDiagnosticsStore)
+ if !ok {
+ writeError(w, http.StatusServiceUnavailable, "diagnostics_unavailable", "Session diagnostics are unavailable.")
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
+ defer cancel()
+ session, err := source.GetSessionDiagnosticsSnapshot(ctx, tenantID(r), chi.URLParam(r, "session_id"))
+ if err != nil {
+ writeStoreError(w, r, err)
+ return
+ }
+ public, err := sessionResponse(session, h.executorURL)
+ if err != nil {
+ writeStoreError(w, r, err)
+ return
+ }
+ response := SessionDiagnostics{Object: "core.session_diagnostics", SessionID: public.ID, Status: public.Status}
+ if public.Status == "failed" {
+ failure := SessionDiagnosticFailure{DiagnosticFailure: DiagnosticFailure{Code: "internal_error", Params: CoreErrorDetails{}}}
+ switch {
+ case session.EnvironmentFailure != nil:
+ failure.Source = "environment"
+ failure.Code = "environment_provisioning_failed"
+ failure.FailedAt = diagnosticTime(session.EnvironmentFailure.FailedAt)
+ failure.Params = provisioningFailureParams(session.EnvironmentFailure.Detail)
+ case session.EnvironmentInputActivity != nil:
+ failure.Source = "environment_input"
+ failure.FailedAt = diagnosticTime(session.EnvironmentInputActivity.LastActiveAt)
+ switch session.EnvironmentInputActivity.Failure {
+ case "":
+ failure.Code = "environment_connection_timeout"
+ case "environment_unavailable":
+ failure.Code = "environment_unavailable"
+ case "model_provider_required":
+ failure.Code = "model_provider_required"
+ }
+ case session.LastTurn != nil:
+ failure.Source, failure.TurnID = "turn", session.LastTurn.ID
+ failure.DiagnosticFailure = *turnDiagnosticFailure(*session.LastTurn)
+ }
+ response.Failure = &failure
+ }
+ w.Header().Set("Cache-Control", "no-store")
+ writeJSON(w, http.StatusOK, response)
+}
+
+// @Summary Retrieve root Turn diagnostics
+// @Description Core key only. At most 1000 root Item receipt timings in public Item order. Receipt intervals are not native execution durations.
+// @Tags Turns
+// @Produce json
+// @Security DeploymentAdminAuth
+// @Param project_id path string true "Project ID"
+// @Param session_id path string true "Session ID"
+// @Param turn_id path string true "Root Turn ID"
+// @Success 200 {object} TurnDiagnostics
+// @Failure 400,401,404,500,503 {object} CoreErrorResponse
+// @Router /core/v1/projects/{project_id}/sessions/{session_id}/turns/{turn_id}/diagnostics [get]
+func (h *Handler) getTurnDiagnostics(w http.ResponseWriter, r *http.Request) {
+ source, ok := h.store.(sessionDiagnosticsStore)
+ if !ok {
+ writeError(w, http.StatusServiceUnavailable, "diagnostics_unavailable", "Session diagnostics are unavailable.")
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
+ defer cancel()
+ snapshot, err := source.GetTurnDiagnosticsSnapshot(ctx, tenantID(r), chi.URLParam(r, "session_id"), chi.URLParam(r, "turn_id"))
+ if err != nil {
+ writeStoreError(w, r, err)
+ return
+ }
+ public, err := turnResponse(snapshot.Session, snapshot.Turn)
+ if err != nil {
+ writeStoreError(w, r, err)
+ return
+ }
+ response := TurnDiagnostics{Object: "core.turn_diagnostics", SessionID: public.SessionID, TurnID: public.ID, Status: public.Status, Failure: turnDiagnosticFailure(snapshot.Turn), Items: []ItemDiagnosticTiming{}, ItemsTruncated: snapshot.ItemsTruncated}
+ for _, value := range snapshot.Items {
+ item := ItemDiagnosticTiming{ItemID: value.ItemID, StartedAt: value.StartedAt, CompletedAt: value.CompletedAt}
+ if value.CompletedAt != nil {
+ duration := value.CompletedAt.Sub(value.StartedAt).Milliseconds()
+ item.ObservedDurationMS = &duration
+ }
+ response.Items = append(response.Items, item)
+ }
+ w.Header().Set("Cache-Control", "no-store")
+ writeJSON(w, http.StatusOK, response)
+}
+
+func diagnosticTime(value time.Time) *time.Time {
+ if value.IsZero() {
+ return nil
+ }
+ return &value
+}
+
+func provisioningFailureParams(detail *store.ProvisioningFailureDetail) CoreErrorDetails {
+ params := CoreErrorDetails{"step": CoreErrorNull(), "index": CoreErrorNull(), "exit_code": CoreErrorNull()}
+ if detail == nil {
+ return params
+ }
+ if detail.Step != nil {
+ params["step"] = CoreErrorString(*detail.Step)
+ }
+ if detail.Index != nil {
+ params["index"] = CoreErrorNumber(float64(*detail.Index))
+ }
+ if detail.ExitCode != nil {
+ params["exit_code"] = CoreErrorNumber(float64(*detail.ExitCode))
+ }
+ return params
+}
diff --git a/services/agents-api/internal/api/session_diagnostics_deadline_test.go b/services/agents-api/internal/api/session_diagnostics_deadline_test.go
new file mode 100644
index 000000000..034aae2e9
--- /dev/null
+++ b/services/agents-api/internal/api/session_diagnostics_deadline_test.go
@@ -0,0 +1,71 @@
+package api
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store"
+ "github.com/google/uuid"
+)
+
+type diagnosticDeadlineStore struct {
+ diagnosticSnapshotStore
+ observed context.Context
+}
+
+func (s *diagnosticDeadlineStore) GetSessionDiagnosticsSnapshot(ctx context.Context, tenant, session string) (store.Session, error) {
+ s.observed = ctx
+ return s.diagnosticSnapshotStore.GetSessionDiagnosticsSnapshot(ctx, tenant, session)
+}
+func (s *diagnosticDeadlineStore) GetTurnDiagnosticsSnapshot(ctx context.Context, tenant, session, turn string) (store.TurnDiagnosticsSnapshot, error) {
+ s.observed = ctx
+ return s.diagnosticSnapshotStore.GetTurnDiagnosticsSnapshot(ctx, tenant, session, turn)
+}
+
+func TestDiagnosticsReadDeadline(t *testing.T) {
+ id, turn := uuid.NewString(), uuid.NewString()
+ session := store.Session{ID: id, Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`), LastTurn: &store.Turn{ID: turn, SessionID: id, Status: store.TurnCompleted}}
+ for _, suffix := range []string{"/diagnostics", "/turns/" + turn + "/diagnostics"} {
+ for _, shorter := range []bool{false, true} {
+ t.Run(suffix+map[bool]string{false: "/server", true: "/caller"}[shorter], func(t *testing.T) {
+ source := &diagnosticDeadlineStore{diagnosticSnapshotStore: diagnosticSnapshotStore{session: session}}
+ h, _, _ := adminTestHandler(t, func(h *Handler) { h.store = source })
+ ctx := t.Context()
+ var callerDeadline time.Time
+ if shorter {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, time.Second)
+ defer cancel()
+ callerDeadline, _ = ctx.Deadline()
+ }
+ r := httptest.NewRequest(http.MethodGet, adminSessionsPath+id+suffix, nil).WithContext(ctx)
+ r.Header.Set("Authorization", "Bearer admin")
+ w := httptest.NewRecorder()
+ before := time.Now()
+ h.ServeHTTP(w, r)
+ after := time.Now()
+ if w.Code != http.StatusOK || source.observed == nil {
+ t.Fatal(w.Code, w.Body)
+ }
+ deadline, ok := source.observed.Deadline()
+ if !ok {
+ t.Fatal("diagnostic snapshot has no deadline")
+ }
+ if shorter {
+ if !deadline.Equal(callerDeadline) {
+ t.Fatal("caller deadline changed", deadline, callerDeadline)
+ }
+ } else if deadline.Before(before.Add(5*time.Second)) || deadline.After(after.Add(5*time.Second)) {
+ t.Fatal("diagnostic snapshot does not have a five-second budget", deadline)
+ }
+ if source.observed.Err() != context.Canceled {
+ t.Fatal("snapshot context retained after response", source.observed.Err())
+ }
+ })
+ }
+ }
+}
diff --git a/services/agents-api/internal/api/session_diagnostics_public_compat_test.go b/services/agents-api/internal/api/session_diagnostics_public_compat_test.go
new file mode 100644
index 000000000..fd0a46825
--- /dev/null
+++ b/services/agents-api/internal/api/session_diagnostics_public_compat_test.go
@@ -0,0 +1,92 @@
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/identity"
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store"
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/migrations"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5/pgxpool"
+)
+
+// This test also runs unchanged against the comparison base using a private
+// overlay. Fixed resource times make its public response bytes comparable.
+func TestDiagnosticPublicCompatibility(t *testing.T) {
+ s, pool := diagnosticDatabase(t)
+ key := callerBinding()
+ auth, err := NewAuthenticator([]APIKey{key})
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := NewHandler(s, auth, "codex")
+ if err != nil {
+ t.Fatal(err)
+ }
+ session, err := s.CreateSession(t.Context(), key.TenantID, store.CreateSessionInput{Creator: identity.Subject{Kind: "service_account", ID: "compat-test"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ turn := uuid.NewString()
+ item := uuid.NewString()
+ if _, err = pool.Exec(t.Context(), "UPDATE sessions SET created_at='2026-09-28T00:00:00Z' WHERE id=$1", session.ID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = pool.Exec(t.Context(), `INSERT INTO turns(id,session_id,status,created_at,started_at,completed_at,outcome) VALUES($1,$2,'failed','2026-09-28T00:00:00Z','2026-09-28T00:00:01Z','2026-09-28T00:00:02Z','{"error_code":"engine_failed","error":"private-secret-canary"}')`, turn, session.ID); err != nil {
+ t.Fatal(err)
+ }
+ payload, _ := json.Marshal(map[string]any{"id": item, "turn_id": turn, "type": "command_execution", "status": "failed", "command": "example", "exit_code": 1, "duration_ms": 5})
+ if _, err = pool.Exec(t.Context(), `INSERT INTO session_items(id,session_id,turn_id,created_at,payload) VALUES($1,$2,$3,'2026-09-28T00:00:01Z',$4)`, item, session.ID, turn, payload); err != nil {
+ t.Fatal(err)
+ }
+ for _, suffix := range []string{"", "/turns/" + turn, "/items"} {
+ w := diagnosticRequest(h, "/v1/agents/sessions/"+session.ID+suffix, "Bearer caller")
+ if w.Code != 200 || strings.Contains(w.Body.String(), "canary") || strings.Contains(w.Body.String(), "observed_duration") || strings.Contains(w.Body.String(), "failure_detail") {
+ t.Fatal(w.Code, w.Body)
+ }
+ body := strings.NewReplacer(session.ID, "SESSION", turn, "TURN", item, "ITEM").Replace(w.Body.String())
+ t.Logf("PUBLIC %s %s", strings.ReplaceAll(suffix, turn, "TURN"), strings.TrimSpace(body))
+ }
+}
+
+func diagnosticRequest(handler http.Handler, path, token string) *httptest.ResponseRecorder {
+ r := httptest.NewRequest(http.MethodGet, path, nil)
+ r.Header.Set("Authorization", token)
+ r.Header.Set("OpenAI-Beta", "agents=v1")
+ w := httptest.NewRecorder()
+ handler.ServeHTTP(w, r)
+ return w
+}
+
+func diagnosticDatabase(t *testing.T) (*store.Store, *pgxpool.Pool) {
+ t.Helper()
+ dsn := os.Getenv("OAC_TEST_DATABASE_URL")
+ if dsn == "" {
+ t.Skip("OAC_TEST_DATABASE_URL is not set; dedicated PostgreSQL required")
+ }
+ cfg, err := pgxpool.ParseConfig(dsn)
+ if err != nil {
+ t.Fatal("invalid test database configuration")
+ }
+ if !strings.HasPrefix(cfg.ConnConfig.Database, "oac_") || !strings.HasSuffix(cfg.ConnConfig.Database, "_tests") {
+ t.Fatal("test database must be named oac_*_tests")
+ }
+ pool, err := pgxpool.NewWithConfig(t.Context(), cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(pool.Close)
+ var productTable *string
+ if err = pool.QueryRow(t.Context(), "SELECT to_regclass('workspaces')::text").Scan(&productTable); err != nil || productTable != nil {
+ t.Fatal("dedicated Core database required", err)
+ }
+ if err = migrations.Apply(t.Context(), dsn); err != nil {
+ t.Fatal(err)
+ }
+ return store.New(pool), pool
+}
diff --git a/services/agents-api/internal/api/session_diagnostics_test.go b/services/agents-api/internal/api/session_diagnostics_test.go
new file mode 100644
index 000000000..74fd5e445
--- /dev/null
+++ b/services/agents-api/internal/api/session_diagnostics_test.go
@@ -0,0 +1,118 @@
+package api
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/identity"
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store"
+ "github.com/google/uuid"
+)
+
+func TestDiagnosticsCoreHandlerDatabaseBoundary(t *testing.T) {
+ s, pool := diagnosticDatabase(t)
+ h, _, tenant := adminTestHandler(t, func(h *Handler) { h.store = s })
+ session, err := s.CreateSession(t.Context(), tenant, store.CreateSessionInput{Creator: identity.Subject{Kind: "service_account", ID: "diagnostic-test"}, Engine: "codex", IdempotencyKey: "diagnostics", Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ receipt, err := s.SubmitMessage(t.Context(), tenant, session.ID, "input", json.RawMessage(`{"text":"input-secret-canary"}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = s.TransitionTurn(t.Context(), tenant, session.ID, receipt.TurnID, store.TurnTransition{ExpectedStatus: store.TurnQueued, Status: store.TurnInProgress}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = s.TransitionTurn(t.Context(), tenant, session.ID, receipt.TurnID, store.TurnTransition{ExpectedStatus: store.TurnInProgress, Status: store.TurnFailed, Outcome: json.RawMessage(`{"error_code":"device_disconnected","error":"Bearer raw-secret-canary https://private.example/key","done":{"native_id":"secret-native-canary"}}`)}); err != nil {
+ t.Fatal(err)
+ }
+ base := adminSessionsPath + session.ID
+ for _, path := range []string{base + "/diagnostics", base + "/turns/" + receipt.TurnID + "/diagnostics"} {
+ if w := diagnosticRequest(h, path, ""); w.Code != 401 {
+ t.Fatal("unauthed diagnostics", w.Code, w.Body)
+ }
+ if w := diagnosticRequest(h, path, "Bearer caller"); w.Code != 401 {
+ t.Fatal("Project credential entered Core route", w.Code, w.Body)
+ }
+ w := diagnosticRequest(h, path, "Bearer admin")
+ if w.Code != 200 || w.Header().Get("Cache-Control") != "no-store" || !strings.Contains(w.Body.String(), `"code":"runtime_disconnected"`) || strings.Contains(w.Body.String(), "canary") || strings.Contains(w.Body.String(), "private.example") {
+ t.Fatal(w.Code, w.Body)
+ }
+ }
+ missing := diagnosticRequest(h, base+"/turns/"+uuid.NewString()+"/diagnostics", "Bearer admin")
+ malformed := diagnosticRequest(h, base+"/turns/malformed/diagnostics", "Bearer admin")
+ if missing.Code != 404 || malformed.Code != 404 || missing.Body.String() != malformed.Body.String() {
+ t.Fatal("missing path semantics changed", missing.Body, malformed.Body)
+ }
+ foreign := diagnosticRequest(h, strings.Replace(base, managementProjectID, uuid.NewString(), 1)+"/diagnostics", "Bearer admin")
+ if foreign.Code != 404 {
+ t.Fatal("foreign project visible", foreign.Code)
+ }
+ if _, err = pool.Exec(t.Context(), "UPDATE sessions SET deleted_at=clock_timestamp() WHERE id=$1", session.ID); err != nil {
+ t.Fatal(err)
+ }
+ for _, path := range []string{base + "/diagnostics", base + "/turns/" + receipt.TurnID + "/diagnostics"} {
+ if w := diagnosticRequest(h, path, "Bearer admin"); w.Code != 404 {
+ t.Fatal("deleted root visible", w.Code, w.Body)
+ }
+ }
+}
+
+type diagnosticSnapshotStore struct {
+ ResourceStore
+ session store.Session
+}
+
+func (s diagnosticSnapshotStore) GetSessionDiagnosticsSnapshot(context.Context, string, string) (store.Session, error) {
+ return s.session, nil
+}
+func (s diagnosticSnapshotStore) GetTurnDiagnosticsSnapshot(context.Context, string, string, string) (store.TurnDiagnosticsSnapshot, error) {
+ return store.TurnDiagnosticsSnapshot{Session: s.session, Turn: *s.session.LastTurn, Items: []store.ItemDiagnosticTiming{}}, nil
+}
+
+func TestDiagnosticsFailurePrecedenceAndUnknownTime(t *testing.T) {
+ id, turnID := uuid.NewString(), uuid.NewString()
+ base := store.Session{ID: id, Configuration: json.RawMessage(`{"agent":{"id":"agent_root","model":"test"},"environment":{"type":"none"}}`), LastTurn: &store.Turn{ID: turnID, SessionID: id, Status: store.TurnFailed, Outcome: json.RawMessage(`{"error_code":"engine_failed","error":"secret-canary"}`)}}
+ for _, tc := range []struct {
+ activity *store.EnvironmentInputActivity
+ code, source string
+ }{{nil, "harness_error", "turn"}, {&store.EnvironmentInputActivity{Status: "failed"}, "environment_connection_timeout", "environment_input"}, {&store.EnvironmentInputActivity{Status: "failed", Failure: "model_provider_required"}, "model_provider_required", "environment_input"}, {&store.EnvironmentInputActivity{Status: "failed", Failure: "secret-canary"}, "internal_error", "environment_input"}} {
+ value := base
+ value.EnvironmentInputActivity = tc.activity
+ h, _, _ := adminTestHandler(t, func(h *Handler) { h.store = diagnosticSnapshotStore{session: value} })
+ w := diagnosticRequest(h, adminSessionsPath+id+"/diagnostics", "Bearer admin")
+ if w.Code != 200 || !strings.Contains(w.Body.String(), `"code":"`+tc.code+`"`) || !strings.Contains(w.Body.String(), `"source":"`+tc.source+`"`) || !strings.Contains(w.Body.String(), `"failed_at":null`) || strings.Contains(w.Body.String(), "canary") {
+ t.Fatal(w.Code, w.Body)
+ }
+ }
+ base.EnvironmentInputActivity = &store.EnvironmentInputActivity{Status: "idle", LastActiveAt: time.Now()}
+ h, _, _ := adminTestHandler(t, func(h *Handler) { h.store = diagnosticSnapshotStore{session: base} })
+ if w := diagnosticRequest(h, adminSessionsPath+id+"/diagnostics", "Bearer admin"); w.Code != 200 || !strings.Contains(w.Body.String(), `"failure":null`) {
+ t.Fatal("public activity precedence changed", w.Code, w.Body)
+ }
+}
+
+func TestDiagnosticsHostedFailureOverridesInputWithoutParsingReason(t *testing.T) {
+ session := hostedFailureSession()
+ session.EnvironmentInputActivity = &store.EnvironmentInputActivity{Status: "failed", Failure: "environment_unavailable", LastActiveAt: time.Now()}
+ session.LastTurn = &store.Turn{ID: uuid.NewString(), SessionID: session.ID, Status: store.TurnFailed, Outcome: json.RawMessage(`{"error_code":"engine_failed"}`)}
+ step, index, exit := "setup", 2, 7
+ for _, detail := range []*store.ProvisioningFailureDetail{nil, {Step: &step, Index: &index, ExitCode: &exit}} {
+ session.EnvironmentFailure = &store.EnvironmentFailure{Reason: "private-secret-canary setup_commands[99] exit 254", Detail: detail}
+ h, _, _ := adminTestHandler(t, func(h *Handler) { h.store = diagnosticSnapshotStore{session: session} })
+ w := diagnosticRequest(h, adminSessionsPath+session.ID+"/diagnostics", "Bearer admin")
+ if w.Code != 200 || strings.Contains(w.Body.String(), "canary") || !strings.Contains(w.Body.String(), `"code":"environment_provisioning_failed"`) || !strings.Contains(w.Body.String(), `"source":"environment"`) {
+ t.Fatal(w.Code, w.Body)
+ }
+ want := `"params":{"exit_code":null,"index":null,"step":null}`
+ if detail != nil {
+ want = `"params":{"exit_code":7,"index":2,"step":"setup"}`
+ }
+ if !strings.Contains(w.Body.String(), want) || !strings.Contains(w.Body.String(), `"failed_at":null`) {
+ t.Fatal("historical reason parsed or time invented", w.Body)
+ }
+ }
+}
diff --git a/services/agents-api/internal/api/turn_diagnostic_failure.go b/services/agents-api/internal/api/turn_diagnostic_failure.go
new file mode 100644
index 000000000..31992df92
--- /dev/null
+++ b/services/agents-api/internal/api/turn_diagnostic_failure.go
@@ -0,0 +1,43 @@
+package api
+
+import (
+ "encoding/json"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store"
+)
+
+func turnDiagnosticFailure(turn store.Turn) *DiagnosticFailure {
+ if turn.Status != store.TurnFailed {
+ return nil
+ }
+ var outcome struct {
+ Code string `json:"error_code"`
+ }
+ if json.Unmarshal(turn.Outcome, &outcome) != nil {
+ outcome.Code = ""
+ }
+ code := "internal_error"
+ switch outcome.Code {
+ case "engine_failed":
+ code = "harness_error"
+ case "model_provider_required":
+ code = "model_provider_required"
+ case "execution_device_unavailable", "execution_unavailable":
+ code = "runtime_unavailable"
+ case "device_disconnected", "event_stream_incomplete":
+ code = "runtime_disconnected"
+ case "preparation_start_failed", "preparation_interrupted":
+ code = "runtime_preparation_failed"
+ case "execution_interrupted":
+ code = "execution_interrupted"
+ case "delivery_unknown", "input_outcome_unknown", "cancel_unconfirmed", "cancel_outcome_unavailable", "function_result_unconfirmed":
+ code = "delivery_unconfirmed"
+ case "invalid_input", "input_not_applied", "message_input_unsupported", "input_invalid_input", "input_run_inactive", "input_input_conflict", "input_input_limit", "input_unsupported", "input_rejected", "input_not_ready", "input_busy":
+ code = "input_rejected"
+ case "invalid_executor_result", "interaction_not_supported", "execution_state_unavailable", "execution_state_changed", "function_call_invalid", "function_result_invalid":
+ code = "executor_protocol_error"
+ case "event_persistence_failed", "artifact_capture_failed":
+ code = "core_storage_failed"
+ }
+ return &DiagnosticFailure{Code: code, Params: CoreErrorDetails{}, FailedAt: diagnosticTime(turn.CompletedAt)}
+}
diff --git a/services/agents-api/internal/api/turn_diagnostic_failure_test.go b/services/agents-api/internal/api/turn_diagnostic_failure_test.go
new file mode 100644
index 000000000..fe719d556
--- /dev/null
+++ b/services/agents-api/internal/api/turn_diagnostic_failure_test.go
@@ -0,0 +1,44 @@
+package api
+
+import (
+ "encoding/json"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store"
+)
+
+func TestDiagnosticFailureWhitelist(t *testing.T) {
+ cases := map[string][]string{
+ "harness_error": {"engine_failed"}, "model_provider_required": {"model_provider_required"}, "runtime_unavailable": {"execution_device_unavailable", "execution_unavailable"}, "runtime_disconnected": {"device_disconnected", "event_stream_incomplete"}, "runtime_preparation_failed": {"preparation_start_failed", "preparation_interrupted"}, "execution_interrupted": {"execution_interrupted"},
+ "delivery_unconfirmed": {"delivery_unknown", "input_outcome_unknown", "cancel_unconfirmed", "cancel_outcome_unavailable", "function_result_unconfirmed"},
+ "input_rejected": {"invalid_input", "input_not_applied", "message_input_unsupported", "input_invalid_input", "input_run_inactive", "input_input_conflict", "input_input_limit", "input_unsupported", "input_rejected", "input_not_ready", "input_busy"},
+ "executor_protocol_error": {"invalid_executor_result", "interaction_not_supported", "execution_state_unavailable", "execution_state_changed", "function_call_invalid", "function_result_invalid"}, "core_storage_failed": {"event_persistence_failed", "artifact_capture_failed"}, "internal_error": {"secret-canary", "input_secret-canary", "execution_state_secret-canary", ""},
+ }
+ catalog, err := os.ReadFile("../../../../contracts/agents-api/core-errors.md")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for want, inputs := range cases {
+ if !strings.Contains(string(catalog), "`"+want+"`") {
+ t.Fatal("uncatalogued diagnostics code", want)
+ }
+ for _, input := range inputs {
+ raw, _ := json.Marshal(map[string]string{"error_code": input, "error": "raw-secret-canary"})
+ got := turnDiagnosticFailure(store.Turn{Status: store.TurnFailed, Outcome: raw})
+ encoded, _ := json.Marshal(got)
+ if got.Code != want || strings.Contains(string(encoded), "canary") {
+ t.Fatal(input, got)
+ }
+ }
+ }
+ if got := turnDiagnosticFailure(store.Turn{Status: store.TurnFailed, Outcome: json.RawMessage(`{"error_code":"engine_failed",`)}); got.Code != "internal_error" {
+ t.Fatal("malformed outcome accepted", got)
+ }
+ for _, status := range []string{store.TurnQueued, store.TurnInProgress, store.TurnWaiting, store.TurnCompleted, store.TurnCancelled} {
+ if got := turnDiagnosticFailure(store.Turn{Status: status, Outcome: json.RawMessage(`{"error_code":"engine_failed"}`)}); got != nil {
+ t.Fatal("nonfailure classified", status, got)
+ }
+ }
+}
diff --git a/services/agents-api/internal/db/queries/environment_connections.sql b/services/agents-api/internal/db/queries/environment_connections.sql
index 044a8ec8c..d9461f24b 100644
--- a/services/agents-api/internal/db/queries/environment_connections.sql
+++ b/services/agents-api/internal/db/queries/environment_connections.sql
@@ -13,7 +13,7 @@ UPDATE environment_connections SET revision = $2 WHERE environment_id = $1;
UPDATE environments SET status = $2 WHERE id = $1;
-- name: RecordEnvironmentFailure :one
-UPDATE environments SET status = 'failed', failure_reason = $2, failed_at = clock_timestamp()
+UPDATE environments SET status = 'failed', failure_reason = $2, failure_detail = $3, failed_at = clock_timestamp()
WHERE id = $1 AND status NOT IN ('failed', 'expired')
RETURNING failed_at;
diff --git a/services/agents-api/internal/db/queries/session_events.sql b/services/agents-api/internal/db/queries/session_events.sql
index 486f3699d..25afdd5d0 100644
--- a/services/agents-api/internal/db/queries/session_events.sql
+++ b/services/agents-api/internal/db/queries/session_events.sql
@@ -30,9 +30,11 @@ DELETE FROM session_events e WHERE e.session_id = $1 AND e.sequence IN (
);
-- name: FinishSessionItems :many
-UPDATE session_items SET payload = jsonb_set(payload, '{status}', '"incomplete"')
+WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS at)
+UPDATE session_items SET payload = jsonb_set(payload, '{status}', '"incomplete"'), settled_at = COALESCE(session_items.settled_at, observed.at)
+FROM observed
WHERE session_id = $1 AND turn_id = $2 AND payload->>'status' = 'in_progress'
-RETURNING *;
+RETURNING session_items.*;
-- name: SessionEventTurn :one
SELECT * FROM turns WHERE session_id = $1 AND id = $2;
diff --git a/services/agents-api/internal/db/queries/session_items.sql b/services/agents-api/internal/db/queries/session_items.sql
index aef74dd39..b1c2c508c 100644
--- a/services/agents-api/internal/db/queries/session_items.sql
+++ b/services/agents-api/internal/db/queries/session_items.sql
@@ -2,13 +2,17 @@
SELECT * FROM session_items WHERE session_id = $1 AND id = $2;
-- name: PutSessionItem :one
-INSERT INTO session_items(id, session_id, turn_id, created_at, payload, position, output_index)
+INSERT INTO session_items(id, session_id, turn_id, created_at, payload, position, output_index, settled_at)
VALUES (sqlc.arg(id), sqlc.arg(session_id), sqlc.arg(turn_id), sqlc.arg(created_at), sqlc.arg(payload),
(SELECT COALESCE(max(position), -1) + 1 FROM session_items WHERE session_id = sqlc.arg(session_id)),
CASE WHEN sqlc.arg(is_output)::boolean THEN
(SELECT COALESCE(max(output_index), -1) + 1 FROM session_items WHERE turn_id = sqlc.arg(turn_id))
- END)
-ON CONFLICT (id) DO UPDATE SET payload = EXCLUDED.payload
+ END,
+ CASE WHEN sqlc.arg(payload)::jsonb->>'status' IN ('completed', 'incomplete', 'failed') THEN sqlc.arg(created_at)::timestamptz END)
+ON CONFLICT (id) DO UPDATE SET payload = EXCLUDED.payload,
+ settled_at = CASE WHEN session_items.payload->>'status' = 'in_progress'
+ AND EXCLUDED.payload->>'status' IN ('completed', 'incomplete', 'failed')
+ THEN COALESCE(session_items.settled_at, EXCLUDED.created_at) ELSE session_items.settled_at END
RETURNING *;
-- name: ListSessionItems :many
@@ -38,3 +42,9 @@ SELECT * FROM turn_events WHERE session_id = $1 AND turn_id = $2 AND ordinal >=
-- name: HasNativeMessageItem :one
SELECT EXISTS(SELECT 1 FROM session_items WHERE turn_id = $1
AND payload->>'role' = 'assistant' AND id <> $2);
+
+-- name: ListTurnItemDiagnostics :many
+SELECT id, created_at, settled_at FROM session_items
+WHERE session_id = $1 AND turn_id = $2
+ORDER BY created_at, position, id
+LIMIT 1001;
diff --git a/services/agents-api/internal/db/sqlc/environment_connections.sql.go b/services/agents-api/internal/db/sqlc/environment_connections.sql.go
index e9c31ad62..57ddde24f 100644
--- a/services/agents-api/internal/db/sqlc/environment_connections.sql.go
+++ b/services/agents-api/internal/db/sqlc/environment_connections.sql.go
@@ -82,7 +82,7 @@ func (q *Queries) ListEnvironmentConnections(ctx context.Context, id pgtype.UUID
}
const recordEnvironmentFailure = `-- name: RecordEnvironmentFailure :one
-UPDATE environments SET status = 'failed', failure_reason = $2, failed_at = clock_timestamp()
+UPDATE environments SET status = 'failed', failure_reason = $2, failure_detail = $3, failed_at = clock_timestamp()
WHERE id = $1 AND status NOT IN ('failed', 'expired')
RETURNING failed_at
`
@@ -90,10 +90,11 @@ RETURNING failed_at
type RecordEnvironmentFailureParams struct {
ID pgtype.UUID `json:"id"`
FailureReason pgtype.Text `json:"failure_reason"`
+ FailureDetail []byte `json:"failure_detail"`
}
func (q *Queries) RecordEnvironmentFailure(ctx context.Context, arg RecordEnvironmentFailureParams) (pgtype.Timestamptz, error) {
- row := q.db.QueryRow(ctx, recordEnvironmentFailure, arg.ID, arg.FailureReason)
+ row := q.db.QueryRow(ctx, recordEnvironmentFailure, arg.ID, arg.FailureReason, arg.FailureDetail)
var failed_at pgtype.Timestamptz
err := row.Scan(&failed_at)
return failed_at, err
diff --git a/services/agents-api/internal/db/sqlc/environments.sql.go b/services/agents-api/internal/db/sqlc/environments.sql.go
index 986ad4933..03b2d24a3 100644
--- a/services/agents-api/internal/db/sqlc/environments.sql.go
+++ b/services/agents-api/internal/db/sqlc/environments.sql.go
@@ -26,7 +26,7 @@ func (q *Queries) CreateEnvironment(ctx context.Context, arg CreateEnvironmentPa
}
const getEnvironment = `-- name: GetEnvironment :one
-SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration
+SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, e.failure_detail, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration
FROM environments e JOIN sessions s ON s.id = e.session_id
WHERE s.tenant_id = $1 AND e.id = $2 AND s.deleted_at IS NULL
`
@@ -52,6 +52,7 @@ func (q *Queries) GetEnvironment(ctx context.Context, arg GetEnvironmentParams)
&i.Environment.CreatedAt,
&i.Environment.FailureReason,
&i.Environment.FailedAt,
+ &i.Environment.FailureDetail,
&i.TenantID,
&i.Configuration,
)
@@ -59,7 +60,7 @@ func (q *Queries) GetEnvironment(ctx context.Context, arg GetEnvironmentParams)
}
const getSessionEnvironment = `-- name: GetSessionEnvironment :one
-SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration
+SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, e.failure_detail, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration
FROM environments e JOIN sessions s ON s.id = e.session_id
WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL
`
@@ -85,6 +86,7 @@ func (q *Queries) GetSessionEnvironment(ctx context.Context, arg GetSessionEnvir
&i.Environment.CreatedAt,
&i.Environment.FailureReason,
&i.Environment.FailedAt,
+ &i.Environment.FailureDetail,
&i.TenantID,
&i.Configuration,
)
diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go
index b093fcecf..f491fa1a7 100644
--- a/services/agents-api/internal/db/sqlc/models.go
+++ b/services/agents-api/internal/db/sqlc/models.go
@@ -83,6 +83,7 @@ type Environment struct {
CreatedAt pgtype.Timestamptz `json:"created_at"`
FailureReason pgtype.Text `json:"failure_reason"`
FailedAt pgtype.Timestamptz `json:"failed_at"`
+ FailureDetail []byte `json:"failure_detail"`
}
type EnvironmentConnection struct {
@@ -418,6 +419,7 @@ type SessionItem struct {
Position int32 `json:"position"`
Payload []byte `json:"payload"`
OutputIndex pgtype.Int4 `json:"output_index"`
+ SettledAt pgtype.Timestamptz `json:"settled_at"`
}
type SessionModelExecution struct {
diff --git a/services/agents-api/internal/db/sqlc/session_events.sql.go b/services/agents-api/internal/db/sqlc/session_events.sql.go
index c1290c507..491b41709 100644
--- a/services/agents-api/internal/db/sqlc/session_events.sql.go
+++ b/services/agents-api/internal/db/sqlc/session_events.sql.go
@@ -31,9 +31,11 @@ func (q *Queries) AppendSessionEvent(ctx context.Context, arg AppendSessionEvent
}
const finishSessionItems = `-- name: FinishSessionItems :many
-UPDATE session_items SET payload = jsonb_set(payload, '{status}', '"incomplete"')
+WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS at)
+UPDATE session_items SET payload = jsonb_set(payload, '{status}', '"incomplete"'), settled_at = COALESCE(session_items.settled_at, observed.at)
+FROM observed
WHERE session_id = $1 AND turn_id = $2 AND payload->>'status' = 'in_progress'
-RETURNING id, session_id, turn_id, created_at, position, payload, output_index
+RETURNING session_items.id, session_items.session_id, session_items.turn_id, session_items.created_at, session_items.position, session_items.payload, session_items.output_index, session_items.settled_at
`
type FinishSessionItemsParams struct {
@@ -58,6 +60,7 @@ func (q *Queries) FinishSessionItems(ctx context.Context, arg FinishSessionItems
&i.Position,
&i.Payload,
&i.OutputIndex,
+ &i.SettledAt,
); err != nil {
return nil, err
}
diff --git a/services/agents-api/internal/db/sqlc/session_items.sql.go b/services/agents-api/internal/db/sqlc/session_items.sql.go
index 5ef09514b..30e328fe2 100644
--- a/services/agents-api/internal/db/sqlc/session_items.sql.go
+++ b/services/agents-api/internal/db/sqlc/session_items.sql.go
@@ -12,7 +12,7 @@ import (
)
const getSessionItem = `-- name: GetSessionItem :one
-SELECT id, session_id, turn_id, created_at, position, payload, output_index FROM session_items WHERE session_id = $1 AND id = $2
+SELECT id, session_id, turn_id, created_at, position, payload, output_index, settled_at FROM session_items WHERE session_id = $1 AND id = $2
`
type GetSessionItemParams struct {
@@ -31,6 +31,7 @@ func (q *Queries) GetSessionItem(ctx context.Context, arg GetSessionItemParams)
&i.Position,
&i.Payload,
&i.OutputIndex,
+ &i.SettledAt,
)
return i, err
}
@@ -175,15 +176,57 @@ func (q *Queries) ListSessionItems(ctx context.Context, arg ListSessionItemsPara
return items, nil
}
+const listTurnItemDiagnostics = `-- name: ListTurnItemDiagnostics :many
+SELECT id, created_at, settled_at FROM session_items
+WHERE session_id = $1 AND turn_id = $2
+ORDER BY created_at, position, id
+LIMIT 1001
+`
+
+type ListTurnItemDiagnosticsParams struct {
+ SessionID pgtype.UUID `json:"session_id"`
+ TurnID pgtype.UUID `json:"turn_id"`
+}
+
+type ListTurnItemDiagnosticsRow struct {
+ ID pgtype.UUID `json:"id"`
+ CreatedAt pgtype.Timestamptz `json:"created_at"`
+ SettledAt pgtype.Timestamptz `json:"settled_at"`
+}
+
+func (q *Queries) ListTurnItemDiagnostics(ctx context.Context, arg ListTurnItemDiagnosticsParams) ([]ListTurnItemDiagnosticsRow, error) {
+ rows, err := q.db.Query(ctx, listTurnItemDiagnostics, arg.SessionID, arg.TurnID)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ items := []ListTurnItemDiagnosticsRow{}
+ for rows.Next() {
+ var i ListTurnItemDiagnosticsRow
+ if err := rows.Scan(&i.ID, &i.CreatedAt, &i.SettledAt); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
const putSessionItem = `-- name: PutSessionItem :one
-INSERT INTO session_items(id, session_id, turn_id, created_at, payload, position, output_index)
+INSERT INTO session_items(id, session_id, turn_id, created_at, payload, position, output_index, settled_at)
VALUES ($1, $2, $3, $4, $5,
(SELECT COALESCE(max(position), -1) + 1 FROM session_items WHERE session_id = $2),
CASE WHEN $6::boolean THEN
(SELECT COALESCE(max(output_index), -1) + 1 FROM session_items WHERE turn_id = $3)
- END)
-ON CONFLICT (id) DO UPDATE SET payload = EXCLUDED.payload
-RETURNING id, session_id, turn_id, created_at, position, payload, output_index
+ END,
+ CASE WHEN $5::jsonb->>'status' IN ('completed', 'incomplete', 'failed') THEN $4::timestamptz END)
+ON CONFLICT (id) DO UPDATE SET payload = EXCLUDED.payload,
+ settled_at = CASE WHEN session_items.payload->>'status' = 'in_progress'
+ AND EXCLUDED.payload->>'status' IN ('completed', 'incomplete', 'failed')
+ THEN COALESCE(session_items.settled_at, EXCLUDED.created_at) ELSE session_items.settled_at END
+RETURNING id, session_id, turn_id, created_at, position, payload, output_index, settled_at
`
type PutSessionItemParams struct {
@@ -213,6 +256,7 @@ func (q *Queries) PutSessionItem(ctx context.Context, arg PutSessionItemParams)
&i.Position,
&i.Payload,
&i.OutputIndex,
+ &i.SettledAt,
)
return i, err
}
diff --git a/services/agents-api/internal/store/provisioning_failure_detail.go b/services/agents-api/internal/store/provisioning_failure_detail.go
new file mode 100644
index 000000000..d3876916f
--- /dev/null
+++ b/services/agents-api/internal/store/provisioning_failure_detail.go
@@ -0,0 +1,42 @@
+package store
+
+// ProvisioningFailureDetail is private, fixed-category evidence from a confirmed
+// initialization receipt. It never contains command text, paths or Runtime output.
+type ProvisioningFailureDetail struct {
+ Step *string `json:"step"`
+ Index *int `json:"index"`
+ ExitCode *int `json:"exit_code"`
+}
+
+func (f ProvisioningFailure) detail() *ProvisioningFailureDetail {
+ return (ProvisioningFailureDetail{Step: &f.Step, Index: &f.Index, ExitCode: &f.ExitCode}).sanitized()
+}
+
+func (f ProvisioningFailureDetail) sanitized() *ProvisioningFailureDetail {
+ if f.Step == nil {
+ return nil
+ }
+ result := &ProvisioningFailureDetail{}
+ switch *f.Step {
+ case ProvisioningSetupCommand:
+ // JSON clients can represent these integer positions exactly.
+ if f.Index != nil && *f.Index >= 0 && int64(*f.Index) <= 9007199254740991 {
+ value := *f.Index
+ result.Index = &value
+ }
+ case ProvisioningPythonPackages, ProvisioningNPMPackages, ProvisioningSystemPackages:
+ case ProvisioningInitialFile, ProvisioningSkill:
+ value := *f.Step
+ result.Step = &value
+ return result
+ default:
+ return nil
+ }
+ value := *f.Step
+ result.Step = &value
+ if f.ExitCode != nil && *f.ExitCode > 0 && *f.ExitCode < 256 {
+ value := *f.ExitCode
+ result.ExitCode = &value
+ }
+ return result
+}
diff --git a/services/agents-api/internal/store/runtime_allocation_state.go b/services/agents-api/internal/store/runtime_allocation_state.go
index 10d7e390a..4fbf54fee 100644
--- a/services/agents-api/internal/store/runtime_allocation_state.go
+++ b/services/agents-api/internal/store/runtime_allocation_state.go
@@ -41,23 +41,23 @@ func (s *Store) SettleRuntimeCreation(ctx context.Context, owner RuntimeAllocati
// Cancellation requests do not prove existing native work has stopped. A live,
// unexpired Environment fails with the generic provisioning reason.
func (s *Store) RequestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation) (RuntimeAllocation, error) {
- return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason, false)
+ return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason, nil, false)
}
// FailRuntimeInitialization is RequestRuntimeCleanup after a confirmed failed
// initialization step: the step's safe reason becomes the Session error. Deleted
// Sessions, expired and already terminal Environments keep their existing outcome.
func (s *Store) FailRuntimeInitialization(ctx context.Context, owner RuntimeAllocation, failure ProvisioningFailure) (RuntimeAllocation, error) {
- return s.requestRuntimeCleanup(ctx, owner, failure.reason(), false)
+ return s.requestRuntimeCleanup(ctx, owner, failure.reason(), failure.detail(), false)
}
// ReleaseAbsentRuntimeCreation consumes provider proof that the original attempt
// is settled and owns no resources. Authority revocation and release commit together.
func (s *Store) ReleaseAbsentRuntimeCreation(ctx context.Context, owner RuntimeAllocation) (RuntimeAllocation, error) {
- return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason, true)
+ return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason, nil, true)
}
-func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation, reason string, absent bool) (RuntimeAllocation, error) {
+func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation, reason string, detail *ProvisioningFailureDetail, absent bool) (RuntimeAllocation, error) {
return s.mutateRuntimeAllocation(ctx, owner, false, func(ctx context.Context, q *sqlc.Queries, row sqlc.RuntimeAllocation) (sqlc.RuntimeAllocation, error) {
if row.State == "released" {
return row, nil
@@ -74,7 +74,7 @@ func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocati
if current.DeletedAt.Valid {
err = cancel()
} else {
- err = terminateRuntimeEnvironment(ctx, q, current, reason, cancel)
+ err = terminateRuntimeEnvironment(ctx, q, current, reason, detail, cancel)
}
if err != nil {
return sqlc.RuntimeAllocation{}, err
diff --git a/services/agents-api/internal/store/runtime_environment_terminal.go b/services/agents-api/internal/store/runtime_environment_terminal.go
index 06e5d8827..4a1414935 100644
--- a/services/agents-api/internal/store/runtime_environment_terminal.go
+++ b/services/agents-api/internal/store/runtime_environment_terminal.go
@@ -2,6 +2,7 @@ package store
import (
"context"
+ "encoding/json"
"fmt"
"time"
@@ -61,29 +62,35 @@ func (f ProvisioningFailure) reason() string {
// EnvironmentFailure is a hosted Environment's recorded provisioning failure. It
// makes the Session failed with this reason and last activity time.
type EnvironmentFailure struct {
- Reason string `json:"reason"`
- FailedAt time.Time `json:"failed_at"`
+ Reason string `json:"reason"`
+ FailedAt time.Time `json:"failed_at"`
+ Detail *ProvisioningFailureDetail `json:"-"`
}
func environmentFailure(row sqlc.Environment) *EnvironmentFailure {
if row.Status != "failed" || !row.FailureReason.Valid || !row.FailedAt.Valid {
return nil
}
- return &EnvironmentFailure{Reason: row.FailureReason.String, FailedAt: row.FailedAt.Time}
+ failure := &EnvironmentFailure{Reason: row.FailureReason.String, FailedAt: row.FailedAt.Time}
+ var detail ProvisioningFailureDetail
+ if json.Unmarshal(row.FailureDetail, &detail) == nil {
+ failure.Detail = detail.sanitized()
+ }
+ return failure
}
// terminateRuntimeEnvironment participates in the allocation's Session transaction.
// Public expiry does not assert compute removal or invent an expired SSE variant.
// A first failure records the hosted provisioning failure; an already terminal
// Environment only settles remaining input, without repeating events.
-func terminateRuntimeEnvironment(ctx context.Context, q *sqlc.Queries, current sqlc.GetRuntimeAllocationRow, reason string, cancel func() error) error {
+func terminateRuntimeEnvironment(ctx context.Context, q *sqlc.Queries, current sqlc.GetRuntimeAllocationRow, reason string, detail *ProvisioningFailureDetail, cancel func() error) error {
row, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: current.TenantID, ID: current.SessionID})
if err != nil {
return err
}
terminal := row.Environment.Status == "expired" || row.Environment.Status == "failed"
if !terminal && !current.Expired {
- return failHostedEnvironment(ctx, q, row, current.SessionID, reason, cancel)
+ return failHostedEnvironment(ctx, q, row, current.SessionID, reason, detail, cancel)
}
return withEnvironmentInputActivity(ctx, q, current.SessionID, func() error {
if !terminal {
@@ -103,8 +110,16 @@ func terminateRuntimeEnvironment(ctx context.Context, q *sqlc.Queries, current s
// safe reason, then one agent.session.failed snapshot. The snapshot captures the
// settled input activity, Usage and the failure, matching later Session reads.
// Pending input settles as failed exactly as before.
-func failHostedEnvironment(ctx context.Context, q *sqlc.Queries, row sqlc.GetSessionEnvironmentRow, session pgtype.UUID, reason string, cancel func() error) error {
- failedAt, err := q.RecordEnvironmentFailure(ctx, sqlc.RecordEnvironmentFailureParams{ID: row.Environment.ID, FailureReason: pgtype.Text{String: reason, Valid: true}})
+func failHostedEnvironment(ctx context.Context, q *sqlc.Queries, row sqlc.GetSessionEnvironmentRow, session pgtype.UUID, reason string, detail *ProvisioningFailureDetail, cancel func() error) error {
+ var rawDetail []byte
+ if detail != nil {
+ var err error
+ rawDetail, err = json.Marshal(detail)
+ if err != nil {
+ return err
+ }
+ }
+ failedAt, err := q.RecordEnvironmentFailure(ctx, sqlc.RecordEnvironmentFailureParams{ID: row.Environment.ID, FailureReason: pgtype.Text{String: reason, Valid: true}, FailureDetail: rawDetail})
if err != nil {
return err
}
diff --git a/services/agents-api/internal/store/sandbox_generations_test.go b/services/agents-api/internal/store/sandbox_generations_test.go
index 8f78c8205..ab9165d6c 100644
--- a/services/agents-api/internal/store/sandbox_generations_test.go
+++ b/services/agents-api/internal/store/sandbox_generations_test.go
@@ -267,6 +267,11 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) {
if _, err = migrations.DownTo(t.Context(), 80); err == nil {
t.Fatal("downgrade erased old owned allocation")
}
+ // Earlier down migrations can commit before the generation guard vetoes
+ // downgrade. Restore the current schema before invoking current Store code.
+ if _, err = migrations.Up(t.Context()); err != nil {
+ t.Fatal(err)
+ }
if _, err = w.RequestRuntimeCleanup(t.Context(), owner); err != nil {
t.Fatal(err)
}
diff --git a/services/agents-api/internal/store/session_diagnostics.go b/services/agents-api/internal/store/session_diagnostics.go
new file mode 100644
index 000000000..11ea8f9bc
--- /dev/null
+++ b/services/agents-api/internal/store/session_diagnostics.go
@@ -0,0 +1,100 @@
+package store
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+)
+
+// ItemDiagnosticTiming records Core database receipt and settlement, never native
+// execution duration. Historical terminal Items can have unknown settlement.
+type ItemDiagnosticTiming struct {
+ ItemID string
+ StartedAt time.Time
+ CompletedAt *time.Time
+}
+
+type TurnDiagnosticsSnapshot struct {
+ Session Session
+ Turn Turn
+ Items []ItemDiagnosticTiming
+ ItemsTruncated bool
+}
+
+// GetSessionDiagnosticsSnapshot keeps all existing Session projections in one
+// read-only snapshot, including the failure precedence used by the public API.
+func (s *Store) GetSessionDiagnosticsSnapshot(ctx context.Context, tenantID, sessionID string) (Session, error) {
+ tenant, err := parseID(tenantID)
+ if err != nil {
+ return Session{}, err
+ }
+ var session Session
+ err = pgx.BeginTxFunc(ctx, s.pool, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}, func(tx pgx.Tx) error {
+ q := s.queries.WithTx(tx)
+ row, err := q.GetSession(ctx, sqlc.GetSessionParams{TenantID: tenant, ID: parsePathID(sessionID)})
+ if err != nil {
+ return err
+ }
+ session, err = sessionFromRow(row)
+ if err != nil {
+ return err
+ }
+ session, err = readSessionActivity(ctx, q, session)
+ return err
+ })
+ if errors.Is(err, pgx.ErrNoRows) {
+ return Session{}, ErrNotFound
+ }
+ return session, err
+}
+
+// GetTurnDiagnosticsSnapshot reads only root Turns and their root Items. Its
+// bounded query and public projection share the same committed snapshot.
+func (s *Store) GetTurnDiagnosticsSnapshot(ctx context.Context, tenantID, sessionID, turnID string) (TurnDiagnosticsSnapshot, error) {
+ params, err := publicTurnLookup(tenantID, sessionID, turnID)
+ if err != nil {
+ return TurnDiagnosticsSnapshot{}, err
+ }
+ result := TurnDiagnosticsSnapshot{Items: []ItemDiagnosticTiming{}}
+ err = pgx.BeginTxFunc(ctx, s.pool, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}, func(tx pgx.Tx) error {
+ q := s.queries.WithTx(tx)
+ turn, err := q.GetTurn(ctx, params)
+ if err != nil {
+ return err
+ }
+ row, err := q.GetSession(ctx, sqlc.GetSessionParams{TenantID: params.TenantID, ID: params.SessionID})
+ if err != nil {
+ return err
+ }
+ result.Session, err = sessionFromRow(row)
+ if err != nil {
+ return err
+ }
+ result.Turn = turnFromRow(turn)
+ rows, err := q.ListTurnItemDiagnostics(ctx, sqlc.ListTurnItemDiagnosticsParams{SessionID: params.SessionID, TurnID: params.ID})
+ if err != nil {
+ return err
+ }
+ result.ItemsTruncated = len(rows) > 1000
+ if result.ItemsTruncated {
+ rows = rows[:1000]
+ }
+ for _, row := range rows {
+ item := ItemDiagnosticTiming{ItemID: uuid.UUID(row.ID.Bytes).String(), StartedAt: row.CreatedAt.Time}
+ if row.SettledAt.Valid {
+ at := row.SettledAt.Time
+ item.CompletedAt = &at
+ }
+ result.Items = append(result.Items, item)
+ }
+ return nil
+ })
+ if errors.Is(err, pgx.ErrNoRows) {
+ return TurnDiagnosticsSnapshot{}, ErrNotFound
+ }
+ return result, err
+}
diff --git a/services/agents-api/internal/store/session_diagnostics_cancellation_test.go b/services/agents-api/internal/store/session_diagnostics_cancellation_test.go
new file mode 100644
index 000000000..f4aae5f19
--- /dev/null
+++ b/services/agents-api/internal/store/session_diagnostics_cancellation_test.go
@@ -0,0 +1,93 @@
+package store
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+)
+
+func TestDiagnosticSnapshotCancellationReleasesTransaction(t *testing.T) {
+ for _, rootTurn := range []bool{false, true} {
+ name, table := "session", "environments"
+ if rootTurn {
+ name, table = "turn", "session_items"
+ }
+ t.Run(name, func(t *testing.T) {
+ s, pool := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "input")
+ blocker, err := pool.Begin(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer blocker.Rollback(context.Background())
+ if _, err = blocker.Exec(t.Context(), "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); err != nil {
+ t.Fatal(err)
+ }
+ // The first Session/Turn lookup establishes a snapshot before the blocked
+ // relation read. A short caller deadline must release it while the lock stays held.
+ ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ var err error
+ if rootTurn {
+ _, err = s.GetTurnDiagnosticsSnapshot(ctx, tenant, session.ID, receipt.TurnID)
+ } else {
+ _, err = s.GetSessionDiagnosticsSnapshot(ctx, tenant, session.ID)
+ }
+ done <- err
+ }()
+ var readerPID int
+ for readerPID == 0 {
+ err = pool.QueryRow(t.Context(), `SELECT COALESCE((SELECT pid FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND backend_xmin IS NOT NULL AND query LIKE $1 LIMIT 1),0)`, "%FROM "+table+"%").Scan(&readerPID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if readerPID != 0 {
+ break
+ }
+ select {
+ case err := <-done:
+ t.Fatal("snapshot did not reach the blocked read", err)
+ case <-time.After(5 * time.Millisecond):
+ }
+ }
+ select {
+ case err := <-done:
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatal("short caller deadline was not preserved", err)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("snapshot read ignored cancellation")
+ }
+ cleanup, stop := context.WithTimeout(t.Context(), 2*time.Second)
+ defer stop()
+ for {
+ var retained bool
+ err = pool.QueryRow(cleanup, "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND xact_start IS NOT NULL)", readerPID).Scan(&retained)
+ if err != nil {
+ t.Fatal("cancelled snapshot retained its transaction", err)
+ }
+ if !retained {
+ break
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ // Cleanup is checked before releasing the blocker: no reader transaction
+ // may depend on the conflicting lock becoming available for cleanup.
+ if err = blocker.Rollback(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if rootTurn {
+ _, err = s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ } else {
+ _, err = s.GetSessionDiagnosticsSnapshot(t.Context(), tenant, session.ID)
+ }
+ if err != nil {
+ t.Fatal("snapshot read did not recover", err)
+ }
+ })
+ }
+}
diff --git a/services/agents-api/internal/store/session_diagnostics_test.go b/services/agents-api/internal/store/session_diagnostics_test.go
new file mode 100644
index 000000000..24c440e24
--- /dev/null
+++ b/services/agents-api/internal/store/session_diagnostics_test.go
@@ -0,0 +1,366 @@
+package store
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device"
+ "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5/pgtype"
+)
+
+func diagnosticToolEvent(id, stage, status string) ExecutionEvent {
+ return ExecutionEvent{Kind: "tool_call", Payload: json.RawMessage(fmt.Sprintf(`{"id":%q,"stage":%q,"observation":{"kind":"command","command":"private-command-canary","status":%q}}`, id, stage, status))}
+}
+
+func TestDiagnosticItemReceiptSettlementAndReplay(t *testing.T) {
+ s, _ := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "start")
+ transition(t, s, tenant, session.ID, receipt.TurnID, TurnQueued, TurnInProgress)
+ before := diagnosticToolEvent("cmd", "before", "in_progress")
+ after := diagnosticToolEvent("cmd", "after", "failed")
+ for i, event := range []ExecutionEvent{before, after} {
+ if err := s.AppendTurnEvents(t.Context(), tenant, session.ID, receipt.TurnID, int32(i+1), []ExecutionEvent{event}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ snap, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil || len(snap.Items) != 2 {
+ t.Fatal(snap, err)
+ }
+ if snap.Items[0].CompletedAt == nil || !snap.Items[0].CompletedAt.Equal(snap.Items[0].StartedAt) {
+ t.Fatal("terminal input did not settle at its receipt", snap.Items[0])
+ }
+ item := snap.Items[1]
+ var received time.Time
+ if err := s.pool.QueryRow(t.Context(), "SELECT created_at FROM turn_events WHERE turn_id=$1 AND ordinal=2", receipt.TurnID).Scan(&received); err != nil {
+ t.Fatal(err)
+ }
+ if item.CompletedAt == nil || !item.CompletedAt.Equal(received) || item.CompletedAt.Before(item.StartedAt) {
+ t.Fatal("Item did not use terminal receipt", item, received)
+ }
+ if err := s.AppendTurnEvents(t.Context(), tenant, session.ID, receipt.TurnID, 2, []ExecutionEvent{after}); err != nil {
+ t.Fatal(err)
+ }
+ // A terminal legacy Item with unknown settlement must remain unknown even on a repeated upsert.
+ runtimeSuspensionSQL(t, s.pool, "UPDATE session_items SET settled_at=NULL WHERE id=$1", item.ItemID)
+ if err := s.AppendTurnEvents(t.Context(), tenant, session.ID, receipt.TurnID, 3, []ExecutionEvent{after}); err != nil {
+ t.Fatal(err)
+ }
+ transition(t, s, tenant, session.ID, receipt.TurnID, TurnInProgress, TurnFailed)
+ snap, err = s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil || snap.Items[1].CompletedAt != nil {
+ t.Fatal("historical settlement synthesized", snap, err)
+ }
+}
+
+func TestDiagnosticForceSettlementIgnoresNativeClock(t *testing.T) {
+ for _, skew := range []time.Duration{-269 * time.Second, 269 * time.Second} {
+ t.Run(skew.String(), func(t *testing.T) {
+ s, w, owner := managedIdleClockFixture(t)
+ turn := uuid.NewString()
+ runtimeSuspensionSQL(t, s.pool, "INSERT INTO turns(id,session_id,status,started_at) VALUES($1,$2,'in_progress',clock_timestamp())", turn, owner.SessionID)
+ events := []ExecutionEvent{diagnosticToolEvent("first", "before", "in_progress"), diagnosticToolEvent("second", "before", "in_progress")}
+ if err := w.AppendTurnEvents(t.Context(), owner.TenantID, owner.SessionID, turn, 1, events); err != nil {
+ t.Fatal(err)
+ }
+ source := runtimeDatabaseTime(t, s).Add(skew).UnixMilli()
+ before := runtimeDatabaseTime(t, s)
+ completed, err := w.CompleteExecution(t.Context(), owner.TenantID, owner.SessionID, turn, TurnCompleted, json.RawMessage(fmt.Sprintf(`{"done":{"source_completed_at_ms":%d}}`, source)), "", 0)
+ after := runtimeDatabaseTime(t, s)
+ if err != nil || completed.CompletedAt.UnixMilli() != source {
+ t.Fatal("public native completion changed", completed, err)
+ }
+ snap, err := s.GetTurnDiagnosticsSnapshot(t.Context(), owner.TenantID, owner.SessionID, turn)
+ if err != nil || len(snap.Items) != 2 {
+ t.Fatal(snap, err)
+ }
+ for _, item := range snap.Items {
+ if item.CompletedAt == nil || item.CompletedAt.Before(before) || item.CompletedAt.After(after) || item.CompletedAt.Before(item.StartedAt) {
+ t.Fatal("force settlement used remote or transaction-start clock", item)
+ }
+ }
+ if !snap.Items[0].CompletedAt.Equal(*snap.Items[1].CompletedAt) {
+ t.Fatal("force settlement has multiple clocks")
+ }
+ })
+ }
+}
+
+func TestDiagnosticSettlementWaitsForSessionLock(t *testing.T) {
+ s, pool := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "start")
+ transition(t, s, tenant, session.ID, receipt.TurnID, TurnQueued, TurnInProgress)
+ tx, err := pool.Begin(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer tx.Rollback(context.Background())
+ if _, err = tx.Exec(t.Context(), "SELECT id FROM sessions WHERE id=$1 FOR UPDATE", session.ID); err != nil {
+ t.Fatal(err)
+ }
+ done := make(chan error, 1)
+ go func() {
+ _, err := s.TransitionTurn(t.Context(), tenant, session.ID, receipt.TurnID, TurnTransition{ExpectedStatus: TurnInProgress, Status: TurnFailed})
+ done <- err
+ }()
+ // Wait for the actual competing transaction to block, not a scheduler delay.
+ deadline := time.After(5 * time.Second)
+ for {
+ var waiting bool
+ err = pool.QueryRow(t.Context(), "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND query LIKE '%sessions%')").Scan(&waiting)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if waiting {
+ break
+ }
+ select {
+ case err := <-done:
+ t.Fatal("terminal transaction did not wait", err)
+ case <-deadline:
+ t.Fatal("terminal transaction never reached lock")
+ case <-time.After(10 * time.Millisecond):
+ }
+ }
+ itemID := uuid.NewString()
+ _, err = tx.Exec(t.Context(), "INSERT INTO session_items(id,session_id,turn_id,created_at,position,payload) VALUES($1,$2,$3,clock_timestamp(),1,$4)", itemID, session.ID, receipt.TurnID, `{"id":"`+itemID+`","turn_id":"`+receipt.TurnID+`","type":"command_execution","status":"in_progress","command":"run"}`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = tx.Commit(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if err = <-done; err != nil {
+ t.Fatal(err)
+ }
+ snap, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ item := snap.Items[len(snap.Items)-1]
+ if item.CompletedAt == nil || item.CompletedAt.Before(item.StartedAt) {
+ t.Fatal("transaction-start settlement predates serialized Item receipt", item)
+ }
+}
+
+func TestDiagnosticTimingBoundOrderAndIsolation(t *testing.T) {
+ s, pool := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "start")
+ // Insert in reverse position order with equal times; response must follow public Item ordering.
+ _, err := pool.Exec(t.Context(), `INSERT INTO session_items(id,session_id,turn_id,created_at,position,payload)
+ SELECT gen_random_uuid(),$1,$2,'2020-01-01T00:00:00Z'::timestamptz,n,'{"status":"completed"}'::jsonb FROM generate_series(1000,1,-1) n`, session.ID, receipt.TurnID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ snap, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil || len(snap.Items) != 1000 || !snap.ItemsTruncated {
+ t.Fatal("unbounded diagnostics", len(snap.Items), snap.ItemsTruncated, err)
+ }
+ for i, item := range snap.Items {
+ var position int
+ if err := pool.QueryRow(t.Context(), "SELECT position FROM session_items WHERE id=$1", item.ItemID).Scan(&position); err != nil {
+ t.Fatal(err)
+ }
+ if position != i+1 || item.CompletedAt != nil {
+ t.Fatal("order or historical settlement changed", position, item)
+ }
+ }
+ runtimeSuspensionSQL(t, pool, "DELETE FROM session_items WHERE turn_id=$1 AND created_at>'2020-01-01T00:00:00Z'", receipt.TurnID)
+ exact, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil || len(exact.Items) != 1000 || exact.ItemsTruncated {
+ t.Fatal("exact limit falsely truncated", len(exact.Items), exact.ItemsTruncated, err)
+ }
+ for _, ids := range [][3]string{{uuid.NewString(), session.ID, receipt.TurnID}, {tenant, "malformed", receipt.TurnID}, {tenant, session.ID, uuid.NewString()}} {
+ if _, err := s.GetTurnDiagnosticsSnapshot(t.Context(), ids[0], ids[1], ids[2]); !errors.Is(err, ErrNotFound) {
+ t.Fatal("scope leaked", ids, err)
+ }
+ }
+ runtimeSuspensionSQL(t, pool, "UPDATE sessions SET deleted_at=clock_timestamp() WHERE id=$1", session.ID)
+ if _, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID); !errors.Is(err, ErrNotFound) {
+ t.Fatal("deleted root visible", err)
+ }
+ if _, err := s.GetSessionDiagnosticsSnapshot(t.Context(), tenant, session.ID); !errors.Is(err, ErrNotFound) {
+ t.Fatal("deleted Session visible", err)
+ }
+}
+
+func TestDiagnosticProvisioningDetailAtomicAndPrivate(t *testing.T) {
+ s, pool := testStore(t)
+ tenant := uuid.NewString()
+ input := environmentInput("safe-detail", "openai_hosted", "/workspace")
+ input.InitialInputs = []Input{messageInput("initial")}
+ session, err := s.CreateSession(t.Context(), tenant, input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ writer := executionLease(t, s).Store()
+ owner, err := writer.ReserveRuntimeAllocation(t.Context(), tenant, session.Environment.ID, uuid.NewString(), device.HashCredential(uuid.NewString()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ constraint := "diagnostics_" + strings.ReplaceAll(uuid.NewString(), "-", "")
+ _, err = pool.Exec(t.Context(), "ALTER TABLE session_events ADD CONSTRAINT "+constraint+" CHECK (session_id <> '"+session.ID+"' OR payload->'event'->>'type' <> 'agent.session.failed') NOT VALID")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ _, _ = pool.Exec(context.Background(), "ALTER TABLE session_events DROP CONSTRAINT IF EXISTS "+constraint)
+ })
+ failure := ProvisioningFailure{Step: ProvisioningSetupCommand, Index: 2, ExitCode: 7}
+ if _, err = writer.FailRuntimeInitialization(t.Context(), owner, failure); err == nil {
+ t.Fatal("failure committed without events")
+ }
+ var detail []byte
+ if err = pool.QueryRow(t.Context(), "SELECT failure_detail FROM environments WHERE id=$1", owner.EnvironmentID).Scan(&detail); err != nil || detail != nil {
+ t.Fatal("partial failure detail", string(detail), err)
+ }
+ runtimeSuspensionSQL(t, pool, "ALTER TABLE session_events DROP CONSTRAINT "+constraint)
+ if _, err = writer.FailRuntimeInitialization(t.Context(), owner, failure); err != nil {
+ t.Fatal(err)
+ }
+ snap, err := s.GetSessionDiagnosticsSnapshot(t.Context(), tenant, session.ID)
+ if err != nil || snap.EnvironmentFailure == nil || snap.EnvironmentFailure.Detail == nil || *snap.EnvironmentFailure.Detail.Index != 2 || *snap.EnvironmentFailure.Detail.ExitCode != 7 {
+ t.Fatal("detail not persisted", snap.EnvironmentFailure, err)
+ }
+ events, err := s.ListSessionEvents(t.Context(), tenant, session.ID, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, _ := json.Marshal(events)
+ if strings.Contains(string(raw), "exit_code") || strings.Contains(string(raw), "failure_detail") {
+ t.Fatal("private fields entered SSE", string(raw))
+ }
+ // Historical reasons are never parsed into structured detail; corrupted private fields are sanitized.
+ runtimeSuspensionSQL(t, pool, "UPDATE environments SET failure_detail=$2 WHERE id=$1", owner.EnvironmentID, `{"step":"secret-provider-token","index":3,"exit_code":2}`)
+ snap, err = s.GetSessionDiagnosticsSnapshot(t.Context(), tenant, session.ID)
+ if err != nil || snap.EnvironmentFailure.Detail != nil {
+ t.Fatal("unsafe private detail projected", snap.EnvironmentFailure, err)
+ }
+}
+
+func TestDiagnosticPutItemRollsBack(t *testing.T) {
+ s, pool := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "start")
+ tx, err := pool.Begin(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := parseID(uuid.NewString())
+ sid, _ := parseID(session.ID)
+ tid, _ := parseID(receipt.TurnID)
+ _, err = sqlc.New(tx).PutSessionItem(t.Context(), sqlc.PutSessionItemParams{ID: id, SessionID: sid, TurnID: tid, CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, Payload: []byte(`{"status":"completed"}`)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = tx.Rollback(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ snap, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil || len(snap.Items) != 1 {
+ t.Fatal("rolled back receipt visible", snap, err)
+ }
+}
+
+func TestDiagnosticFirstSettlementSurvivesStoredStatusRegression(t *testing.T) {
+ s, pool := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "start")
+ transition(t, s, tenant, session.ID, receipt.TurnID, TurnQueued, TurnInProgress)
+ events := []ExecutionEvent{diagnosticToolEvent("cmd", "before", "in_progress"), diagnosticToolEvent("cmd", "after", "completed")}
+ if err := s.AppendTurnEvents(t.Context(), tenant, session.ID, receipt.TurnID, 1, events); err != nil {
+ t.Fatal(err)
+ }
+ first, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ item := first.Items[1]
+ if item.CompletedAt == nil {
+ t.Fatal("missing first settlement")
+ }
+ // Exercise persistence defensively even if a producer bypasses the normal
+ // Item merge's terminal-status preservation.
+ for _, force := range []bool{false, true} {
+ runtimeSuspensionSQL(t, pool, "UPDATE session_items SET payload=jsonb_set(payload,'{status}','\"in_progress\"') WHERE id=$1", item.ItemID)
+ if force {
+ transition(t, s, tenant, session.ID, receipt.TurnID, TurnInProgress, TurnFailed)
+ } else if err := s.AppendTurnEvents(t.Context(), tenant, session.ID, receipt.TurnID, 3, events[1:]); err != nil {
+ t.Fatal(err)
+ }
+ got, err := s.GetTurnDiagnosticsSnapshot(t.Context(), tenant, session.ID, receipt.TurnID)
+ if err != nil || got.Items[1].CompletedAt == nil || !got.Items[1].CompletedAt.Equal(*item.CompletedAt) {
+ t.Fatal("first settlement overwritten", force, got, err)
+ }
+ }
+}
+
+func TestDiagnosticRootReadRejectsActualChildTurn(t *testing.T) {
+ s, w, owner := managedIdleClockFixture(t)
+ root := runtimeSuspensionCompleted(t, s.pool, owner)
+ child, turn := uuid.NewString(), uuid.NewString()
+ runtimeSuspensionSQL(t, s.pool, `INSERT INTO turn_events(session_id,turn_id,ordinal,kind,payload) VALUES($1,$2,1,'subagent','{}')`, owner.SessionID, root)
+ runtimeSuspensionSQL(t, s.pool, `INSERT INTO subagent_identities(id,session_id,device_id,engine,native_id,parent_native_id,native_created_at,first_turn_id,first_event_ordinal) VALUES($1,$2,$3,'codex','child','root',1,$4,1)`, child, owner.SessionID, owner.DeviceID, root)
+ runtimeSuspensionSQL(t, s.pool, `INSERT INTO subagent_turns(id,session_id,subagent_id,native_id,status,created_at) VALUES($1,$2,$3,'child-turn','in_progress',clock_timestamp())`, turn, owner.SessionID, child)
+ if _, err := w.GetTurnDiagnosticsSnapshot(t.Context(), owner.TenantID, owner.SessionID, turn); !errors.Is(err, ErrNotFound) {
+ t.Fatal("child Turn became root diagnostics", err)
+ }
+}
+
+func TestDiagnosticSessionSnapshotConcurrentCommitConsistency(t *testing.T) {
+ s, pool := testStore(t)
+ tenant, session := newTurnSession(t, s)
+ receipt := submitMessage(t, s, tenant, session.ID, "start")
+ runtimeSuspensionSQL(t, pool, `UPDATE sessions SET metadata='{"generation":"0"}' WHERE id=$1`, session.ID)
+ runtimeSuspensionSQL(t, pool, `UPDATE turns SET outcome='{"generation":"0"}' WHERE id=$1`, receipt.TurnID)
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ for i := 1; i <= 150; i++ {
+ tx, err := pool.Begin(ctx)
+ if err != nil {
+ done <- err
+ return
+ }
+ raw := fmt.Sprintf(`{"generation":"%d"}`, i)
+ _, err = tx.Exec(ctx, "UPDATE sessions SET metadata=$2 WHERE id=$1", session.ID, raw)
+ if err == nil {
+ _, err = tx.Exec(ctx, "UPDATE turns SET outcome=$2 WHERE id=$1", receipt.TurnID, raw)
+ }
+ if err != nil {
+ _ = tx.Rollback(context.Background())
+ done <- err
+ return
+ }
+ if err = tx.Commit(ctx); err != nil {
+ done <- err
+ return
+ }
+ }
+ done <- nil
+ }()
+ for i := 0; i < 150; i++ {
+ snapshot, err := s.GetSessionDiagnosticsSnapshot(t.Context(), tenant, session.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var outcome map[string]string
+ if snapshot.LastTurn == nil || json.Unmarshal(snapshot.LastTurn.Outcome, &outcome) != nil || outcome["generation"] != snapshot.Metadata["generation"] {
+ t.Fatal("mixed committed snapshots", snapshot.Metadata, outcome)
+ }
+ }
+ if err := <-done; err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/services/agents-api/migrations/000083_session_diagnostics.sql b/services/agents-api/migrations/000083_session_diagnostics.sql
new file mode 100644
index 000000000..80be75a10
--- /dev/null
+++ b/services/agents-api/migrations/000083_session_diagnostics.sql
@@ -0,0 +1,8 @@
+-- +goose Up
+-- Historical terminal Items retain unknown settlement times.
+ALTER TABLE session_items ADD COLUMN settled_at timestamptz;
+ALTER TABLE environments ADD COLUMN failure_detail jsonb CHECK (failure_detail IS NULL OR jsonb_typeof(failure_detail) = 'object');
+
+-- +goose Down
+ALTER TABLE environments DROP COLUMN failure_detail;
+ALTER TABLE session_items DROP COLUMN settled_at;