diff --git a/UPSTREAM.json b/UPSTREAM.json index ab1fdcb..a2f24c1 100644 --- a/UPSTREAM.json +++ b/UPSTREAM.json @@ -57,6 +57,8 @@ "runtime/capabilities.go": "76c27bdd7ddfa11a1d639915ac2eb2d573b7ccdd687a5f548b3731b7c1e1828f", "runtime/descriptor_unix.go": "df41b6867e9840933f186c93b6bc61861e7ea5252a365455686ea0414bfa0044", "runtime/descriptor_windows.go": "2d9ffefe3af0154fa8042de6b67460d4e86dd3f4cdd9e986f180f7d0c535c9a5", + "runtime/request.go": "198c44fd6c547022a15b6d0d48e4d0130fa8afb687994365115576e4d874550d", + "runtime/request_test.go": "86d8a2feb4ec72e8ed675b9567da2d1f5d628950eeec907c10b9cc1675aa1904", "runtime/runtime.go": "e0624a16ac9f79e8080246042912c73b1588f1b9aeb4f9f95b3711c7acef7e81", "runtime/runtime_test.go": "afd78d122af20148bf30d0db873ff002544189df0dfec0f6167b8cf5cd0d42b1", "runtime/transport.go": "83e2218fb28474e875dafa6943bc5b665acef0565aaf5955fa88b1b4fd21614e", @@ -115,5 +117,11 @@ "windtunnel/doc.go": "44e0bcde632da73e1f8b98beade3a34ca8e0d0ea79cdfb91d131de290b164fc4", "windtunnel/windtunnel_test.go": "d34929ffdb1927b2ee27cc79640b333b0571659d231977b91c3a8898cf79bc42" }, - "schema_version": 1 + "generator": "operatorstack/pitot:project", + "schema_version": 1, + "source": { + "commit": "f3e44076ff395a608f025c7162627e44e795f55f", + "path": "labs/15-pitot", + "repository": "operatorstack/intelligence-flow" + } } diff --git a/adapter-verification.json b/adapter-verification.json new file mode 100644 index 0000000..86ac44b --- /dev/null +++ b/adapter-verification.json @@ -0,0 +1,24 @@ +{ + "schema_version": 6, + "capabilities": [ + {"id": "hook_control", "matrix": "agent_platform"}, + {"id": "consumer_delivery", "matrix": "agent_platform"}, + {"id": "explicit_request", "matrix": "platform"} + ], + "platforms": [ + {"id": "ubuntu", "runner": "ubuntu-latest"}, + {"id": "macos", "runner": "macos-latest"}, + {"id": "windows", "runner": "windows-latest"} + ], + "agents": [ + {"id": "claude", "label": "Claude", "version": "2.1.217", "executable": "claude", "installer": {"kind": "npm", "package": "@anthropic-ai/claude-code"}, "integration": "native_command_hook", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "cursor", "label": "Cursor", "version": "2026.07.20-8cc9c0b", "executable": "agent", "installer": {"kind": "cursor_release", "package": "https://downloads.cursor.com/lab/2026.07.20-8cc9c0b"}, "integration": "native_command_hook", "artifacts": ["integrations/cursor/beforeShellExecution"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "wsl"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "codex", "label": "Codex", "version": "0.145.0", "executable": "codex", "installer": {"kind": "npm", "package": "@openai/codex"}, "integration": "native_command_hook", "artifacts": ["integrations/codex/PreToolUse.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "copilot", "label": "GitHub Copilot CLI", "version": "1.0.73", "executable": "copilot", "installer": {"kind": "npm", "package": "@github/copilot"}, "integration": "native_command_hook", "artifacts": ["integrations/copilot/PreToolUse", "integrations/copilot/PreToolUse.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "gemini", "label": "Gemini", "version": "0.51.0", "executable": "gemini", "installer": {"kind": "npm", "package": "@google/gemini-cli"}, "integration": "native_command_hook", "artifacts": ["integrations/gemini/BeforeTool", "integrations/gemini/BeforeTool.ps1"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "kimi", "label": "Kimi Code", "version": "0.29.0", "executable": "kimi", "installer": {"kind": "kimi_release", "package": "https://code.kimi.com/kimi-code"}, "integration": "native_command_hook", "artifacts": [], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "opencode", "label": "OpenCode", "version": "1.18.4", "executable": "opencode", "installer": {"kind": "npm", "package": "opencode-ai"}, "integration": "opencode_plugin", "artifacts": ["integrations/opencode/pitot.ts"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "pi", "label": "Pi", "version": "0.81.1", "executable": "pi", "installer": {"kind": "npm", "package": "@earendil-works/pi-coding-agent"}, "integration": "pi_extension", "artifacts": ["integrations/pi/pitot.ts"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"}, + {"id": "qwen", "label": "Qwen Code", "version": "0.20.1", "executable": "qwen", "installer": {"kind": "npm", "package": "@qwen-code/qwen-code"}, "integration": "native_command_hook", "artifacts": ["integrations/qwen/PreToolUse", "integrations/qwen/PreToolUse.cjs"], "runtime": {"ubuntu": "native", "macos": "native", "windows": "native"}, "driver": "real_agent_driver.py", "required_mode": "real_cli"} + ] +} diff --git a/integrations/codex/PreToolUse.ps1 b/integrations/codex/PreToolUse.ps1 new file mode 100644 index 0000000..2d3f16a --- /dev/null +++ b/integrations/codex/PreToolUse.ps1 @@ -0,0 +1,39 @@ +param( + [Parameter(Mandatory = $true)][string]$Pitot, + [Parameter(Mandatory = $true)][string]$RealBin, + [Parameter(Mandatory = $true)][string]$Receipt, + [Parameter(Mandatory = $true)][string]$Nonce, + [Parameter(Mandatory = $true)][string]$Runtime +) + +$payload = [Console]::In.ReadToEnd() +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$arguments = @("--real-bin", $RealBin, "--receipt", $Receipt, "--nonce", $Nonce, "hook", "codex", "--runtime", $Runtime) +$startInfo = New-Object System.Diagnostics.ProcessStartInfo +$startInfo.FileName = $Pitot +$startInfo.Arguments = (($arguments | ForEach-Object { '"' + $_.Replace('"', '\"') + '"' }) -join ' ') +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardInput = $true +$startInfo.StandardInputEncoding = [Text.UTF8Encoding]::new($false) +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $startInfo +[void]$process.Start() +$payloadBytes = [Text.UTF8Encoding]::new($false).GetBytes($payload) +$stdin = $process.StandardInput.BaseStream +$stdin.Write($payloadBytes, 0, $payloadBytes.Length) +$stdin.Close() +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() +$process.WaitForExit() +$pitotOutput = ($stdout + $stderr).Trim() +if ($process.ExitCode -eq 0) { + exit 0 +} + +if (-not $pitotOutput) { $pitotOutput = "Pitot rejected the shell request" } +if ($pitotOutput.Length -gt 1024) { $pitotOutput = $pitotOutput.Substring(0, 1024) } +@{ decision = "block"; reason = $pitotOutput } | ConvertTo-Json -Compress +exit 0 diff --git a/integrations/copilot/PreToolUse b/integrations/copilot/PreToolUse new file mode 100644 index 0000000..30af5f7 --- /dev/null +++ b/integrations/copilot/PreToolUse @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${PITOT_BIN:-pitot}" +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" hook copilot 2>&1 >/dev/null); then + printf '%s\n' '{"permissionDecision":"allow","permissionDecisionReason":"Pitot accepted the shell action"}' + exit 0 +fi + +python3 -c 'import json,sys; print(json.dumps({"permissionDecision":"deny","permissionDecisionReason":(sys.argv[1] or "Pitot rejected the shell request")[:1024]},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/integrations/copilot/PreToolUse.ps1 b/integrations/copilot/PreToolUse.ps1 new file mode 100644 index 0000000..1bc5e67 --- /dev/null +++ b/integrations/copilot/PreToolUse.ps1 @@ -0,0 +1,31 @@ +$payload = [Console]::In.ReadToEnd() +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$pitot = if ($env:PITOT_BIN) { $env:PITOT_BIN } else { "pitot" } +$startInfo = New-Object System.Diagnostics.ProcessStartInfo +$startInfo.FileName = $pitot +$startInfo.Arguments = '"hook" "copilot"' +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardInput = $true +$startInfo.StandardInputEncoding = [Text.UTF8Encoding]::new($false) +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $startInfo +[void]$process.Start() +$payloadBytes = [Text.UTF8Encoding]::new($false).GetBytes($payload) +$stdin = $process.StandardInput.BaseStream +$stdin.Write($payloadBytes, 0, $payloadBytes.Length) +$stdin.Close() +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() +$process.WaitForExit() +$pitotOutput = ($stdout + $stderr).Trim() +if ($process.ExitCode -eq 0) { + @{ permissionDecision = "allow"; permissionDecisionReason = "Pitot accepted the shell action" } | ConvertTo-Json -Compress +} else { + if (-not $pitotOutput) { $pitotOutput = "Pitot rejected the shell request" } + if ($pitotOutput.Length -gt 1024) { $pitotOutput = $pitotOutput.Substring(0, 1024) } + @{ permissionDecision = "deny"; permissionDecisionReason = $pitotOutput } | ConvertTo-Json -Compress +} +exit 0 diff --git a/integrations/cursor/beforeShellExecution b/integrations/cursor/beforeShellExecution new file mode 100644 index 0000000..70320ef --- /dev/null +++ b/integrations/cursor/beforeShellExecution @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${1:-${PITOT_BIN:-pitot}}" +PITOT_ARGUMENTS=() +if [ "$#" -ge 5 ]; then + PITOT_ARGUMENTS=(--real-bin "$2" --receipt "$3" --nonce "$4") + RUNTIME_ARGUMENTS=(--runtime "$5") +else + RUNTIME_ARGUMENTS=() +fi +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" "${PITOT_ARGUMENTS[@]}" hook cursor "${RUNTIME_ARGUMENTS[@]}" 2>&1 >/dev/null); then + printf '%s\n' '{"continue":true,"permission":"allow"}' + exit 0 +fi + +python3 -c 'import json,sys; reason=(sys.argv[1] or "Pitot rejected the shell request")[:1024]; print(json.dumps({"continue":True,"permission":"deny","user_message":reason,"agent_message":reason},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/integrations/gemini/BeforeTool b/integrations/gemini/BeforeTool new file mode 100644 index 0000000..3d9c6d7 --- /dev/null +++ b/integrations/gemini/BeforeTool @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${1:-${PITOT_BIN:-pitot}}" +PITOT_ARGUMENTS=() +if [ "$#" -ge 5 ]; then + PITOT_ARGUMENTS=(--real-bin "$2" --receipt "$3" --nonce "$4") + RUNTIME_ARGUMENTS=(--runtime "$5") +else + RUNTIME_ARGUMENTS=() +fi +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" "${PITOT_ARGUMENTS[@]}" hook gemini "${RUNTIME_ARGUMENTS[@]}" 2>&1 >/dev/null); then + printf '%s\n' '{"decision":"allow"}' + exit 0 +fi + +python3 -c 'import json,sys; print(json.dumps({"decision":"deny","reason":(sys.argv[1] or "Pitot rejected the shell request")[:1024]},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/integrations/gemini/BeforeTool.ps1 b/integrations/gemini/BeforeTool.ps1 new file mode 100644 index 0000000..6110b1c --- /dev/null +++ b/integrations/gemini/BeforeTool.ps1 @@ -0,0 +1,41 @@ +param( + [string]$Pitot = $(if ($env:PITOT_BIN) { $env:PITOT_BIN } else { "pitot" }), + [string]$RealBin = "", + [string]$Receipt = "", + [string]$Nonce = "", + [string]$Runtime = "" +) +$payload = [Console]::In.ReadToEnd() +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +$arguments = @() +if ($RealBin) { $arguments += @("--real-bin", $RealBin, "--receipt", $Receipt, "--nonce", $Nonce) } +$arguments += @("hook", "gemini") +if ($Runtime) { $arguments += @("--runtime", $Runtime) } +$startInfo = New-Object System.Diagnostics.ProcessStartInfo +$startInfo.FileName = $Pitot +$startInfo.Arguments = (($arguments | ForEach-Object { '"' + $_.Replace('"', '\"') + '"' }) -join ' ') +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardInput = $true +$startInfo.StandardInputEncoding = [Text.UTF8Encoding]::new($false) +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true +$process = New-Object System.Diagnostics.Process +$process.StartInfo = $startInfo +[void]$process.Start() +$payloadBytes = [Text.UTF8Encoding]::new($false).GetBytes($payload) +$stdin = $process.StandardInput.BaseStream +$stdin.Write($payloadBytes, 0, $payloadBytes.Length) +$stdin.Close() +$stdout = $process.StandardOutput.ReadToEnd() +$stderr = $process.StandardError.ReadToEnd() +$process.WaitForExit() +$pitotOutput = ($stdout + $stderr).Trim() +if ($process.ExitCode -eq 0) { + @{ decision = "allow" } | ConvertTo-Json -Compress +} else { + if (-not $pitotOutput) { $pitotOutput = "Pitot rejected the shell request" } + if ($pitotOutput.Length -gt 1024) { $pitotOutput = $pitotOutput.Substring(0, 1024) } + @{ decision = "deny"; reason = $pitotOutput } | ConvertTo-Json -Compress +} +exit 0 diff --git a/integrations/opencode/pitot.ts b/integrations/opencode/pitot.ts new file mode 100644 index 0000000..3a500db --- /dev/null +++ b/integrations/opencode/pitot.ts @@ -0,0 +1,25 @@ +import { spawnSync } from "node:child_process"; + +// OpenCode runs plugins in-process. This is the genuine synchronous +// tool.execute.before boundary; it is not a Claude PreToolUse simulation. +export const PitotPlugin = async () => ({ + "tool.execute.before": async (input, output) => { + if (input.tool !== "bash") return; + const command = output.args?.command; + const payload = JSON.stringify({ + hook_event_name: "PreToolUse", + tool_name: "Bash", + tool_input: { command: typeof command === "string" ? command : "" }, + }); + const result = spawnSync(process.env.PITOT_BIN || "pitot", ["hook", "opencode"], { + input: payload, + encoding: "utf8", + maxBuffer: 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error((result.stderr || "Pitot rejected the shell request").trim()); + } + }, +}); + +export default PitotPlugin; diff --git a/integrations/pi/pitot.ts b/integrations/pi/pitot.ts new file mode 100644 index 0000000..f802dc2 --- /dev/null +++ b/integrations/pi/pitot.ts @@ -0,0 +1,25 @@ +import { spawnSync } from "node:child_process"; + +// handleToolCall is exported so the shipped boundary can be tested without a +// live Pi session. Pi itself calls the default extension registration below. +export function handleToolCall(event, run = spawnSync) { + if (event.toolName !== "bash") return undefined; + + const payload = JSON.stringify({ + hook_event_name: "tool_call", + tool_name: "bash", + tool_input: { command: event.input?.command ?? "" }, + }); + const result = run(process.env.PITOT_BIN || "pitot", ["hook", "pi"], { + input: payload, + encoding: "utf8", + maxBuffer: 1024 * 1024, + }); + if (result.status === 0) return undefined; + const reason = (result.stderr || "Pitot rejected the shell request").trim(); + return { block: true, reason: reason.slice(0, 1024) }; +} + +export default function pitotExtension(pi) { + pi.on("tool_call", async (event) => handleToolCall(event)); +} diff --git a/integrations/qwen/PreToolUse b/integrations/qwen/PreToolUse new file mode 100644 index 0000000..0d21616 --- /dev/null +++ b/integrations/qwen/PreToolUse @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -uo pipefail + +PITOT_COMMAND="${1:-${PITOT_BIN:-pitot}}" +PITOT_ARGUMENTS=() +if [ "$#" -ge 5 ]; then + PITOT_ARGUMENTS=(--real-bin "$2" --receipt "$3" --nonce "$4") + RUNTIME_ARGUMENTS=(--runtime "$5") +else + RUNTIME_ARGUMENTS=() +fi +PAYLOAD=$(cat) +if PITOT_ERROR=$(printf '%s' "$PAYLOAD" | "$PITOT_COMMAND" "${PITOT_ARGUMENTS[@]}" hook qwen "${RUNTIME_ARGUMENTS[@]}" 2>&1 >/dev/null); then + printf '%s\n' '{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"allow","permissionDecisionReason":"Pitot accepted the shell action"}}' + exit 0 +fi + +python3 -c 'import json,sys; print(json.dumps({"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny","permissionDecisionReason":(sys.argv[1] or "Pitot rejected the shell request")[:1024]}},separators=(",",":")))' "$PITOT_ERROR" +exit 0 diff --git a/integrations/qwen/PreToolUse.cjs b/integrations/qwen/PreToolUse.cjs new file mode 100644 index 0000000..1644042 --- /dev/null +++ b/integrations/qwen/PreToolUse.cjs @@ -0,0 +1,27 @@ +#!/usr/bin/env node +"use strict"; + +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); + +const supplied = process.argv.slice(2); +const pitot = supplied[0] || process.env.PITOT_BIN || "pitot"; +const pitotArgs = supplied.length >= 5 + ? ["--real-bin", supplied[1], "--receipt", supplied[2], "--nonce", supplied[3], "hook", "qwen", "--runtime", supplied[4]] + : ["hook", "qwen"]; +const payload = fs.readFileSync(0); +const result = spawnSync(pitot, pitotArgs, { input: payload, encoding: "utf8", windowsHide: true }); +const detail = `${result.stdout || ""}${result.stderr || ""}`.trim(); +const allowed = result.status === 0 && !result.error; +const reason = allowed + ? "Pitot accepted the shell action" + : (detail || result.error?.message || "Pitot rejected the shell request").slice(0, 1024); + +process.stdout.write(JSON.stringify({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: allowed ? "allow" : "deny", + permissionDecisionReason: reason, + }, +}) + "\n"); + diff --git a/runtime/request.go b/runtime/request.go new file mode 100644 index 0000000..f67749a --- /dev/null +++ b/runtime/request.go @@ -0,0 +1,60 @@ +package runtime + +// Typed request construction. A caller issuing an explicit control request would +// otherwise hand-build the control.requested envelope: set the version and type +// discriminators, mint a correlation id, and marshal the structured body into the +// raw `data` field itself. NewControlRequest and RequestTyped remove that JSON +// boundary — the caller passes a typed payload and gets a correlated response. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/operatorstack/pitot/schema" +) + +// NewControlRequest builds a control.requested envelope for the given request +// kind, marshaling payload into the data field. It stamps the current envelope +// version and control.requested type and mints an unpredictable correlation +// ActionID, so a caller never hand-writes the wire JSON or the boilerplate. A nil +// payload produces an empty data field (a request that carries no structured +// body); a payload that cannot be marshaled is an error. +func NewControlRequest(kind string, payload any) (schema.ControlRequested, error) { + if kind == "" { + return schema.ControlRequested{}, errors.New("pitot: control request kind is required") + } + var data json.RawMessage + if payload != nil { + encoded, err := json.Marshal(payload) + if err != nil { + return schema.ControlRequested{}, fmt.Errorf("pitot: marshal control request data: %w", err) + } + data = encoded + } + actionID, err := NewActionID() + if err != nil { + return schema.ControlRequested{}, err + } + return schema.ControlRequested{ + PitotVersion: schema.Version, + Type: schema.TypeControlRequested, + Kind: kind, + ActionID: actionID, + Data: data, + }, nil +} + +// RequestTyped builds a typed control request for kind and issues it, returning +// the correlated response. It is the typed convenience over Request: the caller +// supplies a structured payload rather than a pre-framed envelope with raw JSON. +// All of Request's guarantees still apply — correlation, the client deadline, and +// the runtime's fail-closed unavailable/timeout defaults. +func (c *Client) RequestTyped(ctx context.Context, kind string, payload any) (schema.ControlResponse, error) { + req, err := NewControlRequest(kind, payload) + if err != nil { + return schema.ControlResponse{}, err + } + return c.Request(ctx, req) +} diff --git a/runtime/request_test.go b/runtime/request_test.go new file mode 100644 index 0000000..608868a --- /dev/null +++ b/runtime/request_test.go @@ -0,0 +1,58 @@ +package runtime + +import ( + "encoding/json" + "testing" + + "github.com/operatorstack/pitot/schema" +) + +func TestNewControlRequestFillsEnvelopeAndMarshalsPayload(t *testing.T) { + type body struct { + Actor string `json:"actor"` + Operation string `json:"operation"` + } + req, err := NewControlRequest("interlock.decide", body{Actor: "publisher", Operation: "artifact.publish"}) + if err != nil { + t.Fatalf("NewControlRequest: %v", err) + } + if req.PitotVersion != schema.Version { + t.Errorf("version = %q, want %q", req.PitotVersion, schema.Version) + } + if req.Type != schema.TypeControlRequested { + t.Errorf("type = %q, want %q", req.Type, schema.TypeControlRequested) + } + if req.Kind != "interlock.decide" { + t.Errorf("kind = %q, want interlock.decide", req.Kind) + } + if req.ActionID == "" { + t.Error("ActionID was not minted") + } + var got body + if err := json.Unmarshal(req.Data, &got); err != nil { + t.Fatalf("data is not the marshaled payload: %v", err) + } + if got.Actor != "publisher" || got.Operation != "artifact.publish" { + t.Errorf("round-tripped payload = %+v, want the input", got) + } +} + +func TestNewControlRequestRequiresKind(t *testing.T) { + if _, err := NewControlRequest("", struct{}{}); err == nil { + t.Fatal("empty kind must be an error") + } +} + +func TestNewControlRequestMintsDistinctActionIDs(t *testing.T) { + a, err := NewControlRequest("k", nil) + if err != nil { + t.Fatal(err) + } + b, err := NewControlRequest("k", nil) + if err != nil { + t.Fatal(err) + } + if a.ActionID == b.ActionID { + t.Fatalf("correlation ids must be unpredictable and distinct, got %q twice", a.ActionID) + } +} diff --git a/sdk/runner.go b/sdk/runner.go new file mode 100644 index 0000000..1c49b0a --- /dev/null +++ b/sdk/runner.go @@ -0,0 +1,73 @@ +package sdk + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + + "github.com/operatorstack/pitot/schema" +) + +type ConsumerHandler func(event schema.Event) + +func RunConsumer(handler ConsumerHandler) { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var event schema.Event + if err := json.Unmarshal(line, &event); err != nil { + fmt.Fprintf(os.Stderr, "Pitot Consumer error: %v\n", err) + continue + } + handler(event) + } +} + +type Outcome struct { + Outcome string + Message string +} + +func Allow(message string) Outcome { + return Outcome{Outcome: schema.OutcomeAllow, Message: message} +} + +func Deny(message string) Outcome { + return Outcome{Outcome: schema.OutcomeDeny, Message: message} +} + +type ControllerHandler func(req schema.ControlRequested) Outcome + +func RunController(controllerID string, handler ControllerHandler) { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var req schema.ControlRequested + if err := json.Unmarshal(line, &req); err != nil { + fmt.Fprintf(os.Stderr, "Pitot Controller error: %v\n", err) + continue + } + + result := handler(req) + + resp := schema.ControlResponse{ + PitotVersion: schema.Version, + Type: schema.TypeControlResponse, + ControllerID: controllerID, + ActionID: req.ActionID, + Outcome: result.Outcome, + Message: result.Message, + } + + if err := json.NewEncoder(os.Stdout).Encode(resp); err != nil { + fmt.Fprintf(os.Stderr, "Pitot Controller error: %v\n", err) + } + } +} diff --git a/tests/cursor_control_proxy.mjs b/tests/cursor_control_proxy.mjs new file mode 100644 index 0000000..e032241 --- /dev/null +++ b/tests/cursor_control_proxy.mjs @@ -0,0 +1,346 @@ +#!/usr/bin/env node +// Pinned Cursor Agent endpoint fixture. This is deliberately separate from the +// JSON dialect handler because Cursor's --endpoint transport upgrades Run to +// cleartext HTTP/2 and protobuf. + +import http2 from "node:http2"; +import http from "node:http"; +import net from "node:net"; +import fs from "node:fs"; + +function parseArgs(argv) { + const values = {}; + for (let index = 2; index < argv.length; index += 2) { + values[argv[index].replace(/^--/, "")] = argv[index + 1]; + } + return values; +} + +function varint(value) { + const bytes = []; + while (value > 0x7f) { + bytes.push((value & 0x7f) | 0x80); + value >>= 7; + } + bytes.push(value); + return Buffer.from(bytes); +} + +function fieldBytes(field, value) { + const payload = Buffer.isBuffer(value) ? value : Buffer.from(value); + return Buffer.concat([varint((field << 3) | 2), varint(payload.length), payload]); +} + +function fieldVarint(field, value) { + return Buffer.concat([varint(field << 3), varint(value)]); +} + +function connectEnvelope(message, flags = 0) { + const header = Buffer.alloc(5); + header[0] = flags; + header.writeUInt32BE(message.length, 1); + return Buffer.concat([header, message]); +} + +function envelopeShapes(body) { + const shapes = []; + let offset = 0; + while (offset + 5 <= body.length) { + const flags = body[offset]; + const length = body.readUInt32BE(offset + 1); + if (offset + 5 + length > body.length) break; + const message = body.subarray(offset + 5, offset + 5 + length); + const printable = (message.toString("latin1").match(/[ -~]{4,}/g) || []) + .map((value) => value.replaceAll(args.nonce, "")) + .filter((value) => /pitot|canary|hook|command|error|fail|denied|not found|nonce/i.test(value)) + .slice(0, 12); + shapes.push({ flags, length, nonce_present: message.includes(Buffer.from(args.nonce)), canary_result_present: message.includes(Buffer.from(`PITOT_CANARY_RESULT ${args.nonce}`)), printable }); + offset += 5 + length; + } + return shapes; +} + +function shellExecution(command, phase = "allow") { + const executable = command.split(" ", 1)[0]; + const argument = command.slice(executable.length + 1); + // ShellCommandParsingResult.ExecutableCommandArg { type, value } + const parsedArgument = Buffer.concat([fieldBytes(1, "word"), fieldBytes(2, argument)]); + // ShellCommandParsingResult.ExecutableCommand { name, args, full_text } + const parsedCommand = Buffer.concat([ + fieldBytes(1, executable), + fieldBytes(2, parsedArgument), + fieldBytes(3, command), + ]); + // ShellCommandParsingResult { executable_commands: 2 } + const parsingResult = fieldBytes(2, parsedCommand); + // agent.v1.ShellArgs { command: 1, tool_call_id: 4 } + const shellArgs = Buffer.concat([ + fieldBytes(1, command), + fieldBytes(4, `pitot-tool-${phase}`), + fieldBytes(5, command), + fieldBytes(8, parsingResult), + fieldVarint(12, 1), + ]); + // agent.v1.ExecServerMessage { id: 1, exec_id: 15, shell_args: 2 } + const execution = Buffer.concat([ + fieldVarint(1, 1), + fieldBytes(15, `pitot-exec-${phase}`), + fieldBytes(2, shellArgs), + ]); + // agent.v1.AgentServerMessage { exec_server_message: 2 } + return fieldBytes(2, execution); +} + +function textUpdate(text) { + // TextDeltaUpdate.text: 1 → InteractionUpdate.text_delta: 1 → + // AgentServerMessage.interaction_update: 1. + return fieldBytes(1, fieldBytes(1, fieldBytes(1, text))); +} + +function turnEnded() { + // TurnEndedUpdate is valid when empty. It is field 14 of InteractionUpdate. + return fieldBytes(1, fieldBytes(14, Buffer.alloc(0))); +} + +function modelDetails() { + return Buffer.concat([1, 3, 4, 5].map((field) => fieldBytes(field, "pitot-control"))); +} + +function unaryResponse(path) { + const model = modelDetails(); + if (path.endsWith("/GetUsableModels") || path.endsWith("/GetDefaultModelForCli")) { + return fieldBytes(1, model); + } + return Buffer.alloc(0); +} + +function outstandingPhase(state, sent) { + if (!state.allow_tool_result_observed && !sent.allow) return "allow"; + if (state.allow_tool_result_observed && !state.denied_result_observed && !sent.deny) return "deny"; + if (state.denied_result_observed && !sent.final) return "final"; + return null; +} + +const args = parseArgs(process.argv); +if (process.argv.includes("--self-test")) { + const command = "pitot-e2e-canary fixture-nonce"; + const message = shellExecution(command); + const framed = connectEnvelope(message); + if (!message.includes(Buffer.from(command)) || framed.readUInt32BE(1) !== message.length) { + throw new Error("Cursor Connect/protobuf fixture is invalid"); + } + const state = { allow_tool_result_observed: false, denied_result_observed: false }; + const firstStream = { allow: false, deny: false, final: false }; + if (outstandingPhase(state, firstStream) !== "allow") throw new Error("allow phase was not selected"); + firstStream.allow = true; + if (outstandingPhase(state, firstStream) !== null) throw new Error("allow phase duplicated on one stream"); + if (outstandingPhase(state, { allow: false, deny: false, final: false }) !== "allow") { + throw new Error("allow phase was not recovered on a replacement stream"); + } + state.allow_tool_result_observed = true; + if (outstandingPhase(state, { allow: false, deny: false, final: false }) !== "deny") { + throw new Error("deny phase was not selected after the allow receipt"); + } + state.denied_result_observed = true; + if (outstandingPhase(state, { allow: false, deny: false, final: false }) !== "final") { + throw new Error("final phase was not selected after the denial receipt"); + } + process.stdout.write("PASS: Cursor Connect/protobuf endpoint fixture\n"); + process.exit(0); +} +const required = ["nonce", "receipt", "ready-file", "canary-command"]; +for (const key of required) { + if (!args[key]) throw new Error(`missing --${key}`); +} + +const receipt = { + schema_version: 1, + agent: "cursor", + protocol: "cursor_connect_proto", + nonce: args.nonce, + initial_prompt_observed: false, + tool_call_response_emitted: false, + tool_result_observed: false, + allow_tool_call_response_emitted: false, + allow_tool_result_observed: false, + deny_tool_call_response_emitted: false, + denied_result_observed: false, + final_response_emitted: false, + endpoint_observed: null, + auxiliary_requests: 0, + cursor_requests: [], + response_attempts: { allow: 0, deny: 0, final: 0 }, + transport_errors: [], +}; + +function save() { + const temporary = `${args.receipt}.tmp`; + fs.mkdirSync(new URL(".", `file://${args.receipt}`).pathname, { recursive: true }); + fs.writeFileSync(temporary, `${JSON.stringify(receipt, null, 2)}\n`); + fs.renameSync(temporary, args.receipt); +} + +const h2Server = http2.createServer(); + +function recordTransportError(scope, error) { + receipt.transport_errors.push({ + scope, + code: String(error?.code || "unknown"), + }); + receipt.transport_errors = receipt.transport_errors.slice(-20); + save(); +} + +h2Server.on("stream", (stream, headers) => { + const path = String(headers[":path"] || "").split("?", 1)[0]; + const contentType = String(headers["content-type"] || ""); + const chunks = []; + let responseStarted = false; + let allowSent = false; + let denySent = false; + let finalSent = false; + stream.on("error", (error) => recordTransportError("run_stream", error)); + stream.on("data", (chunk) => { + chunks.push(chunk); + const body = Buffer.concat(chunks); + receipt.cursor_inbound = { request_bytes: body.length, envelopes: envelopeShapes(body) }; + save(); + if (!responseStarted && body.includes(Buffer.from(args.nonce))) { + responseStarted = true; + receipt.initial_prompt_observed = true; + receipt.endpoint_observed = { + transport: "http2", + method: "POST", + path, + media_type: contentType.split(";", 1)[0].trim().toLowerCase(), + framing: "connect_envelope", + request_shape: { + service: "agent.v1.AgentService", + method: "Run", + stream: "bidirectional", + message: "agent.v1.AgentClientMessage", + }, + }; + receipt.cursor_run = { path, content_type: contentType, request_bytes: body.length }; + save(); + stream.respond({ ":status": 200, "content-type": "application/connect+proto" }); + if (args["response-fault"] === "text") { + receipt.fault_response_emitted = "text"; + save(); + stream.write(connectEnvelope(textUpdate("No tool call"))); + stream.write(connectEnvelope(turnEnded())); + stream.end(connectEnvelope(Buffer.from("{}"), 0x02)); + } + } + if (!responseStarted || args["response-fault"] === "text" || receipt.final_response_emitted) return; + // A released Cursor CLI may reconnect its bidirectional Run stream after + // a transient reset. Delivery is only proven by the matching client tool + // result, so resend the outstanding phase once on each replacement stream. + let phase = outstandingPhase(receipt, { allow: allowSent, deny: denySent, final: finalSent }); + if (phase === "allow") { + allowSent = true; + receipt.tool_call_response_emitted = true; + receipt.allow_tool_call_response_emitted = true; + receipt.response_attempts.allow += 1; + save(); + stream.write(connectEnvelope(shellExecution(`${args["canary-command"]} PITOT_ALLOW ${args.nonce}`, "allow"))); + return; + } + if (!receipt.allow_tool_result_observed && body.includes(Buffer.from(`PITOT_CANARY_RESULT PITOT_ALLOW ${args.nonce}`))) { + receipt.tool_result_observed = true; + receipt.allow_tool_result_observed = true; + save(); + } + phase = outstandingPhase(receipt, { allow: allowSent, deny: denySent, final: finalSent }); + if (phase === "deny") { + denySent = true; + receipt.deny_tool_call_response_emitted = true; + receipt.response_attempts.deny += 1; + save(); + stream.write(connectEnvelope(shellExecution(`${args["canary-command"]} PITOT_DENY ${args.nonce}`, "deny"))); + return; + } + const nativeDenial = body.includes(Buffer.from(`PITOT_DENY ${args.nonce}`)) && + body.includes(Buffer.from("blocked by a hook")); + if (receipt.deny_tool_call_response_emitted && !receipt.denied_result_observed && nativeDenial) { + receipt.denied_result_observed = true; + receipt.final_response_emitted = true; + save(); + } + phase = outstandingPhase(receipt, { allow: allowSent, deny: denySent, final: finalSent }); + if (phase === "final") { + finalSent = true; + receipt.response_attempts.final += 1; + save(); + stream.write(connectEnvelope(textUpdate(`PITOT_E2E_COMPLETE ${args.nonce}`))); + stream.write(connectEnvelope(turnEnded())); + // Connect end-stream envelope. Success metadata is an empty JSON object. + stream.end(connectEnvelope(Buffer.from("{}"), 0x02)); + } + }); + stream.on("end", () => { + if (responseStarted) return; + const body = Buffer.concat(chunks); + receipt.cursor_requests.push({ path, content_type: contentType, request_bytes: body.length }); + receipt.auxiliary_requests = receipt.cursor_requests.length; + save(); + const payload = unaryResponse(path); + stream.respond({ ":status": 200, "content-type": contentType.includes("json") ? "application/json" : "application/proto" }); + stream.end(contentType.includes("json") ? Buffer.from("{}") : payload); + }); +}); +h2Server.on("sessionError", (error) => recordTransportError("h2_session", error)); +h2Server.on("error", (error) => recordTransportError("h2_server", error)); + +const http1Server = http.createServer((request, response) => { + const chunks = []; + request.on("data", (chunk) => chunks.push(chunk)); + request.on("end", () => { + const path = String(request.url || "").split("?", 1)[0]; + const contentType = String(request.headers["content-type"] || ""); + const body = Buffer.concat(chunks); + receipt.cursor_requests.push({ path, content_type: contentType, request_bytes: body.length }); + receipt.auxiliary_requests = receipt.cursor_requests.length; + save(); + const payload = unaryResponse(path); + response.writeHead(200, { "content-type": contentType.includes("json") ? "application/json" : "application/proto" }); + response.end(contentType.includes("json") ? Buffer.from("{}") : payload); + }); +}); +http1Server.on("clientError", (error, socket) => { + recordTransportError("http1_client", error); + socket.destroy(); +}); +http1Server.on("error", (error) => recordTransportError("http1_server", error)); + +const frontServer = net.createServer((client) => { + client.on("error", (error) => recordTransportError("front_client", error)); + client.once("data", (first) => { + const isHttp2 = first.subarray(0, 14).toString() === "PRI * HTTP/2.0"; + const target = isHttp2 ? h2Server.address().port : http1Server.address().port; + const backend = net.connect(target, "127.0.0.1", () => backend.write(first)); + backend.on("error", (error) => { + recordTransportError("front_backend", error); + client.destroy(); + }); + client.pipe(backend).pipe(client); + }); +}); +frontServer.on("error", (error) => recordTransportError("front_server", error)); + +h2Server.listen(0, "127.0.0.1", () => { + http1Server.listen(0, "127.0.0.1", () => { + frontServer.listen(0, "127.0.0.1", () => { + const address = frontServer.address(); + fs.writeFileSync(args["ready-file"], `http://127.0.0.1:${address.port}\n`); + save(); + }); + }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => frontServer.close(() => { + h2Server.close(); + http1Server.close(() => process.exit(0)); + })); +} diff --git a/tests/e2e_claude_cli_test.sh b/tests/e2e_claude_cli_test.sh new file mode 100644 index 0000000..ef20810 --- /dev/null +++ b/tests/e2e_claude_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "claude" diff --git a/tests/e2e_codex_cli_test.sh b/tests/e2e_codex_cli_test.sh new file mode 100644 index 0000000..a5bec37 --- /dev/null +++ b/tests/e2e_codex_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "codex" diff --git a/tests/e2e_copilot_cli_test.sh b/tests/e2e_copilot_cli_test.sh new file mode 100644 index 0000000..c4134bf --- /dev/null +++ b/tests/e2e_copilot_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "copilot" diff --git a/tests/e2e_cursor_cli_test.sh b/tests/e2e_cursor_cli_test.sh new file mode 100644 index 0000000..386c99e --- /dev/null +++ b/tests/e2e_cursor_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "cursor" diff --git a/tests/e2e_gemini_cli_test.sh b/tests/e2e_gemini_cli_test.sh new file mode 100644 index 0000000..9a10ba1 --- /dev/null +++ b/tests/e2e_gemini_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "gemini" diff --git a/tests/e2e_kimi_cli_test.sh b/tests/e2e_kimi_cli_test.sh new file mode 100644 index 0000000..724c9c7 --- /dev/null +++ b/tests/e2e_kimi_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "kimi" diff --git a/tests/e2e_opencode_cli_test.sh b/tests/e2e_opencode_cli_test.sh new file mode 100644 index 0000000..b98a6f9 --- /dev/null +++ b/tests/e2e_opencode_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "opencode" diff --git a/tests/e2e_pi_cli_test.sh b/tests/e2e_pi_cli_test.sh new file mode 100644 index 0000000..53d031c --- /dev/null +++ b/tests/e2e_pi_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "pi" diff --git a/tests/e2e_qwen_cli_test.sh b/tests/e2e_qwen_cli_test.sh new file mode 100644 index 0000000..807187e --- /dev/null +++ b/tests/e2e_qwen_cli_test.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(dirname "$0")/e2e_unified_runner.sh" "qwen" diff --git a/tests/e2e_runtime_cli_test.sh b/tests/e2e_runtime_cli_test.sh new file mode 100644 index 0000000..de2cde6 --- /dev/null +++ b/tests/e2e_runtime_cli_test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${PITOT_E2E_PLATFORM:?PITOT_E2E_PLATFORM is required}" +: "${PITOT_E2E_EVIDENCE:?PITOT_E2E_EVIDENCE is required}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +BUILD_DIR="${RUNNER_TEMP:-$(mktemp -d)}/pitot-runtime-e2e-bin" +mkdir -p "$BUILD_DIR" +PITOT_BINARY="$BUILD_DIR/pitot" +TESTROLE_BINARY="$BUILD_DIR/pitot-testrole" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + PITOT_BINARY="${PITOT_BINARY}.exe" + TESTROLE_BINARY="${TESTROLE_BINARY}.exe" +fi +go build -o "$PITOT_BINARY" "$LAB_DIR/pitot/cmd/pitot" +go build -o "$TESTROLE_BINARY" "$LAB_DIR/pitot/internal/testrole" +PYTHON_COMMAND="python3" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + PYTHON_COMMAND="python" +fi +"$PYTHON_COMMAND" "$SCRIPT_DIR/runtime_capability_driver.py" \ + --pitot "$PITOT_BINARY" \ + --test-role "$TESTROLE_BINARY" \ + --platform "$PITOT_E2E_PLATFORM" \ + --evidence "$PITOT_E2E_EVIDENCE" diff --git a/tests/e2e_unified_runner.sh b/tests/e2e_unified_runner.sh new file mode 100644 index 0000000..7957d5c --- /dev/null +++ b/tests/e2e_unified_runner.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Real released-agent prompt-to-hook E2E. Direct hook invocation is forbidden. +set -euo pipefail + +HOST="${1:-}" +if [[ -z "$HOST" ]]; then + echo "ERROR: a supervised agent ID is required" >&2 + exit 2 +fi +: "${PITOT_E2E_PLATFORM:?PITOT_E2E_PLATFORM is required}" +: "${PITOT_INSTALL_RECEIPT:?PITOT_INSTALL_RECEIPT is required}" +: "${PITOT_E2E_EVIDENCE:?PITOT_E2E_EVIDENCE is required}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PITOT_MAIN="$LAB_DIR/pitot/cmd/pitot" +if [[ ! -d "$PITOT_MAIN" ]]; then + PITOT_MAIN="$LAB_DIR/cmd/pitot" +fi + +BUILD_DIR="${RUNNER_TEMP:-$(mktemp -d)}/pitot-real-e2e-bin" +mkdir -p "$BUILD_DIR" +PITOT_BINARY="$BUILD_DIR/pitot" +WITNESS_BINARY="$BUILD_DIR/pitot-witness" +TESTROLE_BINARY="$BUILD_DIR/pitot-testrole" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + PITOT_BINARY="${PITOT_BINARY}.exe" + WITNESS_BINARY="${WITNESS_BINARY}.exe" + TESTROLE_BINARY="${TESTROLE_BINARY}.exe" + if [[ "$HOST" == "cursor" ]]; then + # Cursor's supported Windows runtime is WSL. Keep both supervised + # executables native to that runtime; crossing back into a Windows Pitot + # process makes Cursor's hook pipe remain open after Pitot has returned. + PITOT_BINARY="$BUILD_DIR/pitot-linux" + WITNESS_BINARY="$BUILD_DIR/pitot-witness-linux" + TESTROLE_BINARY="$BUILD_DIR/pitot-testrole-linux" + fi +fi + +if [[ "${RUNNER_OS:-}" == "Windows" && "$HOST" == "cursor" ]]; then + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$PITOT_BINARY" "$PITOT_MAIN" + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$WITNESS_BINARY" "$SCRIPT_DIR/witness/main.go" + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$TESTROLE_BINARY" "$LAB_DIR/pitot/internal/testrole" +else + go build -o "$PITOT_BINARY" "$PITOT_MAIN" + go build -o "$WITNESS_BINARY" "$SCRIPT_DIR/witness/main.go" + go build -o "$TESTROLE_BINARY" "$LAB_DIR/pitot/internal/testrole" +fi + +DRIVER_ARGS=( + --agent "$HOST" + --platform "$PITOT_E2E_PLATFORM" + --installation "$PITOT_INSTALL_RECEIPT" + --evidence "$PITOT_E2E_EVIDENCE" + --pitot "$PITOT_BINARY" + --witness "$WITNESS_BINARY" + --test-role "$TESTROLE_BINARY" +) +if [[ -n "${PITOT_CAPTURE_OUTPUT:-}" ]]; then + DRIVER_ARGS+=(--capture-output "$PITOT_CAPTURE_OUTPUT") +fi + +DRIVER=(python3 "$SCRIPT_DIR/real_agent_driver.py" "${DRIVER_ARGS[@]}") +NEGATIVE_EVIDENCE="${PITOT_E2E_EVIDENCE}.wrong-response" +if [[ "${RUNNER_OS:-}" == "Windows" && "$HOST" == "cursor" ]]; then + # Run the complete controller in WSL so its temporary home, project, proxy, + # hook, and child processes use the same native filesystem/process boundary. + to_wsl_path() { + local windows_path drive tail + windows_path="$(cygpath -am "$1")" + if [[ ! "$windows_path" =~ ^[A-Za-z]:/ ]]; then + echo "ERROR: cannot map path into WSL: $windows_path" >&2 + return 1 + fi + drive="${windows_path:0:1}" + tail="${windows_path:2}" + printf '/mnt/%s%s' "${drive,,}" "$tail" + } + WSL_ARGS=( + --agent "$HOST" + --platform "$PITOT_E2E_PLATFORM" + --installation "$(to_wsl_path "$PITOT_INSTALL_RECEIPT")" + --evidence "$(to_wsl_path "$PITOT_E2E_EVIDENCE")" + --pitot "$(to_wsl_path "$PITOT_BINARY")" + --witness "$(to_wsl_path "$WITNESS_BINARY")" + --test-role "$(to_wsl_path "$TESTROLE_BINARY")" + ) + if [[ -n "${PITOT_CAPTURE_OUTPUT:-}" ]]; then + WSL_ARGS+=(--capture-output "$(to_wsl_path "$PITOT_CAPTURE_OUTPUT")") + fi + # Git Bash otherwise rewrites /mnt/ arguments as paths beneath its + # own installation before wsl.exe can receive them. + export MSYS2_ARG_CONV_EXCL='*' + DRIVER=( + wsl.exe --distribution Ubuntu -- env + PITOT_SOURCE_SHA="${PITOT_SOURCE_SHA:-}" + python3 "$(to_wsl_path "$SCRIPT_DIR/real_agent_driver.py")" "${WSL_ARGS[@]}" + ) + NEGATIVE_EVIDENCE="$(to_wsl_path "${PITOT_E2E_EVIDENCE}.wrong-response")" +fi + +"${DRIVER[@]}" + +"${DRIVER[@]}" \ + --evidence "$NEGATIVE_EVIDENCE" \ + --response-fault text \ + --expect-incompatible-response +echo "PASS: $HOST rejected incompatible proxy response evidence" diff --git a/tests/endpoint-provenance.json b/tests/endpoint-provenance.json new file mode 100644 index 0000000..960e7c7 --- /dev/null +++ b/tests/endpoint-provenance.json @@ -0,0 +1,1283 @@ +{ + "capture_policy": "pinned_real_cli_capture", + "cells": [ + { + "agent": "claude", + "capture_sha256": "f2eb7045e2370f18521c812d5e5d141f65c41d59d0c488fb8040698d2973c951", + "dialect": "anthropic_messages", + "executable_sha256": "5840c777fd47115e9ca276e165563c6e121e7c7e2b4d86598e0025f8cc37de56", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/messages", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Bash" + ], + "top_level_keys": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "anthropic_messages", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2.1.217" + }, + { + "agent": "claude", + "capture_sha256": "34f96279d0005b279c4287f18a7fa22d8726e8cdcdb3b124ee5b9a53058f0319", + "dialect": "anthropic_messages", + "executable_sha256": "2630fc5dc6db61bc03f86b95daf47766e5ed5b61873f7bb7cfea764c5ac5a9ba", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/messages", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Bash" + ], + "top_level_keys": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "anthropic_messages", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2.1.217" + }, + { + "agent": "claude", + "capture_sha256": "bb70ed5c57eeb11082dbb9738b7abce34496240bd5f16bbd8ded27ef2877b575", + "dialect": "anthropic_messages", + "executable_sha256": "7999fba95dbffe167d9e0a043f29057979a0518ebe89b60c4fcfc6401ea8c424", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/messages", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Bash" + ], + "top_level_keys": [ + "context_management", + "max_tokens", + "messages", + "metadata", + "model", + "output_config", + "stream", + "system", + "thinking", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "anthropic_messages", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2.1.217" + }, + { + "agent": "codex", + "capture_sha256": "5cec7ac20d1e1c550fc1fd1fc5cb96ef1e04b13d1212ea51125eaf2349460788", + "dialect": "openai_responses", + "executable_sha256": "134063e133f0b4244fa3b251acf973d4fe4b4aeeacbdc135211bf480f59f1477", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "exec_command", + "write_stdin", + "update_plan", + "request_user_input", + "view_image", + "multi_agent_v1", + "get_goal", + "create_goal", + "update_goal" + ], + "top_level_keys": [ + "client_metadata", + "include", + "input", + "instructions", + "model", + "parallel_tool_calls", + "prompt_cache_key", + "reasoning", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.145.0" + }, + { + "agent": "codex", + "capture_sha256": "b105f6d402c363f9734553bbaaa7510be71df2f6774f4b1040935dcabcfa1815", + "dialect": "openai_responses", + "executable_sha256": "134063e133f0b4244fa3b251acf973d4fe4b4aeeacbdc135211bf480f59f1477", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "exec_command", + "write_stdin", + "update_plan", + "request_user_input", + "view_image", + "multi_agent_v1", + "get_goal", + "create_goal", + "update_goal" + ], + "top_level_keys": [ + "client_metadata", + "include", + "input", + "instructions", + "model", + "parallel_tool_calls", + "prompt_cache_key", + "reasoning", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.145.0" + }, + { + "agent": "codex", + "capture_sha256": "fe466cdc0cd79fd12db0c06ba7b37d516bbfade7ed22e1fa9c61f740369e4c12", + "dialect": "openai_responses", + "executable_sha256": "c54db6755e710c39703f7c37512f9e35ed41042d8080558d2b84b8d2694323c3", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "shell_command", + "update_plan", + "request_user_input", + "view_image", + "multi_agent_v1", + "get_goal", + "create_goal", + "update_goal" + ], + "top_level_keys": [ + "client_metadata", + "include", + "input", + "instructions", + "model", + "parallel_tool_calls", + "prompt_cache_key", + "reasoning", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.145.0" + }, + { + "agent": "copilot", + "capture_sha256": "69ed781538ee899b31e85061f0c2e8ec3d2ca40244fff5d6a838d1908ce11462", + "dialect": "openai_chat", + "executable_sha256": "0ea824a86be5757533fdb092eff7050871bd7a711a46babde0ffe0e44ac5ad88", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "read_bash", + "stop_bash", + "list_bash", + "apply_patch", + "view", + "fetch_copilot_cli_documentation", + "skill", + "sql", + "session_store_sql", + "read_agent", + "list_agents", + "write_agent", + "rg", + "glob", + "task" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.0.73" + }, + { + "agent": "copilot", + "capture_sha256": "0a30d6ef202ef5743e1c1026122f524747862e1d445d8deeb6403d4eccb9810a", + "dialect": "openai_chat", + "executable_sha256": "0ea824a86be5757533fdb092eff7050871bd7a711a46babde0ffe0e44ac5ad88", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "read_bash", + "stop_bash", + "list_bash", + "apply_patch", + "view", + "fetch_copilot_cli_documentation", + "skill", + "sql", + "session_store_sql", + "read_agent", + "list_agents", + "write_agent", + "rg", + "glob", + "task" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.0.73" + }, + { + "agent": "copilot", + "capture_sha256": "07b91e3b6215218295878f1e449ba61bddcd860de1393756abc1d0b4f502d224", + "dialect": "openai_chat", + "executable_sha256": "959e8ea2c63474956adc1de9c884df9018a686a9c5463e8366b438161c01c8f8", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "powershell", + "read_powershell", + "stop_powershell", + "list_powershell", + "apply_patch", + "view", + "fetch_copilot_cli_documentation", + "skill", + "sql", + "session_store_sql", + "read_agent", + "list_agents", + "write_agent", + "rg", + "glob", + "task" + ], + "top_level_keys": [ + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.0.73" + }, + { + "agent": "cursor", + "capture_sha256": "143a5f1053a9f08cdbd3c9b60d7d9bdc635ecc7fcf13743c995b9929da450b1d", + "dialect": "cursor_connect_proto", + "executable_sha256": "eed61c5224668c9236334c4c68936a16aecc37374b592f59e31eb50433817831", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/agent.v1.AgentService/Run", + "request_shape": { + "message": "agent.v1.AgentClientMessage", + "method": "Run", + "service": "agent.v1.AgentService", + "stream": "bidirectional" + }, + "transport": "http2" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "cursor_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2026.07.20-8cc9c0b" + }, + { + "agent": "cursor", + "capture_sha256": "9cfbafc5b68a8622d6a53e8526f67c2c1283d5a4fa2c329207a4b376e5d60af7", + "dialect": "cursor_connect_proto", + "executable_sha256": "eed61c5224668c9236334c4c68936a16aecc37374b592f59e31eb50433817831", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/agent.v1.AgentService/Run", + "request_shape": { + "message": "agent.v1.AgentClientMessage", + "method": "Run", + "service": "agent.v1.AgentService", + "stream": "bidirectional" + }, + "transport": "http2" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "cursor_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "2026.07.20-8cc9c0b" + }, + { + "agent": "cursor", + "capture_sha256": "9cadbd1499db106aee6479bec218227552d5a8b804ec194f5c550db77aec5763", + "dialect": "cursor_connect_proto", + "executable_sha256": "eed61c5224668c9236334c4c68936a16aecc37374b592f59e31eb50433817831", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "connect_envelope", + "media_type": "application/connect+proto", + "method": "POST", + "path": "/agent.v1.AgentService/Run", + "request_shape": { + "message": "agent.v1.AgentClientMessage", + "method": "Run", + "service": "agent.v1.AgentService", + "stream": "bidirectional" + }, + "transport": "http2" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "cursor_connect_proto", + "framing": "connect_envelope", + "tool_call": "native_shell" + }, + "runtime": "wsl", + "version": "2026.07.20-8cc9c0b" + }, + { + "agent": "gemini", + "capture_sha256": "59a1e83ce73a7a4f8912e0b8780ae084e6c35814952dc4d3930406fc8f1770d7", + "dialect": "gemini_generate_content", + "executable_sha256": "a2533ac23365a1c72b847780b53235d38cf21cc7ee736c98dfa8cbb70b1a9425", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1beta/models/pitot-control:streamGenerateContent", + "request_shape": { + "has_tools": true, + "stream": false, + "tool_names": [ + "update_topic", + "list_directory", + "read_file", + "grep_search", + "glob", + "replace", + "write_file", + "web_fetch", + "run_shell_command", + "list_background_processes", + "read_background_output", + "google_web_search", + "enter_plan_mode", + "invoke_agent", + "activate_skill" + ], + "top_level_keys": [ + "contents", + "generationConfig", + "systemInstruction", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "gemini_generate_content", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.51.0" + }, + { + "agent": "gemini", + "capture_sha256": "f0f7bf0d7b2e88c03194449ccb25b8a7a534ee2d050b5f560fc578df069ef9c9", + "dialect": "gemini_generate_content", + "executable_sha256": "a2533ac23365a1c72b847780b53235d38cf21cc7ee736c98dfa8cbb70b1a9425", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1beta/models/pitot-control:streamGenerateContent", + "request_shape": { + "has_tools": true, + "stream": false, + "tool_names": [ + "update_topic", + "list_directory", + "read_file", + "grep_search", + "glob", + "replace", + "write_file", + "web_fetch", + "run_shell_command", + "list_background_processes", + "read_background_output", + "google_web_search", + "enter_plan_mode", + "invoke_agent", + "activate_skill" + ], + "top_level_keys": [ + "contents", + "generationConfig", + "systemInstruction", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "gemini_generate_content", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.51.0" + }, + { + "agent": "gemini", + "capture_sha256": "e3e0a192ec038c2fbc76ace53288ec14ebb6032954cea331c8eac495f0a928a6", + "dialect": "gemini_generate_content", + "executable_sha256": "ed3d5b269acefc5dffb342df0a214b2351794244c520c124d3556bc5ea6064c5", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1beta/models/pitot-control:streamGenerateContent", + "request_shape": { + "has_tools": true, + "stream": false, + "tool_names": [ + "update_topic", + "list_directory", + "read_file", + "grep_search", + "glob", + "replace", + "write_file", + "web_fetch", + "run_shell_command", + "list_background_processes", + "read_background_output", + "google_web_search", + "enter_plan_mode", + "invoke_agent", + "activate_skill" + ], + "top_level_keys": [ + "contents", + "generationConfig", + "systemInstruction", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "gemini_generate_content", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.51.0" + }, + { + "agent": "kimi", + "capture_sha256": "5c5a51e69a4686eedf0ed78648fbff64ecb850636404173dc6c061acd6b4bcad", + "dialect": "openai_chat", + "executable_sha256": "5cccf53604f20c5499ea10c3094298f49a1ad59fa90cddd9fd7e0ba44815fdd3", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Agent", + "AgentSwarm", + "AskUserQuestion", + "Bash", + "CreateGoal", + "CronCreate", + "CronDelete", + "CronList", + "Edit", + "EnterPlanMode", + "ExitPlanMode", + "FetchURL", + "GetGoal", + "Glob", + "Grep", + "Read", + "SetGoalBudget", + "Skill", + "TaskList", + "TaskOutput", + "TaskStop", + "TodoList", + "UpdateGoal", + "Write" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "prompt_cache_key", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.29.0" + }, + { + "agent": "kimi", + "capture_sha256": "11ece29c94e071bec5f206d9ba00e19233db733be791457d21c7a925bc8d20eb", + "dialect": "openai_chat", + "executable_sha256": "44f0aed58655790b78fe8ac1bf44217f3fc0e65977abb7426f9edca943ddc444", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Agent", + "AgentSwarm", + "AskUserQuestion", + "Bash", + "CreateGoal", + "CronCreate", + "CronDelete", + "CronList", + "Edit", + "EnterPlanMode", + "ExitPlanMode", + "FetchURL", + "GetGoal", + "Glob", + "Grep", + "Read", + "SetGoalBudget", + "Skill", + "TaskList", + "TaskOutput", + "TaskStop", + "TodoList", + "UpdateGoal", + "Write" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "prompt_cache_key", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.29.0" + }, + { + "agent": "kimi", + "capture_sha256": "4e0b6ea5c3c513f1e281a8a2d25e32025d9fb11bf995163fdc74a704b42bf44a", + "dialect": "openai_chat", + "executable_sha256": "c9daf5aec27ca9c35597a14d5cf05692f0aaf1d31d4d825ae6c988c404ecca6e", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "Agent", + "AgentSwarm", + "AskUserQuestion", + "Bash", + "CreateGoal", + "CronCreate", + "CronDelete", + "CronList", + "Edit", + "EnterPlanMode", + "ExitPlanMode", + "FetchURL", + "GetGoal", + "Glob", + "Grep", + "Read", + "SetGoalBudget", + "Skill", + "TaskList", + "TaskOutput", + "TaskStop", + "TodoList", + "UpdateGoal", + "Write" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "prompt_cache_key", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.29.0" + }, + { + "agent": "opencode", + "capture_sha256": "db834ed83cf489dc517c27e2149f9783ae663c20055c2283dabcbdf08ff86457", + "dialect": "openai_responses", + "executable_sha256": "9449af91f517eacc2b0742fa93ae0da64fa6e5db7b714e30c62edea2a8de3f98", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "edit", + "glob", + "grep", + "read", + "skill", + "task", + "todowrite", + "webfetch", + "write" + ], + "top_level_keys": [ + "input", + "model", + "prompt_cache_key", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.18.4" + }, + { + "agent": "opencode", + "capture_sha256": "8c98437528ae0a8f99e007b94cb1176a17943e9fbbd914643f60733152f4ba60", + "dialect": "openai_responses", + "executable_sha256": "6ce6570e7db9a40e7bd3304ebdfff607920bde8cafd2eb5587bd7a26f89ba0b5", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "edit", + "glob", + "grep", + "read", + "skill", + "task", + "todowrite", + "webfetch", + "write" + ], + "top_level_keys": [ + "input", + "model", + "prompt_cache_key", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.18.4" + }, + { + "agent": "opencode", + "capture_sha256": "509555970aabc2e5aa178412295bd0d773467cdeb22d2b95000e2e5ac1d17ad4", + "dialect": "openai_responses", + "executable_sha256": "b53b698473bfa46e09487e485a7f1ad5b4881f8a8b319d3619aa251f3be8ae10", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/responses", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "bash", + "edit", + "glob", + "grep", + "read", + "skill", + "task", + "todowrite", + "webfetch", + "write" + ], + "top_level_keys": [ + "input", + "model", + "prompt_cache_key", + "store", + "stream", + "tool_choice", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_responses", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "1.18.4" + }, + { + "agent": "pi", + "capture_sha256": "fbad33dcce5982fee3ab29cacb251afbce95e15d319cf48f2933031e5a4e0df0", + "dialect": "openai_chat", + "executable_sha256": "af302f231437eaf6f37691bce4b34234fcb626bcb5eb3910d4fc3f6519bf78ca", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read", + "bash", + "edit", + "write" + ], + "top_level_keys": [ + "max_completion_tokens", + "messages", + "model", + "store", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.81.1" + }, + { + "agent": "pi", + "capture_sha256": "c747e94254dc01646aa3624bb95b2a084a9ecc7a3ebb8f82322cceef051bfe0e", + "dialect": "openai_chat", + "executable_sha256": "af302f231437eaf6f37691bce4b34234fcb626bcb5eb3910d4fc3f6519bf78ca", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read", + "bash", + "edit", + "write" + ], + "top_level_keys": [ + "max_completion_tokens", + "messages", + "model", + "store", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.81.1" + }, + { + "agent": "pi", + "capture_sha256": "ef7db6f237ae1ed9a02420404db9555820f6d8ff16b46462103c0436d39e0533", + "dialect": "openai_chat", + "executable_sha256": "7f4c35fbbbe908301c12e06b9e6b7017c6b9cbcc24c34e26615a08f9f60fa53e", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "read", + "bash", + "edit", + "write" + ], + "top_level_keys": [ + "max_completion_tokens", + "messages", + "model", + "store", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.81.1" + }, + { + "agent": "qwen", + "capture_sha256": "f31eb243bbc121471fdecb50d50d177b12e84075cb89c52a1dc7af01ba0b9852", + "dialect": "openai_chat", + "executable_sha256": "eac1ed6f8ed71466fa05cf7b2a63fc0cf252b1ad79183735bba365bbeda3cafb", + "platform": "macos", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "agent", + "edit", + "glob", + "grep_search", + "list_directory", + "notebook_edit", + "read_file", + "run_shell_command", + "skill", + "todo_write", + "tool_search", + "write_file" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.20.1" + }, + { + "agent": "qwen", + "capture_sha256": "27da2424b8153808118fe54ae7de564c0793c4c0151b77934636a69d596e934a", + "dialect": "openai_chat", + "executable_sha256": "eac1ed6f8ed71466fa05cf7b2a63fc0cf252b1ad79183735bba365bbeda3cafb", + "platform": "ubuntu", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "agent", + "edit", + "glob", + "grep_search", + "list_directory", + "notebook_edit", + "read_file", + "run_shell_command", + "skill", + "todo_write", + "tool_search", + "write_file" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.20.1" + }, + { + "agent": "qwen", + "capture_sha256": "79c8dc86ad30a5e67c4259de7f438df2618d05d611bff69b88e4d67c8f64767a", + "dialect": "openai_chat", + "executable_sha256": "a5c19d0c03467074c1ff755ec604422223e1019fa19f8a9bf2f81dd84f32e070", + "platform": "windows", + "provenance": "pinned_real_cli_capture", + "request": { + "framing": "sse", + "media_type": "application/json", + "method": "POST", + "path": "/v1/chat/completions", + "request_shape": { + "has_tools": true, + "stream": true, + "tool_names": [ + "agent", + "edit", + "glob", + "grep_search", + "list_directory", + "notebook_edit", + "read_file", + "run_shell_command", + "skill", + "todo_write", + "tool_search", + "write_file" + ], + "top_level_keys": [ + "max_tokens", + "messages", + "model", + "stream", + "stream_options", + "tools" + ] + }, + "transport": "http1" + }, + "response": { + "acceptance": "nonce_tool_result_round_trip", + "encoder": "openai_chat", + "framing": "sse", + "tool_call": "native_shell" + }, + "runtime": "native", + "version": "0.20.1" + } + ], + "schema_version": 2 +} diff --git a/tests/install_real_agent.py b/tests/install_real_agent.py new file mode 100644 index 0000000..a8b1170 --- /dev/null +++ b/tests/install_real_agent.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Install and attest the exact released CLI declared by Pitot's supervisor.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform as host_platform +import shutil +import subprocess +import tempfile + + +LAB = Path(__file__).resolve().parent.parent +ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) + + +def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=True, text=True, **kwargs) + + +def npm_executable() -> str: + """Resolve npm's platform launcher instead of relying on PATHEXT in Python.""" + name = "npm.cmd" if os.name == "nt" else "npm" + return shutil.which(name) or name + + +def install(agent: dict[str, object], platform: str, runtime: str) -> None: + version = str(agent["version"]) + installer = agent["installer"] + kind, package = installer["kind"], installer["package"] + if runtime == "wsl": + if platform != "windows" or agent["id"] != "cursor": + raise ValueError("WSL is reserved for Cursor on Windows") + url = f"{package}/linux/x64/agent-cli-package.tar.gz" + archive = f"/tmp/pitot-cursor-{version}.tar.gz" + release = f"/usr/local/share/pitot-cursor/{version}" + wsl = ["wsl.exe", "--distribution", "Ubuntu", "--"] + run([*wsl, "curl", "-fsSL", url, "-o", archive]) + run([*wsl, "mkdir", "-p", release, "/usr/local/bin"]) + run([*wsl, "tar", "-xzf", archive, "-C", release, "--strip-components=1"]) + run([*wsl, "chmod", "0755", f"{release}/cursor-agent"]) + run([*wsl, "ln", "-sfn", f"{release}/cursor-agent", "/usr/local/bin/agent"]) + run([*wsl, "/usr/local/bin/agent", "--version"]) + return + if kind == "npm": + run([npm_executable(), "install", "--global", "--ignore-scripts=false", f"{package}@{version}"]) + elif kind == "kimi_release": + if platform == "windows": + # Use the native Kimi Code installer. The legacy /install.ps1 + # endpoint installs the unrelated Python kimi-cli and ignores the + # KIMI_VERSION contract used by the released Kimi Code binary. + command = ( + f"$env:KIMI_VERSION='{version}'; " + "irm https://code.kimi.com/kimi-code/install.ps1 | iex" + ) + run(["powershell", "-NoProfile", "-NonInteractive", "-Command", command]) + else: + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "install.sh" + run(["curl", "-fsSL", "https://code.kimi.com/kimi-code/install.sh", "-o", str(target)]) + environment = {**os.environ, "KIMI_VERSION": version} + run(["bash", str(target)], env=environment) + elif kind == "cursor_release": + os_name = {"ubuntu": "linux", "macos": "darwin"}[platform] + machine = host_platform.machine().lower() + arch = "arm64" if machine in {"arm64", "aarch64"} else "x64" + url = f"{package}/{os_name}/{arch}/agent-cli-package.tar.gz" + destination = Path.home() / ".local" / "bin" + destination.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "cursor.tar.gz" + run(["curl", "-fsSL", url, "-o", str(archive)]) + run(["tar", "-xzf", str(archive), "-C", directory]) + candidates = [path for path in Path(directory).rglob("cursor-agent") if path.is_file()] + if len(candidates) != 1: + raise RuntimeError(f"Cursor archive contained {len(candidates)} agent executables") + release = Path.home() / ".local" / "share" / "pitot-cursor" / version + shutil.copytree(candidates[0].parent, release, dirs_exist_ok=True) + installed = release / "cursor-agent" + installed.chmod(0o755) + link = destination / "agent" + temporary_link = destination / f".agent-{version}.tmp" + if temporary_link.exists() or temporary_link.is_symlink(): + temporary_link.unlink() + temporary_link.symlink_to(installed) + temporary_link.replace(link) + else: + raise ValueError(f"unsupported installer: {kind}") + + +def executable_receipt(agent: dict[str, object], platform: str, runtime: str) -> dict[str, object]: + executable, version = str(agent["executable"]), str(agent["version"]) + if runtime == "wsl": + resolve = ( + f"if command -v {executable} >/dev/null; then command -v {executable}; " + f'elif [ -x "$HOME/.local/bin/{executable}" ]; then printf "%s\\n" "$HOME/.local/bin/{executable}"; ' + "else exit 1; fi" + ) + path = run(["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", resolve], stdout=subprocess.PIPE).stdout.strip() + output = run(["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", f"{path} --version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.strip() + else: + path = "" + if agent["installer"]["kind"] == "npm": + prefix = Path(run([npm_executable(), "prefix", "--global"], stdout=subprocess.PIPE).stdout.strip()) + candidate = prefix / (f"{executable}.cmd" if os.name == "nt" else f"bin/{executable}") + if candidate.is_file(): path = str(candidate) + path = path or shutil.which(executable) or "" + if not path: + suffix = f"{executable}.exe" if os.name == "nt" else executable + home_candidates = ( + Path.home() / ".local" / "bin" / suffix, + Path.home() / ".kimi-code" / "bin" / suffix, + ) + path = next((str(candidate) for candidate in home_candidates if candidate.is_file()), "") + if not path: + raise RuntimeError(f"installed executable {executable!r} is not on PATH") + output = run([path, "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.strip() + if version not in output: + raise RuntimeError(f"{executable} reported {output!r}, expected pinned version {version}") + if runtime == "wsl": + digest = run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "bash", "-lc", f"sha256sum {path} | cut -d' ' -f1"], + stdout=subprocess.PIPE, + ).stdout.strip() + else: + digest = hashlib.sha256(Path(path).read_bytes()).hexdigest() + return { + "schema_version": 1, + "agent": agent["id"], + "version": version, + "executable": path, + "executable_sha256": digest, + "version_output": output, + "installer": agent["installer"]["kind"], + "runtime": runtime, + "platform": platform, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + agent = next((item for item in MANIFEST["agents"] if item["id"] == args.agent), None) + if agent is None: + raise SystemExit(f"unknown supervised agent: {args.agent}") + runtime = agent["runtime"].get(args.platform) + if runtime is None: + raise SystemExit(f"unsupported platform: {args.platform}") + install(agent, args.platform, runtime) + receipt = executable_receipt(agent, args.platform, runtime) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"Installed {args.agent} {agent['version']} at {receipt['executable']} ({runtime})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/mock_anthropic_server.js b/tests/mock_anthropic_server.js new file mode 100644 index 0000000..391d7f3 --- /dev/null +++ b/tests/mock_anthropic_server.js @@ -0,0 +1,213 @@ +import http from 'http'; + +import { isStructuredMetadataRequest } from './mock_anthropic_protocol.js'; + +const PORT = 8080; + +const server = http.createServer((req, res) => { + console.log(`[MOCK API] ${req.method} ${req.url}`); + // Set CORS headers + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Headers', '*'); + res.setHeader('Access-Control-Allow-Methods', '*'); + + if (req.method === 'OPTIONS') { + res.writeHead(200); + res.end(); + return; + } + + if (req.url.startsWith('/v1/messages') && req.method === 'POST') { + let body = ''; + req.on('data', chunk => { + body += chunk; + }); + + req.on('end', () => { + try { + console.log(`[MOCK API] Request Body: ${body}`); + const payload = JSON.parse(body || '{}'); + const messages = payload.messages || []; + const lastMessage = messages[messages.length - 1] || {}; + const requestedModel = payload.model || 'claude-3-5-sonnet'; + + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + + if (isStructuredMetadataRequest(payload)) { + sendTextResponse(res, requestedModel, 'msg_metadata_42', '{"title":"List directory"}'); + return; + } + + // If the last message contains a tool result, we have successfully run the tool! + const hasToolResult = lastMessage.content && lastMessage.content.some(c => c.type === 'tool_result'); + + if (hasToolResult) { + // Send a final message saying we're done + sendSSEEvent(res, 'message_start', { + type: 'message_start', + message: { id: 'msg_done_42', type: 'message', role: 'assistant', content: [], model: requestedModel, stop_reason: null, stop_sequence: null, usage: { input_tokens: 100, output_tokens: 1 } } + }); + sendSSEEvent(res, 'content_block_start', { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' } + }); + sendSSEEvent(res, 'content_block_delta', { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'E2E Verification Complete: Tool executed successfully!' } + }); + sendSSEEvent(res, 'content_block_stop', { type: 'content_block_stop', index: 0 }); + sendSSEEvent(res, 'message_delta', { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 20 } }); + sendSSEEvent(res, 'message_stop', { type: 'message_stop' }); + res.end(); + + // Gracefully shut down the server after a short delay since E2E is complete! + setTimeout(() => { + server.close(() => { + process.exit(0); + }); + }, 1000); + return; + } + + // Instruct Claude Code to execute the Bash tool with 'git status --short' + sendSSEEvent(res, 'message_start', { + type: 'message_start', + message: { id: 'msg_tool_42', type: 'message', role: 'assistant', content: [], model: requestedModel, stop_reason: null, stop_sequence: null, usage: { input_tokens: 50, output_tokens: 1 } } + }); + sendSSEEvent(res, 'content_block_start', { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'toolu_e2e_42', name: 'Bash', input: {} } + }); + sendSSEEvent(res, 'content_block_delta', { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{"command": "git status --short"}' } + }); + sendSSEEvent(res, 'content_block_stop', { type: 'content_block_stop', index: 0 }); + sendSSEEvent(res, 'message_delta', { type: 'message_delta', delta: { stop_reason: 'tool_use', stop_sequence: null }, usage: { output_tokens: 20 } }); + sendSSEEvent(res, 'message_stop', { type: 'message_stop' }); + res.end(); + } catch (err) { + console.error(`[MOCK API ERROR] ${err.stack}`); + res.writeHead(500); + res.end(JSON.stringify({ error: err.message })); + } + }); + } else if (req.url.startsWith('/v1/chat/completions') && req.method === 'POST') { + let body = ''; + req.on('data', chunk => { + body += chunk; + }); + + req.on('end', () => { + try { + console.log(`[MOCK API] OpenAI Request Body: ${body}`); + const payload = JSON.parse(body || '{}'); + const messages = payload.messages || []; + const lastMessage = messages[messages.length - 1] || {}; + const requestedModel = payload.model || 'gpt-4'; + + res.setHeader('Content-Type', 'application/json'); + + // Check if last message contains tool execution response + const isToolResult = lastMessage.role === 'tool' || lastMessage.role === 'function'; + + if (isToolResult) { + // Send final OpenAI-compatible text completion response + res.writeHead(200); + res.end(JSON.stringify({ + id: "chatcmpl-done-42", + object: "chat.completion", + created: 1781881881, + model: requestedModel, + choices: [{ + index: 0, + message: { + role: "assistant", + content: "E2E Verification Complete: Tool executed successfully!" + }, + finish_reason: "stop" + }], + usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 } + })); + + // Gracefully shut down + setTimeout(() => { + server.close(() => { + process.exit(0); + }); + }, 1000); + return; + } + + // Return a tool_calls completion telling Cursor/Codex to execute git status --short + res.writeHead(200); + res.end(JSON.stringify({ + id: "chatcmpl-tool-42", + object: "chat.completion", + created: 1781881881, + model: requestedModel, + choices: [{ + index: 0, + message: { + role: "assistant", + content: null, + tool_calls: [{ + id: "call_e2e_42", + type: "function", + function: { + name: "Bash", + arguments: "{\"command\":\"git status --short\"}" + } + }] + }, + finish_reason: "tool_calls" + }], + usage: { prompt_tokens: 50, completion_tokens: 20, total_tokens: 70 } + })); + } catch (err) { + console.error(`[MOCK API ERROR] ${err.stack}`); + res.writeHead(500); + res.end(JSON.stringify({ error: err.message })); + } + }); + } else { + res.writeHead(404); + res.end(); + } +}); + +function sendSSEEvent(res, eventName, data) { + res.write(`event: ${eventName}\n`); + res.write(`data: ${JSON.stringify(data)}\n\n`); +} + +function sendTextResponse(res, model, messageId, text) { + sendSSEEvent(res, 'message_start', { + type: 'message_start', + message: { id: messageId, type: 'message', role: 'assistant', content: [], model, stop_reason: null, stop_sequence: null, usage: { input_tokens: 10, output_tokens: 1 } } + }); + sendSSEEvent(res, 'content_block_start', { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' } + }); + sendSSEEvent(res, 'content_block_delta', { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text } + }); + sendSSEEvent(res, 'content_block_stop', { type: 'content_block_stop', index: 0 }); + sendSSEEvent(res, 'message_delta', { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 10 } }); + sendSSEEvent(res, 'message_stop', { type: 'message_stop' }); + res.end(); +} + +server.listen(PORT, () => { + console.log(`Mock Anthropic Server running on http://localhost:${PORT}`); +}); diff --git a/tests/model_control_proxy.py b/tests/model_control_proxy.py new file mode 100644 index 0000000..2d7ab36 --- /dev/null +++ b/tests/model_control_proxy.py @@ -0,0 +1,486 @@ +#!/usr/bin/env python3 +"""Deterministic model server for real prompt-to-hook agent E2E sessions.""" + +from __future__ import annotations + +import argparse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import threading +import time +import uuid + + +def contains(value: object, needle: str) -> bool: + return needle in json.dumps(value, separators=(",", ":"), ensure_ascii=True) + + +def tool_result_contains(protocol: str, body: dict[str, object], needle: str) -> bool: + """Search only native tool-result fields, never echoed tool-call arguments.""" + if protocol == "gemini_generate_content": + values = [] + for content in body.get("contents", []): + if not isinstance(content, dict): + continue + for part in content.get("parts", []): + if isinstance(part, dict) and isinstance(part.get("functionResponse"), dict): + values.append(part["functionResponse"].get("response")) + return any(contains(value, needle) for value in values) + if protocol in {"anthropic_messages", "openai_chat"}: + values = [] + for message in body.get("messages", []): + if not isinstance(message, dict): + continue + if protocol == "openai_chat" and message.get("role") == "tool": + values.append(message.get("content")) + for item in message.get("content", []) if isinstance(message.get("content"), list) else []: + if isinstance(item, dict) and item.get("type") == "tool_result": + values.append(item.get("content")) + return any(contains(value, needle) for value in values) + if protocol == "openai_responses": + values = [ + item.get("output") + for item in body.get("input", []) + if isinstance(item, dict) and item.get("type") == "function_call_output" + ] + return any(contains(value, needle) for value in values) + return False + + +class UnknownProtocol(ValueError): + pass + + +def classify_request(path: str, body: object) -> str: + """Classify only from the real CLI's observed request, never a manifest hint.""" + clean_path = path.split("?", 1)[0] + if not isinstance(body, dict): + raise UnknownProtocol("request body is not a JSON object") + candidates: list[str] = [] + if clean_path.endswith("/messages") and isinstance(body.get("messages"), list): + candidates.append("anthropic_messages") + if clean_path.endswith("/responses") and "input" in body: + candidates.append("openai_responses") + if ("generateContent" in clean_path or "streamGenerateContent" in clean_path) and isinstance(body.get("contents"), list): + candidates.append("gemini_generate_content") + if clean_path.endswith("/chat/completions") and isinstance(body.get("messages"), list): + candidates.append("openai_chat") + if len(candidates) != 1: + raise UnknownProtocol(f"request matched {len(candidates)} supported dialects") + return candidates[0] + + +def request_shape(body: dict[str, object]) -> dict[str, object]: + """Return a content-redacted structural fingerprint for review fixtures.""" + return { + "top_level_keys": sorted(body), + "has_tools": bool(body.get("tools")), + "stream": bool(body.get("stream")), + "tool_names": tool_names(body), + } + + +def protobuf_varint(value: int) -> bytes: + encoded = bytearray() + while value > 0x7F: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def protobuf_bytes(field: int, value: bytes) -> bytes: + return protobuf_varint((field << 3) | 2) + protobuf_varint(len(value)) + value + + +def cursor_model_details() -> bytes: + return b"".join(protobuf_bytes(field, b"pitot-control") for field in (1, 3, 4, 5)) + + +def cursor_auxiliary_response(path: str) -> bytes: + model = cursor_model_details() + if path.endswith("/GetUsableModels"): + return protobuf_bytes(1, model) + if path.endswith("/GetDefaultModelForCli"): + return protobuf_bytes(1, model) + return b"" + + +def tool_name(body: dict[str, object]) -> str: + names = tool_names(body) + preferred = ("Bash", "bash", "shell", "run_shell_command", "execute_command", "run_commands") + return next((name for name in preferred if name in names), names[0] if names else "bash") + + +def tool_names(body: dict[str, object]) -> list[str]: + names: list[str] = [] + for item in body.get("tools", []): + if isinstance(item, dict): + function = item.get("function") + name = function.get("name") if isinstance(function, dict) else item.get("name") + if isinstance(name, str): names.append(name) + declarations = item.get("functionDeclarations", []) + if isinstance(declarations, list): + names.extend(value["name"] for value in declarations if isinstance(value, dict) and isinstance(value.get("name"), str)) + return names + + +def tool_arguments(tool: str, command: str) -> dict[str, object]: + if tool == "exec_command": return {"cmd": command} + if tool == "run_commands": return {"commands": [command]} + if tool == "run_shell_command": return {"command": command, "is_background": False} + return {"command": command} + + +def function_response_shapes(value: object) -> list[dict[str, object]]: + found: list[dict[str, object]] = [] + if isinstance(value, dict): + response = value.get("functionResponse") + if isinstance(response, dict): + payload = response.get("response") + found.append({"name": response.get("name"), "response_keys": sorted(payload) if isinstance(payload, dict) else type(payload).__name__}) + for child in value.values(): found.extend(function_response_shapes(child)) + elif isinstance(value, list): + for child in value: found.extend(function_response_shapes(child)) + return found + + +def content_shapes(body: dict[str, object]) -> list[dict[str, object]]: + shapes: list[dict[str, object]] = [] + for content in body.get("contents", []): + if not isinstance(content, dict): continue + parts = content.get("parts", []) + shapes.append({"role": content.get("role"), "parts": [sorted(part) for part in parts if isinstance(part, dict)]}) + return shapes + + +def message_shapes(body: dict[str, object]) -> list[dict[str, object]]: + shapes: list[dict[str, object]] = [] + for message in body.get("messages", []): + if not isinstance(message, dict): continue + content = message.get("content") + serialized = json.dumps(content).lower() + item_shapes = [] + if isinstance(content, list): + for item in content: + if isinstance(item, dict): + item_shapes.append({"keys": sorted(item), "types": {key: type(value).__name__ for key, value in item.items()}}) + else: + item_shapes.append({"type": type(item).__name__}) + markers = ( + "hook", "permission", "denied", "error", "invalid", "background", + "cancel", "reject", "block", "fail", "not found", "exit code", + "timed out", "aborted", + ) + shapes.append({"role": message.get("role"), "keys": sorted(message), "content_type": type(content).__name__, "content_length": len(content) if isinstance(content, (str, list)) else None, "content_items": item_shapes, "markers": [marker for marker in markers if marker in serialized]}) + return shapes + + +def tool_schema_shapes(body: dict[str, object]) -> list[dict[str, object]]: + shapes: list[dict[str, object]] = [] + for item in body.get("tools", []): + if not isinstance(item, dict): continue + function = item.get("function") if isinstance(item.get("function"), dict) else item + params = function.get("parameters", {}) + properties = params.get("properties", {}) if isinstance(params, dict) else {} + property_shapes = {} + if isinstance(properties, dict): + for key, value in properties.items(): + if isinstance(value, dict): + items = value.get("items", {}) + property_shapes[key] = {"type": value.get("type"), "item_type": items.get("type") if isinstance(items, dict) else None, "item_properties": sorted(items.get("properties", {})) if isinstance(items, dict) and isinstance(items.get("properties"), dict) else []} + shapes.append({"name": function.get("name"), "properties": property_shapes}) + return shapes + + +class State: + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.lock = threading.Lock() + self.receipt = { + "schema_version": 1, + "agent": args.agent, + "protocol": None, + "nonce": args.nonce, + "initial_prompt_observed": False, + "tool_call_response_emitted": False, + "tool_result_observed": False, + "allow_tool_call_response_emitted": False, + "allow_tool_result_observed": False, + "deny_tool_call_response_emitted": False, + "denied_result_observed": False, + "final_response_emitted": False, + "selected_tool": None, + "endpoint_observed": None, + "unexpected_request": None, + "auxiliary_requests": 0, + } + + def save(self) -> None: + target = Path(self.args.receipt) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(".tmp") + temporary.write_text(json.dumps(self.receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(target) + + +def anthropic(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + content = [{"type": "text", "text": f"PITOT_E2E_COMPLETE {nonce}"}] if final else [ + {"type": "tool_use", "id": f"pitot_tool_{phase}", "name": tool, "input": tool_arguments(tool, command)} + ] + return {"id": "msg_pitot", "type": "message", "role": "assistant", "model": "pitot-control", "content": content, "stop_reason": "end_turn" if final else "tool_use", "usage": {"input_tokens": 1, "output_tokens": 1}} + + +def chat(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + message: dict[str, object] = {"role": "assistant", "content": f"PITOT_E2E_COMPLETE {nonce}" if final else None} + finish = "stop" if final else "tool_calls" + if not final: + message["tool_calls"] = [{"id": f"pitot_tool_{phase}", "type": "function", "function": {"name": tool, "arguments": json.dumps(tool_arguments(tool, command))}}] + return {"id": "chatcmpl-pitot", "object": "chat.completion", "created": 1, "model": "pitot-control", "choices": [{"index": 0, "message": message, "finish_reason": finish}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}} + + +def responses(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + output = [{"id": "msg_pitot", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": f"PITOT_E2E_COMPLETE {nonce}", "annotations": []}]}] if final else [ + {"id": f"fc_pitot_{phase}", "type": "function_call", "call_id": f"pitot_tool_{phase}", "name": tool, "arguments": json.dumps(tool_arguments(tool, command)), "status": "completed"} + ] + return {"id": "resp_pitot", "object": "response", "created_at": 1, "status": "completed", "model": "pitot-control", "output": output, "parallel_tool_calls": False, "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}} + + +def gemini(tool: str, command: str, final: bool, nonce: str = "", phase: str = "allow") -> dict[str, object]: + part = {"text": f"PITOT_E2E_COMPLETE {nonce}"} if final else {"functionCall": {"name": tool, "args": tool_arguments(tool, command)}, "thoughtSignature": "cGl0b3Q="} + return {"candidates": [{"content": {"role": "model", "parts": [part]}, "finishReason": "STOP"}], "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}} + + +class Handler(BaseHTTPRequestHandler): + server_version = "PitotModelControl/1" + + def log_message(self, format: str, *args: object) -> None: + return + + def do_GET(self) -> None: + state: State = self.server.state # type: ignore[attr-defined] + if self.path in {"/health", "/v1/models"}: + self.reply({"object": "list", "data": [{"id": "pitot-control", "object": "model"}]}) + else: + self.send_error(404) + + def do_POST(self) -> None: + state: State = self.server.state # type: ignore[attr-defined] + try: + length = int(self.headers.get("content-length", "0")) + raw_body = self.rfile.read(length) + body = json.loads(raw_body or b"{}") + except (ValueError, json.JSONDecodeError): + self.send_error(400, "invalid JSON") + return + nonce = state.args.nonce + try: + protocol = classify_request(self.path, body) + except UnknownProtocol as error: + with state.lock: + state.receipt["unexpected_request"] = { + "path": self.path.split("?", 1)[0], + "top_level_keys": sorted(body) if isinstance(body, dict) else [], + "classification_error": str(error), + } + state.save() + self.send_error(422, "unrecognized model request structure") + return + with state.lock: + if state.receipt["protocol"] not in {None, protocol}: + self.send_error(409, "request dialect changed during session") + return + state.receipt["protocol"] = protocol + if not body.get("tools") and state.receipt["auxiliary_requests"] == 0: + state.receipt["auxiliary_requests"] = 1 + state.receipt["auxiliary_request"] = { + "path": self.path.split("?", 1)[0], + "top_level_keys": sorted(body), + "nonce_present": contains(body, nonce), + } + state.save() + if protocol == "anthropic_messages": + payload = anthropic("", "", True) + payload["content"][0]["text"] = '{"title":"Pitot E2E"}' + elif protocol == "openai_responses": payload = responses("", "", True) + elif protocol == "gemini_generate_content": payload = gemini("", "", True) + else: payload = chat("", "", True) + if isinstance(body.get("model"), str): payload["model"] = body["model"] + self.reply(payload, stream=bool(body.get("stream")), protocol=protocol) + return + allow_result = tool_result_contains(protocol, body, f"PITOT_CANARY_RESULT PITOT_ALLOW {nonce}") + denied_result = tool_result_contains(protocol, body, f"PITOT_CONTROLLER_DENY {nonce}") + phase = "allow" + final = False + if not state.receipt["initial_prompt_observed"]: + if not contains(body, nonce): + self.send_error(409, "initial request omitted session nonce") + return + state.receipt["initial_prompt_observed"] = True + state.receipt["endpoint_observed"] = { + "transport": "http1", + "method": "POST", + "path": self.path.split("?", 1)[0], + "media_type": self.headers.get("content-type", "").split(";", 1)[0].strip().lower(), + "framing": "sse" if bool(body.get("stream")) or "streamGenerateContent" in self.path or protocol == "openai_responses" else "json", + "request_shape": request_shape(body), + } + elif allow_result and not state.receipt["allow_tool_result_observed"]: + state.receipt["allow_tool_result_observed"] = True + state.receipt["tool_result_observed"] = True + phase = "deny" + elif denied_result and state.receipt["deny_tool_call_response_emitted"]: + state.receipt["denied_result_observed"] = True + phase = "final" + final = True + else: + state.receipt["unexpected_request"] = { + "path": self.path.split("?", 1)[0], + "top_level_keys": sorted(body), + "nonce_present": contains(body, nonce), + "allow_result_present": allow_result, + "denied_result_present": denied_result, + "function_responses": function_response_shapes(body), + "contents": content_shapes(body), + "messages": message_shapes(body), + } + state.save() + self.send_error(409, "agent request did not advance the supervised allow/deny trajectory") + return + tool = tool_name(body) + state.receipt["selected_tool"] = tool + state.receipt["advertised_tools"] = tool_names(body) + state.receipt["tool_structures"] = [sorted(item) if isinstance(item, dict) else type(item).__name__ for item in body.get("tools", [])] + state.receipt["tool_schemas"] = tool_schema_shapes(body) + command = f"{state.args.canary_command} PITOT_{phase.upper()} {nonce}" + if state.args.response_fault == "text" and not final: + if protocol == "anthropic_messages": payload = anthropic(tool, command, True, nonce, phase) + elif protocol == "openai_responses": payload = responses(tool, command, True, nonce, phase) + elif protocol == "gemini_generate_content": payload = gemini(tool, command, True, nonce, phase) + else: payload = chat(tool, command, True, nonce, phase) + state.receipt["fault_response_emitted"] = "text" + elif protocol == "anthropic_messages": payload = anthropic(tool, command, final, nonce, phase) + elif protocol == "openai_responses": payload = responses(tool, command, final, nonce, phase) + elif protocol == "gemini_generate_content": payload = gemini(tool, command, final, nonce, phase) + elif protocol == "openai_chat": payload = chat(tool, command, final, nonce, phase) + else: + self.send_error(422, "unsupported protocol") + return + if isinstance(body.get("model"), str): + payload["model"] = body["model"] + if state.args.response_fault == "text" and not final: + pass + elif final: + state.receipt["final_response_emitted"] = True + elif phase == "deny": + state.receipt["deny_tool_call_response_emitted"] = True + else: + state.receipt["tool_call_response_emitted"] = True + state.receipt["allow_tool_call_response_emitted"] = True + state.save() + stream = bool(body.get("stream")) or "streamGenerateContent" in self.path or protocol == "openai_responses" + self.reply(payload, stream=stream, protocol=protocol) + + def reply(self, payload: dict[str, object], *, stream: bool = False, protocol: str = "") -> None: + if stream: + if protocol == "openai_responses": + item = payload["output"][0] + events = [("response.created", {"type": "response.created", "sequence_number": 0, "response": {**payload, "status": "in_progress", "output": []}})] + if item["type"] == "function_call": + start = {**item, "arguments": "", "status": "in_progress"} + events.extend([ + ("response.output_item.added", {"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": start}), + ("response.function_call_arguments.delta", {"type": "response.function_call_arguments.delta", "sequence_number": 2, "item_id": item["id"], "output_index": 0, "delta": item["arguments"]}), + ("response.function_call_arguments.done", {"type": "response.function_call_arguments.done", "sequence_number": 3, "item_id": item["id"], "output_index": 0, "arguments": item["arguments"]}), + ("response.output_item.done", {"type": "response.output_item.done", "sequence_number": 4, "output_index": 0, "item": item}), + ]) + else: + text_value = item["content"][0]["text"] + start = {**item, "status": "in_progress", "content": []} + part = {"type": "output_text", "text": "", "annotations": []} + events.extend([ + ("response.output_item.added", {"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": start}), + ("response.content_part.added", {"type": "response.content_part.added", "sequence_number": 2, "item_id": item["id"], "output_index": 0, "content_index": 0, "part": part}), + ("response.output_text.delta", {"type": "response.output_text.delta", "sequence_number": 3, "item_id": item["id"], "output_index": 0, "content_index": 0, "delta": text_value}), + ("response.output_text.done", {"type": "response.output_text.done", "sequence_number": 4, "item_id": item["id"], "output_index": 0, "content_index": 0, "text": text_value}), + ("response.content_part.done", {"type": "response.content_part.done", "sequence_number": 5, "item_id": item["id"], "output_index": 0, "content_index": 0, "part": item["content"][0]}), + ("response.output_item.done", {"type": "response.output_item.done", "sequence_number": 6, "output_index": 0, "item": item}), + ]) + events.append(("response.completed", {"type": "response.completed", "sequence_number": len(events) + 1, "response": payload})) + elif protocol == "anthropic_messages": + block = payload["content"][0] + if block["type"] == "tool_use": + start_block = {"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}} + deltas = [("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"], separators=(',', ':'))}})] + else: + start_block = {"type": "text", "text": ""} + deltas = [("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": block["text"]}})] + events = [ + ("message_start", {"type": "message_start", "message": {**payload, "content": [], "stop_reason": None, "stop_sequence": None}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": start_block}), + *deltas, + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": payload["stop_reason"], "stop_sequence": None}, "usage": {"output_tokens": 1}}), + ("message_stop", {"type": "message_stop"}), + ] + elif protocol == "openai_chat": + message = payload["choices"][0]["message"] + if message.get("tool_calls"): + call = message["tool_calls"][0] + delta = {"role": "assistant", "content": None, "tool_calls": [{"index": 0, **call}]} + else: + delta = {"role": "assistant", "content": message.get("content")} + base = {"id": payload["id"], "object": "chat.completion.chunk", "created": payload["created"], "model": payload["model"]} + events = [ + ("", {**base, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}), + ("", {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": payload["choices"][0]["finish_reason"]}]}), + ] + else: + events = [("", payload)] + encoded = b"".join( + ((f"event: {name}\n" if name else "") + f"data: {json.dumps(item, separators=(',', ':'))}\n\n").encode() + for name, item in events + ) + (b"data: [DONE]\n\n" if protocol == "openai_chat" else b"") + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("content-length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + return + encoded = json.dumps(payload, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def reply_bytes(self, payload: bytes, content_type: str) -> None: + self.send_response(200) + self.send_header("content-type", content_type) + self.send_header("content-length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True) + parser.add_argument("--nonce", required=True) + parser.add_argument("--receipt", required=True) + parser.add_argument("--ready-file", required=True) + parser.add_argument("--canary-command", default="pitot-e2e-canary") + parser.add_argument("--response-fault", choices=("none", "text"), default="none") + args = parser.parse_args() + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.state = State(args) # type: ignore[attr-defined] + Path(args.ready_file).write_text(f"http://127.0.0.1:{server.server_port}\n", encoding="utf-8") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/real_agent_driver.py b/tests/real_agent_driver.py new file mode 100644 index 0000000..28590db --- /dev/null +++ b/tests/real_agent_driver.py @@ -0,0 +1,613 @@ +#!/usr/bin/env python3 +"""Run one released agent through prompt, model, hook, Pitot, and tool result.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import secrets +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import time + + +LAB = Path(__file__).resolve().parent.parent +ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) +ENDPOINT_PROVENANCE = json.loads((LAB / "tests/endpoint-provenance.json").read_text(encoding="utf-8")) + + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def hook_group(event: str, matcher: str, command: str) -> dict[str, object]: + return {"hooks": {event: [{"matcher": matcher, "hooks": [{"name": "pitot", "type": "command", "command": command}]}]}} + + +def wsl_path(path: Path) -> str: + """Map a resolved Windows runner path through WSL's drive automount.""" + native = str(path.resolve()).replace("\\", "/") + matched = re.fullmatch(r"([A-Za-z]):(/.*)", native) + if matched is None: + raise RuntimeError(f"cannot map native path into WSL: {native!r}") + return f"/mnt/{matched.group(1).lower()}{matched.group(2)}" + + +def windows_host() -> bool: + return os.name == "nt" + + +def render_canary_command(executable: str, receipt: str, native_windows: bool) -> str: + if native_windows: + executable = executable.replace("\\", "/") + receipt = receipt.replace("\\", "/") + return subprocess.list2cmdline([executable, "--role", "canary", "--receipt", receipt]) + return shlex.join([executable, "--role", "canary", "--receipt", receipt]) + + +def configure( + agent: str, + home: Path, + project: Path, + witness_command: str, + proxy: str, + *, + pitot_command: str, + witness_receipt: str, + nonce: str, + runtime_path: str, +) -> tuple[list[str], dict[str, str]]: + env: dict[str, str] = { + "HOME": str(home), + "USERPROFILE": str(home), + "OPENAI_API_KEY": "pitot-local-only", + "ANTHROPIC_API_KEY": "pitot-local-only", + "GEMINI_API_KEY": "pitot-local-only", + "GOOGLE_API_KEY": "pitot-local-only", + "PITOT_BIN": witness_command, + "PITOT_RUNTIME": runtime_path, + } + # Gemini, Codex, and Copilot execute command hooks through PowerShell on + # Windows. A quoted executable is only a string there; the call operator is + # required to invoke it. Explicit receipt arguments also survive the hosts' + # intentionally reduced hook environments. + witness_invocation = f'& "{witness_command}"' if windows_host() else f'"{witness_command}"' + witnessed = ( + f'{witness_invocation} --real-bin "{pitot_command}" ' + f'--receipt "{witness_receipt}" --nonce "{nonce}"' + ) + witnessed_direct = ( + f'"{witness_command}" --real-bin "{pitot_command}" ' + f'--receipt "{witness_receipt}" --nonce "{nonce}"' + ) + prompt_flag: list[str] + if agent == "claude": + write_json(home / ".claude/settings.json", hook_group("PreToolUse", "Bash", f'{witnessed_direct} hook claude --runtime "{runtime_path}"')) + env["ANTHROPIC_BASE_URL"] = proxy + env["CLAUDE_CODE_MAX_RETRIES"] = "0" + prompt_flag = ["--print", "--dangerously-skip-permissions", "--tools", "Bash", "--model", "sonnet"] + elif agent == "codex": + if windows_host(): + bridge = LAB / "integrations/codex/PreToolUse.ps1" + hook_command = ( + f'& "{bridge}" ' + f'-Pitot "{witness_command}" -RealBin "{pitot_command}" ' + f'-Receipt "{witness_receipt}" -Nonce "{nonce}" -Runtime "{runtime_path}"' + ) + else: + hook_command = f'{witnessed} hook codex --runtime "{runtime_path}" >/dev/null' + write_json(home / ".codex/hooks.json", hook_group("PreToolUse", "Bash", hook_command)) + (home / ".codex/config.toml").write_text( + f'model = "pitot-control"\nmodel_provider = "pitot"\n[model_providers.pitot]\nname = "Pitot local control"\nbase_url = "{proxy}/v1"\nenv_key = "OPENAI_API_KEY"\nwire_api = "responses"\n', + encoding="utf-8", + ) + env["CODEX_HOME"] = str(home / ".codex") + prompt_flag = ["exec", "--dangerously-bypass-approvals-and-sandbox", "--dangerously-bypass-hook-trust", "--model", "pitot-control"] + elif agent == "copilot": + hooks = home / ".copilot/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/copilot" / ("PreToolUse.ps1" if windows_host() else "PreToolUse") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + hook_command = f'powershell -NoProfile -NonInteractive -File "{target}"' if windows_host() else str(target) + write_json(home / ".copilot/settings.json", hook_group("PreToolUse", "Bash", hook_command)) + env["PITOT_BIN"] = witness_command + env.update({ + "COPILOT_PROVIDER_BASE_URL": f"{proxy}/v1", + "COPILOT_PROVIDER_API_KEY": "pitot-local-only", + "COPILOT_PROVIDER_TYPE": "openai", + "COPILOT_PROVIDER_WIRE_API": "completions", + "COPILOT_PROVIDER_MODEL_ID": "gpt-4o", + "COPILOT_PROVIDER_WIRE_MODEL": "pitot-control", + "COPILOT_MODEL": "gpt-4o", + "COPILOT_OFFLINE": "true", + "COPILOT_HOME": str(home / ".copilot"), + }) + prompt_flag = ["--allow-all-tools", "--model", "gpt-4o", "--no-auto-update", "--no-remote"] + elif agent == "cursor": + source = LAB / "integrations/cursor/beforeShellExecution" + target = project / ".cursor/hooks/beforeShellExecution" + target.parent.mkdir(parents=True, exist_ok=True) + # Cursor runs in WSL on the Windows cell. A Windows checkout may expose + # CRLF bytes through /mnt/, which turns the bridge shebang into + # `bash\r` and makes Cursor fail the hook closed before Pitot can run. + target.write_bytes(source.read_bytes().replace(b"\r\n", b"\n")) + target.chmod(0o755) + hook_command = ( + f'"{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) + write_json(project / ".cursor/hooks.json", {"version": 1, "hooks": {"beforeShellExecution": [{"command": hook_command, "failClosed": True}]}}) + # Cursor exposes an authless CLI mode for endpoint compatibility tests; + # model inference is still supplied only by the pinned local endpoint. + env["CURSOR_AGENT_CLI_AUTHLESS_MODE"] = "true" + env["CURSOR_AUTH_TOKEN"] = "pitot-local-only" + prompt_flag = ["--endpoint", proxy, "--print", "--force"] + elif agent == "gemini": + hooks = home / ".gemini/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/gemini" / ("BeforeTool.ps1" if windows_host() else "BeforeTool") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + if windows_host(): + hook_command = ( + f'& "{target}" ' + f'-Pitot "{witness_command}" -RealBin "{pitot_command}" ' + f'-Receipt "{witness_receipt}" -Nonce "{nonce}" -Runtime "{runtime_path}"' + ) + else: + hook_command = ( + f'"{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) + settings = hook_group("BeforeTool", "run_shell_command", hook_command) + settings["security"] = {"auth": {"selectedType": "gemini-api-key"}, "folderTrust": {"enabled": False}} + write_json(home / ".gemini/settings.json", settings) + env.update({"GOOGLE_GEMINI_BASE_URL": proxy, "GEMINI_CLI_HOME": str(home), "GEMINI_CLI_TRUST_WORKSPACE": "true", "PITOT_BIN": witness_command}) + prompt_flag = ["--skip-trust", "--approval-mode", "yolo", "--model", "pitot-control", "-p"] + elif agent == "kimi": + config = home / ".kimi-code/config.toml" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text( + 'default_model = "pitot-control"\n' + '[providers.pitot]\n' + 'type = "openai"\n' + f'base_url = {json.dumps(proxy + "/v1")}\n' + 'api_key = "pitot-local-only"\n' + '[models."pitot-control"]\n' + 'provider = "pitot"\n' + 'model = "pitot-control"\n' + 'max_context_size = 32768\n' + 'capabilities = ["tool_use"]\n' + '[[hooks]]\n' + 'event = "PreToolUse"\n' + 'matcher = ".*"\n' + f'command = {json.dumps(witnessed_direct + " hook kimi --runtime " + runtime_path)}\n', + encoding="utf-8", + ) + env["KIMI_CODE_HOME"] = str(home / ".kimi-code") + env["KIMI_CODE_NO_AUTO_UPDATE"] = "1" + prompt_flag = [] + elif agent == "qwen": + hooks = home / ".qwen/hooks" + hooks.mkdir(parents=True, exist_ok=True) + source = LAB / "integrations/qwen" / ("PreToolUse.cjs" if windows_host() else "PreToolUse") + target = hooks / source.name + shutil.copy2(source, target) + target.chmod(0o755) + if windows_host(): + hook_command = ( + f'node "{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) + else: + hook_command = ( + f'"{target}" "{witness_command}" "{pitot_command}" ' + f'"{witness_receipt}" "{nonce}" "{runtime_path}"' + ) + settings = hook_group("PreToolUse", "^(Bash|run_shell_command)$", hook_command) + settings["modelProviders"] = {"openai": {"protocol": "openai", "models": [{"id": "pitot-control", "name": "Pitot control", "envKey": "OPENAI_API_KEY", "baseUrl": f"{proxy}/v1"}]}} + settings["security"] = {"auth": {"selectedType": "openai"}} + settings["model"] = {"name": "pitot-control"} + write_json(home / ".qwen/settings.json", settings) + env.update({"OPENAI_API_KEY": "pitot-local-only", "PITOT_BIN": witness_command}) + prompt_flag = ["--model", "pitot-control", "-y", "-p"] + elif agent == "pi": + extension = LAB / "integrations/pi/pitot.ts" + write_json(home / ".pi/agent/models.json", {"providers": {"pitot": {"baseUrl": f"{proxy}/v1", "apiKey": "pitot-local-only", "api": "openai-completions", "models": [{"id": "pitot-control", "name": "Pitot control", "reasoning": False, "input": ["text"], "contextWindow": 32000, "maxTokens": 4096}]}}}) + prompt_flag = ["--no-session", "--print", "--provider", "pitot", "--model", "pitot-control", "-e", str(extension)] + elif agent == "opencode": + plugin = LAB / "integrations/opencode/pitot.ts" + # Use the released binary's bundled OpenAI provider. A custom provider + # would dynamically install @ai-sdk/openai-compatible and make endpoint + # verification depend on a second unpinned network package. + config = home / ".config/opencode/opencode.json" + write_json(config, {"plugin": [f"file://{plugin}"], "provider": {"openai": {"options": {"baseURL": f"{proxy}/v1", "apiKey": "pitot-local-only"}, "models": {"pitot-control": {"name": "Pitot control"}}}}, "model": "openai/pitot-control", "permission": {"bash": "allow"}}) + env["OPENCODE_CONFIG"] = str(config) + prompt_flag = ["--print-logs", "--log-level", "DEBUG", "run", "--model", "openai/pitot-control"] + else: + raise ValueError(f"unsupported agent {agent}") + return prompt_flag, env + + +def prepare_cursor_keychain(home: Path, environment: dict[str, str]) -> None: + """Provide Cursor an isolated unlocked macOS credential store.""" + if sys.platform != "darwin": + return + keychain = home / "Library/Keychains/login.keychain-db" + keychain.parent.mkdir(parents=True, exist_ok=True) + password = "pitot-e2e-local-only" + for command in ( + ["security", "create-keychain", "-p", password, str(keychain)], + ["security", "set-keychain-settings", "-lut", "900", str(keychain)], + ["security", "unlock-keychain", "-p", password, str(keychain)], + ["security", "list-keychains", "-d", "user", "-s", str(keychain)], + ["security", "default-keychain", "-d", "user", "-s", str(keychain)], + ): + subprocess.run(command, env=environment, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + +def capture_record(agent: dict[str, object], platform: str, installation: dict[str, object], proxy: dict[str, object]) -> dict[str, object]: + """Normalize the two receipts without trusting a manifest protocol hint.""" + observed = proxy.get("endpoint_observed") + dialect = proxy.get("protocol") + if not isinstance(observed, dict) or dialect not in { + "anthropic_messages", "openai_chat", "openai_responses", "gemini_generate_content", "cursor_connect_proto", + }: + raise RuntimeError("proxy did not binary-observe a supported request contract") + digest = installation.get("executable_sha256") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise RuntimeError("installation receipt lacks the executable content digest") + core = { + "agent": agent["id"], + "platform": platform, + "runtime": installation["runtime"], + "version": installation["version"], + "executable_sha256": digest, + "dialect": dialect, + "request": observed, + "response": { + "encoder": dialect, + "framing": observed["framing"], + "tool_call": "native_shell", + "acceptance": "nonce_tool_result_round_trip", + }, + "provenance": "pinned_real_cli_capture", + } + return {**core, "capture_sha256": hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":")).encode()).hexdigest()} + + +def validate_capture_fixture(record: dict[str, object]) -> dict[str, object]: + matches = [ + item for item in ENDPOINT_PROVENANCE.get("cells", []) + if item.get("agent") == record["agent"] and item.get("platform") == record["platform"] + ] + if len(matches) != 1 or matches[0] != record: + raise RuntimeError("binary-observed contract drifted from its supervised platform fixture") + return { + "fixture": f"tests/endpoint-provenance.json#{record['agent']}/{record['platform']}", + "fixture_sha256": record["capture_sha256"], + "provenance": record["provenance"], + "dialect": record["dialect"], + "request": record["request"], + "response": record["response"], + "executable_sha256": record["executable_sha256"], + } + + +def json_lines(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def validate_receipts( + agent: dict[str, object], platform: str, nonce: str, installation: dict[str, object], + proxy_path: Path, witness_path: Path, controller_path: Path, consumer_path: Path, + canary_path: Path, runtime_identity: dict[str, object], exit_code: int, output: str, + prompt: str, capture_output: Path | None = None, +) -> dict[str, object]: + proxy = json.loads(proxy_path.read_text(encoding="utf-8")) if proxy_path.is_file() else {"missing": True} + witnesses = json_lines(witness_path) + controller = json_lines(controller_path) + consumers = json_lines(consumer_path) + canary = canary_path.read_text(encoding="utf-8").splitlines() if canary_path.is_file() else [] + proxy_flags = ( + "initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", + "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", + ) + final_marker = f"PITOT_E2E_COMPLETE {nonce}" + if exit_code != 0 or "hook: PreToolUse Failed" in output or final_marker not in output or not all(proxy.get(flag) is True for flag in proxy_flags): + raise RuntimeError(f"agent loop incomplete (exit={exit_code}, proxy={proxy}, witnesses={witnesses})\n{output[-4000:]}") + if proxy.get("nonce") != nonce or len(witnesses) != 2: + raise RuntimeError("proxy and Pitot witness receipts do not identify exactly two hook actions") + if any(item.get("nonce") != nonce or item.get("host") != agent["id"] or item.get("valid") is not True for item in witnesses): + raise RuntimeError("Pitot hook witnesses escaped the nonce-bound real-agent session") + if [item.get("pitot_exit") for item in witnesses] != [0, 2]: + raise RuntimeError(f"real hook did not carry one allow and one deny: {witnesses}") + action_ids = [item.get("action_id") for item in witnesses] + if len(set(action_ids)) != 2 or not all(isinstance(item, str) and item.startswith("act_") for item in action_ids): + raise RuntimeError("hook actions lack unique Pitot correlation ids") + requests = [item.get("value") for item in controller if item.get("receipt_type") == "request"] + responses = [item.get("value") for item in controller if item.get("receipt_type") == "response"] + if len(requests) != 2 or len(responses) != 2: + raise RuntimeError(f"Controller did not receive and resolve both hook actions: {controller}") + if [item.get("action_id") for item in requests] != action_ids or [item.get("action_id") for item in responses] != action_ids: + raise RuntimeError("Controller correlation ids do not match the real hook actions") + if [item.get("outcome") for item in responses] != ["allow", "deny"] or responses[1].get("message") != f"PITOT_CONTROLLER_DENY {nonce}": + raise RuntimeError("external Controller did not produce the nonce-bound allow/deny trajectory") + if len(consumers) != 2 or [item.get("action", {}).get("id") for item in consumers] != action_ids: + raise RuntimeError(f"passive Consumer did not receive both real hook observations: {consumers}") + if any(item.get("content", {}).get("mode") != "sha256" or "full" in item.get("content", {}) for item in consumers): + raise RuntimeError("Consumer projection did not remove full command content") + if canary != [f"PITOT_ALLOW {nonce}"]: + raise RuntimeError(f"canary execution count violated allow/deny control: {canary}") + runtime_public = {key: runtime_identity.get(key) for key in ("schema_version", "instance_id", "pid", "endpoint", "config_sha256")} + if runtime_public["schema_version"] != 1 or not all(runtime_public.get(key) for key in ("instance_id", "endpoint", "config_sha256")): + raise RuntimeError("authenticated Pitot runtime identity is incomplete") + if installation.get("agent") != agent["id"] or installation.get("version") != agent["version"]: + raise RuntimeError("installation receipt does not match supervised manifest") + captured = capture_record(agent, platform, installation, proxy) + if capture_output is not None: + write_json(capture_output, {"schema_version": 1, "accepted": True, "cell": captured}) + endpoint_evidence = { + "fixture": "candidate", + "fixture_sha256": captured["capture_sha256"], + "provenance": captured["provenance"], + "dialect": captured["dialect"], + "request": captured["request"], + "response": captured["response"], + "executable_sha256": captured["executable_sha256"], + } + else: + endpoint_evidence = validate_capture_fixture(captured) + return { + "schema_version": 2, + "agent": agent["id"], + "cli": {"version": installation["version"], "executable": installation["executable"], "executable_sha256": installation["executable_sha256"], "installer": installation["installer"], "runtime": installation["runtime"]}, + "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest(), + "protocol": proxy["protocol"], + "endpoint": endpoint_evidence, + "nonce": nonce, + "receipts": {**{flag: True for flag in proxy_flags}, "consumer_observed": True, "controller_allow_observed": True, "controller_deny_observed": True, "deny_canary_absent": True, "final_output_observed": True, "cli_exit_zero": True}, + "runtime": runtime_public, + "hooks": [{"host": item["host"], "action_kind": item["action_kind"], "action_id": item["action_id"], "pitot_exit": item["pitot_exit"], "nonce": item["nonce"]} for item in witnesses], + "controller": {"id": "e2e-shell-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}, + "consumer": {"id": "e2e-audit", "action_ids": action_ids, "projection": "sha256"}, + "canary": {"executions": canary, "denied_executions": 0}, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--installation", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--pitot", type=Path, required=True) + parser.add_argument("--witness", type=Path, required=True) + parser.add_argument("--test-role", type=Path, required=True) + parser.add_argument("--capture-output", type=Path) + parser.add_argument("--response-fault", choices=("none", "text"), default="none") + parser.add_argument("--expect-incompatible-response", action="store_true") + args = parser.parse_args() + agent = next(item for item in MANIFEST["agents"] if item["id"] == args.agent) + installation = json.loads(args.installation.read_text(encoding="utf-8")) + nonce = secrets.token_hex(16) + prompt = f"Pitot E2E session {nonce}: execute the requested verification command" + with tempfile.TemporaryDirectory(prefix="pitot-real-agent-", ignore_cleanup_errors=True) as temporary: + base = Path(temporary) + runtime = installation["runtime"] + host_controls_wsl = runtime == "wsl" and os.name == "nt" + home, project, bin_dir = base / "home", base / "project", base / "bin" + home.mkdir(); project.mkdir(); bin_dir.mkdir() + canary_receipt = base / "canary.jsonl" + canary_executable = args.test_role.resolve() + cursor_system_canary = False + if args.agent == "cursor" and host_controls_wsl: + occupied = subprocess.run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "test", "-e", "/usr/local/bin/pitot-e2e-canary"], + check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + if occupied.returncode == 0: + raise RuntimeError("refusing to replace an existing WSL canary command") + subprocess.run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "install", "-m", "0755", wsl_path(canary_executable), "/usr/local/bin/pitot-e2e-canary"], + check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + canary_executable = Path("/usr/local/bin/pitot-e2e-canary") + cursor_system_canary = True + elif args.agent == "cursor" and os.name != "nt" and os.access("/usr/local/bin", os.W_OK): + if Path("/usr/local/bin/pitot-e2e-canary").exists(): + raise RuntimeError("refusing to replace an existing system canary command") + shutil.copy2(canary_executable, "/usr/local/bin/pitot-e2e-canary") + Path("/usr/local/bin/pitot-e2e-canary").chmod(0o755) + canary_executable = Path("/usr/local/bin/pitot-e2e-canary") + cursor_system_canary = True + receipt_argument = wsl_path(canary_receipt) if host_controls_wsl else str(canary_receipt) + # Forward slashes keep the same native Windows command valid in + # PowerShell, cmd.exe, and the Unix-like shells selected by hosts. + canary_command = render_canary_command( + str(canary_executable), receipt_argument, windows_host() and not host_controls_wsl, + ) + ready, proxy_receipt, witness_receipt = base / "proxy.url", base / "proxy.json", base / "witness.jsonl" + runtime_descriptor = base / "runtime.json" + runtime_config = base / "pitot.json" + consumer_receipt = base / "consumer.jsonl" + controller_receipt = base / "controller.jsonl" + runtime_config.write_text(json.dumps({ + "consumers": [{ + "id": "e2e-audit", + "command": [str(args.test_role.resolve()), "--role", "consumer", "--receipt", str(consumer_receipt)], + "events": ["action.requested"], + "projection": {"content": "sha256"}, + }], + "controllers": { + "shell": { + "id": "e2e-shell-controller", + "command": [str(args.test_role.resolve()), "--role", "controller", "--id", "e2e-shell-controller", "--receipt", str(controller_receipt), "--nonce", nonce], + "deadline_ms": 5000, + "on_timeout": "deny", + "on_unavailable": "deny", + } + }, + }, indent=2) + "\n", encoding="utf-8") + runtime_process = subprocess.Popen( + [str(args.pitot.resolve()), "run", "--config", str(runtime_config), "--runtime", str(runtime_descriptor)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + for _ in range(600): + if runtime_descriptor.is_file(): break + if runtime_process.poll() is not None: + output = runtime_process.stdout.read() if runtime_process.stdout else "" + raise RuntimeError(f"Pitot runtime exited before becoming ready: {output.strip()}") + time.sleep(0.05) + else: + runtime_process.terminate() + raise RuntimeError("Pitot runtime did not publish its authenticated descriptor") + runtime_identity = json.loads(runtime_descriptor.read_text(encoding="utf-8")) + if args.agent == "cursor": + if host_controls_wsl: + # Keep Cursor and its HTTP/2 Connect control proxy in the same + # supported WSL network namespace. The workflow installs a + # pinned Linux Node runtime specifically for this harness. + proxy_command = [ + "wsl.exe", "--distribution", "Ubuntu", "--", "node", + wsl_path(LAB / "tests/cursor_control_proxy.mjs"), + "--nonce", nonce, + "--receipt", wsl_path(proxy_receipt), + "--ready-file", wsl_path(ready), + "--canary-command", canary_command, + "--response-fault", args.response_fault, + ] + else: + proxy_command = ["node", str(LAB / "tests/cursor_control_proxy.mjs"), "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] + else: + proxy_command = [sys.executable, str(LAB / "tests/model_control_proxy.py"), "--agent", args.agent, "--nonce", nonce, "--receipt", str(proxy_receipt), "--ready-file", str(ready), "--canary-command", canary_command, "--response-fault", args.response_fault] + proxy_process = subprocess.Popen( + proxy_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + for _ in range(600): + if ready.is_file(): break + if proxy_process.poll() is not None: + output = proxy_process.stdout.read() if proxy_process.stdout else "" + raise RuntimeError(f"model-control proxy exited before becoming ready: {output.strip()}") + time.sleep(0.05) + else: + proxy_process.terminate() + output = proxy_process.communicate(timeout=5)[0] if proxy_process.stdout else "" + raise RuntimeError(f"model-control proxy did not become ready: {output.strip()}") + proxy = ready.read_text(encoding="utf-8").strip() + witness_command = wsl_path(args.witness) if host_controls_wsl else str(args.witness.resolve()) + pitot_command = wsl_path(args.pitot) if host_controls_wsl else str(args.pitot.resolve()) + receipt_command = wsl_path(witness_receipt) if host_controls_wsl else str(witness_receipt) + flags, extra_env = configure( + args.agent, + home, + project, + witness_command, + proxy, + pitot_command=pitot_command, + witness_receipt=receipt_command, + nonce=nonce, + runtime_path=str(runtime_descriptor), + ) + environment = {**os.environ, **extra_env, "PATH": str(bin_dir) + os.pathsep + os.environ.get("PATH", ""), "PITOT_REAL_BIN": str(args.pitot.resolve()), "PITOT_WITNESS_RECEIPT": str(witness_receipt), "PITOT_E2E_NONCE": nonce, "PITOT_RUNTIME": str(runtime_descriptor)} + executable = installation["executable"] + if args.agent == "cursor" and not host_controls_wsl: + prepare_cursor_keychain(home, environment) + if host_controls_wsl: + wsl_environment = { + **extra_env, + "HOME": wsl_path(home), + "USERPROFILE": wsl_path(home), + "PITOT_BIN": witness_command, + "PITOT_REAL_BIN": wsl_path(args.pitot), + "PITOT_WITNESS_RECEIPT": wsl_path(witness_receipt), + "PITOT_E2E_NONCE": nonce, + "PITOT_RUNTIME": wsl_path(runtime_descriptor), + } + executable_dir = executable.rsplit("/", 1)[0] + wsl_environment["PATH"] = ":".join((wsl_path(bin_dir), executable_dir, "/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin")) + assignments = [f"{key}={value}" for key, value in wsl_environment.items()] + # Keep every argument distinct across the Windows/WSL boundary. + # A reconstructed `bash -lc` string can alter quoting, expand a + # host PATH, or leave the released agent waiting indefinitely. + command = [ + "wsl.exe", "--distribution", "Ubuntu", "--cd", wsl_path(project), + "--", "env", *assignments, executable, *flags, prompt, + ] + else: + command = [executable, "-p", prompt, *flags] if args.agent in {"copilot", "kimi"} else [executable, *flags, prompt] + try: + completed = subprocess.run(command, cwd=project, env=environment, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=420) + except subprocess.TimeoutExpired as error: + proxy_state = proxy_receipt.read_text(encoding="utf-8") if proxy_receipt.is_file() else '{"missing":true}' + witness_state = witness_receipt.read_text(encoding="utf-8") if witness_receipt.is_file() else '{"missing":true}' + partial_output = error.stdout or "" + raise RuntimeError( + f"released agent timed out; proxy={proxy_state}; witness={witness_state}; " + f"output={partial_output[-4000:]}" + ) from error + # GitHub's Windows Python console defaults to CP-1252 while several + # released CLIs emit Unicode status glyphs. Emit UTF-8 bytes so the + # reporting layer cannot fail after a successful agent session. + sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace")) + sys.stdout.buffer.flush() + if args.expect_incompatible_response: + observed = json.loads(proxy_receipt.read_text(encoding="utf-8")) if proxy_receipt.is_file() else {} + if not ( + observed.get("initial_prompt_observed") is True + and observed.get("fault_response_emitted") == "text" + and observed.get("tool_call_response_emitted") is False + and observed.get("tool_result_observed") is False + and not witness_receipt.exists() + ): + raise RuntimeError("incompatible response unexpectedly entered the hook/tool path") + print("PITOT_INCOMPATIBLE_RESPONSE_REJECTED evidence=binary-observed") + return 0 + evidence = validate_receipts( + agent, args.platform, nonce, installation, proxy_receipt, witness_receipt, + controller_receipt, consumer_receipt, canary_receipt, runtime_identity, + completed.returncode, completed.stdout, prompt, args.capture_output, + ) + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated") + finally: + proxy_process.terminate() + try: proxy_process.wait(timeout=5) + except subprocess.TimeoutExpired: proxy_process.kill() + runtime_process.terminate() + try: runtime_process.wait(timeout=5) + except subprocess.TimeoutExpired: runtime_process.kill() + if cursor_system_canary: + if host_controls_wsl: + subprocess.run( + ["wsl.exe", "--distribution", "Ubuntu", "--", "rm", "-f", "/usr/local/bin/pitot-e2e-canary"], + check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + else: + Path("/usr/local/bin/pitot-e2e-canary").unlink(missing_ok=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/run_e2e_report.py b/tests/run_e2e_report.py new file mode 100644 index 0000000..5e34938 --- /dev/null +++ b/tests/run_e2e_report.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Run one Pitot host E2E script and emit a strict result artifact.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + + +LAB = Path(__file__).resolve().parent.parent +ROOT = LAB.parents[1] if LAB.name == "15-pitot" else LAB +MANIFEST = json.loads((LAB / "adapter-verification.json").read_text(encoding="utf-8")) +AGENTS = {agent["id"] for agent in MANIFEST["agents"]} +PLATFORMS = {platform["id"] for platform in MANIFEST["platforms"]} +RESULT_PATTERN = re.compile(r"^PITOT_E2E_RESULT mode=real_cli evidence=nonce-correlated$", re.MULTILINE) +RUNTIME_RESULT_PATTERN = re.compile(r"^PITOT_RUNTIME_E2E_RESULT capability=explicit_request evidence=nonce-correlated$", re.MULTILINE) + + +def load_evidence(path: Path | None, *, agent: str) -> dict[str, object] | None: + if path is None or not path.is_file(): + return None + value = json.loads(path.read_text(encoding="utf-8")) + required = {"schema_version", "agent", "cli", "prompt_hash", "protocol", "endpoint", "nonce", "receipts", "runtime", "hooks", "controller", "consumer", "canary"} + if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 2 or value["agent"] != agent: + return None + receipts = value.get("receipts") + receipt_fields = { + "initial_prompt_observed", "allow_tool_call_response_emitted", "allow_tool_result_observed", + "deny_tool_call_response_emitted", "denied_result_observed", "final_response_emitted", + "consumer_observed", "controller_allow_observed", "controller_deny_observed", + "deny_canary_absent", "final_output_observed", "cli_exit_zero", + } + if not isinstance(receipts, dict) or set(receipts) != receipt_fields or not all(item is True for item in receipts.values()): + return None + if not re.fullmatch(r"[0-9a-f]{64}", str(value.get("prompt_hash", ""))): + return None + nonce = value.get("nonce") + if not isinstance(nonce, str) or not re.fullmatch(r"[0-9a-f]{32}", nonce): + return None + hooks = value.get("hooks") + if not isinstance(hooks, list) or len(hooks) != 2 or [item.get("pitot_exit") for item in hooks] != [0, 2] or any(item.get("action_kind") != "shell" or item.get("host") != agent or item.get("nonce") != nonce for item in hooks): + return None + endpoint = value.get("endpoint", {}) + endpoint_required = {"fixture", "fixture_sha256", "provenance", "dialect", "request", "response", "executable_sha256"} + if not isinstance(endpoint, dict) or set(endpoint) != endpoint_required: + return None + if endpoint.get("provenance") != "pinned_real_cli_capture" or not re.fullmatch(r"[0-9a-f]{64}", str(endpoint.get("fixture_sha256", ""))): + return None + return value + + +def load_runtime_evidence(path: Path | None, *, platform: str) -> dict[str, object] | None: + if path is None or not path.is_file(): + return None + value = json.loads(path.read_text(encoding="utf-8")) + required = {"schema_version", "capability", "platform", "nonce", "runtime", "controller", "receipts", "commit_sha"} + if not isinstance(value, dict) or set(value) != required or value["schema_version"] != 1 or value["capability"] != "explicit_request" or value["platform"] != platform: + return None + if not isinstance(value["nonce"], str) or not re.fullmatch(r"[0-9a-f]{32}", value["nonce"]): + return None + receipts = value.get("receipts") + if not isinstance(receipts, dict) or set(receipts) != {"request_allow_observed", "request_deny_observed", "correlation_observed"} or not all(item is True for item in receipts.values()): + return None + controller = value.get("controller") + if not isinstance(controller, dict) or controller.get("outcomes") != ["allow", "deny"] or len(controller.get("action_ids", [])) != 2: + return None + return value + + +def result_for(agent: str, platform: str, returncode: int, output: str, evidence_path: Path | None = None) -> dict[str, object]: + if agent not in AGENTS: + raise ValueError(f"unsupported agent: {agent}") + if platform not in PLATFORMS: + raise ValueError(f"unsupported platform: {platform}") + + markers = RESULT_PATTERN.findall(output) + receipt = load_evidence(evidence_path, agent=agent) + passed = returncode == 0 and len(markers) == 1 and receipt is not None + evidence = "binary-observed request, real hook control, projected Consumer, allow/deny canary, and final receipts" if passed else "real-agent control evidence contract failed" + + return { + "schema_version": 2, + "agent": agent, + "platform": platform, + "status": "pass" if passed else "fail", + "verification_mode": "real_cli" if passed else None, + "evidence": evidence, + "cli": receipt["cli"] if passed else None, + "protocol": receipt["protocol"] if passed else None, + "endpoint": receipt["endpoint"] if passed else None, + "prompt_hash": receipt["prompt_hash"] if passed else None, + "nonce": receipt["nonce"] if passed else None, + "receipts": receipt["receipts"] if passed else None, + "runtime": receipt["runtime"] if passed else None, + "hooks": receipt["hooks"] if passed else None, + "controller": receipt["controller"] if passed else None, + "consumer": receipt["consumer"] if passed else None, + "canary": receipt["canary"] if passed else None, + "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), + "run_url": ( + f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}" + f"/actions/runs/{os.environ['GITHUB_RUN_ID']}" + if all( + key in os.environ + for key in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID") + ) + else "local" + ), + } + + +def result_for_runtime(platform: str, returncode: int, output: str, evidence_path: Path | None = None) -> dict[str, object]: + if platform not in PLATFORMS: + raise ValueError(f"unsupported platform: {platform}") + markers = RUNTIME_RESULT_PATTERN.findall(output) + receipt = load_runtime_evidence(evidence_path, platform=platform) + passed = returncode == 0 and len(markers) == 1 and receipt is not None + return { + "schema_version": 2, + "capability": "explicit_request", + "platform": platform, + "status": "pass" if passed else "fail", + "verification_mode": "real_runtime" if passed else None, + "evidence": "real request CLI, authenticated runtime, and correlated allow/deny Controller receipts" if passed else "explicit request evidence contract failed", + "nonce": receipt["nonce"] if passed else None, + "runtime": receipt["runtime"] if passed else None, + "controller": receipt["controller"] if passed else None, + "receipts": receipt["receipts"] if passed else None, + "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), + "run_url": ( + f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}" + f"/actions/runs/{os.environ['GITHUB_RUN_ID']}" + if all(key in os.environ for key in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID")) + else "local" + ), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--agent", choices=sorted(AGENTS)) + target.add_argument("--capability", choices=("explicit_request",)) + parser.add_argument("--platform", required=True, choices=sorted(PLATFORMS)) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("a command is required after --") + + completed = subprocess.run( + command, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + sys.stdout.buffer.write(completed.stdout.encode("utf-8", errors="replace")) + result = ( + result_for(args.agent, args.platform, completed.returncode, completed.stdout, args.evidence) + if args.agent else result_for_runtime(args.platform, completed.returncode, completed.stdout, args.evidence) + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 if result["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/runtime_capability_driver.py b/tests/runtime_capability_driver.py new file mode 100644 index 0000000..8ad3227 --- /dev/null +++ b/tests/runtime_capability_driver.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Prove the real Pitot request CLI against its authenticated runtime.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import secrets +import subprocess +import tempfile +import time + + +def lines(path: Path) -> list[dict[str, object]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pitot", type=Path, required=True) + parser.add_argument("--test-role", type=Path, required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pitot-request-e2e-") as temporary: + nonce = secrets.token_hex(16) + base = Path(temporary) + runtime_path = base / "runtime.json" + receipt = base / "controller.jsonl" + config = base / "pitot.json" + config.write_text(json.dumps({"controllers": {"release.approval": { + "id": "e2e-release-controller", + "command": [str(args.test_role.resolve()), "--role", "controller", "--id", "e2e-release-controller", "--receipt", str(receipt), "--nonce", nonce], + "deadline_ms": 2000, + "on_timeout": "deny", + "on_unavailable": "deny", + }}}) + "\n", encoding="utf-8") + process = subprocess.Popen( + [str(args.pitot.resolve()), "run", "--config", str(config), "--runtime", str(runtime_path)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + try: + for _ in range(300): + if runtime_path.is_file(): break + if process.poll() is not None: + raise RuntimeError(f"runtime exited: {process.stdout.read() if process.stdout else ''}") + time.sleep(0.02) + else: + raise RuntimeError("runtime descriptor was not published") + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + allow = subprocess.run( + [str(args.pitot.resolve()), "request", "release.approval", "--data", json.dumps({"phase": "PITOT_ALLOW", "nonce": nonce}), "--runtime", str(runtime_path)], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + deny = subprocess.run( + [str(args.pitot.resolve()), "request", "release.approval", "--data", json.dumps({"phase": "PITOT_DENY", "nonce": nonce}), "--runtime", str(runtime_path)], + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + allow_value = json.loads(allow.stdout) + deny_value = json.loads(deny.stdout) + controller = lines(receipt) + requests = [item["value"] for item in controller if item.get("receipt_type") == "request"] + responses = [item["value"] for item in controller if item.get("receipt_type") == "response"] + if allow.returncode != 0 or deny.returncode != 2: + raise RuntimeError(f"request exits did not encode allow/deny: {allow.returncode}, {deny.returncode}") + if [allow_value.get("outcome"), deny_value.get("outcome")] != ["allow", "deny"]: + raise RuntimeError("request output did not carry Controller outcomes") + action_ids = [item.get("action_id") for item in requests] + if len(requests) != 2 or len(responses) != 2 or [item.get("action_id") for item in responses] != action_ids: + raise RuntimeError("request and response correlation receipts disagree") + if any(nonce not in json.dumps(item.get("data"), sort_keys=True) for item in requests): + raise RuntimeError("request receipts are not bound to the session nonce") + public_runtime = {key: runtime[key] for key in ("schema_version", "instance_id", "pid", "endpoint", "config_sha256")} + evidence = { + "schema_version": 1, + "capability": "explicit_request", + "platform": args.platform, + "nonce": nonce, + "runtime": public_runtime, + "controller": {"id": "e2e-release-controller", "action_ids": action_ids, "outcomes": ["allow", "deny"]}, + "receipts": {"request_allow_observed": True, "request_deny_observed": True, "correlation_observed": True}, + "commit_sha": os.environ.get("PITOT_SOURCE_SHA", os.environ.get("GITHUB_SHA", "local")), + } + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print("PITOT_RUNTIME_E2E_RESULT capability=explicit_request evidence=nonce-correlated") + finally: + process.terminate() + try: process.wait(timeout=5) + except subprocess.TimeoutExpired: process.kill() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/witness/main.go b/tests/witness/main.go new file mode 100644 index 0000000..f54da7c --- /dev/null +++ b/tests/witness/main.go @@ -0,0 +1,104 @@ +// Command pitot-witness is a transparent executable wrapper used only by the +// real-agent E2E. It proves that the host invoked Pitot without changing the +// bytes or exit status observed by either process. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +type event struct { + Type string `json:"type"` + Host struct { + Name string `json:"name"` + } `json:"host"` + Action *struct { + ID string `json:"id"` + Kind string `json:"kind"` + } `json:"action"` + Content *struct { + Mode string `json:"mode"` + Full json.RawMessage `json:"full"` + } `json:"content"` +} + +func main() { + flags := flag.NewFlagSet("pitot-witness", flag.ContinueOnError) + pitotFlag := flags.String("real-bin", "", "real Pitot executable") + receiptFlag := flags.String("receipt", "", "witness receipt path") + nonceFlag := flags.String("nonce", "", "session nonce") + if err := flags.Parse(os.Args[1:]); err != nil { + os.Exit(125) + } + args := flags.Args() + pitot, receipt, nonce := *pitotFlag, *receiptFlag, *nonceFlag + if pitot == "" { + pitot = os.Getenv("PITOT_REAL_BIN") + } + if receipt == "" { + receipt = os.Getenv("PITOT_WITNESS_RECEIPT") + } + if nonce == "" { + nonce = os.Getenv("PITOT_E2E_NONCE") + } + if pitot == "" || receipt == "" || nonce == "" { + fmt.Fprintln(os.Stderr, "pitot-witness: PITOT_REAL_BIN, PITOT_WITNESS_RECEIPT, and PITOT_E2E_NONCE are required") + os.Exit(125) + } + in, err := io.ReadAll(os.Stdin) + if err != nil { + panic(err) + } + command := exec.Command(pitot, args...) + command.Stdin = bytes.NewReader(in) + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + err = command.Run() + _, _ = os.Stdout.Write(stdout.Bytes()) + _, _ = os.Stderr.Write(stderr.Bytes()) + code := 0 + if err != nil { + if exit, ok := err.(*exec.ExitError); ok { + code = exit.ExitCode() + } else { + code = 125 + } + } + + host := "" + if len(args) >= 2 && args[0] == "hook" { + host = args[1] + } + var observed event + valid := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &observed) == nil && + observed.Type == "action.requested" && observed.Host.Name == host && + observed.Action != nil && observed.Action.Kind == "shell" && + observed.Content != nil && observed.Content.Mode == "full" && + strings.Contains(string(observed.Content.Full), nonce) + actionID := "" + if observed.Action != nil { + actionID = observed.Action.ID + } + record := map[string]any{ + "schema_version": 1, "host": host, "nonce": nonce, "action_kind": "shell", + "action_id": actionID, "pitot_exit": code, "valid": valid, + } + encoded, _ := json.Marshal(record) + if valid { + _ = os.MkdirAll(filepath.Dir(receipt), 0o755) + file, openErr := os.OpenFile(receipt, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if openErr == nil { + _, _ = file.Write(append(encoded, '\n')) + _ = file.Close() + } + } + os.Exit(code) +}