diff --git a/WS5-REMAINDER.md b/WS5-REMAINDER.md new file mode 100644 index 0000000..3f5ef9e --- /dev/null +++ b/WS5-REMAINDER.md @@ -0,0 +1,143 @@ +# WS5 remainder — OpenPhone conformance to Interfaces Protocol v1 + +Status of WS5 (Interfaces v1 rewrite plan, Part III) on `rewrite/protocol-v1`: +the working vertical slice is done — vendored ifp/1 validator/machine/fixtures, +the `openphone-tools.ifp1.json` manifest re-expressing all 37 commands, the +legacy-event -> action-lifecycle adapter with lifecycle tests, and deprecation +notices on the v0 manifests — all wired into `scripts/check.sh` +(see `runtime/protocol/ifp1/` and `runtime/protocol/ifp1/MIGRATION.md`). +This file lists what remains before the WS5 exit gate. + +## 1. Java-side wiring: RuntimeToolBridge / PhoneToolGateway -> ifp/1 + +- `overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeToolBridge.java` + is where the v0 lifecycle lives today: it validates commands against the v0 + manifest, applies autonomy policy, creates `runtime-confirm-*` confirmations + for mutating tools, executes, and returns tool results. Port it to emit and + consume ifp/1 action messages using the mapping in + `runtime/protocol/ifp1/adapter.mjs` (proposal ingestion -> local + confirmation -> `action.approve`/`action.reject` with canonical + `params_hash` binding -> idempotent execution -> `action.result` with + `binding_mismatch`/`action_expired` failure classes). A Java + canonical-JSON + SHA-256 helper must reproduce + `runtime/protocol/ifp1/fixtures/params-hash-vectors.json` exactly. +- `overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/PhoneToolGateway.java` + (and `OpenPhoneOsToolGateway.java`) — the seam from `refactor/app-os-boundary` + is already aligned with the target; the ifp/1 executor should call through + it unchanged. Do not redo the seam. +- Tool-name translation at the boundary: Java command registries key on + `openphone.` while ifp/1 uses `` + (`x-openphone.superseded_command` in `openphone-tools.ifp1.json` is the + lookup bridge). + +## 2. Ed25519 pairing alignment + +The OpenClaw adapter already has Ed25519 device identity: + +- `overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawEd25519.java` +- `overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawDeviceIdentity.java` + +Remaining work: promote this from an OpenClaw-adapter detail to the surface's +ifp/1 identity (spec §6.1–6.3): keypair generated at pairing with the entity's +home, `pair.request`/`pair.grant`/`pair.complete` challenge signature, and a +`control.hello` `payload.proof` signature over the transport nonce +(`x-ifp-nonce` for WebSocket). Key storage should move to the OS keystore +path. No default/dev credential may exist in any configuration. + +## 3. OpenClaw / MCP / CLI adapter migration + +All three still speak v0 and read `runtime/protocol/openphone-commands.json`: + +- `integrations/openclaw-plugin/src/index.ts` (+ `dist/`) — command buckets + (safe read / private read / dangerous) become ifp/1 tool capabilities and + surface-local policy. +- `integrations/mcp-server/src/index.mjs` — MCP tools should be generated + from `openphone-tools.ifp1.json`; the `serverInfo.runtimeProtocol` range + advertisement becomes ifp/1 `control.hello` protocol negotiation semantics. +- `integrations/cli/src/index.mjs` — same manifest switch; `openphone info` + advertises ifp/1. +- The byte-sync package copies under `integrations/cli/runtime/protocol/` and + `integrations/mcp-server/runtime/protocol/` (enforced by + `tests/integrations/runtime-package-contract.mjs`) must then vendor the + ifp1 manifest instead. +- `runtime/protocol/openphone-runtime-tools.mjs` grows an ifp/1 loader (or a + successor module under `ifp1/`) so `mcpTools()`/`commandMap()` consumers + migrate without ad-hoc parsing. + +## 4. Surface/entity ownership stubs (plan WS5 deliverable 3) + +Re-label the system_server memory/context/audit stores (patches 0014–0016) +as surface-owned offline cache + audit evidence; stub the sync boundary so +durable-memory writes emit ifp/1 `event.journal.append` to a runtime endpoint +(pointed at the WS3 runtime in a local harness). OS-owned audit stays +phone-permanent; no framework patch is reverted. + +## 5. Old-manifest deletion gate + +`runtime/protocol/openphone-commands.json` and `openphone-events.json` are +deprecated (`"deprecated": true`, `"superseded_by": "ifp1/"`) but MUST NOT be +deleted until all of the following hold: + +1. Items 1–3 above are done (no Java, plugin, MCP, or CLI reader of the v0 + manifests remains). +2. The v0 sections of `runtime/protocol/validate-runtime-protocol.mjs`, + `openphone-runtime-tools.mjs`, and the v0 contract tests are replaced by + ifp/1 equivalents (including the Android command-registry cross-checks, + re-pointed at `openphone-tools.ifp1.json`). +3. `docs/runtime/runtime-agent-protocol.md` is rewritten for ifp/1 (it names + the v0 manifests as single sources of truth). +4. The ifp/1 fixture corpus has frozen at `v1.0.0-draft` upstream (ADR 0004: + freeze happens after WS4/presence-kit e2e conformance), so the vendored + copies here stop moving. + +Deletion is the clean major-version break (D5): no compat path ships. + +## 6. iOS runtime note + +`ios/` does not exist on this branch (the iOS runtime is in flight on +`add-ios-support`). When that line lands or rebases onto ifp/1 work, add to +its README: "This runtime will consume Interfaces Protocol v1; see +`runtime/protocol/ifp1/`." Per the plan, `add-ios-support` is out of scope +for WS5 and consumes ifp/1 later. + +## 7. Device-in-loop conformance lane (exit gate) + +The WS5 exit gate also requires: existing CI green (full Android build needs +a LineageOS tree — not runnable here), the fixture corpus passing against the +phone-side implementation (device-in-loop or emulator lane), and the +end-to-end local demo with the WS3 runtime (propose -> confirm -> approve +with exact binding -> execute -> audit on both sides; mismatched +approved-vs-executed params rejected by the phone). The node-side halves of +these checks exist today in `tests/integrations/ifp1-adapter-contract.mjs`. + +## Spec divergences found (feed back to WS1 before corpus freeze) + +1. **Spec §8.1 example `params_hash` is wrong.** The example proposal in + `interfaces-protocol-1.md` shows + `sha256:2c26b46b...` for params `{"to": "+15551234567", "body": "on my way"}`, + but that digest is SHA-256 of the literal string `"foo"`. The canonical + encoding `{"body":"on my way","to":"+15551234567"}` hashes to + `sha256:1e920af2c02daa49a449d1ebe938e08f196a9dfa6a3c05b36adbabbd7387193b` + (vector `spec_8_1_example_params` in + `runtime/protocol/ifp1/fixtures/params-hash-vectors.json`). +2. **No canonical-JSON test vectors in the shared fixture corpus.** The + validators only regex-check `sha256:`; nothing cross-checks the + canonicalization itself (unicode handling, nested key sorting). Upstream + `fixtures/params-hash-vectors.json` into + `packages/interfaces-protocol/fixtures/` and both reference validators. +3. **Canonicalization needs one more sentence of precision.** "UTF-8, sorted + keys, no insignificant whitespace" should state: keys sorted at every + nesting depth by Unicode code point, strings serialized without ASCII + escaping of non-ASCII characters (UTF-8 bytes hashed), and numbers + restricted to interoperable forms (e.g. no `1.0` vs `1` ambiguity — or + forbid non-integer numbers in params). +4. **`action.propose` sender lacks generation fencing.** `generation` is + required on `event.*`, `action.approve`, and `action.result`, but a stale + runtime can still propose actions. Probably fine (the approve/result fence + catches it), but worth an explicit spec note. +5. **Risk taxonomy loses information.** v0 distinguishes `risk` from + `confirmation` (`none`/`ask_before_action`/`always`) and `default_exposure`. + ifp/1 folds these into `risk` + surface-local policy — intentional per + §8.2, but the tools manifest needed an `x-` extension + (`x-openphone`) to carry the policy inputs. If other surfaces need the + same, consider a standard optional field on tool capability declarations. diff --git a/docs/AGENT_RUNTIME_V1.md b/docs/AGENT_RUNTIME_V1.md index 65f5725..01718c1 100644 --- a/docs/AGENT_RUNTIME_V1.md +++ b/docs/AGENT_RUNTIME_V1.md @@ -10,8 +10,8 @@ session. - Persist background agent work as first-class jobs. - Wake jobs from alarms, boot, package replacement, and explicit tool calls. - Keep delivery, retries, diagnostics, and failure state with the job. -- Preserve the reviewed autonomy posture: background jobs do not silently - perform state-changing actions. +- Preserve the reviewed autonomy posture: background jobs pause before + state-changing actions and resume only from an exact Android-owned review. - Keep model routing in the registry and schemas instead of Java keyword routing. @@ -45,7 +45,7 @@ An `agent_job` records durable background work. "mode": "notification", "notification_text": "Release notes changed" }, - "status": "active", + "status": "queued", "created_at": 1781480000000, "updated_at": 1781480000000, "next_run_at": 1781483055000, @@ -53,10 +53,27 @@ An `agent_job` records durable background work. "last_run_at": 0, "last_result": "", "failure_count": 0, - "failure_alert_at": 0 + "failure_alert_at": 0, + "phase": "queued", + "progress_text": "Queued", + "progress_current": 0, + "progress_total": 0, + "checkpoint_json": {}, + "pending_confirmation_id": "", + "pending_tool_request_json": {}, + "last_surface_id": "", + "resume_token": "", + "last_event_at": 1781480000000, + "unread_result": false, + "paused_at": 0 } ``` +Lifecycle states are `queued`, `running`, `waiting`, `awaiting_review`, +`paused`, `completed`, `failed`, and `stopped`. Readers continue accepting the +legacy `active` value during migration. Runtime dispatch may also persist +`dispatched` while an external runtime owns the next step. + Job types: - `agent_turn`: run a bounded model/tool loop against a prompt. @@ -77,7 +94,8 @@ model/tool trace and must not be dumped directly into the notification surface. The assistant schedules a single alarm for the next active job. On wake it: -1. repairs stale `running` jobs, +1. expires pending reviews and repairs stale `running` or review-resolution + jobs, 2. marks due jobs `running`, 3. starts bounded background runners, 4. persists completion/failure, @@ -89,13 +107,29 @@ service startup. ## Safety -Background jobs may call observe/read tools and terminal tools. State-changing -tools return `background.confirmation_required` instead of executing. The -foreground reviewed flow remains the only path for actions that mutate device, -account, communication, or external state. - -This is intentionally conservative. The follow-up design can add reviewed -background approvals with a notification action and a resumable job checkpoint. +Background jobs may call observe/read tools and terminal tools directly. A +state-changing tool is never executed by the background model loop. Instead, +OpenPhone: + +1. bounds and sanitizes the exact proposed tool arguments; +2. persists an `awaiting_review` checkpoint and 15-minute confirmation; +3. shows the exact action in an Android notification and AI Home run detail; +4. atomically claims the first Approve or Deny tap; +5. verifies the tool, parameter digest, runtime, phone session, job, expiry, + idempotency key, checkpoint, and resume token; +6. executes only the persisted request after approval, or records a structured + denial/timeout result; +7. resumes the job with that bound result instead of regenerating the action. + +Provider secrets, raw authorization, screenshots, data URLs, large encoded +blobs, and unbounded context are rejected from checkpoints. Audit events store +digests and lifecycle metadata, not raw parameters. If the process dies while +an approved action is resolving, stale-run repair marks the outcome unknown +and does not replay it. A denied action can safely resume after restart. + +The wire shapes are defined by `schemas/agent-job.schema.json` and +`schemas/background-confirmation.schema.json`; recorded valid, tampered, and +secret-bearing fixtures run in `./scripts/check.sh`. ## OpenClaw/Hermes Mapping diff --git a/docs/APP_OS_BOUNDARY.md b/docs/APP_OS_BOUNDARY.md new file mode 100644 index 0000000..bed6f76 --- /dev/null +++ b/docs/APP_OS_BOUNDARY.md @@ -0,0 +1,90 @@ +# App and OS Boundary + +OpenPhone has one product experience with two possible phone capability +profiles: + +- **OpenPhone App** uses Android public APIs and user-granted roles and + permissions. It is the future Play-distributable profile. +- **OpenPhone OS** adds a privileged execution adapter, framework services, + SystemUI surfaces, keyguard configuration, and system policy enforcement. + +Most product logic should be shared. The OS layer should remain a small, +auditable authority rather than becoming a second assistant implementation. + +## Dependency Direction + +```text +AI Home / generated surfaces / model and runtime adapters + | + v + PhoneToolGateway + / \ + v v + public Android app OpenPhone OS adapter + implementation (framework Binder client) + | + v + system_server / SystemUI +``` + +Code above `PhoneToolGateway` must not import `android.openphone` hidden APIs, +construct `FrameworkToolExecutor`, or decide that a privileged action is +authorized. It may request a registered tool, show review UI, and interpret a +structured result. + +The gateway reports a stable profile and whether each tool is supported. A +smaller implementation must deny an unsupported tool explicitly; it must not +silently fall back to accessibility automation or a lower-risk action. + +## Source Ownership + +| Layer | Current source | Responsibility | +| --- | --- | --- | +| Portable product | `assistant/actions`, `model`, `orchestrator`, `runtime`, `surface`, and Compose UI | Conversation, model adapters, tool requests, adaptive UI, runtime sessions, and user review presentation | +| Phone boundary | `assistant/platform/PhoneToolGateway.java` | Stable app-to-phone execution contract and capability-profile seam | +| OpenPhone OS adapter | `assistant/platform/OpenPhoneOsToolGateway.java` and `assistant/agent/FrameworkToolExecutor.java` | Translate registered tools into the hidden OpenPhone framework manager and Android integrations | +| OS authority | `patches/frameworks_base`, Settings patches, SystemUI patches, and SELinux policy | Screen/input authority, secure confirmation, durable audit, island rendering, keyguard-aware behavior, and privileged state | + +The current privileged APK is still the only build artifact. This boundary is +the first extraction step, not a claim that the APK can already be uploaded to +Google Play. + +## Capability Placement + +The public app profile can own AI Home as a user-selected launcher, immersive +window UI, voice and text interaction, generated surfaces, model/runtime +adapters, local history, public Android intents, and scheduled work that fits +normal Android limits. + +The OS profile remains authoritative for: + +- SystemUI-owned island and keyguard-safe rendering; +- non-secure lock-screen defaults and global hardware gestures; +- silent screen context and capture; +- cross-app input and task control; +- secure settings and other signature/privileged operations; +- tamper-resistant confirmation, policy, and audit storage; +- stronger direct-boot and always-available execution guarantees. + +Public Android roles or APIs can cover some phone, SMS, notification, calendar, +contact, and launcher features. Their availability must be represented as +capabilities of the active gateway, never inferred from the package name. + +## Extraction Sequence + +Keep this work in reviewable follow-up changes: + +1. Route runtime and adaptive-surface execution through `PhoneToolGateway`. +2. Split `FrameworkToolExecutor` into public-Android tools and OS-only tools, + with both implementations preserving the same result contract. +3. Move settings and durable stores behind app-owned configuration/storage + interfaces instead of reading arbitrary `Settings.Secure` keys in portable + packages. +4. Add separate app manifests and build targets: a public-SDK Play profile and + the current platform-signed `system_ext` profile. +5. Add a machine-readable capability matrix and run the same runtime/surface + contract suite against both profiles. + +Until those steps are complete, changes must keep the existing OpenPhone OS +behavior green and must not weaken framework confirmation or audit paths to +make the app profile easier to implement. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6ff9669..0e917c7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -51,10 +51,23 @@ Device The current repo implements the first OpenPhone product layer: - `vendor/openphone` product config. -- `OpenPhoneAssistant` privileged app with a chat-style user surface, text and - voice task entry, stateful mic/send/stop composer action, advanced model and - developer controls, task grants, screen context, trace/audit export, OTA - preview controls, and audit controls. +- `OpenPhoneAssistant` privileged app with a dedicated Android Home activity, + a calm voice-first AI Home surface, explicit conventional-launcher App Space + handoff, text and voice task entry, stateful mic/send/stop controls, advanced + model and developer controls, task grants, screen context, trace/audit + export, OTA preview controls, and audit controls. AI Home is an immersive + normal application window rather than an opaque overlay: it hides Android + status/navigation chrome while foreground and allows transient system bars + by edge swipe, while Android retains ownership of secure keyguard, + notifications, IME, recents, emergency UI, and system dialogs. OpenPhone + disables only the non-secure swipe keyguard by default; configuring a + pattern, PIN, or password restores Android's credential gate. +- A `PhoneToolGateway` boundary keeps model, runtime, and adaptive-surface + packages independent from hidden `android.openphone` APIs. The current + `OpenPhoneOsToolGateway` binds that portable contract to the framework-backed + executor; a future public-SDK implementation can expose a smaller supported + tool set without forking the product runtime. See + [APP_OS_BOUNDARY.md](APP_OS_BOUNDARY.md). - Initial capability and policy config files. `scripts/check.sh` validates that the assistant fallback `PolicyEngine` covers every capability in `openphone_capabilities.json` with the same risk class. @@ -66,9 +79,51 @@ The current repo implements the first OpenPhone product layer: - Agent Runtime V1 background jobs. The first assistant-side implementation persists background agent turns, schedules them with AlarmManager, wakes them at boot/package replacement/service startup, delivers notification results, - and blocks state-changing background tools until a foreground reviewed - approval flow exists. The runtime contract is documented in + and pauses state-changing tools behind exact, expiring Android-owned review. + Approval or denial produces a bound checkpoint result; resume cannot + regenerate or substitute the authorized request, duplicate taps cannot + execute twice, and interrupted approval execution fails safe without replay. + The runtime contract is documented in [AGENT_RUNTIME_V1.md](AGENT_RUNTIME_V1.md). +- A unified agent-run projection reads jobs, watchers, commitments, and + foreground execution sessions without creating a second execution store. + AI Home uses it for stable activity bubbles and run detail, while the + SystemUI island reads the same projection for compact recent/live status. + Approval-needed bubbles expose exact review detail and Approve/Deny actions; + queued work can be paused/resumed, and the compact island routes review back + to AI Home. + Inspection and dismissal state is local presentation metadata; source stores + remain authoritative for execution and stop operations. +- The compact OpenPhone island is owned by SystemUI. The assistant publishes a + bounded, privacy-minimized state projection through the OpenPhone agent + Binder service; `system_server` validates it, assigns revision/timestamps, + retains the latest snapshot across assistant UI recreation, and sends + one-way change events to SystemUI. SystemUI renders only a fixed-size + `TYPE_STATUS_BAR_SUB_PANEL` window with non-modal touch behavior, redacts + personal/review state on keyguard, routes detail and approval to AI Home, and + degrades to a generic bounded status when the publisher becomes stale. Its + compact geometry, black capsule, centered typography, and terse state glyphs + preserve the established assistant-island presentation while moving window + ownership into SystemUI. The + assistant's full-screen `TYPE_SYSTEM_ERROR` layer remains only for + non-touchable pointer/glow visualization during active device control; the + old assistant island is a property-gated compatibility renderer. +- Adaptive Surface V1 is a phone-owned, revisioned UI document contract. + Runtimes can request only a bounded semantic component registry; the + assistant validates identity, ownership, expiry, depth/node/text/image + limits, local artifact provenance, registered tool bindings, tool parameter + shapes, accessibility labels, and mutation disclosure before persistence or + rendering. Surface actions resolve against the current revision and then + enter the existing runtime tool bridge, so policy, grants, confirmation, + idempotency, and framework audit remain authoritative. The Compose renderer + has no WebView, HTML, remote image URL, arbitrary color, absolute-position, + or runtime-code path. +- Built-in calendar search and message/notification result paths produce + deterministic Adaptive Surface documents from trusted phone tool results. + Tool artifacts retain session/runtime/capability/sensitivity/expiry + provenance separately from the layout, while the surface repository survives + activity recreation and fails closed on stale, malformed, expired, or + cross-session content. - Runtime Agent Protocol for external agent runtimes. This is an OpenPhone AI layer boundary, not a separate product stack: Android owns sessions, screen context, tool execution, confirmations, and audit, while adapters map remote diff --git a/docs/README.md b/docs/README.md index 38ef2ed..7ba38b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,6 +26,8 @@ it locally with: - [Architecture](ARCHITECTURE.md) — system layers, OS services, agent runtime, and current implementation boundaries. +- [App and OS Boundary](APP_OS_BOUNDARY.md) — shared product layers, the + phone-tool gateway, and the Play-app versus privileged-OS split. - [Capabilities](CAPABILITIES.md) — named capabilities, risk levels, and policy configuration. - [Agent Runtime](AGENT_RUNTIME_V1.md) — background job model, scheduling, diff --git a/docs/TESTING.md b/docs/TESTING.md index dbd33f2..88ff0c3 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -254,6 +254,145 @@ Manual or screenshot-backed checks: - tapping outside the composer dismisses the keyboard; - recent logcat has no `FATAL EXCEPTION` or `AndroidRuntime` crash signature. +## AI Home And App Space Smoke Test + +AI Home changes affect Android Home resolution and require a full product build +or an assistant APK plus the matching Launcher3 patch. An assistant-only APK +push does not remove Launcher3 as a competing Home candidate. + +On a build containing both sides of the change, verify: + +- after a reboot, the direct-boot-aware assistant service reports locked and + defers its island/runtime state while an early `OpenPhoneHomeActivity` + launch renders only its black locked-boot shell; neither path opens + credential-encrypted stores, and both initialize after `USER_UNLOCKED` + without crashing `org.openphone.assistant`; +- clean boot/setup resolves Home to + `org.openphone.assistant/.OpenPhoneHomeActivity`; +- an unsecured owner boots/wakes directly into AI Home without the + non-secure swipe keyguard, while a configured pattern, PIN, or password + continues to show Android's secure keyguard; +- the Home surface owns the full display without status or navigation chrome, + remains usable when no model runtime is configured, and exposes transient + system bars only after an edge swipe; +- holding the orb starts voice capture and release submits it; +- a short tap opens text entry; +- the visible Apps action and a two-finger inward pinch open the explicit + Launcher3 App Space activity; +- Launcher3 initializes Quickstep without a + `OverviewComponentObserver` null-home crash when App Space opens; +- pressing Home from App Space or another app returns to AI Home; +- tapping the compact idle island returns to AI Home; +- queued/running jobs, watchers, commitments, and foreground sessions appear + as stable Home activity bubbles without relaunching Home; +- Home shows no more than three individual activity bubbles plus a `+N` + overflow control, and the compact SystemUI island reports the same live-run + count and attention state; +- tapping a bubble shows its title, source kind, phase, latest progress, and + applicable Pause/Resume/Stop/Dismiss actions; +- stopping supported work updates its source store and survives assistant + process restart; dismissing a terminal result hides only its presentation + and never stops live work; +- terminal results remain visible until inspected and then age out after 24 + hours; attention-required and live work sort ahead of recent terminal work; +- the notification shade, Quick Settings, secure keyguard, IME, recents, + system dialogs, camera, dialer, and emergency surfaces are not covered by a + full-screen OpenPhone overlay; +- TalkBack exposes the voice orb, Apps action, settings/history action, text + composer, review actions, run bubbles, overflow control, and run actions. + +### SystemUI island checks + +Run `node scripts/validate-island-contract.mjs` (also included in +`./scripts/check.sh`). The idle and approval-needed snapshots must pass, while +unknown-mode, secret-bearing, and oversized snapshots must fail. This validator +also checks the framework/SystemUI patch contract: a bounded status-bar +sub-panel, non-modal touch flags, keyguard redaction, stale-state degradation, +and no confirmation execution from the island itself. + +This path requires a full product build containing framework patches `0020` +and `0021`; an assistant-only APK does not add the Binder listener or SystemUI +renderer. On a matching device, verify: + +- idle, listening, working, live-run count, completion, error, and + approval-needed transitions update without recreating SystemUI; +- recreating or killing only the assistant activity leaves the latest + `system_server` snapshot rendered, while a publisher that stays absent beyond + the stale bound becomes a generic `OpenPhone · Offline` chip; +- opening AI Home hides the island and returning to App Space restores it; +- tapping the island dismisses keyguard when necessary and opens AI Home; +- long-press stops only an unlocked active foreground task; approval-needed, + stale, background-only, and locked states route to AI Home instead; +- approval details and personal status are replaced with a generic locked glyph + on keyguard, and Approve/Deny remain available only on AI Home; +- the App Space chip preserves the legacy compact 288 x 90 px proportions on + the Pixel 9a reference density, with a black capsule and terse state glyphs; +- touches immediately outside the fixed chip reach the underlying app, and + notification shade, Quick Settings, IME, recents, camera, dialer, and + emergency UI remain unobstructed; +- active pointer/glow visualization is non-touchable and disappears when + device control finishes. + +### Resumable background review checks + +Run `node scripts/validate-background-review-contract.mjs` (also included in +`./scripts/check.sh`). The known-valid `awaiting_review` job must pass, while a +modified-parameter request and a secret-bearing request must fail. + +On a device, create a background job that proposes a registered state-changing +tool, then verify: + +- the tool does not execute before review and the job becomes + `awaiting_review`; +- its Home bubble and notification show the exact tool and arguments; +- Deny returns `background.action_denied`, queues the same job, and lets it + complete gracefully; +- Approve executes only the persisted request and resumes from its stored tool + result; +- tapping Approve or Deny twice executes at most once; +- force-stopping and restarting the assistant before review preserves the + request and expiry; +- letting the request expire queues a structured timeout continuation and + removes the review notification; +- killing the assistant after an approval claim never replays an outcome whose + completion is unknown; +- audit/context events include binding digests and lifecycle state but no raw + tool parameters. + +### Adaptive Surface V1 checks + +Run `node scripts/validate-surface-contract.mjs` (also included in +`./scripts/check.sh`) and verify the known-valid calendar/message fixtures pass +while external-image, unknown-component, and unknown-action fixtures fail. + +On a device, ask for a calendar list, message summary, and notification +summary, then verify: + +- a deterministic surface replaces prose-only result presentation on AI Home; +- rotating/recreating Home restores the same surface and revision; +- dismissing it survives recreation, and an expired surface is not shown; +- every interactive element has a useful TalkBack label; +- action taps use the registered phone tool, and a stale revision is rejected; +- read-only actions execute through the tool bridge; mutating actions show the + existing local approval UI and cannot execute before approval; +- sensitive message/notification surfaces do not appear over a locked device; +- malformed documents, undeclared actions, arbitrary component types, and + remote image URLs render nothing and create a rejection event. + +Useful resolution checks: + +```bash +adb shell cmd package resolve-activity \ + -a android.intent.action.MAIN \ + -c android.intent.category.HOME + +adb shell am start -W \ + -n org.openphone.assistant/.OpenPhoneHomeActivity + +adb shell am start -W \ + -n com.android.launcher3/com.android.launcher3.uioverrides.QuickstepLauncher +``` + If the mounted APK bytes match the new OTA but PackageManager still reports an older persistent system-app version, treat it as stale `/data/system` package metadata. On the Pixel 9a test device this happened after a v54 OTA: the diff --git a/docs/releases/CHANGELOG.md b/docs/releases/CHANGELOG.md index 5c54364..7abca51 100644 --- a/docs/releases/CHANGELOG.md +++ b/docs/releases/CHANGELOG.md @@ -8,6 +8,79 @@ flow, and device support. ## [Unreleased] +### Added + +- An explicit `PhoneToolGateway` app/OS boundary: runtime and adaptive-surface + execution no longer import hidden OpenPhone framework APIs directly, and the + framework-backed implementation reports its platform profile and supported + tools through the shared contract. +- Dedicated OpenPhone Android Home activity with a calm black AI Home surface, + press-and-hold/release voice input, text fallback, accessible controls, and a + two-finger App Space gesture. +- Explicit App Space handoff to the installed Launcher3 implementation while + preserving Home as a reliable return to AI Home. +- Unified durable-run projection across agent jobs, watchers, commitments, and + foreground sessions, with stable AI Home activity bubbles, overflow/detail + UI, unread/attention states, and source-routed stop controls. +- Adaptive Surface V1 schemas, fail-closed validator, revisioned durable + repository, bounded Compose component renderer, artifact provenance, and + policy-routed surface action dispatcher. +- Deterministic AI Home agenda and message/notification summary surfaces built + from trusted phone tool results. +- Runtime-neutral assistant output envelopes and additive surface + present/replace/dismiss/action-result events, including OpenClaw node-event + mapping and recorded conformance fixtures. +- Resumable background-action review with durable progress/checkpoint fields, + exact expiring approval bindings, Android notification and AI Home review + controls, structured denial/timeout continuation, duplicate-tap gating, + fail-safe interrupted-run repair, and conformance fixtures. +- A versioned compact-island state contract cached by `system_server`, with an + assistant-only publisher, one-way Binder listeners, bounded fields, + secret-like-content rejection, and known-valid/invalid conformance fixtures. +- A SystemUI-owned compact OpenPhone island with a fixed non-modal touch + window, keyguard privacy redaction, stale-publisher degradation, AI Home + routing, and unlocked active-task long-press stop. + +### Changed + +- AI Home now owns the full foreground display: Android status/navigation + chrome stays hidden until a transient edge swipe, and the non-secure swipe + keyguard is disabled by default while configured credentials continue to + invoke Android's secure keyguard. +- OpenPhone Assistant development package bumped to `versionCode=173` / + `versionName=0.1.137-dev` for the app/OS execution boundary. +- Launcher3 remains installed for conventional app access but no longer + advertises itself as a competing Home activity in OpenPhone products. +- Tapping the compact idle OpenPhone island returns directly to AI Home; the + existing pull-down gesture continues to expose detailed status. +- The expanded island Runs tab now reads the same durable-run projection as AI + Home instead of maintaining a jobs-only status view. +- AI Home run detail now supports pause/resume and exact background review; + approval-needed compact-island entries route the user to AI Home. +- The assistant no longer owns the product compact-island window. Its + `TYPE_SYSTEM_ERROR` overlay is reduced to a non-touchable pointer/glow layer + during active device control, with the previous island retained only behind + a product compatibility property. +- OpenPhone Assistant development package bumped to `versionCode=168` / + `versionName=0.1.132-dev` for the SystemUI island integration. + +### Fixed + +- The SystemUI-owned App Space island now preserves the previous compact + geometry, black capsule, centered typography, and icon-first state language + instead of replacing it with a wide, verbose status pill. +- AI Home now renders a minimal black locked-boot shell, while its + direct-boot-aware assistant service remains bindable with a locked status; + both wait for `USER_UNLOCKED` before initializing credential-encrypted + agent, island, or runtime state during early boot. +- Launcher3 App Space keeps its internal HOME resolution at a lower priority, + preventing Quickstep startup crashes while AI Home remains the preferred + Home activity. +- OpenPhone Assistant development package bumped to `versionCode=171` / + `versionName=0.1.135-dev` for the locked-boot fix. +- OpenPhone Assistant development package bumped to `versionCode=172` / + `versionName=0.1.136-dev` for the immersive AI Home shell. + ## [0.0.3] - 2026-07-03 ### Added diff --git a/docs/runtime/openclaw-integration.md b/docs/runtime/openclaw-integration.md index 15018e4..a8ccd18 100644 --- a/docs/runtime/openclaw-integration.md +++ b/docs/runtime/openclaw-integration.md @@ -20,8 +20,17 @@ Responsibilities: `RuntimeToolRequest`s. - Return tool results through `node.invoke.result`. - Send confirmation required/resolved events through existing node event paths. +- Map `openphone.surface.present|replace|dismiss` node events to the generic + runtime surface lifecycle, accepting only explicit assistant-output + envelopes. +- Map local surface invocation, result, and dismissal events back to + `openphone.surface.*` node events. +- Mark active phone sessions failed and leave their last accepted surface + inspectable if the gateway disconnects before completion. The adapter should not define OpenClaw core policy. Policy belongs in the plugin. +Surface validation, rendering, revision checks, and phone-tool execution remain +Android responsibilities rather than plugin or gateway responsibilities. ## Plugin diff --git a/docs/runtime/runtime-agent-protocol.md b/docs/runtime/runtime-agent-protocol.md index 793257c..136ef96 100644 --- a/docs/runtime/runtime-agent-protocol.md +++ b/docs/runtime/runtime-agent-protocol.md @@ -16,6 +16,10 @@ boundary; remote runtimes own agent reasoning. 5. `RuntimeToolBridge` validates the command, applies autonomy policy, creates confirmations for mutating actions, runs the phone tool, and returns the result. +6. A runtime may return plain text unchanged, or explicitly send an + `openphone.assistant_output.v1` envelope containing speech/text and an + optional `openphone.surface.v1` document. OpenPhone never attempts to parse + arbitrary assistant prose as an envelope. ## Stable Concepts @@ -26,6 +30,27 @@ boundary; remote runtimes own agent reasoning. - Attention request: phone-to-runtime user request. - Tool request: runtime-to-phone inspect/action request. - Confirmation: Android-local approval for risky actions. +- Assistant output: runtime-neutral structured result with optional surface. +- Adaptive surface: revisioned, phone-validated UI document whose actions are + bound back to registered phone tools. + +## Adaptive Surface Events + +The additive V1 events are: + +- `runtime.surface.present`; +- `runtime.surface.replace` with `expected_revision`; +- `runtime.surface.dismiss`; +- `runtime.surface.action_result`; +- `phone.surface.action_invoked`; +- `phone.surface.dismissed`. + +The phone verifies transport/runtime/session ownership before accepting a +surface. Replacement and dismissal are revision-bound. Renderer actions resolve +against the current stored revision and use `RuntimeToolBridge`; remote +runtimes cannot bypass local policy or confirmation. Accepted surfaces remain +inspectable if the runtime disconnects, while the phone execution session moves +to a terminal error state. ## Autonomy @@ -45,6 +70,8 @@ session's autonomy. If no session is available, it falls back to - Events: `runtime/protocol/openphone-events.json` - Capabilities: `runtime/protocol/openphone-capabilities.json` - Shape reference: `runtime/protocol/openphone-runtime.schema.json` +- Surface/output contracts: `schemas/openphone-surface.schema.json` and + `schemas/openphone-assistant-output.schema.json` ## Versioning diff --git a/docs/runtime/security-model.md b/docs/runtime/security-model.md index 2213a2e..26a054e 100644 --- a/docs/runtime/security-model.md +++ b/docs/runtime/security-model.md @@ -18,6 +18,13 @@ the authority for local capabilities, confirmations, and audit. execution to `userdebug`/`eng` builds. - Plain `ws://` runtime URLs are rejected unless they target local/private network hosts; public remote runtimes must use `wss://`. +- Structured UI is accepted only through an explicit assistant-output/surface + event. The phone rejects runtime/session mismatches, stale revisions, + unregistered tools, unknown components, excessive documents, remote image + URLs, and sensitive cross-session artifacts before persistence or rendering. +- Surface actions are declarative bindings, not executable runtime code. They + re-enter `RuntimeToolBridge`, including idempotency and Android-local + confirmation. ## Prompt And Context Safety diff --git a/integrations/cli/runtime/protocol/openphone-commands.json b/integrations/cli/runtime/protocol/openphone-commands.json index 84fc903..78ea5b2 100644 --- a/integrations/cli/runtime/protocol/openphone-commands.json +++ b/integrations/cli/runtime/protocol/openphone-commands.json @@ -1,5 +1,7 @@ { "version": 1, + "deprecated": true, + "superseded_by": "ifp1/", "commands": [ { "name": "openphone.device.status", diff --git a/integrations/mcp-server/runtime/protocol/openphone-commands.json b/integrations/mcp-server/runtime/protocol/openphone-commands.json index 84fc903..78ea5b2 100644 --- a/integrations/mcp-server/runtime/protocol/openphone-commands.json +++ b/integrations/mcp-server/runtime/protocol/openphone-commands.json @@ -1,5 +1,7 @@ { "version": 1, + "deprecated": true, + "superseded_by": "ifp1/", "commands": [ { "name": "openphone.device.status", diff --git a/overlay/packages/apps/OpenPhoneAssistant/AndroidManifest.xml b/overlay/packages/apps/OpenPhoneAssistant/AndroidManifest.xml index 09069e7..a271971 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/AndroidManifest.xml +++ b/overlay/packages/apps/OpenPhoneAssistant/AndroidManifest.xml @@ -1,8 +1,8 @@ + android:versionCode="173" + android:versionName="0.1.137-dev"> @@ -56,6 +56,24 @@ + + + + + + + + #0F172A #334155 #64748B + #000000 + #43B7FF diff --git a/overlay/packages/apps/OpenPhoneAssistant/res/values/strings.xml b/overlay/packages/apps/OpenPhoneAssistant/res/values/strings.xml index 00c4aa3..c32e25e 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/res/values/strings.xml +++ b/overlay/packages/apps/OpenPhoneAssistant/res/values/strings.xml @@ -15,4 +15,9 @@ Stop Open Lets OpenPhone understand visible controls during active assistant tasks. + + com.android.launcher3/com.android.launcher3.uioverrides.QuickstepLauncher + com.android.launcher3/com.android.launcher3.Launcher + com.android.launcher3/com.android.launcher3.Launcher3QuickStepGo + diff --git a/overlay/packages/apps/OpenPhoneAssistant/res/values/styles.xml b/overlay/packages/apps/OpenPhoneAssistant/res/values/styles.xml index ec54fec..9609420 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/res/values/styles.xml +++ b/overlay/packages/apps/OpenPhoneAssistant/res/values/styles.xml @@ -14,4 +14,18 @@ false @color/openphone_text_primary + + diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/AssistantActivityBackend.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/AssistantActivityBackend.java index 3efd173..190d756 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/AssistantActivityBackend.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/AssistantActivityBackend.java @@ -1,16 +1,20 @@ package org.openphone.assistant; import android.Manifest; +import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; import android.content.SharedPreferences; import android.content.pm.PackageManager; +import android.graphics.Color; import android.openphone.OpenPhoneAgentManager; import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.os.Looper; import android.os.SystemClock; +import android.os.UserManager; import android.provider.Settings; import android.speech.tts.TextToSpeech; import android.util.Base64; @@ -42,7 +46,14 @@ import org.openphone.assistant.orchestrator.OperatingMode; import org.openphone.assistant.orchestrator.OrchestratorDecision; import org.openphone.assistant.ota.OtaUpdateClient; +import org.openphone.assistant.platform.OpenPhoneOsToolGateway; +import org.openphone.assistant.platform.PhoneToolGateway; import org.openphone.assistant.policy.AppCapabilityPolicy; +import org.openphone.assistant.surface.AdaptiveSurface; +import org.openphone.assistant.surface.AssistantOutput; +import org.openphone.assistant.surface.DeterministicSurfaceFactory; +import org.openphone.assistant.surface.SurfaceMutationResult; +import org.openphone.assistant.surface.SurfaceRepository; import org.openphone.assistant.watchers.OpenPhoneWatcherScheduler; import java.io.IOException; @@ -233,6 +244,8 @@ public void run() { private boolean mRuntimeReplyTtsReady; private boolean mPendingRuntimeVoiceReply; private boolean mVoiceCaptureFinishRequested; + private boolean mWaitingForUserUnlock; + private BroadcastReceiver mUserUnlockReceiver; private boolean mRealtimeVoiceErrorShown; private boolean mPendingVoiceForceClassic; private boolean mVolumeChordPendingClassic; @@ -268,9 +281,18 @@ public void run() { private boolean mComposeAdvancedVisible; private ComposeStateCallbacks mComposeStateCallbacks; + PhoneToolGateway phoneToolGatewayForSurfaces() { + return new OpenPhoneOsToolGateway(this, mAgentManager); + } + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); + UserManager userManager = getSystemService(UserManager.class); + if (userManager != null && !userManager.isUserUnlocked()) { + showLockedBootShell(); + return; + } mAgentManager = getSystemService(OpenPhoneAgentManager.class); mPointerOverlayController = new PointerOverlayController(this); // Wire inline Approve / Deny on the island to this activity's @@ -313,20 +335,35 @@ public void handleOnBackPressed() { } return; } + if (isHomeSurface()) { + return; + } setEnabled(false); getOnBackPressedDispatcher().onBackPressed(); } }); - setContentView(AssistantComposeHost.createView(this)); + setContentView(createActivityContentView()); setServiceIslandVisible(false); mPointerOverlayController.hide(); refreshAll(); applyDebugIntentExtras(getIntent()); } + /** + * Activity-specific content hook. The conversation/settings activity keeps + * the existing Compose host while OpenPhoneHomeActivity supplies the + * deliberately smaller AI Home surface. + */ + protected View createActivityContentView() { + return AssistantComposeHost.createView(this); + } + @Override protected void onResume() { super.onResume(); + if (mWaitingForUserUnlock) { + return; + } sForegroundActivity = this; if (isControlSurface()) { return; @@ -338,15 +375,35 @@ protected void onResume() { OpenPhoneAccessibilityService.ensureEnabled(this); } + @Override + protected void onPause() { + if (mWaitingForUserUnlock) { + super.onPause(); + return; + } + if (isHomeSurface() && !isControlSurface()) { + setServiceIslandVisible(true); + } + super.onPause(); + } + @Override protected void onNewIntent(Intent intent) { super.onNewIntent(intent); setIntent(intent); + if (mWaitingForUserUnlock) { + return; + } applyDebugIntentExtras(intent); } @Override protected void onDestroy() { + unregisterUserUnlockReceiver(); + if (mWaitingForUserUnlock) { + super.onDestroy(); + return; + } shutdownRuntimeReplyTts(); if (isControlSurface()) { super.onDestroy(); @@ -363,6 +420,43 @@ protected void onDestroy() { super.onDestroy(); } + private void showLockedBootShell() { + mWaitingForUserUnlock = true; + View lockedBootView = new View(this); + lockedBootView.setBackgroundColor(Color.BLACK); + setContentView(lockedBootView); + mUserUnlockReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + if (!Intent.ACTION_USER_UNLOCKED.equals(intent.getAction())) { + return; + } + unregisterUserUnlockReceiver(); + recreate(); + } + }; + registerReceiver(mUserUnlockReceiver, new IntentFilter(Intent.ACTION_USER_UNLOCKED), + Context.RECEIVER_NOT_EXPORTED); + UserManager userManager = getSystemService(UserManager.class); + if (userManager != null && userManager.isUserUnlocked()) { + unregisterUserUnlockReceiver(); + recreate(); + return; + } + Log.i(TAG, "AI Home is waiting for credential-encrypted storage to unlock"); + } + + private void unregisterUserUnlockReceiver() { + if (mUserUnlockReceiver == null) { + return; + } + try { + unregisterReceiver(mUserUnlockReceiver); + } catch (IllegalArgumentException ignored) { + } + mUserUnlockReceiver = null; + } + private void setServiceIslandVisible(boolean visible) { Intent intent = new Intent(this, OpenPhoneAssistantService.class); intent.setAction(visible @@ -526,6 +620,10 @@ protected boolean isControlSurface() { return false; } + protected boolean isHomeSurface() { + return false; + } + private boolean isAgentOrVoiceActive() { return mListening || mActiveTaskId != null || mAgentThread != null || mChatThread != null || mRunningChatAdapter != null @@ -757,6 +855,42 @@ public void onComposeMic() { startVoiceAgent(); } + public void onHomeVoiceHoldStarted() { + // AI Home's primary gesture has deterministic push-to-talk semantics: + // hold records, release transcribes and submits. Live sessions remain + // available through the volume gesture and assistant settings. + startVoiceAgent(true, true, "ai_home_voice"); + } + + public void onHomeVoiceHoldFinished() { + if (!mListening || !mVoiceHoldToRecord) { + return; + } + mVoiceCaptureFinishRequested = true; + OpenAiSpeechTranscriber speechTranscriber = mRunningSpeechTranscriber; + if (speechTranscriber != null) { + speechTranscriber.stopRecording(); + } + } + + public void onHomeVoiceHoldCancelled() { + if (mListening || mRunningRealtimeVoiceSession != null + || mRunningGeminiLiveVoiceSession != null) { + stopTask(); + } + } + + public void onHomeTextSubmitted(String text) { + String clean = text == null ? "" : text.trim(); + if (clean.isEmpty()) { + setTaskText("Type a request first."); + updateIsland("Ready"); + return; + } + setCurrentGoalText(clean); + routeMessageFromCurrentMessage("ai_home"); + } + public void onComposeStop() { if (mActiveTaskId == null && mAgentThread == null && mChatThread != null) { cancelChatRun(); @@ -3318,6 +3452,11 @@ public void run() { showConfirmationIfNeeded(result); boolean finished = result.contains("\"status\":\"task.finished\"") || result.contains("\"status\": \"task.finished\""); + if (finished) { + AdaptiveSurface surface = + presentDeterministicSurface(result, taskId); + trajectory.recordSurfacePresented(surface); + } String displayReply = finished ? taskFinishedMessage(result) : agentResultForDisplay(result); appendConversation("OpenPhone", displayReply); @@ -3344,6 +3483,31 @@ public void run() { agentThread.start(); } + private AdaptiveSurface presentDeterministicSurface( + String agentResultJson, String sessionId) { + AdaptiveSurface surface = new DeterministicSurfaceFactory(this) + .fromAgentResult(agentResultJson, sessionId, RuntimeRegistry.BUILTIN); + if (surface == null) { + return null; + } + SurfaceMutationResult result = new SurfaceRepository(this).present(surface); + if (!result.ok) { + Log.w(TAG, "Deterministic surface rejected code=" + result.code); + return null; + } + AssistantOutput output = AssistantOutput.builtin( + sessionId, "", taskFinishedMessage(agentResultJson), surface); + if (output != null) { + recordContextAgentEvent( + "assistant.output.presented", + "Assistant output presented", + output.displayText(), + sessionId, + output.toJson().toString()); + } + return surface; + } + private void confirmPending(boolean approved) { if (mAgentManager == null) { setTaskText("Agent service is unavailable."); diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneAssistantService.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneAssistantService.java index 824a1b1..0ea6eac 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneAssistantService.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneAssistantService.java @@ -1,7 +1,10 @@ package org.openphone.assistant; import android.app.Service; +import android.content.BroadcastReceiver; +import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; import android.os.Build; import android.os.IBinder; import android.os.RemoteException; @@ -24,7 +27,9 @@ import org.openphone.assistant.runtime.RuntimeRegistry; import org.openphone.assistant.model.ModelEndpointConfig; import org.openphone.assistant.model.OpenAiResponsesAgentAdapter; +import org.openphone.assistant.jobs.BackgroundJobReviewManager; import org.openphone.assistant.jobs.OpenPhoneAgentJobScheduler; +import org.openphone.assistant.platform.OpenPhoneOsToolGateway; import org.openphone.assistant.policy.AuditLog; import org.openphone.assistant.policy.PolicyDecision; import org.openphone.assistant.policy.PolicyEngine; @@ -56,6 +61,14 @@ public final class OpenPhoneAssistantService extends Service { "org.openphone.assistant.extra.RUNTIME_ATTENTION_AUTONOMY"; public static final String EXTRA_RUNTIME_ATTENTION_INCLUDE_SCREEN = "org.openphone.assistant.extra.RUNTIME_ATTENTION_INCLUDE_SCREEN"; + public static final String ACTION_RUNTIME_SURFACE_EVENT = + "org.openphone.assistant.action.RUNTIME_SURFACE_EVENT"; + public static final String EXTRA_RUNTIME_SURFACE_RUNTIME = + "org.openphone.assistant.extra.RUNTIME_SURFACE_RUNTIME"; + public static final String EXTRA_RUNTIME_SURFACE_EVENT = + "org.openphone.assistant.extra.RUNTIME_SURFACE_EVENT"; + public static final String EXTRA_RUNTIME_SURFACE_PAYLOAD = + "org.openphone.assistant.extra.RUNTIME_SURFACE_PAYLOAD"; private static final int MAX_PENDING_RUNTIME_VOICE_REPLIES = 16; private static volatile String sLatestRuntimeStatusJson = "{\"status\":\"disabled\",\"manager_status\":\"not_created\"}"; @@ -70,6 +83,8 @@ public final class OpenPhoneAssistantService extends Service { private TextToSpeech mRuntimeReplyTts; private boolean mRuntimeReplyTtsReady; private final Set mPendingRuntimeVoicePhoneSessions = new LinkedHashSet<>(); + private volatile boolean mUnlockedStateReady; + private BroadcastReceiver mUserUnlockReceiver; private final PolicyEngine mPolicyEngine = new PolicyEngine(); private final AuditLog mAuditLog = new AuditLog(TAG); @@ -82,6 +97,9 @@ public final class OpenPhoneAssistantService extends Service { @Override public String getStatus() { String runtimeStatus = "runtime=" + runtimeStatusJson(); + if (!mUnlockedStateReady) { + return "assistant.locked " + runtimeStatus; + } if (mAgentManager == null) { return "assistant.ready framework.unavailable " + runtimeStatus; } @@ -90,6 +108,9 @@ public String getStatus() { @Override public String startTask(String taskRequestJson) throws RemoteException { + if (!mUnlockedStateReady) { + return userLockedResponse(); + } if (mAgentManager != null) { return mAgentManager.startTask(taskRequestJson); } @@ -98,6 +119,9 @@ public String startTask(String taskRequestJson) throws RemoteException { @Override public String getScreenContext(String taskId) throws RemoteException { + if (!mUnlockedStateReady) { + return userLockedResponse(); + } if (mAgentManager != null) { return mAgentManager.getScreenContext(taskId); } @@ -106,6 +130,9 @@ public String getScreenContext(String taskId) throws RemoteException { @Override public String executeAction(String taskId, String actionRequestJson) throws RemoteException { + if (!mUnlockedStateReady) { + return userLockedResponse(); + } if (mAgentManager != null) { return mAgentManager.executeAction(taskId, actionRequestJson); } @@ -115,6 +142,9 @@ public String executeAction(String taskId, String actionRequestJson) throws Remo @Override public String confirmAction(String pendingActionId, boolean approved) throws RemoteException { + if (!mUnlockedStateReady) { + return userLockedResponse(); + } if (mAgentManager != null) { return mAgentManager.confirmAction(pendingActionId, approved); } @@ -133,6 +163,9 @@ public String evaluateCapability(String capabilityId) throws RemoteException { @Override public String getAuditLog(int maxEvents) throws RemoteException { + if (!mUnlockedStateReady) { + return "{\"events\":[],\"source\":\"assistant.user_locked\"}"; + } if (mAgentManager != null) { return mAgentManager.getAuditLog(maxEvents); } @@ -143,6 +176,19 @@ public String getAuditLog(int maxEvents) throws RemoteException { @Override public void onCreate() { super.onCreate(); + if (!isUserUnlocked()) { + waitForUserUnlock(); + return; + } + initializeUnlockedState(); + } + + private void initializeUnlockedState() { + if (mUnlockedStateReady || !isUserUnlocked()) { + return; + } + unregisterUserUnlockReceiver(); + mUnlockedStateReady = true; mPointerOverlayController = new PointerOverlayController(this, this::answerScreenInOverlay); mPointerOverlayController.setConfirmationHandler(new PointerOverlayController.ConfirmationHandler() { @Override @@ -178,6 +224,10 @@ public void deny() { @Override public int onStartCommand(Intent intent, int flags, int startId) { + if (!ensureUnlockedStateReady()) { + Log.i(TAG, "Deferring assistant service command until user unlock"); + return START_STICKY; + } refreshIslandAutonomy(); ensureRuntimeManagerReady(); String action = intent != null ? intent.getAction() : null; @@ -204,6 +254,10 @@ public int onStartCommand(Intent intent, int flags, int startId) { requestRuntimeAttention(intent); return START_STICKY; } + if (ACTION_RUNTIME_SURFACE_EVENT.equals(action)) { + forwardRuntimeSurfaceEvent(intent); + return START_STICKY; + } if (OpenPhoneNotificationController.ACTION_START.equals(action)) { mIslandHiddenByActivity = false; mNotificationTaskId = startNotificationTask(); @@ -221,6 +275,17 @@ public int onStartCommand(Intent intent, int flags, int startId) { OpenPhoneNotificationController.ACTION_EXTERNAL_APPROVE.equals(action)); return START_STICKY; } + if (OpenPhoneNotificationController.ACTION_BACKGROUND_APPROVE.equals(action) + || OpenPhoneNotificationController.ACTION_BACKGROUND_DENY.equals(action)) { + final String confirmationId = intent == null ? "" : intent.getStringExtra( + OpenPhoneNotificationController.EXTRA_BACKGROUND_CONFIRMATION_ID); + final boolean approved = + OpenPhoneNotificationController.ACTION_BACKGROUND_APPROVE.equals(action); + new Thread(() -> BackgroundJobReviewManager.resolve( + OpenPhoneAssistantService.this, confirmationId, approved), + "OpenPhoneBackgroundReview").start(); + return START_STICKY; + } if (!mIslandHiddenByActivity) { mPointerOverlayController.showMicButton(); } @@ -237,6 +302,7 @@ public IBinder onBind(Intent intent) { @Override public void onDestroy() { + unregisterUserUnlockReceiver(); if (mPointerOverlayController != null) { mPointerOverlayController.hide(); } @@ -248,6 +314,56 @@ public void onDestroy() { super.onDestroy(); } + private boolean ensureUnlockedStateReady() { + if (mUnlockedStateReady) { + return true; + } + if (!isUserUnlocked()) { + waitForUserUnlock(); + return false; + } + initializeUnlockedState(); + return mUnlockedStateReady; + } + + private void waitForUserUnlock() { + sLatestRuntimeStatusJson = USER_LOCKED_RUNTIME_STATUS; + if (mUserUnlockReceiver != null) { + return; + } + mUserUnlockReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + if (!Intent.ACTION_USER_UNLOCKED.equals(intent.getAction())) { + return; + } + initializeUnlockedState(); + } + }; + registerReceiver(mUserUnlockReceiver, new IntentFilter(Intent.ACTION_USER_UNLOCKED), + Context.RECEIVER_NOT_EXPORTED); + if (isUserUnlocked()) { + initializeUnlockedState(); + return; + } + Log.i(TAG, "Assistant service is waiting for credential-encrypted storage to unlock"); + } + + private void unregisterUserUnlockReceiver() { + if (mUserUnlockReceiver == null) { + return; + } + try { + unregisterReceiver(mUserUnlockReceiver); + } catch (IllegalArgumentException ignored) { + } + mUserUnlockReceiver = null; + } + + private static String userLockedResponse() { + return "{\"status\":\"denied\",\"reason\":\"user_locked\"}"; + } + private String startNotificationTask() { if (mAgentManager == null) { return null; @@ -280,6 +396,21 @@ private String runtimeStatusJson() { return status; } + private void forwardRuntimeSurfaceEvent(Intent intent) { + if (mRuntimeManager == null || intent == null) { + return; + } + String runtime = cleanRuntime(intent.getStringExtra(EXTRA_RUNTIME_SURFACE_RUNTIME)); + String event = cleanExtra(intent.getStringExtra(EXTRA_RUNTIME_SURFACE_EVENT), ""); + JSONObject payload = parseObject( + intent.getStringExtra(EXTRA_RUNTIME_SURFACE_PAYLOAD)); + if (runtime.isEmpty() || event.isEmpty()) { + return; + } + mRuntimeManager.sendEventToRuntime(runtime, new org.openphone.assistant.runtime.RuntimeEvent( + event, payload)); + } + static String latestRuntimeStatusJson() { return sLatestRuntimeStatusJson; } @@ -291,7 +422,8 @@ private boolean ensureRuntimeManagerReady() { return false; } if (mRuntimeManager == null) { - mRuntimeManager = new RuntimeManager(this, mAgentManager); + mRuntimeManager = new RuntimeManager( + this, new OpenPhoneOsToolGateway(this, mAgentManager)); configureRuntimeCallback(); mRuntimeManager.start(); } @@ -310,7 +442,8 @@ private void reloadRuntimeManager() { return; } if (mRuntimeManager == null) { - mRuntimeManager = new RuntimeManager(this, mAgentManager); + mRuntimeManager = new RuntimeManager( + this, new OpenPhoneOsToolGateway(this, mAgentManager)); } configureRuntimeCallback(); mRuntimeManager.start(); diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeActivity.kt b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeActivity.kt new file mode 100644 index 0000000..8fddd40 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeActivity.kt @@ -0,0 +1,132 @@ +package org.openphone.assistant + +import android.content.ComponentName +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Bundle +import android.view.View +import android.view.WindowInsets +import android.view.WindowInsetsController +import android.view.WindowManager +import org.openphone.assistant.context.ContextIndexStore +import java.util.Locale + +/** + * OpenPhone's normal Android HOME activity. + * + * The activity owns the full display while it is foreground, but remains a + * normal application window rather than a system overlay. Android therefore + * retains ownership of secure keyguard, transient system bars, IME, recents, + * system dialogs, and task navigation. + */ +class OpenPhoneHomeActivity : AssistantActivityBackend() { + override fun onCreate(savedInstanceState: Bundle?) { + configureImmersiveWindow() + super.onCreate(savedInstanceState) + enterImmersiveHome() + } + + override fun onResume() { + super.onResume() + enterImmersiveHome() + } + + override fun onWindowFocusChanged(hasFocus: Boolean) { + super.onWindowFocusChanged(hasFocus) + if (hasFocus) enterImmersiveHome() + } + + override fun isHomeSurface(): Boolean = true + + override fun createActivityContentView(): View = OpenPhoneHomeComposeHost.createView(this) + + fun submitHomeText(text: String) { + if (requestsAppSpace(text)) { + if (!openAppSpace()) { + notifyHomeMessage("App Space is unavailable. Open Home settings to choose a launcher.") + } + return + } + onHomeTextSubmitted(text) + } + + fun openAppSpace(): Boolean { + val candidates = resources.getStringArray(R.array.openphone_app_space_components) + for (flattened in candidates) { + val component = ComponentName.unflattenFromString(flattened) ?: continue + if (!isActivityAvailable(component)) continue + val intent = Intent(Intent.ACTION_MAIN) + .setComponent(component) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED) + return try { + recordHomeTransition("app_space", component.flattenToShortString()) + startActivity(intent) + true + } catch (_: RuntimeException) { + false + } + } + return false + } + + fun openAssistant() { + startActivity( + Intent(this, MainActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP), + ) + } + + private fun isActivityAvailable(component: ComponentName): Boolean = + try { + packageManager.getActivityInfo(component, PackageManager.MATCH_DISABLED_COMPONENTS) + true + } catch (_: PackageManager.NameNotFoundException) { + false + } + + private fun recordHomeTransition(destination: String, detail: String) { + ContextIndexStore(this).recordAgentEvent( + "assistant.home.transition", + "OpenPhone Home transition", + destination, + "", + """{"destination":"$destination","detail":"${detail.replace("\"", "")}"}""", + ) + } + + private fun notifyHomeMessage(message: String) { + OpenPhoneHomeComposeHost.deliverLocalMessage(message) + } + + private fun configureImmersiveWindow() { + window.setDecorFitsSystemWindows(false) + window.attributes = window.attributes.apply { + layoutInDisplayCutoutMode = + WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS + } + } + + private fun enterImmersiveHome() { + configureImmersiveWindow() + window.insetsController?.let { controller -> + controller.systemBarsBehavior = + WindowInsetsController.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + controller.hide( + WindowInsets.Type.statusBars() or WindowInsets.Type.navigationBars(), + ) + } + } + + companion object { + private fun requestsAppSpace(text: String): Boolean { + val clean = text.trim().lowercase(Locale.US) + return clean == "apps" || + clean == "open apps" || + clean == "show apps" || + clean == "show my apps" || + clean == "open my apps" || + clean == "app space" || + clean == "open app space" + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeComposeHost.kt b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeComposeHost.kt new file mode 100644 index 0000000..58a7f09 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeComposeHost.kt @@ -0,0 +1,1177 @@ +package org.openphone.assistant + +import android.view.View +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.gestures.awaitEachGesture +import androidx.compose.foundation.gestures.awaitFirstDown +import androidx.compose.foundation.gestures.awaitLongPressOrCancellation +import androidx.compose.foundation.gestures.detectTransformGestures +import androidx.compose.foundation.gestures.waitForUpOrCancellation +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.BasicTextField +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material3.darkColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.hapticfeedback.HapticFeedbackType +import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.platform.ComposeView +import androidx.compose.ui.platform.LocalHapticFeedback +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.role +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import kotlinx.coroutines.delay +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.openphone.assistant.state.PendingConfirmation +import org.openphone.assistant.jobs.BackgroundJobReviewManager +import org.openphone.assistant.runs.AgentRunProjection +import org.openphone.assistant.runs.AgentRunSummary +import org.openphone.assistant.surface.AdaptiveSurface +import org.openphone.assistant.surface.AdaptiveSurfaceView +import org.openphone.assistant.surface.SurfaceActionDispatcher +import org.openphone.assistant.surface.SurfaceActionResult +import org.openphone.assistant.surface.SurfaceRepository +import org.openphone.assistant.surface.SurfaceRuntimeNotifier +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +enum class HomeAgentMode { + Idle, + Listening, + Thinking, + Running, + Review, + Error, + Result, +} + +data class HomeUiState( + val mode: HomeAgentMode = HomeAgentMode.Idle, + val status: String = "Ready", + val resultText: String = "", + val pending: PendingConfirmation? = null, + val runs: List = emptyList(), + val surface: AdaptiveSurface? = null, + val surfaceActionStatus: String = "", + val surfaceConfirmation: Boolean = false, +) + +object OpenPhoneHomeComposeHost { + private val state = MutableStateFlow(HomeUiState()) + + @JvmStatic + fun createView(activity: OpenPhoneHomeActivity): View { + state.value = HomeUiState() + val runProjection = AgentRunProjection(activity) + val surfaceRepository = SurfaceRepository(activity) + val surfaceDispatcher = SurfaceActionDispatcher( + activity, + activity.phoneToolGatewayForSurfaces(), + ) + activity.setComposeStateCallbacks(object : AssistantActivityBackend.ComposeStateCallbacks { + override fun setTaskStatus(text: String) { + state.update { + it.copy( + status = text.ifBlank { "Ready" }, + mode = modeForStatus(text, it.mode), + ) + } + } + + override fun setContextText(text: String) = Unit + + override fun setAuditText(text: String) = Unit + + override fun setModelDisclosure(text: String) = Unit + + override fun setModelConfig( + useRealtime: Boolean, + useRealtime2: Boolean, + useLiveRealtimeVoice: Boolean, + useGeminiLiveVoice: Boolean, + useBroker: Boolean, + apiKey: String, + geminiApiKey: String, + brokerUrl: String, + brokerToken: String, + ) = Unit + + override fun setOtaStatus(text: String, canDownload: Boolean) = Unit + + override fun setRuntimeStatus( + text: String, + activeTaskId: String?, + running: Boolean, + listening: Boolean, + ) { + state.update { + it.copy( + mode = when { + listening -> HomeAgentMode.Listening + running || activeTaskId != null -> HomeAgentMode.Running + it.pending != null -> HomeAgentMode.Review + text.contains("error", ignoreCase = true) || + text.contains("failed", ignoreCase = true) -> HomeAgentMode.Error + else -> modeForStatus(text, HomeAgentMode.Idle) + }, + status = text.ifBlank { "Ready" }, + ) + } + } + + override fun setAutonomyMode(mode: String) = Unit + + override fun setComposerText(text: String) = Unit + + override fun addConversationMessage(speaker: String, message: String) { + if (speaker != "You" && message.isNotBlank()) { + state.update { + it.copy( + resultText = message.trim(), + status = "Ready", + mode = HomeAgentMode.Result, + ) + } + } + } + + override fun setPendingConfirmation( + actionId: String, + toolName: String, + summary: String, + ) { + state.update { + it.copy( + pending = PendingConfirmation(actionId, toolName, summary), + surfaceConfirmation = false, + status = "Approval needed", + mode = HomeAgentMode.Review, + ) + } + } + + override fun clearPendingConfirmation() { + state.update { + it.copy( + pending = null, + surfaceConfirmation = false, + mode = if (it.mode == HomeAgentMode.Review) HomeAgentMode.Idle else it.mode, + ) + } + } + + override fun showChat() = Unit + }) + return ComposeView(activity).apply { + setContent { + val uiState by state.collectAsState() + val scope = androidx.compose.runtime.rememberCoroutineScope() + LaunchedEffect(runProjection) { + while (true) { + val projection = withContext(Dispatchers.IO) { + runProjection.snapshot(24) to surfaceRepository.currentVisible() + } + state.update { + it.copy( + runs = projection.first, + surface = projection.second, + ) + } + delay(2_000L) + } + } + AiHomeTheme { + OpenPhoneHomeScreen( + state = uiState, + onVoiceStart = activity::onHomeVoiceHoldStarted, + onVoiceFinish = activity::onHomeVoiceHoldFinished, + onVoiceCancel = activity::onHomeVoiceHoldCancelled, + onSubmitText = activity::submitHomeText, + onOpenApps = { + if (!activity.openAppSpace()) { + deliverLocalMessage("App Space is unavailable.") + } + }, + onOpenAssistant = activity::openAssistant, + onApprove = { + val current = state.value + if (!current.surfaceConfirmation) { + activity.onComposeApprove() + } else { + val confirmationId = current.pending?.actionId.orEmpty() + scope.launch { + val result = withContext(Dispatchers.IO) { + surfaceDispatcher.resolveConfirmation( + confirmationId, + true, + ) + } + applySurfaceActionResult(result) + } + } + }, + onDeny = { + val current = state.value + if (!current.surfaceConfirmation) { + activity.onComposeDeny() + } else { + val confirmationId = current.pending?.actionId.orEmpty() + scope.launch { + val result = withContext(Dispatchers.IO) { + surfaceDispatcher.resolveConfirmation( + confirmationId, + false, + ) + } + applySurfaceActionResult(result) + } + } + }, + onSurfaceAction = { surface, actionId, input -> + scope.launch { + state.update { it.copy(surfaceActionStatus = "Working…") } + val result = withContext(Dispatchers.IO) { + surfaceDispatcher.invoke( + surface.surfaceId, + surface.revision, + actionId, + input, + "surface:${surface.surfaceId}:${surface.revision}:$actionId", + ) + } + applySurfaceActionResult(result) + } + }, + onDismissSurface = { surface -> + scope.launch { + withContext(Dispatchers.IO) { + surfaceRepository.dismiss(surface.surfaceId, "user_dismissed") + SurfaceRuntimeNotifier.dismissed( + activity, + surface, + "user_dismissed", + ) + } + state.update { + it.copy( + surface = null, + surfaceActionStatus = "", + pending = if (it.surfaceConfirmation) null else it.pending, + surfaceConfirmation = false, + ) + } + } + }, + onStopRun = { runId -> + scope.launch { + withContext(Dispatchers.IO) { + runProjection.stop(runId) + } + val refreshed = withContext(Dispatchers.IO) { + runProjection.snapshot(24) + } + state.update { it.copy(runs = refreshed) } + } + }, + onPauseRun = { runId -> + scope.launch { + withContext(Dispatchers.IO) { + runProjection.pause(runId) + } + val refreshed = withContext(Dispatchers.IO) { + runProjection.snapshot(24) + } + state.update { it.copy(runs = refreshed) } + } + }, + onResumeRun = { runId -> + scope.launch { + withContext(Dispatchers.IO) { + runProjection.resume(runId) + } + val refreshed = withContext(Dispatchers.IO) { + runProjection.snapshot(24) + } + state.update { it.copy(runs = refreshed) } + } + }, + onResolveRunReview = { run, approved -> + scope.launch { + state.update { + it.copy( + status = if (approved) { + "Executing approved action…" + } else { + "Applying denial…" + }, + mode = HomeAgentMode.Running, + ) + } + val result = withContext(Dispatchers.IO) { + BackgroundJobReviewManager.resolve( + activity, + run.pendingConfirmationId, + approved, + ) + } + val refreshed = withContext(Dispatchers.IO) { + runProjection.snapshot(24) + } + val resolved = JSONObject(result) + state.update { + it.copy( + runs = refreshed, + status = if ( + resolved.optString("status") == + "background.review_resolved" + ) { + "Background job resumed" + } else { + "Review was already resolved" + }, + mode = HomeAgentMode.Idle, + ) + } + } + }, + onReadRun = { runId -> + scope.launch { + withContext(Dispatchers.IO) { + runProjection.markRead(runId) + } + val refreshed = withContext(Dispatchers.IO) { + runProjection.snapshot(24) + } + state.update { it.copy(runs = refreshed) } + } + }, + onDismissRun = { runId -> + scope.launch { + withContext(Dispatchers.IO) { + runProjection.dismiss(runId) + } + val refreshed = withContext(Dispatchers.IO) { + runProjection.snapshot(24) + } + state.update { it.copy(runs = refreshed) } + } + }, + ) + } + } + } + } + + fun deliverLocalMessage(message: String) { + state.update { + it.copy( + mode = HomeAgentMode.Error, + status = message, + resultText = message, + ) + } + } + + private fun applySurfaceActionResult(result: SurfaceActionResult) { + if (result.status == "needs_confirmation") { + val confirmationId = result.result.optString("confirmation_id", "") + val tool = result.result.optString("tool", "") + val summary = result.result.optString( + "summary", + result.message.ifBlank { "Approve this surface action?" }, + ) + state.update { + it.copy( + pending = PendingConfirmation(confirmationId, tool, summary), + surfaceConfirmation = true, + surfaceActionStatus = "Approval required", + mode = HomeAgentMode.Review, + ) + } + return + } + val message = when (result.status) { + "ok" -> "Done" + "denied" -> "Action denied" + "timeout" -> "Approval timed out" + else -> result.message.ifBlank { + result.code.replace('_', ' ').ifBlank { "Action failed" } + } + } + state.update { + it.copy( + pending = if (it.surfaceConfirmation) null else it.pending, + surfaceConfirmation = false, + surfaceActionStatus = message, + mode = if (result.status == "ok") HomeAgentMode.Result else HomeAgentMode.Error, + ) + } + } + + private fun modeForStatus(text: String, fallback: HomeAgentMode): HomeAgentMode { + val clean = text.trim().lowercase(Locale.US) + return when { + clean.startsWith("listening") -> HomeAgentMode.Listening + clean.startsWith("thinking") || clean.startsWith("reading") -> HomeAgentMode.Thinking + clean.startsWith("working") || clean.startsWith("starting") -> HomeAgentMode.Running + clean.contains("approval") || clean.contains("review") -> HomeAgentMode.Review + clean.contains("failed") || clean.contains("error") || + clean.contains("unavailable") -> HomeAgentMode.Error + clean == "ready" || clean.contains("is ready") || clean == "stopped" -> HomeAgentMode.Idle + else -> fallback + } + } +} + +private val AiHomeColors = darkColorScheme( + primary = Color(0xFF43B7FF), + onPrimary = Color.Black, + secondary = Color(0xFF5BE7D2), + onSecondary = Color.Black, + background = Color.Black, + onBackground = Color(0xFFF5F8FF), + surface = Color(0xFF080B12), + onSurface = Color(0xFFF5F8FF), + surfaceVariant = Color(0xFF111827), + onSurfaceVariant = Color(0xFF9CA9BC), + error = Color(0xFFFF6B75), +) + +@Composable +private fun AiHomeTheme(content: @Composable () -> Unit) { + MaterialTheme(colorScheme = AiHomeColors, content = content) +} + +@Composable +private fun OpenPhoneHomeScreen( + state: HomeUiState, + onVoiceStart: () -> Unit, + onVoiceFinish: () -> Unit, + onVoiceCancel: () -> Unit, + onSubmitText: (String) -> Unit, + onOpenApps: () -> Unit, + onOpenAssistant: () -> Unit, + onApprove: () -> Unit, + onDeny: () -> Unit, + onSurfaceAction: (AdaptiveSurface, String, JSONObject) -> Unit, + onDismissSurface: (AdaptiveSurface) -> Unit, + onStopRun: (String) -> Unit, + onPauseRun: (String) -> Unit, + onResumeRun: (String) -> Unit, + onResolveRunReview: (AgentRunSummary, Boolean) -> Unit, + onReadRun: (String) -> Unit, + onDismissRun: (String) -> Unit, +) { + var showTextInput by remember { mutableStateOf(false) } + var composer by remember { mutableStateOf("") } + var cumulativeZoom by remember { mutableStateOf(1f) } + var transitionStarted by remember { mutableStateOf(false) } + var selectedRunId by remember { mutableStateOf(null) } + var showAllRuns by remember { mutableStateOf(false) } + var now by remember { mutableStateOf(Date()) } + val selectedRun = state.runs.firstOrNull { it.id == selectedRunId } + + LaunchedEffect(Unit) { + while (true) { + now = Date() + delay(30_000L) + } + } + + Box( + modifier = Modifier + .fillMaxSize() + .background(Color.Black) + .pointerInput(Unit) { + detectTransformGestures( + panZoomLock = true, + onGesture = { _: Offset, _: Offset, zoom: Float, _: Float -> + cumulativeZoom *= zoom + if (!transitionStarted && cumulativeZoom < 0.72f) { + transitionStarted = true + onOpenApps() + } + if (cumulativeZoom > 0.96f) { + transitionStarted = false + } + }, + ) + } + .imePadding() + .padding(horizontal = 24.dp, vertical = 18.dp), + ) { + HomeHeader( + now = now, + onOpenApps = onOpenApps, + onOpenAssistant = onOpenAssistant, + modifier = Modifier.align(Alignment.TopCenter), + ) + + HomeResult( + state = state, + selectedRun = selectedRun, + showAllRuns = showAllRuns, + onSelectRun = { + selectedRunId = it.id + showAllRuns = false + if (it.unreadResult) onReadRun(it.id) + }, + onCloseRuns = { + selectedRunId = null + showAllRuns = false + }, + onStopRun = onStopRun, + onPauseRun = onPauseRun, + onResumeRun = onResumeRun, + onResolveRunReview = onResolveRunReview, + onDismissRun = onDismissRun, + onApprove = onApprove, + onDeny = onDeny, + onSurfaceAction = onSurfaceAction, + onDismissSurface = onDismissSurface, + modifier = Modifier + .align(Alignment.Center) + .padding(bottom = 112.dp), + ) + + Column( + modifier = Modifier + .align(Alignment.BottomCenter) + .fillMaxWidth(), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + AgentBubbleRow( + runs = state.runs, + onSelectRun = { + selectedRunId = it.id + showAllRuns = false + if (it.unreadResult) onReadRun(it.id) + }, + onShowAll = { + selectedRunId = null + showAllRuns = true + }, + ) + Spacer(Modifier.height(16.dp)) + AnimatedVisibility(visible = showTextInput) { + HomeTextComposer( + text = composer, + onTextChange = { composer = it }, + onSubmit = { + val request = composer.trim() + if (request.isNotEmpty()) { + onSubmitText(request) + composer = "" + showTextInput = false + } + }, + ) + } + Spacer(Modifier.height(18.dp)) + VoiceOrb( + mode = state.mode, + onVoiceStart = onVoiceStart, + onVoiceFinish = onVoiceFinish, + onVoiceCancel = onVoiceCancel, + onShortTap = { showTextInput = true }, + ) + Text( + text = when (state.mode) { + HomeAgentMode.Listening -> "Release to send" + HomeAgentMode.Thinking -> "Thinking" + HomeAgentMode.Running -> "Working" + HomeAgentMode.Review -> "Review needed" + else -> "Hold to talk · tap to type" + }, + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelMedium, + modifier = Modifier.padding(top = 12.dp), + ) + } + } +} + +@Composable +private fun HomeHeader( + now: Date, + onOpenApps: () -> Unit, + onOpenAssistant: () -> Unit, + modifier: Modifier = Modifier, +) { + val time = remember(now) { SimpleDateFormat("h:mm", Locale.getDefault()).format(now) } + val date = remember(now) { SimpleDateFormat("EEE, MMM d", Locale.getDefault()).format(now) } + Row( + modifier = modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + ) { + Column { + Text( + text = time, + color = MaterialTheme.colorScheme.onBackground, + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.Medium, + ) + Text( + text = date, + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelMedium, + ) + } + Spacer(Modifier.weight(1f)) + HomeHeaderAction("Apps", "Open apps", onOpenApps) + Spacer(Modifier.size(8.dp)) + HomeHeaderAction("•••", "Open OpenPhone history and settings", onOpenAssistant) + } +} + +@Composable +private fun HomeHeaderAction(label: String, description: String, onClick: () -> Unit) { + Text( + text = label, + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelLarge, + modifier = Modifier + .clip(RoundedCornerShape(18.dp)) + .clickable(onClick = onClick) + .semantics { + contentDescription = description + role = Role.Button + } + .padding(horizontal = 14.dp, vertical = 10.dp), + ) +} + +@Composable +private fun HomeResult( + state: HomeUiState, + selectedRun: AgentRunSummary?, + showAllRuns: Boolean, + onSelectRun: (AgentRunSummary) -> Unit, + onCloseRuns: () -> Unit, + onStopRun: (String) -> Unit, + onPauseRun: (String) -> Unit, + onResumeRun: (String) -> Unit, + onResolveRunReview: (AgentRunSummary, Boolean) -> Unit, + onDismissRun: (String) -> Unit, + onApprove: () -> Unit, + onDeny: () -> Unit, + onSurfaceAction: (AdaptiveSurface, String, JSONObject) -> Unit, + onDismissSurface: (AdaptiveSurface) -> Unit, + modifier: Modifier = Modifier, +) { + if (selectedRun != null || showAllRuns) { + HomeRunPanel( + runs = if (showAllRuns) state.runs else listOfNotNull(selectedRun), + showAll = showAllRuns, + onSelectRun = onSelectRun, + onClose = onCloseRuns, + onStopRun = onStopRun, + onPauseRun = onPauseRun, + onResumeRun = onResumeRun, + onResolveRunReview = onResolveRunReview, + onDismissRun = onDismissRun, + modifier = modifier, + ) + return + } + val surface = state.surface + if (surface != null && (state.pending == null || state.surfaceConfirmation)) { + Column( + modifier = modifier.fillMaxWidth(), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + AdaptiveSurfaceView( + surface = surface, + actionStatus = state.surfaceActionStatus, + onAction = { actionId, input -> + onSurfaceAction(surface, actionId, input) + }, + onDismiss = { onDismissSurface(surface) }, + ) + if (state.surfaceConfirmation && state.pending != null) { + Text( + state.pending.summary, + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.padding(top = 12.dp), + ) + Row( + modifier = Modifier.padding(top = 10.dp), + horizontalArrangement = Arrangement.spacedBy(12.dp), + ) { + ReviewAction("Deny", false, onDeny) + ReviewAction("Approve", true, onApprove) + } + } + } + return + } + val visibleText = state.pending?.summary + ?: state.resultText.ifBlank { + if (state.mode == HomeAgentMode.Idle) "" else state.status + } + Column( + modifier = modifier + .fillMaxWidth() + .padding(horizontal = 18.dp), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + if (visibleText.isNotBlank()) { + Text( + text = visibleText, + color = if (state.mode == HomeAgentMode.Error) { + MaterialTheme.colorScheme.error + } else { + MaterialTheme.colorScheme.onBackground + }, + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Medium, + ) + } + if (state.pending != null) { + Row( + modifier = Modifier.padding(top = 24.dp), + horizontalArrangement = Arrangement.spacedBy(12.dp), + ) { + ReviewAction("Deny", false, onDeny) + ReviewAction("Approve", true, onApprove) + } + } + } +} + +@Composable +private fun AgentBubbleRow( + runs: List, + onSelectRun: (AgentRunSummary) -> Unit, + onShowAll: () -> Unit, +) { + val visible = runs.take(3) + if (visible.isEmpty()) return + Row( + horizontalArrangement = Arrangement.spacedBy(12.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + visible.forEach { run -> + AgentBubble(run = run, onClick = { onSelectRun(run) }) + } + if (runs.size > visible.size) { + Box( + modifier = Modifier + .size(46.dp) + .clip(CircleShape) + .background(MaterialTheme.colorScheme.surfaceVariant) + .clickable(onClick = onShowAll) + .semantics { + contentDescription = "Show all ${runs.size} OpenPhone runs" + role = Role.Button + }, + contentAlignment = Alignment.Center, + ) { + Text( + "+${runs.size - visible.size}", + color = MaterialTheme.colorScheme.onSurface, + fontWeight = FontWeight.SemiBold, + ) + } + } + } +} + +@Composable +private fun AgentBubble(run: AgentRunSummary, onClick: () -> Unit) { + val accent = when { + run.needsAttention -> Color(0xFFFFC857) + run.status == "failed" || run.failureLike() -> MaterialTheme.colorScheme.error + run.kind == AgentRunSummary.KIND_WATCHER -> Color(0xFF5BE7D2) + run.unreadResult -> Color(0xFF76C9FF) + else -> Color(0xFF7186A8) + } + val label = when (run.kind) { + AgentRunSummary.KIND_WATCHER -> "◎" + AgentRunSummary.KIND_COMMITMENT -> "!" + AgentRunSummary.KIND_SESSION -> "↗" + else -> if (run.isLive) "●" else "✓" + } + Box( + modifier = Modifier + .size(52.dp) + .clip(CircleShape) + .background(Color(0xFF070A10)) + .border(if (run.needsAttention) 3.dp else 2.dp, accent, CircleShape) + .clickable(onClick = onClick) + .semantics { + contentDescription = "${run.title}. ${run.status}." + role = Role.Button + }, + contentAlignment = Alignment.Center, + ) { + Text(label, color = accent, fontWeight = FontWeight.Bold) + if (run.unreadResult || run.needsAttention) { + Box( + modifier = Modifier + .align(Alignment.TopEnd) + .size(11.dp) + .clip(CircleShape) + .background(if (run.needsAttention) Color(0xFFFFC857) else Color.White), + ) + } + } +} + +private fun AgentRunSummary.failureLike(): Boolean = + failureCountFromProgress(progressText) > 0 + +private fun failureCountFromProgress(progress: String): Int = + if (progress.contains("failed", ignoreCase = true) || + progress.contains("error", ignoreCase = true) + ) 1 else 0 + +@Composable +private fun HomeRunPanel( + runs: List, + showAll: Boolean, + onSelectRun: (AgentRunSummary) -> Unit, + onClose: () -> Unit, + onStopRun: (String) -> Unit, + onPauseRun: (String) -> Unit, + onResumeRun: (String) -> Unit, + onResolveRunReview: (AgentRunSummary, Boolean) -> Unit, + onDismissRun: (String) -> Unit, + modifier: Modifier = Modifier, +) { + Column( + modifier = modifier + .fillMaxWidth() + .clip(RoundedCornerShape(28.dp)) + .background(MaterialTheme.colorScheme.surface) + .border(1.dp, Color(0xFF263246), RoundedCornerShape(28.dp)) + .padding(20.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + Text( + if (showAll) "OpenPhone activity" else "Agent run", + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.titleMedium, + fontWeight = FontWeight.SemiBold, + ) + Spacer(Modifier.weight(1f)) + Text( + "Close", + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier + .clip(RoundedCornerShape(14.dp)) + .clickable(onClick = onClose) + .padding(8.dp), + ) + } + Spacer(Modifier.height(12.dp)) + if (runs.isEmpty()) { + Text("No active work", color = MaterialTheme.colorScheme.onSurfaceVariant) + } + runs.take(if (showAll) 8 else 1).forEach { run -> + Column( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(18.dp)) + .clickable { if (showAll) onSelectRun(run) } + .padding(vertical = 10.dp), + ) { + Text( + run.title.ifBlank { "OpenPhone work" }, + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.titleSmall, + fontWeight = FontWeight.SemiBold, + ) + Text( + "${run.kind.replace('_', ' ')} · ${run.phase.ifBlank { run.status }}", + color = if (run.needsAttention) { + Color(0xFFFFC857) + } else { + MaterialTheme.colorScheme.onSurfaceVariant + }, + style = MaterialTheme.typography.labelMedium, + modifier = Modifier.padding(top = 3.dp), + ) + if (run.progressText.isNotBlank()) { + Text( + run.progressText, + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.bodyMedium, + maxLines = if (showAll) 2 else 5, + modifier = Modifier.padding(top = 8.dp), + ) + } + if (!showAll && run.reviewSummary.isNotBlank() && + run.reviewSummary != run.progressText + ) { + Text( + run.reviewSummary, + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.bodyMedium, + maxLines = 6, + modifier = Modifier.padding(top = 10.dp), + ) + } + if (!showAll) { + if (run.pendingConfirmationId.isNotBlank()) { + Row( + modifier = Modifier.padding(top = 14.dp), + horizontalArrangement = Arrangement.spacedBy(10.dp), + ) { + ReviewAction("Deny", false) { + onResolveRunReview(run, false) + } + ReviewAction("Approve", true) { + onResolveRunReview(run, true) + } + } + } + Row( + modifier = Modifier.padding(top = 14.dp), + horizontalArrangement = Arrangement.spacedBy(10.dp), + ) { + if (run.canPause) { + RunAction("Pause", danger = false) { + onPauseRun(run.id) + } + } + if (run.canResume) { + RunAction("Resume", danger = false) { + onResumeRun(run.id) + } + } + if (run.canStop) { + RunAction("Stop", danger = true) { onStopRun(run.id) } + } + if (run.unreadResult || !run.isLive) { + RunAction("Dismiss", danger = false) { + onDismissRun(run.id) + onClose() + } + } + } + } + } + } + if (showAll && runs.size > 8) { + Text( + "${runs.size - 8} more", + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelMedium, + ) + } + } +} + +@Composable +private fun RunAction(label: String, danger: Boolean, onClick: () -> Unit) { + Text( + label, + color = if (danger) MaterialTheme.colorScheme.error + else MaterialTheme.colorScheme.primary, + fontWeight = FontWeight.SemiBold, + modifier = Modifier + .clip(RoundedCornerShape(16.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant) + .clickable(onClick = onClick) + .padding(horizontal = 14.dp, vertical = 9.dp), + ) +} + +@Composable +private fun ReviewAction(label: String, primary: Boolean, onClick: () -> Unit) { + Text( + text = label, + color = if (primary) Color.Black else MaterialTheme.colorScheme.onSurface, + fontWeight = FontWeight.SemiBold, + modifier = Modifier + .clip(RoundedCornerShape(24.dp)) + .background( + if (primary) MaterialTheme.colorScheme.secondary + else MaterialTheme.colorScheme.surfaceVariant, + ) + .clickable(onClick = onClick) + .padding(horizontal = 22.dp, vertical = 13.dp), + ) +} + +@Composable +private fun HomeTextComposer( + text: String, + onTextChange: (String) -> Unit, + onSubmit: () -> Unit, +) { + Row( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(28.dp)) + .background(MaterialTheme.colorScheme.surface) + .border(1.dp, Color(0xFF263246), RoundedCornerShape(28.dp)) + .padding(horizontal = 18.dp, vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + BasicTextField( + value = text, + onValueChange = onTextChange, + textStyle = MaterialTheme.typography.bodyLarge.copy( + color = MaterialTheme.colorScheme.onSurface, + ), + maxLines = 4, + modifier = Modifier + .weight(1f) + .padding(vertical = 10.dp), + decorationBox = { inner -> + Box(contentAlignment = Alignment.CenterStart) { + if (text.isBlank()) { + Text( + "Ask OpenPhone", + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + inner() + } + }, + ) + Text( + text = "Send", + color = if (text.isBlank()) { + MaterialTheme.colorScheme.onSurfaceVariant + } else { + MaterialTheme.colorScheme.primary + }, + fontWeight = FontWeight.SemiBold, + modifier = Modifier + .clip(RoundedCornerShape(18.dp)) + .clickable(enabled = text.isNotBlank(), onClick = onSubmit) + .padding(horizontal = 12.dp, vertical = 9.dp), + ) + } +} + +@Composable +private fun VoiceOrb( + mode: HomeAgentMode, + onVoiceStart: () -> Unit, + onVoiceFinish: () -> Unit, + onVoiceCancel: () -> Unit, + onShortTap: () -> Unit, +) { + val haptic = LocalHapticFeedback.current + val transition = rememberInfiniteTransition(label = "voice-orb") + val pulse by transition.animateFloat( + initialValue = 0.82f, + targetValue = 1f, + animationSpec = infiniteRepeatable( + animation = tween(if (mode == HomeAgentMode.Listening) 620 else 1500), + repeatMode = RepeatMode.Reverse, + ), + label = "voice-orb-pulse", + ) + val active = mode == HomeAgentMode.Listening || + mode == HomeAgentMode.Thinking || + mode == HomeAgentMode.Running + Box( + modifier = Modifier + .size(104.dp) + .semantics { + contentDescription = "Hold to talk to OpenPhone. Tap to type." + role = Role.Button + } + .pointerInput(Unit) { + awaitEachGesture { + val down = awaitFirstDown(requireUnconsumed = false) + val longPress = awaitLongPressOrCancellation(down.id) + if (longPress == null) { + onShortTap() + return@awaitEachGesture + } + haptic.performHapticFeedback(HapticFeedbackType.LongPress) + onVoiceStart() + val up = waitForUpOrCancellation() + if (up == null) { + onVoiceCancel() + } else { + onVoiceFinish() + } + } + }, + contentAlignment = Alignment.Center, + ) { + Canvas(Modifier.fillMaxSize()) { + drawCircle( + brush = Brush.radialGradient( + colors = listOf( + Color(0x6643B7FF), + Color(0x222A7FFF), + Color.Transparent, + ), + ), + radius = size.minDimension * 0.5f * pulse, + ) + drawCircle( + color = if (active) Color(0xFF5BE7D2) else Color(0xFF43B7FF), + radius = size.minDimension * 0.27f, + style = Stroke(width = if (active) 6.dp.toPx() else 3.dp.toPx()), + ) + drawCircle( + brush = Brush.radialGradient( + listOf(Color(0xFF65D7FF), Color(0xFF0A4BCB)), + ), + radius = size.minDimension * 0.21f, + ) + } + Text( + text = when (mode) { + HomeAgentMode.Listening -> "■" + HomeAgentMode.Thinking, HomeAgentMode.Running -> "•••" + else -> "●" + }, + color = Color.White, + fontWeight = FontWeight.Bold, + ) + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneNotificationController.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneNotificationController.java index 397d069..cba8ab8 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneNotificationController.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneNotificationController.java @@ -30,8 +30,14 @@ public final class OpenPhoneNotificationController { "org.openphone.assistant.action.EXTERNAL_APPROVE"; static final String ACTION_EXTERNAL_DENY = "org.openphone.assistant.action.EXTERNAL_DENY"; + static final String ACTION_BACKGROUND_APPROVE = + "org.openphone.assistant.action.BACKGROUND_APPROVE"; + static final String ACTION_BACKGROUND_DENY = + "org.openphone.assistant.action.BACKGROUND_DENY"; static final String EXTRA_EXTERNAL_CONFIRMATION_ID = "org.openphone.assistant.extra.EXTERNAL_CONFIRMATION_ID"; + static final String EXTRA_BACKGROUND_CONFIRMATION_ID = + "org.openphone.assistant.extra.BACKGROUND_CONFIRMATION_ID"; static final int NOTIFICATION_ID = 1001; private static final int COMMITMENT_NOTIFICATION_BASE_ID = 5000; private static final int WATCHER_NOTIFICATION_BASE_ID = 6000; @@ -229,6 +235,65 @@ public static void showAgentJobFailed(Context context, AgentJobRecord job, Strin manager.notify(agentJobNotificationId(job.id), builder.build()); } + public static void showAgentJobReview(Context context, AgentJobRecord job, + JSONObject pendingRequest) { + if (context == null || job == null || pendingRequest == null) { + return; + } + String confirmationId = pendingRequest.optString("confirmation_id", ""); + if (confirmationId.trim().isEmpty()) { + return; + } + NotificationManager manager = context.getSystemService(NotificationManager.class); + if (manager == null) { + return; + } + ensureAgentJobChannel(manager); + String tool = pendingRequest.optString("tool", "OpenPhone action"); + JSONObject params = pendingRequest.optJSONObject("params"); + String exact = params == null ? "{}" : params.toString(); + String detail = pendingRequest.optString("summary", "").trim(); + if (detail.isEmpty()) { + detail = tool + " with " + exact; + } + String expanded = "Job: " + job.title + "\nAction: " + tool + + "\nExact arguments: " + exact; + Notification.Builder builder = new Notification.Builder(context) + .setSmallIcon(R.drawable.ic_openphone_tile) + .setContentTitle("Review background action") + .setContentText(summarize(detail)) + .setStyle(new Notification.BigTextStyle().bigText(expanded)) + .setVisibility(Notification.VISIBILITY_PRIVATE) + .setShowWhen(true) + .setWhen(System.currentTimeMillis()) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setContentIntent(homePendingIntent(context, + jobRequestCode(job.id, 30))) + .addAction(new Notification.Action.Builder( + R.drawable.ic_openphone_tile, + "Approve", + backgroundReviewAction(context, ACTION_BACKGROUND_APPROVE, + confirmationId, job.id, 31)).build()) + .addAction(new Notification.Action.Builder( + R.drawable.ic_openphone_tile, + "Deny", + backgroundReviewAction(context, ACTION_BACKGROUND_DENY, + confirmationId, job.id, 32)).build()); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + builder.setChannelId(AGENT_JOB_CHANNEL_ID); + } + manager.notify(agentJobNotificationId(job.id), builder.build()); + } + + public static void cancelAgentJobReview(Context context, long jobId) { + NotificationManager manager = context == null + ? null : context.getSystemService(NotificationManager.class); + if (manager != null) { + manager.cancel(agentJobNotificationId(jobId)); + } + } + public static void showRuntimeMessage(Context context, String runtime, String title, String text, String messageId) { if (context == null) { @@ -385,6 +450,29 @@ private static PendingIntent runtimeAction(Context context, String action, runtimeRequestCode(confirmationId) + actionCode, intent, flags); } + private static PendingIntent backgroundReviewAction(Context context, String action, + String confirmationId, long jobId, int actionCode) { + Intent intent = new Intent(context, OpenPhoneTriggerReceiver.class); + intent.setAction(action); + intent.putExtra(EXTRA_BACKGROUND_CONFIRMATION_ID, confirmationId); + int flags = PendingIntent.FLAG_UPDATE_CURRENT; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags |= PendingIntent.FLAG_IMMUTABLE; + } + return PendingIntent.getBroadcast(context, + jobRequestCode(jobId, actionCode), intent, flags); + } + + private static PendingIntent homePendingIntent(Context context, int requestCode) { + Intent intent = new Intent(context, OpenPhoneHomeActivity.class); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_CLEAR_TOP); + int flags = PendingIntent.FLAG_UPDATE_CURRENT; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + flags |= PendingIntent.FLAG_IMMUTABLE; + } + return PendingIntent.getActivity(context, requestCode, intent, flags); + } + private static void ensureCommitmentChannel(Context context, NotificationManager manager) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { NotificationChannel channel = new NotificationChannel( diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/PointerOverlayController.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/PointerOverlayController.java index 8b1bc80..a74e679 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/PointerOverlayController.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/PointerOverlayController.java @@ -31,10 +31,11 @@ import org.json.JSONException; import org.json.JSONObject; import org.openphone.assistant.context.ContextIndexStore; +import org.openphone.assistant.island.IslandStateRepository; import org.openphone.assistant.runtime.RuntimeConfig; import org.openphone.assistant.runtime.RuntimeRegistry; -import org.openphone.assistant.jobs.AgentJobRecord; -import org.openphone.assistant.jobs.AgentJobStore; +import org.openphone.assistant.runs.AgentRunProjection; +import org.openphone.assistant.runs.AgentRunSummary; import org.openphone.assistant.watchers.OpenPhoneWatcherScheduler; import org.openphone.assistant.watchers.WatcherRecord; import org.openphone.assistant.watchers.WatcherStore; @@ -78,6 +79,7 @@ interface ConfirmationHandler { private static final long ISLAND_RESIZE_MS = 260L; private static final long THINKING_DOTS_INTERVAL_MS = 420L; private static final long REPLY_AUTO_COLLAPSE_MS = 7000L; + private static final long SYSTEM_UI_STATE_REFRESH_MS = 2000L; private static final int REPLY_MAX_LINES = 8; private static final int CAMERA_RESERVED_WIDTH = 96; private static final int CAMERA_ISLAND_FALLBACK_TOP = 42; @@ -90,6 +92,7 @@ interface ConfirmationHandler { private static final long DONE_VISIBLE_MS = 2200; private static final Set sControllers = Collections.newSetFromMap(new WeakHashMap<>()); + private static PointerOverlayController sIslandOwner; private static String sLatestUserMessage = ""; private static String sLatestAssistantMessage = ""; private static long sLatestConversationAtMillis; @@ -105,6 +108,7 @@ interface ConfirmationHandler { private final Handler mHandler = new Handler(Looper.getMainLooper()); private final Runnable mWatchdogHide = this::hide; private final ScreenAnswerProvider mScreenAnswerProvider; + private final IslandStateRepository mIslandStateRepository; private ConfirmationHandler mConfirmationHandler; private WindowManager mWindowManager; private FrameLayout mRoot; @@ -135,6 +139,7 @@ interface ConfirmationHandler { private TextView mDenyButton; private TextView mActionLabel; private String mMode = "idle"; + private String mActiveTaskId = ""; private String mStateDetail = ""; private String mTranscriptText = ""; private String mReplyText = ""; @@ -144,6 +149,7 @@ interface ConfirmationHandler { private boolean mInspectExpanded; private boolean mLargeExpanded; private boolean mIslandDragExpanded; + private boolean mIslandVisible; private float mIslandTouchDownY; private int mStatusTab = STATUS_TAB_CHAT; private int mThinkingDotsFrame; @@ -163,6 +169,17 @@ public void run() { mHandler.postDelayed(this, THINKING_DOTS_INTERVAL_MS); } }; + private final Runnable mSystemUiStateRefresh = new Runnable() { + @Override + public void run() { + if (!mIslandVisible || !isIslandOwner() + || !mIslandStateRepository.isSystemUiOwned()) { + return; + } + publishSystemUiState(); + mHandler.postDelayed(this, SYSTEM_UI_STATE_REFRESH_MS); + } + }; private ValueAnimator mIslandResizeAnimator; PointerOverlayController(Context context) { @@ -172,6 +189,7 @@ public void run() { PointerOverlayController(Context context, ScreenAnswerProvider screenAnswerProvider) { mContext = context.getApplicationContext(); mScreenAnswerProvider = screenAnswerProvider; + mIslandStateRepository = new IslandStateRepository(mContext); synchronized (sControllers) { sControllers.add(this); } @@ -185,6 +203,7 @@ void setConfirmationHandler(ConfirmationHandler handler) { void show(String taskId) { mHandler.post(() -> { mMode = "action_running"; + mActiveTaskId = taskId == null ? "" : taskId.trim(); mInspectExpanded = false; ensurePointerLayer(); showPointerDot(); @@ -196,6 +215,19 @@ void show(String taskId) { void hide() { mHandler.post(() -> { + boolean publishHidden; + synchronized (sControllers) { + publishHidden = sIslandOwner == null || sIslandOwner == this; + if (sIslandOwner == this) { + sIslandOwner = null; + } + } + mIslandVisible = false; + mHandler.removeCallbacks(mSystemUiStateRefresh); + if (publishHidden) { + mIslandStateRepository.publish( + mMode, false, mActiveTaskId, mWatchingCount); + } mHandler.removeCallbacks(mWatchdogHide); mHandler.removeCallbacks(mReplyAutoCollapse); stopThinkingDots(); @@ -406,6 +438,7 @@ void showMicButton() { private void showMicButtonNow() { stopThinkingDots(); mMode = mWatchingCount > 0 ? "watching" : "idle"; + mActiveTaskId = ""; mInspectExpanded = false; mLargeExpanded = false; mTranscriptText = ""; @@ -554,23 +587,29 @@ private void ensurePointerLayer() { } private void ensureIslandWindow() { - if (mWindowManager == null) { - mWindowManager = mContext.getSystemService(WindowManager.class); - } - if (mWindowManager == null) { - return; - } - // Only one island may exist across all controller instances - // (service + activities); the latest claimant wins. + mIslandVisible = true; ArrayList controllers; synchronized (sControllers) { + sIslandOwner = this; controllers = new ArrayList<>(sControllers); } + // Only one island state owner may exist across service and activity + // controller instances. The latest claimant wins. for (PointerOverlayController other : controllers) { if (other != this) { other.removeIslandNow(); } } + if (mIslandStateRepository.isSystemUiOwned()) { + startSystemUiStateRefresh(); + return; + } + if (mWindowManager == null) { + mWindowManager = mContext.getSystemService(WindowManager.class); + } + if (mWindowManager == null) { + return; + } if (mIslandRoot != null) { return; } @@ -786,14 +825,25 @@ private void handleIslandTap(View view) { mInspectExpanded = false; mLargeExpanded = false; } else { - mInspectExpanded = true; - mLargeExpanded = false; - mStatusTab = STATUS_TAB_CHAT; + launchAiHome(); + return; } updateIslandViews(); } } + private void launchAiHome() { + Intent intent = new Intent(mContext, OpenPhoneHomeActivity.class); + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK + | Intent.FLAG_ACTIVITY_SINGLE_TOP + | Intent.FLAG_ACTIVITY_CLEAR_TOP); + try { + mContext.startActivity(intent); + } catch (RuntimeException e) { + Log.w(TAG, "Unable to launch OpenPhone Home", e); + } + } + private boolean handleIslandLongPress(View view) { if (!isActiveMode()) { return false; @@ -861,6 +911,13 @@ private void removeAiSheet() { } private void removeIslandNow() { + mIslandVisible = false; + mHandler.removeCallbacks(mSystemUiStateRefresh); + synchronized (sControllers) { + if (sIslandOwner == this) { + sIslandOwner = null; + } + } mHandler.removeCallbacks(mWatchdogHide); if (mIslandResizeAnimator != null) { mIslandResizeAnimator.cancel(); @@ -973,6 +1030,10 @@ private static void styleTabButton(TextView button, boolean selected) { } private void updateIslandViews() { + publishSystemUiState(); + if (mIslandStateRepository.isSystemUiOwned()) { + return; + } if (mLeftIslandText == null || mRightIslandText == null || mIslandBodyText == null || mIslandChatColumn == null) { return; @@ -994,6 +1055,26 @@ private void updateIslandViews() { } } + private void startSystemUiStateRefresh() { + mHandler.removeCallbacks(mSystemUiStateRefresh); + publishSystemUiState(); + mHandler.postDelayed(mSystemUiStateRefresh, SYSTEM_UI_STATE_REFRESH_MS); + } + + private void publishSystemUiState() { + if (!isIslandOwner()) { + return; + } + mIslandStateRepository.publish( + mMode, mIslandVisible, mActiveTaskId, mWatchingCount); + } + + private boolean isIslandOwner() { + synchronized (sControllers) { + return sIslandOwner == this; + } + } + private void updateBodyContentViews(IslandPresentation presentation) { boolean showChat = presentation.showTabs && mStatusTab == STATUS_TAB_CHAT; boolean showRuntime = presentation.showTabs && mStatusTab == STATUS_TAB_RUNTIME; @@ -1336,18 +1417,28 @@ private String watchersStatusBody() { } private String runsStatusBody() { - List jobs = activeJobs(5); - int jobCount = activeJobCount(); + List runs = projectedRuns(50); + int runCount = runs.size(); StringBuilder body = new StringBuilder(); - body.append("Queued agent tasks that keep working after current chat."); - if (jobCount <= 0) { - body.append("\n\n◌\nNo queued runs"); + body.append("Durable agent work across jobs, commitments, and sessions."); + if (runCount <= 0) { + body.append("\n\n◌\nNo recent runs"); return body.toString(); } - body.append("\nRuns: ").append(jobCount).append(" active"); - for (AgentJobRecord job : jobs) { - body.append("\n").append(compactStatusLine(job.id, job.title, - job.type, job.status, job.nextRunAtMillis)); + int liveCount = 0; + for (AgentRunSummary run : runs) { + if (run.isLive()) { + liveCount++; + } + } + body.append("\nRuns: ").append(liveCount).append(" live"); + int visible = Math.min(5, runs.size()); + for (int i = 0; i < visible; i++) { + AgentRunSummary run = runs.get(i); + body.append("\n").append(compactRunStatusLine(run)); + } + if (runs.size() > visible) { + body.append("\n").append(runs.size() - visible).append(" more"); } return body.toString(); } @@ -1483,16 +1574,14 @@ private List activeWatchers(int limit) { } } - private List activeJobs(int limit) { - List out = new ArrayList<>(); + private List projectedRuns(int limit) { + List out = new ArrayList<>(); try { - for (AgentJobRecord job : new AgentJobStore(mContext).list("", 50)) { - if (!isLiveJob(job)) { + for (AgentRunSummary run : new AgentRunProjection(mContext).snapshot(limit)) { + if (AgentRunSummary.KIND_WATCHER.equals(run.kind)) { continue; } - if (out.size() < Math.max(1, limit)) { - out.add(job); - } + out.add(run); } } catch (RuntimeException ignored) { } @@ -1501,21 +1590,14 @@ private List activeJobs(int limit) { private int activeJobCount() { int count = 0; - try { - for (AgentJobRecord job : new AgentJobStore(mContext).list("", 50)) { - if (isLiveJob(job)) { - count++; - } + for (AgentRunSummary run : projectedRuns(50)) { + if (run.isLive()) { + count++; } - } catch (RuntimeException ignored) { } return count; } - private static boolean isLiveJob(AgentJobRecord job) { - return job != null && ("active".equals(job.status) || "running".equals(job.status)); - } - private String latestConversationBody() { String[] pair = latestConversationPair(); return formatLatestConversation(pair[0], pair[1]); @@ -1645,6 +1727,25 @@ private static String compactStatusLine(long id, String title, String type, Stri return line.toString(); } + private static String compactRunStatusLine(AgentRunSummary run) { + String cleanTitle = shortText(firstNonEmpty(run.title, run.kind, "Agent run"), 44); + String phase = shortText(firstNonEmpty(run.phase, run.status, ""), 18); + String timing = relativeTime(run.nextRunAtMillis); + StringBuilder line = new StringBuilder(); + line.append(run.needsAttention ? "! " : "- "); + line.append(cleanTitle); + if (!phase.isEmpty()) { + line.append(" · ").append(phase); + } + if (!run.pendingConfirmationId.isEmpty()) { + line.append(" · approval needed · Open AI Home"); + } + if (!timing.isEmpty()) { + line.append(" ").append(timing); + } + return line.toString(); + } + private static String relativeTime(long epochMillis) { if (epochMillis <= 0) { return ""; diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/agent/TrajectoryRecorder.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/agent/TrajectoryRecorder.java index c0dbf8a..5619b85 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/agent/TrajectoryRecorder.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/agent/TrajectoryRecorder.java @@ -11,6 +11,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.openphone.assistant.surface.AdaptiveSurface; import java.io.File; import java.io.FileInputStream; @@ -101,6 +102,23 @@ public synchronized void recordAgentResult(String resultJson) { } } + public synchronized void recordSurfacePresented(AdaptiveSurface surface) { + if (surface == null) { + return; + } + try { + append("surface_presented", new JSONObject() + .put("surface_id", surface.surfaceId) + .put("revision", surface.revision) + .put("session_id", surface.sessionId) + .put("runtime", surface.runtime) + .put("presentation", surface.presentation) + .put("sensitivity", surface.sensitivity)); + } catch (JSONException e) { + Log.w(TAG, "Failed to record surface presentation", e); + } + } + public String sessionPath() { return mSessionDir.getAbsolutePath(); } diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandState.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandState.java new file mode 100644 index 0000000..ce0d4a7 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandState.java @@ -0,0 +1,69 @@ +package org.openphone.assistant.island; + +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Bounded, privacy-minimized projection published for the SystemUI island. + * + *

system_server validates this shape, discards unknown fields, and assigns + * the authoritative revision and publication timestamps.

+ */ +public final class IslandState { + public static final String SCHEMA = "openphone.island_state.v1"; + + private final String mMode; + private final String mLabel; + private final String mTitle; + private final String mDetail; + private final boolean mVisible; + private final boolean mNeedsAttention; + private final String mSensitivity; + private final String mActiveTaskId; + private final String mPendingConfirmationId; + private final int mLiveRuns; + private final long mUpdatedAtMillis; + + public IslandState(String mode, String label, String title, String detail, + boolean visible, boolean needsAttention, String sensitivity, + String activeTaskId, String pendingConfirmationId, int liveRuns, + long updatedAtMillis) { + mMode = bounded(mode, 32); + mLabel = bounded(label, 48); + mTitle = bounded(title, 80); + mDetail = bounded(detail, 160); + mVisible = visible; + mNeedsAttention = needsAttention; + mSensitivity = bounded(sensitivity, 16); + mActiveTaskId = bounded(activeTaskId, 96); + mPendingConfirmationId = bounded(pendingConfirmationId, 128); + mLiveRuns = Math.max(0, Math.min(liveRuns, 100)); + mUpdatedAtMillis = Math.max(0L, updatedAtMillis); + } + + public String toJson() { + try { + return new JSONObject() + .put("schema", SCHEMA) + .put("mode", mMode) + .put("label", mLabel) + .put("title", mTitle) + .put("detail", mDetail) + .put("visible", mVisible) + .put("needs_attention", mNeedsAttention) + .put("sensitivity", mSensitivity) + .put("active_task_id", mActiveTaskId) + .put("pending_confirmation_id", mPendingConfirmationId) + .put("live_runs", mLiveRuns) + .put("updated_at", mUpdatedAtMillis) + .toString(); + } catch (JSONException e) { + throw new IllegalStateException(e); + } + } + + private static String bounded(String value, int maxLength) { + String clean = value == null ? "" : value.trim(); + return clean.length() <= maxLength ? clean : clean.substring(0, maxLength); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandStateRepository.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandStateRepository.java new file mode 100644 index 0000000..9c3a4d4 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandStateRepository.java @@ -0,0 +1,172 @@ +package org.openphone.assistant.island; + +import android.content.Context; +import android.openphone.OpenPhoneAgentManager; +import android.os.SystemProperties; +import android.util.Log; + +import org.openphone.assistant.runs.AgentRunProjection; +import org.openphone.assistant.runs.AgentRunSummary; + +import java.util.List; + +/** + * Projects assistant and durable-run state into the compact SystemUI contract. + * + *

The projection deliberately contains no transcript, reply, screenshot, + * notification body, tool parameters, or result payload. AI Home remains the + * review surface for those richer details.

+ */ +public final class IslandStateRepository { + private static final String TAG = "OpenPhoneIslandState"; + private static final String SYSTEM_UI_PROPERTY = "ro.openphone.systemui_island"; + private static final long MIN_REPEAT_PUBLISH_MS = 1500L; + + private final OpenPhoneAgentManager mAgentManager; + private final AgentRunProjection mRunProjection; + private String mLastProjectionKey = ""; + private long mLastPublishedAtMillis; + + public IslandStateRepository(Context context) { + Context app = context.getApplicationContext(); + mAgentManager = app.getSystemService(OpenPhoneAgentManager.class); + mRunProjection = new AgentRunProjection(app); + } + + public boolean isSystemUiOwned() { + return SystemProperties.getBoolean(SYSTEM_UI_PROPERTY, false); + } + + public void publish(String requestedMode, boolean visible, String activeTaskId, + int watchingCount) { + if (!isSystemUiOwned() || mAgentManager == null) { + return; + } + + String mode = cleanMode(requestedMode); + int liveRuns = 0; + boolean needsAttention = "needs_review".equals(mode) || "error".equals(mode); + String pendingConfirmationId = ""; + try { + List runs = mRunProjection.snapshot(50); + for (AgentRunSummary run : runs) { + if (run.isLive()) { + liveRuns++; + } + if (pendingConfirmationId.isEmpty() + && !run.pendingConfirmationId.isEmpty()) { + pendingConfirmationId = run.pendingConfirmationId; + } + needsAttention |= run.needsAttention; + } + } catch (RuntimeException e) { + Log.w(TAG, "Unable to project durable runs for island", e); + } + liveRuns = Math.max(liveRuns, Math.max(0, watchingCount)); + + if (("idle".equals(mode) || "watching".equals(mode)) + && !pendingConfirmationId.isEmpty()) { + mode = "needs_review"; + } else if ("idle".equals(mode) && liveRuns > 0) { + mode = "watching"; + } + + IslandCopy copy = copyForMode(mode, liveRuns); + String sensitivity = "idle".equals(mode) && liveRuns == 0 && !needsAttention + ? "public" : "personal"; + long now = System.currentTimeMillis(); + String projectionKey = mode + "|" + visible + "|" + needsAttention + "|" + + clean(activeTaskId) + "|" + pendingConfirmationId + "|" + liveRuns; + if (projectionKey.equals(mLastProjectionKey) + && now - mLastPublishedAtMillis < MIN_REPEAT_PUBLISH_MS) { + return; + } + IslandState state = new IslandState( + mode, + copy.label, + copy.title, + copy.detail, + visible, + needsAttention, + sensitivity, + activeTaskId, + pendingConfirmationId, + liveRuns, + now); + try { + mAgentManager.publishIslandState(state.toJson()); + mLastProjectionKey = projectionKey; + mLastPublishedAtMillis = now; + } catch (RuntimeException e) { + Log.w(TAG, "Unable to publish SystemUI island state", e); + } + } + + private static IslandCopy copyForMode(String mode, int liveRuns) { + switch (mode) { + case "listening": + return new IslandCopy("Listening", "Listening", "Voice capture active"); + case "transcript": + return new IslandCopy("OpenPhone", "Request captured", + "Voice request captured"); + case "thinking": + return new IslandCopy("OpenPhone", "Thinking", "Working on your request"); + case "realtime": + return new IslandCopy("OpenPhone", "Live", "Realtime session active"); + case "action_running": + return new IslandCopy("OpenPhone", "Working", "Task running"); + case "answer_ready": + return new IslandCopy("OpenPhone", "Done", "Task complete"); + case "reply": + return new IslandCopy("OpenPhone", "Ready", "Response ready"); + case "needs_review": + return new IslandCopy("Review", "Approval needed", + "Open AI Home to review"); + case "error": + return new IslandCopy("OpenPhone", "Needs attention", + "Open AI Home for details"); + case "watching": + return new IslandCopy("OpenPhone", "Background activity", + liveRuns > 0 ? liveRuns + " active" : "Watching"); + case "idle": + default: + return new IslandCopy("OpenPhone", "Ready", ""); + } + } + + private static String cleanMode(String mode) { + String clean = clean(mode); + switch (clean) { + case "idle": + case "listening": + case "transcript": + case "thinking": + case "realtime": + case "action_running": + case "answer_ready": + case "reply": + case "needs_review": + case "error": + case "watching": + return clean; + default: + return "idle"; + } + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } + + private static final class IslandCopy { + final String label; + final String title; + final String detail; + + IslandCopy(String label, String title, String detail) { + this.label = label; + this.title = title; + this.detail = detail; + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobRecord.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobRecord.java index 574ac99..6a2ed57 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobRecord.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobRecord.java @@ -18,12 +18,28 @@ public final class AgentJobRecord { public final String lastResult; public final int failureCount; public final long failureAlertAtMillis; + public final String phase; + public final String progressText; + public final int progressCurrent; + public final int progressTotal; + public final String checkpointJson; + public final String pendingConfirmationId; + public final String pendingToolRequestJson; + public final String lastSurfaceId; + public final String resumeToken; + public final long lastEventAtMillis; + public final boolean unreadResult; + public final long pausedAtMillis; AgentJobRecord(long id, String type, String title, String prompt, String payloadJson, String scheduleJson, String sessionTarget, String deliveryJson, String status, long createdAtMillis, long updatedAtMillis, long nextRunAtMillis, long runningAtMillis, long lastRunAtMillis, String lastResult, int failureCount, - long failureAlertAtMillis) { + long failureAlertAtMillis, String phase, String progressText, + int progressCurrent, int progressTotal, String checkpointJson, + String pendingConfirmationId, String pendingToolRequestJson, + String lastSurfaceId, String resumeToken, long lastEventAtMillis, + boolean unreadResult, long pausedAtMillis) { this.id = id; this.type = type; this.title = title; @@ -41,5 +57,17 @@ public final class AgentJobRecord { this.lastResult = lastResult; this.failureCount = failureCount; this.failureAlertAtMillis = failureAlertAtMillis; + this.phase = phase; + this.progressText = progressText; + this.progressCurrent = progressCurrent; + this.progressTotal = progressTotal; + this.checkpointJson = checkpointJson; + this.pendingConfirmationId = pendingConfirmationId; + this.pendingToolRequestJson = pendingToolRequestJson; + this.lastSurfaceId = lastSurfaceId; + this.resumeToken = resumeToken; + this.lastEventAtMillis = lastEventAtMillis; + this.unreadResult = unreadResult; + this.pausedAtMillis = pausedAtMillis; } } diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobStore.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobStore.java index 9c9aaff..912730f 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobStore.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobStore.java @@ -18,6 +18,10 @@ public final class AgentJobStore { private static final String KEY_JOBS = "jobs"; private static final String KEY_NEXT_ID = "next_id"; private static final int MAX_JOBS = 200; + private static final int MAX_CHECKPOINT_CHARS = 12000; + private static final int MAX_PENDING_REQUEST_CHARS = 12000; + private static final Object STORE_LOCK = new Object(); + private static final Object REVIEW_LOCK = STORE_LOCK; private final SharedPreferences mPrefs; @@ -29,42 +33,57 @@ public AgentJobStore(Context context) { public synchronized long createJob(String type, String title, String prompt, String payloadJson, String scheduleJson, String sessionTarget, String deliveryJson, long nextRunAtMillis) { - String cleanTitle = safe(title).trim(); - if (cleanTitle.isEmpty()) { - return -1L; - } - long now = System.currentTimeMillis(); - long id = Math.max(1L, mPrefs.getLong(KEY_NEXT_ID, 1L)); - JSONArray jobs = readJobs(); - JSONObject job = new JSONObject(); - try { - job.put("id", id) - .put("type", normalizeType(type)) - .put("title", cleanTitle) - .put("prompt", safe(prompt)) - .put("payload_json", objectOrEmpty(payloadJson)) - .put("schedule_json", objectOrEmpty(scheduleJson)) - .put("session_target", safe(sessionTarget).isEmpty() - ? "main" : safe(sessionTarget)) - .put("delivery_json", objectOrEmpty(defaultDelivery(deliveryJson))) - .put("status", "active") - .put("created_at", now) - .put("updated_at", now) - .put("next_run_at", Math.max(0L, nextRunAtMillis)) - .put("running_at", 0L) - .put("last_run_at", 0L) - .put("last_result", "") - .put("failure_count", 0) - .put("failure_alert_at", 0L); - } catch (JSONException e) { - return -1L; - } - jobs.put(job); - trimOldTerminalJobs(jobs); - if (!writeJobs(jobs, id + 1L)) { - return -1L; + synchronized (STORE_LOCK) { + String cleanTitle = safe(title).trim(); + if (cleanTitle.isEmpty()) { + return -1L; + } + long now = System.currentTimeMillis(); + long id = Math.max(1L, mPrefs.getLong(KEY_NEXT_ID, 1L)); + JSONArray jobs = readJobs(); + JSONObject job = new JSONObject(); + try { + job.put("id", id) + .put("type", normalizeType(type)) + .put("title", cleanTitle) + .put("prompt", safe(prompt)) + .put("payload_json", objectOrEmpty(payloadJson)) + .put("schedule_json", objectOrEmpty(scheduleJson)) + .put("session_target", safe(sessionTarget).isEmpty() + ? "main" : safe(sessionTarget)) + .put("delivery_json", objectOrEmpty( + defaultDelivery(deliveryJson))) + .put("status", "queued") + .put("created_at", now) + .put("updated_at", now) + .put("next_run_at", Math.max(0L, nextRunAtMillis)) + .put("running_at", 0L) + .put("last_run_at", 0L) + .put("last_result", "") + .put("failure_count", 0) + .put("failure_alert_at", 0L) + .put("phase", "queued") + .put("progress_text", "Queued") + .put("progress_current", 0) + .put("progress_total", 0) + .put("checkpoint_json", new JSONObject()) + .put("pending_confirmation_id", "") + .put("pending_tool_request_json", new JSONObject()) + .put("last_surface_id", "") + .put("resume_token", "") + .put("last_event_at", now) + .put("unread_result", false) + .put("paused_at", 0L); + } catch (JSONException e) { + return -1L; + } + jobs.put(job); + trimOldTerminalJobs(jobs); + if (!writeJobs(jobs, id + 1L)) { + return -1L; + } + return id; } - return id; } public synchronized List due(long nowMillis, int limit) { @@ -72,7 +91,7 @@ public synchronized List due(long nowMillis, int limit) { JSONArray jobs = readJobs(); for (int i = 0; i < jobs.length() && out.size() < Math.max(1, limit); i++) { JSONObject job = jobs.optJSONObject(i); - if (job == null || !"active".equals(job.optString("status", ""))) { + if (job == null || !isQueued(job.optString("status", ""))) { continue; } long nextRunAt = job.optLong("next_run_at", 0L); @@ -123,7 +142,7 @@ public synchronized long nextRunAt(long nowMillis) { JSONArray jobs = readJobs(); for (int i = 0; i < jobs.length(); i++) { JSONObject job = jobs.optJSONObject(i); - if (job == null || !"active".equals(job.optString("status", ""))) { + if (job == null || !isQueued(job.optString("status", ""))) { continue; } long due = job.optLong("next_run_at", 0L); @@ -139,27 +158,41 @@ public synchronized long nextRunAt(long nowMillis) { public synchronized boolean markRunning(long id, long nowMillis) { return updateJob(id, job -> { - if (!"active".equals(job.optString("status", ""))) { + if (!isQueued(job.optString("status", ""))) { return false; } job.put("status", "running") + .put("phase", "running") + .put("progress_text", "Working") .put("running_at", nowMillis) - .put("updated_at", nowMillis); + .put("updated_at", nowMillis) + .put("last_event_at", nowMillis); return true; }); } public synchronized boolean markCompleted(long id, String result, long nowMillis) { return updateJob(id, job -> { + if (!"running".equals(job.optString("status", ""))) { + return false; + } JSONObject schedule = job.optJSONObject("schedule_json"); long interval = schedule == null ? 0L : schedule.optLong("interval_ms", 0L); - job.put("status", interval > 0 ? "active" : "completed") + job.put("status", interval > 0 ? "queued" : "completed") + .put("phase", interval > 0 ? "queued" : "completed") + .put("progress_text", interval > 0 ? "Waiting for next run" : "Completed") .put("updated_at", nowMillis) .put("last_run_at", nowMillis) .put("running_at", 0L) .put("last_result", truncate(result)) .put("failure_count", 0) - .put("failure_alert_at", 0L); + .put("failure_alert_at", 0L) + .put("checkpoint_json", new JSONObject()) + .put("pending_confirmation_id", "") + .put("pending_tool_request_json", new JSONObject()) + .put("resume_token", "") + .put("last_event_at", nowMillis) + .put("unread_result", interval <= 0); if (interval > 0) { job.put("next_run_at", nowMillis + Math.max(interval, 15_000L)); } else { @@ -172,11 +205,15 @@ public synchronized boolean markCompleted(long id, String result, long nowMillis public synchronized boolean markDispatched(long id, String result, long nowMillis) { return updateJob(id, job -> { job.put("status", "dispatched") + .put("phase", "waiting_for_runtime") + .put("progress_text", "Sent to runtime") .put("updated_at", nowMillis) .put("last_run_at", nowMillis) .put("running_at", 0L) .put("last_result", truncate(result)) - .put("next_run_at", 0L); + .put("next_run_at", 0L) + .put("last_event_at", nowMillis) + .put("unread_result", true); return true; }); } @@ -184,14 +221,22 @@ public synchronized boolean markDispatched(long id, String result, long nowMilli public synchronized boolean markFailed(long id, String reason, long nextRunAtMillis, int failureCount, long failureAlertAtMillis, long nowMillis) { return updateJob(id, job -> { - job.put("status", "active") + if ("stopped".equals(job.optString("status", ""))) { + return false; + } + boolean terminal = failureCount >= 6; + job.put("status", terminal ? "failed" : "queued") + .put("phase", terminal ? "failed" : "waiting") + .put("progress_text", truncate(reason)) .put("updated_at", nowMillis) .put("last_run_at", nowMillis) .put("running_at", 0L) .put("last_result", truncate(reason)) .put("next_run_at", nextRunAtMillis) .put("failure_count", Math.max(1, failureCount)) - .put("failure_alert_at", failureAlertAtMillis); + .put("failure_alert_at", failureAlertAtMillis) + .put("last_event_at", nowMillis) + .put("unread_result", terminal); return true; }); } @@ -200,32 +245,293 @@ public synchronized boolean stop(long id) { long now = System.currentTimeMillis(); return updateJob(id, job -> { job.put("status", "stopped") + .put("phase", "stopped") + .put("progress_text", "Stopped") .put("updated_at", now) .put("running_at", 0L) - .put("next_run_at", 0L); + .put("next_run_at", 0L) + .put("checkpoint_json", new JSONObject()) + .put("pending_confirmation_id", "") + .put("pending_tool_request_json", new JSONObject()) + .put("resume_token", "") + .put("last_event_at", now) + .put("unread_result", true); + return true; + }); + } + + public synchronized AgentJobRecord find(long id) { + JSONArray jobs = readJobs(); + for (int i = 0; i < jobs.length(); i++) { + JSONObject job = jobs.optJSONObject(i); + if (job != null && job.optLong("id", -1L) == id) { + return fromJson(job); + } + } + return null; + } + + public synchronized AgentJobRecord findByConfirmationId(String confirmationId) { + String cleanId = safe(confirmationId).trim(); + if (cleanId.isEmpty()) { + return null; + } + JSONArray jobs = readJobs(); + for (int i = 0; i < jobs.length(); i++) { + JSONObject job = jobs.optJSONObject(i); + if (job != null && cleanId.equals( + job.optString("pending_confirmation_id", ""))) { + return fromJson(job); + } + } + return null; + } + + public boolean markAwaitingReview(long id, JSONObject checkpoint, + JSONObject pendingRequest, String confirmationId, String resumeToken, + String progressText, long nowMillis) { + String checkpointRaw = boundedJson(checkpoint, MAX_CHECKPOINT_CHARS); + String pendingRaw = boundedJson(pendingRequest, MAX_PENDING_REQUEST_CHARS); + if (checkpointRaw == null || pendingRaw == null + || safe(confirmationId).trim().isEmpty() + || safe(resumeToken).trim().isEmpty()) { + return false; + } + synchronized (REVIEW_LOCK) { + return updateJob(id, job -> { + if (!"running".equals(job.optString("status", ""))) { + return false; + } + job.put("status", "awaiting_review") + .put("phase", "awaiting_review") + .put("progress_text", truncate(progressText)) + .put("checkpoint_json", new JSONObject(checkpointRaw)) + .put("pending_confirmation_id", confirmationId) + .put("pending_tool_request_json", new JSONObject(pendingRaw)) + .put("resume_token", resumeToken) + .put("running_at", 0L) + .put("next_run_at", 0L) + .put("updated_at", nowMillis) + .put("last_event_at", nowMillis) + .put("unread_result", false); + return true; + }); + } + } + + /** + * Atomically claims a still-pending review. Only the first tap can move it + * to resolving; later taps observe no claim and cannot execute the tool. + */ + public ReviewClaim claimReview(String confirmationId, boolean approved, long nowMillis) { + String cleanId = safe(confirmationId).trim(); + if (cleanId.isEmpty()) { + return null; + } + synchronized (REVIEW_LOCK) { + JSONArray jobs = readJobs(); + for (int i = 0; i < jobs.length(); i++) { + JSONObject job = jobs.optJSONObject(i); + if (job == null || !"awaiting_review".equals( + job.optString("status", ""))) { + continue; + } + if (!cleanId.equals(job.optString("pending_confirmation_id", ""))) { + continue; + } + JSONObject pending = job.optJSONObject("pending_tool_request_json"); + if (pending == null || !"pending".equals( + pending.optString("review_state", ""))) { + return null; + } + long expiresAt = pending.optLong("expires_at", 0L); + if (expiresAt <= nowMillis) { + try { + applyReviewResolution(job, pending, "timeout", + "{\"status\":\"background.confirmation_timeout\"}", + nowMillis); + } catch (JSONException e) { + Log.w(TAG, "failed to expire claimed review", e); + return null; + } + writeJobs(jobs, mPrefs.getLong(KEY_NEXT_ID, 1L)); + return null; + } + try { + pending.put("review_state", "resolving") + .put("decision", approved ? "approved" : "denied") + .put("reviewed_at", nowMillis); + job.put("status", "waiting") + .put("phase", approved ? "approval_executing" : "denial_resolving") + .put("progress_text", approved + ? "Executing approved action" : "Applying denial") + .put("pending_tool_request_json", pending) + .put("updated_at", nowMillis) + .put("last_event_at", nowMillis); + } catch (JSONException e) { + return null; + } + if (!writeJobs(jobs, mPrefs.getLong(KEY_NEXT_ID, 1L))) { + return null; + } + return new ReviewClaim(fromJson(job), objectOrEmpty(pending.toString()), approved); + } + } + return null; + } + + public boolean completeReview(long jobId, String confirmationId, String resolution, + String toolResult, long nowMillis) { + synchronized (REVIEW_LOCK) { + return updateJob(jobId, job -> { + if (!safe(confirmationId).equals( + job.optString("pending_confirmation_id", ""))) { + return false; + } + JSONObject pending = job.optJSONObject("pending_tool_request_json"); + if (pending == null || !"resolving".equals( + pending.optString("review_state", ""))) { + return false; + } + applyReviewResolution(job, pending, resolution, toolResult, nowMillis); + return true; + }); + } + } + + public int expirePendingReviews(long nowMillis) { + final int[] expired = {0}; + final boolean persisted; + synchronized (REVIEW_LOCK) { + persisted = updateAllJobs(job -> { + if (!"awaiting_review".equals(job.optString("status", ""))) { + return false; + } + JSONObject pending = job.optJSONObject("pending_tool_request_json"); + if (pending == null || pending.optLong("expires_at", 0L) > nowMillis) { + return false; + } + try { + applyReviewResolution(job, pending, "timeout", + "{\"status\":\"background.confirmation_timeout\"," + + "\"reason\":\"approval_expired\"}", nowMillis); + } catch (JSONException e) { + Log.w(TAG, "failed to expire background review", e); + return false; + } + expired[0]++; + return true; + }); + } + return persisted ? expired[0] : 0; + } + + public synchronized List reviewJobIdsExpiringBy(long nowMillis) { + List ids = new ArrayList<>(); + JSONArray jobs = readJobs(); + for (int i = 0; i < jobs.length(); i++) { + JSONObject job = jobs.optJSONObject(i); + if (job == null + || !"awaiting_review".equals(job.optString("status", ""))) { + continue; + } + JSONObject pending = job.optJSONObject("pending_tool_request_json"); + if (pending != null && pending.optLong("expires_at", 0L) <= nowMillis) { + ids.add(job.optLong("id", -1L)); + } + } + return ids; + } + + public synchronized boolean pause(long id) { + long now = System.currentTimeMillis(); + return updateJob(id, job -> { + String status = job.optString("status", ""); + if (!isQueued(status)) { + return false; + } + job.put("status", "paused") + .put("phase", "paused") + .put("progress_text", "Paused") + .put("paused_at", now) + .put("next_run_at", 0L) + .put("updated_at", now) + .put("last_event_at", now); + return true; + }); + } + + public synchronized boolean resume(long id) { + long now = System.currentTimeMillis(); + return updateJob(id, job -> { + if (!"paused".equals(job.optString("status", ""))) { + return false; + } + job.put("status", "queued") + .put("phase", "queued") + .put("progress_text", "Queued") + .put("paused_at", 0L) + .put("next_run_at", now) + .put("updated_at", now) + .put("last_event_at", now); return true; }); } public synchronized int repairStuck(long staleBeforeMillis, long nowMillis) { final int[] repaired = {0}; - updateAllJobs(job -> { + boolean persisted = updateAllJobs(job -> { if ("running".equals(job.optString("status", "")) && job.optLong("running_at", 0L) > 0 && job.optLong("running_at", 0L) < staleBeforeMillis) { int failures = job.optInt("failure_count", 0) + 1; - job.put("status", "active") + job.put("status", "queued") + .put("phase", "waiting") + .put("progress_text", "Recovered after interruption") .put("running_at", 0L) .put("updated_at", nowMillis) .put("last_result", "stuck_running_repaired") .put("failure_count", failures) - .put("next_run_at", nowMillis + backoffMillis(failures)); + .put("next_run_at", nowMillis + backoffMillis(failures)) + .put("last_event_at", nowMillis); + repaired[0]++; + return true; + } + JSONObject pending = job.optJSONObject("pending_tool_request_json"); + if ("waiting".equals(job.optString("status", "")) + && pending != null + && "resolving".equals(pending.optString("review_state", "")) + && pending.optLong("reviewed_at", 0L) > 0 + && pending.optLong("reviewed_at", 0L) < staleBeforeMillis) { + if ("denied".equals(pending.optString("decision", ""))) { + applyReviewResolution(job, pending, "denied_after_restart", + "{\"status\":\"background.action_denied\"," + + "\"reason\":\"user_denied\"}", nowMillis); + repaired[0]++; + return true; + } + pending.put("review_state", "resolved") + .put("resolution", "interrupted_unknown") + .put("resolved_at", nowMillis); + job.put("status", "failed") + .put("phase", "review_resolution_unknown") + .put("progress_text", + "Approved action was interrupted; it was not replayed") + .put("running_at", 0L) + .put("updated_at", nowMillis) + .put("last_result", + "background.review_interrupted_result_unknown") + .put("pending_confirmation_id", "") + .put("pending_tool_request_json", pending) + .put("next_run_at", 0L) + .put("last_event_at", nowMillis) + .put("unread_result", true); repaired[0]++; return true; } return false; }); - return repaired[0]; + return persisted ? repaired[0] : 0; } public static long backoffMillis(int failureCount) { @@ -252,7 +558,20 @@ static JSONObject toJson(AgentJobRecord job) { .put("last_run_at", job.lastRunAtMillis) .put("last_result", job.lastResult) .put("failure_count", job.failureCount) - .put("failure_alert_at", job.failureAlertAtMillis); + .put("failure_alert_at", job.failureAlertAtMillis) + .put("phase", job.phase) + .put("progress_text", job.progressText) + .put("progress_current", job.progressCurrent) + .put("progress_total", job.progressTotal) + .put("checkpoint_json", objectOrEmpty(job.checkpointJson)) + .put("pending_confirmation_id", job.pendingConfirmationId) + .put("pending_tool_request_json", + objectOrEmpty(job.pendingToolRequestJson)) + .put("last_surface_id", job.lastSurfaceId) + .put("resume_token", job.resumeToken) + .put("last_event_at", job.lastEventAtMillis) + .put("unread_result", job.unreadResult) + .put("paused_at", job.pausedAtMillis); } catch (JSONException ignored) { } return out; @@ -264,48 +583,65 @@ private interface JobUpdater { private boolean updateJob(long id, JobUpdater updater) { final boolean[] changed = {false}; - updateAllJobs(job -> { + boolean persisted = updateAllJobs(job -> { if (job.optLong("id", -1L) != id) { return false; } changed[0] = updater.update(job); return changed[0]; }); - return changed[0]; + return changed[0] && persisted; } private boolean updateAllJobs(JobUpdater updater) { - JSONArray jobs = readJobs(); - boolean changed = false; - for (int i = 0; i < jobs.length(); i++) { - JSONObject job = jobs.optJSONObject(i); - if (job == null) { - continue; - } - try { - changed |= updater.update(job); - } catch (JSONException e) { - Log.w(TAG, "job update failed", e); + synchronized (STORE_LOCK) { + JSONArray jobs = readJobs(); + boolean changed = false; + for (int i = 0; i < jobs.length(); i++) { + JSONObject job = jobs.optJSONObject(i); + if (job == null) { + continue; + } + try { + changed |= updater.update(job); + } catch (JSONException e) { + Log.w(TAG, "job update failed", e); + } } + return !changed || writeJobs(jobs, mPrefs.getLong(KEY_NEXT_ID, 1L)); } - return !changed || writeJobs(jobs, mPrefs.getLong(KEY_NEXT_ID, 1L)); } private JSONArray readJobs() { - String raw = mPrefs.getString(KEY_JOBS, "[]"); - try { - return new JSONArray(raw == null || raw.isEmpty() ? "[]" : raw); - } catch (JSONException e) { - Log.w(TAG, "agent job store corrupt; ignoring", e); - return new JSONArray(); + synchronized (STORE_LOCK) { + String raw = mPrefs.getString(KEY_JOBS, "[]"); + try { + JSONArray jobs = new JSONArray(raw == null || raw.isEmpty() ? "[]" : raw); + boolean migrated = false; + for (int i = 0; i < jobs.length(); i++) { + JSONObject job = jobs.optJSONObject(i); + if (job != null) { + migrated |= ensureLifecycleDefaults(job); + } + } + if (migrated) { + mPrefs.edit().putString(KEY_JOBS, jobs.toString()).commit(); + } + return jobs; + } catch (JSONException e) { + Log.w(TAG, "agent job store corrupt; ignoring", e); + return new JSONArray(); + } } } private boolean writeJobs(JSONArray jobs, long nextId) { - return mPrefs.edit() - .putString(KEY_JOBS, jobs.toString()) - .putLong(KEY_NEXT_ID, Math.max(1L, nextId)) - .commit(); + synchronized (STORE_LOCK) { + return mPrefs.edit() + .putString(KEY_JOBS, jobs.toString()) + .putLong(KEY_NEXT_ID, Math.max(1L, nextId)) + .commit(); + } } private static AgentJobRecord fromJson(JSONObject job) { @@ -326,7 +662,19 @@ private static AgentJobRecord fromJson(JSONObject job) { job.optLong("last_run_at", 0L), job.optString("last_result", ""), job.optInt("failure_count", 0), - job.optLong("failure_alert_at", 0L)); + job.optLong("failure_alert_at", 0L), + job.optString("phase", job.optString("status", "")), + job.optString("progress_text", ""), + job.optInt("progress_current", 0), + job.optInt("progress_total", 0), + stringify(job.optJSONObject("checkpoint_json")), + job.optString("pending_confirmation_id", ""), + stringify(job.optJSONObject("pending_tool_request_json")), + job.optString("last_surface_id", ""), + job.optString("resume_token", ""), + job.optLong("last_event_at", job.optLong("updated_at", 0L)), + job.optBoolean("unread_result", false), + job.optLong("paused_at", 0L)); } private static String normalizeType(String type) { @@ -359,6 +707,84 @@ private static String truncate(String value) { return clean.length() <= 4000 ? clean : clean.substring(0, 4000); } + private static boolean isQueued(String status) { + return "active".equals(status) || "queued".equals(status); + } + + private static String boundedJson(JSONObject object, int maxChars) { + String raw = object == null ? "{}" : object.toString(); + return raw.length() <= maxChars ? raw : null; + } + + private static void applyReviewResolution(JSONObject job, JSONObject pending, + String resolution, String toolResult, long nowMillis) throws JSONException { + JSONObject checkpoint = job.optJSONObject("checkpoint_json"); + if (checkpoint == null) { + checkpoint = new JSONObject(); + } + String boundedResult = truncate(toolResult); + checkpoint.put("resume_pending", true) + .put("review_resolution", safe(resolution)) + .put("tool", pending.optString("tool", "")) + .put("params_digest", pending.optString("params_digest", "")) + .put("idempotency_key", pending.optString("idempotency_key", "")) + .put("tool_result", boundedResult) + .put("resolved_at", nowMillis); + pending.put("review_state", "resolved") + .put("resolution", safe(resolution)) + .put("resolved_at", nowMillis); + job.put("status", "queued") + .put("phase", "resuming") + .put("progress_text", "Resuming after review") + .put("checkpoint_json", checkpoint) + .put("pending_confirmation_id", "") + .put("pending_tool_request_json", pending) + .put("running_at", 0L) + .put("next_run_at", nowMillis) + .put("updated_at", nowMillis) + .put("last_event_at", nowMillis); + } + + private static boolean ensureLifecycleDefaults(JSONObject job) throws JSONException { + boolean changed = false; + long updatedAt = job.optLong("updated_at", System.currentTimeMillis()); + changed |= putDefault(job, "phase", job.optString("status", "queued")); + changed |= putDefault(job, "progress_text", ""); + changed |= putDefault(job, "progress_current", 0); + changed |= putDefault(job, "progress_total", 0); + changed |= putDefault(job, "checkpoint_json", new JSONObject()); + changed |= putDefault(job, "pending_confirmation_id", ""); + changed |= putDefault(job, "pending_tool_request_json", new JSONObject()); + changed |= putDefault(job, "last_surface_id", ""); + changed |= putDefault(job, "resume_token", ""); + changed |= putDefault(job, "last_event_at", updatedAt); + changed |= putDefault(job, "unread_result", false); + changed |= putDefault(job, "paused_at", 0L); + return changed; + } + + private static boolean putDefault(JSONObject object, String key, Object value) + throws JSONException { + if (object.has(key)) { + return false; + } + object.put(key, value); + return true; + } + + public static final class ReviewClaim { + public final AgentJobRecord job; + public final JSONObject pendingRequest; + public final boolean approved; + + ReviewClaim(AgentJobRecord job, JSONObject pendingRequest, boolean approved) { + this.job = job; + this.pendingRequest = objectOrEmpty( + pendingRequest == null ? "{}" : pendingRequest.toString()); + this.approved = approved; + } + } + private static String safe(String value) { return value == null ? "" : value; } diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewContract.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewContract.java new file mode 100644 index 0000000..391add7 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewContract.java @@ -0,0 +1,334 @@ +package org.openphone.assistant.jobs; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +/** Exact, bounded approval binding for a background mutating tool request. */ +public final class BackgroundJobReviewContract { + public static final long REVIEW_TTL_MILLIS = 15L * 60L * 1000L; + private static final int MAX_REQUEST_CHARS = 12000; + + private BackgroundJobReviewContract() { + } + + public static PreparedRequest prepare(AgentJobRecord job, String tool, + JSONObject arguments, long nowMillis) { + if (job == null || job.id <= 0 || clean(tool).isEmpty() || arguments == null) { + return null; + } + if (containsBlockedMaterial(arguments)) { + return null; + } + JSONObject params = copy(arguments); + if (params.toString().length() > MAX_REQUEST_CHARS) { + return null; + } + String confirmationId = "job-confirm-" + UUID.randomUUID(); + String resumeToken = "job-resume-" + UUID.randomUUID(); + String idempotencyKey = "job:" + job.id + ":" + UUID.randomUUID(); + String sessionId = "background-job:" + job.id; + long expiresAt = nowMillis + REVIEW_TTL_MILLIS; + String paramsDigest = sha256(clean(tool) + "\n" + canonical(params)); + String bindingDigest = bindingDigest(job.id, "builtin", sessionId, tool, + paramsDigest, idempotencyKey, expiresAt); + JSONObject pending = new JSONObject(); + JSONObject checkpoint = new JSONObject(); + try { + pending.put("schema", "openphone.background_confirmation.v1") + .put("confirmation_id", confirmationId) + .put("job_id", job.id) + .put("runtime", "builtin") + .put("phone_session_id", sessionId) + .put("tool", clean(tool)) + .put("params", params) + .put("params_digest", paramsDigest) + .put("binding_digest", bindingDigest) + .put("idempotency_key", idempotencyKey) + .put("created_at", nowMillis) + .put("expires_at", expiresAt) + .put("review_state", "pending") + .put("summary", summary(tool, params)); + checkpoint.put("schema", "openphone.background_checkpoint.v1") + .put("job_id", job.id) + .put("phase", "awaiting_review") + .put("tool", clean(tool)) + .put("params_digest", paramsDigest) + .put("binding_digest", bindingDigest) + .put("idempotency_key", idempotencyKey) + .put("resume_token", resumeToken) + .put("created_at", nowMillis) + .put("resume_pending", false); + } catch (JSONException e) { + return null; + } + if (pending.toString().length() > MAX_REQUEST_CHARS + || checkpoint.toString().length() > MAX_REQUEST_CHARS) { + return null; + } + return new PreparedRequest( + confirmationId, resumeToken, pending, checkpoint, paramsDigest, bindingDigest); + } + + public static boolean verify(AgentJobStore.ReviewClaim claim, long nowMillis) { + if (claim == null || claim.job == null || claim.pendingRequest == null) { + return false; + } + JSONObject request = claim.pendingRequest; + JSONObject params = request.optJSONObject("params"); + JSONObject checkpoint = object(claim.job.checkpointJson); + long createdAt = request.optLong("created_at", 0L); + long expiresAt = request.optLong("expires_at", 0L); + if (!"openphone.background_confirmation.v1".equals( + request.optString("schema", "")) + || request.optLong("job_id", -1L) != claim.job.id + || !"builtin".equals(request.optString("runtime", "")) + || !("background-job:" + claim.job.id).equals( + request.optString("phone_session_id", "")) + || clean(request.optString("tool", "")).isEmpty() + || params == null + || containsBlockedMaterial(params) + || request.toString().length() > MAX_REQUEST_CHARS + || createdAt <= 0L + || expiresAt <= nowMillis + || expiresAt <= createdAt + || expiresAt - createdAt > REVIEW_TTL_MILLIS + || !"resolving".equals(request.optString("review_state", "")) + || !(claim.approved ? "approved" : "denied").equals( + request.optString("decision", "")) + || clean(request.optString("idempotency_key", "")).isEmpty() + || !"openphone.background_checkpoint.v1".equals( + checkpoint.optString("schema", "")) + || checkpoint.optLong("job_id", -1L) != claim.job.id + || !"awaiting_review".equals(checkpoint.optString("phase", "")) + || !clean(request.optString("tool", "")).equals( + checkpoint.optString("tool", "")) + || !clean(request.optString("idempotency_key", "")).equals( + checkpoint.optString("idempotency_key", "")) + || clean(claim.job.resumeToken).isEmpty() + || !clean(claim.job.resumeToken).equals( + checkpoint.optString("resume_token", ""))) { + return false; + } + String paramsDigest = sha256(request.optString("tool", "") + + "\n" + canonical(params)); + if (!constantEquals(paramsDigest, request.optString("params_digest", "")) + || !constantEquals(paramsDigest, + checkpoint.optString("params_digest", ""))) { + return false; + } + String binding = bindingDigest( + claim.job.id, + request.optString("runtime", ""), + request.optString("phone_session_id", ""), + request.optString("tool", ""), + paramsDigest, + request.optString("idempotency_key", ""), + expiresAt); + return constantEquals(binding, request.optString("binding_digest", "")) + && constantEquals(binding, + checkpoint.optString("binding_digest", "")); + } + + public static boolean matchesResume(AgentJobRecord job, String tool, JSONObject params) { + if (job == null || params == null) { + return false; + } + JSONObject checkpoint = object(job.checkpointJson); + if (!"openphone.background_checkpoint.v1".equals( + checkpoint.optString("schema", "")) + || checkpoint.optLong("job_id", -1L) != job.id + || clean(job.resumeToken).isEmpty() + || !clean(job.resumeToken).equals( + checkpoint.optString("resume_token", "")) + || !checkpoint.optBoolean("resume_pending", false) + || !clean(tool).equals(checkpoint.optString("tool", ""))) { + return false; + } + String digest = sha256(clean(tool) + "\n" + canonical(params)); + return constantEquals(digest, checkpoint.optString("params_digest", "")); + } + + public static String resumeToolResult(AgentJobRecord job) { + return object(job == null ? "{}" : job.checkpointJson) + .optString("tool_result", + "{\"status\":\"background.review_result_missing\"}"); + } + + public static String resumePrompt(AgentJobRecord job) { + JSONObject checkpoint = object(job == null ? "{}" : job.checkpointJson); + if (!checkpoint.optBoolean("resume_pending", false)) { + return job == null ? "" : job.prompt; + } + String resolution = checkpoint.optString("review_resolution", "resolved"); + String tool = checkpoint.optString("tool", ""); + String result = checkpoint.optString("tool_result", "{}"); + return clean(job.prompt) + + "\n\nOpenPhone durable review checkpoint (trusted phone state):\n" + + "The exact " + tool + " request was " + resolution + ".\n" + + "Its bound tool result is:\n" + truncate(result, 4000) + "\n" + + "Continue from this result and finish gracefully. Do not execute or ask " + + "for the same action again unless the user request requires a distinct " + + "new action."; + } + + private static String bindingDigest(long jobId, String runtime, String sessionId, + String tool, String paramsDigest, String idempotencyKey, long expiresAt) { + return sha256(jobId + "\n" + clean(runtime) + "\n" + clean(sessionId) + "\n" + + clean(tool) + "\n" + clean(paramsDigest) + "\n" + + clean(idempotencyKey) + "\n" + expiresAt); + } + + private static String summary(String tool, JSONObject params) { + String visible = canonical(params); + return truncate(clean(tool).replace('_', ' ') + " with " + visible, 1000); + } + + private static boolean containsBlockedMaterial(Object value) { + if (value instanceof JSONObject) { + JSONObject object = (JSONObject) value; + Iterator keys = object.keys(); + while (keys.hasNext()) { + String key = keys.next(); + String lower = key.toLowerCase(Locale.US); + if (lower.contains("api_key") || lower.contains("authorization") + || "auth".equals(lower) || lower.contains("credential") + || lower.contains("private_key") + || lower.contains("token") || lower.contains("secret") + || lower.contains("password") || lower.contains("cookie") + || lower.contains("screenshot") + || ("data".equals(lower) + && "base64".equalsIgnoreCase(object.optString("encoding", "")))) { + return true; + } + if (containsBlockedMaterial(object.opt(key))) { + return true; + } + } + } else if (value instanceof JSONArray) { + JSONArray array = (JSONArray) value; + for (int i = 0; i < array.length(); i++) { + if (containsBlockedMaterial(array.opt(i))) { + return true; + } + } + } else if (value instanceof String) { + String text = ((String) value).trim(); + String lower = text.toLowerCase(Locale.US); + if (lower.startsWith("bearer ") || lower.startsWith("basic ") + || lower.contains("-----begin private key-----") + || lower.startsWith("data:image/")) { + return true; + } + if (text.length() > 2048 && text.matches("^[A-Za-z0-9+/=_-]+$")) { + return true; + } + } + return false; + } + + private static String canonical(Object value) { + if (value == null || JSONObject.NULL.equals(value)) { + return "null"; + } + if (value instanceof JSONObject) { + JSONObject object = (JSONObject) value; + List keys = new ArrayList<>(); + Iterator iterator = object.keys(); + while (iterator.hasNext()) { + keys.add(iterator.next()); + } + Collections.sort(keys); + StringBuilder out = new StringBuilder("{"); + for (String key : keys) { + if (out.length() > 1) out.append(','); + out.append(JSONObject.quote(key)).append(':') + .append(canonical(object.opt(key))); + } + return out.append('}').toString(); + } + if (value instanceof JSONArray) { + JSONArray array = (JSONArray) value; + StringBuilder out = new StringBuilder("["); + for (int i = 0; i < array.length(); i++) { + if (i > 0) out.append(','); + out.append(canonical(array.opt(i))); + } + return out.append(']').toString(); + } + if (value instanceof String) { + return JSONObject.quote((String) value); + } + return String.valueOf(value); + } + + private static String sha256(String value) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(clean(value).getBytes(StandardCharsets.UTF_8)); + StringBuilder out = new StringBuilder(digest.length * 2); + for (byte item : digest) { + out.append(String.format(Locale.US, "%02x", item & 0xff)); + } + return out.toString(); + } catch (NoSuchAlgorithmException e) { + return ""; + } + } + + private static boolean constantEquals(String left, String right) { + byte[] a = clean(left).getBytes(StandardCharsets.UTF_8); + byte[] b = clean(right).getBytes(StandardCharsets.UTF_8); + return MessageDigest.isEqual(a, b); + } + + private static JSONObject copy(JSONObject source) { + return object(source == null ? "{}" : source.toString()); + } + + private static JSONObject object(String raw) { + try { + return new JSONObject(raw == null || raw.trim().isEmpty() ? "{}" : raw); + } catch (JSONException e) { + return new JSONObject(); + } + } + + private static String truncate(String value, int max) { + String clean = clean(value); + return clean.length() <= max ? clean : clean.substring(0, max - 1) + "…"; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } + + public static final class PreparedRequest { + public final String confirmationId; + public final String resumeToken; + public final JSONObject pendingRequest; + public final JSONObject checkpoint; + public final String paramsDigest; + public final String bindingDigest; + + PreparedRequest(String confirmationId, String resumeToken, JSONObject pendingRequest, + JSONObject checkpoint, String paramsDigest, String bindingDigest) { + this.confirmationId = confirmationId; + this.resumeToken = resumeToken; + this.pendingRequest = copy(pendingRequest); + this.checkpoint = copy(checkpoint); + this.paramsDigest = paramsDigest; + this.bindingDigest = bindingDigest; + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewManager.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewManager.java new file mode 100644 index 0000000..30b15b2 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewManager.java @@ -0,0 +1,295 @@ +package org.openphone.assistant.jobs; + +import android.app.KeyguardManager; +import android.content.Context; +import android.openphone.OpenPhoneAgentManager; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.OpenPhoneNotificationController; +import org.openphone.assistant.actions.ToolCatalog; +import org.openphone.assistant.agent.FrameworkToolExecutor; +import org.openphone.assistant.context.ContextIndexStore; + +import java.util.Iterator; +import java.util.List; + +/** + * Resolves Android-owned background approvals from the exact persisted request. + * + * The store's atomic claim is the execution gate: a second notification or UI + * tap cannot receive a claim and therefore cannot execute the action twice. + */ +public final class BackgroundJobReviewManager { + private static final String TAG = "OpenPhoneJobReview"; + + private BackgroundJobReviewManager() { + } + + public static String resolve(Context context, String confirmationId, boolean approved) { + if (context == null) { + return result("background.review_error", "context_unavailable"); + } + Context app = context.getApplicationContext(); + KeyguardManager keyguard = app.getSystemService(KeyguardManager.class); + if (keyguard != null && keyguard.isDeviceLocked()) { + return result("background.review_unlock_required", + "unlock_device_to_review"); + } + AgentJobStore store = new AgentJobStore(app); + long now = System.currentTimeMillis(); + AgentJobRecord pendingJob = store.findByConfirmationId(confirmationId); + boolean selectedExpired = pendingJob != null + && parseObject(pendingJob.pendingToolRequestJson) + .optLong("expires_at", 0L) <= now; + List expiringReviewIds = store.reviewJobIdsExpiringBy(now); + int expired = store.expirePendingReviews(now); + if (expired > 0) { + for (Long jobId : expiringReviewIds) { + OpenPhoneNotificationController.cancelAgentJobReview( + app, jobId == null ? -1L : jobId); + } + OpenPhoneAgentJobScheduler.checkNow(app); + } + if (expired > 0 && selectedExpired) { + return result("background.review_expired", "approval_expired"); + } + AgentJobStore.ReviewClaim claim = store.claimReview(confirmationId, approved, now); + if (claim == null) { + return result("background.review_already_resolved", "claim_unavailable"); + } + + JSONObject request = claim.pendingRequest; + String tool = request.optString("tool", ""); + String resolution = approved ? "approved" : "denied"; + String toolResult = approved + ? result("background.review_error", "execution_not_started") + : result("background.action_denied", "user_denied"); + String taskId = ""; + OpenPhoneAgentManager manager = null; + try { + if (!BackgroundJobReviewContract.verify(claim, System.currentTimeMillis())) { + resolution = "binding_invalid"; + toolResult = result("background.review_error", + "approval_binding_invalid_or_expired"); + } else if (!approved) { + resolution = "denied"; + } else { + ToolCatalog catalog = ToolCatalog.get(); + if (!catalog.isLoaded() || !catalog.isAllowedTool(tool) + || !catalog.isStateChangingTool(tool) + || catalog.isTerminalTool(tool)) { + resolution = "tool_invalid"; + toolResult = result("background.review_error", + "tool_not_eligible_for_background_approval"); + } else { + manager = app.getSystemService(OpenPhoneAgentManager.class); + if (manager == null) { + resolution = "framework_unavailable"; + toolResult = result("background.review_error", + "framework_unavailable"); + } else { + taskId = startApprovedTask(manager, claim.job, tool, + catalog.capabilityForTool(tool)); + if (taskId.isEmpty()) { + resolution = "task_start_failed"; + toolResult = result("background.review_error", + "approved_task_start_failed"); + } else { + FrameworkToolExecutor executor = + new FrameworkToolExecutor(app, manager); + JSONObject params = request.optJSONObject("params"); + toolResult = executor.execute(taskId, tool, + params == null ? new JSONObject() : copy(params)); + toolResult = confirmFrameworkActionIfNeeded( + manager, toolResult); + resolution = resultSucceeded(toolResult) + ? "approved" : "approved_action_failed"; + } + } + } + } + } catch (RuntimeException e) { + resolution = "execution_error"; + toolResult = result("background.review_error", + "execution_" + e.getClass().getSimpleName()); + Log.w(TAG, "background review execution failed", e); + } finally { + if (manager != null && !taskId.isEmpty()) { + try { + manager.stopTask(taskId, + "{\"reason\":\"background_review_execution_finished\"}"); + } catch (RuntimeException ignored) { + } + } + } + + long resolvedAt = System.currentTimeMillis(); + boolean completed = store.completeReview( + claim.job.id, confirmationId, resolution, toolResult, resolvedAt); + recordAudit(app, claim, resolution, toolResult, completed, resolvedAt); + OpenPhoneNotificationController.cancelAgentJobReview( + app, claim.job.id); + if (completed) { + OpenPhoneAgentJobScheduler.checkNow(app); + } + return reviewResult(completed, claim.job.id, resolution, toolResult); + } + + private static String startApprovedTask(OpenPhoneAgentManager manager, + AgentJobRecord job, String tool, String capability) { + try { + JSONObject task = new JSONObject() + .put("goal", "Execute approved background action: " + tool) + .put("user_visible", false) + .put("background_allowed", true) + .put("runtime", "builtin") + .put("phone_session_id", "background-job:" + job.id) + .put("approved_capabilities", new JSONArray() + .put("tasks.observe") + .put("screen.read.visible") + .put(capability)); + return new JSONObject(manager.startTask(task.toString())) + .optString("task_id", ""); + } catch (JSONException | RuntimeException e) { + return ""; + } + } + + private static String confirmFrameworkActionIfNeeded( + OpenPhoneAgentManager manager, String rawResult) { + JSONObject parsed = parseObject(rawResult); + String pendingActionId = findStringRecursive(parsed, "pending_action_id"); + String status = parsed.optString("status", ""); + String state = parsed.optString("state", ""); + if (pendingActionId.isEmpty() + || (!status.contains("confirmation") + && !state.contains("confirmation"))) { + return rawResult; + } + try { + return manager.confirmAction(pendingActionId, true); + } catch (RuntimeException e) { + return result("background.review_error", + "framework_confirmation_failed"); + } + } + + private static boolean resultSucceeded(String rawResult) { + JSONObject result = parseObject(rawResult); + String status = result.optString("status", ""); + return !status.isEmpty() + && !result.has("error") + && !status.contains("denied") + && !status.contains("failed") + && !status.contains("error") + && !status.contains("confirmation"); + } + + private static void recordAudit(Context context, AgentJobStore.ReviewClaim claim, + String resolution, String toolResult, boolean completed, long nowMillis) { + JSONObject request = claim.pendingRequest; + JSONObject payload = new JSONObject(); + try { + payload.put("schema", "openphone.background_review_audit.v1") + .put("job_id", claim.job.id) + .put("confirmation_id", + request.optString("confirmation_id", "")) + .put("runtime", request.optString("runtime", "")) + .put("phone_session_id", + request.optString("phone_session_id", "")) + .put("tool", request.optString("tool", "")) + .put("params_digest", + request.optString("params_digest", "")) + .put("binding_digest", + request.optString("binding_digest", "")) + .put("idempotency_key", + request.optString("idempotency_key", "")) + .put("resolution", resolution) + .put("result_status", + parseObject(toolResult).optString("status", "")) + .put("checkpoint_committed", completed) + .put("resolved_at", nowMillis); + } catch (JSONException ignored) { + } + try { + new ContextIndexStore(context).recordAgentEvent( + "assistant.background_job.review_resolved", + "Background action review resolved", + resolution, + "background-job:" + claim.job.id, + payload.toString()); + } catch (RuntimeException e) { + Log.w(TAG, "background review audit write failed", e); + } + } + + private static String reviewResult(boolean completed, long jobId, + String resolution, String toolResult) { + try { + return new JSONObject() + .put("status", completed + ? "background.review_resolved" + : "background.review_commit_failed") + .put("job_id", jobId) + .put("resolution", resolution) + .put("tool_result", parseObject(toolResult)) + .toString(); + } catch (JSONException e) { + return result("background.review_error", "result_encoding_failed"); + } + } + + private static String result(String status, String reason) { + try { + return new JSONObject() + .put("status", status) + .put("reason", reason) + .toString(); + } catch (JSONException e) { + return "{\"status\":\"background.review_error\"}"; + } + } + + private static JSONObject copy(JSONObject object) { + return parseObject(object == null ? "{}" : object.toString()); + } + + private static JSONObject parseObject(String raw) { + try { + return new JSONObject(raw == null || raw.trim().isEmpty() ? "{}" : raw); + } catch (JSONException e) { + return new JSONObject(); + } + } + + private static String findStringRecursive(Object value, String key) { + if (value instanceof JSONObject) { + JSONObject object = (JSONObject) value; + if (object.has(key)) { + String found = object.optString(key, ""); + if (!found.isEmpty() && !"null".equals(found)) { + return found; + } + } + Iterator keys = object.keys(); + while (keys.hasNext()) { + String found = findStringRecursive(object.opt(keys.next()), key); + if (!found.isEmpty()) { + return found; + } + } + } else if (value instanceof JSONArray) { + JSONArray array = (JSONArray) value; + for (int i = 0; i < array.length(); i++) { + String found = findStringRecursive(array.opt(i), key); + if (!found.isEmpty()) { + return found; + } + } + } + return ""; + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/OpenPhoneAgentJobScheduler.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/OpenPhoneAgentJobScheduler.java index e5a1162..208956a 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/OpenPhoneAgentJobScheduler.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/OpenPhoneAgentJobScheduler.java @@ -18,12 +18,15 @@ import org.openphone.assistant.OpenPhoneAssistantService; import org.openphone.assistant.actions.ToolCatalog; import org.openphone.assistant.agent.FrameworkToolExecutor; +import org.openphone.assistant.context.ContextIndexStore; import org.openphone.assistant.runtime.RuntimeConfig; import org.openphone.assistant.model.ModelAdapter; import org.openphone.assistant.model.ModelEndpointConfig; import org.openphone.assistant.model.OpenAiResponsesAgentAdapter; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; public final class OpenPhoneAgentJobScheduler { private static final String TAG = "OpenPhoneAgentJobs"; @@ -45,6 +48,15 @@ public static void checkNow(Context context) { Context appContext = context.getApplicationContext(); AgentJobStore store = new AgentJobStore(appContext); long now = System.currentTimeMillis(); + List expiringReviewIds = store.reviewJobIdsExpiringBy(now); + int expired = store.expirePendingReviews(now); + if (expired > 0) { + for (Long jobId : expiringReviewIds) { + OpenPhoneNotificationController.cancelAgentJobReview( + appContext, jobId == null ? -1L : jobId); + } + Log.i(TAG, "Expired background reviews: " + expired); + } int repaired = store.repairStuck(now - STUCK_TIMEOUT_MILLIS, now); if (repaired > 0) { Log.w(TAG, "Repaired stuck jobs: " + repaired); @@ -81,6 +93,10 @@ public void run() { private static void runJob(Context context, AgentJobRecord job) { AgentJobStore store = new AgentJobStore(context); + AgentJobRecord persistedJob = store.find(job.id); + if (persistedJob != null) { + job = persistedJob; + } long now = System.currentTimeMillis(); if ("heartbeat".equals(job.type)) { store.markCompleted(job.id, "heartbeat", now); @@ -95,6 +111,9 @@ private static void runJob(Context context, AgentJobRecord job) { RuntimeConfig runtimeConfig = RuntimeConfig.load(context); String backgroundRuntime = AssistantBrainConfig.routeBackgroundRuntime( context, runtimeConfig); + if (parseOrEmpty(job.checkpointJson).optBoolean("resume_pending", false)) { + backgroundRuntime = AssistantBrainConfig.BUILTIN; + } if (!AssistantBrainConfig.BUILTIN.equals(backgroundRuntime)) { sendBackgroundJobToRuntime(context, store, job, backgroundRuntime, runtimeConfig); scheduleNext(context, store); @@ -111,28 +130,68 @@ private static void runJob(Context context, AgentJobRecord job) { return; } String taskId = null; + AtomicBoolean reviewPending = new AtomicBoolean(false); try { - String response = agentManager.startTask(taskRequestJson(job)); + final AgentJobRecord runJob = job; + final String runPrompt = BackgroundJobReviewContract.resumePrompt(runJob); + String response = agentManager.startTask(taskRequestJson(runJob, runPrompt)); taskId = parseString(response, "task_id"); if (taskId == null || taskId.isEmpty()) { failJob(context, store, job, "task_start_failed"); return; } FrameworkToolExecutor toolExecutor = new FrameworkToolExecutor(context, agentManager); - OpenAiResponsesAgentAdapter adapter = new OpenAiResponsesAgentAdapter(endpointConfig); + OpenAiResponsesAgentAdapter adapter = + new OpenAiResponsesAgentAdapter(endpointConfig, true); final String activeTaskId = taskId; - String result = adapter.runTask(activeTaskId, job.prompt, + final AtomicBoolean resumeConsumed = new AtomicBoolean(false); + final AtomicReference setupFailure = new AtomicReference<>(""); + String result = adapter.runTask(activeTaskId, runPrompt, new ModelAdapter.ToolExecutor() { @Override public String callTool(String toolName, String argumentsJson) { - if (isStateChangingBackgroundTool(toolName)) { - return "{\"status\":\"background.confirmation_required\"," - + "\"reason\":\"state_changing_tool_blocked\"," - + "\"tool\":\"" + jsonEscape(toolName) + "\"}"; - } try { - return toolExecutor.execute(activeTaskId, toolName, - new JSONObject(argumentsJson == null ? "{}" : argumentsJson)); + JSONObject arguments = new JSONObject( + argumentsJson == null ? "{}" : argumentsJson); + if (isStateChangingBackgroundTool(toolName)) { + if (BackgroundJobReviewContract.matchesResume( + runJob, toolName, arguments) + && resumeConsumed.compareAndSet(false, true)) { + return BackgroundJobReviewContract.resumeToolResult(runJob); + } + long requestedAt = System.currentTimeMillis(); + BackgroundJobReviewContract.PreparedRequest prepared = + BackgroundJobReviewContract.prepare( + runJob, toolName, arguments, requestedAt); + if (prepared == null) { + setupFailure.compareAndSet("", + "review_request_rejected_sensitive_or_oversized"); + return "{\"status\":\"denied\"," + + "\"code\":\"background.review_request_rejected\"," + + "\"reason\":\"sensitive_or_oversized_checkpoint\"}"; + } + boolean saved = store.markAwaitingReview( + runJob.id, + prepared.checkpoint, + prepared.pendingRequest, + prepared.confirmationId, + prepared.resumeToken, + "Approval needed: " + toolName, + requestedAt); + if (!saved) { + setupFailure.compareAndSet("", + "review_checkpoint_persist_failed"); + return "{\"status\":\"denied\"," + + "\"code\":\"background.review_persist_failed\"}"; + } + reviewPending.set(true); + OpenPhoneNotificationController.showAgentJobReview( + context, runJob, prepared.pendingRequest); + recordReviewRequested( + context, runJob, prepared, toolName, requestedAt); + return confirmationRequired(prepared, toolName); + } + return toolExecutor.execute(activeTaskId, toolName, arguments); } catch (JSONException e) { return "{\"status\":\"error\",\"reason\":\"bad_tool_json\"}"; } @@ -140,19 +199,33 @@ public String callTool(String toolName, String argumentsJson) { @Override public boolean isCancelled() { - return false; + AgentJobRecord current = store.find(runJob.id); + return current == null || "stopped".equals(current.status); } }); - store.markCompleted(job.id, result, System.currentTimeMillis()); - if (shouldNotify(job)) { - OpenPhoneNotificationController.showAgentJobFinished(context, job, result); + if (reviewPending.get()) { + Log.i(TAG, "Background job awaiting review: " + runJob.id); + } else if (!setupFailure.get().isEmpty()) { + failJob(context, store, runJob, setupFailure.get()); + } else { + boolean completed = store.markCompleted( + runJob.id, result, System.currentTimeMillis()); + if (completed && shouldNotify(runJob)) { + OpenPhoneNotificationController.showAgentJobFinished( + context, runJob, result); + } } } catch (RuntimeException e) { - failJob(context, store, job, "job_error:" + e.getClass().getSimpleName()); + if (!reviewPending.get()) { + failJob(context, store, job, + "job_error:" + e.getClass().getSimpleName()); + } } finally { if (agentManager != null && taskId != null && !taskId.isEmpty()) { try { - agentManager.stopTask(taskId, "{\"reason\":\"background_job_finished\"}"); + agentManager.stopTask(taskId, reviewPending.get() + ? "{\"reason\":\"background_job_paused_for_review\"}" + : "{\"reason\":\"background_job_finished\"}"); } catch (RuntimeException ignored) { } } @@ -210,7 +283,12 @@ private static void failJob(Context context, AgentJobStore store, int failures = job.failureCount + 1; long nextRunAt = now + AgentJobStore.backoffMillis(failures); long failureAlertAt = failures >= 3 ? now : job.failureAlertAtMillis; - store.markFailed(job.id, reason, nextRunAt, failures, failureAlertAt, now); + boolean failed = store.markFailed( + job.id, reason, nextRunAt, failures, failureAlertAt, now); + if (!failed) { + scheduleNext(context, store); + return; + } Log.w(TAG, "Agent job failed: " + job.id + " " + reason); if (failures >= 3 && shouldNotify(job)) { OpenPhoneNotificationController.showAgentJobFailed(context, job, reason); @@ -263,13 +341,72 @@ private static ModelEndpointConfig backgroundEndpointConfig(Context context) { return ModelEndpointConfig.directOpenAi(apiKey == null ? "" : apiKey); } - private static String taskRequestJson(AgentJobRecord job) { - return "{" - + "\"goal\":\"" + jsonEscape(job.prompt) + "\"," - + "\"user_visible\":false," - + "\"background_allowed\":true," - + "\"approved_capabilities\":[\"tasks.observe\",\"screen.read.visible\"]" - + "}"; + private static String taskRequestJson(AgentJobRecord job, String prompt) { + try { + return new JSONObject() + .put("goal", prompt == null ? job.prompt : prompt) + .put("user_visible", false) + .put("background_allowed", true) + .put("runtime", "builtin") + .put("phone_session_id", "background-job:" + job.id) + .put("approved_capabilities", + new org.json.JSONArray() + .put("tasks.observe") + .put("screen.read.visible")) + .toString(); + } catch (JSONException e) { + return "{}"; + } + } + + private static String confirmationRequired( + BackgroundJobReviewContract.PreparedRequest prepared, String toolName) { + try { + return new JSONObject() + .put("status", "confirmation_required") + .put("code", "background.confirmation_required") + .put("reason", "state_changing_background_tool") + .put("tool", toolName) + .put("confirmation_id", prepared.confirmationId) + .put("params_digest", prepared.paramsDigest) + .put("binding_digest", prepared.bindingDigest) + .put("expires_at", + prepared.pendingRequest.optLong("expires_at", 0L)) + .toString(); + } catch (JSONException e) { + return "{\"status\":\"confirmation_required\"," + + "\"code\":\"background.confirmation_required\"}"; + } + } + + private static void recordReviewRequested(Context context, AgentJobRecord job, + BackgroundJobReviewContract.PreparedRequest prepared, + String toolName, long requestedAt) { + JSONObject payload = new JSONObject(); + try { + payload.put("schema", "openphone.background_review_audit.v1") + .put("job_id", job.id) + .put("confirmation_id", prepared.confirmationId) + .put("runtime", "builtin") + .put("phone_session_id", "background-job:" + job.id) + .put("tool", toolName) + .put("params_digest", prepared.paramsDigest) + .put("binding_digest", prepared.bindingDigest) + .put("expires_at", + prepared.pendingRequest.optLong("expires_at", 0L)) + .put("requested_at", requestedAt); + } catch (JSONException ignored) { + } + try { + new ContextIndexStore(context).recordAgentEvent( + "assistant.background_job.review_requested", + "Background action needs review", + toolName, + "background-job:" + job.id, + payload.toString()); + } catch (RuntimeException e) { + Log.w(TAG, "background review audit write failed", e); + } } private static boolean shouldNotify(AgentJobRecord job) { diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/OpenPhoneOsToolGateway.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/OpenPhoneOsToolGateway.java new file mode 100644 index 0000000..e461a86 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/OpenPhoneOsToolGateway.java @@ -0,0 +1,64 @@ +package org.openphone.assistant.platform; + +import android.content.Context; +import android.openphone.OpenPhoneAgentManager; + +import org.json.JSONObject; +import org.openphone.assistant.agent.FrameworkToolExecutor; + +/** OpenPhone OS implementation of the portable phone-tool boundary. */ +public final class OpenPhoneOsToolGateway implements PhoneToolGateway { + public static final String PROFILE = "openphone_os"; + + private final OpenPhoneAgentManager mAgentManager; + private final FrameworkToolExecutor mToolExecutor; + + public OpenPhoneOsToolGateway(Context context, OpenPhoneAgentManager agentManager) { + Context app = context.getApplicationContext(); + mAgentManager = agentManager; + mToolExecutor = new FrameworkToolExecutor(app, agentManager); + } + + @Override + public String profile() { + return PROFILE; + } + + @Override + public boolean isAvailable() { + return mAgentManager != null; + } + + @Override + public boolean supportsTool(String toolName) { + return isAvailable() && toolName != null && !toolName.trim().isEmpty(); + } + + @Override + public String startTask(String taskJson) { + if (!isAvailable()) { + return unavailable(); + } + return mAgentManager.startTask(taskJson); + } + + @Override + public String executeTool(String taskId, String toolName, JSONObject arguments) { + if (!isAvailable()) { + return unavailable(); + } + return mToolExecutor.execute(taskId, toolName, arguments); + } + + @Override + public String confirmAction(String pendingActionId, boolean approved) { + if (!isAvailable()) { + return unavailable(); + } + return mAgentManager.confirmAction(pendingActionId, approved); + } + + private static String unavailable() { + return "{\"error\":\"phone_platform_unavailable\"}"; + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/PhoneToolGateway.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/PhoneToolGateway.java new file mode 100644 index 0000000..ec5c358 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/PhoneToolGateway.java @@ -0,0 +1,31 @@ +package org.openphone.assistant.platform; + +import org.json.JSONObject; + +/** + * Capability boundary between portable OpenPhone product logic and the phone + * implementation that executes tools. + * + *

The runtime and adaptive-surface layers depend only on this contract. + * OpenPhone OS supplies a framework-backed implementation; a Play-distributed + * build can supply a public-API implementation with a smaller capability set. + */ +public interface PhoneToolGateway { + /** Stable implementation profile exposed for diagnostics. */ + String profile(); + + /** Whether this implementation can currently accept phone work. */ + boolean isAvailable(); + + /** Whether this implementation supports the registered tool. */ + boolean supportsTool(String toolName); + + /** Starts a policy-scoped phone task and returns its JSON result. */ + String startTask(String taskJson); + + /** Executes one registered phone tool and returns its JSON result. */ + String executeTool(String taskId, String toolName, JSONObject arguments); + + /** Resolves an OS-owned pending action and returns its JSON result. */ + String confirmAction(String pendingActionId, boolean approved); +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runs/AgentRunProjection.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runs/AgentRunProjection.java new file mode 100644 index 0000000..c06d4dc --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runs/AgentRunProjection.java @@ -0,0 +1,382 @@ +package org.openphone.assistant.runs; + +import android.content.Context; +import android.content.SharedPreferences; + +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.commitments.CommitmentRecord; +import org.openphone.assistant.commitments.CommitmentStore; +import org.openphone.assistant.jobs.AgentJobRecord; +import org.openphone.assistant.jobs.AgentJobStore; +import org.openphone.assistant.jobs.OpenPhoneAgentJobScheduler; +import org.openphone.assistant.OpenPhoneNotificationController; +import org.openphone.assistant.session.PhoneExecutionSession; +import org.openphone.assistant.session.PhoneSessionStore; +import org.openphone.assistant.watchers.WatcherRecord; +import org.openphone.assistant.watchers.WatcherStore; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * Central read model for durable OpenPhone work. + * + * Source stores remain authoritative. Read/dismiss state belongs to this + * projection because it is presentation state, not runtime execution state. + */ +public final class AgentRunProjection { + private static final String PREFS = "openphone_run_projection"; + private static final String READ_PREFIX = "read:"; + private static final String HIDDEN_PREFIX = "hidden:"; + private static final long TERMINAL_VISIBILITY_MILLIS = 24L * 60L * 60L * 1000L; + + private final AgentJobStore mJobs; + private final WatcherStore mWatchers; + private final CommitmentStore mCommitments; + private final PhoneSessionStore mSessions; + private final SharedPreferences mPrefs; + private final Context mContext; + + public AgentRunProjection(Context context) { + Context app = context.getApplicationContext(); + mContext = app; + mJobs = new AgentJobStore(app); + mWatchers = new WatcherStore(app); + mCommitments = new CommitmentStore(app); + mSessions = new PhoneSessionStore(app); + mPrefs = app.getSharedPreferences(PREFS, Context.MODE_PRIVATE); + } + + public List snapshot(int limit) { + long now = System.currentTimeMillis(); + ArrayList out = new ArrayList<>(); + appendJobs(out, now); + appendWatchers(out); + appendCommitments(out, now); + appendSessions(out, now); + Collections.sort(out, RUN_ORDER); + int bounded = Math.max(1, Math.min(limit, 100)); + if (out.size() <= bounded) { + return out; + } + return new ArrayList<>(out.subList(0, bounded)); + } + + public boolean markRead(String stableId) { + String clean = clean(stableId); + if (clean.isEmpty()) { + return false; + } + return mPrefs.edit().putLong(READ_PREFIX + clean, System.currentTimeMillis()).commit(); + } + + public boolean dismiss(String stableId) { + String clean = clean(stableId); + if (clean.isEmpty()) { + return false; + } + return mPrefs.edit() + .putLong(READ_PREFIX + clean, System.currentTimeMillis()) + .putLong(HIDDEN_PREFIX + clean, System.currentTimeMillis()) + .commit(); + } + + public boolean stop(String stableId) { + ParsedId parsed = ParsedId.parse(stableId); + if (parsed == null) { + return false; + } + if (AgentRunSummary.KIND_JOB.equals(parsed.kind)) { + AgentJobRecord job = mJobs.find(parsed.longId); + boolean stopped = mJobs.stop(parsed.longId); + if (stopped && job != null && !job.pendingConfirmationId.isEmpty()) { + OpenPhoneNotificationController.cancelAgentJobReview( + mContext, job.id); + } + return stopped; + } + if (AgentRunSummary.KIND_WATCHER.equals(parsed.kind)) { + return mWatchers.stop(parsed.longId); + } + if (AgentRunSummary.KIND_COMMITMENT.equals(parsed.kind)) { + return mCommitments.dismiss(parsed.longId); + } + return false; + } + + public boolean pause(String stableId) { + ParsedId parsed = ParsedId.parse(stableId); + boolean paused = parsed != null + && AgentRunSummary.KIND_JOB.equals(parsed.kind) + && mJobs.pause(parsed.longId); + if (paused) { + OpenPhoneAgentJobScheduler.scheduleNext(mContext); + } + return paused; + } + + public boolean resume(String stableId) { + ParsedId parsed = ParsedId.parse(stableId); + boolean resumed = parsed != null + && AgentRunSummary.KIND_JOB.equals(parsed.kind) + && mJobs.resume(parsed.longId); + if (resumed) { + OpenPhoneAgentJobScheduler.checkNow(mContext); + } + return resumed; + } + + private void appendJobs(List out, long now) { + for (AgentJobRecord job : mJobs.list("", 50)) { + String stableId = stableId(AgentRunSummary.KIND_JOB, job.id); + if (isHidden(stableId)) { + continue; + } + boolean terminal = isTerminal(job.status); + boolean unread = terminal && job.unreadResult && !isRead(stableId); + if (terminal && !unread + && now - Math.max(job.updatedAtMillis, job.lastRunAtMillis) + > TERMINAL_VISIBILITY_MILLIS) { + continue; + } + JSONObject payload = objectOrEmpty(job.payloadJson); + String phase = firstNonEmpty(job.phase, + payload.optString("phase", ""), job.status); + String progress = firstNonEmpty(job.progressText, + payload.optString("progress_text", "")); + if (progress.isEmpty()) { + progress = terminal ? summarizeResult(job.lastResult) : job.prompt; + } + String pending = firstNonEmpty(job.pendingConfirmationId, + payload.optString("pending_confirmation_id", "")); + JSONObject pendingRequest = objectOrEmpty(job.pendingToolRequestJson); + String reviewSummary = pending.isEmpty() ? "" : firstNonEmpty( + pendingRequest.optString("summary", ""), + pendingRequest.optString("tool", "")); + String surface = firstNonEmpty(job.lastSurfaceId, + payload.optString("surface_id", "")); + boolean attention = "awaiting_review".equals(job.status) + || job.failureCount > 0 || !pending.isEmpty() + || "failed".equals(job.status); + out.add(new AgentRunSummary( + stableId, + AgentRunSummary.KIND_JOB, + Long.toString(job.id), + job.title, + job.status, + phase, + progress, + "background_job", + job.sessionTarget, + job.createdAtMillis, + job.updatedAtMillis, + job.nextRunAtMillis, + attention, + unread, + pending, + reviewSummary, + surface, + "queued".equals(job.status) || "active".equals(job.status), + "paused".equals(job.status), + !terminal)); + } + } + + private void appendWatchers(List out) { + for (WatcherRecord watcher : mWatchers.active(50)) { + String stableId = stableId(AgentRunSummary.KIND_WATCHER, watcher.id); + if (isHidden(stableId)) { + continue; + } + boolean attention = watcher.failureCount > 0; + out.add(new AgentRunSummary( + stableId, + AgentRunSummary.KIND_WATCHER, + Long.toString(watcher.id), + watcher.title, + watcher.status, + "running".equals(watcher.status) ? "checking" : "watching", + watcher.type, + "watcher", + watcher.sessionTarget, + watcher.createdAtMillis, + watcher.updatedAtMillis, + watcher.nextRunAtMillis, + attention, + false, + "", + "", + "", + false, + false, + true)); + } + } + + private void appendCommitments(List out, long now) { + for (CommitmentRecord commitment : mCommitments.active(50)) { + String stableId = stableId(AgentRunSummary.KIND_COMMITMENT, commitment.id); + if (isHidden(stableId)) { + continue; + } + boolean due = commitment.dueAtMillis > 0 && commitment.dueAtMillis <= now; + if (!due && commitment.updatedAtMillis + TERMINAL_VISIBILITY_MILLIS < now) { + continue; + } + out.add(new AgentRunSummary( + stableId, + AgentRunSummary.KIND_COMMITMENT, + Long.toString(commitment.id), + commitment.title, + commitment.status, + due ? "due" : "waiting", + commitment.description, + "commitment", + "", + commitment.createdAtMillis, + commitment.updatedAtMillis, + commitment.dueAtMillis, + due, + due && !isRead(stableId), + "", + "", + "", + false, + false, + true)); + } + } + + private void appendSessions(List out, long now) { + for (PhoneExecutionSession session : mSessions.list(50)) { + String stableId = sessionStableId(session.phoneSessionId); + if (isHidden(stableId)) { + continue; + } + boolean terminal = isSessionTerminal(session.status); + boolean unread = terminal && !isRead(stableId); + if (terminal && (!unread + || now - session.updatedAtMillis > TERMINAL_VISIBILITY_MILLIS)) { + continue; + } + boolean attention = "awaiting_confirmation".equals(session.status) + || "failed".equals(session.status); + out.add(new AgentRunSummary( + stableId, + AgentRunSummary.KIND_SESSION, + session.phoneSessionId, + session.summary.isEmpty() ? "Runtime session" : session.summary, + session.status, + session.status, + session.source, + session.source, + session.runtimeKind, + session.createdAtMillis, + session.updatedAtMillis, + 0L, + attention, + unread, + "", + "", + "", + false, + false, + false)); + } + } + + private boolean isRead(String stableId) { + return mPrefs.contains(READ_PREFIX + stableId); + } + + private boolean isHidden(String stableId) { + return mPrefs.contains(HIDDEN_PREFIX + stableId); + } + + private static String stableId(String kind, long id) { + return kind + ":" + id; + } + + private static String sessionStableId(String sessionId) { + return AgentRunSummary.KIND_SESSION + ":" + clean(sessionId); + } + + private static boolean isTerminal(String status) { + return "completed".equals(status) + || "failed".equals(status) + || "stopped".equals(status) + || "dispatched".equals(status); + } + + private static boolean isSessionTerminal(String status) { + return "completed".equals(status) + || "failed".equals(status) + || "stopped".equals(status) + || "final".equals(status); + } + + private static JSONObject objectOrEmpty(String raw) { + try { + return new JSONObject(raw == null || raw.trim().isEmpty() ? "{}" : raw); + } catch (JSONException e) { + return new JSONObject(); + } + } + + private static String summarizeResult(String value) { + String clean = clean(value).replaceAll("\\s+", " "); + return clean.length() <= 240 ? clean : clean.substring(0, 240) + "…"; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } + + private static String firstNonEmpty(String... values) { + for (String value : values) { + if (value != null && !value.trim().isEmpty()) { + return value.trim(); + } + } + return ""; + } + + private static final Comparator RUN_ORDER = (left, right) -> { + int attention = Boolean.compare(right.needsAttention, left.needsAttention); + if (attention != 0) { + return attention; + } + int live = Boolean.compare(right.isLive(), left.isLive()); + if (live != 0) { + return live; + } + return Long.compare(right.updatedAtMillis, left.updatedAtMillis); + }; + + private static final class ParsedId { + final String kind; + final long longId; + + ParsedId(String kind, long longId) { + this.kind = kind; + this.longId = longId; + } + + static ParsedId parse(String stableId) { + String clean = clean(stableId); + int separator = clean.indexOf(':'); + if (separator <= 0 || separator >= clean.length() - 1) { + return null; + } + String kind = clean.substring(0, separator); + try { + return new ParsedId(kind, Long.parseLong(clean.substring(separator + 1))); + } catch (NumberFormatException e) { + return null; + } + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runs/AgentRunSummary.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runs/AgentRunSummary.java new file mode 100644 index 0000000..e1bb701 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runs/AgentRunSummary.java @@ -0,0 +1,72 @@ +package org.openphone.assistant.runs; + +/** Read-only projection of durable OpenPhone work for Home and SystemUI. */ +public final class AgentRunSummary { + public static final String KIND_JOB = "job"; + public static final String KIND_WATCHER = "watcher"; + public static final String KIND_COMMITMENT = "commitment"; + public static final String KIND_SESSION = "foreground_session"; + + public final String id; + public final String kind; + public final String sourceId; + public final String title; + public final String status; + public final String phase; + public final String progressText; + public final String origin; + public final String runtime; + public final long createdAtMillis; + public final long updatedAtMillis; + public final long nextRunAtMillis; + public final boolean needsAttention; + public final boolean unreadResult; + public final String pendingConfirmationId; + public final String reviewSummary; + public final String surfaceId; + public final boolean canPause; + public final boolean canResume; + public final boolean canStop; + + public AgentRunSummary(String id, String kind, String sourceId, String title, + String status, String phase, String progressText, String origin, String runtime, + long createdAtMillis, long updatedAtMillis, long nextRunAtMillis, + boolean needsAttention, boolean unreadResult, String pendingConfirmationId, + String reviewSummary, String surfaceId, boolean canPause, boolean canResume, + boolean canStop) { + this.id = clean(id); + this.kind = clean(kind); + this.sourceId = clean(sourceId); + this.title = clean(title); + this.status = clean(status); + this.phase = clean(phase); + this.progressText = clean(progressText); + this.origin = clean(origin); + this.runtime = clean(runtime); + this.createdAtMillis = createdAtMillis; + this.updatedAtMillis = updatedAtMillis; + this.nextRunAtMillis = nextRunAtMillis; + this.needsAttention = needsAttention; + this.unreadResult = unreadResult; + this.pendingConfirmationId = clean(pendingConfirmationId); + this.reviewSummary = clean(reviewSummary); + this.surfaceId = clean(surfaceId); + this.canPause = canPause; + this.canResume = canResume; + this.canStop = canStop; + } + + public boolean isLive() { + return "active".equals(status) + || "queued".equals(status) + || "running".equals(status) + || "waiting".equals(status) + || "waiting_for_runtime".equals(status) + || "awaiting_review".equals(status) + || "created".equals(status); + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeCallback.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeCallback.java index 545ef33..b4e68a8 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeCallback.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeCallback.java @@ -1,5 +1,19 @@ package org.openphone.assistant.runtime; +import org.json.JSONObject; +import org.openphone.assistant.surface.AssistantOutput; + public interface RuntimeCallback { void onRuntimeMessage(String runtime, String sessionKey, String message, boolean terminal); + + default void onRuntimeOutput(String runtime, String sessionKey, + AssistantOutput output, boolean terminal) { + if (output != null && !output.displayText().isEmpty()) { + onRuntimeMessage(runtime, sessionKey, output.displayText(), terminal); + } + } + + default void onRuntimeSurfaceEvent(String runtime, String sessionKey, + String event, JSONObject payload) { + } } diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeManager.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeManager.java index d943f2e..2cbecf1 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeManager.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeManager.java @@ -1,16 +1,20 @@ package org.openphone.assistant.runtime; import android.content.Context; -import android.openphone.OpenPhoneAgentManager; import android.util.Log; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.openphone.assistant.OpenPhoneNotificationController; +import org.openphone.assistant.platform.PhoneToolGateway; import org.openphone.assistant.runtime.adapters.openclaw.OpenClawRuntimeAdapter; import org.openphone.assistant.session.PhoneExecutionSession; import org.openphone.assistant.session.PhoneSessionStore; +import org.openphone.assistant.surface.AdaptiveSurface; +import org.openphone.assistant.surface.AssistantOutput; +import org.openphone.assistant.surface.SurfaceMutationResult; +import org.openphone.assistant.surface.SurfaceRepository; import java.util.ArrayList; import java.util.List; @@ -20,25 +24,29 @@ public final class RuntimeManager implements RuntimeConfirmationCallback { private static final String TAG = "OpenPhoneRuntime"; private final Context mContext; - private final OpenPhoneAgentManager mAgentManager; + private final PhoneToolGateway mPhoneGateway; private final RuntimeToolBridge mToolBridge; private final PhoneSessionStore mSessionStore; + private final SurfaceRepository mSurfaceRepository; private final List mAdapters = new ArrayList<>(); private RuntimeCallback mRuntimeCallback; private String mStatus = "disabled"; - public RuntimeManager(Context context, OpenPhoneAgentManager agentManager) { + public RuntimeManager(Context context, PhoneToolGateway phoneGateway) { mContext = context; - mAgentManager = agentManager; + mPhoneGateway = phoneGateway; mSessionStore = new PhoneSessionStore(context); - mToolBridge = new RuntimeToolBridge(context, agentManager, mSessionStore); + mSurfaceRepository = new SurfaceRepository(context); + mToolBridge = new RuntimeToolBridge(context, phoneGateway, mSessionStore); mToolBridge.setConfirmationCallback(this); } public synchronized void start() { stopLocked(); RuntimeConfig config = RuntimeConfig.load(mContext); - if (mAgentManager == null) { + if (mPhoneGateway == null || !mPhoneGateway.isAvailable()) { + // Preserve the existing wire/status value while the Play profile + // is introduced behind the new gateway contract. mStatus = "framework_unavailable"; return; } @@ -54,6 +62,18 @@ public void onRuntimeMessage(String runtime, String sessionKey, String message, boolean terminal) { dispatchRuntimeMessage(runtime, sessionKey, message, terminal); } + + @Override + public void onRuntimeOutput(String runtime, String sessionKey, + AssistantOutput output, boolean terminal) { + acceptRuntimeOutput(runtime, sessionKey, output, terminal, "auto", -1); + } + + @Override + public void onRuntimeSurfaceEvent(String runtime, String sessionKey, + String event, JSONObject payload) { + handleRuntimeSurfaceEvent(runtime, sessionKey, event, payload); + } })); } if (mAdapters.isEmpty()) { @@ -88,6 +108,8 @@ public synchronized String statusJson() { .put("status", aggregateStatus) .put("manager_status", aggregateStatus) .put("lifecycle_status", mStatus) + .put("phone_platform", mPhoneGateway == null + ? "unavailable" : mPhoneGateway.profile()) .put("updated_at_ms", System.currentTimeMillis()) .put("adapters", adapters) .toString(); @@ -197,12 +219,157 @@ private void sendResolutionToRuntime(RuntimeToolRequest request, private void dispatchRuntimeMessage(String runtime, String sessionKey, String message, boolean terminal) { + String phoneSessionId = PhoneSessionStore.extractPhoneSessionId(sessionKey); + if (!phoneSessionId.isEmpty()) { + String lower = message == null ? "" : message.toLowerCase(); + String status = terminal + ? (lower.contains("error") || lower.contains("failed") + || lower.contains("disconnected") ? "failed" : "completed") + : "running"; + mSessionStore.touchStatus(phoneSessionId, status); + } RuntimeCallback callback = mRuntimeCallback; if (callback != null) { callback.onRuntimeMessage(runtime, sessionKey, message, terminal); } } + private void dispatchRuntimeOutput(String runtime, String sessionKey, + AssistantOutput output, boolean terminal) { + RuntimeCallback callback = mRuntimeCallback; + if (callback != null) { + callback.onRuntimeOutput(runtime, sessionKey, output, terminal); + } + } + + private synchronized void handleRuntimeSurfaceEvent(String runtime, String sessionKey, + String event, JSONObject payload) { + if ("runtime.surface.dismiss".equals(event)) { + handleRuntimeSurfaceDismiss(runtime, sessionKey, payload); + return; + } + if (!"runtime.surface.present".equals(event) + && !"runtime.surface.replace".equals(event)) { + sendSurfaceProtocolResult(runtime, null, "event_unsupported", + "Unsupported runtime surface event.", event); + return; + } + AssistantOutput output = AssistantOutput.fromJson( + payload == null ? null : payload.optJSONObject("output")); + if (output == null) { + sendSurfaceProtocolResult(runtime, null, "output_invalid", + "Surface event requires an explicit assistant output envelope.", event); + return; + } + int expectedRevision = payload.optInt("expected_revision", -1); + acceptRuntimeOutput(runtime, sessionKey, output, false, + "runtime.surface.replace".equals(event) ? "replace" : "present", + expectedRevision); + } + + private synchronized void acceptRuntimeOutput(String runtime, String sessionKey, + AssistantOutput output, boolean terminal, String operation, int expectedRevision) { + if (output == null) { + sendSurfaceProtocolResult(runtime, null, "output_invalid", + "Assistant output is malformed.", operation); + return; + } + AdaptiveSurface surface = output.surface; + if (surface != null) { + if (!RuntimeRegistry.normalize(runtime).equals( + RuntimeRegistry.normalize(surface.runtime))) { + sendSurfaceProtocolResult(runtime, surface, "runtime_mismatch", + "Surface runtime ownership does not match its transport.", operation); + return; + } + if (!output.sessionId.equals(surface.sessionId) + || (!sessionKey.isEmpty() && !sessionKey.contains(output.sessionId))) { + sendSurfaceProtocolResult(runtime, surface, "session_mismatch", + "Surface session ownership does not match its output.", operation); + return; + } + SurfaceMutationResult mutation; + AdaptiveSurface current = mSurfaceRepository.getActive(surface.surfaceId); + if ("replace".equals(operation)) { + int expected = expectedRevision < 1 + ? surface.revision - 1 : expectedRevision; + mutation = mSurfaceRepository.replace(surface, expected); + } else if ("present".equals(operation)) { + mutation = mSurfaceRepository.present(surface); + } else if (current == null) { + mutation = mSurfaceRepository.present(surface); + } else if (surface.revision == current.revision + && surface.toJson().toString().equals(current.toJson().toString())) { + mutation = SurfaceMutationResult.ok("already_presented", current); + } else { + mutation = mSurfaceRepository.replace(surface, current.revision); + } + if (!mutation.ok) { + sendSurfaceProtocolResult(runtime, surface, mutation.code, + mutation.message, operation); + return; + } + sendSurfaceProtocolResult(runtime, surface, mutation.code, + "Surface accepted by the phone.", operation); + } + if (!output.sessionId.isEmpty()) { + mSessionStore.touchStatus(output.sessionId, terminal ? "completed" : "running"); + } + dispatchRuntimeOutput(runtime, sessionKey, output, terminal); + } + + private void handleRuntimeSurfaceDismiss(String runtime, String sessionKey, + JSONObject payload) { + String surfaceId = payload == null ? "" : payload.optString("surface_id", ""); + AdaptiveSurface surface = mSurfaceRepository.getActive(surfaceId); + if (surface == null) { + sendSurfaceProtocolResult(runtime, null, "surface_not_active", + "Surface is missing, expired, or already dismissed.", "dismiss"); + return; + } + if (!RuntimeRegistry.normalize(runtime).equals( + RuntimeRegistry.normalize(surface.runtime))) { + sendSurfaceProtocolResult(runtime, surface, "runtime_mismatch", + "Runtime cannot dismiss another runtime's surface.", "dismiss"); + return; + } + String claimedSession = payload.optString("session_id", ""); + if (claimedSession.isEmpty()) { + claimedSession = PhoneSessionStore.extractPhoneSessionId(sessionKey); + } + if (!claimedSession.isEmpty() && !claimedSession.equals(surface.sessionId)) { + sendSurfaceProtocolResult(runtime, surface, "session_mismatch", + "Runtime cannot dismiss another session's surface.", "dismiss"); + return; + } + int revision = payload.optInt("revision", 0); + if (revision != surface.revision) { + sendSurfaceProtocolResult(runtime, surface, "stale_revision", + "Runtime dismissal revision is stale.", "dismiss"); + return; + } + SurfaceMutationResult result = mSurfaceRepository.dismiss( + surface.surfaceId, payload.optString("reason", "runtime_dismissed")); + sendSurfaceProtocolResult(runtime, surface, result.code, + result.ok ? "Surface dismissed." : result.message, "dismiss"); + } + + private void sendSurfaceProtocolResult(String runtime, AdaptiveSurface surface, + String status, String message, String operation) { + JSONObject payload = new JSONObject(); + try { + payload.put("surface_id", surface == null ? "" : surface.surfaceId) + .put("revision", surface == null ? 0 : surface.revision) + .put("session_id", surface == null ? "" : surface.sessionId) + .put("operation", operation == null ? "" : operation) + .put("status", status == null ? "" : status) + .put("message", message == null ? "" : message); + } catch (JSONException ignored) { + } + sendEventToRuntime(runtime, + new RuntimeEvent("runtime.surface.action_result", payload)); + } + private void sendResultToRuntime(String runtime, RuntimeToolRequest request, RuntimeToolResult result) { for (RuntimeAdapter adapter : mAdapters) { @@ -212,7 +379,7 @@ private void sendResultToRuntime(String runtime, RuntimeToolRequest request, } } - private void sendEventToRuntime(String runtime, RuntimeEvent event) { + public synchronized void sendEventToRuntime(String runtime, RuntimeEvent event) { if (runtime == null || runtime.isEmpty() || event == null) { return; } diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeToolBridge.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeToolBridge.java index 9b113e2..b24ce80 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeToolBridge.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeToolBridge.java @@ -1,14 +1,13 @@ package org.openphone.assistant.runtime; import android.content.Context; -import android.openphone.OpenPhoneAgentManager; import android.util.Log; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.openphone.assistant.actions.ToolCatalog; -import org.openphone.assistant.agent.FrameworkToolExecutor; +import org.openphone.assistant.platform.PhoneToolGateway; import org.openphone.assistant.session.PhoneExecutionSession; import org.openphone.assistant.session.PhoneSessionStore; @@ -29,8 +28,7 @@ public final class RuntimeToolBridge { private static final String TAG = "OpenPhoneRuntime"; private static final int MAX_COMPLETED_IDEMPOTENCY_RESULTS = 128; - private final OpenPhoneAgentManager mAgentManager; - private final FrameworkToolExecutor mToolExecutor; + private final PhoneToolGateway mPhoneGateway; private final PhoneSessionStore mSessionStore; private final Map mTaskIdsByRuntimeSession = new HashMap<>(); private final Map mPendingConfirmations = @@ -55,14 +53,13 @@ public Thread newThread(Runnable runnable) { }); private RuntimeConfirmationCallback mConfirmationCallback; - public RuntimeToolBridge(Context context, OpenPhoneAgentManager agentManager) { - this(context, agentManager, new PhoneSessionStore(context)); + public RuntimeToolBridge(Context context, PhoneToolGateway phoneGateway) { + this(context, phoneGateway, new PhoneSessionStore(context)); } - public RuntimeToolBridge(Context context, OpenPhoneAgentManager agentManager, + public RuntimeToolBridge(Context context, PhoneToolGateway phoneGateway, PhoneSessionStore sessionStore) { - mAgentManager = agentManager; - mToolExecutor = new FrameworkToolExecutor(context, agentManager); + mPhoneGateway = phoneGateway; mSessionStore = sessionStore == null ? new PhoneSessionStore(context) : sessionStore; } @@ -137,6 +134,23 @@ private RuntimeToolResult executeInternal(RuntimeToolRequest request, logResult(request, result); return result; } + if (!mPhoneGateway.supportsTool(request.tool())) { + JSONObject details = new JSONObject(); + try { + details.put("tool", request.tool()) + .put("phone_platform", mPhoneGateway.profile()); + } catch (JSONException ignored) { + } + RuntimeToolResult result = RuntimeToolResult.denied( + request.requestId(), + "tool_not_supported_by_phone_platform", + "The active phone platform does not support this tool.", + details, + auditId); + markSessionForResult(request, result); + logResult(request, result); + return result; + } RuntimeToolResult completed = completedResultFor(request, auditId); if (completed != null) { markSessionForResult(request, completed); @@ -196,9 +210,9 @@ private RuntimeToolResult executeInternal(RuntimeToolRequest request, } markSession(request, "running"); - String rawResult = mToolExecutor.execute(taskId, request.tool(), params); + String rawResult = mPhoneGateway.executeTool(taskId, request.tool(), params); if (approvedMutation) { - rawResult = confirmFrameworkActionIfNeeded(rawResult); + rawResult = confirmPhoneActionIfNeeded(rawResult); } RuntimeToolResult result = normalizeToolResult(request, rawResult, auditId, taskId); if (approvedMutation) { @@ -323,7 +337,9 @@ private RuntimeConfirmationResolution timeoutPendingConfirmationLocked( } private RuntimeToolResult validate(RuntimeToolRequest request, String auditId) { - if (mAgentManager == null) { + if (mPhoneGateway == null || !mPhoneGateway.isAvailable()) { + // Keep the established result code stable during this + // dependency-only refactor. return RuntimeToolResult.error(request.requestId(), "framework_unavailable", "OpenPhone framework service is not available."); } @@ -388,7 +404,7 @@ private String startRuntimeTask(RuntimeToolRequest request) { .put("approved_capabilities", new JSONArray() .put("tasks.observe") .put("screen.read.visible")); - JSONObject response = parseObject(mAgentManager.startTask(task.toString())); + JSONObject response = parseObject(mPhoneGateway.startTask(task.toString())); return response.optString("task_id", ""); } catch (JSONException | RuntimeException e) { Log.w(TAG, "runtime task start failed runtime=" + request.runtime() @@ -397,7 +413,7 @@ private String startRuntimeTask(RuntimeToolRequest request) { } } - private String confirmFrameworkActionIfNeeded(String rawResult) { + private String confirmPhoneActionIfNeeded(String rawResult) { JSONObject parsed = parseObject(rawResult); String pendingActionId = findStringRecursive(parsed, "pending_action_id"); if (pendingActionId.isEmpty() || "null".equals(pendingActionId)) { @@ -409,7 +425,7 @@ private String confirmFrameworkActionIfNeeded(String rawResult) { return rawResult; } try { - return mAgentManager.confirmAction(pendingActionId, true); + return mPhoneGateway.confirmAction(pendingActionId, true); } catch (RuntimeException e) { return errorJson("framework_confirmation_failed", e.getClass().getSimpleName()); } diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawRuntimeAdapter.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawRuntimeAdapter.java index b6ca708..1f536bd 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawRuntimeAdapter.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawRuntimeAdapter.java @@ -17,11 +17,14 @@ import org.openphone.assistant.runtime.RuntimeToolResult; import org.openphone.assistant.runtime.RuntimeTransport; import org.openphone.assistant.session.PhoneSessionStore; +import org.openphone.assistant.surface.AssistantOutput; import java.net.URI; import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; @@ -50,6 +53,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) { return size() > MAX_SEEN_RUNTIME_MESSAGES; } }; + private final Set mActiveSessionKeys = new LinkedHashSet<>(); private final AtomicReference mStatus = new AtomicReference<>("idle"); private volatile String mConnectNonce = ""; private volatile boolean mConnectSent; @@ -106,6 +110,7 @@ public void onClosed(String reason) { if (!status().startsWith("auth_failed")) { mStatus.set("closed"); } + emitDisconnectedSessions(reason); } @Override @@ -146,7 +151,7 @@ public synchronized void sendEvent(RuntimeEvent event) { if (mClient == null || !"connected".equals(status())) { return; } - sendRpc("node.event", buildNodeEventParams(event.event(), event.payload())); + sendRpc("node.event", buildNodeEventParams(wireEvent(event.event()), event.payload())); } @Override @@ -210,6 +215,7 @@ cleanAttentionId, cleanSource(source), message, return ""; } subscribeToSessionFanout(sessionKey); + mActiveSessionKeys.add(sessionKey); sendRpc("node.event", buildNodeEventParams("agent.request", payload)); return sessionKey; } @@ -306,7 +312,13 @@ private String buildAgentRequestMessage(String phoneSessionId, String attentionI .append("or openphone.notifications.open.\n") .append("- Mutating phone actions may require local confirmation on ") .append("the Android device; wait for the tool result and report ") - .append("whether it was approved, denied, or completed."); + .append("whether it was approved, denied, or completed.\n") + .append("- When a visual result is materially clearer than prose and the ") + .append("runtime transport supports it, send an explicit ") + .append("openphone.assistant_output.v1 object through ") + .append("openphone.surface.present or openphone.surface.replace. ") + .append("Do not paste a surface JSON document into ordinary assistant text; ") + .append("plain text remains the fallback."); if (includeScreen) { out.append("\n- This request asked to include the screen. Before answering ") .append("screen/app/UI questions, read the phone screen with ") @@ -440,6 +452,18 @@ private void handleMessage(String raw) { } return; } + if ("node.event".equals(event)) { + String nestedEvent = payload.optString("event", ""); + JSONObject nestedPayload = payload.optJSONObject("payload"); + if (nestedPayload == null) { + nestedPayload = OpenClawJson.parseObject( + payload.optString("payloadJSON", "{}")); + } + if (isSurfaceWireEvent(nestedEvent)) { + handleSurfaceEvent(nestedEvent, nestedPayload); + } + return; + } if ("chat".equals(event)) { handleChatEvent(payload); return; @@ -450,6 +474,10 @@ private void handleMessage(String raw) { } if ("node.invoke.request".equals(event)) { handleInvoke(payload); + return; + } + if (isSurfaceWireEvent(event)) { + handleSurfaceEvent(event, payload); } return; } @@ -480,6 +508,11 @@ private void handleChatEvent(JSONObject payload) { if (!terminal) { return; } + AssistantOutput output = explicitAssistantOutput(payload); + if (output != null) { + emitRuntimeOutput(sessionKeyFromPayload(payload), output, true); + return; + } String text = extractMessageText(payload); if ("error".equals(state)) { String errorText = OpenClawJson.firstNonEmpty(payload.optString("errorMessage", ""), @@ -498,6 +531,7 @@ private void handleChatEvent(JSONObject payload) { return; } emitRuntimeMessage(sessionKey, text, true); + markSessionTerminal(sessionKey); } private void handleAgentEvent(JSONObject payload) { @@ -507,6 +541,11 @@ private void handleAgentEvent(JSONObject payload) { String stream = payload.optString("stream", ""); String sessionKey = sessionKeyFromPayload(payload); if ("assistant".equals(stream)) { + AssistantOutput output = explicitAssistantOutput(payload); + if (output != null) { + emitRuntimeOutput(sessionKey, output, false); + return; + } String text = agentText(payload, payload.optJSONObject("data")); if (!text.isEmpty()) { rememberLatestAgentText(sessionKey, text); @@ -534,6 +573,7 @@ private void handleAgentEvent(JSONObject payload) { emitRuntimeMessageOnce("agent:" + sessionKey + ":final:" + text, sessionKey, text, true); } + markSessionTerminal(sessionKey); } return; } @@ -542,6 +582,7 @@ private void handleAgentEvent(JSONObject payload) { String displayError = openClawErrorForUser(error); emitRuntimeMessageOnce("agent:" + sessionKey + ":terminal:" + displayError, sessionKey, displayError, true); + markSessionTerminal(sessionKey); } } @@ -586,6 +627,107 @@ private void emitRuntimeMessage(String sessionKey, String message, boolean termi } } + private void emitRuntimeOutput(String sessionKey, AssistantOutput output, boolean terminal) { + if (mCallback == null || output == null) { + return; + } + try { + mCallback.onRuntimeOutput(name(), sessionKey == null ? "" : sessionKey, + output, terminal); + } catch (RuntimeException e) { + Log.w(TAG, "runtime output callback failed", e); + } + } + + private void handleSurfaceEvent(String wireEvent, JSONObject payload) { + if (mCallback == null || payload == null) { + return; + } + String event = protocolSurfaceEvent(wireEvent); + String sessionKey = sessionKeyFromPayload(payload); + JSONObject normalized = OpenClawJson.parseObject(payload.toString()); + if ("runtime.surface.present".equals(event) + || "runtime.surface.replace".equals(event)) { + AssistantOutput output = explicitAssistantOutput(payload); + if (output == null) { + Log.w(TAG, "Ignoring surface event without explicit assistant output"); + return; + } + try { + normalized.put("output", output.toJson()); + } catch (JSONException ignored) { + } + } + try { + mCallback.onRuntimeSurfaceEvent(name(), sessionKey, event, normalized); + } catch (RuntimeException e) { + Log.w(TAG, "runtime surface callback failed", e); + } + } + + private static AssistantOutput explicitAssistantOutput(JSONObject payload) { + if (payload == null) { + return null; + } + AssistantOutput direct = AssistantOutput.fromJson(payload); + if (direct != null) { + return direct; + } + for (String key : new String[] { + "output", "openphone_output", "openphoneAssistantOutput" + }) { + AssistantOutput nested = AssistantOutput.fromJson(payload.optJSONObject(key)); + if (nested != null) { + return nested; + } + } + JSONObject message = payload.optJSONObject("message"); + if (message != null) { + AssistantOutput nested = explicitAssistantOutput(message); + if (nested != null) { + return nested; + } + JSONObject metadata = message.optJSONObject("metadata"); + nested = AssistantOutput.fromJson(metadata == null + ? null : metadata.optJSONObject("openphone_output")); + if (nested != null) { + return nested; + } + } + JSONObject data = payload.optJSONObject("data"); + return data == null ? null : AssistantOutput.fromJson( + data.optJSONObject("openphone_output")); + } + + private static boolean isSurfaceWireEvent(String event) { + return "openphone.surface.present".equals(event) + || "openphone.surface.replace".equals(event) + || "openphone.surface.dismiss".equals(event) + || "runtime.surface.present".equals(event) + || "runtime.surface.replace".equals(event) + || "runtime.surface.dismiss".equals(event); + } + + private static String protocolSurfaceEvent(String event) { + if (event != null && event.startsWith("openphone.surface.")) { + return "runtime.surface." + event.substring("openphone.surface.".length()); + } + return event == null ? "" : event; + } + + private static String wireEvent(String event) { + if ("runtime.surface.action_result".equals(event)) { + return "openphone.surface.action_result"; + } + if ("phone.surface.action_invoked".equals(event)) { + return "openphone.surface.action_invoked"; + } + if ("phone.surface.dismissed".equals(event)) { + return "openphone.surface.dismissed"; + } + return event == null ? "" : event; + } + private static String extractMessageText(JSONObject payload) { JSONObject message = payload.optJSONObject("message"); if (message == null) { @@ -753,6 +895,29 @@ private synchronized String takeLatestAgentText(String sessionKey) { return text == null ? "" : text; } + private synchronized void markSessionTerminal(String sessionKey) { + mActiveSessionKeys.remove(sessionKey == null ? "" : sessionKey.trim()); + } + + private void emitDisconnectedSessions(String reason) { + Set sessions; + synchronized (this) { + sessions = new LinkedHashSet<>(mActiveSessionKeys); + mActiveSessionKeys.clear(); + } + if (sessions.isEmpty()) { + return; + } + String detail = safePromptText(reason, 120); + String message = detail.isEmpty() + ? "OpenClaw disconnected before completing the request." + : "OpenClaw disconnected before completing the request: " + detail; + for (String sessionKey : sessions) { + emitRuntimeMessageOnce( + "disconnect:" + sessionKey, sessionKey, message, true); + } + } + private String runtimeSessionId(JSONObject payload, JSONObject params) { String sessionKey = OpenClawJson.firstString(payload, params, "sessionKey", "session_key"); diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/session/PhoneSessionStore.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/session/PhoneSessionStore.java index 3f6fec6..c5a7199 100644 --- a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/session/PhoneSessionStore.java +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/session/PhoneSessionStore.java @@ -8,6 +8,8 @@ import org.json.JSONException; import org.json.JSONObject; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; public final class PhoneSessionStore { @@ -101,16 +103,9 @@ public synchronized PhoneExecutionSession find(String phoneSessionId) { } public synchronized String listJson(int limit) { - JSONArray sessions = readSessions(); JSONArray out = new JSONArray(); - int emitted = 0; - for (int i = sessions.length() - 1; i >= 0 && emitted < Math.max(1, limit); i--) { - JSONObject session = sessions.optJSONObject(i); - if (session == null) { - continue; - } - out.put(session); - emitted++; + for (PhoneExecutionSession session : list(limit)) { + out.put(session.toJson()); } try { return new JSONObject().put("sessions", out).toString(); @@ -119,6 +114,20 @@ public synchronized String listJson(int limit) { } } + public synchronized List list(int limit) { + JSONArray sessions = readSessions(); + ArrayList out = new ArrayList<>(); + int boundedLimit = Math.max(1, Math.min(limit, MAX_SESSIONS)); + for (int i = sessions.length() - 1; i >= 0 && out.size() < boundedLimit; i--) { + PhoneExecutionSession session = + PhoneExecutionSession.fromJson(sessions.optJSONObject(i)); + if (session != null) { + out.add(session); + } + } + return out; + } + public static String extractPhoneSessionId(String runtimeSessionId) { String clean = clean(runtimeSessionId); if (clean.isEmpty()) { diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/AdaptiveSurface.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/AdaptiveSurface.java new file mode 100644 index 0000000..2daa83d --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/AdaptiveSurface.java @@ -0,0 +1,110 @@ +package org.openphone.assistant.surface; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Parsed, immutable phone-owned Adaptive Surface V1 document. */ +public final class AdaptiveSurface { + public static final String SCHEMA = "openphone.surface.v1"; + + public final String surfaceId; + public final int revision; + public final String sessionId; + public final String runtime; + public final String title; + public final String presentation; + public final long expiresAtMillis; + public final String sensitivity; + public final long createdAtMillis; + public final SurfaceComponent body; + public final Map actions; + public final JSONArray artifacts; + + private final JSONObject mJson; + + private AdaptiveSurface(JSONObject json, String surfaceId, int revision, String sessionId, + String runtime, String title, String presentation, long expiresAtMillis, + String sensitivity, long createdAtMillis, SurfaceComponent body, + Map actions, JSONArray artifacts) { + mJson = SurfaceComponent.copy(json); + this.surfaceId = clean(surfaceId); + this.revision = revision; + this.sessionId = clean(sessionId); + this.runtime = clean(runtime); + this.title = clean(title); + this.presentation = clean(presentation); + this.expiresAtMillis = expiresAtMillis; + this.sensitivity = clean(sensitivity); + this.createdAtMillis = createdAtMillis; + this.body = body; + this.actions = Collections.unmodifiableMap(actions); + this.artifacts = SurfaceComponent.copy(artifacts); + } + + public static AdaptiveSurface fromJson(String raw) { + if (raw == null || raw.trim().isEmpty()) { + return null; + } + try { + return fromJson(new JSONObject(raw)); + } catch (JSONException e) { + return null; + } + } + + public static AdaptiveSurface fromJson(JSONObject json) { + if (json == null) { + return null; + } + JSONObject actionsJson = json.optJSONObject("actions"); + Map actions = new LinkedHashMap<>(); + if (actionsJson != null) { + JSONArray names = actionsJson.names(); + if (names != null) { + for (int i = 0; i < names.length(); i++) { + String id = names.optString(i, ""); + SurfaceAction action = SurfaceAction.fromJson( + id, actionsJson.optJSONObject(id)); + if (action != null) { + actions.put(id, action); + } + } + } + } + JSONArray artifacts = json.optJSONArray("artifacts"); + if (artifacts == null) { + artifacts = new JSONArray(); + } + return new AdaptiveSurface( + json, + json.optString("surface_id", ""), + json.optInt("revision", 0), + json.optString("session_id", ""), + json.optString("runtime", ""), + json.optString("title", ""), + json.optString("presentation", ""), + json.optLong("expires_at", 0L), + json.optString("sensitivity", ""), + json.optLong("created_at", 0L), + SurfaceComponent.fromJson(json.optJSONObject("body")), + actions, + artifacts); + } + + public JSONObject toJson() { + return SurfaceComponent.copy(mJson); + } + + public boolean isExpired(long nowMillis) { + return expiresAtMillis > 0L && expiresAtMillis <= nowMillis; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/AssistantOutput.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/AssistantOutput.java new file mode 100644 index 0000000..10c1fef --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/AssistantOutput.java @@ -0,0 +1,109 @@ +package org.openphone.assistant.surface; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +/** Explicit runtime-neutral assistant output envelope. */ +public final class AssistantOutput { + public static final String SCHEMA = "openphone.assistant_output.v1"; + + public final String sessionId; + public final String speech; + public final String text; + public final AdaptiveSurface surface; + public final JSONObject backgroundRun; + + private final JSONObject mJson; + + private AssistantOutput(JSONObject json, String sessionId, String speech, String text, + AdaptiveSurface surface, JSONObject backgroundRun) { + mJson = SurfaceComponent.copy(json); + this.sessionId = clean(sessionId); + this.speech = clean(speech); + this.text = clean(text); + this.surface = surface; + this.backgroundRun = SurfaceComponent.copy(backgroundRun); + } + + /** + * Parses only an object already identified as an output envelope. + * Callers must never pass arbitrary assistant prose here. + */ + public static AssistantOutput fromJson(JSONObject json) { + if (json == null || !SCHEMA.equals(json.optString("schema", ""))) { + return null; + } + Set allowed = new HashSet<>(Arrays.asList( + "schema", "session_id", "speech", "text", "surface", "background_run")); + Iterator keys = json.keys(); + while (keys.hasNext()) { + if (!allowed.contains(keys.next())) { + return null; + } + } + if (!(json.opt("session_id") instanceof String) + || !(json.opt("speech") instanceof String) + || !(json.opt("text") instanceof String)) { + return null; + } + String sessionId = clean(json.optString("session_id", "")); + String speech = clean(json.optString("speech", "")); + String text = clean(json.optString("text", "")); + if (sessionId.isEmpty() || sessionId.length() > 160 + || speech.length() > 4000 || text.length() > 12000) { + return null; + } + AdaptiveSurface surface = AdaptiveSurface.fromJson(json.optJSONObject("surface")); + if (json.has("surface") && !JSONObject.NULL.equals(json.opt("surface")) + && surface == null) { + return null; + } + if (surface != null && !sessionId.equals(surface.sessionId)) { + return null; + } + Object background = json.opt("background_run"); + if (background != null && !JSONObject.NULL.equals(background) + && !(background instanceof JSONObject)) { + return null; + } + return new AssistantOutput( + json, + sessionId, + speech, + text, + surface, + background instanceof JSONObject ? (JSONObject) background : new JSONObject()); + } + + public static AssistantOutput builtin(String sessionId, String speech, String text, + AdaptiveSurface surface) { + JSONObject out = new JSONObject(); + try { + out.put("schema", SCHEMA) + .put("session_id", clean(sessionId)) + .put("speech", clean(speech)) + .put("text", clean(text)) + .put("surface", surface == null ? JSONObject.NULL : surface.toJson()) + .put("background_run", JSONObject.NULL); + } catch (JSONException ignored) { + } + return fromJson(out); + } + + public JSONObject toJson() { + return SurfaceComponent.copy(mJson); + } + + public String displayText() { + return text.isEmpty() ? speech : text; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/DeterministicSurfaceFactory.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/DeterministicSurfaceFactory.java new file mode 100644 index 0000000..7fba20d --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/DeterministicSurfaceFactory.java @@ -0,0 +1,340 @@ +package org.openphone.assistant.surface; + +import android.content.Context; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.actions.ToolCatalog; + +import java.util.UUID; + +/** + * Deterministic phone renderer input for trusted calendar/message/notification + * tool results. This does not ask a model to generate layout. + */ +public final class DeterministicSurfaceFactory { + private static final long DEFAULT_TTL_MILLIS = 30L * 60L * 1000L; + + private final SurfaceArtifactStore mArtifacts; + + public DeterministicSurfaceFactory(Context context) { + mArtifacts = new SurfaceArtifactStore(context); + } + + public AdaptiveSurface fromAgentResult(String agentResultJson, String sessionId, + String runtime) { + JSONObject result = object(agentResultJson); + JSONArray steps = result.optJSONArray("steps"); + if (steps == null) { + return null; + } + for (int i = steps.length() - 1; i >= 0; i--) { + JSONObject step = steps.optJSONObject(i); + if (step == null) { + continue; + } + String tool = step.optString("tool", ""); + if (!isSupported(tool)) { + continue; + } + JSONObject toolResult = objectValue(step.opt("tool_result")); + AdaptiveSurface surface = fromToolResult(tool, toolResult, sessionId, runtime); + if (surface != null) { + return surface; + } + } + return null; + } + + public AdaptiveSurface fromToolResult(String tool, JSONObject toolResult, + String sessionId, String runtime) { + String cleanSession = clean(sessionId); + String cleanRuntime = clean(runtime); + if (cleanSession.isEmpty() || cleanRuntime.isEmpty() || toolResult == null) { + return null; + } + long now = System.currentTimeMillis(); + long expiresAt = now + DEFAULT_TTL_MILLIS; + String capability = ToolCatalog.get().capabilityForTool(tool); + String sensitivity = tool.startsWith("calendar_") ? "personal" : "sensitive"; + String artifactId = mArtifacts.put( + cleanSession, cleanRuntime, tool, capability, sensitivity, + toolResult, expiresAt); + JSONObject surface; + if ("calendar_search".equals(tool) + && "calendar.search.results".equals(toolResult.optString("status", ""))) { + surface = calendarSurface(toolResult, cleanSession, cleanRuntime, now, expiresAt); + } else if (("messages_search".equals(tool) + && "messages.search.results".equals(toolResult.optString("status", ""))) + || ("messages_summary".equals(tool) + && "messages.summary".equals(toolResult.optString("status", "")))) { + surface = messageSurface(tool, toolResult, cleanSession, cleanRuntime, now, expiresAt); + } else if (("notifications_summary".equals(tool) + && "notifications.summary".equals(toolResult.optString("status", ""))) + || ("notifications_list".equals(tool) + && toolResult.optString("status", "").startsWith("notifications."))) { + surface = notificationSurface( + toolResult, cleanSession, cleanRuntime, now, expiresAt); + } else { + return null; + } + if (!artifactId.isEmpty()) { + try { + surface.put("artifacts", new JSONArray().put(new JSONObject() + .put("artifact_id", artifactId) + .put("path", "$"))); + } catch (JSONException ignored) { + } + } + return AdaptiveSurface.fromJson(surface); + } + + private static JSONObject calendarSurface(JSONObject result, String sessionId, + String runtime, long createdAt, long expiresAt) { + JSONArray events = result.optJSONArray("events"); + JSONArray items = new JSONArray(); + JSONObject actions = new JSONObject(); + int count = events == null ? 0 : Math.min(events.length(), 12); + for (int i = 0; i < count; i++) { + JSONObject event = events.optJSONObject(i); + if (event == null) { + continue; + } + String title = first(event.optString("title", ""), "Untitled event"); + String subtitle = join( + event.optString("start_local", ""), + event.optString("location", "")); + String actionId = "calendar_event_" + i; + try { + items.put(new JSONObject() + .put("type", "list_item") + .put("title", title) + .put("subtitle", subtitle) + .put("icon", "calendar") + .put("action_id", actionId)); + actions.put(actionId, new JSONObject() + .put("label", "Find event") + .put("tool", "calendar_search") + .put("params", new JSONObject() + .put("query", title) + .put("limit", 8)) + .put("reason", "User selected an event from the phone agenda")); + } catch (JSONException ignored) { + } + } + JSONObject body = column(new JSONArray() + .put(text("headline", count == 0 ? "No matching events" : "Your agenda")) + .put(new JSONObjectBuilder("list").array("items", items).build())); + return root("Agenda", "personal", sessionId, runtime, createdAt, expiresAt, body, actions); + } + + private static JSONObject messageSurface(String tool, JSONObject result, String sessionId, + String runtime, long createdAt, long expiresAt) { + JSONArray source = "messages_summary".equals(tool) + ? result.optJSONArray("threads") : result.optJSONArray("messages"); + JSONArray children = new JSONArray() + .put(disclosure("messages.read", "From messages stored on this phone")); + int count = source == null ? 0 : Math.min(source.length(), 10); + JSONObject actions = new JSONObject(); + for (int i = 0; i < count; i++) { + JSONObject item = source.optJSONObject(i); + if (item == null) { + continue; + } + String address = first(item.optString("address", ""), "Unknown sender"); + String body; + if ("messages_summary".equals(tool)) { + JSONArray samples = item.optJSONArray("samples"); + JSONObject sample = samples == null ? null : samples.optJSONObject(0); + body = sample == null ? "" : sample.optString("body", ""); + } else { + body = item.optString("body", ""); + } + String actionId = "message_thread_" + i; + try { + children.put(new JSONObjectBuilder("card") + .array("children", new JSONArray() + .put(text("title", address)) + .put(text("body", body)) + .put(new JSONObjectBuilder("button") + .string("label", "Find thread") + .string("action_id", actionId) + .string("content_description", + "Find message thread with " + address) + .build())) + .build()); + actions.put(actionId, new JSONObject() + .put("label", "Find thread") + .put("tool", "messages_search") + .put("params", new JSONObject() + .put("query", address) + .put("limit", 8)) + .put("reason", "User selected a message summary item")); + } catch (JSONException ignored) { + } + } + if (count == 0) { + children.put(text("body", first(result.optString("summary", ""), + "No matching messages."))); + } + return root("Messages", "sensitive", sessionId, runtime, createdAt, expiresAt, + column(children), actions); + } + + private static JSONObject notificationSurface(JSONObject result, String sessionId, + String runtime, long createdAt, long expiresAt) { + JSONArray groups = result.optJSONArray("groups"); + JSONArray children = new JSONArray() + .put(disclosure("notifications.read", "From recent phone notifications")); + JSONObject actions = new JSONObject(); + int count = groups == null ? 0 : Math.min(groups.length(), 10); + for (int i = 0; i < count; i++) { + JSONObject group = groups.optJSONObject(i); + if (group == null) { + continue; + } + String app = first(group.optString("app", ""), "Notification"); + String title = first(group.optString("title", ""), app); + String sample = group.optString("sample_text", ""); + String actionId = "notification_" + i; + try { + children.put(new JSONObjectBuilder("list_item") + .string("title", title) + .string("subtitle", sample) + .string("icon", "notification") + .string("action_id", actionId) + .build()); + actions.put(actionId, new JSONObject() + .put("label", "Open notification") + .put("tool", "notifications_open") + .put("params", new JSONObject().put("query", title)) + .put("reason", "User selected a notification summary item") + .put("requires_confirmation", true)); + } catch (JSONException ignored) { + } + } + if (count == 0) { + children.put(text("body", first(result.optString("summary", ""), + "No recent notifications."))); + } + return root("Notifications", "sensitive", sessionId, runtime, createdAt, expiresAt, + column(children), actions); + } + + private static JSONObject root(String title, String sensitivity, String sessionId, + String runtime, long createdAt, long expiresAt, JSONObject body, + JSONObject actions) { + JSONObject root = new JSONObject(); + try { + root.put("schema", AdaptiveSurface.SCHEMA) + .put("surface_id", "surface-" + UUID.randomUUID()) + .put("revision", 1) + .put("session_id", sessionId) + .put("runtime", runtime) + .put("title", title) + .put("presentation", "full") + .put("expires_at", expiresAt) + .put("sensitivity", sensitivity) + .put("created_at", createdAt) + .put("body", body) + .put("actions", actions); + } catch (JSONException ignored) { + } + return root; + } + + private static JSONObject column(JSONArray children) { + return new JSONObjectBuilder("column") + .string("spacing", "md") + .array("children", children) + .build(); + } + + private static JSONObject text(String style, String value) { + return new JSONObjectBuilder("text") + .string("style", style) + .string("text", truncate(value, 1000)) + .build(); + } + + private static JSONObject disclosure(String capability, String value) { + return new JSONObjectBuilder("capability_disclosure") + .string("capability", capability) + .string("text", value) + .build(); + } + + private static String join(String first, String second) { + String left = clean(first); + String right = clean(second); + if (left.isEmpty()) return right; + if (right.isEmpty()) return left; + return left + " · " + right; + } + + private static String first(String value, String fallback) { + String clean = clean(value); + return clean.isEmpty() ? fallback : clean; + } + + private static String truncate(String value, int max) { + String clean = clean(value); + return clean.length() <= max ? clean : clean.substring(0, max - 1) + "…"; + } + + private static boolean isSupported(String tool) { + return "calendar_search".equals(tool) + || "messages_search".equals(tool) + || "messages_summary".equals(tool) + || "notifications_list".equals(tool) + || "notifications_summary".equals(tool); + } + + private static JSONObject object(String raw) { + try { + return new JSONObject(raw == null || raw.trim().isEmpty() ? "{}" : raw); + } catch (JSONException e) { + return new JSONObject(); + } + } + + private static JSONObject objectValue(Object value) { + if (value instanceof JSONObject) { + return SurfaceComponent.copy((JSONObject) value); + } + return object(value instanceof String ? (String) value : ""); + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } + + private static final class JSONObjectBuilder { + private final JSONObject mObject = new JSONObject(); + + JSONObjectBuilder(String type) { + string("type", type); + } + + JSONObjectBuilder string(String key, String value) { + try { + mObject.put(key, truncate(value, 1000)); + } catch (JSONException ignored) { + } + return this; + } + + JSONObjectBuilder array(String key, JSONArray value) { + try { + mObject.put(key, value == null ? new JSONArray() : value); + } catch (JSONException ignored) { + } + return this; + } + + JSONObject build() { + return mObject; + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceAction.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceAction.java new file mode 100644 index 0000000..2d18b93 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceAction.java @@ -0,0 +1,56 @@ +package org.openphone.assistant.surface; + +import org.json.JSONException; +import org.json.JSONObject; + +/** A declarative action binding. Execution always goes through the phone tool bridge. */ +public final class SurfaceAction { + public final String id; + public final String label; + public final String tool; + public final JSONObject params; + public final String reason; + public final boolean requiresConfirmation; + + public SurfaceAction(String id, String label, String tool, JSONObject params, + String reason, boolean requiresConfirmation) { + this.id = clean(id); + this.label = clean(label); + this.tool = clean(tool); + this.params = SurfaceComponent.copy(params); + this.reason = clean(reason); + this.requiresConfirmation = requiresConfirmation; + } + + public static SurfaceAction fromJson(String id, JSONObject json) { + if (json == null) { + return null; + } + return new SurfaceAction( + id, + json.optString("label", ""), + json.optString("tool", ""), + json.optJSONObject("params"), + json.optString("reason", ""), + json.optBoolean("requires_confirmation", false)); + } + + public JSONObject toJson() { + JSONObject out = new JSONObject(); + try { + out.put("label", label) + .put("tool", tool) + .put("params", SurfaceComponent.copy(params)) + .put("reason", reason); + if (requiresConfirmation) { + out.put("requires_confirmation", true); + } + } catch (JSONException ignored) { + } + return out; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceActionDispatcher.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceActionDispatcher.java new file mode 100644 index 0000000..1a61c03 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceActionDispatcher.java @@ -0,0 +1,184 @@ +package org.openphone.assistant.surface; + +import android.content.Context; + +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.actions.ActionRegistry; +import org.openphone.assistant.actions.ToolCatalog; +import org.openphone.assistant.platform.PhoneToolGateway; +import org.openphone.assistant.runtime.RuntimeIdentity; +import org.openphone.assistant.runtime.RuntimeConfirmationResolution; +import org.openphone.assistant.runtime.RuntimeToolBridge; +import org.openphone.assistant.runtime.RuntimeToolRequest; +import org.openphone.assistant.runtime.RuntimeToolResult; + +import java.util.Iterator; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * Resolves a revision-bound action then delegates to RuntimeToolBridge. + * + * The renderer never executes intents, framework actions, or runtime code + * directly. Existing policy, confirmation, grants, idempotency, and audit + * behavior therefore remain authoritative. + */ +public final class SurfaceActionDispatcher { + private final SurfaceRepository mRepository; + private final SurfaceEventLog mEvents; + private final RuntimeToolBridge mToolBridge; + private final Context mContext; + private final Map mPending = new HashMap<>(); + + public SurfaceActionDispatcher(Context context, PhoneToolGateway phoneGateway) { + Context app = context.getApplicationContext(); + mContext = app; + mRepository = new SurfaceRepository(app); + mEvents = new SurfaceEventLog(app); + mToolBridge = phoneGateway == null || !phoneGateway.isAvailable() + ? null : new RuntimeToolBridge(app, phoneGateway); + } + + public synchronized SurfaceActionResult invoke(String surfaceId, int revision, + String actionId, JSONObject input, String idempotencyKey) { + SurfaceMutationResult resolved = + mRepository.resolveInvocation(surfaceId, revision, actionId); + if (!resolved.ok || resolved.surface == null) { + return SurfaceActionResult.error(resolved.code, resolved.message); + } + AdaptiveSurface surface = resolved.surface; + SurfaceAction action = surface.actions.get(clean(actionId)); + if (action == null) { + return SurfaceActionResult.error("action_not_found", "Surface action was not found."); + } + if (mToolBridge == null) { + return SurfaceActionResult.error( + "agent_service_unavailable", "OpenPhone agent service is unavailable."); + } + JSONObject params = SurfaceComponent.copy(action.params); + SurfaceActionResult inputResult = mergeValidatedInput(action.tool, params, input); + if (inputResult != null) { + return inputResult; + } + try { + params.put("reason", action.reason) + .put("_surface_id", surface.surfaceId) + .put("_surface_revision", surface.revision) + .put("_surface_action_id", action.id); + } catch (JSONException e) { + return SurfaceActionResult.error("params_invalid", "Could not bind surface action."); + } + String key = clean(idempotencyKey); + if (key.isEmpty()) { + key = "surface:" + surface.surfaceId + ":" + surface.revision + + ":" + action.id + ":" + UUID.randomUUID(); + } + RuntimeToolRequest request = new RuntimeToolRequest( + "surface-request-" + UUID.randomUUID(), + surface.runtime, + "surface:" + surface.sessionId, + surface.sessionId, + new RuntimeIdentity(surface.runtime, "phone_surface", surface.runtime), + action.tool, + params, + action.reason, + "confirm", + key, + 30000L, + ""); + JSONObject eventPayload = new JSONObject(); + try { + eventPayload.put("action_id", action.id) + .put("tool", action.tool) + .put("idempotency_key", key); + } catch (JSONException ignored) { + } + mEvents.record(SurfaceEventLog.ACTION_INVOKED, surface, + "Invoked " + action.label, eventPayload); + SurfaceRuntimeNotifier.actionInvoked(mContext, surface, action.id, key); + RuntimeToolResult runtimeResult = mToolBridge.execute(request); + SurfaceActionResult result = SurfaceActionResult.fromRuntime(runtimeResult); + String confirmationId = result.result.optString("confirmation_id", ""); + if (!confirmationId.isEmpty()) { + mPending.put(confirmationId, new PendingSurfaceAction(surface, action.id)); + } + mEvents.record(SurfaceEventLog.ACTION_RESULT, surface, + result.status + (result.code.isEmpty() ? "" : ": " + result.code), + result.toJson()); + SurfaceRuntimeNotifier.actionResult(mContext, surface, action.id, result); + return result; + } + + public synchronized SurfaceActionResult resolveConfirmation( + String confirmationId, boolean approved) { + if (mToolBridge == null) { + return SurfaceActionResult.error( + "agent_service_unavailable", "OpenPhone agent service is unavailable."); + } + RuntimeConfirmationResolution resolution = + mToolBridge.resolveConfirmation(clean(confirmationId), approved); + if (resolution == null || resolution.result() == null) { + return SurfaceActionResult.error( + "confirmation_result_missing", "Confirmation returned no result."); + } + SurfaceActionResult result = SurfaceActionResult.fromRuntime(resolution.result()); + PendingSurfaceAction pending = mPending.remove(clean(confirmationId)); + if (pending != null) { + SurfaceRuntimeNotifier.actionResult( + mContext, pending.surface, pending.actionId, result); + } + return result; + } + + private static SurfaceActionResult mergeValidatedInput(String tool, JSONObject params, + JSONObject input) { + if (input == null || input.length() == 0) { + return null; + } + if (input.length() > 16) { + return SurfaceActionResult.error("input_too_large", "Surface input has too many keys."); + } + ActionRegistry.ActionMetadata metadata = null; + for (ActionRegistry.ActionMetadata candidate : ToolCatalog.get().tools()) { + if (tool.equals(candidate.modelTool)) { + metadata = candidate; + break; + } + } + if (metadata == null) { + return SurfaceActionResult.error("tool_metadata_missing", "Tool metadata is missing."); + } + JSONObject properties = metadata.inputSchema.optJSONObject("properties"); + Iterator keys = input.keys(); + try { + while (keys.hasNext()) { + String key = keys.next(); + if (key.startsWith("_") || "reason".equals(key) + || properties == null || !properties.has(key)) { + return SurfaceActionResult.error( + "input_key_invalid", "Surface input key is not allowed: " + key); + } + params.put(key, input.opt(key)); + } + } catch (JSONException e) { + return SurfaceActionResult.error("input_invalid", "Could not merge surface input."); + } + return null; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } + + private static final class PendingSurfaceAction { + final AdaptiveSurface surface; + final String actionId; + + PendingSurfaceAction(AdaptiveSurface surface, String actionId) { + this.surface = surface; + this.actionId = actionId; + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceActionResult.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceActionResult.java new file mode 100644 index 0000000..ad7f6ce --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceActionResult.java @@ -0,0 +1,48 @@ +package org.openphone.assistant.surface; + +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.runtime.RuntimeToolResult; + +/** Result returned to the renderer/runtime after a bound surface action. */ +public final class SurfaceActionResult { + public final String status; + public final String code; + public final String message; + public final JSONObject result; + + private SurfaceActionResult(String status, String code, String message, JSONObject result) { + this.status = clean(status); + this.code = clean(code); + this.message = clean(message); + this.result = SurfaceComponent.copy(result); + } + + public static SurfaceActionResult fromRuntime(RuntimeToolResult result) { + if (result == null) { + return error("runtime_result_missing", "Phone tool returned no result."); + } + return new SurfaceActionResult( + result.status(), result.errorCode(), result.errorMessage(), result.result()); + } + + public static SurfaceActionResult error(String code, String message) { + return new SurfaceActionResult("error", code, message, new JSONObject()); + } + + public JSONObject toJson() { + JSONObject out = new JSONObject(); + try { + out.put("status", status) + .put("code", code) + .put("message", message) + .put("result", SurfaceComponent.copy(result)); + } catch (JSONException ignored) { + } + return out; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceArtifactStore.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceArtifactStore.java new file mode 100644 index 0000000..f5e55f3 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceArtifactStore.java @@ -0,0 +1,103 @@ +package org.openphone.assistant.surface; + +import android.content.Context; +import android.content.SharedPreferences; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.UUID; + +/** + * Small metadata/data store for trusted tool artifacts referenced by surfaces. + * + * It deliberately does not fetch URLs. Artifacts are created by phone tools, + * session-bound, sensitivity-labelled, and expire independently. + */ +public final class SurfaceArtifactStore { + private static final String PREFS = "openphone_surface_artifacts"; + private static final String KEY_ARTIFACTS = "artifacts"; + private static final int MAX_ARTIFACTS = 64; + + private final SharedPreferences mPrefs; + + public SurfaceArtifactStore(Context context) { + mPrefs = context.getApplicationContext().getSharedPreferences( + PREFS, Context.MODE_PRIVATE); + } + + public synchronized String put(String sessionId, String runtime, String tool, + String capability, String sensitivity, JSONObject data, long expiresAtMillis) { + String cleanSession = clean(sessionId); + if (cleanSession.isEmpty() || data == null) { + return ""; + } + long now = System.currentTimeMillis(); + String id = "artifact-" + UUID.randomUUID(); + JSONObject artifact = new JSONObject(); + try { + artifact.put("artifact_id", id) + .put("session_id", cleanSession) + .put("runtime", clean(runtime)) + .put("tool", clean(tool)) + .put("capability", clean(capability)) + .put("sensitivity", clean(sensitivity)) + .put("created_at", now) + .put("expires_at", expiresAtMillis) + .put("data", SurfaceComponent.copy(data)); + } catch (JSONException e) { + return ""; + } + JSONArray stored = read(); + JSONArray next = new JSONArray().put(artifact); + for (int i = 0; i < stored.length() && next.length() < MAX_ARTIFACTS; i++) { + JSONObject item = stored.optJSONObject(i); + if (item == null || expired(item, now)) { + continue; + } + next.put(item); + } + return write(next) ? id : ""; + } + + public synchronized JSONObject metadata(String artifactId) { + long now = System.currentTimeMillis(); + JSONArray artifacts = read(); + for (int i = 0; i < artifacts.length(); i++) { + JSONObject item = artifacts.optJSONObject(i); + if (item != null && clean(artifactId).equals(item.optString("artifact_id", "")) + && !expired(item, now)) { + return SurfaceComponent.copy(item); + } + } + return null; + } + + public synchronized boolean belongsToSession(String artifactId, String sessionId) { + JSONObject artifact = metadata(artifactId); + return artifact != null + && clean(sessionId).equals(artifact.optString("session_id", "")); + } + + private JSONArray read() { + try { + return new JSONArray(mPrefs.getString(KEY_ARTIFACTS, "[]")); + } catch (JSONException e) { + return new JSONArray(); + } + } + + private boolean write(JSONArray artifacts) { + return mPrefs.edit().putString(KEY_ARTIFACTS, artifacts.toString()).commit(); + } + + private static boolean expired(JSONObject artifact, long now) { + long expiresAt = artifact.optLong("expires_at", 0L); + return expiresAt > 0L && expiresAt <= now; + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceComponent.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceComponent.java new file mode 100644 index 0000000..6c1b5ab --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceComponent.java @@ -0,0 +1,64 @@ +package org.openphone.assistant.surface; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** Immutable JSON-backed node from the bounded Adaptive Surface V1 registry. */ +public final class SurfaceComponent { + private final JSONObject mJson; + + private SurfaceComponent(JSONObject json) { + mJson = copy(json); + } + + public static SurfaceComponent fromJson(JSONObject json) { + return json == null ? null : new SurfaceComponent(json); + } + + public String type() { + return mJson.optString("type", ""); + } + + public String string(String key) { + return mJson.optString(key, ""); + } + + public boolean bool(String key, boolean fallback) { + return mJson.has(key) ? mJson.optBoolean(key, fallback) : fallback; + } + + public double number(String key, double fallback) { + return mJson.has(key) ? mJson.optDouble(key, fallback) : fallback; + } + + public JSONArray array(String key) { + JSONArray value = mJson.optJSONArray(key); + return value == null ? new JSONArray() : copy(value); + } + + public JSONObject object(String key) { + JSONObject value = mJson.optJSONObject(key); + return value == null ? new JSONObject() : copy(value); + } + + public JSONObject toJson() { + return copy(mJson); + } + + static JSONObject copy(JSONObject source) { + try { + return new JSONObject(source == null ? "{}" : source.toString()); + } catch (JSONException e) { + return new JSONObject(); + } + } + + static JSONArray copy(JSONArray source) { + try { + return new JSONArray(source == null ? "[]" : source.toString()); + } catch (JSONException e) { + return new JSONArray(); + } + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceEventLog.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceEventLog.java new file mode 100644 index 0000000..cae3303 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceEventLog.java @@ -0,0 +1,87 @@ +package org.openphone.assistant.surface; + +import android.content.Context; +import android.content.SharedPreferences; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.context.ContextIndexStore; + +/** Durable, bounded surface lifecycle log plus context-index audit projection. */ +public final class SurfaceEventLog { + public static final String PRESENTED = "assistant.surface.presented"; + public static final String REPLACED = "assistant.surface.replaced"; + public static final String DISMISSED = "assistant.surface.dismissed"; + public static final String ACTION_INVOKED = "assistant.surface.action_invoked"; + public static final String ACTION_RESULT = "assistant.surface.action_result"; + public static final String REJECTED = "assistant.surface.rejected"; + + private static final String PREFS = "openphone_surface_events"; + private static final String KEY_EVENTS = "events"; + private static final int MAX_EVENTS = 100; + + private final SharedPreferences mPrefs; + private final ContextIndexStore mContextIndex; + + public SurfaceEventLog(Context context) { + Context app = context.getApplicationContext(); + mPrefs = app.getSharedPreferences(PREFS, Context.MODE_PRIVATE); + mContextIndex = new ContextIndexStore(app); + } + + public synchronized void record(String event, AdaptiveSurface surface, + String detail, JSONObject payload) { + long now = System.currentTimeMillis(); + JSONObject eventJson = new JSONObject(); + try { + eventJson.put("schema", "openphone.surface_event.v1") + .put("timestamp_ms", now) + .put("event", clean(event)) + .put("surface_id", surface == null ? "" : surface.surfaceId) + .put("revision", surface == null ? 0 : surface.revision) + .put("session_id", surface == null ? "" : surface.sessionId) + .put("runtime", surface == null ? "" : surface.runtime) + .put("detail", clean(detail)) + .put("payload", payload == null + ? new JSONObject() : SurfaceComponent.copy(payload)); + } catch (JSONException ignored) { + } + JSONArray existing = read(); + JSONArray next = new JSONArray().put(eventJson); + for (int i = 0; i < existing.length() && next.length() < MAX_EVENTS; i++) { + JSONObject item = existing.optJSONObject(i); + if (item != null) { + next.put(item); + } + } + mPrefs.edit().putString(KEY_EVENTS, next.toString()).apply(); + mContextIndex.recordAgentEvent( + clean(event), + surface == null ? "Adaptive surface" : surface.title, + clean(detail), + surface == null ? "" : surface.sessionId, + eventJson.toString()); + } + + public synchronized String listJson(int limit) { + JSONArray source = read(); + JSONArray out = new JSONArray(); + for (int i = 0; i < source.length() && out.length() < Math.max(1, limit); i++) { + out.put(source.opt(i)); + } + return out.toString(); + } + + private JSONArray read() { + try { + return new JSONArray(mPrefs.getString(KEY_EVENTS, "[]")); + } catch (JSONException e) { + return new JSONArray(); + } + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceMutationResult.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceMutationResult.java new file mode 100644 index 0000000..7727c86 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceMutationResult.java @@ -0,0 +1,29 @@ +package org.openphone.assistant.surface; + +/** Result for repository lifecycle operations. */ +public final class SurfaceMutationResult { + public final boolean ok; + public final String code; + public final String message; + public final AdaptiveSurface surface; + + private SurfaceMutationResult(boolean ok, String code, String message, + AdaptiveSurface surface) { + this.ok = ok; + this.code = clean(code); + this.message = clean(message); + this.surface = surface; + } + + public static SurfaceMutationResult ok(String code, AdaptiveSurface surface) { + return new SurfaceMutationResult(true, code, "", surface); + } + + public static SurfaceMutationResult error(String code, String message) { + return new SurfaceMutationResult(false, code, message, null); + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRenderer.kt b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRenderer.kt new file mode 100644 index 0000000..9ed96b9 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRenderer.kt @@ -0,0 +1,529 @@ +package org.openphone.assistant.surface + +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.BasicTextField +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateMapOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.role +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import org.json.JSONObject + +/** + * Renderer for the validated, bounded Surface V1 component registry. + * + * This code intentionally has no WebView, HTML, arbitrary color, absolute + * positioning, remote URL, or runtime code path. + */ +@Composable +fun AdaptiveSurfaceView( + surface: AdaptiveSurface, + actionStatus: String, + onAction: (String, JSONObject) -> Unit, + onDismiss: () -> Unit, + modifier: Modifier = Modifier, +) { + val inputs = remember(surface.surfaceId, surface.revision) { + mutableStateMapOf() + } + Column( + modifier = modifier + .fillMaxWidth() + .clip(RoundedCornerShape(28.dp)) + .background(MaterialTheme.colorScheme.surface) + .border(1.dp, Color(0xFF263246), RoundedCornerShape(28.dp)) + .padding(20.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + Column(modifier = Modifier.weight(1f)) { + Text( + surface.title.ifBlank { "OpenPhone" }, + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.SemiBold, + ) + Text( + "Generated by ${surface.runtime} · ${surface.sensitivity}", + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelSmall, + ) + } + Text( + "Close", + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier + .clip(RoundedCornerShape(14.dp)) + .clickable(onClick = onDismiss) + .semantics { + contentDescription = "Dismiss ${surface.title} surface" + role = Role.Button + } + .padding(8.dp), + ) + } + Spacer(Modifier.height(14.dp)) + Column( + modifier = Modifier + .fillMaxWidth() + .verticalScroll(rememberScrollState()), + ) { + SurfaceNode( + node = surface.body, + inputs = inputs, + onAction = { actionId -> + onAction(actionId, JSONObject(inputs.toMap())) + }, + ) + if (actionStatus.isNotBlank()) { + Text( + actionStatus, + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelMedium, + modifier = Modifier.padding(top = 14.dp), + ) + } + } + } +} + +@Composable +private fun SurfaceNode( + node: SurfaceComponent, + inputs: MutableMap, + onAction: (String) -> Unit, +) { + val children = node.array("children").components() + val items = node.array("items").components() + val spacing = spacing(node.string("spacing")) + when (node.type()) { + "column", "scroll", "list", "timeline" -> Column( + modifier = Modifier.fillMaxWidth(), + verticalArrangement = Arrangement.spacedBy(spacing), + ) { + (if (children.isNotEmpty()) children else items).forEach { + SurfaceNode(it, inputs, onAction) + } + } + + "row" -> Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(spacing), + verticalAlignment = Alignment.CenterVertically, + ) { + children.forEach { SurfaceNode(it, inputs, onAction) } + } + + "spacer" -> Spacer(Modifier.height(spacing.coerceAtLeast(8.dp))) + "divider" -> Box( + Modifier + .fillMaxWidth() + .height(1.dp) + .background(Color(0xFF263246)), + ) + + "text" -> SurfaceText(node) + "icon" -> SurfaceIcon(node) + "image" -> SafeImagePlaceholder(node) + "badge" -> SurfaceBadge(node) + "progress" -> LinearProgressIndicator( + progress = { node.number("progress", 0.0).toFloat().coerceIn(0f, 1f) }, + modifier = Modifier + .fillMaxWidth() + .semantics { + contentDescription = node.string("content_description") + .ifBlank { "Progress" } + }, + color = tone(node.string("tone")), + trackColor = MaterialTheme.colorScheme.surfaceVariant, + ) + + "card" -> Column( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(20.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant) + .padding(if (node.string("density") == "compact") 12.dp else 16.dp), + verticalArrangement = Arrangement.spacedBy(spacing.coerceAtLeast(8.dp)), + ) { + children.forEach { SurfaceNode(it, inputs, onAction) } + } + + "list_item" -> SurfaceListItem(node, onAction) + "button", "chip", "native_handoff", "confirmation" -> + SurfaceButton(node, onAction) + + "choice_group" -> SurfaceChoices(node, inputs) + "toggle" -> SurfaceToggle(node, inputs, onAction) + "text_input" -> SurfaceTextInput(node, inputs) + "capability_disclosure" -> SurfaceDisclosure(node) + "error" -> SurfaceError(node) + } +} + +@Composable +private fun SurfaceText(node: SurfaceComponent) { + val style = when (node.string("style")) { + "title" -> MaterialTheme.typography.titleMedium + "headline" -> MaterialTheme.typography.headlineSmall + "caption" -> MaterialTheme.typography.labelMedium + else -> MaterialTheme.typography.bodyLarge + } + Text( + node.string("text"), + color = tone(node.string("tone")), + style = style, + fontWeight = if (node.string("style") in setOf("title", "headline")) { + FontWeight.SemiBold + } else { + FontWeight.Normal + }, + ) +} + +@Composable +private fun SurfaceIcon(node: SurfaceComponent) { + val glyph = when (node.string("icon")) { + "calendar" -> "▣" + "message" -> "✉" + "notification" -> "◉" + "person" -> "●" + "phone" -> "⌕" + "location" -> "⌖" + "clock" -> "◷" + "check" -> "✓" + "warning" -> "!" + "error" -> "×" + "open" -> "↗" + else -> "✦" + } + Text( + glyph, + color = tone(node.string("tone")), + fontWeight = FontWeight.Bold, + modifier = Modifier.semantics { + contentDescription = node.string("content_description") + .ifBlank { node.string("icon") } + }, + ) +} + +@Composable +private fun SafeImagePlaceholder(node: SurfaceComponent) { + Box( + modifier = Modifier + .fillMaxWidth() + .height(132.dp) + .clip(RoundedCornerShape(18.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant) + .semantics { + contentDescription = node.string("content_description") + .ifBlank { "Phone-owned image artifact" } + }, + contentAlignment = Alignment.Center, + ) { + Text("Image artifact", color = MaterialTheme.colorScheme.onSurfaceVariant) + } +} + +@Composable +private fun SurfaceBadge(node: SurfaceComponent) { + Text( + node.string("label").ifBlank { node.string("text") }, + color = Color.Black, + style = MaterialTheme.typography.labelMedium, + fontWeight = FontWeight.SemiBold, + modifier = Modifier + .clip(RoundedCornerShape(999.dp)) + .background(tone(node.string("tone"))) + .padding(horizontal = 10.dp, vertical = 5.dp), + ) +} + +@Composable +private fun SurfaceListItem(node: SurfaceComponent, onAction: (String) -> Unit) { + val actionId = node.string("action_id") + val label = node.string("content_description") + .ifBlank { listOf(node.string("title"), node.string("subtitle")).joinToString(". ") } + Row( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)) + .then( + if (actionId.isNotBlank()) { + Modifier.clickable { onAction(actionId) } + } else { + Modifier + }, + ) + .semantics { + contentDescription = label + if (actionId.isNotBlank()) role = Role.Button + } + .padding(vertical = 12.dp, horizontal = 8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + if (node.string("icon").isNotBlank()) { + SurfaceIcon(SurfaceComponent.fromJson(JSONObject() + .put("type", "icon") + .put("icon", node.string("icon")) + .put("content_description", node.string("icon")))) + Spacer(Modifier.size(12.dp)) + } + Column(modifier = Modifier.weight(1f)) { + Text( + node.string("title"), + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.titleSmall, + fontWeight = FontWeight.SemiBold, + ) + if (node.string("subtitle").isNotBlank()) { + Text( + node.string("subtitle"), + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.bodyMedium, + ) + } + } + if (actionId.isNotBlank()) { + Text("›", color = MaterialTheme.colorScheme.primary) + } + } +} + +@Composable +private fun SurfaceButton(node: SurfaceComponent, onAction: (String) -> Unit) { + val actionId = node.string("action_id") + val label = node.string("label") + .ifBlank { node.string("title").ifBlank { node.string("text") } } + val danger = node.string("tone") == "danger" || node.type() == "confirmation" + Text( + label, + color = if (danger) Color.White else Color.Black, + fontWeight = FontWeight.SemiBold, + modifier = Modifier + .clip(RoundedCornerShape(if (node.type() == "chip") 999.dp else 18.dp)) + .background(if (danger) Color(0xFF8D2632) else MaterialTheme.colorScheme.primary) + .clickable { onAction(actionId) } + .semantics { + contentDescription = node.string("content_description").ifBlank { label } + role = Role.Button + } + .padding(horizontal = 16.dp, vertical = 11.dp), + ) +} + +@Composable +private fun SurfaceChoices(node: SurfaceComponent, inputs: MutableMap) { + val key = node.string("input_key") + val options = node.array("options") + Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { + if (node.string("label").isNotBlank()) { + Text(node.string("label"), color = MaterialTheme.colorScheme.onSurface) + } + for (index in 0 until options.length()) { + val option = options.optJSONObject(index) ?: continue + val id = option.optString("id", "") + val label = option.optString("label", "") + val selected = inputs[key] == id + Text( + label, + color = if (selected) Color.Black else MaterialTheme.colorScheme.onSurface, + modifier = Modifier + .clip(RoundedCornerShape(999.dp)) + .background( + if (selected) MaterialTheme.colorScheme.secondary + else MaterialTheme.colorScheme.surfaceVariant, + ) + .clickable { inputs[key] = id } + .semantics { + contentDescription = "$label, ${if (selected) "selected" else "not selected"}" + role = Role.RadioButton + } + .padding(horizontal = 14.dp, vertical = 9.dp), + ) + } + } +} + +@Composable +private fun SurfaceToggle( + node: SurfaceComponent, + inputs: MutableMap, + onAction: (String) -> Unit, +) { + val key = node.string("input_key") + val initial = node.bool("checked", false) + val checked = (inputs[key] as? Boolean) ?: initial + val label = node.string("label") + Row( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)) + .clickable { + inputs[key] = !checked + if (node.string("action_id").isNotBlank()) onAction(node.string("action_id")) + } + .semantics { + contentDescription = "$label, ${if (checked) "on" else "off"}" + role = Role.Switch + } + .padding(vertical = 10.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Text(label, color = MaterialTheme.colorScheme.onSurface, modifier = Modifier.weight(1f)) + Box( + Modifier + .size(42.dp, 24.dp) + .clip(RoundedCornerShape(999.dp)) + .background( + if (checked) MaterialTheme.colorScheme.secondary + else MaterialTheme.colorScheme.surfaceVariant, + ), + contentAlignment = if (checked) Alignment.CenterEnd else Alignment.CenterStart, + ) { + Box( + Modifier + .padding(3.dp) + .size(18.dp) + .clip(CircleShape) + .background(if (checked) Color.Black else Color.White), + ) + } + } +} + +@Composable +private fun SurfaceTextInput(node: SurfaceComponent, inputs: MutableMap) { + val key = node.string("input_key") + var value by remember(key) { + mutableStateOf((inputs[key] as? String) ?: node.string("value")) + } + Column { + if (node.string("label").isNotBlank()) { + Text( + node.string("label"), + color = MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.labelMedium, + ) + } + BasicTextField( + value = value, + onValueChange = { + value = it.take(500) + inputs[key] = value + }, + textStyle = MaterialTheme.typography.bodyLarge.copy( + color = MaterialTheme.colorScheme.onSurface, + ), + modifier = Modifier + .fillMaxWidth() + .padding(top = 6.dp) + .clip(RoundedCornerShape(16.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant) + .semantics { + contentDescription = node.string("content_description") + .ifBlank { + node.string("label").ifBlank { node.string("placeholder") } + } + } + .padding(horizontal = 14.dp, vertical = 12.dp), + decorationBox = { inner -> + if (value.isEmpty()) { + Text( + node.string("placeholder"), + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + inner() + }, + ) + } +} + +@Composable +private fun SurfaceDisclosure(node: SurfaceComponent) { + Column( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(14.dp)) + .background(Color(0xFF10201D)) + .padding(12.dp), + ) { + Text( + node.string("text"), + color = MaterialTheme.colorScheme.secondary, + style = MaterialTheme.typography.labelMedium, + ) + Text( + node.string("capability"), + color = MaterialTheme.colorScheme.onSurfaceVariant, + style = MaterialTheme.typography.labelSmall, + ) + } +} + +@Composable +private fun SurfaceError(node: SurfaceComponent) { + Text( + node.string("message").ifBlank { node.string("text") }, + color = MaterialTheme.colorScheme.error, + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(14.dp)) + .background(Color(0xFF2A1115)) + .padding(12.dp), + ) +} + +private fun org.json.JSONArray.components(): List = + buildList { + for (index in 0 until length()) { + SurfaceComponent.fromJson(optJSONObject(index))?.let(::add) + } + } + +private fun spacing(token: String) = when (token) { + "none" -> 0.dp + "xs" -> 4.dp + "sm" -> 8.dp + "lg" -> 20.dp + "xl" -> 28.dp + else -> 12.dp +} + +@Composable +private fun tone(token: String): Color = when (token) { + "secondary" -> MaterialTheme.colorScheme.secondary + "warning" -> Color(0xFFFFC857) + "danger" -> MaterialTheme.colorScheme.error + else -> MaterialTheme.colorScheme.onSurface +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRepository.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRepository.java new file mode 100644 index 0000000..ae04272 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRepository.java @@ -0,0 +1,234 @@ +package org.openphone.assistant.surface; + +import android.app.KeyguardManager; +import android.content.Context; +import android.content.SharedPreferences; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** Revisioned durable repository for phone-validated adaptive surfaces. */ +public final class SurfaceRepository { + private static final String PREFS = "openphone_adaptive_surfaces"; + private static final String KEY_ENTRIES = "entries"; + private static final int MAX_ENTRIES = 16; + + private final SharedPreferences mPrefs; + private final SurfaceValidator mValidator; + private final SurfaceEventLog mEvents; + private final KeyguardManager mKeyguard; + + public SurfaceRepository(Context context) { + Context app = context.getApplicationContext(); + mPrefs = app.getSharedPreferences(PREFS, Context.MODE_PRIVATE); + mValidator = new SurfaceValidator(app); + mEvents = new SurfaceEventLog(app); + mKeyguard = app.getSystemService(KeyguardManager.class); + } + + public synchronized SurfaceMutationResult present(AdaptiveSurface surface) { + SurfaceValidationResult validation = mValidator.validate(surface); + if (!validation.valid) { + mEvents.record(SurfaceEventLog.REJECTED, surface, + validation.code + ": " + validation.message, null); + return SurfaceMutationResult.error(validation.code, validation.message); + } + JSONArray entries = read(); + JSONObject existing = findEntry(entries, surface.surfaceId); + if (existing != null) { + AdaptiveSurface current = entrySurface(existing); + if (current != null && current.revision == surface.revision + && current.toJson().toString().equals(surface.toJson().toString()) + && "active".equals(existing.optString("state", ""))) { + return SurfaceMutationResult.ok("already_presented", current); + } + return SurfaceMutationResult.error( + "surface_exists", "Use replace with the expected revision."); + } + entries = prepend(entries, entry(surface, "active", "", System.currentTimeMillis())); + if (!write(entries)) { + return SurfaceMutationResult.error("storage_failed", "Could not persist surface."); + } + mEvents.record(SurfaceEventLog.PRESENTED, surface, "Surface presented.", null); + return SurfaceMutationResult.ok("presented", surface); + } + + public synchronized SurfaceMutationResult replace(AdaptiveSurface surface, + int expectedRevision) { + SurfaceValidationResult validation = mValidator.validate(surface); + if (!validation.valid) { + mEvents.record(SurfaceEventLog.REJECTED, surface, + validation.code + ": " + validation.message, null); + return SurfaceMutationResult.error(validation.code, validation.message); + } + JSONArray entries = read(); + int index = findEntryIndex(entries, surface.surfaceId); + if (index < 0) { + return SurfaceMutationResult.error("surface_not_found", "Surface was not found."); + } + JSONObject existing = entries.optJSONObject(index); + AdaptiveSurface current = entrySurface(existing); + if (current == null || !"active".equals(existing.optString("state", ""))) { + return SurfaceMutationResult.error("surface_not_active", "Surface is not active."); + } + if (current.revision != expectedRevision) { + return SurfaceMutationResult.error("stale_revision", + "Expected revision does not match the current surface."); + } + if (!current.sessionId.equals(surface.sessionId) + || surface.revision != expectedRevision + 1) { + return SurfaceMutationResult.error("replacement_invalid", + "Replacement must keep session ownership and increment revision by one."); + } + JSONArray next = new JSONArray(); + next.put(entry(surface, "active", "", System.currentTimeMillis())); + for (int i = 0; i < entries.length() && next.length() < MAX_ENTRIES; i++) { + if (i != index) { + next.put(entries.opt(i)); + } + } + if (!write(next)) { + return SurfaceMutationResult.error("storage_failed", "Could not persist surface."); + } + mEvents.record(SurfaceEventLog.REPLACED, surface, "Surface replaced.", null); + return SurfaceMutationResult.ok("replaced", surface); + } + + public synchronized SurfaceMutationResult dismiss(String surfaceId, String reason) { + JSONArray entries = read(); + int index = findEntryIndex(entries, clean(surfaceId)); + if (index < 0) { + return SurfaceMutationResult.error("surface_not_found", "Surface was not found."); + } + JSONObject existing = entries.optJSONObject(index); + AdaptiveSurface surface = entrySurface(existing); + if (surface == null) { + return SurfaceMutationResult.error("surface_malformed", "Stored surface is malformed."); + } + if ("dismissed".equals(existing.optString("state", ""))) { + return SurfaceMutationResult.ok("already_dismissed", surface); + } + try { + existing.put("state", "dismissed") + .put("dismissed_reason", clean(reason)) + .put("updated_at", System.currentTimeMillis()); + } catch (JSONException e) { + return SurfaceMutationResult.error("storage_failed", "Could not update surface."); + } + if (!write(entries)) { + return SurfaceMutationResult.error("storage_failed", "Could not persist dismissal."); + } + mEvents.record(SurfaceEventLog.DISMISSED, surface, + clean(reason).isEmpty() ? "Surface dismissed." : clean(reason), null); + return SurfaceMutationResult.ok("dismissed", surface); + } + + public synchronized AdaptiveSurface getActive(String surfaceId) { + JSONObject entry = findEntry(read(), clean(surfaceId)); + if (entry == null || !"active".equals(entry.optString("state", ""))) { + return null; + } + AdaptiveSurface surface = entrySurface(entry); + return surface == null || surface.isExpired(System.currentTimeMillis()) ? null : surface; + } + + public synchronized AdaptiveSurface currentVisible() { + JSONArray entries = read(); + long now = System.currentTimeMillis(); + boolean locked = mKeyguard != null && mKeyguard.isDeviceLocked(); + for (int i = 0; i < entries.length(); i++) { + JSONObject entry = entries.optJSONObject(i); + if (entry == null || !"active".equals(entry.optString("state", ""))) { + continue; + } + AdaptiveSurface surface = entrySurface(entry); + if (surface == null || surface.isExpired(now)) { + continue; + } + if (locked && ("sensitive".equals(surface.sensitivity) + || "restricted".equals(surface.sensitivity))) { + continue; + } + return surface; + } + return null; + } + + public synchronized SurfaceMutationResult resolveInvocation(String surfaceId, + int revision, String actionId) { + AdaptiveSurface surface = getActive(surfaceId); + if (surface == null) { + return SurfaceMutationResult.error( + "surface_not_active", "Surface is missing, dismissed, or expired."); + } + if (surface.revision != revision) { + return SurfaceMutationResult.error( + "stale_revision", "Surface changed; reopen it before acting."); + } + if (!surface.actions.containsKey(clean(actionId))) { + return SurfaceMutationResult.error("action_not_found", "Surface action was not found."); + } + SurfaceValidationResult validation = mValidator.validate(surface); + if (!validation.valid) { + return SurfaceMutationResult.error(validation.code, validation.message); + } + return SurfaceMutationResult.ok("invocation_resolved", surface); + } + + private static JSONObject entry(AdaptiveSurface surface, String state, + String dismissedReason, long updatedAt) { + JSONObject out = new JSONObject(); + try { + out.put("surface", surface.toJson()) + .put("state", state) + .put("dismissed_reason", dismissedReason) + .put("updated_at", updatedAt); + } catch (JSONException ignored) { + } + return out; + } + + private static AdaptiveSurface entrySurface(JSONObject entry) { + return entry == null ? null : AdaptiveSurface.fromJson(entry.optJSONObject("surface")); + } + + private static JSONArray prepend(JSONArray entries, JSONObject first) { + JSONArray out = new JSONArray().put(first); + for (int i = 0; i < entries.length() && out.length() < MAX_ENTRIES; i++) { + out.put(entries.opt(i)); + } + return out; + } + + private static JSONObject findEntry(JSONArray entries, String surfaceId) { + int index = findEntryIndex(entries, surfaceId); + return index < 0 ? null : entries.optJSONObject(index); + } + + private static int findEntryIndex(JSONArray entries, String surfaceId) { + for (int i = 0; i < entries.length(); i++) { + AdaptiveSurface candidate = entrySurface(entries.optJSONObject(i)); + if (candidate != null && surfaceId.equals(candidate.surfaceId)) { + return i; + } + } + return -1; + } + + private JSONArray read() { + try { + return new JSONArray(mPrefs.getString(KEY_ENTRIES, "[]")); + } catch (JSONException e) { + return new JSONArray(); + } + } + + private boolean write(JSONArray entries) { + return mPrefs.edit().putString(KEY_ENTRIES, entries.toString()).commit(); + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRuntimeNotifier.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRuntimeNotifier.java new file mode 100644 index 0000000..00be72a --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceRuntimeNotifier.java @@ -0,0 +1,80 @@ +package org.openphone.assistant.surface; + +import android.content.Context; +import android.content.Intent; + +import org.json.JSONException; +import org.json.JSONObject; +import org.openphone.assistant.OpenPhoneAssistantService; + +/** Sends local renderer lifecycle/action results to the owning runtime service. */ +public final class SurfaceRuntimeNotifier { + private SurfaceRuntimeNotifier() { + } + + public static void actionInvoked(Context context, AdaptiveSurface surface, + String actionId, String idempotencyKey) { + JSONObject payload = base(surface); + try { + payload.put("action_id", clean(actionId)) + .put("idempotency_key", clean(idempotencyKey)); + } catch (JSONException ignored) { + } + send(context, surface, "phone.surface.action_invoked", payload); + } + + public static void actionResult(Context context, AdaptiveSurface surface, + String actionId, SurfaceActionResult result) { + JSONObject payload = base(surface); + try { + payload.put("action_id", clean(actionId)) + .put("status", result == null ? "error" : result.status) + .put("message", result == null ? "" : result.message) + .put("result", result == null ? new JSONObject() : result.toJson()); + } catch (JSONException ignored) { + } + send(context, surface, "runtime.surface.action_result", payload); + } + + public static void dismissed(Context context, AdaptiveSurface surface, String reason) { + JSONObject payload = base(surface); + try { + payload.put("reason", clean(reason)); + } catch (JSONException ignored) { + } + send(context, surface, "phone.surface.dismissed", payload); + } + + private static JSONObject base(AdaptiveSurface surface) { + JSONObject payload = new JSONObject(); + try { + payload.put("surface_id", surface == null ? "" : surface.surfaceId) + .put("revision", surface == null ? 0 : surface.revision) + .put("session_id", surface == null ? "" : surface.sessionId) + .put("runtime", surface == null ? "" : surface.runtime); + } catch (JSONException ignored) { + } + return payload; + } + + private static void send(Context context, AdaptiveSurface surface, + String event, JSONObject payload) { + if (context == null || surface == null || "builtin".equals(surface.runtime)) { + return; + } + Intent intent = new Intent(context, OpenPhoneAssistantService.class) + .setAction(OpenPhoneAssistantService.ACTION_RUNTIME_SURFACE_EVENT) + .putExtra(OpenPhoneAssistantService.EXTRA_RUNTIME_SURFACE_RUNTIME, surface.runtime) + .putExtra(OpenPhoneAssistantService.EXTRA_RUNTIME_SURFACE_EVENT, event) + .putExtra(OpenPhoneAssistantService.EXTRA_RUNTIME_SURFACE_PAYLOAD, + payload.toString()); + try { + context.startService(intent); + } catch (RuntimeException ignored) { + } + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceValidationResult.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceValidationResult.java new file mode 100644 index 0000000..7e7ce26 --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceValidationResult.java @@ -0,0 +1,26 @@ +package org.openphone.assistant.surface; + +/** Fail-closed validation result with a stable machine-readable error code. */ +public final class SurfaceValidationResult { + public final boolean valid; + public final String code; + public final String message; + + private SurfaceValidationResult(boolean valid, String code, String message) { + this.valid = valid; + this.code = clean(code); + this.message = clean(message); + } + + public static SurfaceValidationResult ok() { + return new SurfaceValidationResult(true, "ok", ""); + } + + public static SurfaceValidationResult reject(String code, String message) { + return new SurfaceValidationResult(false, code, message); + } + + private static String clean(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceValidator.java b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceValidator.java new file mode 100644 index 0000000..3a945aa --- /dev/null +++ b/overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/surface/SurfaceValidator.java @@ -0,0 +1,452 @@ +package org.openphone.assistant.surface; + +import android.content.Context; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.openphone.assistant.actions.ActionRegistry; +import org.openphone.assistant.actions.ToolCatalog; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.regex.Pattern; + +/** Phone-owned structural, policy-binding, and resource validator for Surface V1. */ +public final class SurfaceValidator { + public static final int MAX_BYTES = 256 * 1024; + public static final int MAX_NODES = 120; + public static final int MAX_DEPTH = 12; + public static final int MAX_TEXT_CHARS = 24000; + public static final int MAX_IMAGES = 4; + public static final int MAX_ACTIONS = 24; + + private static final Pattern SURFACE_ID = + Pattern.compile("^surface-[A-Za-z0-9._-]{1,120}$"); + private static final Pattern ACTION_ID = + Pattern.compile("^[A-Za-z][A-Za-z0-9._-]{0,119}$"); + private static final Pattern ARTIFACT_ID = + Pattern.compile("^artifact-[A-Za-z0-9._-]{1,120}$"); + private static final Set PRESENTATIONS = set("full", "sheet", "inline"); + private static final Set SENSITIVITY = + set("public", "personal", "sensitive", "restricted"); + private static final Set COMPONENT_TYPES = set( + "column", "row", "spacer", "divider", "scroll", + "text", "icon", "image", "badge", "progress", + "card", "list", "list_item", "timeline", + "button", "chip", "choice_group", "toggle", "text_input", + "native_handoff", "confirmation", "capability_disclosure", "error"); + private static final Set COMPONENT_KEYS = set( + "type", "children", "items", "text", "title", "subtitle", "label", + "message", "content_description", "style", "tone", "density", "spacing", + "action_id", "input_key", "placeholder", "value", "checked", "progress", + "artifact", "icon", "options", "native_target", "capability"); + private static final Set ROOT_KEYS = set( + "schema", "surface_id", "revision", "session_id", "runtime", "title", + "presentation", "expires_at", "sensitivity", "body", "actions", + "artifacts", "created_at"); + private static final Set ACTION_KEYS = set( + "label", "tool", "params", "reason", "requires_confirmation"); + private static final Set TEXT_STYLES = + set("title", "headline", "body", "caption"); + private static final Set TONES = + set("primary", "secondary", "warning", "danger"); + private static final Set DENSITIES = set("compact", "comfortable"); + private static final Set SPACING = set("none", "xs", "sm", "md", "lg", "xl"); + private static final Set ICONS = set( + "calendar", "message", "notification", "person", "phone", "location", + "clock", "check", "warning", "error", "open", "agent"); + private static final Set NATIVE_TARGETS = + set("app_space", "calendar", "messages", "notifications", "dialer", "settings"); + private static final Set INTERACTIVE_TYPES = + set("button", "chip", "choice_group", "toggle", "text_input", "native_handoff", + "confirmation", "list_item"); + + private final SurfaceArtifactStore mArtifacts; + + public SurfaceValidator(Context context) { + mArtifacts = new SurfaceArtifactStore(context); + } + + public SurfaceValidationResult validate(AdaptiveSurface surface) { + if (surface == null) { + return reject("surface_missing", "Surface document is missing or malformed."); + } + JSONObject raw = surface.toJson(); + if (raw.toString().getBytes(StandardCharsets.UTF_8).length > MAX_BYTES) { + return reject("surface_too_large", "Surface exceeds the byte limit."); + } + String unknownRoot = firstUnknownKey(raw, ROOT_KEYS); + if (!unknownRoot.isEmpty()) { + return reject("unknown_root_property", "Unknown surface property: " + unknownRoot); + } + if (!AdaptiveSurface.SCHEMA.equals(raw.optString("schema", ""))) { + return reject("schema_mismatch", "Unsupported surface schema."); + } + if (!SURFACE_ID.matcher(surface.surfaceId).matches()) { + return reject("surface_id_invalid", "Surface identity is invalid."); + } + if (surface.revision < 1) { + return reject("revision_invalid", "Surface revision must be positive."); + } + if (surface.sessionId.isEmpty() || surface.sessionId.length() > 160 + || surface.runtime.isEmpty() || surface.runtime.length() > 80) { + return reject("owner_invalid", "Surface session/runtime ownership is invalid."); + } + if (surface.title.length() > 160 + || !PRESENTATIONS.contains(surface.presentation) + || !SENSITIVITY.contains(surface.sensitivity) + || surface.expiresAtMillis < 0L) { + return reject("presentation_invalid", "Surface presentation metadata is invalid."); + } + if (surface.isExpired(System.currentTimeMillis())) { + return reject("surface_expired", "Surface has expired."); + } + if (surface.actions.size() > MAX_ACTIONS) { + return reject("too_many_actions", "Surface has too many actions."); + } + + Set declaredArtifacts = declaredArtifacts(surface); + if (declaredArtifacts == null) { + return reject("artifact_declaration_invalid", "Artifact declaration is invalid."); + } + ValidationState state = new ValidationState(); + SurfaceValidationResult bodyResult = validateComponent( + raw.optJSONObject("body"), surface, declaredArtifacts, state, 1); + if (!bodyResult.valid) { + return bodyResult; + } + if (state.nodes > MAX_NODES || state.textChars > MAX_TEXT_CHARS + || state.images > MAX_IMAGES) { + return reject("resource_limit", "Surface exceeds a renderer resource limit."); + } + + ToolCatalog catalog = ToolCatalog.get(); + if (!surface.actions.isEmpty() && !catalog.isLoaded()) { + return reject("tool_registry_unavailable", + "Phone tool registry is unavailable; actions fail closed."); + } + JSONObject rawActions = raw.optJSONObject("actions"); + for (SurfaceAction action : surface.actions.values()) { + JSONObject rawAction = rawActions == null + ? null : rawActions.optJSONObject(action.id); + SurfaceValidationResult actionResult = + validateAction(action, rawAction, catalog); + if (!actionResult.valid) { + return actionResult; + } + } + for (String referenced : state.referencedActions) { + if (!surface.actions.containsKey(referenced)) { + return reject("undeclared_action", + "Component references undeclared action: " + referenced); + } + } + for (String declared : surface.actions.keySet()) { + if (!state.referencedActions.contains(declared)) { + return reject("unbound_action", + "Action is not bound to a component: " + declared); + } + } + return SurfaceValidationResult.ok(); + } + + private SurfaceValidationResult validateComponent(JSONObject node, AdaptiveSurface surface, + Set declaredArtifacts, ValidationState state, int depth) { + if (node == null) { + return reject("component_missing", "Surface component must be an object."); + } + if (depth > MAX_DEPTH) { + return reject("surface_too_deep", "Surface exceeds the depth limit."); + } + state.nodes++; + if (state.nodes > MAX_NODES) { + return reject("too_many_nodes", "Surface exceeds the node limit."); + } + String unknown = firstUnknownKey(node, COMPONENT_KEYS); + if (!unknown.isEmpty()) { + return reject("unknown_component_property", + "Unknown component property: " + unknown); + } + String type = node.optString("type", ""); + if (!COMPONENT_TYPES.contains(type)) { + return reject("unknown_component", "Unknown component type: " + type); + } + SurfaceValidationResult tokens = validateTokens(node); + if (!tokens.valid) { + return tokens; + } + for (String key : new String[] { + "text", "title", "subtitle", "label", "message", + "content_description", "placeholder" + }) { + if (!node.has(key)) { + continue; + } + Object value = node.opt(key); + if (!(value instanceof String)) { + return reject("text_type_invalid", key + " must be a string."); + } + int length = ((String) value).length(); + int perFieldLimit = ("text".equals(key) || "message".equals(key)) ? 1000 : 500; + if (length > perFieldLimit) { + return reject("node_text_too_long", key + " exceeds its text limit."); + } + state.textChars += length; + if (state.textChars > MAX_TEXT_CHARS) { + return reject("document_text_too_long", "Surface text exceeds its limit."); + } + } + String actionId = node.optString("action_id", ""); + if (!actionId.isEmpty()) { + if (!ACTION_ID.matcher(actionId).matches()) { + return reject("action_id_invalid", "Component action identity is invalid."); + } + state.referencedActions.add(actionId); + } + if (INTERACTIVE_TYPES.contains(type) + && actionId.isEmpty() + && !"choice_group".equals(type) + && !"text_input".equals(type)) { + return reject("interactive_action_missing", + "Interactive component requires an action binding."); + } + if (INTERACTIVE_TYPES.contains(type) + && accessibleLabel(node).isEmpty()) { + return reject("accessibility_label_missing", + "Interactive component requires an accessible label."); + } + if ("text_input".equals(type) + && !ACTION_ID.matcher(node.optString("input_key", "")).matches()) { + return reject("input_key_invalid", "Text input requires a stable input key."); + } + if ("progress".equals(type)) { + double progress = node.optDouble("progress", -1.0); + if (progress < 0.0 || progress > 1.0) { + return reject("progress_invalid", "Progress must be between zero and one."); + } + } + if ("image".equals(type)) { + state.images++; + JSONObject artifact = node.optJSONObject("artifact"); + SurfaceValidationResult artifactResult = validateArtifactReference( + artifact, surface, declaredArtifacts); + if (!artifactResult.valid) { + return artifactResult; + } + } + if ("icon".equals(type) && !ICONS.contains(node.optString("icon", ""))) { + return reject("icon_invalid", "Unknown semantic icon."); + } + if ("native_handoff".equals(type) + && !NATIVE_TARGETS.contains(node.optString("native_target", ""))) { + return reject("native_target_invalid", "Native handoff target is not allowed."); + } + JSONArray options = node.optJSONArray("options"); + if (options != null) { + if (options.length() > 12) { + return reject("too_many_options", "Choice component has too many options."); + } + for (int i = 0; i < options.length(); i++) { + JSONObject option = options.optJSONObject(i); + if (option == null || option.optString("id", "").isEmpty() + || option.optString("label", "").isEmpty()) { + return reject("choice_option_invalid", "Choice option is malformed."); + } + } + } + for (String childKey : new String[] {"children", "items"}) { + JSONArray children = node.optJSONArray(childKey); + if (children == null) { + continue; + } + if (children.length() > 32) { + return reject("too_many_children", "Component has too many children."); + } + for (int i = 0; i < children.length(); i++) { + SurfaceValidationResult childResult = validateComponent( + children.optJSONObject(i), surface, declaredArtifacts, state, depth + 1); + if (!childResult.valid) { + return childResult; + } + } + } + return SurfaceValidationResult.ok(); + } + + private SurfaceValidationResult validateAction(SurfaceAction action, JSONObject raw, + ToolCatalog catalog) { + if (action == null || !ACTION_ID.matcher(action.id).matches() + || action.label.isEmpty() || action.label.length() > 80 + || action.tool.isEmpty() || action.tool.length() > 80 + || action.reason.isEmpty() || action.reason.length() > 240) { + return reject("action_invalid", "Surface action is malformed."); + } + if (raw == null) { + return reject("action_invalid", "Surface action is missing."); + } + String unknown = firstUnknownKey(raw, ACTION_KEYS); + if (!unknown.isEmpty()) { + return reject("unknown_action_property", "Unknown action property: " + unknown); + } + if (!catalog.isAllowedTool(action.tool)) { + return reject("unknown_tool", "Surface action uses an unregistered tool."); + } + ActionRegistry.ActionMetadata metadata = null; + for (ActionRegistry.ActionMetadata candidate : catalog.tools()) { + if (action.tool.equals(candidate.modelTool)) { + metadata = candidate; + break; + } + } + if (metadata == null) { + return reject("tool_metadata_missing", "Tool metadata is unavailable."); + } + if (catalog.isStateChangingTool(action.tool) && !action.requiresConfirmation) { + return reject("mutation_disclosure_missing", + "Mutating surface actions must declare confirmation."); + } + for (String required : metadata.requiredInputs) { + if ("reason".equals(required)) { + continue; + } + if (!action.params.has(required)) { + return reject("tool_param_missing", "Missing required tool parameter: " + required); + } + } + JSONObject properties = metadata.inputSchema.optJSONObject("properties"); + Iterator keys = action.params.keys(); + while (keys.hasNext()) { + String key = keys.next(); + if (properties == null || !properties.has(key)) { + return reject("tool_param_unknown", "Unknown tool parameter: " + key); + } + JSONObject property = properties.optJSONObject(key); + if (property != null && !matchesType(action.params.opt(key), + property.optString("type", ""))) { + return reject("tool_param_type", "Invalid type for tool parameter: " + key); + } + } + return SurfaceValidationResult.ok(); + } + + private Set declaredArtifacts(AdaptiveSurface surface) { + Set declared = new HashSet<>(); + for (int i = 0; i < surface.artifacts.length(); i++) { + JSONObject reference = surface.artifacts.optJSONObject(i); + if (!validArtifactReferenceShape(reference)) { + return null; + } + declared.add(reference.optString("artifact_id", "")); + } + return declared; + } + + private SurfaceValidationResult validateArtifactReference(JSONObject reference, + AdaptiveSurface surface, Set declaredArtifacts) { + if (!validArtifactReferenceShape(reference)) { + return reject("artifact_reference_invalid", "Artifact reference is malformed."); + } + String artifactId = reference.optString("artifact_id", ""); + if (!declaredArtifacts.contains(artifactId)) { + return reject("artifact_undeclared", "Component uses an undeclared artifact."); + } + if (!mArtifacts.belongsToSession(artifactId, surface.sessionId)) { + return reject("artifact_session_mismatch", + "Artifact is missing, expired, or belongs to another session."); + } + return SurfaceValidationResult.ok(); + } + + private static boolean validArtifactReferenceShape(JSONObject reference) { + if (reference == null || reference.length() != 2) { + return false; + } + String artifactId = reference.optString("artifact_id", ""); + String path = reference.optString("path", ""); + return ARTIFACT_ID.matcher(artifactId).matches() + && path.matches("^\\$([.][A-Za-z0-9_-]+|\\[[0-9]+\\])*$") + && path.length() <= 240; + } + + private static SurfaceValidationResult validateTokens(JSONObject node) { + if (node.has("style") && !TEXT_STYLES.contains(node.optString("style", ""))) { + return reject("style_invalid", "Unknown semantic text style."); + } + if (node.has("tone") && !TONES.contains(node.optString("tone", ""))) { + return reject("tone_invalid", "Unknown semantic tone."); + } + if (node.has("density") && !DENSITIES.contains(node.optString("density", ""))) { + return reject("density_invalid", "Unknown density token."); + } + if (node.has("spacing") && !SPACING.contains(node.optString("spacing", ""))) { + return reject("spacing_invalid", "Unknown spacing token."); + } + return SurfaceValidationResult.ok(); + } + + private static String accessibleLabel(JSONObject node) { + for (String key : new String[] { + "content_description", "label", "title", "text", "message", "placeholder" + }) { + String value = node.optString(key, "").trim(); + if (!value.isEmpty()) { + return value; + } + } + return ""; + } + + private static boolean matchesType(Object value, String type) { + if (type == null || type.isEmpty()) { + return true; + } + if ("string".equals(type)) { + return value instanceof String; + } + if ("integer".equals(type)) { + return value instanceof Integer || value instanceof Long; + } + if ("number".equals(type)) { + return value instanceof Number; + } + if ("boolean".equals(type)) { + return value instanceof Boolean; + } + if ("object".equals(type)) { + return value instanceof JSONObject; + } + if ("array".equals(type)) { + return value instanceof JSONArray; + } + return false; + } + + private static String firstUnknownKey(JSONObject object, Set allowed) { + Iterator keys = object.keys(); + while (keys.hasNext()) { + String key = keys.next(); + if (!allowed.contains(key)) { + return key; + } + } + return ""; + } + + private static SurfaceValidationResult reject(String code, String message) { + return SurfaceValidationResult.reject(code, message); + } + + private static Set set(String... values) { + return new HashSet<>(Arrays.asList(values)); + } + + private static final class ValidationState { + int nodes; + int textChars; + int images; + final Set referencedActions = new HashSet<>(); + } +} diff --git a/overlay/vendor/openphone/overlay/frameworks/base/core/res/res/values/config.xml b/overlay/vendor/openphone/overlay/frameworks/base/core/res/res/values/config.xml new file mode 100644 index 0000000..c6dac24 --- /dev/null +++ b/overlay/vendor/openphone/overlay/frameworks/base/core/res/res/values/config.xml @@ -0,0 +1,10 @@ + + + + true + diff --git a/overlay/vendor/openphone/products/openphone_common.mk b/overlay/vendor/openphone/products/openphone_common.mk index ae80789..7c2dac1 100644 --- a/overlay/vendor/openphone/products/openphone_common.mk +++ b/overlay/vendor/openphone/products/openphone_common.mk @@ -24,5 +24,6 @@ PRODUCT_COPY_FILES += \ PRODUCT_SYSTEM_EXT_PROPERTIES += \ ro.openphone.version=0.1.0-dev \ + ro.openphone.systemui_island=true \ ro.openphone.source_available=true \ ro.openphone.commercial_license_required=true diff --git a/patches/frameworks_base/0020-OpenPhone-add-durable-island-state-contract.patch b/patches/frameworks_base/0020-OpenPhone-add-durable-island-state-contract.patch new file mode 100644 index 0000000..0c61cc3 --- /dev/null +++ b/patches/frameworks_base/0020-OpenPhone-add-durable-island-state-contract.patch @@ -0,0 +1,414 @@ +From 6867fb1fa13b96fddae6e1094f53d4f43c432ad1 Mon Sep 17 00:00:00 2001 +From: OpenPhone +Date: Sat, 18 Jul 2026 17:12:55 -0700 +Subject: [PATCH 20/21] OpenPhone: add durable island state contract + +--- + .../openphone/IOpenPhoneAgentService.aidl | 6 + + .../IOpenPhoneIslandStateListener.aidl | 14 + + .../openphone/OpenPhoneAgentManager.java | 49 ++++ + .../OpenPhoneAgentManagerService.java | 251 ++++++++++++++++++ + 4 files changed, 320 insertions(+) + create mode 100644 core/java/android/openphone/IOpenPhoneIslandStateListener.aidl + +diff --git a/core/java/android/openphone/IOpenPhoneAgentService.aidl b/core/java/android/openphone/IOpenPhoneAgentService.aidl +index 2d1d9943..8ab48776 100644 +--- a/core/java/android/openphone/IOpenPhoneAgentService.aidl ++++ b/core/java/android/openphone/IOpenPhoneAgentService.aidl +@@ -8,6 +8,8 @@ + + package android.openphone; + ++import android.openphone.IOpenPhoneIslandStateListener; ++ + /** @hide */ + interface IOpenPhoneAgentService { + String getServiceStatus(); +@@ -23,4 +25,8 @@ interface IOpenPhoneAgentService { + String getAuditLog(int maxEvents); + String submitUiTreeSnapshot(String snapshotJson); + String getUiTreeSnapshot(); ++ String getIslandState(); ++ String publishIslandState(String stateJson); ++ void registerIslandStateListener(IOpenPhoneIslandStateListener listener); ++ void unregisterIslandStateListener(IOpenPhoneIslandStateListener listener); + } +diff --git a/core/java/android/openphone/IOpenPhoneIslandStateListener.aidl b/core/java/android/openphone/IOpenPhoneIslandStateListener.aidl +new file mode 100644 +index 00000000..595149b7 +--- /dev/null ++++ b/core/java/android/openphone/IOpenPhoneIslandStateListener.aidl +@@ -0,0 +1,14 @@ ++/* ++ * Copyright 2026 OpenPhone contributors. ++ * ++ * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 ++ * ++ * Commercial use requires a separate written license. ++ */ ++ ++package android.openphone; ++ ++/** @hide */ ++oneway interface IOpenPhoneIslandStateListener { ++ void onIslandStateChanged(String stateJson); ++} +diff --git a/core/java/android/openphone/OpenPhoneAgentManager.java b/core/java/android/openphone/OpenPhoneAgentManager.java +index 7ef47657..992143ba 100644 +--- a/core/java/android/openphone/OpenPhoneAgentManager.java ++++ b/core/java/android/openphone/OpenPhoneAgentManager.java +@@ -168,4 +168,53 @@ public final class OpenPhoneAgentManager { + throw e.rethrowFromSystemServer(); + } + } ++ ++ /** ++ * Return the latest normalized compact-island projection held by ++ * system_server. ++ * ++ * @hide ++ */ ++ @NonNull ++ public String getIslandState() { ++ try { ++ return mService.getIslandState(); ++ } catch (RemoteException e) { ++ throw e.rethrowFromSystemServer(); ++ } ++ } ++ ++ /** ++ * Publish a bounded compact-island projection. Only the OpenPhone ++ * assistant package may publish; system_server normalizes the payload, ++ * assigns its revision and timestamps, and discards unknown fields. ++ * ++ * @hide ++ */ ++ @NonNull ++ public String publishIslandState(@NonNull String stateJson) { ++ try { ++ return mService.publishIslandState(stateJson); ++ } catch (RemoteException e) { ++ throw e.rethrowFromSystemServer(); ++ } ++ } ++ ++ /** @hide */ ++ public void registerIslandStateListener(@NonNull IOpenPhoneIslandStateListener listener) { ++ try { ++ mService.registerIslandStateListener(listener); ++ } catch (RemoteException e) { ++ throw e.rethrowFromSystemServer(); ++ } ++ } ++ ++ /** @hide */ ++ public void unregisterIslandStateListener(@NonNull IOpenPhoneIslandStateListener listener) { ++ try { ++ mService.unregisterIslandStateListener(listener); ++ } catch (RemoteException e) { ++ throw e.rethrowFromSystemServer(); ++ } ++ } + } +diff --git a/services/core/java/com/android/server/openphone/OpenPhoneAgentManagerService.java b/services/core/java/com/android/server/openphone/OpenPhoneAgentManagerService.java +index b44bd1eb..4be18f8b 100644 +--- a/services/core/java/com/android/server/openphone/OpenPhoneAgentManagerService.java ++++ b/services/core/java/com/android/server/openphone/OpenPhoneAgentManagerService.java +@@ -18,8 +18,11 @@ import android.net.Uri; + import android.graphics.Bitmap; + import android.hardware.input.InputManager; + import android.openphone.IOpenPhoneAgentService; ++import android.openphone.IOpenPhoneIslandStateListener; + import android.os.Binder; + import android.os.Environment; ++import android.os.RemoteCallbackList; ++import android.os.RemoteException; + import android.os.SystemClock; + import android.os.UserHandle; + import android.util.AtomicFile; +@@ -113,6 +116,15 @@ public final class OpenPhoneAgentManagerService extends SystemService { + private static final long DEFAULT_SWIPE_MS = 250; + private static final int DEFAULT_SCREENSHOT_QUALITY = 70; + private static final int DEFAULT_SCREENSHOT_MAX_DIMENSION = 720; ++ private static final int MAX_ISLAND_STATE_BYTES = 16 * 1024; ++ private static final String ISLAND_STATE_SCHEMA = "openphone.island_state.v1"; ++ private static final String ASSISTANT_PACKAGE = "org.openphone.assistant"; ++ private static final Set ALLOWED_ISLAND_MODES = Set.of( ++ "idle", "listening", "transcript", "thinking", "realtime", ++ "action_running", "answer_ready", "reply", "needs_review", ++ "error", "watching", "unavailable"); ++ private static final Set ALLOWED_ISLAND_SENSITIVITY = Set.of( ++ "public", "personal", "sensitive", "restricted"); + + private final Context mContext; + private final Object mAuditLock = new Object(); +@@ -129,6 +141,11 @@ public final class OpenPhoneAgentManagerService extends SystemService { + private String mUiTreeSnapshot = ""; + private long mUiTreeSnapshotElapsedMs = 0L; + private int mUiTreeSnapshotProducerUid = 0; ++ private final Object mIslandStateLock = new Object(); ++ private final RemoteCallbackList mIslandStateListeners = ++ new RemoteCallbackList<>(); ++ private String mIslandState = defaultIslandState(); ++ private long mIslandStateRevision = 0L; + private final String mBootId = UUID.randomUUID().toString(); + private long mAuditSeq = 0L; + private String mAuditChainHead = AUDIT_GENESIS_HASH; +@@ -333,6 +350,221 @@ public final class OpenPhoneAgentManagerService extends SystemService { + } + } + ++ @Override ++ public String getIslandState() { ++ enforceManageTasks(); ++ synchronized (mIslandStateLock) { ++ return mIslandState; ++ } ++ } ++ ++ @Override ++ public String publishIslandState(String stateJson) { ++ enforceManageTasks(); ++ enforceAssistantIslandPublisher(); ++ final String normalized; ++ synchronized (mIslandStateLock) { ++ final long nextRevision = mIslandStateRevision + 1L; ++ normalized = normalizeIslandState(stateJson, nextRevision); ++ mIslandState = normalized; ++ mIslandStateRevision = nextRevision; ++ } ++ notifyIslandStateListeners(normalized); ++ return normalized; ++ } ++ ++ @Override ++ public void registerIslandStateListener(IOpenPhoneIslandStateListener listener) { ++ enforceManageTasks(); ++ if (listener == null) { ++ throw new IllegalArgumentException("Island state listener must not be null"); ++ } ++ mIslandStateListeners.register(listener); ++ final String current; ++ synchronized (mIslandStateLock) { ++ current = mIslandState; ++ } ++ try { ++ listener.onIslandStateChanged(current); ++ } catch (RemoteException ignored) { ++ mIslandStateListeners.unregister(listener); ++ } ++ } ++ ++ @Override ++ public void unregisterIslandStateListener(IOpenPhoneIslandStateListener listener) { ++ enforceManageTasks(); ++ if (listener != null) { ++ mIslandStateListeners.unregister(listener); ++ } ++ } ++ ++ private void notifyIslandStateListeners(String stateJson) { ++ final int count = mIslandStateListeners.beginBroadcast(); ++ try { ++ for (int index = 0; index < count; index++) { ++ try { ++ mIslandStateListeners.getBroadcastItem(index) ++ .onIslandStateChanged(stateJson); ++ } catch (RemoteException ignored) { ++ // RemoteCallbackList removes dead binders automatically. ++ } ++ } ++ } finally { ++ mIslandStateListeners.finishBroadcast(); ++ } ++ } ++ ++ private String normalizeIslandState(String stateJson, long revision) { ++ if (stateJson == null ++ || stateJson.getBytes(StandardCharsets.UTF_8).length > MAX_ISLAND_STATE_BYTES) { ++ throw new IllegalArgumentException("Island state exceeds bounded payload"); ++ } ++ final JSONObject input; ++ try { ++ input = new JSONObject(stateJson); ++ } catch (JSONException e) { ++ throw new IllegalArgumentException("Invalid island state JSON", e); ++ } ++ try { ++ final String schema = requiredBoundedString(input, "schema", 64); ++ if (!ISLAND_STATE_SCHEMA.equals(schema)) { ++ throw new IllegalArgumentException("Unsupported island state schema"); ++ } ++ final String mode = requiredBoundedString(input, "mode", 32); ++ if (!ALLOWED_ISLAND_MODES.contains(mode)) { ++ throw new IllegalArgumentException("Unsupported island mode"); ++ } ++ final String sensitivity = requiredBoundedString( ++ input, "sensitivity", 16); ++ if (!ALLOWED_ISLAND_SENSITIVITY.contains(sensitivity)) { ++ throw new IllegalArgumentException("Unsupported island sensitivity"); ++ } ++ final String label = boundedString(input, "label", 48); ++ final String title = boundedString(input, "title", 80); ++ final String detail = boundedString(input, "detail", 160); ++ rejectSecretMaterial(label); ++ rejectSecretMaterial(title); ++ rejectSecretMaterial(detail); ++ final int liveRuns = strictBoundedInt(input, "live_runs", 0, 100); ++ final JSONObject output = new JSONObject() ++ .put("schema", ISLAND_STATE_SCHEMA) ++ .put("revision", revision) ++ .put("mode", mode) ++ .put("label", label) ++ .put("title", title) ++ .put("detail", detail) ++ .put("visible", strictBoolean(input, "visible")) ++ .put("needs_attention", strictBoolean(input, "needs_attention")) ++ .put("sensitivity", sensitivity) ++ .put("active_task_id", ++ boundedString(input, "active_task_id", 96)) ++ .put("pending_confirmation_id", ++ boundedString(input, "pending_confirmation_id", 128)) ++ .put("live_runs", liveRuns) ++ .put("updated_at", strictNonNegativeLong(input, "updated_at")) ++ .put("published_at", System.currentTimeMillis()) ++ .put("published_elapsed_ms", SystemClock.elapsedRealtime()) ++ .put("source", "assistant"); ++ return output.toString(); ++ } catch (JSONException e) { ++ throw new IllegalArgumentException("Invalid island state fields", e); ++ } ++ } ++ ++ private static String defaultIslandState() { ++ try { ++ return new JSONObject() ++ .put("schema", ISLAND_STATE_SCHEMA) ++ .put("revision", 0) ++ .put("mode", "unavailable") ++ .put("label", "OpenPhone") ++ .put("title", "OpenPhone") ++ .put("detail", "Assistant unavailable") ++ .put("visible", true) ++ .put("needs_attention", false) ++ .put("sensitivity", "public") ++ .put("active_task_id", "") ++ .put("pending_confirmation_id", "") ++ .put("live_runs", 0) ++ .put("updated_at", 0) ++ .put("published_at", System.currentTimeMillis()) ++ .put("published_elapsed_ms", SystemClock.elapsedRealtime()) ++ .put("source", "system_server") ++ .toString(); ++ } catch (JSONException e) { ++ throw new IllegalStateException(e); ++ } ++ } ++ ++ private static String requiredBoundedString(JSONObject input, String key, int max) ++ throws JSONException { ++ if (!input.has(key)) { ++ throw new JSONException("Missing " + key); ++ } ++ return boundedString(input, key, max); ++ } ++ ++ private static String boundedString(JSONObject input, String key, int max) ++ throws JSONException { ++ if (!input.has(key) || input.isNull(key)) { ++ return ""; ++ } ++ final Object value = input.get(key); ++ if (!(value instanceof String)) { ++ throw new JSONException(key + " must be a string"); ++ } ++ final String clean = ((String) value).trim(); ++ if (clean.length() > max) { ++ throw new JSONException(key + " exceeds " + max + " characters"); ++ } ++ return clean; ++ } ++ ++ private static boolean strictBoolean(JSONObject input, String key) throws JSONException { ++ final Object value = input.get(key); ++ if (!(value instanceof Boolean)) { ++ throw new JSONException(key + " must be a boolean"); ++ } ++ return (Boolean) value; ++ } ++ ++ private static int strictBoundedInt(JSONObject input, String key, int min, int max) ++ throws JSONException { ++ final Object value = input.get(key); ++ if (!(value instanceof Number)) { ++ throw new JSONException(key + " must be a number"); ++ } ++ final long parsed = ((Number) value).longValue(); ++ if (((Number) value).doubleValue() != parsed || parsed < min || parsed > max) { ++ throw new JSONException(key + " is outside bounds"); ++ } ++ return (int) parsed; ++ } ++ ++ private static long strictNonNegativeLong(JSONObject input, String key) ++ throws JSONException { ++ final Object value = input.get(key); ++ if (!(value instanceof Number)) { ++ throw new JSONException(key + " must be a number"); ++ } ++ final long parsed = ((Number) value).longValue(); ++ if (((Number) value).doubleValue() != parsed || parsed < 0L) { ++ throw new JSONException(key + " must be non-negative"); ++ } ++ return parsed; ++ } ++ ++ private static void rejectSecretMaterial(String value) { ++ final String lower = value.toLowerCase(Locale.ROOT); ++ if (lower.contains("api_key") || lower.contains("access_token") ++ || lower.contains("bearer ") || lower.contains("private key") ++ || lower.contains("password=")) { ++ throw new IllegalArgumentException( ++ "Secret-like material is not allowed in island state"); ++ } ++ } ++ + private static String emptyUiTreeSnapshot(String reason) { + try { + return new JSONObject() +@@ -376,6 +608,20 @@ public final class OpenPhoneAgentManagerService extends SystemService { + "OpenPhone task management requires MANAGE_AGENT_TASKS"); + } + ++ private void enforceAssistantIslandPublisher() { ++ final int callerUid = Binder.getCallingUid(); ++ final String[] packages = mContext.getPackageManager().getPackagesForUid(callerUid); ++ if (packages != null) { ++ for (String packageName : packages) { ++ if (ASSISTANT_PACKAGE.equals(packageName)) { ++ return; ++ } ++ } ++ } ++ throw new SecurityException( ++ "Only the OpenPhone assistant may publish island state"); ++ } ++ + private void enforceReadAuditLog() { + mContext.enforceCallingOrSelfPermission( + PERMISSION_READ_AGENT_AUDIT_LOG, +@@ -393,6 +639,11 @@ public final class OpenPhoneAgentManagerService extends SystemService { + json.put("pending_actions", pendingActions); + json.put("active_tasks", activeTaskIds.length()); + json.put("active_task_ids", activeTaskIds); ++ synchronized (mIslandStateLock) { ++ json.put("island_state_revision", mIslandStateRevision); ++ json.put("island_state_listeners", ++ mIslandStateListeners.getRegisteredCallbackCount()); ++ } + json.put("audit_storage", "durable"); + json.put("audit_path", auditPath); + } catch (JSONException e) { +-- +2.54.0 diff --git a/patches/frameworks_base/0021-OpenPhone-render-compact-island-in-SystemUI.patch b/patches/frameworks_base/0021-OpenPhone-render-compact-island-in-SystemUI.patch new file mode 100644 index 0000000..3794872 --- /dev/null +++ b/patches/frameworks_base/0021-OpenPhone-render-compact-island-in-SystemUI.patch @@ -0,0 +1,507 @@ +From 38f9073ca70344ee029b7f0fe4c0eabdbffa55cc Mon Sep 17 00:00:00 2001 +From: OpenPhone +Date: Sat, 18 Jul 2026 17:15:21 -0700 +Subject: [PATCH 21/21] OpenPhone: render compact island in SystemUI + +--- + .../openphone/OpenPhoneIslandController.java | 456 ++++++++++++++++++ + .../openphone/OpenPhoneSystemUIModule.kt | 9 + + 2 files changed, 465 insertions(+) + create mode 100644 packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneIslandController.java + +diff --git a/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneIslandController.java b/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneIslandController.java +new file mode 100644 +index 00000000..2fb9b890 +--- /dev/null ++++ b/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneIslandController.java +@@ -0,0 +1,456 @@ ++/* ++ * Copyright (C) 2026 OpenPhone ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); ++ * you may not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software distributed under the ++ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY ++ * KIND, either express or implied. See the License for the specific language governing ++ * permissions and limitations under the License. ++ */ ++ ++package com.android.systemui.openphone; ++ ++import android.app.KeyguardManager; ++import android.content.BroadcastReceiver; ++import android.content.ComponentName; ++import android.content.Context; ++import android.content.Intent; ++import android.content.IntentFilter; ++import android.graphics.PixelFormat; ++import android.graphics.Typeface; ++import android.graphics.drawable.GradientDrawable; ++import android.openphone.IOpenPhoneIslandStateListener; ++import android.openphone.OpenPhoneAgentManager; ++import android.os.Handler; ++import android.os.Looper; ++import android.os.SystemClock; ++import android.util.Log; ++import android.view.Gravity; ++import android.view.HapticFeedbackConstants; ++import android.view.View; ++import android.view.WindowManager; ++import android.widget.LinearLayout; ++import android.widget.TextView; ++ ++import com.android.systemui.CoreStartable; ++import com.android.systemui.dagger.SysUISingleton; ++import com.android.systemui.plugins.ActivityStarter; ++ ++import org.json.JSONException; ++import org.json.JSONObject; ++ ++import java.io.PrintWriter; ++import java.util.Set; ++ ++import javax.inject.Inject; ++ ++/** ++ * SystemUI-owned compact renderer for the OpenPhone island. ++ * ++ *

The renderer consumes a small projection cached by system_server. It owns ++ * only a bounded touchable window: touches outside the chip continue directly ++ * to the app underneath. Review and private content are always redacted while ++ * the device is locked.

++ */ ++@SysUISingleton ++public final class OpenPhoneIslandController implements CoreStartable { ++ private static final String TAG = "OpenPhoneIsland"; ++ private static final String ISLAND_SCHEMA = "openphone.island_state.v1"; ++ private static final long STATE_STALE_MS = 2 * 60 * 1000L; ++ private static final long STALE_CHECK_MS = 30 * 1000L; ++ private static final Set ACTIVE_MODES = Set.of( ++ "listening", "thinking", "realtime", "action_running"); ++ ++ private final Context mContext; ++ private final ActivityStarter mActivityStarter; ++ private final Handler mMainHandler = new Handler(Looper.getMainLooper()); ++ private final WindowManager mWindowManager; ++ private final KeyguardManager mKeyguardManager; ++ private final OpenPhoneAgentManager mAgentManager; ++ ++ private LinearLayout mRoot; ++ private TextView mLeftText; ++ private TextView mRightText; ++ private IslandState mState = IslandState.unavailable(); ++ private boolean mStarted; ++ ++ private final IOpenPhoneIslandStateListener mListener = ++ new IOpenPhoneIslandStateListener.Stub() { ++ @Override ++ public void onIslandStateChanged(String stateJson) { ++ mMainHandler.post(() -> acceptState(stateJson)); ++ } ++ }; ++ ++ private final Runnable mStaleCheck = new Runnable() { ++ @Override ++ public void run() { ++ render(); ++ mMainHandler.postDelayed(this, STALE_CHECK_MS); ++ } ++ }; ++ ++ private final BroadcastReceiver mLockStateReceiver = new BroadcastReceiver() { ++ @Override ++ public void onReceive(Context context, Intent intent) { ++ mMainHandler.post(OpenPhoneIslandController.this::render); ++ } ++ }; ++ ++ @Inject ++ public OpenPhoneIslandController(Context context, ActivityStarter activityStarter) { ++ mContext = context; ++ mActivityStarter = activityStarter; ++ mWindowManager = context.getSystemService(WindowManager.class); ++ mKeyguardManager = context.getSystemService(KeyguardManager.class); ++ mAgentManager = context.getSystemService(OpenPhoneAgentManager.class); ++ } ++ ++ @Override ++ public void start() { ++ if (mStarted) { ++ return; ++ } ++ mStarted = true; ++ ++ IntentFilter lockFilter = new IntentFilter(); ++ lockFilter.addAction(Intent.ACTION_SCREEN_ON); ++ lockFilter.addAction(Intent.ACTION_SCREEN_OFF); ++ lockFilter.addAction(Intent.ACTION_USER_PRESENT); ++ try { ++ mContext.registerReceiver( ++ mLockStateReceiver, lockFilter, Context.RECEIVER_NOT_EXPORTED); ++ } catch (RuntimeException e) { ++ Log.w(TAG, "Unable to observe lock state", e); ++ } ++ ++ if (mAgentManager == null || mWindowManager == null) { ++ Log.w(TAG, "Island dependencies unavailable; showing bounded degraded state"); ++ render(); ++ } else { ++ try { ++ mAgentManager.registerIslandStateListener(mListener); ++ acceptState(mAgentManager.getIslandState()); ++ } catch (RuntimeException e) { ++ Log.w(TAG, "Unable to subscribe to island state", e); ++ render(); ++ } ++ } ++ mMainHandler.postDelayed(mStaleCheck, STALE_CHECK_MS); ++ } ++ ++ private void acceptState(String stateJson) { ++ IslandState candidate = IslandState.parse(stateJson); ++ if (candidate == null) { ++ Log.w(TAG, "Ignoring invalid island state"); ++ return; ++ } ++ if (candidate.revision < mState.revision) { ++ return; ++ } ++ mState = candidate; ++ render(); ++ } ++ ++ private void render() { ++ // A hidden state is an explicit Home-space ownership decision, not a ++ // health signal. Preserve it even if the assistant later disappears. ++ if (!mState.visible) { ++ removeView(); ++ return; ++ } ++ IslandState state = isStale(mState) ? IslandState.unavailable() : mState; ++ ensureView(); ++ if (mRoot == null) { ++ return; ++ } ++ ++ boolean locked = mKeyguardManager != null && mKeyguardManager.isDeviceLocked(); ++ boolean redact = locked ++ && (!"public".equals(state.sensitivity) ++ || state.needsAttention ++ || !state.pendingConfirmationId.isEmpty()); ++ RenderedState rendered = renderedState(state, redact); ++ mLeftText.setText(rendered.left); ++ mRightText.setText(rendered.right); ++ mRightText.setTextColor(rendered.accentColor); ++ mRoot.setBackground(chipBackground(state.needsAttention && !redact)); ++ mRoot.setContentDescription(rendered.contentDescription); ++ } ++ ++ private RenderedState renderedState(IslandState state, boolean redact) { ++ if (redact) { ++ return new RenderedState("AI", "⌾", 0xfff4f7f8, ++ "OpenPhone. Unlock to review private activity."); ++ } ++ if (state.needsAttention || "needs_review".equals(state.mode)) { ++ return new RenderedState("!", "", 0xffffcc6c, ++ "OpenPhone approval needed. Double tap to review."); ++ } ++ switch (state.mode) { ++ case "listening": ++ return new RenderedState("●", "■", 0xffff6b6b, ++ "OpenPhone is listening. Double tap to open."); ++ case "thinking": ++ return new RenderedState("", "...", 0xff9ab8ff, ++ "OpenPhone is thinking. Double tap to open."); ++ case "realtime": ++ return new RenderedState("⚡", "...", 0xffffcc6c, ++ "OpenPhone realtime session is active. Double tap to open."); ++ case "action_running": ++ return new RenderedState("●", "■", 0xffff6b6b, ++ "OpenPhone task running. Double tap to open; long press to stop."); ++ case "answer_ready": ++ return new RenderedState("", "✓", 0xff20e36a, ++ "OpenPhone finished. Double tap to open."); ++ case "transcript": ++ case "reply": ++ return new RenderedState("AI", "✓", 0xff72e0c4, ++ "OpenPhone response ready. Double tap to open."); ++ case "error": ++ return new RenderedState("!", "", 0xffff6b6b, ++ "OpenPhone needs attention. Double tap to open."); ++ case "watching": ++ return new RenderedState("AI", ++ state.liveRuns > 1 ? "◎ " + state.liveRuns : "◎", ++ 0xff9ab8ff, ++ "OpenPhone background activity. Double tap to open."); ++ case "unavailable": ++ return new RenderedState("AI", "○", 0xffaeb8bf, ++ "OpenPhone assistant unavailable. Double tap to open."); ++ case "idle": ++ default: ++ return new RenderedState("AI", ++ state.liveRuns > 0 ? "◎ " + state.liveRuns : "◉", ++ 0xff72e0c4, ++ "OpenPhone ready. Double tap to open."); ++ } ++ } ++ ++ private void ensureView() { ++ if (mRoot != null || mWindowManager == null) { ++ return; ++ } ++ LinearLayout root = new LinearLayout(mContext); ++ root.setOrientation(LinearLayout.HORIZONTAL); ++ root.setGravity(Gravity.CENTER_VERTICAL); ++ root.setPadding(dp(4), 0, dp(4), 0); ++ root.setClickable(true); ++ root.setFocusable(true); ++ root.setElevation(0); ++ root.setOnClickListener(view -> launchAiHome()); ++ root.setOnLongClickListener(this::handleLongPress); ++ ++ TextView left = islandText(Gravity.CENTER); ++ TextView right = islandText(Gravity.CENTER); ++ root.addView(left, new LinearLayout.LayoutParams( ++ 0, LinearLayout.LayoutParams.MATCH_PARENT, 1f)); ++ View cameraSpace = new View(mContext); ++ root.addView(cameraSpace, new LinearLayout.LayoutParams( ++ dp(37), LinearLayout.LayoutParams.MATCH_PARENT)); ++ root.addView(right, new LinearLayout.LayoutParams( ++ 0, LinearLayout.LayoutParams.MATCH_PARENT, 1f)); ++ ++ WindowManager.LayoutParams params = new WindowManager.LayoutParams( ++ dp(110), ++ dp(34), ++ WindowManager.LayoutParams.TYPE_STATUS_BAR_SUB_PANEL, ++ WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE ++ | WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL ++ | WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN, ++ PixelFormat.TRANSLUCENT); ++ params.gravity = Gravity.TOP | Gravity.CENTER_HORIZONTAL; ++ params.y = dp(16); ++ params.layoutInDisplayCutoutMode = ++ WindowManager.LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS; ++ params.setFitInsetsTypes(0); ++ params.setTitle("OpenPhoneIsland"); ++ ++ try { ++ mWindowManager.addView(root, params); ++ mRoot = root; ++ mLeftText = left; ++ mRightText = right; ++ } catch (RuntimeException e) { ++ Log.w(TAG, "Unable to add bounded island window", e); ++ } ++ } ++ ++ private boolean handleLongPress(View view) { ++ view.performHapticFeedback(HapticFeedbackConstants.LONG_PRESS); ++ boolean locked = mKeyguardManager != null && mKeyguardManager.isDeviceLocked(); ++ IslandState state = mState; ++ if (locked || isStale(state) || state.needsAttention || state.activeTaskId.isEmpty() ++ || !ACTIVE_MODES.contains(state.mode) || mAgentManager == null) { ++ launchAiHome(); ++ return true; ++ } ++ try { ++ mAgentManager.stopTask( ++ state.activeTaskId, ++ "{\"reason\":\"systemui_island_long_press\"}"); ++ } catch (RuntimeException e) { ++ Log.w(TAG, "Unable to stop island task", e); ++ launchAiHome(); ++ } ++ return true; ++ } ++ ++ private void launchAiHome() { ++ Intent intent = new Intent(); ++ intent.setComponent(new ComponentName( ++ "org.openphone.assistant", ++ "org.openphone.assistant.OpenPhoneHomeActivity")); ++ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK ++ | Intent.FLAG_ACTIVITY_SINGLE_TOP ++ | Intent.FLAG_ACTIVITY_CLEAR_TOP); ++ mActivityStarter.postStartActivityDismissingKeyguard(intent, 0); ++ } ++ ++ private void removeView() { ++ if (mRoot == null || mWindowManager == null) { ++ return; ++ } ++ try { ++ mWindowManager.removeView(mRoot); ++ } catch (RuntimeException e) { ++ Log.w(TAG, "Unable to remove island window", e); ++ } ++ mRoot = null; ++ mLeftText = null; ++ mRightText = null; ++ } ++ ++ private TextView islandText(int gravity) { ++ TextView view = new TextView(mContext); ++ view.setTextColor(0xfff4f7f8); ++ view.setTextSize(14); ++ view.setTypeface(Typeface.DEFAULT_BOLD); ++ view.setGravity(Gravity.CENTER_VERTICAL | gravity); ++ view.setSingleLine(true); ++ view.setEllipsize(android.text.TextUtils.TruncateAt.END); ++ return view; ++ } ++ ++ private GradientDrawable chipBackground(boolean attention) { ++ GradientDrawable background = new GradientDrawable(); ++ background.setColor(0xff000000); ++ background.setCornerRadius(dp(17)); ++ if (attention) { ++ background.setStroke(dp(1), 0xffffcc6c); ++ } ++ return background; ++ } ++ ++ private boolean isStale(IslandState state) { ++ if (state.publishedElapsedMs <= 0L) { ++ return true; ++ } ++ long age = SystemClock.elapsedRealtime() - state.publishedElapsedMs; ++ return age < -10_000L || age > STATE_STALE_MS; ++ } ++ ++ private int dp(int value) { ++ return Math.round(value * mContext.getResources().getDisplayMetrics().density); ++ } ++ ++ @Override ++ public void dump(PrintWriter pw, String[] args) { ++ pw.println("OpenPhone island:"); ++ pw.println(" started=" + mStarted); ++ pw.println(" attached=" + (mRoot != null)); ++ pw.println(" revision=" + mState.revision); ++ pw.println(" mode=" + mState.mode); ++ pw.println(" visible=" + mState.visible); ++ pw.println(" stale=" + isStale(mState)); ++ pw.println(" needsAttention=" + mState.needsAttention); ++ pw.println(" liveRuns=" + mState.liveRuns); ++ } ++ ++ private static final class RenderedState { ++ final String left; ++ final String right; ++ final int accentColor; ++ final String contentDescription; ++ ++ RenderedState(String left, String right, int accentColor, String contentDescription) { ++ this.left = left; ++ this.right = right; ++ this.accentColor = accentColor; ++ this.contentDescription = contentDescription; ++ } ++ } ++ ++ private static final class IslandState { ++ final long revision; ++ final String mode; ++ final String label; ++ final boolean visible; ++ final boolean needsAttention; ++ final String sensitivity; ++ final String activeTaskId; ++ final String pendingConfirmationId; ++ final int liveRuns; ++ final long publishedElapsedMs; ++ ++ IslandState(long revision, String mode, String label, boolean visible, ++ boolean needsAttention, String sensitivity, String activeTaskId, ++ String pendingConfirmationId, int liveRuns, long publishedElapsedMs) { ++ this.revision = revision; ++ this.mode = clean(mode); ++ this.label = clean(label); ++ this.visible = visible; ++ this.needsAttention = needsAttention; ++ this.sensitivity = clean(sensitivity); ++ this.activeTaskId = clean(activeTaskId); ++ this.pendingConfirmationId = clean(pendingConfirmationId); ++ this.liveRuns = Math.max(0, Math.min(liveRuns, 100)); ++ this.publishedElapsedMs = publishedElapsedMs; ++ } ++ ++ static IslandState parse(String raw) { ++ try { ++ JSONObject json = new JSONObject(raw); ++ if (!ISLAND_SCHEMA.equals(json.getString("schema"))) { ++ return null; ++ } ++ return new IslandState( ++ json.getLong("revision"), ++ json.getString("mode"), ++ json.optString("label", ""), ++ json.getBoolean("visible"), ++ json.getBoolean("needs_attention"), ++ json.getString("sensitivity"), ++ json.optString("active_task_id", ""), ++ json.optString("pending_confirmation_id", ""), ++ json.getInt("live_runs"), ++ json.getLong("published_elapsed_ms")); ++ } catch (JSONException | RuntimeException e) { ++ return null; ++ } ++ } ++ ++ static IslandState unavailable() { ++ return new IslandState( ++ 0L, ++ "unavailable", ++ "OpenPhone", ++ true, ++ false, ++ "public", ++ "", ++ "", ++ 0, ++ SystemClock.elapsedRealtime()); ++ } ++ ++ private static String clean(String value) { ++ return value == null ? "" : value.trim(); ++ } ++ } ++} +diff --git a/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneSystemUIModule.kt b/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneSystemUIModule.kt +index a9ada9e8..a21ea41c 100644 +--- a/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneSystemUIModule.kt ++++ b/packages/SystemUI/src/com/android/systemui/openphone/OpenPhoneSystemUIModule.kt +@@ -16,6 +16,7 @@ + + package com.android.systemui.openphone + ++import com.android.systemui.CoreStartable + import com.android.systemui.qs.QsEventLogger + import com.android.systemui.qs.pipeline.shared.TileSpec + import com.android.systemui.qs.shared.model.TileCategory +@@ -27,11 +28,19 @@ import com.android.systemui.res.R + import dagger.Binds + import dagger.Module + import dagger.Provides ++import dagger.multibindings.ClassKey + import dagger.multibindings.IntoMap + import dagger.multibindings.StringKey + + @Module + interface OpenPhoneSystemUIModule { ++ @Binds ++ @IntoMap ++ @ClassKey(OpenPhoneIslandController::class) ++ fun bindOpenPhoneIslandController( ++ controller: OpenPhoneIslandController, ++ ): CoreStartable ++ + @Binds + @IntoMap + @StringKey(OpenPhoneAgentTile.TILE_SPEC) +-- +2.54.0 diff --git a/patches/packages_apps_Launcher3/0001-OpenPhone-make-AI-Home-the-default-home-surface.patch b/patches/packages_apps_Launcher3/0001-OpenPhone-make-AI-Home-the-default-home-surface.patch new file mode 100644 index 0000000..8127d38 --- /dev/null +++ b/patches/packages_apps_Launcher3/0001-OpenPhone-make-AI-Home-the-default-home-surface.patch @@ -0,0 +1,56 @@ +From 7f5f6f8a0ec4e79f2c5be3f13dbf8a08d0cda801 Mon Sep 17 00:00:00 2001 +From: OpenPhone +Date: Sat, 18 Jul 2026 18:05:00 -0700 +Subject: [PATCH] OpenPhone: prefer AI Home as the default home surface + +Keep Launcher3's HOME intent resolvable inside Quickstep, which uses its own +HOME activity to initialize overview even when another launcher is the +default. Give that filter a lower priority so OpenPhoneHomeActivity remains +the deterministic default while AI Home can launch Launcher3 explicitly as +App Space. + +Change-Id: I5d2b98f67bc5fe81472841fd35e536af983a411c +--- + AndroidManifest.xml | 2 +- + go/AndroidManifest-launcher.xml | 2 +- + quickstep/AndroidManifest-launcher.xml | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/AndroidManifest.xml b/AndroidManifest.xml +--- a/AndroidManifest.xml ++++ b/AndroidManifest.xml +@@ -62,7 +62,7 @@ + android:taskAffinity="" + android:exported="true" + android:enabled="true"> +- ++ + + + +diff --git a/go/AndroidManifest-launcher.xml b/go/AndroidManifest-launcher.xml +--- a/go/AndroidManifest-launcher.xml ++++ b/go/AndroidManifest-launcher.xml +@@ -55,7 +55,7 @@ + android:taskAffinity="" + android:exported="true" + android:enabled="true"> +- ++ + + + +diff --git a/quickstep/AndroidManifest-launcher.xml b/quickstep/AndroidManifest-launcher.xml +--- a/quickstep/AndroidManifest-launcher.xml ++++ b/quickstep/AndroidManifest-launcher.xml +@@ -54,7 +54,7 @@ + android:taskAffinity="" + android:exported="true" + android:enabled="true"> +- ++ + + + +-- +2.50.1 diff --git a/runtime/protocol/ifp1/MIGRATION.md b/runtime/protocol/ifp1/MIGRATION.md new file mode 100644 index 0000000..279ef14 --- /dev/null +++ b/runtime/protocol/ifp1/MIGRATION.md @@ -0,0 +1,105 @@ +# Migration: OpenPhone runtime protocol v0 -> Interfaces Protocol v1 (`ifp/1`) + +- **Status:** in progress on `rewrite/protocol-v1` (WS5 of the Interfaces v1 + rewrite plan; ADR 0004 in silentspeech-app + `docs/architecture/adr/0004-interfaces-protocol-v1-greenfield.md`). +- **Spec:** silentspeech-app + `packages/interfaces-protocol/spec/interfaces-protocol-1.md` + (v1.0.0-draft — fixture corpus not yet frozen). +- **This directory** vendors the ifp/1 TypeScript reference validator and + state machine (`validator.ts`, `machine.ts`), the conformance fixture + corpus (`fixtures/`), the OpenPhone tool capability manifest + (`openphone-tools.ifp1.json`), and the legacy-event adapter + (`adapter.mjs`). Tests: + `tests/integrations/ifp1-conformance-contract.mjs` and + `tests/integrations/ifp1-adapter-contract.mjs`, run by + `scripts/check-runtime-protocol.sh` (and so by `scripts/check.sh`). + +## Deprecation of the v0 manifests + +`runtime/protocol/openphone-commands.json` and +`runtime/protocol/openphone-events.json` now carry top-level +`"deprecated": true` and `"superseded_by": "ifp1/"`. The v0 loader +(`openphone-runtime-tools.mjs`), the manifest validator +(`validate-runtime-protocol.mjs`), and all contract tests tolerate the +extra top-level keys (verified; the vendored copies under +`integrations/cli/runtime/protocol/` and +`integrations/mcp-server/runtime/protocol/` are byte-synced as the +package contract requires). The old manifests are NOT deleted yet: the +Java side (`RuntimeToolBridge`, `OpenClawCommandRegistry`) and the +OpenClaw plugin still read them. Deletion is gated on the remainder items +in `WS5-REMAINDER.md` at the repo root. + +## Concept mapping + +| Runtime protocol v0 | ifp/1 | +|---|---| +| Command (`openphone.`) | Tool capability `tool:` declared at hello/pairing (spec §6.4); invoked via the action lifecycle (spec §8) | +| `runtime.tool.requested` event | `action.propose` (runtime -> surface) | +| `runtime.confirmation.required` event | No wire message: confirmation is surface-local authority (spec §8.2); the runtime only observes the outcome | +| `runtime.confirmation.resolved` event | `action.approve` (with exact `params_hash` binding + `confirmation.mode`/`ref`) or `action.reject` (`reason_class`, terminal) | +| `runtime.tool.result` event | `action.result` (`executed`/`failed`; `binding_mismatch` if executed params diverge from the approved hash; `action_expired` past `expires_at`) | +| Autonomy `observe_only` / `ask_before_action` / `trusted_actions` | Surface-local policy, NOT negotiable over the wire (spec §8.2). `trusted_actions` auto-approvals become `confirmation.mode: "policy_auto"` | +| `confirmation: "always"` command metadata | `risk: "high"` on the tool declaration and on `action.propose` | +| Ed25519 device identity (OpenClaw adapter) | ifp/1 pairing (spec §6.1–6.3): keypair at pairing, challenge signature, `control.hello` proof over the connection nonce | +| Manifest integer `version` + range negotiation | Envelope `ifp: "1"` major version; `control.hello payload.protocol {min,max}` negotiation | +| `deprecated`/`superseded_by` per command | Same policy, adopted by the ifp/1 type registry (spec §3) | + +## Command -> tool mapping (all 37) + +The ifp/1 tool name is the v0 command name without the `openphone.` prefix. +Aliases are retired: ifp/1 has exactly one name per tool (old aliases are +recorded in each manifest entry's `x-openphone.superseded_aliases`). +`risk` carries over from the v0 manifest unchanged; the v0 `confirmation` +and `default_exposure` columns become surface-local policy inputs and are +preserved under `x-openphone` for the Java bridge migration. + +| v0 command | v0 aliases | ifp/1 tool | risk | v0 confirmation | v0 exposure | +|---|---|---|---|---|---| +| `openphone.device.status` | `device.status`, `device.info` | `device.status` | medium | none | private_read | +| `openphone.apps.search` | `device.apps` | `apps.search` | low | none | safe_read | +| `openphone.screen.get` | `canvas.snapshot` | `screen.get` | low | none | safe_read | +| `openphone.screen.understand_local` | `openphone.local.screen_understanding` | `screen.understand_local` | low | none | safe_read | +| `openphone.notifications.list` | `notifications.list` | `notifications.list` | medium | none | private_read | +| `openphone.notifications.search` | `notifications.search` | `notifications.search` | medium | none | private_read | +| `openphone.notifications.open` | `notifications.open` | `notifications.open` | medium | ask_before_action | dangerous | +| `openphone.contacts.search` | `contacts.search` | `contacts.search` | medium | none | private_read | +| `openphone.calendar.search` | `calendar.events` | `calendar.search` | medium | none | private_read | +| `openphone.calendar.add` | `calendar.add` | `calendar.add` | medium | ask_before_action | dangerous | +| `openphone.calendar.update` | `calendar.update` | `calendar.update` | medium | ask_before_action | dangerous | +| `openphone.calendar.delete` | `calendar.delete` | `calendar.delete` | high | always | dangerous | +| `openphone.messages.search` | `sms.search` | `messages.search` | medium | none | private_read | +| `openphone.messages.draft` | `sms.draft` | `messages.draft` | medium | ask_before_action | dangerous | +| `openphone.messages.send` | `sms.send` | `messages.send` | high | always | dangerous | +| `openphone.calls.search` | `callLog.search` | `calls.search` | medium | none | private_read | +| `openphone.calls.place` | `calls.place` | `calls.place` | high | always | dangerous | +| `openphone.memory.search` | — | `memory.search` | low | none | private_read | +| `openphone.memory.save` | — | `memory.save` | medium | ask_before_action | dangerous | +| `openphone.watchers.list` | — | `watchers.list` | low | none | private_read | +| `openphone.watchers.create` | — | `watchers.create` | medium | ask_before_action | dangerous | +| `openphone.watchers.stop` | — | `watchers.stop` | medium | ask_before_action | dangerous | +| `openphone.jobs.list` | — | `jobs.list` | low | none | safe_read | +| `openphone.jobs.create` | — | `jobs.create` | medium | ask_before_action | dangerous | +| `openphone.jobs.stop` | — | `jobs.stop` | medium | ask_before_action | dangerous | +| `openphone.app.open` | — | `app.open` | low | ask_before_action | dangerous | +| `openphone.url.open` | — | `url.open` | medium | ask_before_action | dangerous | +| `openphone.ui.tap` | — | `ui.tap` | medium | ask_before_action | dangerous | +| `openphone.ui.tap_element` | — | `ui.tap_element` | medium | ask_before_action | dangerous | +| `openphone.ui.long_press` | — | `ui.long_press` | medium | ask_before_action | dangerous | +| `openphone.ui.long_press_element` | — | `ui.long_press_element` | medium | ask_before_action | dangerous | +| `openphone.ui.swipe` | — | `ui.swipe` | medium | ask_before_action | dangerous | +| `openphone.ui.type_text` | — | `ui.type_text` | medium | ask_before_action | dangerous | +| `openphone.input.press_key` | — | `input.press_key` | medium | ask_before_action | dangerous | +| `openphone.clipboard.set` | — | `clipboard.set` | low | ask_before_action | dangerous | +| `openphone.clipboard.paste` | — | `clipboard.paste` | medium | ask_before_action | dangerous | +| `openphone.share.text` | — | `share.text` | high | always | dangerous | + +## params_hash + +`action.propose`/`action.approve` bind exact parameters with +`params_hash` = `sha256:` + hex SHA-256 of the canonical JSON encoding of +`params` (UTF-8, keys sorted lexicographically at every depth, no +insignificant whitespace) — spec §8.1. Reference implementation: +`adapter.mjs` `canonicalJson()`/`paramsHash()`; shared test vectors: +`fixtures/params-hash-vectors.json` (the first vector is the spec §8.1 +example params object). diff --git a/runtime/protocol/ifp1/adapter.mjs b/runtime/protocol/ifp1/adapter.mjs new file mode 100644 index 0000000..1efe384 --- /dev/null +++ b/runtime/protocol/ifp1/adapter.mjs @@ -0,0 +1,226 @@ +/** + * ifp/1 action-lifecycle adapter for the OpenPhone runtime protocol v0. + * + * Maps the legacy runtime events onto Interfaces Protocol v1 action messages + * (spec section 8 of packages/interfaces-protocol/spec/interfaces-protocol-1.md + * in silentspeech-app, vendored alongside as validator.ts/machine.ts): + * + * runtime.tool.requested -> action.propose (runtime -> surface) + * runtime.confirmation.required -> (no wire message: confirmation is + * surface-local state, spec section 8.2; + * the runtime only sees the outcome) + * runtime.confirmation.resolved -> action.approve | action.reject + * (surface -> runtime, exact params_hash + * binding, spec section 8.3) + * runtime.tool.result -> action.result (surface -> runtime, + * binding_mismatch on divergence, + * spec section 8.4) + * + * params_hash is "sha256:" + hex SHA-256 of the canonical JSON encoding of + * params (UTF-8, lexicographically sorted keys at every depth, no + * insignificant whitespace) — spec section 8.1. Shared test vectors live in + * fixtures/params-hash-vectors.json. + */ + +import { createHash, randomBytes } from "node:crypto"; + +export const OLD_COMMAND_PREFIX = "openphone."; + +/** Canonical JSON encoding per spec section 8.1. */ +export function canonicalJson(value) { + if (value === null || typeof value === "number" || typeof value === "boolean") { + return JSON.stringify(value); + } + if (typeof value === "string") { + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalJson(item ?? null)).join(",")}]`; + } + if (typeof value === "object") { + const keys = Object.keys(value).sort(); + const body = keys + .filter((key) => value[key] !== undefined) + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(","); + return `{${body}}`; + } + throw new TypeError(`value not representable in canonical JSON: ${typeof value}`); +} + +/** "sha256:" over the canonical JSON encoding of params. */ +export function paramsHash(params) { + const canonical = canonicalJson(params); + return `sha256:${createHash("sha256").update(canonical, "utf8").digest("hex")}`; +} + +/** Old command name ("openphone.messages.send") -> ifp/1 tool name ("messages.send"). */ +export function toolNameFromCommand(command) { + const name = String(command ?? ""); + return name.startsWith(OLD_COMMAND_PREFIX) ? name.slice(OLD_COMMAND_PREFIX.length) : name; +} + +function newMessageId(prefix = "msg") { + return `${prefix}_${randomBytes(12).toString("hex")}`; +} + +function isoNow(clock) { + return (clock ? new Date(clock()) : new Date()).toISOString().replace(/\.\d{3}Z$/u, "Z"); +} + +function requireContext(context, fields) { + for (const field of fields) { + if (context?.[field] === undefined || context?.[field] === null) { + throw new Error(`adapter context missing required field: ${field}`); + } + } +} + +/** + * runtime.tool.requested -> action.propose. + * + * toolRequest is the legacy toolRequest shape + * (runtime/protocol/openphone-runtime.schema.json $defs.toolRequest): + * { request_id, runtime, runtime_session_id, tool, params, reason?, + * idempotency_key?, timeout_ms? }. + * + * context: { entity, session, risk, expiresAt?, ttlMs?, clock? }. + * risk comes from the ifp1 tools manifest entry for the tool. + */ +export function proposeFromToolRequest(toolRequest, context) { + requireContext(context, ["entity", "session", "risk"]); + const params = toolRequest.params ?? {}; + const ttlMs = context.ttlMs ?? 5 * 60 * 1000; + const now = context.clock ? context.clock() : Date.now(); + const expiresAt = context.expiresAt + ?? new Date(now + ttlMs).toISOString().replace(/\.\d{3}Z$/u, "Z"); + return { + ifp: "1", + id: newMessageId(), + ts: isoNow(context.clock), + type: "action.propose", + entity: context.entity, + session: context.session, + device: null, + correlation: toolRequest.request_id ?? null, + payload: { + action_id: `act_${(toolRequest.request_id ?? newMessageId("act")).replace(/[^A-Za-z0-9]/gu, "").slice(-24) || randomBytes(8).toString("hex")}`, + tool: toolNameFromCommand(toolRequest.tool), + params, + params_hash: paramsHash(params), + reason: toolRequest.reason ?? null, + risk: context.risk, + idempotency_key: toolRequest.idempotency_key ?? toolRequest.request_id, + expires_at: expiresAt, + }, + }; +} + +/** + * runtime.confirmation.resolved -> action.approve | action.reject. + * + * resolution: { approved: boolean, confirmation_id?, reason_class? } — the + * Android-local confirmation outcome. proposal is the action.propose envelope + * this resolution answers. executedParams are the exact params the surface + * will execute (defaults to the proposal's params); the approve message + * carries their hash so the runtime's exact-binding check (spec section 8.3) + * catches any divergence as binding_mismatch. + * + * context: { entity, session, device, generation, mode?, clock? }. + * mode defaults to "user_confirmed"; pass "policy_auto" for + * trusted_actions/no-confirmation tools. + */ +export function actionMessageFromConfirmationResolution(resolution, proposal, context) { + requireContext(context, ["entity", "session", "device"]); + const base = { + ifp: "1", + id: newMessageId(), + ts: isoNow(context.clock), + entity: context.entity, + session: context.session, + device: context.device, + parent: proposal.id ?? null, + }; + if (!resolution.approved) { + return { + ...base, + type: "action.reject", + payload: { + action_id: proposal.payload.action_id, + reason_class: resolution.reason_class ?? "user_denied", + }, + }; + } + requireContext(context, ["generation"]); + const executedParams = resolution.executed_params ?? proposal.payload.params; + return { + ...base, + type: "action.approve", + generation: context.generation, + payload: { + action_id: proposal.payload.action_id, + params_hash: paramsHash(executedParams), + confirmation: { + mode: context.mode ?? "user_confirmed", + ref: resolution.confirmation_id ?? null, + }, + }, + }; +} + +/** + * runtime.tool.result -> action.result. + * + * toolResult is the legacy toolResult shape + * (runtime/protocol/openphone-runtime.schema.json $defs.toolResult): + * { request_id, status: "ok" | "error" | ..., result?, error? }. + * + * Before reporting execution, the surface re-checks exact binding: if the + * params it executed (context.executedParams) hash differently from the + * approved params_hash (context.approvedParamsHash), the result MUST be + * failed/binding_mismatch and nothing may execute (spec section 8.4). Expiry + * (context.expiresAt vs message ts) fails with action_expired. + * + * context: { entity, session, device, generation, approvedParamsHash?, + * executedParams?, expiresAt?, evidence?, clock? }. + */ +export function resultFromToolResult(toolResult, proposal, context) { + requireContext(context, ["entity", "session", "device", "generation"]); + const ts = isoNow(context.clock); + const base = { + ifp: "1", + id: newMessageId(), + ts, + type: "action.result", + entity: context.entity, + session: context.session, + device: context.device, + generation: context.generation, + parent: proposal.id ?? null, + }; + const payload = { + action_id: proposal.payload.action_id, + status: "failed", + error_class: null, + evidence: context.evidence ?? null, + }; + + if (context.approvedParamsHash && context.executedParams !== undefined) { + if (paramsHash(context.executedParams) !== context.approvedParamsHash) { + payload.error_class = "binding_mismatch"; + return { ...base, payload }; + } + } + + const expiresAt = context.expiresAt ?? proposal.payload.expires_at; + if (expiresAt && ts > expiresAt) { + payload.error_class = "action_expired"; + return { ...base, payload }; + } + + const ok = toolResult.status === "ok" || toolResult.status === "success" + || toolResult.status === "executed"; + payload.status = ok ? "executed" : "failed"; + payload.error_class = ok ? null : "internal"; + return { ...base, payload }; +} diff --git a/runtime/protocol/ifp1/fixtures/messages.json b/runtime/protocol/ifp1/fixtures/messages.json new file mode 100644 index 0000000..387a7aa --- /dev/null +++ b/runtime/protocol/ifp1/fixtures/messages.json @@ -0,0 +1,568 @@ +{ + "corpus": "ifp/1 single-message conformance", + "version": "1.0.0-draft", + "cases": [ + { + "name": "valid_minimal_control_ping", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_ping000001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.ping", + "entity": "ent_abc123", + "payload": {} + } + }, + { + "name": "valid_event_text_message", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_evt0000001", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "generation": 1, + "provenance": { + "sensor": "dev_mac_a1b2:keyboard", + "captured_at": "2026-07-21T00:00:01Z", + "chain": [ + { + "stage": "capture", + "model": null + } + ] + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_mac_a1b2:ses_s1a2b3:0", + "text": "hello" + } + } + }, + { + "name": "reject_wrong_major_version", + "expect": "reject", + "error_class": "unsupported_version", + "message": { + "ifp": "2", + "id": "msg_v2test0001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.ping", + "entity": "ent_abc123", + "payload": {} + } + }, + { + "name": "reject_event_missing_provenance", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_evtnoprov1", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "generation": 1, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "k1", + "text": "hello" + } + } + }, + { + "name": "reject_event_missing_labels", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_evtnolab01", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "generation": 1, + "provenance": { + "sensor": "dev_mac_a1b2:keyboard", + "captured_at": "2026-07-21T00:00:01Z" + }, + "payload": { + "seq": 0, + "dedup": "k1", + "text": "hello" + } + } + }, + { + "name": "reject_event_missing_generation", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_evtnogen01", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "provenance": { + "sensor": "dev_mac_a1b2:keyboard", + "captured_at": "2026-07-21T00:00:01Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "k1", + "text": "hello" + } + } + }, + { + "name": "reject_event_missing_dedup", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_evtnodedup", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "generation": 1, + "provenance": { + "sensor": "dev_mac_a1b2:keyboard", + "captured_at": "2026-07-21T00:00:01Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "text": "hello" + } + } + }, + { + "name": "reject_ext_key_without_x_prefix", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_extbad0001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.ping", + "entity": "ent_abc123", + "payload": {}, + "ext": { + "vendor": "nope" + } + } + }, + { + "name": "accept_ext_key_with_x_prefix", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_extok00001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.ping", + "entity": "ent_abc123", + "payload": {}, + "ext": { + "x-vendor": "ok" + } + } + }, + { + "name": "accept_unknown_message_type", + "expect": "accept", + "note": "Receivers must ignore unknown types, not reject them (spec §2). Validators accept the envelope; routing may no-op.", + "message": { + "ifp": "1", + "id": "msg_unknown001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.future_thing", + "entity": "ent_abc123", + "payload": { + "anything": true + } + } + }, + { + "name": "reject_bad_id_format", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "not a valid id!", + "ts": "2026-07-21T00:00:00Z", + "type": "control.ping", + "entity": "ent_abc123", + "payload": {} + } + }, + { + "name": "valid_action_propose", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_actprop001", + "ts": "2026-07-21T00:00:02Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "payload": { + "action_id": "act_00000001", + "tool": "messages.send", + "params": { + "to": "+15551234567", + "body": "on my way" + }, + "params_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "reason": "You asked me to tell Dana you're leaving.", + "risk": "medium", + "idempotency_key": "act_00000001", + "expires_at": "2026-07-21T00:05:00Z" + } + } + }, + { + "name": "reject_action_propose_missing_params_hash", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_actprop002", + "ts": "2026-07-21T00:00:02Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "payload": { + "action_id": "act_00000002", + "tool": "messages.send", + "params": {}, + "risk": "low", + "idempotency_key": "act_00000002", + "expires_at": "2026-07-21T00:05:00Z" + } + } + }, + { + "name": "reject_action_approve_missing_generation", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_actappr001", + "ts": "2026-07-21T00:00:03Z", + "type": "action.approve", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "payload": { + "action_id": "act_00000001", + "params_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "confirmation": { + "mode": "user_confirmed", + "ref": "aud_1" + } + } + } + }, + { + "name": "valid_action_reject_user_denied", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_actrej0001", + "ts": "2026-07-21T00:00:03Z", + "type": "action.reject", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "payload": { + "action_id": "act_00000001", + "reason_class": "user_denied" + } + } + }, + { + "name": "reject_action_reject_bad_reason_class", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_actrej0002", + "ts": "2026-07-21T00:00:03Z", + "type": "action.reject", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "payload": { + "action_id": "act_00000001", + "reason_class": "did_not_feel_like_it" + } + } + }, + { + "name": "valid_pair_request_null_entity", + "expect": "accept", + "note": "pair.request is the one message allowed a null entity (spec §2).", + "message": { + "ifp": "1", + "id": "msg_pairreq001", + "ts": "2026-07-21T00:00:00Z", + "type": "pair.request", + "entity": null, + "payload": { + "pairing_code": "483-921", + "public_key": "JDInBjWvZbYAt-lFQxZbV0jkyDrkejNzq5zA5PJ07UE", + "capabilities": [ + { + "type": "console", + "properties": {} + } + ], + "label": "kitchen mac" + } + } + }, + { + "name": "reject_pair_request_bad_public_key", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_pairreq002", + "ts": "2026-07-21T00:00:00Z", + "type": "pair.request", + "entity": null, + "payload": { + "pairing_code": "483-921", + "public_key": "tooshort", + "capabilities": [] + } + } + }, + { + "name": "reject_event_null_entity", + "expect": "reject", + "error_class": "invalid_envelope", + "note": "Only pair.request and control.hello may omit entity.", + "message": { + "ifp": "1", + "id": "msg_evtnoent01", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": null, + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "generation": 1, + "provenance": { + "sensor": "dev_mac_a1b2:keyboard", + "captured_at": "2026-07-21T00:00:01Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "k1", + "text": "hi" + } + } + }, + { + "name": "valid_control_hello", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_hello00001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.hello", + "entity": null, + "payload": { + "device_id": "dev_mac_a1b2", + "proof": "c2lnbmF0dXJlLW92ZXItbm9uY2U", + "capabilities": [ + { + "type": "console", + "properties": {} + }, + { + "type": "tool:messages.send", + "properties": {} + } + ], + "protocol": { + "min": 1, + "max": 1 + } + } + } + }, + { + "name": "reject_error_message_unknown_class", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_err0000001", + "ts": "2026-07-21T00:00:00Z", + "type": "error", + "entity": "ent_abc123", + "payload": { + "class": "made_up_error", + "detail": null + } + } + }, + { + "name": "valid_error_stale_generation", + "expect": "accept", + "message": { + "ifp": "1", + "id": "msg_err0000002", + "ts": "2026-07-21T00:00:00Z", + "type": "error", + "entity": "ent_abc123", + "parent": "msg_evt0000001", + "payload": { + "class": "stale_generation", + "detail": "home moved", + "current_generation": 4 + } + } + }, + { + "name": "reject_bad_consent_label", + "expect": "reject", + "error_class": "invalid_envelope", + "message": { + "ifp": "1", + "id": "msg_evtbadlab1", + "ts": "2026-07-21T00:00:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_s1a2b3", + "device": "dev_mac_a1b2", + "generation": 1, + "provenance": { + "sensor": "dev_mac_a1b2:keyboard", + "captured_at": "2026-07-21T00:00:01Z" + }, + "labels": { + "consent": "whatever_we_want", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "k1", + "text": "hi" + } + } + }, + { + "name": "reject_ts_with_numeric_offset", + "expect": "reject", + "error_class": "invalid_envelope", + "note": "Z suffix required so timestamp ordering is well-defined (spec section 2).", + "message": { + "ifp": "1", + "id": "msg_tsoffset01", + "ts": "2026-07-21T00:00:00+02:00", + "type": "control.ping", + "entity": "ent_abc123", + "payload": {} + } + }, + { + "name": "accept_error_with_null_entity", + "expect": "accept", + "note": "An error responder rejecting an unparseable message may not know the entity.", + "message": { + "ifp": "1", + "id": "msg_errnoent01", + "ts": "2026-07-21T00:00:00Z", + "type": "error", + "entity": null, + "payload": { + "class": "invalid_envelope", + "detail": "unparseable" + } + } + }, + { + "name": "accept_provenance_on_control_message", + "expect": "accept", + "note": "Provenance is required on events, tolerated elsewhere (spec section 2).", + "message": { + "ifp": "1", + "id": "msg_provctl001", + "ts": "2026-07-21T00:00:00Z", + "type": "control.ping", + "entity": "ent_abc123", + "provenance": { + "sensor": "dev_x1y2:mic", + "captured_at": "2026-07-21T00:00:00Z" + }, + "payload": {} + } + }, + { + "name": "accept_control_ack_duplicate", + "expect": "accept", + "note": "control.ack acknowledges messages with no substantive reply (spec section 3).", + "message": { + "ifp": "1", + "id": "msg_ack0000001", + "ts": "2026-07-21T00:00:05Z", + "type": "control.ack", + "entity": "ent_abc123", + "payload": { + "of": "msg_evt0000001", + "verdict": "acknowledged_duplicate", + "stored": false + } + } + }, + { + "name": "accept_pair_complete_null_entity", + "expect": "accept", + "note": "pair.complete may omit envelope entity; it is learned from pair.grant and echoed in the signed payload (spec section 6.2 step 3).", + "message": { + "ifp": "1", + "id": "msg_paircmp001", + "ts": "2026-07-21T00:00:06Z", + "type": "pair.complete", + "entity": null, + "payload": { + "device_id": "dev_kitchen01", + "entity": "ent_abc123", + "signature": "c2lnbmF0dXJlLW92ZXItY2hhbGxlbmdlLWRldmljZS1lbnRpdHk" + } + } + } + ] +} \ No newline at end of file diff --git a/runtime/protocol/ifp1/fixtures/params-hash-vectors.json b/runtime/protocol/ifp1/fixtures/params-hash-vectors.json new file mode 100644 index 0000000..b35df3d --- /dev/null +++ b/runtime/protocol/ifp1/fixtures/params-hash-vectors.json @@ -0,0 +1,37 @@ +{ + "corpus": "ifp/1 canonical params_hash test vectors (spec section 8.1)", + "version": "1.0.0-draft", + "note": "params_hash = 'sha256:' + hex SHA-256 of the canonical JSON encoding of params: UTF-8, lexicographically sorted object keys at every depth, no insignificant whitespace. These vectors are shared between the OpenPhone adapter tests and the upstream interfaces-protocol fixture corpus; upstream them to packages/interfaces-protocol/fixtures/ before the corpus freezes (WS1).", + "vectors": [ + { + "name": "spec_8_1_example_params", + "params": { "to": "+15551234567", "body": "on my way" }, + "canonical": "{\"body\":\"on my way\",\"to\":\"+15551234567\"}", + "params_hash": "sha256:1e920af2c02daa49a449d1ebe938e08f196a9dfa6a3c05b36adbabbd7387193b" + }, + { + "name": "empty_params", + "params": {}, + "canonical": "{}", + "params_hash": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + { + "name": "nested_objects_and_arrays_sorted_at_every_depth", + "params": { "z": { "b": 2, "a": 1 }, "a": [3, { "y": true, "x": null }] }, + "canonical": "{\"a\":[3,{\"x\":null,\"y\":true}],\"z\":{\"a\":1,\"b\":2}}", + "params_hash": "sha256:ec6c43224edc1d346f2e8e60a653b47df2346dabf8d9e4f8d7b378845125a250" + }, + { + "name": "unicode_is_utf8_not_ascii_escaped", + "params": { "body": "café → 😀" }, + "canonical": "{\"body\":\"café → 😀\"}", + "params_hash": "sha256:c8e0a199ed18106634a6a12bd76a9a2211e68028bc5846d208a6a8c77234557c" + }, + { + "name": "key_order_of_input_is_irrelevant", + "params": { "body": "on my way", "to": "+15551234567" }, + "canonical": "{\"body\":\"on my way\",\"to\":\"+15551234567\"}", + "params_hash": "sha256:1e920af2c02daa49a449d1ebe938e08f196a9dfa6a3c05b36adbabbd7387193b" + } + ] +} diff --git a/runtime/protocol/ifp1/fixtures/sequences.json b/runtime/protocol/ifp1/fixtures/sequences.json new file mode 100644 index 0000000..484382b --- /dev/null +++ b/runtime/protocol/ifp1/fixtures/sequences.json @@ -0,0 +1,698 @@ +{ + "corpus": "ifp/1 stateful sequence conformance", + "version": "1.0.0-draft", + "note": "Each scenario is a list of steps applied to a fresh conformance state machine. 'give' delivers a message; 'expect' asserts the machine's verdict and observable state. Validators implement the reference state machine described in spec §6-§8.", + "initial_state": { + "home_generation": 3, + "known_dedup": [], + "sessions": {} + }, + "scenarios": [ + { + "name": "replay_dedup_accept_then_drop", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_r10000", + "ts": "2026-07-21T01:00:00Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_r10000", + "device": "dev_d100", + "generation": 3, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:00:00Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_d100:ses_r10000:0", + "text": "a" + } + }, + "expect": { + "verdict": "accepted", + "stored": true + } + }, + { + "give": { + "ifp": "1", + "id": "msg_r20000", + "ts": "2026-07-21T01:00:05Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_r10000", + "device": "dev_d100", + "generation": 3, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:00:00Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_d100:ses_r10000:0", + "text": "a" + } + }, + "expect": { + "verdict": "acknowledged_duplicate", + "stored": false + } + } + ] + }, + { + "name": "ordering_violation_seq_regression_new_dedup", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_o10000", + "ts": "2026-07-21T01:01:00Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_o10000", + "device": "dev_d100", + "generation": 3, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:01:00Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 5, + "dedup": "dev_d100:ses_o10000:5", + "text": "later" + } + }, + "expect": { + "verdict": "accepted" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_o20000", + "ts": "2026-07-21T01:01:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_o10000", + "device": "dev_d100", + "generation": 3, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:01:01Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 2, + "dedup": "dev_d100:ses_o10000:2-new", + "text": "earlier" + } + }, + "expect": { + "verdict": "rejected", + "error_class": "ordering_violation" + } + } + ] + }, + { + "name": "stale_generation_fence", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_g10000", + "ts": "2026-07-21T01:02:00Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_g10000", + "device": "dev_d100", + "generation": 2, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:02:00Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_d100:ses_g10000:0", + "text": "stale writer" + } + }, + "expect": { + "verdict": "rejected", + "error_class": "stale_generation", + "error_payload_includes": { + "current_generation": 3 + } + } + }, + { + "give": { + "ifp": "1", + "id": "msg_g20000", + "ts": "2026-07-21T01:02:01Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_g10000", + "device": "dev_d100", + "generation": 3, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:02:01Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_d100:ses_g10000:0b", + "text": "resynced" + } + }, + "expect": { + "verdict": "accepted" + } + } + ] + }, + { + "name": "action_lifecycle_approve_execute", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_a10000", + "ts": "2026-07-21T01:03:00Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_a10000", + "payload": { + "action_id": "act_life0001", + "tool": "messages.send", + "params": { + "to": "+15551234567", + "body": "hi" + }, + "params_hash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "risk": "medium", + "idempotency_key": "act_life0001", + "expires_at": "2026-07-21T01:08:00Z" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "requested" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_a20000", + "ts": "2026-07-21T01:03:05Z", + "type": "action.approve", + "entity": "ent_abc123", + "session": "ses_a10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_life0001", + "params_hash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "confirmation": { + "mode": "user_confirmed", + "ref": "aud_local_1" + } + } + }, + "expect": { + "verdict": "accepted", + "action_state": "approved" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_a30000", + "ts": "2026-07-21T01:03:07Z", + "type": "action.result", + "entity": "ent_abc123", + "session": "ses_a10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_life0001", + "status": "executed", + "error_class": null, + "evidence": "aud_local_2" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "executed" + } + } + ] + }, + { + "name": "action_lifecycle_device_rejection_is_terminal", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_b10000", + "ts": "2026-07-21T01:04:00Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_b10000", + "payload": { + "action_id": "act_rej00001", + "tool": "calls.place", + "params": { + "to": "+15550000000" + }, + "params_hash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "risk": "high", + "idempotency_key": "act_rej00001", + "expires_at": "2026-07-21T01:09:00Z" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "requested" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_b20000", + "ts": "2026-07-21T01:04:02Z", + "type": "action.reject", + "entity": "ent_abc123", + "session": "ses_b10000", + "device": "dev_d100", + "payload": { + "action_id": "act_rej00001", + "reason_class": "user_denied" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "rejected" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_b30000", + "ts": "2026-07-21T01:04:10Z", + "type": "action.approve", + "entity": "ent_abc123", + "session": "ses_b10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_rej00001", + "params_hash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "confirmation": { + "mode": "user_confirmed", + "ref": "aud_x" + } + } + }, + "expect": { + "verdict": "rejected", + "error_class": "invalid_envelope", + "note": "terminal state; approval after rejection is a protocol violation" + } + } + ] + }, + { + "name": "action_binding_mismatch", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_c10000", + "ts": "2026-07-21T01:05:00Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_c10000", + "payload": { + "action_id": "act_bind0001", + "tool": "messages.send", + "params": { + "to": "+15551234567", + "body": "original" + }, + "params_hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "risk": "medium", + "idempotency_key": "act_bind0001", + "expires_at": "2026-07-21T01:10:00Z" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "requested" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_c20000", + "ts": "2026-07-21T01:05:03Z", + "type": "action.approve", + "entity": "ent_abc123", + "session": "ses_c10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_bind0001", + "params_hash": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "confirmation": { + "mode": "user_confirmed", + "ref": "aud_y" + } + } + }, + "expect": { + "verdict": "rejected", + "error_class": "binding_mismatch", + "action_state": "requested" + } + } + ] + }, + { + "name": "action_expiry", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_d10000", + "ts": "2026-07-21T01:06:00Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_d10000", + "payload": { + "action_id": "act_exp00001", + "tool": "messages.send", + "params": { + "to": "+15551234567", + "body": "late" + }, + "params_hash": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "risk": "low", + "idempotency_key": "act_exp00001", + "expires_at": "2026-07-21T01:06:30Z" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "requested" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_d20000", + "ts": "2026-07-21T01:07:00Z", + "type": "action.approve", + "entity": "ent_abc123", + "session": "ses_d10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_exp00001", + "params_hash": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "confirmation": { + "mode": "user_confirmed", + "ref": "aud_z" + } + } + }, + "expect": { + "verdict": "rejected", + "error_class": "action_expired", + "action_state": "expired" + } + } + ] + }, + { + "name": "action_cancel_before_execution", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_e10000", + "ts": "2026-07-21T01:08:00Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_e10000", + "payload": { + "action_id": "act_can00001", + "tool": "messages.send", + "params": { + "to": "+15551234567", + "body": "wait no" + }, + "params_hash": "sha256:6666666666666666666666666666666666666666666666666666666666666666", + "risk": "medium", + "idempotency_key": "act_can00001", + "expires_at": "2026-07-21T01:13:00Z" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "requested" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_e20000", + "ts": "2026-07-21T01:08:05Z", + "type": "action.cancel", + "entity": "ent_abc123", + "session": "ses_e10000", + "payload": { + "action_id": "act_can00001" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "cancelled" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_e30000", + "ts": "2026-07-21T01:08:08Z", + "type": "action.result", + "entity": "ent_abc123", + "session": "ses_e10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_can00001", + "status": "executed", + "error_class": null, + "evidence": null + } + }, + "expect": { + "verdict": "rejected", + "error_class": "invalid_envelope", + "note": "execution after cancellation is a protocol violation" + } + } + ] + }, + { + "name": "duplicate_action_result_idempotent", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_f10000", + "ts": "2026-07-21T01:09:00Z", + "type": "action.propose", + "entity": "ent_abc123", + "session": "ses_f10000", + "payload": { + "action_id": "act_idem0001", + "tool": "messages.send", + "params": { + "to": "+15551234567", + "body": "once" + }, + "params_hash": "sha256:7777777777777777777777777777777777777777777777777777777777777777", + "risk": "medium", + "idempotency_key": "act_idem0001", + "expires_at": "2026-07-21T01:14:00Z" + } + }, + "expect": { + "verdict": "accepted" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_f20000", + "ts": "2026-07-21T01:09:02Z", + "type": "action.approve", + "entity": "ent_abc123", + "session": "ses_f10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_idem0001", + "params_hash": "sha256:7777777777777777777777777777777777777777777777777777777777777777", + "confirmation": { + "mode": "policy_auto", + "ref": null + } + } + }, + "expect": { + "verdict": "accepted", + "action_state": "approved" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_f30000", + "ts": "2026-07-21T01:09:04Z", + "type": "action.result", + "entity": "ent_abc123", + "session": "ses_f10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_idem0001", + "status": "executed", + "error_class": null, + "evidence": "aud_1" + } + }, + "expect": { + "verdict": "accepted", + "action_state": "executed" + } + }, + { + "give": { + "ifp": "1", + "id": "msg_f40000", + "ts": "2026-07-21T01:09:06Z", + "type": "action.result", + "entity": "ent_abc123", + "session": "ses_f10000", + "device": "dev_d100", + "generation": 3, + "payload": { + "action_id": "act_idem0001", + "status": "executed", + "error_class": null, + "evidence": "aud_1" + } + }, + "expect": { + "verdict": "acknowledged_duplicate", + "action_state": "executed" + } + } + ] + }, + { + "name": "duplicate_dedup_wins_over_stale_generation", + "note": "Rule precedence (spec section 7): dedup check precedes the generation fence — a replayed pre-transfer event is acknowledged, not fenced.", + "steps": [ + { + "give": { + "ifp": "1", + "id": "msg_p10000", + "ts": "2026-07-21T01:10:00Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_p10000", + "device": "dev_d100", + "generation": 3, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:10:00Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_d100:ses_p10000:0", + "text": "before transfer" + } + }, + "expect": { + "verdict": "accepted", + "stored": true + } + }, + { + "give": { + "ifp": "1", + "id": "msg_p20000", + "ts": "2026-07-21T01:10:05Z", + "type": "event.text.message", + "entity": "ent_abc123", + "session": "ses_p10000", + "device": "dev_d100", + "generation": 2, + "provenance": { + "sensor": "dev_d100:kb", + "captured_at": "2026-07-21T01:10:00Z" + }, + "labels": { + "consent": "product_operation", + "retention": "standard" + }, + "payload": { + "seq": 0, + "dedup": "dev_d100:ses_p10000:0", + "text": "before transfer" + } + }, + "expect": { + "verdict": "acknowledged_duplicate", + "stored": false + } + } + ] + } + ] +} \ No newline at end of file diff --git a/runtime/protocol/ifp1/machine.ts b/runtime/protocol/ifp1/machine.ts new file mode 100644 index 0000000..cb5e87c Binary files /dev/null and b/runtime/protocol/ifp1/machine.ts differ diff --git a/runtime/protocol/ifp1/openphone-tools.ifp1.json b/runtime/protocol/ifp1/openphone-tools.ifp1.json new file mode 100644 index 0000000..7f42ed6 --- /dev/null +++ b/runtime/protocol/ifp1/openphone-tools.ifp1.json @@ -0,0 +1,1685 @@ +{ + "ifp": "1", + "manifest": "openphone-tools", + "version": "1.0.0-draft", + "description": "OpenPhone surface tool capability declarations for Interfaces Protocol v1 (ifp/1). Each tool is declared at hello/pairing as a capability of type 'tool:' (spec section 6.4) and invoked via the action lifecycle (spec section 8): action.propose -> action.approve/action.reject -> action.result. risk maps directly onto action.propose payload.risk. Supersedes runtime/protocol/openphone-commands.json (37 commands).", + "tools": [ + { + "tool": "device.status", + "capability": "tool:device.status", + "risk": "medium", + "description": "Gather a call context card from call log, contacts, messages, and calendar, including missed calls not yet returned.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional person, business, trip, or confirmation query." + }, + "number": { + "type": "string", + "description": "Optional phone number." + }, + "limit": { + "type": "integer", + "description": "Maximum number of rows per source." + }, + "reason": { + "type": "string", + "description": "Why phone context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.device.status", + "superseded_aliases": [ + "device.status", + "device.info" + ], + "android_tool": "phone_context", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "calls.read" + } + }, + { + "tool": "apps.search", + "capability": "tool:apps.search", + "risk": "low", + "description": "Search launchable installed Android apps by label, package, or activity.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional app label, package, or activity query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of apps to return." + }, + "include_system": { + "type": "boolean", + "description": "Include system apps." + }, + "reason": { + "type": "string", + "description": "Why installed app context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.apps.search", + "superseded_aliases": [ + "device.apps" + ], + "android_tool": "apps_search", + "confirmation": "none", + "default_exposure": "safe_read", + "surface_capability": "apps.read" + } + }, + { + "tool": "screen.get", + "capability": "tool:screen.get", + "risk": "low", + "description": "Read the current phone screen and optional screenshot.", + "params_schema": { + "type": "object", + "properties": { + "include_screenshot": { + "type": "boolean", + "description": "Include screenshot image bytes." + }, + "include_activity": { + "type": "boolean", + "description": "Include foreground app/activity." + }, + "include_ui_tree": { + "type": "boolean", + "description": "Include accessibility UI tree." + }, + "reason": { + "type": "string", + "description": "Why this observation is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.screen.get", + "superseded_aliases": [ + "canvas.snapshot" + ], + "android_tool": "get_screen", + "confirmation": "none", + "default_exposure": "safe_read", + "surface_capability": "screen.read.visible" + } + }, + { + "tool": "screen.understand_local", + "capability": "tool:screen.understand_local", + "risk": "low", + "description": "Run bounded local screen understanding on the phone without returning screenshot bytes.", + "params_schema": { + "type": "object", + "properties": { + "max_visible_text": { + "type": "integer", + "description": "Maximum visible text entries to return, capped by the phone." + }, + "max_interactive_elements": { + "type": "integer", + "description": "Maximum interactive elements to return, capped by the phone." + }, + "timeout_ms": { + "type": "integer", + "description": "Requested compute timeout in milliseconds, capped by the phone." + }, + "reason": { + "type": "string", + "description": "Why local screen understanding is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.screen.understand_local", + "superseded_aliases": [ + "openphone.local.screen_understanding" + ], + "android_tool": "local_screen_understanding", + "confirmation": "none", + "default_exposure": "safe_read", + "surface_capability": "screen.read.visible" + } + }, + { + "tool": "notifications.list", + "capability": "tool:notifications.list", + "risk": "medium", + "description": "List recent indexed Android notifications.", + "params_schema": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "description": "Maximum number of notifications to return." + }, + "reason": { + "type": "string", + "description": "Why notification context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.notifications.list", + "superseded_aliases": [ + "notifications.list" + ], + "android_tool": "notifications_list", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "notifications.read" + } + }, + { + "tool": "notifications.search", + "capability": "tool:notifications.search", + "risk": "medium", + "description": "Search recent indexed Android notifications.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of notifications to return." + }, + "reason": { + "type": "string", + "description": "Why notification context is needed." + } + }, + "required": [ + "query", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.notifications.search", + "superseded_aliases": [ + "notifications.search" + ], + "android_tool": "notifications_search", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "notifications.read" + } + }, + { + "tool": "notifications.open", + "capability": "tool:notifications.open", + "risk": "medium", + "description": "Open an active Android notification through its content intent.", + "params_schema": { + "type": "object", + "properties": { + "notification_key": { + "type": "string", + "description": "Optional notification key." + }, + "package": { + "type": "string", + "description": "Optional source package." + }, + "query": { + "type": "string", + "description": "Optional title/text/package search query." + }, + "reason": { + "type": "string", + "description": "Why this notification should open." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.notifications.open", + "superseded_aliases": [ + "notifications.open" + ], + "android_tool": "notifications_open", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "notifications.act" + } + }, + { + "tool": "contacts.search", + "capability": "tool:contacts.search", + "risk": "medium", + "description": "Search Android contacts by name, phone, email, or organization.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional name, phone, or email query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of contacts to return." + }, + "include_details": { + "type": "boolean", + "description": "Include phone and email details." + }, + "reason": { + "type": "string", + "description": "Why contact context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.contacts.search", + "superseded_aliases": [ + "contacts.search" + ], + "android_tool": "contacts_search", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "contacts.read" + } + }, + { + "tool": "calendar.search", + "capability": "tool:calendar.search", + "risk": "medium", + "description": "Search Android calendar events in a bounded time window.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional search query." + }, + "start_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "end_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "limit": { + "type": "integer", + "description": "Maximum number of events to return." + }, + "reason": { + "type": "string", + "description": "Why calendar context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.calendar.search", + "superseded_aliases": [ + "calendar.events" + ], + "android_tool": "calendar_search", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "calendar.read" + } + }, + { + "tool": "calendar.add", + "capability": "tool:calendar.add", + "risk": "medium", + "description": "Create an Android calendar event in the user's writable calendar.", + "params_schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Event title." + }, + "description": { + "type": "string", + "description": "Optional event details." + }, + "location": { + "type": "string", + "description": "Optional location." + }, + "start_at": { + "type": "integer", + "description": "Unix time in milliseconds." + }, + "end_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "duration_minutes": { + "type": "integer", + "description": "Duration if end_at is omitted." + }, + "all_day": { + "type": "boolean", + "description": "Whether this is an all-day event." + }, + "calendar_id": { + "type": "integer", + "description": "Optional target calendar id." + }, + "reason": { + "type": "string", + "description": "Why this event should be created." + } + }, + "required": [ + "title", + "start_at", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.calendar.add", + "superseded_aliases": [ + "calendar.add" + ], + "android_tool": "calendar_create_event", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "calendar.write" + } + }, + { + "tool": "calendar.update", + "capability": "tool:calendar.update", + "risk": "medium", + "description": "Update fields of an existing Android calendar event (title, time, location, description). Recurring events update the whole series.", + "params_schema": { + "type": "object", + "properties": { + "event_id": { + "type": "integer", + "description": "Calendar event id from calendar_search." + }, + "title": { + "type": "string", + "description": "New event title." + }, + "description": { + "type": "string", + "description": "New event details." + }, + "location": { + "type": "string", + "description": "New location." + }, + "start_at": { + "type": "integer", + "description": "New start, Unix time in milliseconds." + }, + "end_at": { + "type": "integer", + "description": "New end, Unix time in milliseconds." + }, + "duration_minutes": { + "type": "integer", + "description": "New duration if end_at is omitted." + }, + "all_day": { + "type": "boolean", + "description": "Whether this is an all-day event." + }, + "reason": { + "type": "string", + "description": "Why this event should be changed." + } + }, + "required": [ + "event_id", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.calendar.update", + "superseded_aliases": [ + "calendar.update" + ], + "android_tool": "calendar_update_event", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "calendar.write" + } + }, + { + "tool": "calendar.delete", + "capability": "tool:calendar.delete", + "risk": "high", + "description": "Delete an Android calendar event permanently. Recurring events delete the whole series.", + "params_schema": { + "type": "object", + "properties": { + "event_id": { + "type": "integer", + "description": "Calendar event id from calendar_search." + }, + "reason": { + "type": "string", + "description": "Why this event should be deleted." + } + }, + "required": [ + "event_id", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.calendar.delete", + "superseded_aliases": [ + "calendar.delete" + ], + "android_tool": "calendar_delete_event", + "confirmation": "always", + "default_exposure": "dangerous", + "surface_capability": "calendar.delete" + } + }, + { + "tool": "messages.search", + "capability": "tool:messages.search", + "risk": "medium", + "description": "Search Android SMS message history by sender, body, or address.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional sender, body, or address query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of messages to return." + }, + "thread_id": { + "type": "integer", + "description": "Optional SMS thread id, or 0." + }, + "reason": { + "type": "string", + "description": "Why message context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.messages.search", + "superseded_aliases": [ + "sms.search" + ], + "android_tool": "messages_search", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "messages.read" + } + }, + { + "tool": "messages.draft", + "capability": "tool:messages.draft", + "risk": "medium", + "description": "Prepare a structured SMS draft for review without sending it.", + "params_schema": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "Phone number if known." + }, + "body": { + "type": "string", + "description": "Draft message body." + }, + "contact_query": { + "type": "string", + "description": "Contact name if recipient should be resolved." + }, + "reason": { + "type": "string", + "description": "Why this draft is needed." + } + }, + "required": [ + "body", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.messages.draft", + "superseded_aliases": [ + "sms.draft" + ], + "android_tool": "messages_draft", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "messages.draft" + } + }, + { + "tool": "messages.send", + "capability": "tool:messages.send", + "risk": "high", + "description": "Send an SMS message to a phone number.", + "params_schema": { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "Phone number." + }, + "body": { + "type": "string", + "description": "Message body." + }, + "reason": { + "type": "string", + "description": "Why this SMS should be sent." + } + }, + "required": [ + "to", + "body", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.messages.send", + "superseded_aliases": [ + "sms.send" + ], + "android_tool": "messages_send", + "confirmation": "always", + "default_exposure": "dangerous", + "surface_capability": "messages.send" + } + }, + { + "tool": "calls.search", + "capability": "tool:calls.search", + "risk": "medium", + "description": "Search Android call log entries by number, contact name (joined from contacts when the log has none), type, or date range.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional number or cached contact name." + }, + "type": { + "type": "string", + "description": "Optional call type filter: missed, incoming, outgoing, voicemail, rejected, or blocked." + }, + "limit": { + "type": "integer", + "description": "Maximum number of calls to return." + }, + "since": { + "type": "integer", + "description": "Only calls at or after this Unix time in milliseconds, or 0." + }, + "until": { + "type": "integer", + "description": "Only calls at or before this Unix time in milliseconds, or 0." + }, + "reason": { + "type": "string", + "description": "Why call history context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.calls.search", + "superseded_aliases": [ + "callLog.search" + ], + "android_tool": "calls_search", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "calls.read" + } + }, + { + "tool": "calls.place", + "capability": "tool:calls.place", + "risk": "high", + "description": "Place a phone call to a number.", + "params_schema": { + "type": "object", + "properties": { + "number": { + "type": "string", + "description": "Phone number if known." + }, + "contact_query": { + "type": "string", + "description": "Contact name if number should be resolved." + }, + "reason": { + "type": "string", + "description": "Why this call should be placed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.calls.place", + "superseded_aliases": [ + "calls.place" + ], + "android_tool": "calls_place", + "confirmation": "always", + "default_exposure": "dangerous", + "surface_capability": "calls.place" + } + }, + { + "tool": "memory.search", + "capability": "tool:memory.search", + "risk": "low", + "description": "Search durable OpenPhone memories: user preferences, facts, standing instructions, and long-lived personal context. Use before answering questions that depend on remembered user preferences or identity.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of memories to return." + }, + "reason": { + "type": "string", + "description": "Why this memory is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.memory.search", + "superseded_aliases": [], + "android_tool": "memory_search", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "memory.read" + } + }, + { + "tool": "memory.save", + "capability": "tool:memory.save", + "risk": "medium", + "description": "Save explicit durable user memory: stable preferences, facts, or standing instructions that should affect future phone behavior. Do not use for one-off reminders or future conditions; use commitments or watchers for those.", + "params_schema": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "fact, preference, or standing_instruction." + }, + "subject": { + "type": "string", + "description": "Memory subject, usually user." + }, + "text": { + "type": "string", + "description": "Durable memory text." + }, + "confidence": { + "type": "number", + "description": "Confidence from 0 to 1." + }, + "reason": { + "type": "string", + "description": "Why this should become memory." + } + }, + "required": [ + "text", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.memory.save", + "superseded_aliases": [], + "android_tool": "memory_save", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "memory.write" + } + }, + { + "tool": "watchers.list", + "capability": "tool:watchers.list", + "risk": "low", + "description": "List active OpenPhone watchers: external-condition monitors that may alert later. Use when the user asks what OpenPhone is watching or what conditions are being monitored. Do not use for queued background agent jobs; use background_job_list for those.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of watchers to return." + }, + "reason": { + "type": "string", + "description": "Why watcher context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.watchers.list", + "superseded_aliases": [], + "android_tool": "watcher_list", + "confirmation": "none", + "default_exposure": "private_read", + "surface_capability": "watchers.read" + } + }, + { + "tool": "watchers.create", + "capability": "tool:watchers.create", + "risk": "medium", + "description": "Create a durable OpenPhone watcher: a future trigger plus an optional reaction. Use for proactive monitoring of message replies/no-reply deadlines, calls/no-call deadlines, notification matches, web/page changes, semantic page conditions, and requests like tell me when, remind me if, watch this, let me know, or make sure I do not forget. If the user asks the phone to do something when the trigger happens, store that reaction in delivery using delivery.tool plus delivery.arguments, or delivery.prompt for a background agent job. Tool arguments may reference trigger data with templates such as {{event.number}}, {{event.address}}, {{event.title}}, {{event.text}}, {{event.url}}, {{watcher.id}}, and {{watcher.title}}. For requests to monitor incoming SMS/messages and check, research, classify, summarize, or otherwise reason about each message, create a message watcher with match_any=true and recurring=true, then use delivery.prompt so the fired watcher starts a background agent job; the prompt should include {{event.text}} and may tell the agent to use browser_search or browser_fetch_page for internet context before notifying the user. Do not use watcher_create for explicit run-as-soon-as-possible queued agent work; use background_job_create for that.", + "params_schema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Generic source such as web, notification, message, call, time, or screen." + }, + "evaluator": { + "type": "string", + "description": "hash_change, text_contains, or semantic_match." + }, + "query": { + "type": "string", + "description": "Condition text for text or semantic evaluators." + }, + "url": { + "type": "string", + "description": "Web URL when source is web." + }, + "type": { + "type": "string", + "description": "Compatibility type: time, notification, web_change, message_reply, or call_back." + }, + "address": { + "type": "string", + "description": "Phone number or sender to watch when source is message." + }, + "number": { + "type": "string", + "description": "Phone number to watch when source is call. Omit and set match_any=true for every caller." + }, + "match_any": { + "type": "boolean", + "description": "For call watchers, true means match any phone number. Use for requests like every incoming call or any caller." + }, + "direction": { + "type": "string", + "description": "For call watchers: any (default), incoming, or outgoing." + }, + "thread_id": { + "type": "integer", + "description": "SMS thread id to watch when source is message." + }, + "deadline_at": { + "type": "integer", + "description": "Unix milliseconds deadline for message or call watchers; with notify_on no_reply or no_call, fires if nothing arrived by then." + }, + "notify_on": { + "type": "string", + "description": "For message watchers: reply (default) or no_reply. For call watchers: call (default; alert when a call with the number appears) or no_call (alert only if the deadline passes without one)." + }, + "title": { + "type": "string", + "description": "Short watcher title." + }, + "condition": { + "type": "object", + "description": "Structured condition." + }, + "schedule": { + "type": "object", + "description": "Schedule, including next_run_at or interval_ms." + }, + "delivery": { + "type": "object", + "description": "Delivery or reaction details. Use {\"mode\":\"notification\"} for alerts, {\"tool\":\"model_tool_name\",\"arguments\":{...}} for a stored tool reaction, or {\"mode\":\"background_job\",\"prompt\":\"...\"} for an observe-first background agent turn. For SMS research/classification/evaluation watchers, use delivery.prompt with {{event.text}} and browser_search/browser_fetch_page instructions rather than hardcoding the check in the watcher. Set notify=false or mode=silent to avoid a notification after the reaction." + }, + "tool": { + "type": "string", + "description": "Shortcut for delivery.tool: the model-facing tool to run when the watcher fires." + }, + "arguments": { + "type": "object", + "description": "Shortcut for delivery.arguments. Values may use event templates such as {{event.number}}." + }, + "prompt": { + "type": "string", + "description": "Shortcut for delivery.prompt when the watcher should enqueue a background agent job after the trigger." + }, + "recurring": { + "type": "boolean", + "description": "True when the watcher should continue after matching, such as every matching call or notification." + }, + "sms_body": { + "type": "string", + "description": "Legacy shortcut only; prefer delivery.tool with delivery.arguments for new watcher reactions." + }, + "next_run_at": { + "type": "integer", + "description": "Unix time in milliseconds, or 0." + }, + "interval_ms": { + "type": "integer", + "description": "Polling interval in milliseconds if relevant." + }, + "reason": { + "type": "string", + "description": "Why this watcher should exist." + } + }, + "required": [ + "title", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.watchers.create", + "superseded_aliases": [], + "android_tool": "watcher_create", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "watchers.write" + } + }, + { + "tool": "watchers.stop", + "capability": "tool:watchers.stop", + "risk": "medium", + "description": "Stop active OpenPhone watchers. Use watcher_id when the user names a specific watcher id. Use scope=all or all=true when the user asks to turn off, disable, clear, remove, delete, cancel, stop, or kill all watchers. Use query when the user describes a watcher by title, source, or condition instead of giving an id.", + "params_schema": { + "type": "object", + "properties": { + "watcher_id": { + "type": "integer", + "description": "Specific watcher id. Omit when stopping all watchers or a query-matched set." + }, + "scope": { + "type": "string", + "description": "Use all to stop every active watcher." + }, + "all": { + "type": "boolean", + "description": "True to stop every active watcher." + }, + "query": { + "type": "string", + "description": "Optional search text to stop matching active watchers when the user described the watcher but did not provide an id." + }, + "reason": { + "type": "string", + "description": "Why this watcher should stop." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.watchers.stop", + "superseded_aliases": [], + "android_tool": "watcher_stop", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "watchers.write" + } + }, + { + "tool": "jobs.list", + "capability": "tool:jobs.list", + "risk": "low", + "description": "List durable OpenPhone background agent jobs, including active, running, completed, failed, and stopped jobs. Use when the user asks what OpenPhone is working on in the background or what background jobs are active.", + "params_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Optional search query." + }, + "limit": { + "type": "integer", + "description": "Maximum number of jobs to return." + }, + "reason": { + "type": "string", + "description": "Why background job context is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.jobs.list", + "superseded_aliases": [], + "android_tool": "background_job_list", + "confirmation": "none", + "default_exposure": "safe_read", + "surface_capability": "tasks.observe" + } + }, + { + "tool": "jobs.create", + "capability": "tool:jobs.create", + "risk": "medium", + "description": "Create a durable OpenPhone background agent job: a queued agent turn that can run after the current conversation. Use for explicit requests like create a background job, do this in the background, run this as soon as possible, keep working and notify me, periodically summarize, check later by running an agent task, or run a non-urgent retrieval task. Do not use watcher_create for these queued agent jobs. Background jobs may observe and summarize; state-changing tools require foreground review.", + "params_schema": { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Short user-visible job title." + }, + "prompt": { + "type": "string", + "description": "The agent turn to run when the job wakes." + }, + "type": { + "type": "string", + "description": "agent_turn, system_event, or heartbeat. Default agent_turn." + }, + "schedule": { + "type": "object", + "description": "Schedule fields such as next_run_at or interval_ms." + }, + "run_at": { + "type": "integer", + "description": "Unix milliseconds when the job should first run. Omit or 0 to run soon." + }, + "next_run_at": { + "type": "integer", + "description": "Unix milliseconds when the job should next run." + }, + "interval_ms": { + "type": "integer", + "description": "Optional recurrence interval in milliseconds." + }, + "session_target": { + "type": "string", + "description": "main, isolated, current, or session:." + }, + "delivery": { + "type": "object", + "description": "Delivery mode. Use {\"mode\":\"notification\"}, {\"mode\":\"silent\"}, or {\"mode\":\"none\"}. If the user gives exact notification wording, include it as notification_text, text, or message." + }, + "notification_text": { + "type": "string", + "description": "Optional exact notification body to show when the background job completes. Use this when the user says to notify them with an exact phrase." + }, + "payload": { + "type": "object", + "description": "Optional structured payload for future runtime integrations." + }, + "reason": { + "type": "string", + "description": "Why this background job should exist." + } + }, + "required": [ + "title", + "prompt", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.jobs.create", + "superseded_aliases": [], + "android_tool": "background_job_create", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "background.run" + } + }, + { + "tool": "jobs.stop", + "capability": "tool:jobs.stop", + "risk": "medium", + "description": "Stop a durable OpenPhone background agent job.", + "params_schema": { + "type": "object", + "properties": { + "job_id": { + "type": "integer", + "description": "Background job id." + }, + "reason": { + "type": "string", + "description": "Why this background job should stop." + } + }, + "required": [ + "job_id", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.jobs.stop", + "superseded_aliases": [], + "android_tool": "background_job_stop", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "background.run" + } + }, + { + "tool": "app.open", + "capability": "tool:app.open", + "risk": "low", + "description": "Open an installed Android app. This tool completes only simple goals whose whole intent is to open, launch, or show the app. If the user wants something done inside the app, such as playing media, searching, selecting content, changing settings, ordering, booking, posting, sending, or any other in-app workflow, use the autonomous multi-step agent: open the app as one step, then keep observing and acting until the requested result is visibly complete or blocked.", + "params_schema": { + "type": "object", + "properties": { + "package_or_label": { + "type": "string", + "description": "Package name or app label." + }, + "label": { + "type": "string", + "description": "Optional explicit app label when package_or_label is a package name." + }, + "reason": { + "type": "string", + "description": "Why this app is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.app.open", + "superseded_aliases": [], + "android_tool": "open_app", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "apps.launch" + } + }, + { + "tool": "url.open", + "capability": "tool:url.open", + "risk": "medium", + "description": "Open a URL directly in the browser.", + "params_schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "URL to open." + }, + "reason": { + "type": "string", + "description": "Why this URL is needed." + } + }, + "required": [ + "url", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.url.open", + "superseded_aliases": [], + "android_tool": "open_url", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "network.use" + } + }, + { + "tool": "ui.tap", + "capability": "tool:ui.tap", + "risk": "medium", + "description": "Tap absolute screen coordinates.", + "params_schema": { + "type": "object", + "properties": { + "x": { + "type": "number", + "description": "X coordinate." + }, + "y": { + "type": "number", + "description": "Y coordinate." + }, + "reason": { + "type": "string", + "description": "Why this coordinate should be tapped." + } + }, + "required": [ + "x", + "y", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.ui.tap", + "superseded_aliases": [], + "android_tool": "tap", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "ui.tap_element", + "capability": "tool:ui.tap_element", + "risk": "medium", + "description": "Tap an enabled interactive element by screen element id.", + "params_schema": { + "type": "object", + "properties": { + "element_id": { + "type": "string", + "description": "Element id from interactive_elements." + }, + "reason": { + "type": "string", + "description": "Why this element should be tapped." + } + }, + "required": [ + "element_id", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.ui.tap_element", + "superseded_aliases": [], + "android_tool": "tap_element", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "ui.long_press", + "capability": "tool:ui.long_press", + "risk": "medium", + "description": "Long press absolute screen coordinates.", + "params_schema": { + "type": "object", + "properties": { + "x": { + "type": "number", + "description": "X coordinate." + }, + "y": { + "type": "number", + "description": "Y coordinate." + }, + "duration_ms": { + "type": "integer", + "description": "Press duration." + }, + "reason": { + "type": "string", + "description": "Why this coordinate should be long-pressed." + } + }, + "required": [ + "x", + "y", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.ui.long_press", + "superseded_aliases": [], + "android_tool": "long_press", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "ui.long_press_element", + "capability": "tool:ui.long_press_element", + "risk": "medium", + "description": "Long press an enabled interactive element by screen element id.", + "params_schema": { + "type": "object", + "properties": { + "element_id": { + "type": "string", + "description": "Element id from interactive_elements." + }, + "duration_ms": { + "type": "integer", + "description": "Press duration." + }, + "reason": { + "type": "string", + "description": "Why this element should be long-pressed." + } + }, + "required": [ + "element_id", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.ui.long_press_element", + "superseded_aliases": [], + "android_tool": "long_press_element", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "ui.swipe", + "capability": "tool:ui.swipe", + "risk": "medium", + "description": "Swipe from one coordinate to another.", + "params_schema": { + "type": "object", + "properties": { + "start_x": { + "type": "number", + "description": "Start X." + }, + "start_y": { + "type": "number", + "description": "Start Y." + }, + "end_x": { + "type": "number", + "description": "End X." + }, + "end_y": { + "type": "number", + "description": "End Y." + }, + "reason": { + "type": "string", + "description": "Why this swipe is needed." + } + }, + "required": [ + "start_x", + "start_y", + "end_x", + "end_y", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.ui.swipe", + "superseded_aliases": [], + "android_tool": "swipe", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "ui.type_text", + "capability": "tool:ui.type_text", + "risk": "medium", + "description": "Type text into the focused field.", + "params_schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to type." + }, + "reason": { + "type": "string", + "description": "Why this text is needed." + } + }, + "required": [ + "text", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.ui.type_text", + "superseded_aliases": [], + "android_tool": "type_text", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "input.press_key", + "capability": "tool:input.press_key", + "risk": "medium", + "description": "Press a supported Android navigation or IME key.", + "params_schema": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "back, home, recents, enter, search, or go." + }, + "reason": { + "type": "string", + "description": "Why this key is needed." + } + }, + "required": [ + "key", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.input.press_key", + "superseded_aliases": [], + "android_tool": "press_key", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "input.perform" + } + }, + { + "tool": "clipboard.set", + "capability": "tool:clipboard.set", + "risk": "low", + "description": "Set clipboard text.", + "params_schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Clipboard text." + }, + "reason": { + "type": "string", + "description": "Why clipboard is needed." + } + }, + "required": [ + "text", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.clipboard.set", + "superseded_aliases": [], + "android_tool": "set_clipboard", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "clipboard.write" + } + }, + { + "tool": "clipboard.paste", + "capability": "tool:clipboard.paste", + "risk": "medium", + "description": "Paste clipboard into the focused field.", + "params_schema": { + "type": "object", + "properties": { + "reason": { + "type": "string", + "description": "Why paste is needed." + } + }, + "required": [ + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.clipboard.paste", + "superseded_aliases": [], + "android_tool": "paste", + "confirmation": "ask_before_action", + "default_exposure": "dangerous", + "surface_capability": "clipboard.read" + } + }, + { + "tool": "share.text", + "capability": "tool:share.text", + "risk": "high", + "description": "Share text through Android's mediated share flow.", + "params_schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to share." + }, + "chooser_title": { + "type": "string", + "description": "Optional chooser title." + }, + "reason": { + "type": "string", + "description": "Why sharing is needed." + } + }, + "required": [ + "text", + "reason" + ] + }, + "result_schema": { + "type": "object", + "additionalProperties": true + }, + "x-openphone": { + "superseded_command": "openphone.share.text", + "superseded_aliases": [], + "android_tool": "share_text", + "confirmation": "always", + "default_exposure": "dangerous", + "surface_capability": "share.content" + } + } + ] +} diff --git a/runtime/protocol/ifp1/validator.ts b/runtime/protocol/ifp1/validator.ts new file mode 100644 index 0000000..4b9d6b4 --- /dev/null +++ b/runtime/protocol/ifp1/validator.ts @@ -0,0 +1,301 @@ +/** + * VENDORED from silentspeech-app packages/interfaces-protocol/ts/src/validator.ts + * (rewrite/interfaces-v1, ifp/1 v1.0.0-draft). Do not edit here; upstream is + * the source of truth until the fixture corpus freezes (ADR 0004). Re-vendor + * by copying the upstream file and re-running scripts/check.sh. + */ +/** + * Structural validation of ifp/1 envelopes. + * + * Dependency-free by design: rules are transcribed from + * schemas/ifp-1.schema.json, and tests assert against the shared fixture + * corpus, so drift between schema and code shows up as a fixture failure. + * Must produce verdicts identical to the Python validator (spec §11). + */ + +export const ERROR_CLASSES = new Set([ + "unsupported_version", + "invalid_envelope", + "unauthorized_device", + "pairing_failed", + "unknown_type", + "stale_generation", + "ordering_violation", + "action_expired", + "binding_mismatch", + "capability_missing", + "policy_denied", + "user_denied", + "rate_limited", + "internal", +]); + +const CONSENT_LABELS = new Set([ + "transient_inference", + "product_operation", + "user_memory", + "research_contribution", + "personal_model_training", + "shared_model_training", + "operational_audit", +]); + +const RETENTION_LABELS = new Set(["transient", "standard", "extended", "permanent"]); + +const ACTION_REJECT_REASONS = new Set([ + "capability_missing", + "policy_denied", + "user_denied", + "expired", + "other", +]); + +const RISK_LEVELS = new Set(["low", "medium", "high"]); + +const ENTITYLESS_TYPES = new Set(["pair.request", "pair.complete", "control.hello", "error"]); + +const ID_RE = /^[a-z]+_[A-Za-z0-9]{6,64}$/; +const TYPE_RE = /^[a-z][a-z0-9_.]*$/; +// Z suffix required (spec section 2): keeps ordering well-defined on the wire. +const TS_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/; +const SHA256_RE = /^sha256:[0-9a-f]{64}$/; +const PUBKEY_RE = /^[A-Za-z0-9_-]{43}$/; +const ENTITY_RE = /^ent_[A-Za-z0-9]{6,64}$/; +const SESSION_RE = /^ses_[A-Za-z0-9]{6,64}$/; +const DEVICE_RE = /^dev_[A-Za-z0-9_]{4,64}$/; + +const ENVELOPE_FIELDS = new Set([ + "ifp", + "id", + "ts", + "type", + "entity", + "session", + "device", + "generation", + "parent", + "correlation", + "provenance", + "labels", + "payload", + "ext", +]); + +export interface ValidationResult { + ok: boolean; + errorClass: string | null; + detail: string | null; +} + +const accept = (): ValidationResult => ({ ok: true, errorClass: null, detail: null }); +const reject = (errorClass: string, detail: string): ValidationResult => ({ + ok: false, + errorClass, + detail, +}); + +type Obj = Record; + +const isObj = (v: unknown): v is Obj => + typeof v === "object" && v !== null && !Array.isArray(v); +const isStr = (v: unknown): v is string => typeof v === "string"; +const isInt = (v: unknown): v is number => Number.isInteger(v); + +function checkProvenance(prov: unknown): string | null { + if (!isObj(prov)) return "provenance must be an object"; + if (!isStr(prov.sensor) || !prov.sensor) return "provenance.sensor required"; + if (!isStr(prov.captured_at) || !TS_RE.test(prov.captured_at)) { + return "provenance.captured_at must be RFC 3339"; + } + if (prov.chain !== undefined && prov.chain !== null) { + if (!Array.isArray(prov.chain)) return "provenance.chain must be a list"; + for (const entry of prov.chain) { + if (!isObj(entry) || !isStr(entry.stage)) return "provenance.chain entries require stage"; + } + } + return null; +} + +function checkLabels(labels: unknown): string | null { + if (!isObj(labels)) return "labels must be an object"; + if (!CONSENT_LABELS.has(labels.consent as string)) return "labels.consent not in registry"; + if (!RETENTION_LABELS.has(labels.retention as string)) return "labels.retention not in registry"; + return null; +} + +function checkCapabilities(caps: unknown): string | null { + if (!Array.isArray(caps)) return "capabilities must be a list"; + for (const cap of caps) { + if (!isObj(cap) || !isStr(cap.type) || !cap.type) return "capability entries require type"; + } + return null; +} + +export function validateMessage(msg: unknown): ValidationResult { + if (!isObj(msg)) return reject("invalid_envelope", "message must be an object"); + + if (msg.ifp !== "1") return reject("unsupported_version", "ifp must be '1'"); + + for (const key of Object.keys(msg)) { + if (!ENVELOPE_FIELDS.has(key)) { + return reject("invalid_envelope", `unknown envelope field: ${key}`); + } + } + + for (const field of ["id", "ts", "type"] as const) { + if (!isStr(msg[field])) return reject("invalid_envelope", `${field} required`); + } + if (!ID_RE.test(msg.id as string)) return reject("invalid_envelope", "id format invalid"); + if (!TS_RE.test(msg.ts as string)) return reject("invalid_envelope", "ts must be RFC 3339"); + if (!TYPE_RE.test(msg.type as string)) return reject("invalid_envelope", "type format invalid"); + + const payload = msg.payload; + if (!isObj(payload)) return reject("invalid_envelope", "payload must be an object"); + + const ext = msg.ext ?? {}; + if (!isObj(ext)) return reject("invalid_envelope", "ext must be an object"); + for (const key of Object.keys(ext)) { + if (!key.startsWith("x-")) return reject("invalid_envelope", `ext key without x- prefix: ${key}`); + } + + const mtype = msg.type as string; + const entity = msg.entity ?? null; + const session = msg.session ?? null; + const device = msg.device ?? null; + const generation = msg.generation ?? null; + + if (entity !== null && (!isStr(entity) || !ENTITY_RE.test(entity))) { + return reject("invalid_envelope", "entity format invalid"); + } + if (session !== null && (!isStr(session) || !SESSION_RE.test(session))) { + return reject("invalid_envelope", "session format invalid"); + } + if (device !== null && (!isStr(device) || !DEVICE_RE.test(device))) { + return reject("invalid_envelope", "device format invalid"); + } + if (generation !== null && (!isInt(generation) || (generation as number) < 1)) { + return reject("invalid_envelope", "generation must be int >= 1"); + } + + if (entity === null && !ENTITYLESS_TYPES.has(mtype)) { + return reject("invalid_envelope", "entity required on this message type"); + } + + if (mtype.startsWith("event.")) { + if (session === null) return reject("invalid_envelope", "event requires session"); + if (generation === null) return reject("invalid_envelope", "event requires generation"); + const provErr = checkProvenance(msg.provenance); + if (provErr) return reject("invalid_envelope", provErr); + const labelErr = checkLabels(msg.labels); + if (labelErr) return reject("invalid_envelope", labelErr); + const seq = payload.seq; + if (!isInt(seq) || (seq as number) < 0) { + return reject("invalid_envelope", "event payload.seq must be int >= 0"); + } + if (!isStr(payload.dedup) || !payload.dedup) { + return reject("invalid_envelope", "event payload.dedup required"); + } + } + + if (mtype.startsWith("action.") && session === null) { + return reject("invalid_envelope", "action requires session"); + } + + if ((mtype === "action.approve" || mtype === "action.result") && generation === null) { + return reject("invalid_envelope", `${mtype} requires generation`); + } + + if (mtype === "action.propose") { + for (const field of ["action_id", "tool", "params", "params_hash", "risk", "idempotency_key", "expires_at"]) { + if (!(field in payload)) return reject("invalid_envelope", `action.propose payload.${field} required`); + } + if (!isObj(payload.params)) return reject("invalid_envelope", "params must be an object"); + if (!isStr(payload.params_hash) || !SHA256_RE.test(payload.params_hash)) { + return reject("invalid_envelope", "params_hash must be sha256:"); + } + if (!RISK_LEVELS.has(payload.risk as string)) return reject("invalid_envelope", "risk not in registry"); + if (!isStr(payload.expires_at) || !TS_RE.test(payload.expires_at)) { + return reject("invalid_envelope", "expires_at must be RFC 3339"); + } + if (!isStr(payload.tool) || !payload.tool) return reject("invalid_envelope", "tool required"); + if (!isStr(payload.idempotency_key) || !payload.idempotency_key) { + return reject("invalid_envelope", "idempotency_key required"); + } + } + + if (mtype === "action.approve") { + for (const field of ["action_id", "params_hash", "confirmation"]) { + if (!(field in payload)) return reject("invalid_envelope", `action.approve payload.${field} required`); + } + if (!isStr(payload.params_hash) || !SHA256_RE.test(payload.params_hash)) { + return reject("invalid_envelope", "params_hash must be sha256:"); + } + const conf = payload.confirmation; + if (!isObj(conf) || (conf.mode !== "user_confirmed" && conf.mode !== "policy_auto")) { + return reject("invalid_envelope", "confirmation.mode invalid"); + } + } + + if (mtype === "action.reject") { + if (!isStr(payload.action_id)) return reject("invalid_envelope", "action.reject payload.action_id required"); + if (!ACTION_REJECT_REASONS.has(payload.reason_class as string)) { + return reject("invalid_envelope", "reason_class not in registry"); + } + } + + if (mtype === "action.result") { + if (!isStr(payload.action_id)) return reject("invalid_envelope", "action.result payload.action_id required"); + if (payload.status !== "executed" && payload.status !== "failed") { + return reject("invalid_envelope", "status invalid"); + } + const errClass = payload.error_class ?? null; + if (errClass !== null && !ERROR_CLASSES.has(errClass as string)) { + return reject("invalid_envelope", "error_class not in registry"); + } + } + + if (mtype === "action.cancel") { + if (!isStr(payload.action_id)) return reject("invalid_envelope", "action.cancel payload.action_id required"); + } + + if (mtype === "pair.request") { + for (const field of ["pairing_code", "public_key", "capabilities"]) { + if (!(field in payload)) return reject("invalid_envelope", `pair.request payload.${field} required`); + } + if (!isStr(payload.pairing_code) || payload.pairing_code.length < 6) { + return reject("invalid_envelope", "pairing_code too short"); + } + if (!isStr(payload.public_key) || !PUBKEY_RE.test(payload.public_key)) { + return reject("invalid_envelope", "public_key must be 32 bytes base64url"); + } + const capErr = checkCapabilities(payload.capabilities); + if (capErr) return reject("invalid_envelope", capErr); + } + + if (mtype === "control.hello") { + for (const field of ["device_id", "proof", "capabilities", "protocol"]) { + if (!(field in payload)) return reject("invalid_envelope", `control.hello payload.${field} required`); + } + if (!isStr(payload.proof) || !payload.proof) return reject("invalid_envelope", "proof required"); + const capErr = checkCapabilities(payload.capabilities); + if (capErr) return reject("invalid_envelope", capErr); + const proto = payload.protocol; + if ( + !isObj(proto) || + !isInt(proto.min) || + !isInt(proto.max) || + (proto.min as number) < 1 || + (proto.max as number) < (proto.min as number) + ) { + return reject("invalid_envelope", "protocol range invalid"); + } + } + + if (mtype === "error") { + if (!ERROR_CLASSES.has(payload.class as string)) { + return reject("invalid_envelope", "error class not in registry"); + } + } + + return accept(); +} diff --git a/runtime/protocol/openphone-commands.json b/runtime/protocol/openphone-commands.json index 84fc903..78ea5b2 100644 --- a/runtime/protocol/openphone-commands.json +++ b/runtime/protocol/openphone-commands.json @@ -1,5 +1,7 @@ { "version": 1, + "deprecated": true, + "superseded_by": "ifp1/", "commands": [ { "name": "openphone.device.status", diff --git a/runtime/protocol/openphone-events.json b/runtime/protocol/openphone-events.json index ad8c7ab..5a02420 100644 --- a/runtime/protocol/openphone-events.json +++ b/runtime/protocol/openphone-events.json @@ -1,5 +1,7 @@ { "version": 1, + "deprecated": true, + "superseded_by": "ifp1/", "events": [ { "name": "runtime.presence.online", "direction": "phone_to_runtime" }, { "name": "runtime.presence.offline", "direction": "phone_to_runtime" }, @@ -12,7 +14,13 @@ { "name": "runtime.tool.result", "direction": "phone_to_runtime" }, { "name": "runtime.confirmation.required", "direction": "phone_to_runtime" }, { "name": "runtime.confirmation.resolved", "direction": "phone_to_runtime" }, - { "name": "runtime.session.updated", "direction": "bidirectional" } + { "name": "runtime.session.updated", "direction": "bidirectional" }, + { "name": "runtime.surface.present", "direction": "runtime_to_phone" }, + { "name": "runtime.surface.replace", "direction": "runtime_to_phone" }, + { "name": "runtime.surface.dismiss", "direction": "runtime_to_phone" }, + { "name": "runtime.surface.action_result", "direction": "phone_to_runtime" }, + { "name": "phone.surface.action_invoked", "direction": "phone_to_runtime" }, + { "name": "phone.surface.dismissed", "direction": "phone_to_runtime" } ], "transport_mappings": [ { @@ -23,6 +31,12 @@ { "protocol": "runtime.presence.offline", "wire": "openphone.presence.offline" }, { "protocol": "runtime.confirmation.required", "wire": "openphone.confirmation.required" }, { "protocol": "runtime.confirmation.resolved", "wire": "openphone.confirmation.resolved" }, + { "protocol": "runtime.surface.present", "wire": "openphone.surface.present" }, + { "protocol": "runtime.surface.replace", "wire": "openphone.surface.replace" }, + { "protocol": "runtime.surface.dismiss", "wire": "openphone.surface.dismiss" }, + { "protocol": "runtime.surface.action_result", "wire": "openphone.surface.action_result" }, + { "protocol": "phone.surface.action_invoked", "wire": "openphone.surface.action_invoked" }, + { "protocol": "phone.surface.dismissed", "wire": "openphone.surface.dismissed" }, { "protocol": "runtime.presence.online", "wire": "node.presence.alive" } ] } diff --git a/runtime/protocol/openphone-runtime.schema.json b/runtime/protocol/openphone-runtime.schema.json index a84c933..51165a7 100644 --- a/runtime/protocol/openphone-runtime.schema.json +++ b/runtime/protocol/openphone-runtime.schema.json @@ -22,6 +22,27 @@ }, "confirmation": { "$ref": "#/$defs/confirmation" + }, + "assistant_output": { + "$ref": "#/$defs/assistantOutput" + }, + "surface_present": { + "$ref": "#/$defs/surfacePresent" + }, + "surface_replace": { + "$ref": "#/$defs/surfaceReplace" + }, + "surface_dismiss": { + "$ref": "#/$defs/surfaceDismiss" + }, + "surface_action_result": { + "$ref": "#/$defs/surfaceActionResult" + }, + "surface_action_invocation": { + "$ref": "#/$defs/surfaceActionInvocation" + }, + "phone_surface_dismissed": { + "$ref": "#/$defs/phoneSurfaceDismissed" } }, "required": [ @@ -190,6 +211,143 @@ "minimum": 1 } } + }, + "assistantOutput": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "session_id", "speech", "text"], + "properties": { + "schema": { + "const": "openphone.assistant_output.v1" + }, + "session_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "speech": { + "type": "string", + "maxLength": 4000 + }, + "text": { + "type": "string", + "maxLength": 12000 + }, + "surface": { + "type": ["object", "null"] + }, + "background_run": { + "type": ["object", "null"] + } + } + }, + "surfacePresent": { + "type": "object", + "additionalProperties": true, + "required": ["output"], + "properties": { + "output": { + "$ref": "#/$defs/assistantOutput" + }, + "sessionKey": { + "type": "string" + } + } + }, + "surfaceReplace": { + "type": "object", + "additionalProperties": true, + "required": ["output", "expected_revision"], + "properties": { + "output": { + "$ref": "#/$defs/assistantOutput" + }, + "expected_revision": { + "type": "integer", + "minimum": 1 + }, + "sessionKey": { + "type": "string" + } + } + }, + "surfaceDismiss": { + "type": "object", + "additionalProperties": true, + "required": ["surface_id", "revision", "reason"], + "properties": { + "surface_id": { + "type": "string" + }, + "revision": { + "type": "integer", + "minimum": 1 + }, + "reason": { + "type": "string" + }, + "sessionKey": { + "type": "string" + } + } + }, + "surfaceActionResult": { + "type": "object", + "additionalProperties": true, + "required": ["surface_id", "revision", "status"], + "properties": { + "surface_id": { + "type": "string" + }, + "revision": { + "type": "integer", + "minimum": 0 + }, + "action_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "result": { + "type": "object" + }, + "message": { + "type": "string" + } + } + }, + "surfaceActionInvocation": { + "type": "object", + "additionalProperties": true, + "required": [ + "surface_id", + "revision", + "session_id", + "runtime", + "action_id", + "idempotency_key" + ], + "properties": { + "surface_id": { "type": "string" }, + "revision": { "type": "integer", "minimum": 1 }, + "session_id": { "type": "string" }, + "runtime": { "type": "string" }, + "action_id": { "type": "string" }, + "idempotency_key": { "type": "string" } + } + }, + "phoneSurfaceDismissed": { + "type": "object", + "additionalProperties": true, + "required": ["surface_id", "revision", "session_id", "runtime", "reason"], + "properties": { + "surface_id": { "type": "string" }, + "revision": { "type": "integer", "minimum": 1 }, + "session_id": { "type": "string" }, + "runtime": { "type": "string" }, + "reason": { "type": "string" } + } } } } diff --git a/runtime/protocol/validate-runtime-protocol.mjs b/runtime/protocol/validate-runtime-protocol.mjs index 6798081..8ffa9bd 100755 --- a/runtime/protocol/validate-runtime-protocol.mjs +++ b/runtime/protocol/validate-runtime-protocol.mjs @@ -24,6 +24,15 @@ const androidCommandRegistryPath = path.join( root, "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawCommandRegistry.java", ); +const openClawAdapterPath = path.join( + root, + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/adapters/openclaw/OpenClawRuntimeAdapter.java", +); +const surfaceFixturePaths = [ + "surface-present-event.json", + "surface-replace-event.json", + "surface-dismiss-event.json", +].map((name) => path.join(root, "tests/fixtures/runtime", name)); function fail(message) { console.error(`check-runtime-protocol: ${message}`); @@ -95,6 +104,62 @@ if (eventsManifest.version !== 1 || !Array.isArray(eventsManifest.events)) { fail("openphone-events.json must contain version=1 and events[]"); } unique(eventsManifest.events.map((entry) => entry.name), "event"); +const eventDirections = new Map( + eventsManifest.events.map((entry) => [entry.name, entry.direction]), +); +const expectedSurfaceEvents = new Map([ + ["runtime.surface.present", "runtime_to_phone"], + ["runtime.surface.replace", "runtime_to_phone"], + ["runtime.surface.dismiss", "runtime_to_phone"], + ["runtime.surface.action_result", "phone_to_runtime"], + ["phone.surface.action_invoked", "phone_to_runtime"], + ["phone.surface.dismissed", "phone_to_runtime"], +]); +for (const [event, direction] of expectedSurfaceEvents) { + if (eventDirections.get(event) !== direction) { + fail(`missing or misdirected surface event: ${event} -> ${direction}`); + } +} +const openClawMappings = new Map( + (eventsManifest.transport_mappings ?? []) + .filter((mapping) => mapping.runtime === "openclaw") + .flatMap((mapping) => mapping.events ?? []) + .map((mapping) => [mapping.protocol, mapping.wire]), +); +for (const event of expectedSurfaceEvents.keys()) { + if (!openClawMappings.has(event)) { + fail(`OpenClaw transport mapping missing surface event: ${event}`); + } +} + +const surfaceFixtures = surfaceFixturePaths.map(readJson); +const [presentFixture, replaceFixture, dismissFixture] = surfaceFixtures; +if (presentFixture.event !== "runtime.surface.present" + || presentFixture.payload?.output?.schema !== "openphone.assistant_output.v1" + || presentFixture.payload?.output?.surface?.schema !== "openphone.surface.v1" + || presentFixture.payload?.output?.session_id + !== presentFixture.payload?.output?.surface?.session_id) { + fail("runtime surface present fixture is malformed"); +} +if (replaceFixture.event !== "runtime.surface.replace" + || replaceFixture.payload?.expected_revision !== 1 + || replaceFixture.payload?.output?.surface?.revision !== 2 + || replaceFixture.payload?.output?.surface?.surface_id + !== presentFixture.payload?.output?.surface?.surface_id) { + fail("runtime surface replace fixture must increment the same surface revision"); +} +if (dismissFixture.event !== "runtime.surface.dismiss" + || dismissFixture.payload?.revision !== replaceFixture.payload?.output?.surface?.revision + || dismissFixture.payload?.surface_id + !== replaceFixture.payload?.output?.surface?.surface_id) { + fail("runtime surface dismiss fixture must target the replacement revision"); +} +const openClawAdapterSource = fs.readFileSync(openClawAdapterPath, "utf8"); +for (const wire of openClawMappings.values()) { + if (wire.startsWith("openphone.surface.") && !openClawAdapterSource.includes(wire)) { + fail(`Android OpenClaw adapter does not map surface wire event: ${wire}`); + } +} const capabilitiesManifest = readJson(capabilitiesPath); if (capabilitiesManifest.version !== 1 || !Array.isArray(capabilitiesManifest.capabilities)) { diff --git a/schemas/README.md b/schemas/README.md index 767b974..5a9b04a 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -14,6 +14,9 @@ release tooling, and eval tooling must agree on: - audit events and audit evidence exports; - trajectory events; - background agent jobs and task reports; +- exact, expiring background-action confirmations; +- the privacy-minimized SystemUI compact-island state projection; +- phone-owned adaptive surfaces and runtime-neutral assistant outputs; - OTA feed metadata. `scripts/check.sh` loads `action-registry`, `action-request`, `audit-event`, @@ -25,10 +28,22 @@ schemas they validate against: - `scripts/validate-ota-feed.sh` — `ota-feed.schema.json`; - `scripts/validate-agent-eval-report.sh` — `agent-eval-report.schema.json`; - `scripts/validate-audit-evidence-export.sh` — `audit-evidence.schema.json` - and `audit-event.schema.json`. + and `audit-event.schema.json`; +- `scripts/validate-surface-contract.mjs` — + `openphone-surface.schema.json` and + `openphone-assistant-output.schema.json`, including known-valid and + known-invalid conformance fixtures. +- `scripts/validate-background-review-contract.mjs` — + `agent-job.schema.json` and `background-confirmation.schema.json`, including + exact parameter and approval-binding digests, checkpoint linkage, expiry, + size, and blocked-secret checks. +- `scripts/validate-island-contract.mjs` — + `openphone-island-state.schema.json`, bounded/privacy-safe state fixtures, + Binder listener/publisher markers, SystemUI window/touch/lock policy, and + assistant overlay ownership reduction. Editing enums, required keys, or const markers in those schemas changes validator behavior directly. The remaining schemas (`action-result`, -`agent-job`, `agent-task`, `app-policy`, `audit-log`, `model-tool`) document +`agent-task`, `app-policy`, `audit-log`, `model-tool`) document contracts but are not yet wired into any validator; `scripts/check.sh` only verifies they exist. diff --git a/schemas/agent-job.schema.json b/schemas/agent-job.schema.json index 9a7ffc5..17a02d1 100644 --- a/schemas/agent-job.schema.json +++ b/schemas/agent-job.schema.json @@ -20,7 +20,19 @@ "last_run_at", "last_result", "failure_count", - "failure_alert_at" + "failure_alert_at", + "phase", + "progress_text", + "progress_current", + "progress_total", + "checkpoint_json", + "pending_confirmation_id", + "pending_tool_request_json", + "last_surface_id", + "resume_token", + "last_event_at", + "unread_result", + "paused_at" ], "properties": { "id": { "type": "integer", "minimum": 1 }, @@ -45,7 +57,18 @@ "additionalProperties": true }, "status": { - "enum": ["active", "running", "completed", "failed", "stopped"] + "enum": [ + "active", + "queued", + "running", + "waiting", + "awaiting_review", + "paused", + "completed", + "failed", + "stopped", + "dispatched" + ] }, "created_at": { "type": "integer", "minimum": 0 }, "updated_at": { "type": "integer", "minimum": 0 }, @@ -54,7 +77,19 @@ "last_run_at": { "type": "integer", "minimum": 0 }, "last_result": { "type": "string" }, "failure_count": { "type": "integer", "minimum": 0 }, - "failure_alert_at": { "type": "integer", "minimum": 0 } + "failure_alert_at": { "type": "integer", "minimum": 0 }, + "phase": { "type": "string", "maxLength": 120 }, + "progress_text": { "type": "string", "maxLength": 4000 }, + "progress_current": { "type": "integer", "minimum": 0 }, + "progress_total": { "type": "integer", "minimum": 0 }, + "checkpoint_json": { "type": "object" }, + "pending_confirmation_id": { "type": "string", "maxLength": 200 }, + "pending_tool_request_json": { "type": "object" }, + "last_surface_id": { "type": "string", "maxLength": 160 }, + "resume_token": { "type": "string", "maxLength": 200 }, + "last_event_at": { "type": "integer", "minimum": 0 }, + "unread_result": { "type": "boolean" }, + "paused_at": { "type": "integer", "minimum": 0 } }, "additionalProperties": false } diff --git a/schemas/background-confirmation.schema.json b/schemas/background-confirmation.schema.json new file mode 100644 index 0000000..4c9caa9 --- /dev/null +++ b/schemas/background-confirmation.schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/background-confirmation.schema.json", + "title": "OpenPhone Background Confirmation V1", + "type": "object", + "required": [ + "schema", + "confirmation_id", + "job_id", + "runtime", + "phone_session_id", + "tool", + "params", + "params_digest", + "binding_digest", + "idempotency_key", + "created_at", + "expires_at", + "review_state", + "summary" + ], + "properties": { + "schema": { "const": "openphone.background_confirmation.v1" }, + "confirmation_id": { + "type": "string", + "pattern": "^job-confirm-[A-Za-z0-9._-]+$", + "maxLength": 200 + }, + "job_id": { "type": "integer", "minimum": 1 }, + "runtime": { "const": "builtin" }, + "phone_session_id": { + "type": "string", + "pattern": "^background-job:[1-9][0-9]*$", + "maxLength": 160 + }, + "tool": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,79}$" + }, + "params": { + "type": "object", + "maxProperties": 64 + }, + "params_digest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "binding_digest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "idempotency_key": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "created_at": { "type": "integer", "minimum": 0 }, + "expires_at": { "type": "integer", "minimum": 0 }, + "review_state": { + "enum": ["pending", "resolving", "resolved"] + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "decision": { + "enum": ["approved", "denied"] + }, + "reviewed_at": { "type": "integer", "minimum": 0 }, + "resolution": { "type": "string", "maxLength": 120 }, + "resolved_at": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": false +} diff --git a/schemas/openphone-assistant-output.schema.json b/schemas/openphone-assistant-output.schema.json new file mode 100644 index 0000000..9acbbcb --- /dev/null +++ b/schemas/openphone-assistant-output.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/openphone-assistant-output.schema.json", + "title": "OpenPhone Assistant Output V1", + "type": "object", + "required": ["schema", "session_id", "speech", "text"], + "properties": { + "schema": { + "const": "openphone.assistant_output.v1" + }, + "session_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "speech": { + "type": "string", + "maxLength": 4000 + }, + "text": { + "type": "string", + "maxLength": 12000 + }, + "surface": { + "oneOf": [ + { + "$ref": "openphone-surface.schema.json" + }, + { + "type": "null" + } + ] + }, + "background_run": { + "type": ["object", "null"] + } + }, + "additionalProperties": false +} diff --git a/schemas/openphone-island-state.schema.json b/schemas/openphone-island-state.schema.json new file mode 100644 index 0000000..2129a8b --- /dev/null +++ b/schemas/openphone-island-state.schema.json @@ -0,0 +1,99 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/openphone-island-state.schema.json", + "title": "OpenPhone SystemUI Island State V1", + "type": "object", + "required": [ + "schema", + "revision", + "mode", + "label", + "title", + "detail", + "visible", + "needs_attention", + "sensitivity", + "active_task_id", + "pending_confirmation_id", + "live_runs", + "updated_at", + "published_at", + "published_elapsed_ms", + "source" + ], + "properties": { + "schema": { + "const": "openphone.island_state.v1" + }, + "revision": { + "type": "integer", + "minimum": 0 + }, + "mode": { + "enum": [ + "idle", + "listening", + "transcript", + "thinking", + "realtime", + "action_running", + "answer_ready", + "reply", + "needs_review", + "error", + "watching", + "unavailable" + ] + }, + "label": { + "type": "string", + "maxLength": 48 + }, + "title": { + "type": "string", + "maxLength": 80 + }, + "detail": { + "type": "string", + "maxLength": 160 + }, + "visible": { + "type": "boolean" + }, + "needs_attention": { + "type": "boolean" + }, + "sensitivity": { + "enum": ["public", "personal", "sensitive", "restricted"] + }, + "active_task_id": { + "type": "string", + "maxLength": 96 + }, + "pending_confirmation_id": { + "type": "string", + "maxLength": 128 + }, + "live_runs": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "updated_at": { + "type": "integer", + "minimum": 0 + }, + "published_at": { + "type": "integer", + "minimum": 0 + }, + "published_elapsed_ms": { + "type": "integer", + "minimum": 0 + }, + "source": { + "enum": ["assistant", "system_server"] + } + }, + "additionalProperties": false +} diff --git a/schemas/openphone-surface.schema.json b/schemas/openphone-surface.schema.json new file mode 100644 index 0000000..994d3e0 --- /dev/null +++ b/schemas/openphone-surface.schema.json @@ -0,0 +1,284 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openphone.dev/schemas/openphone-surface.schema.json", + "title": "OpenPhone Adaptive Surface V1", + "type": "object", + "required": [ + "schema", + "surface_id", + "revision", + "session_id", + "runtime", + "title", + "presentation", + "expires_at", + "sensitivity", + "body", + "actions" + ], + "properties": { + "schema": { + "const": "openphone.surface.v1" + }, + "surface_id": { + "type": "string", + "pattern": "^surface-[A-Za-z0-9._-]{1,120}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "session_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "runtime": { + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "title": { + "type": "string", + "maxLength": 160 + }, + "presentation": { + "enum": ["full", "sheet", "inline"] + }, + "expires_at": { + "type": "integer", + "minimum": 0 + }, + "sensitivity": { + "enum": ["public", "personal", "sensitive", "restricted"] + }, + "body": { + "$ref": "#/$defs/component" + }, + "actions": { + "type": "object", + "maxProperties": 24, + "additionalProperties": { + "$ref": "#/$defs/action" + } + }, + "artifacts": { + "type": "array", + "maxItems": 16, + "items": { + "$ref": "#/$defs/artifactReference" + } + }, + "created_at": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "$defs": { + "artifactReference": { + "type": "object", + "required": ["artifact_id", "path"], + "properties": { + "artifact_id": { + "type": "string", + "pattern": "^artifact-[A-Za-z0-9._-]{1,120}$" + }, + "path": { + "type": "string", + "pattern": "^\\$([.][A-Za-z0-9_-]+|\\[[0-9]+\\])*$", + "maxLength": 240 + } + }, + "additionalProperties": false + }, + "action": { + "type": "object", + "required": ["label", "tool", "params", "reason"], + "properties": { + "label": { + "type": "string", + "minLength": 1, + "maxLength": 80 + }, + "tool": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,79}$" + }, + "params": { + "type": "object", + "maxProperties": 32 + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "requires_confirmation": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + "component": { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "enum": [ + "column", + "row", + "spacer", + "divider", + "scroll", + "text", + "icon", + "image", + "badge", + "progress", + "card", + "list", + "list_item", + "timeline", + "button", + "chip", + "choice_group", + "toggle", + "text_input", + "native_handoff", + "confirmation", + "capability_disclosure", + "error" + ] + }, + "children": { + "type": "array", + "maxItems": 32, + "items": { + "$ref": "#/$defs/component" + } + }, + "items": { + "type": "array", + "maxItems": 32, + "items": { + "$ref": "#/$defs/component" + } + }, + "text": { + "type": "string", + "maxLength": 1000 + }, + "title": { + "type": "string", + "maxLength": 240 + }, + "subtitle": { + "type": "string", + "maxLength": 500 + }, + "label": { + "type": "string", + "maxLength": 120 + }, + "message": { + "type": "string", + "maxLength": 1000 + }, + "content_description": { + "type": "string", + "maxLength": 240 + }, + "style": { + "enum": ["title", "headline", "body", "caption"] + }, + "tone": { + "enum": ["primary", "secondary", "warning", "danger"] + }, + "density": { + "enum": ["compact", "comfortable"] + }, + "spacing": { + "enum": ["none", "xs", "sm", "md", "lg", "xl"] + }, + "action_id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,119}$" + }, + "input_key": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,79}$" + }, + "placeholder": { + "type": "string", + "maxLength": 160 + }, + "value": { + "type": ["string", "number", "boolean"] + }, + "checked": { + "type": "boolean" + }, + "progress": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "artifact": { + "$ref": "#/$defs/artifactReference" + }, + "icon": { + "enum": [ + "calendar", + "message", + "notification", + "person", + "phone", + "location", + "clock", + "check", + "warning", + "error", + "open", + "agent" + ] + }, + "options": { + "type": "array", + "maxItems": 12, + "items": { + "type": "object", + "required": ["id", "label"], + "properties": { + "id": { + "type": "string", + "maxLength": 80 + }, + "label": { + "type": "string", + "maxLength": 120 + } + }, + "additionalProperties": false + } + }, + "native_target": { + "enum": [ + "app_space", + "calendar", + "messages", + "notifications", + "dialer", + "settings" + ] + }, + "capability": { + "type": "string", + "maxLength": 120 + } + }, + "additionalProperties": false + } + } +} diff --git a/schemas/trajectory-event.schema.json b/schemas/trajectory-event.schema.json index eaba44b..61dee11 100644 --- a/schemas/trajectory-event.schema.json +++ b/schemas/trajectory-event.schema.json @@ -21,7 +21,11 @@ "task_started", "tool_call", "tool_result", - "agent_result" + "agent_result", + "surface_presented", + "surface_action_invoked", + "surface_action_result", + "surface_dismissed" ] }, "payload": { diff --git a/scripts/check-runtime-protocol.sh b/scripts/check-runtime-protocol.sh index f9b123b..3e71122 100755 --- a/scripts/check-runtime-protocol.sh +++ b/scripts/check-runtime-protocol.sh @@ -9,6 +9,8 @@ node --check "$root/runtime/protocol/openphone-runtime-tools.mjs" node --check "$root/integrations/adb/openphone-adb-transport.mjs" node --check "$root/integrations/cli/src/index.mjs" node --check "$root/integrations/mcp-server/src/index.mjs" +node --no-warnings=ExperimentalWarning "$root/tests/integrations/ifp1-conformance-contract.mjs" +node --no-warnings=ExperimentalWarning "$root/tests/integrations/ifp1-adapter-contract.mjs" node "$root/tests/integrations/adb-stateful-gating-contract.mjs" node "$root/tests/integrations/openclaw-plugin-policy-contract.mjs" node "$root/tests/integrations/runtime-cli-contract.mjs" diff --git a/scripts/check.sh b/scripts/check.sh index f824b1e..64d4213 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -19,6 +19,7 @@ required=( docs/README.md docs/AI_FIRST_ENGINEERING.md docs/AGENT_RUNTIME_V1.md + docs/APP_OS_BOUNDARY.md docs/ARCHITECTURE.md docs/BUILD.md docs/CAPABILITIES.md @@ -49,12 +50,16 @@ required=( schemas/action-result.schema.json schemas/agent-eval-report.schema.json schemas/agent-job.schema.json + schemas/background-confirmation.schema.json schemas/agent-task.schema.json schemas/app-policy.schema.json schemas/audit-evidence.schema.json schemas/audit-event.schema.json schemas/audit-log.schema.json schemas/model-tool.schema.json + schemas/openphone-assistant-output.schema.json + schemas/openphone-island-state.schema.json + schemas/openphone-surface.schema.json schemas/ota-feed.schema.json schemas/screen-context.schema.json schemas/trajectory-event.schema.json @@ -91,6 +96,11 @@ required=( scripts/run-assistant-task.sh scripts/pull-latest-trajectory.sh scripts/check-runtime-protocol.sh + scripts/validate-background-review-contract.mjs + scripts/validate-app-os-boundary.mjs + scripts/validate-home-shell-contract.mjs + scripts/validate-island-contract.mjs + scripts/validate-surface-contract.mjs scripts/smoke-test-openclaw-device-failures.sh scripts/smoke-test-openclaw-runtime.sh scripts/setup-model-broker-tls.sh @@ -118,17 +128,43 @@ required=( runtime/protocol/openphone-runtime-tools.mjs runtime/protocol/openphone-runtime.schema.json runtime/protocol/validate-runtime-protocol.mjs + runtime/protocol/ifp1/openphone-tools.ifp1.json + runtime/protocol/ifp1/adapter.mjs + runtime/protocol/ifp1/validator.ts + runtime/protocol/ifp1/machine.ts + runtime/protocol/ifp1/MIGRATION.md + runtime/protocol/ifp1/fixtures/messages.json + runtime/protocol/ifp1/fixtures/sequences.json + runtime/protocol/ifp1/fixtures/params-hash-vectors.json integrations/adb/openphone-adb-transport.mjs integrations/cli/README.md integrations/cli/package.json integrations/cli/src/index.mjs tests/README.md tests/integrations/adb-stateful-gating-contract.mjs + tests/integrations/ifp1-conformance-contract.mjs + tests/integrations/ifp1-adapter-contract.mjs tests/integrations/runtime-cli-contract.mjs tests/integrations/runtime-mcp-contract.mjs tests/integrations/runtime-protocol-versioning-contract.mjs tests/integrations/openclaw-plugin-policy-contract.mjs tests/integrations/runtime-package-contract.mjs + tests/fixtures/surfaces/calendar-agenda.json + tests/fixtures/surfaces/message-summary.json + tests/fixtures/runtime/surface-present-event.json + tests/fixtures/runtime/surface-replace-event.json + tests/fixtures/runtime/surface-dismiss-event.json + tests/fixtures/surfaces/invalid-external-image.json + tests/fixtures/surfaces/invalid-unknown-action.json + tests/fixtures/surfaces/invalid-unknown-component.json + tests/fixtures/jobs/background-awaiting-review.json + tests/fixtures/jobs/invalid-background-review-tampered.json + tests/fixtures/jobs/invalid-background-review-secret.json + tests/fixtures/island/idle.json + tests/fixtures/island/needs-review.json + tests/fixtures/island/invalid-mode.json + tests/fixtures/island/invalid-secret.json + tests/fixtures/island/invalid-oversized.json integrations/mcp-server/README.md integrations/mcp-server/package.json integrations/mcp-server/src/index.mjs @@ -183,6 +219,8 @@ required=( overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneQuickSettingsTileService.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneTriggerReceiver.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/PointerOverlayController.java + overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandState.java + overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandStateRepository.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/IOpenPhoneAssistant.aidl overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/agent/FrameworkToolExecutor.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/agent/TrajectoryRecorder.java @@ -191,6 +229,8 @@ required=( overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/model/LocalHeuristicModelAdapter.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/model/OpenAiRealtimeAdapter.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/ota/OtaUpdateClient.java + overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/PhoneToolGateway.java + overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/OpenPhoneOsToolGateway.java overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/policy/AppCapabilityPolicy.java patches/frameworks_base/0001-OpenPhone-add-agent-manager-framework-service.patch patches/frameworks_base/0002-OpenPhone-add-foreground-context-and-audit-mediation.patch @@ -204,6 +244,8 @@ required=( patches/frameworks_base/0010-OpenPhone-capture-screenshots-as-system-server.patch patches/frameworks_base/0011-OpenPhone-add-SystemUI-agent-QS-tile.patch patches/frameworks_base/0012-OpenPhone-add-mediated-open-url-action.patch + patches/frameworks_base/0020-OpenPhone-add-durable-island-state-contract.patch + patches/frameworks_base/0021-OpenPhone-render-compact-island-in-SystemUI.patch patches/packages_apps_Settings/0001-OpenPhone-add-About-phone-version-surface.patch patches/packages_apps_Settings/0002-OpenPhone-add-settings-dashboard.patch patches/packages_apps_Settings/0003-OpenPhone-add-Settings-hosted-audit-and-grant-pages.patch @@ -1113,6 +1155,11 @@ if grep -R "SPDX-license-identifier-Apache-2.0" \ fi "$root/scripts/check-runtime-protocol.sh" +node "$root/scripts/validate-surface-contract.mjs" +node "$root/scripts/validate-background-review-contract.mjs" +node "$root/scripts/validate-app-os-boundary.mjs" +node "$root/scripts/validate-home-shell-contract.mjs" +node "$root/scripts/validate-island-contract.mjs" "$root/scripts/check-assistant-java.sh" printf 'OpenPhone repo checks passed.\n' diff --git a/scripts/download-mindthegapps.sh b/scripts/download-mindthegapps.sh index 5a4291b..51563f4 100755 --- a/scripts/download-mindthegapps.sh +++ b/scripts/download-mindthegapps.sh @@ -63,7 +63,10 @@ metadata="$output_dir/mindthegapps-${android_version}-${arch}-latest.json" info "Fetching latest release metadata: $repo" curl -fsSL "$api_url" -o "$metadata" -readarray -t assets < <(python3 - <<'PY' "$metadata" +assets=() +while IFS= read -r asset; do + assets+=("$asset") +done < <(python3 - <<'PY' "$metadata" import json import sys diff --git a/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java b/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java index b16b498..02d5f58 100644 --- a/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java +++ b/scripts/javacheck-stubs/android/openphone/OpenPhoneAgentManager.java @@ -60,4 +60,8 @@ public String submitUiTreeSnapshot(String snapshotJson) { public String getUiTreeSnapshot() { throw new UnsupportedOperationException("compile-check stub"); } + + public String publishIslandState(String stateJson) { + throw new UnsupportedOperationException("compile-check stub"); + } } diff --git a/scripts/javacheck-stubs/android/os/SystemProperties.java b/scripts/javacheck-stubs/android/os/SystemProperties.java new file mode 100644 index 0000000..3f4fe02 --- /dev/null +++ b/scripts/javacheck-stubs/android/os/SystemProperties.java @@ -0,0 +1,12 @@ +// Compile-check stub for the hidden platform API used by the privileged +// assistant. The real class is provided by frameworks/base. +package android.os; + +public final class SystemProperties { + private SystemProperties() { + } + + public static boolean getBoolean(String key, boolean defaultValue) { + throw new UnsupportedOperationException("compile-check stub"); + } +} diff --git a/scripts/javacheck-stubs/org/openphone/assistant/OpenPhoneHomeActivity.java b/scripts/javacheck-stubs/org/openphone/assistant/OpenPhoneHomeActivity.java new file mode 100644 index 0000000..22535e7 --- /dev/null +++ b/scripts/javacheck-stubs/org/openphone/assistant/OpenPhoneHomeActivity.java @@ -0,0 +1,5 @@ +// Compile-check stub for the Kotlin OpenPhoneHomeActivity class. +package org.openphone.assistant; + +public class OpenPhoneHomeActivity extends AssistantActivityBackend { +} diff --git a/scripts/javacheck-stubs/org/openphone/assistant/R.java b/scripts/javacheck-stubs/org/openphone/assistant/R.java index ecab4df..35e8209 100644 --- a/scripts/javacheck-stubs/org/openphone/assistant/R.java +++ b/scripts/javacheck-stubs/org/openphone/assistant/R.java @@ -25,4 +25,11 @@ public static final class drawable { private drawable() { } } + + public static final class array { + public static final int openphone_app_space_components = 0; + + private array() { + } + } } diff --git a/scripts/validate-app-os-boundary.mjs b/scripts/validate-app-os-boundary.mjs new file mode 100644 index 0000000..487de58 --- /dev/null +++ b/scripts/validate-app-os-boundary.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const assistant = path.join( + root, + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant", +); + +function read(relative) { + return fs.readFileSync(path.join(root, relative), "utf8"); +} + +function fail(message) { + throw new Error(message); +} + +function sourceFiles(relativeDirectory) { + const files = []; + function visit(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(entryPath); + } else if (entry.isFile() + && (entry.name.endsWith(".java") || entry.name.endsWith(".kt"))) { + files.push(entryPath); + } + } + } + visit(path.join(assistant, relativeDirectory)); + return files; +} + +const portableDirectories = ["actions", "model", "orchestrator", "runtime", "surface"]; +const forbiddenPortableMarkers = [ + "import android.openphone.", + "FrameworkToolExecutor", + "OpenPhoneOsToolGateway", +]; + +for (const directory of portableDirectories) { + for (const file of sourceFiles(directory)) { + const source = fs.readFileSync(file, "utf8"); + for (const marker of forbiddenPortableMarkers) { + if (source.includes(marker)) { + fail(`${path.relative(root, file)} crosses the app/OS boundary via ${marker}`); + } + } + } +} + +const contract = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/PhoneToolGateway.java", +); +for (const marker of [ + "String profile()", + "boolean isAvailable()", + "boolean supportsTool(String toolName)", + "String startTask(String taskJson)", + "String executeTool(String taskId, String toolName, JSONObject arguments)", + "String confirmAction(String pendingActionId, boolean approved)", +]) { + if (!contract.includes(marker)) { + fail(`PhoneToolGateway is missing ${marker}`); + } +} +if (contract.includes("android.openphone")) { + fail("PhoneToolGateway must remain independent of hidden OpenPhone APIs"); +} + +const osGateway = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/platform/OpenPhoneOsToolGateway.java", +); +for (const marker of [ + "implements PhoneToolGateway", + "import android.openphone.OpenPhoneAgentManager", + "new FrameworkToolExecutor", + 'PROFILE = "openphone_os"', +]) { + if (!osGateway.includes(marker)) { + fail(`OpenPhone OS gateway is missing ${marker}`); + } +} + +const runtimeBridge = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/runtime/RuntimeToolBridge.java", +); +for (const marker of [ + "mPhoneGateway.startTask", + "mPhoneGateway.executeTool", + "mPhoneGateway.confirmAction", + "mPhoneGateway.supportsTool", +]) { + if (!runtimeBridge.includes(marker)) { + fail(`RuntimeToolBridge does not route through ${marker}`); + } +} + +console.log("App/OS boundary contract checks passed."); diff --git a/scripts/validate-background-review-contract.mjs b/scripts/validate-background-review-contract.mjs new file mode 100644 index 0000000..25cf0bd --- /dev/null +++ b/scripts/validate-background-review-contract.mjs @@ -0,0 +1,245 @@ +#!/usr/bin/env node + +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const confirmationSchema = readJson("schemas/background-confirmation.schema.json"); +const jobSchema = readJson("schemas/agent-job.schema.json"); + +function readJson(relative) { + return JSON.parse(fs.readFileSync(path.join(root, relative), "utf8")); +} + +function fail(message) { + throw new Error(message); +} + +function canonical(value) { + if (value === null) return "null"; + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (typeof value === "object") { + return `{${Object.keys(value).sort() + .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function sha256(value) { + return crypto.createHash("sha256").update(value.trim()).digest("hex"); +} + +function validateShallowSchema(value, schema, label) { + if (!value || Array.isArray(value) || typeof value !== "object") { + fail(`${label} must be an object`); + } + for (const required of schema.required ?? []) { + if (!(required in value)) fail(`${label} is missing ${required}`); + } + if (schema.additionalProperties === false) { + for (const key of Object.keys(value)) { + if (!(key in schema.properties)) fail(`${label} has unknown property ${key}`); + } + } + for (const [key, property] of Object.entries(schema.properties ?? {})) { + if (!(key in value)) continue; + const item = value[key]; + if (property.type === "object" + && (!item || Array.isArray(item) || typeof item !== "object")) { + fail(`${label}.${key} must be an object`); + } + if (property.type === "string" && typeof item !== "string") { + fail(`${label}.${key} must be a string`); + } + if (property.type === "integer" && !Number.isSafeInteger(item)) { + fail(`${label}.${key} must be an integer`); + } + if (property.type === "boolean" && typeof item !== "boolean") { + fail(`${label}.${key} must be a boolean`); + } + if ("const" in property && item !== property.const) { + fail(`${label}.${key} must equal ${property.const}`); + } + if (property.enum && !property.enum.includes(item)) { + fail(`${label}.${key} is outside its enum`); + } + if (typeof item === "string") { + if (property.minLength && item.length < property.minLength) { + fail(`${label}.${key} is too short`); + } + if (property.maxLength && item.length > property.maxLength) { + fail(`${label}.${key} is too long`); + } + if (property.pattern && !(new RegExp(property.pattern)).test(item)) { + fail(`${label}.${key} does not match its pattern`); + } + } + if (Number.isSafeInteger(item) && property.minimum !== undefined + && item < property.minimum) { + fail(`${label}.${key} is below its minimum`); + } + } +} + +function containsBlockedMaterial(value) { + if (Array.isArray(value)) return value.some(containsBlockedMaterial); + if (value && typeof value === "object") { + return Object.entries(value).some(([key, nested]) => { + const lower = key.toLowerCase(); + return lower.includes("api_key") + || lower.includes("authorization") + || lower === "auth" + || lower.includes("credential") + || lower.includes("private_key") + || lower.includes("token") + || lower.includes("secret") + || lower.includes("password") + || lower.includes("cookie") + || lower.includes("screenshot") + || containsBlockedMaterial(nested); + }); + } + if (typeof value === "string") { + const clean = value.trim(); + const lower = clean.toLowerCase(); + return lower.startsWith("bearer ") + || lower.startsWith("basic ") + || lower.includes("-----begin private key-----") + || lower.startsWith("data:image/") + || (clean.length > 2048 && /^[A-Za-z0-9+/=_-]+$/.test(clean)); + } + return false; +} + +function validateConfirmation(request) { + validateShallowSchema(request, confirmationSchema, "background confirmation"); + if (request.phone_session_id !== `background-job:${request.job_id}`) { + fail("confirmation crosses its bound job session"); + } + if (request.expires_at <= request.created_at + || request.expires_at - request.created_at > 15 * 60 * 1000) { + fail("confirmation expiry exceeds the bounded review window"); + } + if (containsBlockedMaterial(request.params)) { + fail("confirmation persists blocked secret or screenshot material"); + } + if (JSON.stringify(request).length > 12000) { + fail("confirmation exceeds its persisted bound"); + } + const paramsDigest = sha256( + `${request.tool}\n${canonical(request.params)}`, + ); + if (request.params_digest !== paramsDigest) { + fail("confirmation params digest does not match exact parameters"); + } + const bindingDigest = sha256([ + request.job_id, + request.runtime, + request.phone_session_id, + request.tool, + request.params_digest, + request.idempotency_key, + request.expires_at, + ].join("\n")); + if (request.binding_digest !== bindingDigest) { + fail("confirmation binding digest does not match bound request"); + } +} + +function validateAwaitingJob(job) { + validateShallowSchema(job, jobSchema, "agent job"); + if (job.status !== "awaiting_review" || job.phase !== "awaiting_review") { + fail("fixture must persist the awaiting-review lifecycle"); + } + if (!job.pending_confirmation_id || !job.resume_token) { + fail("awaiting job must persist confirmation and resume identifiers"); + } + validateConfirmation(job.pending_tool_request_json); + const request = job.pending_tool_request_json; + const checkpoint = job.checkpoint_json; + for (const [checkpointKey, requestKey] of [ + ["job_id", "job_id"], + ["tool", "tool"], + ["params_digest", "params_digest"], + ["binding_digest", "binding_digest"], + ["idempotency_key", "idempotency_key"], + ]) { + if (checkpoint[checkpointKey] !== request[requestKey]) { + fail(`checkpoint ${checkpointKey} drifted from pending request`); + } + } + if (checkpoint.resume_token !== job.resume_token + || checkpoint.resume_pending !== false) { + fail("awaiting checkpoint has an invalid resume binding"); + } + const checkpointSafetyView = { ...checkpoint }; + delete checkpointSafetyView.resume_token; + if (JSON.stringify(checkpoint).length > 12000 + || containsBlockedMaterial(checkpointSafetyView)) { + fail("checkpoint is unsafe or unbounded"); + } +} + +if (confirmationSchema.properties.schema.const + !== "openphone.background_confirmation.v1") { + fail("background confirmation schema marker drifted"); +} +for (const status of [ + "queued", + "running", + "waiting", + "awaiting_review", + "paused", + "completed", + "failed", + "stopped", +]) { + if (!jobSchema.properties.status.enum.includes(status)) { + fail(`agent job schema is missing lifecycle state ${status}`); + } +} + +validateAwaitingJob(readJson( + "tests/fixtures/jobs/background-awaiting-review.json", +)); +for (const name of [ + "invalid-background-review-tampered.json", + "invalid-background-review-secret.json", +]) { + let rejected = false; + try { + validateConfirmation(readJson(`tests/fixtures/jobs/${name}`)); + } catch { + rejected = true; + } + if (!rejected) fail(`invalid review fixture was accepted: ${name}`); +} + +const storeSource = fs.readFileSync(path.join( + root, + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/AgentJobStore.java", +), "utf8"); +const managerSource = fs.readFileSync(path.join( + root, + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/jobs/BackgroundJobReviewManager.java", +), "utf8"); +if (!storeSource.includes("REVIEW_LOCK") + || !storeSource.includes('.put("review_state", "resolving")')) { + fail("background review claim must atomically gate duplicate taps"); +} +if (managerSource.indexOf("claimReview(") < 0 + || managerSource.indexOf("claimReview(") + > managerSource.indexOf("executor.execute(")) { + fail("tool execution must happen only after an atomic review claim"); +} +const auditMethod = managerSource.slice( + managerSource.indexOf("private static void recordAudit"), +); +if (auditMethod.includes('.put("params",')) { + fail("background review audit must not persist raw tool parameters"); +} + +console.log("Background review contract checks passed."); diff --git a/scripts/validate-home-shell-contract.mjs b/scripts/validate-home-shell-contract.mjs new file mode 100644 index 0000000..00b9dfb --- /dev/null +++ b/scripts/validate-home-shell-contract.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +function read(relative) { + return fs.readFileSync(path.join(root, relative), "utf8"); +} + +function fail(message) { + throw new Error(message); +} + +const activity = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeActivity.kt", +); +for (const marker of [ + "setDecorFitsSystemWindows(false)", + "LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS", + "BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE", + "WindowInsets.Type.statusBars()", + "WindowInsets.Type.navigationBars()", + "override fun onWindowFocusChanged", +]) { + if (!activity.includes(marker)) { + fail(`AI Home immersive window contract is missing ${marker}`); + } +} + +const homeTheme = read( + "overlay/packages/apps/OpenPhoneAssistant/res/values/styles.xml", +); +for (const marker of [ + 'true', + 'always', +]) { + if (!homeTheme.includes(marker)) { + fail(`AI Home cold-start theme is missing ${marker}`); + } +} + +const homeCompose = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/OpenPhoneHomeComposeHost.kt", +); +for (const forbidden of ["statusBarsPadding", "navigationBarsPadding"]) { + if (homeCompose.includes(forbidden)) { + fail(`AI Home still reserves Android system-bar space: ${forbidden}`); + } +} + +const frameworkOverlay = read( + "overlay/vendor/openphone/overlay/frameworks/base/core/res/res/values/config.xml", +); +if (!frameworkOverlay.includes( + 'true', +)) { + fail("OpenPhone does not disable the non-secure lockscreen by default"); +} +for (const forbidden of [ + "config_enableCredentialFactoryResetProtection", + "config_supportsInsecureLockScreen", + "config_disableWeaverOnUnsecuredUsers", +]) { + if (frameworkOverlay.includes(forbidden)) { + fail(`lockscreen overlay weakens an unrelated security control: ${forbidden}`); + } +} + +console.log("AI Home immersive-shell contract checks passed."); diff --git a/scripts/validate-island-contract.mjs b/scripts/validate-island-contract.mjs new file mode 100755 index 0000000..29c483f --- /dev/null +++ b/scripts/validate-island-contract.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const schema = readJson("schemas/openphone-island-state.schema.json"); + +function read(relative) { + return fs.readFileSync(path.join(root, relative), "utf8"); +} + +function readJson(relative) { + return JSON.parse(read(relative)); +} + +function fail(message) { + throw new Error(message); +} + +function validate(value, label) { + if (!value || Array.isArray(value) || typeof value !== "object") { + fail(`${label} must be an object`); + } + for (const required of schema.required) { + if (!(required in value)) fail(`${label} is missing ${required}`); + } + for (const key of Object.keys(value)) { + if (!(key in schema.properties)) fail(`${label} has unknown property ${key}`); + } + for (const [key, property] of Object.entries(schema.properties)) { + if (!(key in value)) continue; + const item = value[key]; + if (property.type === "string" && typeof item !== "string") { + fail(`${label}.${key} must be a string`); + } + if (property.type === "boolean" && typeof item !== "boolean") { + fail(`${label}.${key} must be a boolean`); + } + if (property.type === "integer" && !Number.isSafeInteger(item)) { + fail(`${label}.${key} must be an integer`); + } + if ("const" in property && item !== property.const) { + fail(`${label}.${key} must equal ${property.const}`); + } + if (property.enum && !property.enum.includes(item)) { + fail(`${label}.${key} is outside its enum`); + } + if (typeof item === "string" && property.maxLength + && item.length > property.maxLength) { + fail(`${label}.${key} is too long`); + } + if (Number.isSafeInteger(item) && property.minimum !== undefined + && item < property.minimum) { + fail(`${label}.${key} is below its minimum`); + } + if (Number.isSafeInteger(item) && property.maximum !== undefined + && item > property.maximum) { + fail(`${label}.${key} is above its maximum`); + } + } + const compactCopy = [ + value.label, + value.title, + value.detail, + ].join("\n").toLowerCase(); + if (compactCopy.includes("bearer ") + || compactCopy.includes("api_key") + || compactCopy.includes("access_token") + || compactCopy.includes("private key") + || compactCopy.includes("password=")) { + fail(`${label} contains secret-like material`); + } + if (Buffer.byteLength(JSON.stringify(value), "utf8") > 16 * 1024) { + fail(`${label} exceeds the Binder payload bound`); + } +} + +for (const name of ["idle.json", "needs-review.json"]) { + validate(readJson(`tests/fixtures/island/${name}`), name); +} +for (const name of [ + "invalid-mode.json", + "invalid-secret.json", + "invalid-oversized.json", +]) { + let rejected = false; + try { + validate(readJson(`tests/fixtures/island/${name}`), name); + } catch { + rejected = true; + } + if (!rejected) fail(`invalid island fixture was accepted: ${name}`); +} + +const frameworkPatch = read( + "patches/frameworks_base/0020-OpenPhone-add-durable-island-state-contract.patch", +); +for (const marker of [ + "IOpenPhoneIslandStateListener", + "registerIslandStateListener", + "publishIslandState", + "MAX_ISLAND_STATE_BYTES", + "enforceAssistantIslandPublisher", + "RemoteCallbackList", +]) { + if (!frameworkPatch.includes(marker)) { + fail(`framework island contract is missing ${marker}`); + } +} + +const systemUiPatch = read( + "patches/frameworks_base/0021-OpenPhone-render-compact-island-in-SystemUI.patch", +); +for (const marker of [ + "TYPE_STATUS_BAR_SUB_PANEL", + "FLAG_NOT_TOUCH_MODAL", + "isDeviceLocked", + "Unlock to review private activity", + "STATE_STALE_MS", + "postStartActivityDismissingKeyguard", + "dp(110)", + "dp(34)", + "\"AI\",", + "\"◎ \" + state.liveRuns", + "background.setColor(0xff000000)", +]) { + if (!systemUiPatch.includes(marker)) { + fail(`SystemUI island renderer is missing ${marker}`); + } +} +for (const forbidden of [ + "FLAG_WATCH_OUTSIDE_TOUCH", + "WindowManager.LayoutParams.MATCH_PARENT", + ".confirmAction(", +]) { + if (systemUiPatch.includes(forbidden)) { + fail(`SystemUI island renderer contains forbidden behavior: ${forbidden}`); + } +} + +const pointerSource = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/PointerOverlayController.java", +); +const ensureStart = pointerSource.indexOf("private void ensureIslandWindow()"); +const ensureEnd = pointerSource.indexOf("private void handleIslandTap", ensureStart); +const ensureIsland = pointerSource.slice(ensureStart, ensureEnd); +if (ensureIsland.indexOf("isSystemUiOwned()") < 0 + || ensureIsland.indexOf("isSystemUiOwned()") + > ensureIsland.indexOf("TYPE_SYSTEM_ERROR")) { + fail("assistant island window is not gated behind SystemUI ownership"); +} +const pointerStart = pointerSource.indexOf("private void ensurePointerLayer()"); +const pointerEnd = pointerSource.indexOf("private void ensureIslandWindow()", pointerStart); +const pointerLayer = pointerSource.slice(pointerStart, pointerEnd); +if (!pointerLayer.includes("FLAG_NOT_TOUCHABLE")) { + fail("assistant pointer visualization must remain non-touchable"); +} + +const repositorySource = read( + "overlay/packages/apps/OpenPhoneAssistant/src/org/openphone/assistant/island/IslandStateRepository.java", +); +for (const forbidden of ["mTranscriptText", "mReplyText"]) { + if (repositorySource.includes(forbidden)) { + fail(`island projection carries forbidden rich content: ${forbidden}`); + } +} +if (!repositorySource.includes("ro.openphone.systemui_island") + || !read("overlay/vendor/openphone/products/openphone_common.mk") + .includes("ro.openphone.systemui_island=true")) { + fail("SystemUI island product ownership is not enabled"); +} + +console.log("SystemUI island contract checks passed."); diff --git a/scripts/validate-surface-contract.mjs b/scripts/validate-surface-contract.mjs new file mode 100755 index 0000000..959a879 --- /dev/null +++ b/scripts/validate-surface-contract.mjs @@ -0,0 +1,219 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const schemaPath = path.join(root, "schemas/openphone-surface.schema.json"); +const outputSchemaPath = path.join( + root, + "schemas/openphone-assistant-output.schema.json", +); +const fixturesDir = path.join(root, "tests/fixtures/surfaces"); +const actionRegistry = JSON.parse( + fs.readFileSync( + path.join(root, "overlay/vendor/openphone/config/openphone_action_registry.json"), + "utf8", + ), +); +const actionsByTool = new Map( + actionRegistry.actions.map((action) => [action.model_tool, action]), +); +const schema = JSON.parse(fs.readFileSync(schemaPath, "utf8")); +const outputSchema = JSON.parse(fs.readFileSync(outputSchemaPath, "utf8")); + +const componentTypes = new Set( + schema.$defs.component.properties.type.enum, +); +const allowedComponentKeys = new Set( + Object.keys(schema.$defs.component.properties), +); +const allowedRootKeys = new Set(Object.keys(schema.properties)); +const allowedActionKeys = new Set( + Object.keys(schema.$defs.action.properties), +); +const textKeys = ["text", "title", "subtitle", "label", "message", "placeholder"]; +const limits = { + nodes: 120, + depth: 12, + text: 24000, + images: 4, + actions: schema.properties.actions.maxProperties, + bytes: 256 * 1024, +}; + +function fail(message) { + throw new Error(message); +} + +function string(value, label, max, required = false) { + if (typeof value !== "string" || (required && value.trim().length === 0)) { + fail(`${label} must be ${required ? "a non-empty " : "a "}string`); + } + if (value.length > max) fail(`${label} exceeds ${max} characters`); +} + +function validateComponent(component, state, depth = 1) { + if (!component || Array.isArray(component) || typeof component !== "object") { + fail("surface component must be an object"); + } + if (depth > limits.depth) fail(`surface exceeds depth ${limits.depth}`); + state.nodes += 1; + if (state.nodes > limits.nodes) fail(`surface exceeds ${limits.nodes} nodes`); + if (!componentTypes.has(component.type)) { + fail(`unknown component type: ${String(component.type)}`); + } + for (const key of Object.keys(component)) { + if (!allowedComponentKeys.has(key)) fail(`unknown component property: ${key}`); + } + for (const key of textKeys) { + if (key in component) { + const max = schema.$defs.component.properties[key].maxLength; + string(component[key], `component.${key}`, max); + state.text += component[key].length; + } + } + if (state.text > limits.text) fail(`surface exceeds ${limits.text} text characters`); + if (component.type === "image") { + state.images += 1; + if (state.images > limits.images) fail(`surface exceeds ${limits.images} images`); + if (!component.artifact) fail("image requires a local artifact reference"); + } + if (component.action_id) state.actionIds.add(component.action_id); + for (const key of ["children", "items"]) { + if (key in component) { + if (!Array.isArray(component[key]) || component[key].length > 32) { + fail(`${component.type}.${key} must contain at most 32 components`); + } + for (const child of component[key]) validateComponent(child, state, depth + 1); + } + } +} + +function validateSurface(surface) { + if (!surface || Array.isArray(surface) || typeof surface !== "object") { + fail("surface must be an object"); + } + if (Buffer.byteLength(JSON.stringify(surface), "utf8") > limits.bytes) { + fail(`surface exceeds ${limits.bytes} bytes`); + } + for (const key of Object.keys(surface)) { + if (!allowedRootKeys.has(key)) fail(`unknown surface property: ${key}`); + } + if (surface.schema !== "openphone.surface.v1") fail("invalid surface schema"); + string(surface.surface_id, "surface_id", 128, true); + if (!/^surface-[A-Za-z0-9._-]{1,120}$/.test(surface.surface_id)) { + fail("invalid surface_id"); + } + if (!Number.isInteger(surface.revision) || surface.revision < 1) { + fail("revision must be a positive integer"); + } + string(surface.session_id, "session_id", 160, true); + string(surface.runtime, "runtime", 80, true); + string(surface.title, "title", 160); + if (!["full", "sheet", "inline"].includes(surface.presentation)) { + fail("invalid presentation"); + } + if (!["public", "personal", "sensitive", "restricted"].includes(surface.sensitivity)) { + fail("invalid sensitivity"); + } + if (!Number.isSafeInteger(surface.expires_at) || surface.expires_at < 0) { + fail("expires_at must be a non-negative integer"); + } + const actions = surface.actions; + if (!actions || Array.isArray(actions) || typeof actions !== "object") { + fail("actions must be an object"); + } + if (Object.keys(actions).length > limits.actions) { + fail(`surface exceeds ${limits.actions} actions`); + } + const state = { nodes: 0, text: 0, images: 0, actionIds: new Set() }; + validateComponent(surface.body, state); + for (const [actionId, action] of Object.entries(actions)) { + if (!/^[A-Za-z][A-Za-z0-9._-]{0,119}$/.test(actionId)) fail("invalid action id"); + if (!action || Array.isArray(action) || typeof action !== "object") { + fail(`action ${actionId} must be an object`); + } + for (const key of Object.keys(action)) { + if (!allowedActionKeys.has(key)) fail(`unknown action property: ${key}`); + } + string(action.label, `${actionId}.label`, 80, true); + string(action.tool, `${actionId}.tool`, 80, true); + string(action.reason, `${actionId}.reason`, 240, true); + if (!action.params || Array.isArray(action.params) || typeof action.params !== "object") { + fail(`${actionId}.params must be an object`); + } + const registered = actionsByTool.get(action.tool); + if (!registered) fail(`${actionId}.tool is not registered`); + const inputSchema = registered.input_schema_json ?? {}; + const properties = inputSchema.properties ?? {}; + for (const required of inputSchema.required ?? []) { + if (required !== "reason" && !(required in action.params)) { + fail(`${actionId} is missing required tool parameter ${required}`); + } + } + for (const key of Object.keys(action.params)) { + if (!(key in properties)) fail(`${actionId} has unknown tool parameter ${key}`); + } + if (registered.kind === "action" && action.requires_confirmation !== true) { + fail(`${actionId} must disclose confirmation for a mutating tool`); + } + } + for (const actionId of state.actionIds) { + if (!(actionId in actions)) fail(`component references undeclared action: ${actionId}`); + } + for (const actionId of Object.keys(actions)) { + if (!state.actionIds.has(actionId)) fail(`action is not referenced by UI: ${actionId}`); + } + return state; +} + +if (schema.properties.schema.const !== "openphone.surface.v1") { + fail("surface schema marker drifted"); +} +if (outputSchema.properties.schema.const !== "openphone.assistant_output.v1") { + fail("assistant output schema marker drifted"); +} + +const validFixtures = ["calendar-agenda.json", "message-summary.json"]; +for (const name of validFixtures) { + validateSurface(JSON.parse(fs.readFileSync(path.join(fixturesDir, name), "utf8"))); +} + +const invalidFixtures = [ + "invalid-external-image.json", + "invalid-unknown-component.json", + "invalid-unknown-action.json", +]; +for (const name of invalidFixtures) { + let rejected = false; + try { + validateSurface(JSON.parse(fs.readFileSync(path.join(fixturesDir, name), "utf8"))); + } catch { + rejected = true; + } + if (!rejected) fail(`invalid fixture was accepted: ${name}`); +} + +for (const name of ["surface-present-event.json", "surface-replace-event.json"]) { + const event = JSON.parse( + fs.readFileSync(path.join(root, "tests/fixtures/runtime", name), "utf8"), + ); + const output = event.payload?.output; + if (!output || output.schema !== "openphone.assistant_output.v1" + || typeof output.session_id !== "string" + || typeof output.speech !== "string" + || typeof output.text !== "string") { + fail(`runtime fixture has invalid assistant output: ${name}`); + } + if (output.surface) { + validateSurface(output.surface); + if (output.surface.session_id !== output.session_id) { + fail(`runtime fixture surface crosses output session: ${name}`); + } + } +} + +console.log("Adaptive surface contract checks passed."); diff --git a/tests/fixtures/island/idle.json b/tests/fixtures/island/idle.json new file mode 100644 index 0000000..42dba75 --- /dev/null +++ b/tests/fixtures/island/idle.json @@ -0,0 +1,18 @@ +{ + "schema": "openphone.island_state.v1", + "revision": 12, + "mode": "idle", + "label": "OpenPhone", + "title": "Ready", + "detail": "", + "visible": true, + "needs_attention": false, + "sensitivity": "public", + "active_task_id": "", + "pending_confirmation_id": "", + "live_runs": 0, + "updated_at": 1784410000000, + "published_at": 1784410000001, + "published_elapsed_ms": 22000, + "source": "assistant" +} diff --git a/tests/fixtures/island/invalid-mode.json b/tests/fixtures/island/invalid-mode.json new file mode 100644 index 0000000..755a300 --- /dev/null +++ b/tests/fixtures/island/invalid-mode.json @@ -0,0 +1,18 @@ +{ + "schema": "openphone.island_state.v1", + "revision": 14, + "mode": "execute_without_review", + "label": "OpenPhone", + "title": "Working", + "detail": "Task running", + "visible": true, + "needs_attention": false, + "sensitivity": "personal", + "active_task_id": "", + "pending_confirmation_id": "", + "live_runs": 1, + "updated_at": 1784410002000, + "published_at": 1784410002001, + "published_elapsed_ms": 24000, + "source": "assistant" +} diff --git a/tests/fixtures/island/invalid-oversized.json b/tests/fixtures/island/invalid-oversized.json new file mode 100644 index 0000000..e1486fe --- /dev/null +++ b/tests/fixtures/island/invalid-oversized.json @@ -0,0 +1,18 @@ +{ + "schema": "openphone.island_state.v1", + "revision": 16, + "mode": "error", + "label": "OpenPhone", + "title": "Needs attention", + "detail": "This deliberately oversized island detail is invalid because it crosses the compact projection boundary and attempts to carry substantially more text than SystemUI needs. Rich result bodies belong in AI Home and must never be transported in this state contract.", + "visible": true, + "needs_attention": true, + "sensitivity": "personal", + "active_task_id": "", + "pending_confirmation_id": "", + "live_runs": 0, + "updated_at": 1784410004000, + "published_at": 1784410004001, + "published_elapsed_ms": 26000, + "source": "assistant" +} diff --git a/tests/fixtures/island/invalid-secret.json b/tests/fixtures/island/invalid-secret.json new file mode 100644 index 0000000..9619a54 --- /dev/null +++ b/tests/fixtures/island/invalid-secret.json @@ -0,0 +1,18 @@ +{ + "schema": "openphone.island_state.v1", + "revision": 15, + "mode": "reply", + "label": "OpenPhone", + "title": "Ready", + "detail": "Authorization: Bearer private-session-value", + "visible": true, + "needs_attention": false, + "sensitivity": "restricted", + "active_task_id": "", + "pending_confirmation_id": "", + "live_runs": 0, + "updated_at": 1784410003000, + "published_at": 1784410003001, + "published_elapsed_ms": 25000, + "source": "assistant" +} diff --git a/tests/fixtures/island/needs-review.json b/tests/fixtures/island/needs-review.json new file mode 100644 index 0000000..8a34753 --- /dev/null +++ b/tests/fixtures/island/needs-review.json @@ -0,0 +1,18 @@ +{ + "schema": "openphone.island_state.v1", + "revision": 13, + "mode": "needs_review", + "label": "Review", + "title": "Approval needed", + "detail": "Open AI Home to review", + "visible": true, + "needs_attention": true, + "sensitivity": "personal", + "active_task_id": "task-912345", + "pending_confirmation_id": "job-confirm-fixture-42", + "live_runs": 1, + "updated_at": 1784410001000, + "published_at": 1784410001001, + "published_elapsed_ms": 23000, + "source": "assistant" +} diff --git a/tests/fixtures/jobs/background-awaiting-review.json b/tests/fixtures/jobs/background-awaiting-review.json new file mode 100644 index 0000000..d5f5592 --- /dev/null +++ b/tests/fixtures/jobs/background-awaiting-review.json @@ -0,0 +1,62 @@ +{ + "id": 42, + "type": "agent_turn", + "title": "Clean up duplicate calendar event", + "prompt": "Remove the duplicate calendar event.", + "payload_json": {}, + "schedule_json": {}, + "session_target": "main", + "delivery_json": { + "mode": "notification" + }, + "status": "awaiting_review", + "created_at": 1784400000000, + "updated_at": 1784400600000, + "next_run_at": 0, + "running_at": 0, + "last_run_at": 0, + "last_result": "", + "failure_count": 0, + "failure_alert_at": 0, + "phase": "awaiting_review", + "progress_text": "Approval needed: calendar_delete_event", + "progress_current": 0, + "progress_total": 0, + "checkpoint_json": { + "schema": "openphone.background_checkpoint.v1", + "job_id": 42, + "phase": "awaiting_review", + "tool": "calendar_delete_event", + "params_digest": "9f293c6f544ebcd106ac33dd942773bf8b945f9f8e981f533583bd0ebb651b68", + "binding_digest": "195b6e1f2f5fc1f31ce429b2e22654571f3ebfda9da064eb6c7c9313b0b7f9d9", + "idempotency_key": "job:42:fixture-delete-event", + "resume_token": "job-resume-fixture-42", + "created_at": 1784400600000, + "resume_pending": false + }, + "pending_confirmation_id": "job-confirm-fixture-42", + "pending_tool_request_json": { + "schema": "openphone.background_confirmation.v1", + "confirmation_id": "job-confirm-fixture-42", + "job_id": 42, + "runtime": "builtin", + "phone_session_id": "background-job:42", + "tool": "calendar_delete_event", + "params": { + "event_id": 123, + "reason": "Remove duplicate meeting" + }, + "params_digest": "9f293c6f544ebcd106ac33dd942773bf8b945f9f8e981f533583bd0ebb651b68", + "binding_digest": "195b6e1f2f5fc1f31ce429b2e22654571f3ebfda9da064eb6c7c9313b0b7f9d9", + "idempotency_key": "job:42:fixture-delete-event", + "created_at": 1784400600000, + "expires_at": 1784401500000, + "review_state": "pending", + "summary": "calendar delete event with {\"event_id\":123,\"reason\":\"Remove duplicate meeting\"}" + }, + "last_surface_id": "", + "resume_token": "job-resume-fixture-42", + "last_event_at": 1784400600000, + "unread_result": false, + "paused_at": 0 +} diff --git a/tests/fixtures/jobs/invalid-background-review-secret.json b/tests/fixtures/jobs/invalid-background-review-secret.json new file mode 100644 index 0000000..a4caef5 --- /dev/null +++ b/tests/fixtures/jobs/invalid-background-review-secret.json @@ -0,0 +1,19 @@ +{ + "schema": "openphone.background_confirmation.v1", + "confirmation_id": "job-confirm-secret", + "job_id": 42, + "runtime": "builtin", + "phone_session_id": "background-job:42", + "tool": "open_url", + "params": { + "authorization": "Bearer must-not-be-checkpointed", + "url": "https://example.com" + }, + "params_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "binding_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "idempotency_key": "job:42:secret", + "created_at": 1784400600000, + "expires_at": 1784401500000, + "review_state": "pending", + "summary": "Unsafe request" +} diff --git a/tests/fixtures/jobs/invalid-background-review-tampered.json b/tests/fixtures/jobs/invalid-background-review-tampered.json new file mode 100644 index 0000000..2aeef4e --- /dev/null +++ b/tests/fixtures/jobs/invalid-background-review-tampered.json @@ -0,0 +1,19 @@ +{ + "schema": "openphone.background_confirmation.v1", + "confirmation_id": "job-confirm-tampered", + "job_id": 42, + "runtime": "builtin", + "phone_session_id": "background-job:42", + "tool": "calendar_delete_event", + "params": { + "event_id": 999, + "reason": "Tampered after approval" + }, + "params_digest": "9f293c6f544ebcd106ac33dd942773bf8b945f9f8e981f533583bd0ebb651b68", + "binding_digest": "195b6e1f2f5fc1f31ce429b2e22654571f3ebfda9da064eb6c7c9313b0b7f9d9", + "idempotency_key": "job:42:fixture-delete-event", + "created_at": 1784400600000, + "expires_at": 1784401500000, + "review_state": "pending", + "summary": "Tampered request" +} diff --git a/tests/fixtures/runtime/surface-dismiss-event.json b/tests/fixtures/runtime/surface-dismiss-event.json new file mode 100644 index 0000000..4e7e66d --- /dev/null +++ b/tests/fixtures/runtime/surface-dismiss-event.json @@ -0,0 +1,9 @@ +{ + "event": "runtime.surface.dismiss", + "payload": { + "sessionKey": "openphone:fixture:phone-sess-fixture", + "surface_id": "surface-runtime-fixture", + "revision": 2, + "reason": "runtime_completed" + } +} diff --git a/tests/fixtures/runtime/surface-present-event.json b/tests/fixtures/runtime/surface-present-event.json new file mode 100644 index 0000000..0b4b769 --- /dev/null +++ b/tests/fixtures/runtime/surface-present-event.json @@ -0,0 +1,35 @@ +{ + "event": "runtime.surface.present", + "payload": { + "sessionKey": "openphone:fixture:phone-sess-fixture", + "output": { + "schema": "openphone.assistant_output.v1", + "session_id": "phone-sess-fixture", + "speech": "Here is your afternoon.", + "text": "You have one event.", + "surface": { + "schema": "openphone.surface.v1", + "surface_id": "surface-runtime-fixture", + "revision": 1, + "session_id": "phone-sess-fixture", + "runtime": "openclaw", + "title": "Today", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "personal", + "body": { + "type": "column", + "children": [ + { + "type": "text", + "style": "headline", + "text": "Your afternoon" + } + ] + }, + "actions": {} + }, + "background_run": null + } + } +} diff --git a/tests/fixtures/runtime/surface-replace-event.json b/tests/fixtures/runtime/surface-replace-event.json new file mode 100644 index 0000000..e9cfad2 --- /dev/null +++ b/tests/fixtures/runtime/surface-replace-event.json @@ -0,0 +1,31 @@ +{ + "event": "runtime.surface.replace", + "payload": { + "sessionKey": "openphone:fixture:phone-sess-fixture", + "expected_revision": 1, + "output": { + "schema": "openphone.assistant_output.v1", + "session_id": "phone-sess-fixture", + "speech": "", + "text": "The agenda changed.", + "surface": { + "schema": "openphone.surface.v1", + "surface_id": "surface-runtime-fixture", + "revision": 2, + "session_id": "phone-sess-fixture", + "runtime": "openclaw", + "title": "Today", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "personal", + "body": { + "type": "text", + "style": "body", + "text": "Updated agenda" + }, + "actions": {} + }, + "background_run": null + } + } +} diff --git a/tests/fixtures/surfaces/calendar-agenda.json b/tests/fixtures/surfaces/calendar-agenda.json new file mode 100644 index 0000000..feb7b52 --- /dev/null +++ b/tests/fixtures/surfaces/calendar-agenda.json @@ -0,0 +1,46 @@ +{ + "schema": "openphone.surface.v1", + "surface_id": "surface-calendar-fixture", + "revision": 1, + "session_id": "phone-sess-fixture", + "runtime": "builtin", + "title": "Today", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "personal", + "created_at": 1784419200000, + "body": { + "type": "column", + "spacing": "md", + "children": [ + { + "type": "text", + "style": "headline", + "text": "Your afternoon" + }, + { + "type": "list", + "items": [ + { + "type": "list_item", + "title": "Design review", + "subtitle": "2:00 PM · Meet", + "icon": "calendar", + "action_id": "search_event" + } + ] + } + ] + }, + "actions": { + "search_event": { + "label": "Open event", + "tool": "calendar_search", + "params": { + "query": "Design review", + "limit": 8 + }, + "reason": "User selected an event from the generated agenda" + } + } +} diff --git a/tests/fixtures/surfaces/invalid-external-image.json b/tests/fixtures/surfaces/invalid-external-image.json new file mode 100644 index 0000000..f8d013e --- /dev/null +++ b/tests/fixtures/surfaces/invalid-external-image.json @@ -0,0 +1,16 @@ +{ + "schema": "openphone.surface.v1", + "surface_id": "surface-invalid-image", + "revision": 1, + "session_id": "phone-sess-fixture", + "runtime": "builtin", + "title": "Unsafe", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "public", + "body": { + "type": "image", + "text": "https://example.com/tracker.png" + }, + "actions": {} +} diff --git a/tests/fixtures/surfaces/invalid-unknown-action.json b/tests/fixtures/surfaces/invalid-unknown-action.json new file mode 100644 index 0000000..2ade670 --- /dev/null +++ b/tests/fixtures/surfaces/invalid-unknown-action.json @@ -0,0 +1,24 @@ +{ + "schema": "openphone.surface.v1", + "surface_id": "surface-invalid-action", + "revision": 1, + "session_id": "phone-sess-fixture", + "runtime": "builtin", + "title": "Unsafe", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "public", + "body": { + "type": "button", + "label": "Run unknown action", + "action_id": "unknown_action" + }, + "actions": { + "unknown_action": { + "label": "Run unknown action", + "tool": "unregistered_phone_tool", + "params": {}, + "reason": "This must fail closed" + } + } +} diff --git a/tests/fixtures/surfaces/invalid-unknown-component.json b/tests/fixtures/surfaces/invalid-unknown-component.json new file mode 100644 index 0000000..acd8ab6 --- /dev/null +++ b/tests/fixtures/surfaces/invalid-unknown-component.json @@ -0,0 +1,16 @@ +{ + "schema": "openphone.surface.v1", + "surface_id": "surface-invalid-component", + "revision": 1, + "session_id": "phone-sess-fixture", + "runtime": "builtin", + "title": "Unsafe", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "public", + "body": { + "type": "webview", + "text": "" + }, + "actions": {} +} diff --git a/tests/fixtures/surfaces/message-summary.json b/tests/fixtures/surfaces/message-summary.json new file mode 100644 index 0000000..b108f5c --- /dev/null +++ b/tests/fixtures/surfaces/message-summary.json @@ -0,0 +1,53 @@ +{ + "schema": "openphone.surface.v1", + "surface_id": "surface-message-fixture", + "revision": 1, + "session_id": "phone-sess-fixture", + "runtime": "builtin", + "title": "Messages", + "presentation": "full", + "expires_at": 4102444800000, + "sensitivity": "sensitive", + "body": { + "type": "column", + "spacing": "md", + "children": [ + { + "type": "capability_disclosure", + "capability": "messages.read", + "text": "From recent messages on this phone" + }, + { + "type": "card", + "children": [ + { + "type": "text", + "style": "title", + "text": "Sam" + }, + { + "type": "text", + "style": "body", + "text": "Can we move dinner to 7?" + }, + { + "type": "button", + "label": "Find thread", + "action_id": "find_thread" + } + ] + } + ] + }, + "actions": { + "find_thread": { + "label": "Find thread", + "tool": "messages_search", + "params": { + "query": "Sam", + "limit": 8 + }, + "reason": "User selected a message summary item" + } + } +} diff --git a/tests/integrations/ifp1-adapter-contract.mjs b/tests/integrations/ifp1-adapter-contract.mjs new file mode 100644 index 0000000..f1eab9f --- /dev/null +++ b/tests/integrations/ifp1-adapter-contract.mjs @@ -0,0 +1,283 @@ +#!/usr/bin/env node + +// Exercises the ifp/1 action-lifecycle adapter +// (runtime/protocol/ifp1/adapter.mjs) end to end against the vendored +// reference validator and state machine: the legacy runtime events +// (runtime.tool.requested / runtime.confirmation.resolved / +// runtime.tool.result) become ifp/1 action messages that a conformant +// runtime accepts, and the failure paths (user rejection, binding mismatch, +// expiry) produce the spec-mandated terminal outcomes. + +import assert from "node:assert/strict"; + +import { + actionMessageFromConfirmationResolution, + canonicalJson, + paramsHash, + proposeFromToolRequest, + resultFromToolResult, + toolNameFromCommand, +} from "../../runtime/protocol/ifp1/adapter.mjs"; +import { validateMessage } from "../../runtime/protocol/ifp1/validator.ts"; +import { ConformanceMachine } from "../../runtime/protocol/ifp1/machine.ts"; + +let cases = 0; +const check = (name, fn) => { + fn(); + cases += 1; +}; + +const GENERATION = 3; +const baseContext = { + entity: "ent_test01", + session: "ses_test01", + device: "dev_phone_01", + generation: GENERATION, +}; + +// Deterministic clock inside the proposal's validity window. +const T0 = Date.parse("2026-07-21T02:00:00Z"); +let nowMs = T0; +const clock = () => nowMs; + +function legacyToolRequest(overrides = {}) { + return { + request_id: "req-42-abc", + runtime: "openclaw", + runtime_session_id: "rs-1", + tool: "openphone.messages.send", + params: { to: "+15551234567", body: "on my way" }, + reason: "user asked to notify Dana", + ...overrides, + }; +} + +function propose(overrides = {}) { + return proposeFromToolRequest(legacyToolRequest(overrides), { + ...baseContext, + risk: "high", + clock, + }); +} + +// --- canonical hashing (spec 8.1) ------------------------------------------- +check("canonical json sorts keys at every depth", () => { + assert.equal( + canonicalJson({ z: { b: 2, a: 1 }, a: [3, { y: true, x: null }] }), + '{"a":[3,{"x":null,"y":true}],"z":{"a":1,"b":2}}', + ); +}); + +check("params_hash matches the shared spec 8.1 vector", () => { + assert.equal( + paramsHash({ to: "+15551234567", body: "on my way" }), + "sha256:1e920af2c02daa49a449d1ebe938e08f196a9dfa6a3c05b36adbabbd7387193b", + ); +}); + +check("tool names drop the openphone. prefix", () => { + assert.equal(toolNameFromCommand("openphone.messages.send"), "messages.send"); + assert.equal(toolNameFromCommand("messages.send"), "messages.send"); +}); + +// --- runtime.tool.requested -> action.propose -------------------------------- +check("tool request maps to a structurally valid action.propose", () => { + const proposal = propose(); + const verdict = validateMessage(proposal); + assert.equal(verdict.ok, true, `${verdict.errorClass}: ${verdict.detail}`); + assert.equal(proposal.type, "action.propose"); + assert.equal(proposal.payload.tool, "messages.send"); + assert.equal(proposal.payload.risk, "high"); + assert.equal(proposal.payload.params_hash, paramsHash(proposal.payload.params)); + assert.equal(proposal.correlation, "req-42-abc"); + assert.equal(proposal.payload.idempotency_key, "req-42-abc"); + assert.ok(proposal.payload.expires_at > proposal.ts, "expiry must be in the future"); +}); + +// --- full approve -> execute lifecycle --------------------------------------- +check("approve then execute walks requested -> approved -> executed", () => { + const machine = new ConformanceMachine(GENERATION); + const proposal = propose(); + assert.equal(machine.step(proposal).actionState, "requested"); + + nowMs = T0 + 5_000; + const approve = actionMessageFromConfirmationResolution( + { approved: true, confirmation_id: "runtime-confirm-1" }, + proposal, + { ...baseContext, clock }, + ); + assert.equal(validateMessage(approve).ok, true); + assert.equal(approve.type, "action.approve"); + assert.equal(approve.payload.params_hash, proposal.payload.params_hash); + assert.equal(approve.payload.confirmation.mode, "user_confirmed"); + assert.equal(approve.payload.confirmation.ref, "runtime-confirm-1"); + const approveStep = machine.step(approve); + assert.equal(approveStep.verdict, "accepted"); + assert.equal(approveStep.actionState, "approved"); + + nowMs = T0 + 8_000; + const result = resultFromToolResult( + { request_id: "req-42-abc", status: "ok", result: { sent: true } }, + proposal, + { + ...baseContext, + approvedParamsHash: approve.payload.params_hash, + executedParams: proposal.payload.params, + evidence: "audit-evt-77", + clock, + }, + ); + assert.equal(validateMessage(result).ok, true); + assert.equal(result.payload.status, "executed"); + assert.equal(result.payload.error_class, null); + assert.equal(result.payload.evidence, "audit-evt-77"); + const resultStep = machine.step(result); + assert.equal(resultStep.verdict, "accepted"); + assert.equal(resultStep.actionState, "executed"); +}); + +// --- user rejection is terminal ---------------------------------------------- +check("user denial maps to action.reject and is terminal", () => { + const machine = new ConformanceMachine(GENERATION); + nowMs = T0; + const proposal = propose({ request_id: "req-43-rej" }); + machine.step(proposal); + + nowMs = T0 + 2_000; + const reject = actionMessageFromConfirmationResolution( + { approved: false, confirmation_id: "runtime-confirm-2" }, + proposal, + { ...baseContext, clock }, + ); + assert.equal(validateMessage(reject).ok, true); + assert.equal(reject.type, "action.reject"); + assert.equal(reject.payload.reason_class, "user_denied"); + const rejectStep = machine.step(reject); + assert.equal(rejectStep.verdict, "accepted"); + assert.equal(rejectStep.actionState, "rejected"); + + // A later approval of the rejected action is a protocol violation. + const lateApprove = actionMessageFromConfirmationResolution( + { approved: true }, + proposal, + { ...baseContext, clock }, + ); + const lateStep = machine.step(lateApprove); + assert.equal(lateStep.verdict, "rejected"); + assert.equal(lateStep.actionState, "rejected"); +}); + +// --- params_hash mismatch => binding_mismatch --------------------------------- +check("approve with tampered params is rejected as binding_mismatch", () => { + const machine = new ConformanceMachine(GENERATION); + nowMs = T0; + const proposal = propose({ request_id: "req-44-bind" }); + machine.step(proposal); + + nowMs = T0 + 2_000; + const tamperedApprove = actionMessageFromConfirmationResolution( + { + approved: true, + executed_params: { to: "+15559999999", body: "on my way" }, + }, + proposal, + { ...baseContext, clock }, + ); + assert.equal(validateMessage(tamperedApprove).ok, true); + assert.notEqual(tamperedApprove.payload.params_hash, proposal.payload.params_hash); + const step = machine.step(tamperedApprove); + assert.equal(step.verdict, "rejected"); + assert.equal(step.errorClass, "binding_mismatch"); + assert.equal(step.actionState, "requested", "binding failure leaves the action requested (spec 8.3)"); +}); + +check("executed params diverging from approval fail as binding_mismatch without executing", () => { + nowMs = T0; + const proposal = propose({ request_id: "req-45-exec" }); + const approvedHash = proposal.payload.params_hash; + + nowMs = T0 + 3_000; + const result = resultFromToolResult( + { request_id: "req-45-exec", status: "ok" }, + proposal, + { + ...baseContext, + approvedParamsHash: approvedHash, + executedParams: { to: "+15550000000", body: "on my way" }, + clock, + }, + ); + assert.equal(validateMessage(result).ok, true); + assert.equal(result.payload.status, "failed"); + assert.equal(result.payload.error_class, "binding_mismatch"); +}); + +// --- expiry ------------------------------------------------------------------- +check("approval after expires_at is rejected as action_expired", () => { + const machine = new ConformanceMachine(GENERATION); + nowMs = T0; + const proposal = propose({ request_id: "req-46-exp" }); + machine.step(proposal); + + nowMs = Date.parse(proposal.payload.expires_at) + 60_000; + const lateApprove = actionMessageFromConfirmationResolution( + { approved: true }, + proposal, + { ...baseContext, clock }, + ); + const step = machine.step(lateApprove); + assert.equal(step.verdict, "rejected"); + assert.equal(step.errorClass, "action_expired"); + assert.equal(step.actionState, "expired"); +}); + +check("execution attempted past expiry reports failed/action_expired", () => { + nowMs = T0; + const proposal = propose({ request_id: "req-47-exp" }); + nowMs = Date.parse(proposal.payload.expires_at) + 60_000; + const result = resultFromToolResult( + { request_id: "req-47-exp", status: "ok" }, + proposal, + { ...baseContext, clock }, + ); + assert.equal(result.payload.status, "failed"); + assert.equal(result.payload.error_class, "action_expired"); +}); + +// --- error path from the legacy bridge ---------------------------------------- +check("legacy error results map to failed/internal", () => { + const machine = new ConformanceMachine(GENERATION); + nowMs = T0; + const proposal = propose({ request_id: "req-48-err" }); + machine.step(proposal); + nowMs = T0 + 1_000; + machine.step(actionMessageFromConfirmationResolution( + { approved: true }, proposal, { ...baseContext, clock }, + )); + nowMs = T0 + 2_000; + const result = resultFromToolResult( + { request_id: "req-48-err", status: "error", error: { code: "tool_failed" } }, + proposal, + { ...baseContext, clock }, + ); + assert.equal(result.payload.status, "failed"); + assert.equal(result.payload.error_class, "internal"); + const step = machine.step(result); + assert.equal(step.verdict, "accepted"); + assert.equal(step.actionState, "failed"); +}); + +// --- policy_auto confirmations -------------------------------------------------- +check("trusted_actions resolutions map to policy_auto confirmation mode", () => { + nowMs = T0; + const proposal = propose({ request_id: "req-49-auto" }); + const approve = actionMessageFromConfirmationResolution( + { approved: true }, + proposal, + { ...baseContext, mode: "policy_auto", clock }, + ); + assert.equal(validateMessage(approve).ok, true); + assert.equal(approve.payload.confirmation.mode, "policy_auto"); +}); + +console.log(`ifp/1 adapter contract passed (${cases} cases).`); diff --git a/tests/integrations/ifp1-conformance-contract.mjs b/tests/integrations/ifp1-conformance-contract.mjs new file mode 100644 index 0000000..4b88a71 --- /dev/null +++ b/tests/integrations/ifp1-conformance-contract.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node + +// Runs the vendored ifp/1 conformance fixture corpus through the vendored +// TypeScript reference validator and state machine +// (runtime/protocol/ifp1/validator.ts, machine.ts). Verdicts must match the +// fixtures exactly — this is the OpenPhone side of spec section 11 +// ("a surface is conformant when it passes the fixture corpus"). + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +import { validateMessage } from "../../runtime/protocol/ifp1/validator.ts"; +import { ConformanceMachine } from "../../runtime/protocol/ifp1/machine.ts"; +import { paramsHash, toolNameFromCommand } from "../../runtime/protocol/ifp1/adapter.mjs"; + +const ifp1Root = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../../runtime/protocol/ifp1"); +const readJson = (file) => JSON.parse(fs.readFileSync(path.join(ifp1Root, file), "utf8")); + +let cases = 0; + +// --- fixtures/messages.json: single-message accept/reject verdicts --------- +const messages = readJson("fixtures/messages.json"); +for (const c of messages.cases) { + const result = validateMessage(c.message); + if (c.expect === "accept") { + assert.equal(result.ok, true, `${c.name}: expected accept, got ${result.errorClass}: ${result.detail}`); + } else { + assert.equal(result.ok, false, `${c.name}: expected reject, got accept`); + assert.equal(result.errorClass, c.error_class, `${c.name}: ${result.detail ?? ""}`); + } + cases += 1; +} + +// --- fixtures/sequences.json: stateful scenarios ---------------------------- +const sequences = readJson("fixtures/sequences.json"); +for (const scenario of sequences.scenarios) { + const machine = new ConformanceMachine(sequences.initial_state.home_generation); + scenario.steps.forEach((step, i) => { + const result = machine.step(step.give); + const expect = step.expect; + const label = `${scenario.name} step ${i}`; + assert.equal(result.verdict, expect.verdict, `${label}: error=${result.errorClass}`); + if ("error_class" in expect) assert.equal(result.errorClass, expect.error_class, label); + if ("action_state" in expect) assert.equal(result.actionState, expect.action_state, label); + if ("stored" in expect) assert.equal(result.stored, expect.stored, label); + if ("error_payload_includes" in expect) { + assert.ok(result.errorPayload, label); + for (const [key, value] of Object.entries(expect.error_payload_includes)) { + assert.deepEqual(result.errorPayload[key], value, label); + } + } + }); + cases += 1; +} + +// --- fixtures/params-hash-vectors.json: canonical hashing (spec 8.1) -------- +const vectors = readJson("fixtures/params-hash-vectors.json"); +for (const vector of vectors.vectors) { + assert.equal(paramsHash(vector.params), vector.params_hash, `params-hash vector: ${vector.name}`); + cases += 1; +} + +// --- openphone-tools.ifp1.json: manifest re-expresses all 37 commands ------- +const toolsManifest = readJson("openphone-tools.ifp1.json"); +const oldManifest = JSON.parse( + fs.readFileSync(path.join(ifp1Root, "..", "openphone-commands.json"), "utf8"), +); +assert.equal(toolsManifest.ifp, "1"); +assert.equal(toolsManifest.tools.length, oldManifest.commands.length); +assert.equal(toolsManifest.tools.length, 37, "the v0 manifest declares 37 commands"); + +const oldByName = new Map(oldManifest.commands.map((c) => [c.name, c])); +const seenTools = new Set(); +for (const tool of toolsManifest.tools) { + assert.ok(!seenTools.has(tool.tool), `duplicate ifp/1 tool: ${tool.tool}`); + seenTools.add(tool.tool); + + const oldName = tool["x-openphone"].superseded_command; + const old = oldByName.get(oldName); + assert.ok(old, `ifp/1 tool ${tool.tool} references unknown old command ${oldName}`); + assert.equal(tool.tool, toolNameFromCommand(old.name), `tool name must be old name minus prefix: ${oldName}`); + assert.equal(tool.capability, `tool:${tool.tool}`, `capability declaration mismatch: ${tool.tool}`); + assert.ok(["low", "medium", "high"].includes(tool.risk), `risk not in ifp/1 registry: ${tool.tool}`); + assert.equal(tool.risk, old.risk, `risk must carry over from the v0 manifest: ${tool.tool}`); + assert.deepEqual(tool.params_schema, old.input_schema, `params_schema must carry over: ${tool.tool}`); + cases += 1; +} +for (const old of oldManifest.commands) { + assert.ok(seenTools.has(toolNameFromCommand(old.name)), `old command not re-expressed: ${old.name}`); +} + +// Risk levels must be consistent with the v0 confirmation policy they encode: +// every "always"-confirm command is high risk; every high-risk tool confirms. +for (const tool of toolsManifest.tools) { + const old = oldByName.get(tool["x-openphone"].superseded_command); + if (old.confirmation === "always") assert.equal(tool.risk, "high", `always-confirm must be high risk: ${tool.tool}`); + if (tool.risk === "high") assert.equal(old.confirmation, "always", `high risk must always confirm: ${tool.tool}`); +} + +console.log(`ifp/1 conformance contract passed (${cases} cases: ${messages.cases.length} messages, ${sequences.scenarios.length} sequences, ${vectors.vectors.length} hash vectors, ${toolsManifest.tools.length} tool mappings).`);