From f4ce31048f20b96c668ff0d18be7e9082556acde Mon Sep 17 00:00:00 2001 From: Vishay Agarwal Date: Wed, 18 Mar 2026 00:04:54 -0700 Subject: [PATCH] refactor for demo actions (spotify, gmail, texting) --- .dockerignore | 6 + .env.jarvis.example | 6 + .gitignore | 4 + Dockerfile | 66 +++ README.md | 68 ++- browser_sidecar/server.mjs | 168 ++++++-- docker-compose.yml | 29 ++ jarvis_rs/src/agent.rs | 222 ++++++++-- jarvis_rs/src/agent_core.rs | 266 ++++++++++++ jarvis_rs/src/app.rs | 9 +- jarvis_rs/src/automation.rs | 451 ++++++++++++++------ jarvis_rs/src/bin/agent_core_server.rs | 53 +++ jarvis_rs/src/bin/desktop_demo.rs | 457 ++++++++++++++++++++ jarvis_rs/src/bin/scenario_runner.rs | 560 ++++++++++++++++++++++++ jarvis_rs/src/browser_sidecar.rs | 125 +++++- jarvis_rs/src/config.rs | 30 +- jarvis_rs/src/executor/command.rs | 20 +- jarvis_rs/src/executor/systems.rs | 35 +- jarvis_rs/src/intent/mod.rs | 2 +- jarvis_rs/src/intent/schema.rs | 2 +- jarvis_rs/src/intent/validator.rs | 19 +- jarvis_rs/src/lib.rs | 1 + jarvis_rs/src/llm/mod.rs | 5 +- jarvis_rs/src/llm/ollama_client.rs | 6 +- jarvis_rs/src/llm/provider.rs | 461 +++++++++++++++----- jarvis_rs/src/llm/schema.rs | 7 + jarvis_rs/src/speech.rs | 21 +- main.py | 5 + requirements.txt | 5 + scripts/bootstrap_macos.sh | 109 +++++ scripts/doctor.sh | 8 + scripts/run_jarvis.sh | 38 ++ target/.rustc_info.json | 2 +- tools/ocr.swift | 85 ++++ voice_os/__init__.py | 1 + voice_os/agent.py | 566 +++++++++++++++++++++++++ voice_os/browser.py | 80 ++++ voice_os/cli.py | 173 ++++++++ voice_os/config.py | 158 +++++++ voice_os/control.py | 287 +++++++++++++ voice_os/intent.py | 195 +++++++++ voice_os/llm.py | 109 +++++ voice_os/memory.py | 30 ++ voice_os/planner.py | 176 ++++++++ voice_os/runtime.py | 34 ++ voice_os/schemas.py | 116 +++++ voice_os/verifier.py | 83 ++++ voice_os/vision.py | 123 ++++++ voice_os/voice.py | 141 ++++++ 49 files changed, 5235 insertions(+), 388 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.jarvis.example create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 jarvis_rs/src/agent_core.rs create mode 100644 jarvis_rs/src/bin/agent_core_server.rs create mode 100644 jarvis_rs/src/bin/desktop_demo.rs create mode 100644 jarvis_rs/src/bin/scenario_runner.rs create mode 100644 requirements.txt create mode 100755 scripts/bootstrap_macos.sh create mode 100755 scripts/doctor.sh create mode 100755 scripts/run_jarvis.sh create mode 100644 tools/ocr.swift create mode 100644 voice_os/__init__.py create mode 100644 voice_os/agent.py create mode 100644 voice_os/browser.py create mode 100644 voice_os/cli.py create mode 100644 voice_os/config.py create mode 100644 voice_os/control.py create mode 100644 voice_os/intent.py create mode 100644 voice_os/llm.py create mode 100644 voice_os/memory.py create mode 100644 voice_os/planner.py create mode 100644 voice_os/runtime.py create mode 100644 voice_os/schemas.py create mode 100644 voice_os/verifier.py create mode 100644 voice_os/vision.py create mode 100644 voice_os/voice.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bff34f9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +target +Downloads +scenario_artifacts +.tooling/browser-profile +jarvis_rs/target diff --git a/.env.jarvis.example b/.env.jarvis.example new file mode 100644 index 0000000..6e8236c --- /dev/null +++ b/.env.jarvis.example @@ -0,0 +1,6 @@ +JARVIS_FALLBACK_BASE_URL=https://api.openai.com/v1 +JARVIS_FALLBACK_MODEL=your-model-name +JARVIS_FALLBACK_API_KEY=your-api-key +ELEVENLABS_API_KEY=your-elevenlabs-api-key +ELEVENLABS_VOICE_ID=your-elevenlabs-voice-id +JARVIS_DISABLE_LLM=0 diff --git a/.gitignore b/.gitignore index 1dfde52..eb14ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ /target .tooling/ +.venv/ +__pycache__/ +artifacts/ +scenario_artifacts/ *.wav transcript.txt browser_sidecar/node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b35050f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +FROM rust:1.86-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + libasound2-dev \ + libgl1-mesa-dev \ + libglu1-mesa-dev \ + libudev-dev \ + libwayland-dev \ + libx11-dev \ + libxcursor-dev \ + libxinerama-dev \ + libxkbcommon-dev \ + libxrandr-dev \ + nodejs \ + npm \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY Cargo.toml Cargo.lock ./ +COPY jarvis_rs/Cargo.toml jarvis_rs/Cargo.toml +COPY browser_sidecar/package.json browser_sidecar/package-lock.json browser_sidecar/ +RUN mkdir -p jarvis_rs/src && printf "fn main() {}\n" > jarvis_rs/src/main.rs +RUN printf "pub fn placeholder() {}\n" > jarvis_rs/src/lib.rs +RUN cargo build --release --bin agent_core_server || true + +COPY . . +RUN npm ci --prefix browser_sidecar +RUN cargo build --release --bin agent_core_server --bin scenario_runner + +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libasound2 \ + libgl1 \ + libglu1-mesa \ + libudev1 \ + libwayland-client0 \ + libx11-6 \ + libxcursor1 \ + libxinerama1 \ + libxkbcommon0 \ + libxrandr2 \ + nodejs \ + npm \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /app/target/release/agent_core_server /usr/local/bin/agent_core_server +COPY --from=builder /app/target/release/scenario_runner /usr/local/bin/scenario_runner +COPY --from=builder /app/browser_sidecar /app/browser_sidecar + +ENV JARVIS_AGENT_CORE_BIND=0.0.0.0:8788 + +EXPOSE 8788 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -fsS http://127.0.0.1:8788/health || exit 1 + +CMD ["/usr/local/bin/agent_core_server"] diff --git a/README.md b/README.md index cb8add7..8729ae1 100644 --- a/README.md +++ b/README.md @@ -1 +1,67 @@ -*in progress* +# Jarvis Voice OS + +Voice-first desktop agent for macOS with: + +- local Whisper STT via `whisper-cli` +- browser control through a Playwright sidecar +- screen observation through screenshot + Vision OCR +- grounded desktop actions via `pyautogui` +- planner/executor/verifier loop with deterministic demo-safe planning and optional local Qwen intent help + +## Quickstart + +Use the real repo root: + +```bash +cd /Users/agarwal/coding/jarvis/jarvis +./scripts/bootstrap_macos.sh +./scripts/run_jarvis.sh run --voice +``` + +For a text-triggered command: + +```bash +./scripts/run_jarvis.sh run --text "Search YouTube for NBA highlights" +``` + +For continuous voice capture: + +```bash +./scripts/run_jarvis.sh voice-loop +``` + +To run the local required test suite: + +```bash +./scripts/doctor.sh +``` + +## Architecture + +```text +Microphone / text input + -> Whisper STT + -> Intent resolver (heuristics first, local LLM fallback) + -> Planner + -> Observe screen/browser + -> Execute tool + -> Re-observe + -> Verify + -> TTS response + +Tools: + open_app + browser_attach_or_launch + browser_open + browser_assert + press_hotkey + type_text + read_screen +``` + +## Notes + +- `bootstrap_macos.sh` creates `.venv`, installs Python and browser sidecar dependencies, compiles the OCR helper, and downloads local models if needed. +- Grant Accessibility, Screen Recording, Automation, and Microphone permissions when macOS prompts. +- Google Chrome is the default browser target. Set `JARVIS_CHROME_EXECUTABLE` if Chrome lives somewhere else. +- If `ELEVENLABS_API_KEY` and `ELEVENLABS_VOICE_ID` are set, Jarvis will use ElevenLabs TTS; otherwise it falls back to macOS `say`. diff --git a/browser_sidecar/server.mjs b/browser_sidecar/server.mjs index f08e702..6288a65 100644 --- a/browser_sidecar/server.mjs +++ b/browser_sidecar/server.mjs @@ -15,6 +15,48 @@ let context = null; let page = null; let mode = 'uninitialized'; +function isRecoverableSessionError(error) { + const message = error instanceof Error ? error.message : String(error); + const normalized = message.toLowerCase(); + return normalized.includes('target page, context or browser has been closed') + || normalized.includes('browser has been closed') + || normalized.includes('browser context closed') + || normalized.includes('context has been closed') + || normalized.includes('target closed') + || normalized.includes('connection closed') + || normalized.includes('browser disconnected') + || normalized.includes('has been collected to prevent unbounded heap growth'); +} + +async function clearSessionState() { + page = null; + context = null; + browser = null; + mode = 'uninitialized'; +} + +async function closeSessionState() { + const activeContext = context; + const activeBrowser = browser; + await clearSessionState(); + + if (activeContext) { + try { + await activeContext.close(); + } catch (_error) { + // Best-effort cleanup so we can relaunch a fresh session. + } + } + + if (activeBrowser && typeof activeBrowser.isConnected === 'function' && activeBrowser.isConnected()) { + try { + await activeBrowser.close(); + } catch (_error) { + // CDP-attached browsers may reject close; keep cleanup best-effort. + } + } +} + function jsonResponse(res, statusCode, payload) { res.writeHead(statusCode, { 'content-type': 'application/json' }); res.end(JSON.stringify(payload)); @@ -33,49 +75,72 @@ async function readBody(req) { } async function getActivePage() { - if (page && !page.isClosed()) { - return page; - } + try { + if (page && !page.isClosed()) { + return page; + } - if (!context) { - return null; - } + if (!context) { + return null; + } + + const pages = context.pages(); + if (pages.length > 0) { + page = pages[0]; + return page; + } - const pages = context.pages(); - if (pages.length > 0) { - page = pages[0]; + page = await context.newPage(); return page; + } catch (error) { + if (isRecoverableSessionError(error)) { + await clearSessionState(); + return null; + } + throw error; } - - page = await context.newPage(); - return page; } async function describePage() { - const activePage = await getActivePage(); - if (!activePage) { + try { + const activePage = await getActivePage(); + if (!activePage) { + return { + ok: true, + ready: false, + mode, + browserName: 'Google Chrome', + }; + } + + const visibleText = await activePage.evaluate(() => { + const text = document.body ? document.body.innerText || '' : ''; + return text.replace(/\s+/g, ' ').trim().slice(0, 6000); + }); + return { ok: true, - ready: false, + ready: true, mode, browserName: 'Google Chrome', + url: activePage.url(), + title: await activePage.title(), + visibleText, }; + } catch (error) { + if (isRecoverableSessionError(error)) { + await clearSessionState(); + return { + ok: false, + ready: false, + recoverable: true, + error: error instanceof Error ? error.message : String(error), + mode, + browserName: 'Google Chrome', + }; + } + throw error; } - - const visibleText = await activePage.evaluate(() => { - const text = document.body ? document.body.innerText || '' : ''; - return text.replace(/\s+/g, ' ').trim().slice(0, 6000); - }); - - return { - ok: true, - ready: true, - mode, - browserName: 'Google Chrome', - url: activePage.url(), - title: await activePage.title(), - visibleText, - }; } async function tryAttach() { @@ -93,6 +158,7 @@ async function tryAttach() { page = await getActivePage(); return true; } catch (_error) { + await clearSessionState(); return false; } } @@ -114,9 +180,11 @@ async function launchDedicatedProfile() { } async function ensureSession() { - if (context) { - await getActivePage(); - return describePage(); + if (context || browser || page) { + const existingPage = await getActivePage(); + if (existingPage) { + return describePage(); + } } if (!fs.existsSync(chromeExecutable)) { @@ -131,6 +199,11 @@ async function ensureSession() { return describePage(); } +async function recoverSession() { + await closeSessionState(); + return ensureSession(); +} + async function browserOpen(payload) { const targetUrl = payload.url; if (!targetUrl) { @@ -213,6 +286,29 @@ async function browserExtractText(payload) { }; } +async function browserEval(payload) { + const activePage = await getActivePage(); + if (!payload.script) { + throw new Error('browser_eval requires script'); + } + + const evaluation = await activePage.evaluate( + ({ script, arg }) => { + const fn = new Function('arg', script); + return fn(arg ?? null); + }, + { + script: String(payload.script), + arg: payload.arg ?? null, + } + ); + + return { + ...(await describePage()), + evaluation, + }; +} + async function browserAssert(payload) { const snapshot = await describePage(); const url = String(snapshot.url || '').toLowerCase(); @@ -246,6 +342,9 @@ async function dispatch(pathname, payload) { if (pathname === '/browser/attach_or_launch') { return ensureSession(); } + if (pathname === '/browser/reset') { + return recoverSession(); + } await ensureSession(); if (pathname === '/browser/open') { return browserOpen(payload); @@ -265,6 +364,9 @@ async function dispatch(pathname, payload) { if (pathname === '/browser/extract_text') { return browserExtractText(payload); } + if (pathname === '/browser/eval') { + return browserEval(payload); + } if (pathname === '/browser/assert') { return browserAssert(payload); } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..217bbe8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +services: + agent-core: + build: + context: . + dockerfile: Dockerfile + ports: + - "8788:8788" + environment: + JARVIS_AGENT_CORE_BIND: 0.0.0.0:8788 + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8788/health"] + interval: 30s + timeout: 5s + retries: 3 + + scenario-runner: + build: + context: . + dockerfile: Dockerfile + command: ["/usr/local/bin/scenario_runner"] + profiles: ["test"] + + browser-sidecar: + image: node:20-bookworm-slim + working_dir: /app/browser_sidecar + volumes: + - ./browser_sidecar:/app/browser_sidecar + command: ["sh", "-lc", "npm ci && node server.mjs"] + profiles: ["sidecar"] diff --git a/jarvis_rs/src/agent.rs b/jarvis_rs/src/agent.rs index 21abaf1..99d48ff 100644 --- a/jarvis_rs/src/agent.rs +++ b/jarvis_rs/src/agent.rs @@ -4,16 +4,22 @@ use std::sync::Arc; use tokio::runtime::Handle; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use crate::agent_core::{ + evidence_from_observation, verify_plan_completion, ActionSpec, RecoveryDecision, RecoveryKind, + TaskJournalEntry, +}; use crate::automation::{ AppTarget, AutomationBackend, BrowserTarget, RiskLevel, ToolCallRequest, ToolCallResult, }; use crate::config::AppConfig; use crate::llm::{ - CompletionStatus, HeuristicPlannerProvider, HeuristicWorkerProvider, LocalLlamaPlannerProvider, - LocalLlamaWorkerProvider, Observation, PlannerProvider, PlannerStack, TaskPlan, TaskState, - WorkerAction, WorkerProvider, WorkerStack, + CompletionStatus, HeuristicPlannerProvider, HeuristicWorkerProvider, HostedPlannerProvider, + HostedWorkerProvider, LocalLlamaPlannerProvider, LocalLlamaWorkerProvider, Observation, + PlannerProvider, PlannerStack, TaskPlan, TaskState, WorkerAction, WorkerProvider, WorkerStack, +}; +use crate::sms::{ + start_approval_webhook, NoopSmsService, SmsApprovalAction, SmsService, TwilioSmsService, }; -use crate::sms::{start_approval_webhook, NoopSmsService, SmsApprovalAction, SmsService, TwilioSmsService}; use crate::speech::{MacOsSayTextToSpeech, SpeechToText, TextToSpeech, WhisperCommandSpeechToText}; #[derive(Debug, Clone)] @@ -287,7 +293,9 @@ impl AgentRuntime { } AgentCommand::StartListening => { self.emit(AgentEvent::Listening(true)); - self.emit(AgentEvent::Status("Listening for voice input...".to_string())); + self.emit(AgentEvent::Status( + "Listening for voice input...".to_string(), + )); match self .stt .record_and_transcribe(&self.config.recording_path, self.config.capture_seconds) @@ -302,7 +310,10 @@ impl AgentRuntime { } Err(error) => { self.emit(AgentEvent::Listening(false)); - self.emit(AgentEvent::Error(format!("Voice capture failed: {}", error))); + self.emit(AgentEvent::Error(format!( + "Voice capture failed: {}", + error + ))); } } } @@ -374,9 +385,18 @@ impl AgentRuntime { return Ok(()); } - self.notify_sms(&format!("Jarvis started: {}", plan.summary)).await; - - let state = TaskState::new(input, plan); + self.notify_sms(&format!("Jarvis started: {}", plan.summary)) + .await; + + let mut state = TaskState::new(input, plan); + state.journal.push(TaskJournalEntry { + phase: "plan".to_string(), + message: format!("Started task '{}'", state.plan.summary), + action: None, + evidence: None, + verification: None, + recovery: None, + }); self.execute_task_loop(state).await } @@ -408,6 +428,18 @@ impl AgentRuntime { "Next tool: {} ({:?})", request.name, risk ))); + state.journal.push(TaskJournalEntry { + phase: "action".to_string(), + message: format!("Scheduling tool {}", request.name), + action: Some(ActionSpec { + name: request.name.clone(), + expected_outcome: request.expected_outcome.clone(), + target_identity: request.target_identity.clone(), + }), + evidence: None, + verification: None, + recovery: None, + }); if requires_user_approval(&request, risk) { let message = format!( "Approval required before {}. Approve in the app or reply YES by SMS.", @@ -416,13 +448,11 @@ impl AgentRuntime { state.awaiting_approval = true; state.completion_status = CompletionStatus::AwaitingApproval; state.block_current_step(); - self.pending = Some(PendingApproval { - state, - request, - }); + self.pending = Some(PendingApproval { state, request }); self.emit(AgentEvent::ApprovalRequired(message.clone())); self.notify_sms(&message).await; - self.say("I need approval before the next irreversible step.").await; + self.say("I need approval before the next irreversible step.") + .await; return Ok(()); } @@ -440,8 +470,13 @@ impl AgentRuntime { state.advance_step(); if state.current_step().is_none() { let message = format!("Task completed: {}", state.plan.summary); - self.complete_task(&message).await; - return Ok(()); + match self.attempt_completion(state, message).await? { + Some(next_state) => { + state = next_state; + continue; + } + None => return Ok(()), + } } } WorkerAction::Replan { reason } => { @@ -458,6 +493,18 @@ impl AgentRuntime { ) .await?; self.emit(AgentEvent::TaskUpdated(replanned.summary.clone())); + state.journal.push(TaskJournalEntry { + phase: "recovery".to_string(), + message: format!("Replanning because {}", reason), + action: None, + evidence: None, + verification: None, + recovery: Some(RecoveryDecision { + kind: RecoveryKind::Replan, + reason, + next_hint: Some(replanned.summary.clone()), + }), + }); state.reset_for_replan(replanned); } WorkerAction::Complete { message } => { @@ -468,8 +515,10 @@ impl AgentRuntime { )); continue; } - self.complete_task(&message).await; - return Ok(()); + match self.attempt_completion(state, message).await? { + Some(next_state) => state = next_state, + None => return Ok(()), + } } } } @@ -481,15 +530,26 @@ impl AgentRuntime { ) } - async fn execute_tool(&mut self, request: &ToolCallRequest, mut state: TaskState) -> Result { + async fn execute_tool( + &mut self, + request: &ToolCallRequest, + mut state: TaskState, + ) -> Result { self.emit(AgentEvent::ToolLog(format!( "Running tool {}", request.name ))); state.last_expected_outcome = request.expected_outcome.clone(); - let result = self.automation.call_tool(request.clone()).await.with_context(|| { - format!("tool {} failed while executing '{}'", request.name, state.plan.summary) - })?; + let result = self + .automation + .call_tool(request.clone()) + .await + .with_context(|| { + format!( + "tool {} failed while executing '{}'", + request.name, state.plan.summary + ) + })?; self.handle_tool_result(&result, &mut state).await; Ok(state) } @@ -517,16 +577,88 @@ impl AgentRuntime { proof_passed: result.proof_passed, observed_outcome: result.observed_outcome.clone(), }); + if let Some(observation) = state.last_observation.as_ref() { + state.journal.push(TaskJournalEntry { + phase: "observe".to_string(), + message: result.summary.clone(), + action: None, + evidence: Some(evidence_from_observation(observation)), + verification: None, + recovery: None, + }); + } self.emit(AgentEvent::ObservationUpdated(result.summary.clone())); if let Ok(app_target) = serde_json::from_value::(result.observation.clone()) { state.active_app = Some(app_target); } - if let Ok(browser_target) = serde_json::from_value::(result.observation.clone()) { + if let Ok(browser_target) = + serde_json::from_value::(result.observation.clone()) + { state.active_browser = Some(browser_target); } } + async fn attempt_completion( + &mut self, + mut state: TaskState, + message: String, + ) -> Result> { + let verification = verify_plan_completion(&state.plan, state.last_observation.as_ref()); + state.final_verification = Some(verification.clone()); + state.journal.final_verification = Some(verification.clone()); + state.journal.push(TaskJournalEntry { + phase: "verify".to_string(), + message: verification.summary.clone(), + action: None, + evidence: state + .last_observation + .as_ref() + .map(evidence_from_observation), + verification: Some(verification.clone()), + recovery: None, + }); + + if verification.satisfied { + self.complete_task(&message).await; + return Ok(None); + } + + self.emit(AgentEvent::Status(format!( + "Final verifier requested recovery: {}", + verification.summary + ))); + let capabilities = self.automation.capabilities(); + let replanned = self + .planner + .create_plan( + &format!( + "Original request: {}\nFinal verifier summary: {}\nMissing criteria: {:?}\nLast observation: {:?}", + state.user_request, + verification.summary, + verification.missing_criteria, + state.last_observation + ), + &capabilities, + ) + .await?; + state.journal.push(TaskJournalEntry { + phase: "recovery".to_string(), + message: "Final verifier triggered a recovery replan".to_string(), + action: None, + evidence: None, + verification: Some(verification), + recovery: Some(RecoveryDecision { + kind: RecoveryKind::Replan, + reason: "Final verifier could not confirm completion".to_string(), + next_hint: Some(replanned.summary.clone()), + }), + }); + self.emit(AgentEvent::TaskUpdated(replanned.summary.clone())); + state.reset_for_replan(replanned); + Ok(Some(state)) + } + async fn complete_task(&self, message: &str) { self.emit(AgentEvent::Completed(message.to_string())); self.notify_sms(message).await; @@ -556,23 +688,45 @@ impl AgentRuntime { } fn build_planner_stack(config: &AppConfig) -> Arc { - let primary = Arc::new(HeuristicPlannerProvider) as Arc; - let fallback = Arc::new(LocalLlamaPlannerProvider::new( + let mut providers = Vec::>::new(); + if let (Some(base_url), Some(model)) = ( + config.provider.fallback_base_url.clone(), + config.provider.fallback_model.clone(), + ) { + providers.push(Arc::new(HostedPlannerProvider::new( + base_url, + model, + config.provider.fallback_api_key.clone(), + ))); + } + providers.push(Arc::new(LocalLlamaPlannerProvider::new( config.provider.planner_endpoint.clone(), config.provider.planner_model.clone(), config.provider.fallback_api_key.clone(), - )) as Arc; - Arc::new(PlannerStack::new(primary, fallback)) + ))); + providers.push(Arc::new(HeuristicPlannerProvider)); + Arc::new(PlannerStack::new(providers)) } fn build_worker_stack(config: &AppConfig) -> Arc { - let primary = Arc::new(HeuristicWorkerProvider) as Arc; - let fallback = Arc::new(LocalLlamaWorkerProvider::new( + let mut providers = Vec::>::new(); + if let (Some(base_url), Some(model)) = ( + config.provider.fallback_base_url.clone(), + config.provider.fallback_model.clone(), + ) { + providers.push(Arc::new(HostedWorkerProvider::new( + base_url, + model, + config.provider.fallback_api_key.clone(), + ))); + } + providers.push(Arc::new(LocalLlamaWorkerProvider::new( config.provider.worker_endpoint.clone(), config.provider.worker_model.clone(), config.provider.fallback_api_key.clone(), - )) as Arc; - Arc::new(WorkerStack::new(primary, fallback)) + ))); + providers.push(Arc::new(HeuristicWorkerProvider)); + Arc::new(WorkerStack::new(providers)) } fn build_tts(config: &AppConfig) -> Arc { @@ -640,10 +794,8 @@ fn requires_user_approval(request: &ToolCallRequest, risk: RiskLevel) -> bool { return true; } - matches!( - request.name.as_str(), - "filesystem_delete" | "shell_run" - ) || matches!(risk, RiskLevel::Critical) + matches!(request.name.as_str(), "filesystem_delete" | "shell_run") + || matches!(risk, RiskLevel::Critical) } fn step_proved(state: &TaskState) -> bool { diff --git a/jarvis_rs/src/agent_core.rs b/jarvis_rs/src/agent_core.rs new file mode 100644 index 0000000..b332b0f --- /dev/null +++ b/jarvis_rs/src/agent_core.rs @@ -0,0 +1,266 @@ +use serde::{Deserialize, Serialize}; + +use crate::automation::AutomationCapabilities; +use crate::config::AppConfig; +use crate::llm::schema::{Observation, TaskPlan}; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ActionSpec { + pub name: String, + #[serde(default)] + pub expected_outcome: Option, + #[serde(default)] + pub target_identity: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ObservationEvidence { + pub source: String, + pub summary: String, + #[serde(default)] + pub details: Option, + #[serde(default)] + pub target_identity: Option, + #[serde(default)] + pub observed_outcome: Option, + #[serde(default)] + pub proof_passed: bool, + #[serde(default)] + pub success: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct VerificationResult { + pub satisfied: bool, + pub confidence: f32, + pub summary: String, + #[serde(default)] + pub matched_criteria: Vec, + #[serde(default)] + pub missing_criteria: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecoveryKind { + RetryStep, + RefocusApp, + ReacquireTarget, + Replan, + Abort, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecoveryDecision { + pub kind: RecoveryKind, + pub reason: String, + #[serde(default)] + pub next_hint: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct TaskJournalEntry { + pub phase: String, + pub message: String, + #[serde(default)] + pub action: Option, + #[serde(default)] + pub evidence: Option, + #[serde(default)] + pub verification: Option, + #[serde(default)] + pub recovery: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct TaskJournal { + #[serde(default)] + pub entries: Vec, + #[serde(default)] + pub final_verification: Option, +} + +impl TaskJournal { + pub fn push(&mut self, entry: TaskJournalEntry) { + self.entries.push(entry); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct PermissionStatus { + pub accessibility: bool, + pub automation: bool, + pub microphone: bool, + pub browser_sidecar: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RuntimeManifest { + pub platform: String, + pub primary_browser: String, + pub planner_provider: String, + pub worker_provider: String, + #[serde(default)] + pub fallback_model: Option, + #[serde(default)] + pub allowed_paths: Vec, + pub permissions: PermissionStatus, + #[serde(default)] + pub setup_items: Vec, + #[serde(default)] + pub diagnostics: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScenarioSpec { + pub id: String, + pub summary: String, + #[serde(default)] + pub success_criteria: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScenarioOutcome { + pub id: String, + pub success: bool, + pub summary: String, + #[serde(default)] + pub verification: Option, + #[serde(default)] + pub artifact_paths: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RunReport { + pub runtime: RuntimeManifest, + #[serde(default)] + pub scenarios: Vec, +} + +pub fn build_runtime_manifest( + config: &AppConfig, + capabilities: &AutomationCapabilities, + diagnostics: &[String], + planner_provider: &str, + worker_provider: &str, +) -> RuntimeManifest { + let accessibility_ready = !capabilities.setup_items.iter().any(|item| { + item.to_ascii_lowercase() + .contains("accessibility permission is blocked") + }); + let browser_sidecar_ready = capabilities.browser_automation_ready + && !capabilities.setup_items.iter().any(|item| { + item.to_ascii_lowercase() + .contains("browser sidecar is not running yet") + }); + RuntimeManifest { + platform: std::env::consts::OS.to_string(), + primary_browser: config.primary_browser.clone(), + planner_provider: planner_provider.to_string(), + worker_provider: worker_provider.to_string(), + fallback_model: config.provider.fallback_model.clone(), + allowed_paths: config + .allowed_paths + .iter() + .map(|path| path.display().to_string()) + .collect(), + permissions: PermissionStatus { + accessibility: accessibility_ready, + automation: capabilities.applescript_available && accessibility_ready, + microphone: true, + browser_sidecar: browser_sidecar_ready, + }, + setup_items: capabilities.setup_items.clone(), + diagnostics: diagnostics.to_vec(), + } +} + +pub fn evidence_from_observation(observation: &Observation) -> ObservationEvidence { + ObservationEvidence { + source: observation.source.clone(), + summary: observation.summary.clone(), + details: observation.details.clone(), + target_identity: observation.target_identity.clone(), + observed_outcome: observation.observed_outcome.clone(), + proof_passed: observation.proof_passed, + success: observation.success, + } +} + +pub fn verify_plan_completion( + plan: &TaskPlan, + observation: Option<&Observation>, +) -> VerificationResult { + let Some(observation) = observation else { + return VerificationResult { + satisfied: false, + confidence: 0.0, + summary: "No observation was available to verify the final task state".to_string(), + matched_criteria: Vec::new(), + missing_criteria: plan.success_criteria.clone(), + }; + }; + + let evidence = [ + observation.summary.as_str(), + observation.details.as_deref().unwrap_or_default(), + observation.observed_outcome.as_deref().unwrap_or_default(), + observation.target_identity.as_deref().unwrap_or_default(), + ] + .join(" ") + .to_ascii_lowercase(); + + let mut matched = Vec::new(); + let mut missing = Vec::new(); + for criterion in &plan.success_criteria { + let tokens = criterion + .split_whitespace() + .filter(|token| token.len() > 3) + .map(|token| token.to_ascii_lowercase()) + .collect::>(); + if tokens.is_empty() || tokens.iter().any(|token| evidence.contains(token)) { + matched.push(criterion.clone()); + } else { + missing.push(criterion.clone()); + } + } + + let satisfied = observation.success && observation.proof_passed && missing.is_empty(); + let confidence = if satisfied { + 0.95 + } else if observation.proof_passed { + 0.55 + } else { + 0.15 + }; + VerificationResult { + satisfied, + confidence, + summary: if satisfied { + "Final verifier confirmed the requested task outcome".to_string() + } else { + "Final verifier could not confirm every success criterion".to_string() + }, + matched_criteria: matched, + missing_criteria: missing, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn verifier_requires_observation() { + let plan = TaskPlan { + assistant_message: String::new(), + summary: "test".to_string(), + goal: "test".to_string(), + success_criteria: vec!["Finder is frontmost".to_string()], + steps: Vec::new(), + }; + let result = verify_plan_completion(&plan, None); + assert!(!result.satisfied); + assert_eq!(result.missing_criteria.len(), 1); + } +} diff --git a/jarvis_rs/src/app.rs b/jarvis_rs/src/app.rs index e9dfe5b..fd7c5fe 100644 --- a/jarvis_rs/src/app.rs +++ b/jarvis_rs/src/app.rs @@ -195,7 +195,8 @@ impl eframe::App for JarvisApp { .desired_width(f32::INFINITY) .hint_text("Tell Jarvis what to do"), ); - if response.lost_focus() && ui.input(|input| input.key_pressed(egui::Key::Enter)) + if response.lost_focus() + && ui.input(|input| input.key_pressed(egui::Key::Enter)) { self.submit_text(); } @@ -209,7 +210,11 @@ impl eframe::App for JarvisApp { let _ = self.commands.send(AgentCommand::StartListening); } if ui - .button(if self.speech_muted { "Unmute Speech" } else { "Mute Speech" }) + .button(if self.speech_muted { + "Unmute Speech" + } else { + "Mute Speech" + }) .clicked() { self.speech_muted = !self.speech_muted; diff --git a/jarvis_rs/src/automation.rs b/jarvis_rs/src/automation.rs index 407cd7c..22f2943 100644 --- a/jarvis_rs/src/automation.rs +++ b/jarvis_rs/src/automation.rs @@ -9,7 +9,9 @@ use std::process::Command; use std::thread; use std::time::Duration; -use crate::browser_sidecar::{browser_target_from_value, BrowserSidecarDiagnostics, BrowserSidecarManager}; +use crate::browser_sidecar::{ + browser_target_from_value, BrowserSidecarDiagnostics, BrowserSidecarManager, +}; use crate::config::AppConfig; #[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] @@ -211,7 +213,38 @@ impl MacAutomationBackend { } fn is_path_allowed(&self, path: &Path) -> bool { - self.allowed_paths.iter().any(|allowed| path.starts_with(allowed)) + self.allowed_paths + .iter() + .any(|allowed| path.starts_with(allowed)) + } + + fn resolve_path(&self, raw: &str) -> Result { + let path = PathBuf::from(raw); + if path.is_absolute() { + Ok(path) + } else { + Ok(std::env::current_dir() + .context("failed to resolve current working directory")? + .join(path)) + } + } + + fn ensure_path_allowed(&self, raw: &str) -> Result { + let path = self.resolve_path(raw)?; + if self.is_path_allowed(&path) { + Ok(path) + } else { + bail!( + "path {} is outside the allowed automation roots", + path.display() + ) + } + } + + fn ensure_move_paths_allowed(&self, from: &str, to: &str) -> Result<(PathBuf, PathBuf)> { + let from_path = self.ensure_path_allowed(from)?; + let to_path = self.ensure_path_allowed(to)?; + Ok((from_path, to_path)) } fn summary( @@ -241,14 +274,28 @@ impl MacAutomationBackend { bail!("app name cannot be empty"); } - let normalized = normalize_name(requested); - if let Some(bundle_id) = self.app_aliases.get(&normalized) { - return Ok(AppTarget { - requested_name: requested.to_string(), - display_name: canonical_display_name(bundle_id), - bundle_id: Some(bundle_id.clone()), - path: None, - }); + let candidates = app_name_candidates(requested); + for candidate in &candidates { + if let Some(bundle_id) = self.app_aliases.get(candidate) { + return Ok(AppTarget { + requested_name: requested.to_string(), + display_name: canonical_display_name(bundle_id), + bundle_id: Some(bundle_id.clone()), + path: None, + }); + } + } + + #[cfg(target_os = "macos")] + for running_name in self.running_app_names()? { + if app_name_matches(&candidates, &running_name) { + return Ok(AppTarget { + requested_name: requested.to_string(), + display_name: running_name, + bundle_id: None, + path: None, + }); + } } for directory in app_search_paths() { @@ -271,7 +318,7 @@ impl MacAutomationBackend { .and_then(|value| value.to_str()) .unwrap_or_default() .to_string(); - if normalize_name(&display_name).contains(&normalized) { + if app_name_matches(&candidates, &display_name) { return Ok(AppTarget { requested_name: requested.to_string(), display_name, @@ -286,8 +333,38 @@ impl MacAutomationBackend { bail!("could not resolve application '{}'", requested); } + #[cfg(target_os = "macos")] + fn running_app_names(&self) -> Result> { + let output = self.run_applescript_lines(&vec![ + r#"tell application "System Events""#.to_string(), + r#"get the name of every application process whose background only is false"# + .to_string(), + "end tell".to_string(), + ])?; + Ok(output + .split(',') + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_string) + .collect()) + } + #[cfg(target_os = "macos")] fn activate_app_target(&self, target: &AppTarget) -> Result<()> { + let _ = self.run_applescript_lines(&vec![format!( + r#"tell application "{}" to activate"#, + escape_applescript(&target.display_name) + )]); + + for _ in 0..5 { + thread::sleep(Duration::from_millis(250)); + if let Ok(frontmost) = self.frontmost_app_name() { + if frontmost == target.display_name { + return Ok(()); + } + } + } + let mut command = Command::new("open"); if let Some(bundle_id) = &target.bundle_id { command.arg("-b").arg(bundle_id); @@ -303,28 +380,6 @@ impl MacAutomationBackend { bail!("macOS failed to launch {}", target.display_name); } - for _ in 0..5 { - thread::sleep(Duration::from_millis(250)); - if let Ok(frontmost) = self.frontmost_app_name() { - if frontmost == target.display_name { - return Ok(()); - } - } - } - - if let Some(bundle_id) = &target.bundle_id { - let _ = self.run_applescript_lines(&vec![ - format!(r#"tell application id "{}" to activate"#, escape_applescript(bundle_id)), - ]); - } else { - let _ = self.run_applescript_lines(&vec![ - format!( - r#"tell application "{}" to activate"#, - escape_applescript(&target.display_name) - ), - ]); - } - for _ in 0..8 { thread::sleep(Duration::from_millis(250)); if let Ok(frontmost) = self.frontmost_app_name() { @@ -352,23 +407,11 @@ impl MacAutomationBackend { #[cfg(target_os = "macos")] fn frontmost_app_name(&self) -> Result { - let output = Command::new(Self::shell_program()) - .args(Self::shell_args("lsappinfo info -only name `lsappinfo front`")) - .output() - .context("failed to run lsappinfo")?; - if !output.status.success() { - bail!( - "failed to query frontmost app: {}", - String::from_utf8_lossy(&output.stderr).trim() - ); - } - let stdout = String::from_utf8_lossy(&output.stdout); - let name = stdout - .split('"') - .nth(3) - .map(str::to_string) - .or_else(|| stdout.split('=').nth(1).map(|value| value.trim_matches('"').trim().to_string())) - .unwrap_or_else(|| stdout.trim().to_string()); + let name = self.run_applescript_lines(&vec![ + r#"tell application "System Events""#.to_string(), + r#"return name of first application process whose frontmost is true"#.to_string(), + "end tell".to_string(), + ])?; if name.is_empty() { bail!("frontmost app query returned an empty name"); } @@ -406,7 +449,8 @@ impl MacAutomationBackend { let lines = vec![ r#"tell application "Google Chrome""#.to_string(), "activate".to_string(), - r#"if (count of windows) = 0 then error "Google Chrome has no open windows""#.to_string(), + r#"if (count of windows) = 0 then error "Google Chrome has no open windows""# + .to_string(), format!( r#"set resultText to execute active tab of front window javascript "{}""#, escape_applescript(script) @@ -464,7 +508,10 @@ impl MacAutomationBackend { let observation = self.browser_sidecar.call(route, request_arguments).await?; let mut result = ToolCallResult { name: tool_name.to_string(), - success: observation.get("ok").and_then(Value::as_bool).unwrap_or(true), + success: observation + .get("ok") + .and_then(Value::as_bool) + .unwrap_or(true), summary: summary_template.to_string(), output: Some(observation.to_string()), artifact_path: None, @@ -478,12 +525,17 @@ impl MacAutomationBackend { proof_passed: observation .get("matched") .and_then(Value::as_bool) - .unwrap_or(true), + .unwrap_or(false), observed_outcome: observation .get("title") .and_then(Value::as_str) .map(str::to_string) - .or_else(|| observation.get("url").and_then(Value::as_str).map(str::to_string)), + .or_else(|| { + observation + .get("url") + .and_then(Value::as_str) + .map(str::to_string) + }), }; if let Some(extracted) = observation.get("extractedText").and_then(Value::as_str) { @@ -680,6 +732,7 @@ impl AutomationBackend for MacAutomationBackend { "window_title": window_title, }), ); + result.proof_passed = false; result.observed_outcome = Some(app_name.clone()); Ok(result) } @@ -699,12 +752,14 @@ impl AutomationBackend for MacAutomationBackend { "end tell".to_string(), ]; self.run_applescript_lines(&script)?; - Ok(Self::summary( + let mut result = Self::summary( "ui_click", format!("Clicked {}", args.label), None, json!({"label": args.label}), - )) + ); + result.proof_passed = false; + Ok(result) } "ui_type" => { let args: UiTypeArgs = serde_json::from_value(request.arguments)?; @@ -713,12 +768,14 @@ impl AutomationBackend for MacAutomationBackend { format!(r#"keystroke "{}""#, escape_applescript(&args.text)), "end tell".to_string(), ])?; - Ok(Self::summary( + let mut result = Self::summary( "ui_type", "Typed text".to_string(), None, json!({"text_len": args.text.chars().count()}), - )) + ); + result.proof_passed = false; + Ok(result) } "ui_press_key" => { let args: UiPressKeyArgs = serde_json::from_value(request.arguments)?; @@ -728,12 +785,14 @@ impl AutomationBackend for MacAutomationBackend { key_command, "end tell".to_string(), ])?; - Ok(Self::summary( + let mut result = Self::summary( "ui_press_key", format!("Pressed {}", args.key), None, json!({"key": args.key, "modifiers": args.modifiers}), - )) + ); + result.proof_passed = false; + Ok(result) } "ui_select_menu" => { let args: UiSelectMenuArgs = serde_json::from_value(request.arguments)?; @@ -748,26 +807,33 @@ impl AutomationBackend for MacAutomationBackend { "end tell".to_string(), "end tell".to_string(), ])?; - Ok(Self::summary( + let mut result = Self::summary( "ui_select_menu", format!("Selected {} > {}", args.menu, args.item), None, json!({"menu": args.menu, "item": args.item}), - )) + ); + result.proof_passed = false; + Ok(result) } "ui_scroll" => { let args: UiScrollArgs = serde_json::from_value(request.arguments)?; - self.run_applescript_lines(&vec![ - r#"tell application "System Events""#.to_string(), - format!("key code 125 {}", if args.amount < 0 { "" } else { "" }), - "end tell".to_string(), - ])?; - Ok(Self::summary( + let steps = args.amount.abs().max(1).min(40); + let key_code = if args.amount < 0 { 126 } else { 125 }; + let mut script = vec![r#"tell application "System Events""#.to_string()]; + for _ in 0..steps { + script.push(format!("key code {}", key_code)); + } + script.push("end tell".to_string()); + self.run_applescript_lines(&script)?; + let mut result = Self::summary( "ui_scroll", format!("Requested scroll {}", args.amount), None, json!({"amount": args.amount}), - )) + ); + result.proof_passed = false; + Ok(result) } "browser_attach_or_launch" => self.call_browser_tool( "browser_attach_or_launch", @@ -848,28 +914,34 @@ impl AutomationBackend for MacAutomationBackend { ).await, "chrome_screenshot" | "take_screenshot" => { let args: ScreenshotArgs = serde_json::from_value(request.arguments)?; + let target_path = self.ensure_path_allowed(&args.path)?; #[cfg(target_os = "macos")] - let status = Command::new("screencapture").arg("-x").arg(&args.path).status(); + let status = Command::new("screencapture") + .arg("-x") + .arg(&target_path) + .status(); #[cfg(not(target_os = "macos"))] let status = Command::new(Self::shell_program()) .args(Self::shell_args(&format!( "which gnome-screenshot >/dev/null && gnome-screenshot -f '{}'", - args.path + target_path.display() ))) .status(); - status.with_context(|| format!("failed to capture screenshot to {}", args.path))?; + status.with_context(|| { + format!("failed to capture screenshot to {}", target_path.display()) + })?; Ok(ToolCallResult { name: request.name, success: true, - summary: format!("Captured screenshot to {}", args.path), + summary: format!("Captured screenshot to {}", target_path.display()), output: None, - artifact_path: Some(PathBuf::from(&args.path)), - observation: json!({ "path": args.path }), + artifact_path: Some(target_path.clone()), + observation: json!({ "path": target_path.display().to_string() }), retryable: true, target_identity: None, error_code: None, proof_passed: true, - observed_outcome: Some(args.path), + observed_outcome: Some(target_path.display().to_string()), }) } "mail_compose" => { @@ -884,8 +956,8 @@ impl AutomationBackend for MacAutomationBackend { ) }) .unwrap_or_default(); - let output = self.run_applescript_lines(&vec![ - r#"tell application id "com.apple.mail""#.to_string(), + self.run_applescript_lines(&vec![ + r#"tell application "Mail""#.to_string(), "activate".to_string(), format!( r#"set newMessage to make new outgoing message with properties {{visible:true, subject:"{}", content:"{}"}}"#, @@ -894,18 +966,47 @@ impl AutomationBackend for MacAutomationBackend { ), "tell newMessage".to_string(), recipient_line, - "delay 0.2".to_string(), - "set verifiedSubject to subject of newMessage".to_string(), - "set verifiedContent to content of newMessage".to_string(), - "set verifiedRecipients to address of every to recipient of newMessage".to_string(), - r#"return verifiedSubject & linefeed & (verifiedRecipients as string) & linefeed & verifiedContent"#.to_string(), "end tell".to_string(), "end tell".to_string(), ])?; - let mut lines = output.lines(); - let verified_subject = lines.next().unwrap_or_default().to_string(); - let verified_recipients = lines.next().unwrap_or_default().to_string(); - let verified_content = lines.collect::>().join("\n"); + let verification_script = vec![ + r#"tell application "Mail""#.to_string(), + format!( + r#"set matchingMessages to every outgoing message whose subject is "{}""#, + escape_applescript(&args.subject) + ), + r#"if (count of matchingMessages) = 0 then error "No matching Mail draft was found yet""# + .to_string(), + "set verifiedMessage to item (count of matchingMessages) of matchingMessages" + .to_string(), + "set verifiedSubject to subject of verifiedMessage".to_string(), + "set verifiedContent to content of verifiedMessage".to_string(), + "set verifiedRecipients to address of every to recipient of verifiedMessage" + .to_string(), + r#"return verifiedSubject & linefeed & (verifiedRecipients as string) & linefeed & verifiedContent"# + .to_string(), + "end tell".to_string(), + ]; + + let mut verification_error = None; + let mut verified_subject = String::new(); + let mut verified_recipients = String::new(); + let mut verified_content = String::new(); + for _ in 0..8 { + thread::sleep(Duration::from_millis(250)); + match self.run_applescript_lines(&verification_script) { + Ok(output) => { + let mut lines = output.lines(); + verified_subject = lines.next().unwrap_or_default().to_string(); + verified_recipients = lines.next().unwrap_or_default().to_string(); + verified_content = lines.collect::>().join("\n"); + verification_error = None; + break; + } + Err(error) => verification_error = Some(error.to_string()), + } + } + let proof_passed = verified_subject == args.subject && verified_content.contains(&args.body) && args @@ -920,12 +1021,17 @@ impl AutomationBackend for MacAutomationBackend { } else { "Prepared Mail draft (verification incomplete)" }, - Some(verified_content.clone()), + Some(if verified_content.is_empty() { + args.body.clone() + } else { + verified_content.clone() + }), json!({ "to": args.to, "subject": args.subject, "body": args.body, "verified_recipients": verified_recipients, + "verification_error": verification_error, }), ); result.target_identity = Some("com.apple.mail".to_string()); @@ -946,7 +1052,7 @@ impl AutomationBackend for MacAutomationBackend { thread::sleep(Duration::from_millis(800)); let frontmost = self.frontmost_app_name().unwrap_or_default(); let output = self.run_applescript_lines(&vec![ - r#"tell application id "com.apple.MobileSMS""#.to_string(), + r#"tell application "Messages""#.to_string(), "set windowCount to count of windows".to_string(), "set frontWindowName to name of front window".to_string(), "return (windowCount as string) & linefeed & frontWindowName".to_string(), @@ -982,77 +1088,90 @@ impl AutomationBackend for MacAutomationBackend { } "filesystem_list" => { let args: PathArgs = serde_json::from_value(request.arguments)?; - let entries = fs::read_dir(&args.path) - .with_context(|| format!("failed to list {}", args.path))? + let path = self.ensure_path_allowed(&args.path)?; + let entries = fs::read_dir(&path) + .with_context(|| format!("failed to list {}", path.display()))? .filter_map(|entry| entry.ok()) .map(|entry| entry.file_name().to_string_lossy().to_string()) .collect::>(); Ok(Self::summary( "filesystem_list", - format!("Listed {}", args.path), + format!("Listed {}", path.display()), Some(entries.join("\n")), - json!({ "path": args.path, "entries": entries }), + json!({ "path": path.display().to_string(), "entries": entries }), )) } "filesystem_create_folder" => { let args: PathArgs = serde_json::from_value(request.arguments)?; - fs::create_dir_all(&args.path) - .with_context(|| format!("failed to create {}", args.path))?; + let path = self.ensure_path_allowed(&args.path)?; + fs::create_dir_all(&path) + .with_context(|| format!("failed to create {}", path.display()))?; Ok(Self::summary( "filesystem_create_folder", - format!("Created folder {}", args.path), + format!("Created folder {}", path.display()), None, - json!({ "path": args.path }), + json!({ "path": path.display().to_string() }), )) } "filesystem_read_file" => { let args: PathArgs = serde_json::from_value(request.arguments)?; - let contents = - fs::read_to_string(&args.path).with_context(|| format!("failed to read {}", args.path))?; + let path = self.ensure_path_allowed(&args.path)?; + let contents = fs::read_to_string(&path) + .with_context(|| format!("failed to read {}", path.display()))?; Ok(Self::summary( "filesystem_read_file", - format!("Read {}", args.path), + format!("Read {}", path.display()), Some(contents.clone()), - json!({ "path": args.path, "size": contents.len() }), + json!({ "path": path.display().to_string(), "size": contents.len() }), )) } "filesystem_write_file" => { let args: WriteFileArgs = serde_json::from_value(request.arguments)?; - fs::write(&args.path, &args.contents) - .with_context(|| format!("failed to write {}", args.path))?; + let path = self.ensure_path_allowed(&args.path)?; + fs::write(&path, &args.contents) + .with_context(|| format!("failed to write {}", path.display()))?; Ok(Self::summary( "filesystem_write_file", - format!("Wrote {}", args.path), + format!("Wrote {}", path.display()), None, - json!({ "path": args.path, "size": args.contents.len() }), + json!({ "path": path.display().to_string(), "size": args.contents.len() }), )) } "filesystem_move" => { let args: MovePathArgs = serde_json::from_value(request.arguments)?; - fs::rename(&args.from, &args.to) - .with_context(|| format!("failed to move {} to {}", args.from, args.to))?; + let (from_path, to_path) = self.ensure_move_paths_allowed(&args.from, &args.to)?; + fs::rename(&from_path, &to_path).with_context(|| { + format!( + "failed to move {} to {}", + from_path.display(), + to_path.display() + ) + })?; Ok(Self::summary( "filesystem_move", - format!("Moved {} to {}", args.from, args.to), + format!("Moved {} to {}", from_path.display(), to_path.display()), None, - json!({ "from": args.from, "to": args.to }), + json!({ + "from": from_path.display().to_string(), + "to": to_path.display().to_string() + }), )) } "filesystem_delete" => { let args: PathArgs = serde_json::from_value(request.arguments)?; - let path = PathBuf::from(&args.path); + let path = self.ensure_path_allowed(&args.path)?; if path.is_dir() { fs::remove_dir_all(&path) - .with_context(|| format!("failed to remove directory {}", args.path))?; + .with_context(|| format!("failed to remove directory {}", path.display()))?; } else { fs::remove_file(&path) - .with_context(|| format!("failed to remove file {}", args.path))?; + .with_context(|| format!("failed to remove file {}", path.display()))?; } Ok(Self::summary( "filesystem_delete", - format!("Deleted {}", args.path), + format!("Deleted {}", path.display()), None, - json!({ "path": args.path }), + json!({ "path": path.display().to_string() }), )) } "shell_run" => { @@ -1093,7 +1212,13 @@ impl AutomationBackend for MacAutomationBackend { .get("path") .and_then(Value::as_str) .map(PathBuf::from) - .or_else(|| request.arguments.get("to").and_then(Value::as_str).map(PathBuf::from)); + .or_else(|| { + request + .arguments + .get("to") + .and_then(Value::as_str) + .map(PathBuf::from) + }); match path { Some(path) if self.is_path_allowed(&path) => RiskLevel::Medium, Some(_) => RiskLevel::High, @@ -1115,8 +1240,12 @@ impl AutomationBackend for MacAutomationBackend { RiskLevel::High } } - "ui_click" | "ui_type" | "ui_press_key" | "ui_select_menu" | "ui_scroll" => RiskLevel::High, - "chrome_eval" | "chrome_click" | "chrome_type" | "browser_click" | "browser_fill" => RiskLevel::Medium, + "ui_click" | "ui_type" | "ui_press_key" | "ui_select_menu" | "ui_scroll" => { + RiskLevel::High + } + "chrome_eval" | "chrome_click" | "chrome_type" | "browser_click" | "browser_fill" => { + RiskLevel::Medium + } "messages_compose" | "mail_compose" => RiskLevel::Medium, _ => request.risk, } @@ -1149,7 +1278,10 @@ impl AutomationBackend for MacAutomationBackend { } fn app_search_paths() -> Vec { - let mut paths = vec![PathBuf::from("/Applications"), PathBuf::from("/System/Applications")]; + let mut paths = vec![ + PathBuf::from("/Applications"), + PathBuf::from("/System/Applications"), + ]; if let Some(home) = std::env::var_os("HOME").map(PathBuf::from) { paths.push(home.join("Applications")); } @@ -1181,6 +1313,47 @@ fn normalize_name(value: &str) -> String { .join(" ") } +fn app_name_candidates(value: &str) -> Vec { + let normalized = normalize_name(value); + if normalized.is_empty() { + return Vec::new(); + } + + let mut candidates = vec![normalized.clone()]; + let tokens: Vec<&str> = normalized.split_whitespace().collect(); + let significant_tokens: Vec<&str> = tokens + .iter() + .copied() + .filter(|token| !is_ignorable_app_token(token)) + .collect(); + let significant = significant_tokens.join(" "); + if !significant.is_empty() && significant != normalized { + candidates.push(significant); + } + + candidates +} + +fn app_name_matches(candidates: &[String], display_name: &str) -> bool { + let normalized_display_name = normalize_name(display_name); + candidates.iter().any(|candidate| { + normalized_display_name.contains(candidate) + || candidate.contains(&normalized_display_name) + || normalized_display_name.split_whitespace().any(|token| { + candidate + .split_whitespace() + .any(|candidate_token| candidate_token == token) + }) + }) +} + +fn is_ignorable_app_token(token: &str) -> bool { + matches!( + token, + "a" | "an" | "the" | "app" | "application" | "local" | "desktop" + ) +} + fn canonical_display_name(bundle_id: &str) -> String { match bundle_id { "com.google.Chrome" => "Google Chrome", @@ -1239,7 +1412,9 @@ fn applescript_key_command(key: &str, modifiers: &[String]) -> Result { "down" => format!("key code 125{}", using), "left" => format!("key code 123{}", using), "right" => format!("key code 124{}", using), - value if value.len() == 1 => format!(r#"keystroke "{}"{}"#, escape_applescript(value), using), + value if value.len() == 1 => { + format!(r#"keystroke "{}"{}"#, escape_applescript(value), using) + } _ => bail!("unsupported key '{}'", key), }; @@ -1283,6 +1458,21 @@ mod tests { assert_ne!(target.display_name, "Finder"); } + #[test] + fn resolves_voice_friendly_alias_with_filler_words() { + let backend = MacAutomationBackend::new(vec![std::env::temp_dir()]); + let target = backend.resolve_app_target("local Chrome").unwrap(); + + assert_eq!(target.bundle_id.as_deref(), Some("com.google.Chrome")); + assert_eq!(target.display_name, "Google Chrome"); + } + + #[test] + fn bidirectional_app_name_matching_handles_longer_display_names() { + let candidates = app_name_candidates("Arduino IDE"); + assert!(app_name_matches(&candidates, "Arduino Uno IDE")); + } + #[test] fn unknown_app_returns_error() { let backend = MacAutomationBackend::new(vec![std::env::temp_dir()]); @@ -1313,4 +1503,27 @@ mod tests { assert!(result.output.unwrap_or_default().contains("demo.txt")); } + + #[tokio::test] + async fn filesystem_write_rejects_paths_outside_allowlist() { + let backend = MacAutomationBackend::new(vec![std::env::temp_dir().join("allowed-only")]); + let denied_path = std::env::temp_dir() + .join("outside-allowed") + .join("blocked.txt"); + let result = backend + .call_tool(ToolCallRequest { + name: "filesystem_write_file".to_string(), + arguments: json!({ + "path": denied_path.display().to_string(), + "contents": "blocked", + }), + risk: RiskLevel::Low, + requires_confirmation: false, + target_identity: None, + expected_outcome: None, + }) + .await; + + assert!(result.is_err()); + } } diff --git a/jarvis_rs/src/bin/agent_core_server.rs b/jarvis_rs/src/bin/agent_core_server.rs new file mode 100644 index 0000000..b75cc05 --- /dev/null +++ b/jarvis_rs/src/bin/agent_core_server.rs @@ -0,0 +1,53 @@ +use axum::{routing::get, Json, Router}; +use serde_json::json; +use std::sync::Arc; + +use jarvis_rs::agent_core::build_runtime_manifest; +use jarvis_rs::automation::{AutomationBackend, LocalAutomationBackend}; +use jarvis_rs::config::AppConfig; +use jarvis_rs::logging; + +#[derive(Clone)] +struct AppState { + manifest: Arc, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + logging::init(); + + let config = AppConfig::from_env(); + let automation = LocalAutomationBackend::from_config(&config); + let capabilities = automation.capabilities(); + let runtime_manifest = build_runtime_manifest( + &config, + &capabilities, + &capabilities.setup_items, + "runtime-server", + "runtime-server", + ); + let bind = + std::env::var("JARVIS_AGENT_CORE_BIND").unwrap_or_else(|_| "127.0.0.1:8788".to_string()); + let app = Router::new() + .route("/health", get(health)) + .route("/manifest", get(manifest_route)) + .with_state(AppState { + manifest: Arc::new(runtime_manifest), + }); + let listener = tokio::net::TcpListener::bind(&bind).await?; + axum::serve(listener, app).await?; + Ok(()) +} + +async fn health() -> Json { + Json(json!({ + "ok": true, + "service": "jarvis-agent-core" + })) +} + +async fn manifest_route( + axum::extract::State(state): axum::extract::State, +) -> Json { + Json((*state.manifest).clone()) +} diff --git a/jarvis_rs/src/bin/desktop_demo.rs b/jarvis_rs/src/bin/desktop_demo.rs new file mode 100644 index 0000000..59a4e44 --- /dev/null +++ b/jarvis_rs/src/bin/desktop_demo.rs @@ -0,0 +1,457 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{anyhow, bail, Context, Result}; +use serde::Serialize; +use serde_json::{json, Value}; +use tokio::time::{sleep, Duration}; + +use jarvis_rs::automation::{ + AutomationBackend, LocalAutomationBackend, ToolCallRequest, ToolCallResult, +}; +use jarvis_rs::browser_sidecar::BrowserSidecarManager; +use jarvis_rs::config::AppConfig; +use jarvis_rs::logging; + +const GMAIL_INSPECTION_SCRIPT: &str = r#" +const expected = arg || {}; +const normalize = value => String(value || '').replace(/\s+/g, ' ').trim(); +const lower = value => normalize(value).toLowerCase(); +const pageText = normalize(document.body ? document.body.innerText || '' : ''); +const subjectInput = document.querySelector('input[name="subjectbox"]'); +const subjectValue = normalize(subjectInput && 'value' in subjectInput ? subjectInput.value : ''); +const bodyCandidates = Array.from(document.querySelectorAll('div[role="textbox"], div[aria-label*="Message Body"], textarea')) + .map(node => normalize('value' in node ? node.value : node.innerText || node.textContent || '')) + .filter(Boolean); +const bodyValue = bodyCandidates.find(Boolean) || ''; +const recipientCandidates = Array.from(document.querySelectorAll('[email], [data-hovercard-id], input[aria-label*="To"], textarea')) + .map(node => normalize(node.getAttribute && (node.getAttribute('email') || node.getAttribute('data-hovercard-id')) || ('value' in node ? node.value : node.innerText || node.textContent || ''))) + .filter(Boolean); +const recipient = lower(expected.to || ''); +const subject = normalize(expected.subject || ''); +const body = normalize(expected.body || ''); +return { + host: location.host, + href: location.href, + title: document.title, + is_gmail: location.host.includes('mail.google.com'), + is_login_page: location.host.includes('accounts.google.com'), + compose_open: Boolean(subjectInput) || lower(pageText).includes('send'), + recipient_match: !recipient || recipientCandidates.some(candidate => lower(candidate).includes(recipient)) || lower(pageText).includes(recipient), + subject_match: !subject || subjectValue === subject || pageText.includes(subject), + body_match: !body || bodyValue.includes(body) || pageText.includes(body), + subject_value: subjectValue, + body_value: bodyValue, + recipient_candidates: recipientCandidates.slice(0, 10), + page_excerpt: pageText.slice(0, 500), +}; +"#; + +#[derive(Debug)] +struct DemoConfig { + gmail_to: String, + gmail_subject: String, + gmail_body: String, + message_to: String, + message_body: String, + artifacts_dir: PathBuf, +} + +#[derive(Debug, Serialize)] +struct DemoReport { + artifacts_dir: String, + gmail: DemoStepReport, + messages: DemoStepReport, +} + +#[derive(Debug, Serialize)] +struct DemoStepReport { + success: bool, + summary: String, + screenshot_path: Option, + details: Value, +} + +#[tokio::main] +async fn main() -> Result<()> { + logging::init(); + let config = parse_args()?; + fs::create_dir_all(&config.artifacts_dir) + .with_context(|| format!("failed to create {}", config.artifacts_dir.display()))?; + + println!("Artifacts: {}", config.artifacts_dir.display()); + + let app_config = AppConfig::from_env(); + let automation = LocalAutomationBackend::from_config(&app_config); + let browser = BrowserSidecarManager::new(app_config.browser.clone()); + + let gmail = run_gmail_demo(&automation, &browser, &config).await?; + let messages = run_messages_demo(&automation, &config).await?; + let report = DemoReport { + artifacts_dir: config.artifacts_dir.display().to_string(), + gmail, + messages, + }; + + let report_path = config.artifacts_dir.join("desktop-demo-report.json"); + fs::write(&report_path, serde_json::to_string_pretty(&report)?) + .with_context(|| format!("failed to write {}", report_path.display()))?; + + println!("Report: {}", report_path.display()); + if report.gmail.success && report.messages.success { + println!("Desktop demo passed."); + Ok(()) + } else { + bail!("desktop demo failed; see {}", report_path.display()); + } +} + +async fn run_gmail_demo( + automation: &LocalAutomationBackend, + browser: &BrowserSidecarManager, + config: &DemoConfig, +) -> Result { + println!("Step 1/2: opening Google Chrome and Gmail compose"); + run_tool( + automation, + "app_activate", + json!({ "app_name": "Google Chrome" }), + ) + .await + .context("failed to activate Google Chrome")?; + browser + .call("/browser/attach_or_launch", json!({})) + .await + .context("failed to attach to or launch browser automation")?; + run_tool( + automation, + "app_activate", + json!({ "app_name": "Google Chrome" }), + ) + .await + .context("failed to bring Chrome frontmost after browser attach")?; + + browser + .call( + "/browser/open", + json!({ + "url": gmail_compose_url(&config.gmail_to, &config.gmail_subject, &config.gmail_body), + }), + ) + .await + .context("failed to open Gmail compose URL")?; + + let gmail_state = wait_for_gmail_state(browser, config).await?; + let gmail_screenshot = + capture_screenshot(automation, &config.artifacts_dir.join("gmail-compose.png")) + .await + .ok(); + + let evaluation = gmail_state + .get("evaluation") + .cloned() + .unwrap_or_else(|| json!({})); + let is_login_page = evaluation + .get("is_login_page") + .and_then(Value::as_bool) + .unwrap_or(false); + let passed = evaluation + .get("is_gmail") + .and_then(Value::as_bool) + .unwrap_or(false) + && !is_login_page + && evaluation + .get("compose_open") + .and_then(Value::as_bool) + .unwrap_or(false) + && evaluation + .get("recipient_match") + .and_then(Value::as_bool) + .unwrap_or(false) + && evaluation + .get("subject_match") + .and_then(Value::as_bool) + .unwrap_or(false) + && evaluation + .get("body_match") + .and_then(Value::as_bool) + .unwrap_or(false); + + let summary = if passed { + format!("Verified Gmail compose draft for {}", config.gmail_to) + } else if is_login_page { + "Gmail compose did not open because the browser session is not signed in to Gmail" + .to_string() + } else { + "Gmail compose did not verify all requested fields".to_string() + }; + + Ok(DemoStepReport { + success: passed, + summary, + screenshot_path: gmail_screenshot.map(|path| path.display().to_string()), + details: gmail_state, + }) +} + +async fn run_messages_demo( + automation: &LocalAutomationBackend, + config: &DemoConfig, +) -> Result { + println!("Step 2/2: opening Messages draft"); + let result = run_tool( + automation, + "messages_compose", + json!({ + "recipient": config.message_to, + "body": config.message_body, + }), + ) + .await + .context("failed to prepare Messages draft")?; + + let screenshot_path = capture_screenshot( + automation, + &config.artifacts_dir.join("messages-compose.png"), + ) + .await + .ok(); + let window_name = result + .observation + .get("window_name") + .and_then(Value::as_str) + .unwrap_or_default(); + let recipient_verified = recipient_matches_window(window_name, &config.message_to); + let passed = result.proof_passed && recipient_verified; + let summary = if passed { + format!("Verified Messages draft window for {}", config.message_to) + } else { + format!( + "Messages draft opened, but the window title did not reliably match {}", + config.message_to + ) + }; + + Ok(DemoStepReport { + success: passed, + summary, + screenshot_path: screenshot_path.map(|path| path.display().to_string()), + details: json!({ + "tool_summary": result.summary, + "tool_observation": result.observation, + "tool_output": result.output, + "recipient_title_verified": recipient_verified, + }), + }) +} + +async fn run_tool( + automation: &LocalAutomationBackend, + name: &str, + arguments: Value, +) -> Result { + automation + .call_tool(ToolCallRequest { + name: name.to_string(), + arguments, + risk: jarvis_rs::automation::RiskLevel::Low, + requires_confirmation: false, + target_identity: None, + expected_outcome: None, + }) + .await +} + +async fn capture_screenshot(automation: &LocalAutomationBackend, path: &Path) -> Result { + run_tool( + automation, + "take_screenshot", + json!({ "path": path.display().to_string() }), + ) + .await + .with_context(|| format!("failed to capture screenshot to {}", path.display()))?; + Ok(path.to_path_buf()) +} + +async fn wait_for_gmail_state( + browser: &BrowserSidecarManager, + config: &DemoConfig, +) -> Result { + let payload = json!({ + "script": GMAIL_INSPECTION_SCRIPT, + "arg": { + "to": config.gmail_to, + "subject": config.gmail_subject, + "body": config.gmail_body, + } + }); + + for _ in 0..20 { + let state = browser + .call("/browser/eval", payload.clone()) + .await + .context("failed to inspect Gmail compose state")?; + let evaluation = state + .get("evaluation") + .cloned() + .unwrap_or_else(|| json!({})); + let is_login_page = evaluation + .get("is_login_page") + .and_then(Value::as_bool) + .unwrap_or(false); + let compose_open = evaluation + .get("compose_open") + .and_then(Value::as_bool) + .unwrap_or(false); + if is_login_page || compose_open { + return Ok(state); + } + sleep(Duration::from_millis(500)).await; + } + + browser + .call("/browser/eval", payload) + .await + .context("failed to inspect Gmail compose state after retries") +} + +fn gmail_compose_url(to: &str, subject: &str, body: &str) -> String { + format!( + "https://mail.google.com/mail/?view=cm&fs=1&tf=1&to={}&su={}&body={}", + percent_encode_for_url(to), + percent_encode_for_url(subject), + percent_encode_for_url(body) + ) +} + +fn percent_encode_for_url(value: &str) -> String { + value + .bytes() + .map(|byte| match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (byte as char).to_string() + } + b' ' => "%20".to_string(), + other => format!("%{:02X}", other), + }) + .collect::() +} + +fn recipient_matches_window(window_name: &str, recipient: &str) -> bool { + let normalized_window = normalize_for_match(window_name); + let normalized_recipient = normalize_for_match(recipient); + if normalized_window.is_empty() || normalized_recipient.is_empty() { + return false; + } + + if normalized_window.contains(&normalized_recipient) { + return true; + } + + let recipient_digits: String = recipient.chars().filter(|ch| ch.is_ascii_digit()).collect(); + let window_digits: String = window_name + .chars() + .filter(|ch| ch.is_ascii_digit()) + .collect(); + recipient_digits.len() >= 4 + && window_digits.len() >= 4 + && window_digits.ends_with(&recipient_digits[recipient_digits.len() - 4..]) +} + +fn normalize_for_match(value: &str) -> String { + value + .to_ascii_lowercase() + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '@' || *ch == '.') + .collect() +} + +fn parse_args() -> Result { + let args = env::args().skip(1).collect::>(); + if args.iter().any(|arg| arg == "--help" || arg == "-h") { + print_usage(); + std::process::exit(0); + } + + let mut gmail_to = None; + let mut gmail_subject = None; + let mut gmail_body = None; + let mut message_to = None; + let mut message_body = None; + let mut artifacts_dir = None; + + let mut index = 0; + while index < args.len() { + let flag = &args[index]; + let value = args + .get(index + 1) + .ok_or_else(|| anyhow!("missing value for {}", flag))?; + match flag.as_str() { + "--gmail-to" => gmail_to = Some(value.clone()), + "--gmail-subject" => gmail_subject = Some(value.clone()), + "--gmail-body" => gmail_body = Some(value.clone()), + "--message-to" => message_to = Some(value.clone()), + "--message-body" => message_body = Some(value.clone()), + "--artifacts-dir" => artifacts_dir = Some(PathBuf::from(value)), + _ => bail!("unknown argument '{}'", flag), + } + index += 2; + } + + Ok(DemoConfig { + gmail_to: gmail_to.ok_or_else(|| anyhow!("--gmail-to is required"))?, + gmail_subject: gmail_subject.ok_or_else(|| anyhow!("--gmail-subject is required"))?, + gmail_body: gmail_body.ok_or_else(|| anyhow!("--gmail-body is required"))?, + message_to: message_to.ok_or_else(|| anyhow!("--message-to is required"))?, + message_body: message_body.ok_or_else(|| anyhow!("--message-body is required"))?, + artifacts_dir: artifacts_dir.unwrap_or_else(default_artifacts_dir), + }) +} + +fn default_artifacts_dir() -> PathBuf { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + env::current_dir() + .unwrap_or_else(|_| PathBuf::from(".")) + .join(".tooling") + .join("desktop-demo") + .join(format!("{}", millis)) +} + +fn print_usage() { + println!( + "Usage: cargo run -p jarvis_rs --bin desktop_demo -- \ + --gmail-to person@example.com \ + --gmail-subject \"Demo subject\" \ + --gmail-body \"Demo email body\" \ + --message-to \"+15551234567\" \ + --message-body \"Demo iMessage body\" \ + [--artifacts-dir /absolute/path]" + ); + println!("The demo stops at visible drafts. It does not send email or messages."); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gmail_compose_url_encodes_expected_fields() { + let url = gmail_compose_url("demo@example.com", "Hello there", "Line 1 & 2"); + assert!(url.contains("to=demo%40example.com")); + assert!(url.contains("su=Hello%20there")); + assert!(url.contains("body=Line%201%20%26%202")); + } + + #[test] + fn recipient_match_accepts_phone_suffix() { + assert!(recipient_matches_window( + "Chat with +1 (555) 123-4567", + "+15551234567" + )); + } +} diff --git a/jarvis_rs/src/bin/scenario_runner.rs b/jarvis_rs/src/bin/scenario_runner.rs new file mode 100644 index 0000000..a1a77af --- /dev/null +++ b/jarvis_rs/src/bin/scenario_runner.rs @@ -0,0 +1,560 @@ +use anyhow::{bail, Context, Result}; +use serde_json::{json, Value}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use jarvis_rs::agent_core::{ + build_runtime_manifest, RunReport, ScenarioOutcome, VerificationResult, +}; +use jarvis_rs::automation::{ + AutomationBackend, LocalAutomationBackend, RiskLevel, ToolCallRequest, ToolCallResult, +}; +use jarvis_rs::browser_sidecar::BrowserSidecarManager; +use jarvis_rs::config::AppConfig; +use jarvis_rs::llm::provider::OpenAiCompatibleClient; +use jarvis_rs::logging; + +#[tokio::main] +async fn main() -> Result<()> { + logging::init(); + + let artifacts_dir = artifacts_dir()?; + fs::create_dir_all(&artifacts_dir) + .with_context(|| format!("failed to create {}", artifacts_dir.display()))?; + + let config = AppConfig::from_env(); + let automation = LocalAutomationBackend::from_config(&config); + let browser = BrowserSidecarManager::new(config.browser.clone()); + let capabilities = automation.capabilities(); + let runtime = build_runtime_manifest( + &config, + &capabilities, + &capabilities.setup_items, + "scenario-runner", + "scenario-runner", + ); + + let mut scenarios = Vec::new(); + scenarios.push(run_app_focus_task(&automation, &artifacts_dir).await?); + scenarios.push(run_browser_task(&automation, &browser, &artifacts_dir).await?); + scenarios.push(run_filesystem_task(&automation, &config, &artifacts_dir).await?); + scenarios.push(run_mail_draft_task(&automation, &config, &artifacts_dir).await?); + scenarios.push(run_multi_app_task(&automation, &browser, &config, &artifacts_dir).await?); + + let report = RunReport { runtime, scenarios }; + let report_path = artifacts_dir.join("scenario-run-report.json"); + fs::write(&report_path, serde_json::to_string_pretty(&report)?) + .with_context(|| format!("failed to write {}", report_path.display()))?; + + println!("Scenario artifacts: {}", artifacts_dir.display()); + println!("Scenario report: {}", report_path.display()); + + if report.scenarios.iter().all(|scenario| scenario.success) { + println!("All scenario tasks passed."); + Ok(()) + } else { + bail!( + "one or more scenario tasks failed; see {}", + report_path.display() + ); + } +} + +async fn run_app_focus_task( + automation: &LocalAutomationBackend, + artifacts_dir: &Path, +) -> Result { + let activate = run_tool(automation, "app_activate", json!({ "app_name": "Finder" })).await; + let snapshot = run_tool(automation, "window_snapshot", json!({})).await; + let screenshot = capture_screenshot(automation, &artifacts_dir.join("app-focus-finder.png")) + .await + .ok(); + + let success = activate + .as_ref() + .map(|result| result.proof_passed) + .unwrap_or(false) + && tool_observation_str(&snapshot, "app_name") + .map(|value| value == "Finder") + .unwrap_or(false); + let verification = verification( + success, + "Finder became the frontmost application", + "Finder was not confirmed as the frontmost application", + ); + let detail_path = write_detail( + artifacts_dir, + "app-focus-finder.json", + &json!({ + "activate": result_json(&activate), + "snapshot": result_json(&snapshot), + "verification": verification.clone(), + }), + )?; + + Ok(ScenarioOutcome { + id: "app_focus_finder".to_string(), + success, + summary: verification.summary.clone(), + verification: Some(verification), + artifact_paths: artifact_paths(&[Some(detail_path), screenshot]), + }) +} + +async fn run_browser_task( + automation: &LocalAutomationBackend, + browser: &BrowserSidecarManager, + artifacts_dir: &Path, +) -> Result { + let activate = run_tool( + automation, + "app_activate", + json!({ "app_name": "Google Chrome" }), + ) + .await; + let attach = browser.call("/browser/attach_or_launch", json!({})).await; + let open = browser + .call("/browser/open", json!({ "url": "https://example.com" })) + .await; + let assertion = browser + .call( + "/browser/assert", + json!({ "title_contains": "Example Domain", "text_contains": "Example Domain" }), + ) + .await; + let screenshot = capture_screenshot(automation, &artifacts_dir.join("browser-example.png")) + .await + .ok(); + + let success = activate + .as_ref() + .map(|result| result.proof_passed) + .unwrap_or(false) + && attach + .as_ref() + .map(|value| value["ok"] == true) + .unwrap_or(false) + && open + .as_ref() + .map(|value| value["ok"] == true) + .unwrap_or(false) + && value_bool(&assertion, "matched").unwrap_or(false); + let verification = verification( + success, + "Browser workflow opened Example Domain and verified visible state", + "Browser workflow did not verify the requested page state", + ); + let detail_path = write_detail( + artifacts_dir, + "browser-example.json", + &json!({ + "activate": result_json(&activate), + "attach": value_json(&attach), + "open": value_json(&open), + "assertion": value_json(&assertion), + "verification": verification.clone(), + }), + )?; + + Ok(ScenarioOutcome { + id: "browser_example_domain".to_string(), + success, + summary: verification.summary.clone(), + verification: Some(verification), + artifact_paths: artifact_paths(&[Some(detail_path), screenshot]), + }) +} + +async fn run_filesystem_task( + automation: &LocalAutomationBackend, + config: &AppConfig, + artifacts_dir: &Path, +) -> Result { + let work_dir = artifacts_dir.join("filesystem-roundtrip"); + let source_path = work_dir.join("generated-note.txt"); + let moved_path = work_dir.join("moved-note.txt"); + let (generated_text, generator) = generate_task_text( + config, + "Write a concise two-sentence status note about proving an autonomous filesystem workflow.", + ) + .await; + + let create = run_tool( + automation, + "filesystem_create_folder", + json!({ "path": work_dir.display().to_string() }), + ) + .await; + let write = run_tool( + automation, + "filesystem_write_file", + json!({ + "path": source_path.display().to_string(), + "contents": generated_text, + }), + ) + .await; + let read = run_tool( + automation, + "filesystem_read_file", + json!({ "path": source_path.display().to_string() }), + ) + .await; + let mv = run_tool( + automation, + "filesystem_move", + json!({ + "from": source_path.display().to_string(), + "to": moved_path.display().to_string(), + }), + ) + .await; + let list = run_tool( + automation, + "filesystem_list", + json!({ "path": work_dir.display().to_string() }), + ) + .await; + + let success = create.is_ok() + && write.is_ok() + && tool_output(&read) + .map(|output| !output.trim().is_empty()) + .unwrap_or(false) + && mv.is_ok() + && tool_output(&list) + .map(|output| output.contains("moved-note.txt")) + .unwrap_or(false); + let verification = verification( + success, + "Filesystem roundtrip created, read, moved, and listed generated content", + "Filesystem roundtrip did not complete every required step", + ); + let detail_path = write_detail( + artifacts_dir, + "filesystem-roundtrip.json", + &json!({ + "generator": generator, + "create": result_json(&create), + "write": result_json(&write), + "read": result_json(&read), + "move": result_json(&mv), + "list": result_json(&list), + "verification": verification.clone(), + }), + )?; + + Ok(ScenarioOutcome { + id: "filesystem_roundtrip".to_string(), + success, + summary: verification.summary.clone(), + verification: Some(verification), + artifact_paths: artifact_paths(&[Some(detail_path)]), + }) +} + +async fn run_mail_draft_task( + automation: &LocalAutomationBackend, + config: &AppConfig, + artifacts_dir: &Path, +) -> Result { + let (generated_body, generator) = generate_task_text( + config, + "Write a short professional email body that says Jarvis completed a no-help autonomy validation run.", + ) + .await; + let result = run_tool( + automation, + "mail_compose", + json!({ + "to": Option::::None, + "subject": "Jarvis autonomy validation draft", + "body": generated_body, + }), + ) + .await; + let screenshot = capture_screenshot(automation, &artifacts_dir.join("mail-draft.png")) + .await + .ok(); + let success = result + .as_ref() + .map(|value| value.proof_passed) + .unwrap_or(false); + let verification = verification( + success, + "Mail draft with generated text was created and verified", + "Mail draft generation did not verify successfully", + ); + let detail_path = write_detail( + artifacts_dir, + "mail-draft.json", + &json!({ + "generator": generator, + "result": result_json(&result), + "verification": verification.clone(), + }), + )?; + + Ok(ScenarioOutcome { + id: "mail_generated_draft".to_string(), + success, + summary: verification.summary.clone(), + verification: Some(verification), + artifact_paths: artifact_paths(&[Some(detail_path), screenshot]), + }) +} + +async fn run_multi_app_task( + automation: &LocalAutomationBackend, + browser: &BrowserSidecarManager, + config: &AppConfig, + artifacts_dir: &Path, +) -> Result { + let summary_file = artifacts_dir.join("multi-app-summary.txt"); + let browser_open = browser + .call("/browser/open", json!({ "url": "https://example.com" })) + .await; + let extracted = browser + .call("/browser/extract_text", json!({ "max_chars": 400 })) + .await; + let extracted_text = + value_str(&extracted, "extractedText").unwrap_or_else(|| "Example Domain".to_string()); + let prompt = format!( + "Summarize this web page in two concise sentences for an autonomy regression note:\n\n{}", + extracted_text + ); + let (generated_summary, generator) = generate_task_text(config, &prompt).await; + let write = run_tool( + automation, + "filesystem_write_file", + json!({ + "path": summary_file.display().to_string(), + "contents": generated_summary.clone(), + }), + ) + .await; + let mail = run_tool( + automation, + "mail_compose", + json!({ + "to": Option::::None, + "subject": "Multi-app autonomy summary", + "body": generated_summary, + }), + ) + .await; + let screenshot = capture_screenshot(automation, &artifacts_dir.join("multi-app-mail.png")) + .await + .ok(); + + let success = browser_open + .as_ref() + .map(|value| value["ok"] == true) + .unwrap_or(false) + && value_str(&extracted, "extractedText") + .map(|text| !text.trim().is_empty()) + .unwrap_or(false) + && write.is_ok() + && mail + .as_ref() + .map(|value| value.proof_passed) + .unwrap_or(false); + let verification = verification( + success, + "Multi-app workflow crossed browser, filesystem, and Mail with generated output", + "Multi-app workflow did not verify across every subsystem", + ); + let detail_path = write_detail( + artifacts_dir, + "multi-app-summary.json", + &json!({ + "generator": generator, + "browser_open": value_json(&browser_open), + "extracted": value_json(&extracted), + "write": result_json(&write), + "mail": result_json(&mail), + "verification": verification.clone(), + }), + )?; + + Ok(ScenarioOutcome { + id: "multi_app_browser_file_mail".to_string(), + success, + summary: verification.summary.clone(), + verification: Some(verification), + artifact_paths: artifact_paths(&[Some(detail_path), screenshot]), + }) +} + +async fn run_tool( + automation: &LocalAutomationBackend, + name: &str, + arguments: Value, +) -> Result { + automation + .call_tool(ToolCallRequest { + name: name.to_string(), + arguments, + risk: RiskLevel::Low, + requires_confirmation: false, + target_identity: None, + expected_outcome: None, + }) + .await +} + +async fn capture_screenshot(automation: &LocalAutomationBackend, path: &Path) -> Result { + let _ = run_tool( + automation, + "take_screenshot", + json!({ "path": path.display().to_string() }), + ) + .await?; + if path.exists() { + Ok(path.to_path_buf()) + } else { + bail!( + "screenshot command returned without creating {}", + path.display() + ); + } +} + +async fn generate_task_text(config: &AppConfig, prompt: &str) -> (String, String) { + let mut clients = Vec::new(); + if let (Some(base_url), Some(model)) = ( + config.provider.fallback_base_url.clone(), + config.provider.fallback_model.clone(), + ) { + clients.push(( + "hosted-openai".to_string(), + OpenAiCompatibleClient::new(base_url, model, config.provider.fallback_api_key.clone()), + )); + } + clients.push(( + "local-worker".to_string(), + OpenAiCompatibleClient::new( + config.provider.worker_endpoint.clone(), + config.provider.worker_model.clone(), + config.provider.fallback_api_key.clone(), + ), + )); + + for (label, client) in clients { + match client + .complete_text( + "You write concise desktop-task output. Return plain text only.", + prompt, + ) + .await + { + Ok(text) if !text.trim().is_empty() => return (text.trim().to_string(), label), + _ => continue, + } + } + + let fallback = prompt + .split_whitespace() + .take(24) + .collect::>() + .join(" "); + ( + format!("Fallback generated text: {}", fallback), + "deterministic-fallback".to_string(), + ) +} + +fn artifacts_dir() -> Result { + let current_dir = std::env::current_dir().context("failed to read current directory")?; + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock drifted before unix epoch")? + .as_secs(); + Ok(current_dir + .join("scenario_artifacts") + .join(format!("run-{}", stamp))) +} + +fn write_detail(artifacts_dir: &Path, file_name: &str, payload: &Value) -> Result { + let path = artifacts_dir.join(file_name); + fs::write(&path, serde_json::to_string_pretty(payload)?) + .with_context(|| format!("failed to write {}", path.display()))?; + Ok(path) +} + +fn artifact_paths(values: &[Option]) -> Vec { + values + .iter() + .flatten() + .map(|path| path.display().to_string()) + .collect() +} + +fn verification(success: bool, success_summary: &str, failure_summary: &str) -> VerificationResult { + VerificationResult { + satisfied: success, + confidence: if success { 0.95 } else { 0.2 }, + summary: if success { + success_summary.to_string() + } else { + failure_summary.to_string() + }, + matched_criteria: if success { + vec![success_summary.to_string()] + } else { + Vec::new() + }, + missing_criteria: if success { + Vec::new() + } else { + vec![failure_summary.to_string()] + }, + } +} + +fn result_json(result: &Result) -> Value { + match result { + Ok(value) => { + serde_json::to_value(value).unwrap_or_else(|_| json!({"error": "serialization failed"})) + } + Err(error) => json!({ "error": error.to_string() }), + } +} + +fn value_json(result: &Result) -> Value { + match result { + Ok(value) => value.clone(), + Err(error) => json!({ "error": error.to_string() }), + } +} + +fn tool_output(result: &Result) -> Option { + result.as_ref().ok().and_then(|value| value.output.clone()) +} + +fn tool_observation_str(result: &Result, field: &str) -> Option { + result + .as_ref() + .ok() + .and_then(|value| value.observation.get(field)) + .and_then(Value::as_str) + .map(str::to_string) +} + +fn value_str(result: &Result, field: &str) -> Option { + result + .as_ref() + .ok() + .and_then(|value| value.get(field)) + .and_then(Value::as_str) + .map(str::to_string) +} + +fn value_bool(result: &Result, field: &str) -> Option { + result + .as_ref() + .ok() + .and_then(|value| value.get(field)) + .and_then(Value::as_bool) +} diff --git a/jarvis_rs/src/browser_sidecar.rs b/jarvis_rs/src/browser_sidecar.rs index b356905..6e94d19 100644 --- a/jarvis_rs/src/browser_sidecar.rs +++ b/jarvis_rs/src/browser_sidecar.rs @@ -56,14 +56,18 @@ impl BrowserSidecarManager { let node_ready = self.config.node_binary.exists(); let npm_ready = self.config.npm_binary.exists(); let chrome_ready = self.config.chrome_executable.exists(); - let sidecar_files_ready = self.config.sidecar_dir.exists() && self.config.sidecar_entry.exists(); + let sidecar_files_ready = + self.config.sidecar_dir.exists() && self.config.sidecar_entry.exists(); let playwright_installed = self .config .sidecar_dir .join("node_modules/playwright-core") .exists(); let health = self.health_sync_value(); - let sidecar_running = health.is_some(); + let sidecar_running = health + .as_ref() + .map(Self::payload_is_healthy) + .unwrap_or(false); let browser_mode = health .as_ref() .and_then(|value| value.get("mode")) @@ -90,7 +94,10 @@ impl BrowserSidecarManager { )); } if !sidecar_running { - setup_items.push("Browser sidecar is not running yet; it will be started on the first browser task".to_string()); + setup_items.push( + "Browser sidecar is not running yet; it will be started on the first browser task" + .to_string(), + ); } BrowserSidecarDiagnostics { @@ -111,13 +118,21 @@ impl BrowserSidecarManager { return Ok(health); } + if let Some(recovered) = self.recover_existing_sidecar().await? { + return Ok(recovered); + } + self.bootstrap_prerequisites()?; + self.cleanup_stale_listener()?; self.spawn_sidecar_if_needed()?; for _ in 0..30 { if let Some(health) = self.health().await? { return Ok(health); } + if let Some(recovered) = self.recover_existing_sidecar().await? { + return Ok(recovered); + } thread::sleep(Duration::from_millis(200)); } @@ -137,7 +152,7 @@ impl BrowserSidecarManager { let response = self.http.post(url).json(&payload).send().await?; let status = response.status(); let value: Value = response.json().await?; - if !status.is_success() { + if !status.is_success() || !Self::payload_is_healthy(&value) { let message = value .get("error") .and_then(Value::as_str) @@ -166,7 +181,12 @@ impl BrowserSidecarManager { self.config.sidecar_entry.display() ); } - if !self.config.sidecar_dir.join("node_modules/playwright-core").exists() { + if !self + .config + .sidecar_dir + .join("node_modules/playwright-core") + .exists() + { bail!( "Playwright sidecar dependency is missing. Install it in {} before browser tasks can run.", self.config.sidecar_dir.display() @@ -188,22 +208,18 @@ impl BrowserSidecarManager { *child_guard = None; } - std::fs::create_dir_all(&self.config.dedicated_profile_dir) - .with_context(|| { - format!( - "failed to create browser profile directory {}", - self.config.dedicated_profile_dir.display() - ) - })?; + std::fs::create_dir_all(&self.config.dedicated_profile_dir).with_context(|| { + format!( + "failed to create browser profile directory {}", + self.config.dedicated_profile_dir.display() + ) + })?; let child = Command::new(&self.config.node_binary) .arg(&self.config.sidecar_entry) .current_dir(&self.config.sidecar_dir) .env("JARVIS_BROWSER_HOST", &self.config.sidecar_host) - .env( - "JARVIS_BROWSER_PORT", - self.config.sidecar_port.to_string(), - ) + .env("JARVIS_BROWSER_PORT", self.config.sidecar_port.to_string()) .env( "JARVIS_CHROME_EXECUTABLE", self.config.chrome_executable.display().to_string(), @@ -231,12 +247,81 @@ impl BrowserSidecarManager { Ok(()) } + fn cleanup_stale_listener(&self) -> Result<()> { + let port_arg = format!("-iTCP:{}", self.config.sidecar_port); + let output = match Command::new("lsof") + .arg("-t") + .arg("-nP") + .arg(port_arg) + .arg("-sTCP:LISTEN") + .output() + { + Ok(output) => output, + Err(_) => return Ok(()), + }; + + if !output.status.success() { + return Ok(()); + } + + let mut child_guard = self + .child + .lock() + .map_err(|_| anyhow!("browser sidecar child mutex poisoned"))?; + + for pid in String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::trim) + .filter(|pid| !pid.is_empty()) + { + if let Some(existing) = child_guard.as_mut() { + if existing.id().to_string() == pid { + let _ = existing.kill(); + let _ = existing.wait(); + *child_guard = None; + continue; + } + } + + let _ = Command::new("kill").arg("-TERM").arg(pid).status(); + } + + thread::sleep(Duration::from_millis(250)); + Ok(()) + } + async fn health(&self) -> Result> { - let url = format!("{}/health", self.config.sidecar_endpoint.trim_end_matches('/')); + let url = format!( + "{}/health", + self.config.sidecar_endpoint.trim_end_matches('/') + ); match self.http.get(url).send().await { Ok(response) if response.status().is_success() => { let value = response.json::().await?; - Ok(Some(value)) + if Self::payload_is_healthy(&value) { + Ok(Some(value)) + } else { + Ok(None) + } + } + Ok(_) => Ok(None), + Err(_) => Ok(None), + } + } + + async fn recover_existing_sidecar(&self) -> Result> { + let url = format!( + "{}/browser/reset", + self.config.sidecar_endpoint.trim_end_matches('/') + ); + match self.http.post(url).json(&json!({})).send().await { + Ok(response) if response.status().is_success() => { + let value = response.json::().await?; + if Self::payload_is_healthy(&value) { + Ok(Some(value)) + } else { + Ok(None) + } } Ok(_) => Ok(None), Err(_) => Ok(None), @@ -266,6 +351,10 @@ impl BrowserSidecarManager { serde_json::from_slice(&output.stdout).ok() } + fn payload_is_healthy(value: &Value) -> bool { + value.get("ok").and_then(Value::as_bool).unwrap_or(true) + } + pub fn config_paths(&self) -> Vec { vec![ self.config.sidecar_dir.clone(), diff --git a/jarvis_rs/src/config.rs b/jarvis_rs/src/config.rs index dc0756e..f0ba40c 100644 --- a/jarvis_rs/src/config.rs +++ b/jarvis_rs/src/config.rs @@ -1,5 +1,6 @@ use std::env; use std::path::PathBuf; +use std::process::Command; #[derive(Clone, Debug)] pub struct ProviderConfig { @@ -66,6 +67,10 @@ impl Default for AppConfig { let browser_sidecar_dir = project_root.join("browser_sidecar"); let browser_port = 4317; let chrome_debug_port = 9222; + let node_binary = + detect_binary_path("node").unwrap_or_else(|| local_node_root.join("bin/node")); + let npm_binary = + detect_binary_path("npm").unwrap_or_else(|| local_node_root.join("bin/npm")); let mut allowed_paths = vec![current_dir.clone()]; allowed_paths.push(home_dir.join("Desktop")); @@ -106,8 +111,8 @@ impl Default for AppConfig { sidecar_port: browser_port, sidecar_dir: browser_sidecar_dir.clone(), sidecar_entry: browser_sidecar_dir.join("server.mjs"), - node_binary: local_node_root.join("bin/node"), - npm_binary: local_node_root.join("bin/npm"), + node_binary, + npm_binary, chrome_executable: PathBuf::from( "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", ), @@ -321,6 +326,19 @@ fn project_root_candidates() -> Vec { candidates } +fn detect_binary_path(name: &str) -> Option { + let output = Command::new("which").arg(name).output().ok()?; + if !output.status.success() { + return None; + } + let path = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if path.is_empty() { + None + } else { + Some(PathBuf::from(path)) + } +} + fn auto_detect_whisper_model() -> Option { let mut candidates = Vec::new(); @@ -329,8 +347,12 @@ fn auto_detect_whisper_model() -> Option { candidates.push(root.join(".tooling/models/ggml-base.en.bin")); } - candidates.push(PathBuf::from("/opt/homebrew/share/whisper-cpp/for-tests-ggml-tiny.bin")); - candidates.push(PathBuf::from("/usr/local/share/whisper-cpp/for-tests-ggml-tiny.bin")); + candidates.push(PathBuf::from( + "/opt/homebrew/share/whisper-cpp/for-tests-ggml-tiny.bin", + )); + candidates.push(PathBuf::from( + "/usr/local/share/whisper-cpp/for-tests-ggml-tiny.bin", + )); candidates.into_iter().find(|path| path.exists()) } diff --git a/jarvis_rs/src/executor/command.rs b/jarvis_rs/src/executor/command.rs index c7f4bf4..31f88c7 100644 --- a/jarvis_rs/src/executor/command.rs +++ b/jarvis_rs/src/executor/command.rs @@ -1,10 +1,9 @@ +use crate::intent::validator::ValidatedParams; use anyhow::Result; use std::process::Command; -use crate::intent::validator::ValidatedParams; pub fn execute(intent: ValidatedParams) -> Result<()> { match intent { - ValidatedParams::OpenApplication(params) => { println!("Opening application: {}", params.app_name); @@ -17,13 +16,14 @@ pub fn execute(intent: ValidatedParams) -> Result<()> { ValidatedParams::ListFiles(params) => { println!("Listing files in: {}", params.directory); - Command::new("ls") - .arg(params.directory) - .spawn()?; + Command::new("ls").arg(params.directory).spawn()?; } ValidatedParams::CreateFolder(params) => { - println!("Creating folder: {}/{}", params.directory, params.folder_name); + println!( + "Creating folder: {}/{}", + params.directory, params.folder_name + ); Command::new("mkdir") .arg(format!("{}/{}", params.directory, params.folder_name)) @@ -33,9 +33,7 @@ pub fn execute(intent: ValidatedParams) -> Result<()> { ValidatedParams::DeleteFile(params) => { println!("Deleting file: {}", params.path); - Command::new("rm") - .arg(params.path) - .spawn()?; + Command::new("rm").arg(params.path).spawn()?; } ValidatedParams::SearchWeb(params) => { @@ -43,9 +41,7 @@ pub fn execute(intent: ValidatedParams) -> Result<()> { let url = format!("https://www.google.com/search?q={}", params.query); - Command::new("open") - .arg(url) - .spawn()?; + Command::new("open").arg(url).spawn()?; } } diff --git a/jarvis_rs/src/executor/systems.rs b/jarvis_rs/src/executor/systems.rs index 4614a30..6e2d806 100644 --- a/jarvis_rs/src/executor/systems.rs +++ b/jarvis_rs/src/executor/systems.rs @@ -1,21 +1,14 @@ -use std::process::Command; use anyhow::Result; +use std::process::Command; pub fn open_application(app_name: &str) -> Result<()> { - - Command::new("open") - .arg("-a") - .arg(app_name) - .spawn()?; + Command::new("open").arg("-a").arg(app_name).spawn()?; Ok(()) } pub fn list_files(path: &str) -> Result { - - let output = Command::new("ls") - .arg(path) - .output()?; + let output = Command::new("ls").arg(path).output()?; let result = String::from_utf8_lossy(&output.stdout).to_string(); @@ -23,30 +16,24 @@ pub fn list_files(path: &str) -> Result { } pub fn create_folder(name: &str) -> Result<()> { - - Command::new("mkdir") - .arg(name) - .spawn()?; + Command::new("mkdir").arg(name).spawn()?; Ok(()) } pub fn delete_file(path: &str) -> Result<()> { - - Command::new("rm") - .arg(path) - .spawn()?; + Command::new("rm").arg(path).spawn()?; Ok(()) } pub fn search_web(query: &str) -> Result<()> { + let url = format!( + "https://www.google.com/search?q={}", + query.replace(" ", "+") + ); - let url = format!("https://www.google.com/search?q={}", query.replace(" ", "+")); - - Command::new("open") - .arg(url) - .spawn()?; + Command::new("open").arg(url).spawn()?; Ok(()) -} \ No newline at end of file +} diff --git a/jarvis_rs/src/intent/mod.rs b/jarvis_rs/src/intent/mod.rs index 88278f2..d5b3d47 100644 --- a/jarvis_rs/src/intent/mod.rs +++ b/jarvis_rs/src/intent/mod.rs @@ -2,4 +2,4 @@ pub mod schema; pub mod validator; pub use schema::*; -pub use validator::*; \ No newline at end of file +pub use validator::*; diff --git a/jarvis_rs/src/intent/schema.rs b/jarvis_rs/src/intent/schema.rs index e14af3f..867e4ba 100644 --- a/jarvis_rs/src/intent/schema.rs +++ b/jarvis_rs/src/intent/schema.rs @@ -24,4 +24,4 @@ pub struct IntentObject { pub parameters: serde_json::Value, pub risk_level: RiskLevel, pub requires_confirmation: bool, -} \ No newline at end of file +} diff --git a/jarvis_rs/src/intent/validator.rs b/jarvis_rs/src/intent/validator.rs index 0cf787f..1c5451c 100644 --- a/jarvis_rs/src/intent/validator.rs +++ b/jarvis_rs/src/intent/validator.rs @@ -1,6 +1,6 @@ -use serde::{Deserialize}; -use anyhow::Result; use super::{IntentObject, IntentType}; +use anyhow::Result; +use serde::Deserialize; #[derive(Debug, Deserialize)] pub struct OpenApplicationParams { @@ -40,32 +40,27 @@ pub enum ValidatedParams { pub fn validate_parameters(intent: &IntentObject) -> Result { match intent.intent { IntentType::OpenApplication => { - let params: OpenApplicationParams = - serde_json::from_value(intent.parameters.clone())?; + let params: OpenApplicationParams = serde_json::from_value(intent.parameters.clone())?; Ok(ValidatedParams::OpenApplication(params)) } IntentType::ListFiles => { - let params: ListFilesParams = - serde_json::from_value(intent.parameters.clone())?; + let params: ListFilesParams = serde_json::from_value(intent.parameters.clone())?; Ok(ValidatedParams::ListFiles(params)) } IntentType::CreateFolder => { - let params: CreateFolderParams = - serde_json::from_value(intent.parameters.clone())?; + let params: CreateFolderParams = serde_json::from_value(intent.parameters.clone())?; Ok(ValidatedParams::CreateFolder(params)) } IntentType::DeleteFile => { - let params: DeleteFileParams = - serde_json::from_value(intent.parameters.clone())?; + let params: DeleteFileParams = serde_json::from_value(intent.parameters.clone())?; Ok(ValidatedParams::DeleteFile(params)) } IntentType::SearchWeb => { - let params: SearchWebParams = - serde_json::from_value(intent.parameters.clone())?; + let params: SearchWebParams = serde_json::from_value(intent.parameters.clone())?; Ok(ValidatedParams::SearchWeb(params)) } } diff --git a/jarvis_rs/src/lib.rs b/jarvis_rs/src/lib.rs index c311f87..4fb3a40 100644 --- a/jarvis_rs/src/lib.rs +++ b/jarvis_rs/src/lib.rs @@ -1,4 +1,5 @@ pub mod agent; +pub mod agent_core; pub mod app; pub mod automation; pub mod browser_sidecar; diff --git a/jarvis_rs/src/llm/mod.rs b/jarvis_rs/src/llm/mod.rs index 561bf80..06e79e2 100644 --- a/jarvis_rs/src/llm/mod.rs +++ b/jarvis_rs/src/llm/mod.rs @@ -4,8 +4,9 @@ pub mod provider; pub mod schema; pub use provider::{ - HeuristicPlannerProvider, HeuristicWorkerProvider, LocalLlamaPlannerProvider, - LocalLlamaWorkerProvider, PlannerProvider, PlannerStack, WorkerProvider, WorkerStack, + HeuristicPlannerProvider, HeuristicWorkerProvider, HostedPlannerProvider, HostedWorkerProvider, + LocalLlamaPlannerProvider, LocalLlamaWorkerProvider, OpenAiCompatibleClient, PlannerProvider, + PlannerStack, WorkerProvider, WorkerStack, }; pub use schema::{ CompletionStatus, Observation, PlanStep, StepStatus, TaskPlan, TaskState, WorkerAction, diff --git a/jarvis_rs/src/llm/ollama_client.rs b/jarvis_rs/src/llm/ollama_client.rs index bb24946..d39f21e 100644 --- a/jarvis_rs/src/llm/ollama_client.rs +++ b/jarvis_rs/src/llm/ollama_client.rs @@ -35,11 +35,7 @@ impl OllamaClient { let json: serde_json::Value = response.json().await?; - let output = json["response"] - .as_str() - .unwrap_or("") - .trim() - .to_string(); + let output = json["response"].as_str().unwrap_or("").trim().to_string(); Ok(output) } diff --git a/jarvis_rs/src/llm/provider.rs b/jarvis_rs/src/llm/provider.rs index cfda264..79f81dc 100644 --- a/jarvis_rs/src/llm/provider.rs +++ b/jarvis_rs/src/llm/provider.rs @@ -1,6 +1,7 @@ use std::sync::Arc; +use std::time::Duration; -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use async_trait::async_trait; use reqwest::Client; use serde::de::DeserializeOwned; @@ -9,22 +10,30 @@ use serde_json::json; use crate::automation::{AutomationCapabilities, RiskLevel, ToolCallRequest}; use super::prompt::{build_planner_prompt, build_worker_prompt}; -use super::schema::{PlanStep, TaskPlan, TaskState, WorkerDecision, WorkerAction}; +use super::schema::{PlanStep, TaskPlan, TaskState, WorkerAction, WorkerDecision}; #[async_trait] pub trait PlannerProvider: Send + Sync { - async fn create_plan(&self, task_input: &str, capabilities: &AutomationCapabilities) -> Result; + async fn create_plan( + &self, + task_input: &str, + capabilities: &AutomationCapabilities, + ) -> Result; fn provider_name(&self) -> &'static str; } #[async_trait] pub trait WorkerProvider: Send + Sync { - async fn next_action(&self, state: &TaskState, capabilities: &AutomationCapabilities) -> Result; + async fn next_action( + &self, + state: &TaskState, + capabilities: &AutomationCapabilities, + ) -> Result; fn provider_name(&self) -> &'static str; } #[derive(Clone)] -struct OpenAiCompatibleClient { +pub struct OpenAiCompatibleClient { base_url: String, model: String, api_key: Option, @@ -32,16 +41,23 @@ struct OpenAiCompatibleClient { } impl OpenAiCompatibleClient { - fn new(base_url: String, model: String, api_key: Option) -> Self { + pub fn new(base_url: String, model: String, api_key: Option) -> Self { Self { base_url, model, api_key, - http: Client::new(), + http: Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .unwrap_or_else(|_| Client::new()), } } - async fn parse_json(&self, system_prompt: &str, user_prompt: &str) -> Result { + pub fn model_name(&self) -> &str { + &self.model + } + + async fn chat_completion(&self, system_prompt: &str, user_prompt: &str) -> Result { let endpoint = format!("{}/chat/completions", self.base_url.trim_end_matches('/')); let mut request = self.http.post(endpoint).json(&json!({ "model": self.model, @@ -56,14 +72,26 @@ impl OpenAiCompatibleClient { } let response = request.send().await?.error_for_status()?; let body: serde_json::Value = response.json().await?; - let content = body["choices"][0]["message"]["content"] + Ok(body["choices"][0]["message"]["content"] .as_str() .unwrap_or_default() .trim() - .to_string(); + .to_string()) + } + + async fn parse_json( + &self, + system_prompt: &str, + user_prompt: &str, + ) -> Result { + let content = self.chat_completion(system_prompt, user_prompt).await?; serde_json::from_str(&content) .with_context(|| format!("model {} returned invalid JSON: {}", self.model, content)) } + + pub async fn complete_text(&self, system_prompt: &str, user_prompt: &str) -> Result { + self.chat_completion(system_prompt, user_prompt).await + } } #[derive(Clone)] @@ -81,7 +109,11 @@ impl LocalLlamaPlannerProvider { #[async_trait] impl PlannerProvider for LocalLlamaPlannerProvider { - async fn create_plan(&self, task_input: &str, capabilities: &AutomationCapabilities) -> Result { + async fn create_plan( + &self, + task_input: &str, + capabilities: &AutomationCapabilities, + ) -> Result { self.client .parse_json( "You are Jarvis Planner. Return strict JSON only.", @@ -95,6 +127,39 @@ impl PlannerProvider for LocalLlamaPlannerProvider { } } +#[derive(Clone)] +pub struct HostedPlannerProvider { + client: OpenAiCompatibleClient, +} + +impl HostedPlannerProvider { + pub fn new(base_url: String, model: String, api_key: Option) -> Self { + Self { + client: OpenAiCompatibleClient::new(base_url, model, api_key), + } + } +} + +#[async_trait] +impl PlannerProvider for HostedPlannerProvider { + async fn create_plan( + &self, + task_input: &str, + capabilities: &AutomationCapabilities, + ) -> Result { + self.client + .parse_json( + "You are Jarvis Planner. Return strict JSON only.", + &build_planner_prompt(task_input, capabilities), + ) + .await + } + + fn provider_name(&self) -> &'static str { + "hosted-openai-planner" + } +} + #[derive(Clone)] pub struct LocalLlamaWorkerProvider { client: OpenAiCompatibleClient, @@ -110,7 +175,11 @@ impl LocalLlamaWorkerProvider { #[async_trait] impl WorkerProvider for LocalLlamaWorkerProvider { - async fn next_action(&self, state: &TaskState, capabilities: &AutomationCapabilities) -> Result { + async fn next_action( + &self, + state: &TaskState, + capabilities: &AutomationCapabilities, + ) -> Result { self.client .parse_json( "You are Jarvis Worker. Return strict JSON only.", @@ -124,17 +193,59 @@ impl WorkerProvider for LocalLlamaWorkerProvider { } } +#[derive(Clone)] +pub struct HostedWorkerProvider { + client: OpenAiCompatibleClient, +} + +impl HostedWorkerProvider { + pub fn new(base_url: String, model: String, api_key: Option) -> Self { + Self { + client: OpenAiCompatibleClient::new(base_url, model, api_key), + } + } +} + +#[async_trait] +impl WorkerProvider for HostedWorkerProvider { + async fn next_action( + &self, + state: &TaskState, + capabilities: &AutomationCapabilities, + ) -> Result { + self.client + .parse_json( + "You are Jarvis Worker. Return strict JSON only.", + &build_worker_prompt(state, capabilities), + ) + .await + } + + fn provider_name(&self) -> &'static str { + "hosted-openai-worker" + } +} + #[derive(Debug, Default)] pub struct HeuristicPlannerProvider; #[async_trait] impl PlannerProvider for HeuristicPlannerProvider { - async fn create_plan(&self, task_input: &str, capabilities: &AutomationCapabilities) -> Result { + async fn create_plan( + &self, + task_input: &str, + capabilities: &AutomationCapabilities, + ) -> Result { let lowered = task_input.to_ascii_lowercase(); - if lowered.contains("schoology") || lowered.contains("biology assignment") || lowered.contains("assignment") { + if lowered.contains("schoology") + || lowered.contains("biology assignment") + || lowered.contains("assignment") + { return Ok(TaskPlan { - assistant_message: "I’m opening Chrome, getting into Schoology, and finding the assignment.".to_string(), + assistant_message: + "I’m opening Chrome, getting into Schoology, and finding the assignment." + .to_string(), summary: "Open the latest Schoology assignment".to_string(), goal: task_input.trim().to_string(), success_criteria: vec![ @@ -143,18 +254,44 @@ impl PlannerProvider for HeuristicPlannerProvider { "The assignment content is summarized before any submission".to_string(), ], steps: vec![ - step("activate_chrome", "Open Chrome", "Resolve and activate Google Chrome", "Chrome is frontmost"), - step("open_schoology", "Open Schoology", "Open Schoology in Chrome", "Schoology is visible in Chrome"), - step("open_assignments", "Open assignments", "Navigate to the assignments section", "The assignments view is visible"), - step("open_biology", "Open biology assignment", "Open the latest biology-related assignment", "The biology assignment is open"), - step("summarize_assignment", "Summarize assignment", "Read the assignment page and summarize the work", "The assignment details are summarized"), + step( + "activate_chrome", + "Open Chrome", + "Resolve and activate Google Chrome", + "Chrome is frontmost", + ), + step( + "open_schoology", + "Open Schoology", + "Open Schoology in Chrome", + "Schoology is visible in Chrome", + ), + step( + "open_assignments", + "Open assignments", + "Navigate to the assignments section", + "The assignments view is visible", + ), + step( + "open_biology", + "Open biology assignment", + "Open the latest biology-related assignment", + "The biology assignment is open", + ), + step( + "summarize_assignment", + "Summarize assignment", + "Read the assignment page and summarize the work", + "The assignment details are summarized", + ), ], }); } if lowered.contains("email") || lowered.contains("mail ") { let recipient = extract_after_keyword(task_input, "to"); - let subject = extract_after_keyword(task_input, "about").unwrap_or_else(|| "Jarvis follow-up".to_string()); + let subject = extract_after_keyword(task_input, "about") + .unwrap_or_else(|| "Jarvis follow-up".to_string()); return Ok(TaskPlan { assistant_message: "I’m preparing a Mail draft.".to_string(), summary: "Compose an email draft".to_string(), @@ -165,7 +302,12 @@ impl PlannerProvider for HeuristicPlannerProvider { "Jarvis stops before sending".to_string(), ], steps: vec![ - step("activate_mail", "Open Mail", "Activate Mail", "Mail is frontmost"), + step( + "activate_mail", + "Open Mail", + "Activate Mail", + "Mail is frontmost", + ), step( "compose_mail", "Compose mail", @@ -190,22 +332,43 @@ impl PlannerProvider for HeuristicPlannerProvider { "Jarvis stops before sending".to_string(), ], steps: vec![ - step("activate_messages", "Open Messages", "Activate Messages", "Messages is frontmost"), - step("compose_message", "Compose message", "Create a draft message", "The message draft is prepared"), + step( + "activate_messages", + "Open Messages", + "Activate Messages", + "Messages is frontmost", + ), + step( + "compose_message", + "Compose message", + "Create a draft message", + "The message draft is prepared", + ), ], }); } if lowered.contains("open ") { - let app_name = extract_after_keyword(task_input, "open").unwrap_or_else(|| capabilities.primary_browser.clone()); + let app_name = extract_after_keyword(task_input, "open") + .unwrap_or_else(|| capabilities.primary_browser.clone()); return Ok(TaskPlan { assistant_message: format!("I’m opening {}.", app_name), summary: format!("Open {}", app_name), goal: task_input.trim().to_string(), success_criteria: vec![format!("{} is frontmost", app_name)], steps: vec![ - step("resolve_app", "Resolve app", &format!("Resolve the requested app '{}'", app_name), "The requested app target is resolved"), - step("activate_app", "Activate app", &format!("Activate the requested app '{}'", app_name), "The requested app is frontmost"), + step( + "resolve_app", + "Resolve app", + &format!("Resolve the requested app '{}'", app_name), + "The requested app target is resolved", + ), + step( + "activate_app", + "Activate app", + &format!("Activate the requested app '{}'", app_name), + "The requested app is frontmost", + ), ], }); } @@ -215,21 +378,31 @@ impl PlannerProvider for HeuristicPlannerProvider { assistant_message: "I’m opening Chrome for that search.".to_string(), summary: "Open browser search".to_string(), goal: task_input.trim().to_string(), - success_criteria: vec!["Chrome is open".to_string(), "The target page or search results are visible".to_string()], + success_criteria: vec![ + "Chrome is open".to_string(), + "The target page or search results are visible".to_string(), + ], steps: vec![ - step("activate_chrome", "Open Chrome", "Resolve and activate Chrome", "Chrome is frontmost"), - step("browser_search", "Run search", "Open the relevant search or URL in Chrome", "The relevant page is visible"), + step( + "activate_chrome", + "Open Chrome", + "Resolve and activate Chrome", + "Chrome is frontmost", + ), + step( + "browser_search", + "Run search", + "Open the relevant search or URL in Chrome", + "The relevant page is visible", + ), ], }); } - Ok(TaskPlan { - assistant_message: format!("I understood '{}', but I need a more concrete desktop action.", task_input), - summary: "Clarify request".to_string(), - goal: task_input.trim().to_string(), - success_criteria: vec!["No desktop action required".to_string()], - steps: Vec::new(), - }) + Err(anyhow!( + "heuristic planner has no deterministic workflow for request '{}'", + task_input.trim() + )) } fn provider_name(&self) -> &'static str { @@ -242,7 +415,11 @@ pub struct HeuristicWorkerProvider; #[async_trait] impl WorkerProvider for HeuristicWorkerProvider { - async fn next_action(&self, state: &TaskState, _capabilities: &AutomationCapabilities) -> Result { + async fn next_action( + &self, + state: &TaskState, + _capabilities: &AutomationCapabilities, + ) -> Result { let Some(step) = state.current_step() else { return Ok(WorkerDecision { assistant_message: Some("The plan has no remaining steps.".to_string()), @@ -269,7 +446,11 @@ impl WorkerProvider for HeuristicWorkerProvider { } } else { WorkerAction::Tool { - request: tool("app_resolve", json!({ "app_name": app_name }), RiskLevel::Low), + request: tool( + "app_resolve", + json!({ "app_name": app_name }), + RiskLevel::Low, + ), } } } @@ -285,7 +466,11 @@ impl WorkerProvider for HeuristicWorkerProvider { reason: format!("{} did not become frontmost after activation", app_name), } } else { - let mut request = tool("app_activate", json!({ "app_name": app_name }), RiskLevel::Low); + let mut request = tool( + "app_activate", + json!({ "app_name": app_name }), + RiskLevel::Low, + ); request.expected_outcome = Some("requested app is frontmost".to_string()); WorkerAction::Tool { request } } @@ -300,17 +485,26 @@ impl WorkerProvider for HeuristicWorkerProvider { reason: "Chrome did not become frontmost after activation".to_string(), } } else { - let mut request = tool("app_activate", json!({ "app_name": "Google Chrome" }), RiskLevel::Low); + let mut request = tool( + "app_activate", + json!({ "app_name": "Google Chrome" }), + RiskLevel::Low, + ); request.expected_outcome = Some("Google Chrome is frontmost".to_string()); WorkerAction::Tool { request } } } "open_schoology" => { - if state.last_tool_name.as_deref() == Some("browser_assert") && observation_proved(state) { + if state.last_tool_name.as_deref() == Some("browser_assert") + && observation_proved(state) + { WorkerAction::AdvanceStep { note: "Schoology is open".to_string(), } - } else if matches!(state.last_tool_name.as_deref(), Some("browser_open") | Some("chrome_open_tab")) { + } else if matches!( + state.last_tool_name.as_deref(), + Some("browser_open") | Some("chrome_open_tab") + ) { let mut request = tool( "browser_assert", json!({ "url_contains": "schoology", "text_contains": "schoology" }), @@ -329,11 +523,16 @@ impl WorkerProvider for HeuristicWorkerProvider { } } "open_assignments" => { - if state.last_tool_name.as_deref() == Some("browser_assert") && observation_proved(state) { + if state.last_tool_name.as_deref() == Some("browser_assert") + && observation_proved(state) + { WorkerAction::AdvanceStep { note: "Opened the assignments view".to_string(), } - } else if matches!(state.last_tool_name.as_deref(), Some("browser_click") | Some("chrome_click")) { + } else if matches!( + state.last_tool_name.as_deref(), + Some("browser_click") | Some("chrome_click") + ) { let mut request = tool( "browser_assert", json!({ "url_contains": "assign", "text_contains": "assignment" }), @@ -352,17 +551,23 @@ impl WorkerProvider for HeuristicWorkerProvider { } } "open_biology" => { - if state.last_tool_name.as_deref() == Some("browser_assert") && observation_proved(state) { + if state.last_tool_name.as_deref() == Some("browser_assert") + && observation_proved(state) + { WorkerAction::AdvanceStep { note: "Opened the biology assignment".to_string(), } - } else if matches!(state.last_tool_name.as_deref(), Some("browser_click") | Some("chrome_click")) { + } else if matches!( + state.last_tool_name.as_deref(), + Some("browser_click") | Some("chrome_click") + ) { let mut request = tool( "browser_assert", json!({ "text_contains": "biology" }), RiskLevel::Low, ); - request.expected_outcome = Some("The biology assignment page is open".to_string()); + request.expected_outcome = + Some("The biology assignment page is open".to_string()); WorkerAction::Tool { request } } else { let mut request = tool( @@ -370,7 +575,8 @@ impl WorkerProvider for HeuristicWorkerProvider { json!({ "text": "biology" }), RiskLevel::Medium, ); - request.expected_outcome = Some("The latest biology assignment opens".to_string()); + request.expected_outcome = + Some("The latest biology assignment opens".to_string()); WorkerAction::Tool { request } } } @@ -400,13 +606,19 @@ impl WorkerProvider for HeuristicWorkerProvider { reason: "Mail did not become frontmost after activation".to_string(), } } else { - let mut request = tool("app_activate", json!({ "app_name": "Mail" }), RiskLevel::Low); + let mut request = tool( + "app_activate", + json!({ "app_name": "Mail" }), + RiskLevel::Low, + ); request.expected_outcome = Some("Mail is frontmost".to_string()); WorkerAction::Tool { request } } } "compose_mail" => { - if state.last_tool_name.as_deref() == Some("mail_compose") && observation_proved(state) { + if state.last_tool_name.as_deref() == Some("mail_compose") + && observation_proved(state) + { WorkerAction::Complete { message: "Mail draft prepared. I stopped before sending.".to_string(), } @@ -428,11 +640,11 @@ impl WorkerProvider for HeuristicWorkerProvider { risk: RiskLevel::Medium, requires_confirmation: false, target_identity: Some("com.apple.mail".to_string()), - expected_outcome: Some("A visible Mail draft exists with the requested details".to_string()), + expected_outcome: Some( + "A visible Mail draft exists with the requested details".to_string(), + ), }; - WorkerAction::Tool { - request, - } + WorkerAction::Tool { request } } } "activate_messages" => { @@ -445,13 +657,19 @@ impl WorkerProvider for HeuristicWorkerProvider { reason: "Messages did not become frontmost after activation".to_string(), } } else { - let mut request = tool("app_activate", json!({ "app_name": "Messages" }), RiskLevel::Low); + let mut request = tool( + "app_activate", + json!({ "app_name": "Messages" }), + RiskLevel::Low, + ); request.expected_outcome = Some("Messages is frontmost".to_string()); WorkerAction::Tool { request } } } "compose_message" => { - if state.last_tool_name.as_deref() == Some("messages_compose") && observation_proved(state) { + if state.last_tool_name.as_deref() == Some("messages_compose") + && observation_proved(state) + { WorkerAction::Complete { message: "Message draft prepared. I stopped before sending.".to_string(), } @@ -472,44 +690,50 @@ impl WorkerProvider for HeuristicWorkerProvider { risk: RiskLevel::Medium, requires_confirmation: false, target_identity: Some("com.apple.MobileSMS".to_string()), - expected_outcome: Some("A visible Messages draft exists for the requested recipient".to_string()), + expected_outcome: Some( + "A visible Messages draft exists for the requested recipient" + .to_string(), + ), }; - WorkerAction::Tool { - request, - } + WorkerAction::Tool { request } } } "browser_search" => { - if state.last_tool_name.as_deref() == Some("browser_assert") && observation_proved(state) { + if state.last_tool_name.as_deref() == Some("browser_assert") + && observation_proved(state) + { WorkerAction::Complete { message: "Opened the browser target.".to_string(), } - } else if matches!(state.last_tool_name.as_deref(), Some("browser_open") | Some("chrome_open_tab")) { + } else if matches!( + state.last_tool_name.as_deref(), + Some("browser_open") | Some("chrome_open_tab") + ) { let mut request = tool("browser_snapshot", json!({}), RiskLevel::Low); request.expected_outcome = Some("The relevant page is visible".to_string()); WorkerAction::Tool { request } } else if state.last_tool_name.as_deref() == Some("browser_snapshot") { - WorkerAction::Complete { - message: "Opened the browser target.".to_string(), + WorkerAction::Replan { + reason: + "Browser snapshot observed a page, but nothing verified that it satisfied the request" + .to_string(), } } else { - let search_url = if let Some(url) = state - .user_request - .split_whitespace() - .find(|token| token.starts_with("http://") || token.starts_with("https://")) - { - url.to_string() - } else { - format!( - "https://www.google.com/search?q={}", - state.user_request.replace(' ', "+") - ) - }; - let mut request = tool("browser_open", json!({ "url": search_url }), RiskLevel::Low); + let search_url = + if let Some(url) = state.user_request.split_whitespace().find(|token| { + token.starts_with("http://") || token.starts_with("https://") + }) { + url.to_string() + } else { + format!( + "https://www.google.com/search?q={}", + state.user_request.replace(' ', "+") + ) + }; + let mut request = + tool("browser_open", json!({ "url": search_url }), RiskLevel::Low); request.expected_outcome = Some("The browser target is visible".to_string()); - WorkerAction::Tool { - request, - } + WorkerAction::Tool { request } } } _ => { @@ -568,30 +792,30 @@ fn observation_proved(state: &TaskState) -> bool { } pub struct PlannerStack { - primary: Arc, - fallback: Arc, + providers: Vec>, } impl PlannerStack { - pub fn new(primary: Arc, fallback: Arc) -> Self { - Self { primary, fallback } + pub fn new(providers: Vec>) -> Self { + Self { providers } } } #[async_trait] impl PlannerProvider for PlannerStack { - async fn create_plan(&self, task_input: &str, capabilities: &AutomationCapabilities) -> Result { - match self.primary.create_plan(task_input, capabilities).await { - Ok(plan) => Ok(plan), - Err(primary_error) => match self.fallback.create_plan(task_input, capabilities).await { - Ok(plan) => Ok(plan), - Err(fallback_error) => bail!( - "planner failed with primary={} fallback={}", - primary_error, - fallback_error - ), - }, + async fn create_plan( + &self, + task_input: &str, + capabilities: &AutomationCapabilities, + ) -> Result { + let mut errors = Vec::new(); + for provider in &self.providers { + match provider.create_plan(task_input, capabilities).await { + Ok(plan) => return Ok(plan), + Err(error) => errors.push(format!("{}={}", provider.provider_name(), error)), + } } + bail!("planner failed across providers: {}", errors.join(" | ")) } fn provider_name(&self) -> &'static str { @@ -600,30 +824,30 @@ impl PlannerProvider for PlannerStack { } pub struct WorkerStack { - primary: Arc, - fallback: Arc, + providers: Vec>, } impl WorkerStack { - pub fn new(primary: Arc, fallback: Arc) -> Self { - Self { primary, fallback } + pub fn new(providers: Vec>) -> Self { + Self { providers } } } #[async_trait] impl WorkerProvider for WorkerStack { - async fn next_action(&self, state: &TaskState, capabilities: &AutomationCapabilities) -> Result { - match self.primary.next_action(state, capabilities).await { - Ok(decision) => Ok(decision), - Err(primary_error) => match self.fallback.next_action(state, capabilities).await { - Ok(decision) => Ok(decision), - Err(fallback_error) => bail!( - "worker failed with primary={} fallback={}", - primary_error, - fallback_error - ), - }, + async fn next_action( + &self, + state: &TaskState, + capabilities: &AutomationCapabilities, + ) -> Result { + let mut errors = Vec::new(); + for provider in &self.providers { + match provider.next_action(state, capabilities).await { + Ok(decision) => return Ok(decision), + Err(error) => errors.push(format!("{}={}", provider.provider_name(), error)), + } } + bail!("worker failed across providers: {}", errors.join(" | ")) } fn provider_name(&self) -> &'static str { @@ -737,6 +961,16 @@ mod tests { assert!(matches!(decision.action, WorkerAction::AdvanceStep { .. })); } + #[tokio::test] + async fn heuristic_planner_errors_on_unknown_requests() { + let provider = HeuristicPlannerProvider; + let result = provider + .create_plan("coordinate a vague autonomous workflow", &caps()) + .await; + + assert!(result.is_err()); + } + #[tokio::test] async fn compose_mail_requires_verified_draft_before_completion() { let worker = HeuristicWorkerProvider; @@ -744,7 +978,10 @@ mod tests { .create_plan("send an email to teacher about missing homework", &caps()) .await .unwrap(); - let mut state = TaskState::new("send an email to teacher about missing homework".to_string(), plan); + let mut state = TaskState::new( + "send an email to teacher about missing homework".to_string(), + plan, + ); state.active_step_index = 1; state.last_tool_name = Some("mail_compose".to_string()); state.last_observation = Some(crate::llm::schema::Observation { diff --git a/jarvis_rs/src/llm/schema.rs b/jarvis_rs/src/llm/schema.rs index 1538e0d..878ca9f 100644 --- a/jarvis_rs/src/llm/schema.rs +++ b/jarvis_rs/src/llm/schema.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +use crate::agent_core::{TaskJournal, VerificationResult}; use crate::automation::{AppTarget, BrowserTarget, ToolCallRequest, UiElementRef}; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -89,6 +90,10 @@ pub struct TaskState { pub cached_ui_targets: Vec, #[serde(default)] pub last_expected_outcome: Option, + #[serde(default)] + pub journal: TaskJournal, + #[serde(default)] + pub final_verification: Option, } impl TaskState { @@ -106,6 +111,8 @@ impl TaskState { active_browser: None, cached_ui_targets: Vec::new(), last_expected_outcome: None, + journal: TaskJournal::default(), + final_verification: None, }; state.sync_step_statuses(); state diff --git a/jarvis_rs/src/speech.rs b/jarvis_rs/src/speech.rs index c7ecdc5..bbe7522 100644 --- a/jarvis_rs/src/speech.rs +++ b/jarvis_rs/src/speech.rs @@ -1,8 +1,8 @@ use anyhow::{bail, Context, Result}; use async_trait::async_trait; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; -use std::io::ErrorKind; use std::fs; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::{Arc, Mutex}; @@ -136,7 +136,10 @@ pub fn record_wav(path: &Path, seconds: u64) -> Result<()> { Ok(()) } -fn write_samples_f32(writer: &Arc>>>>, data: &[f32]) { +fn write_samples_f32( + writer: &Arc>>>>, + data: &[f32], +) { if let Ok(mut writer) = writer.lock() { if let Some(writer) = writer.as_mut() { for sample in data { @@ -147,7 +150,10 @@ fn write_samples_f32(writer: &Arc>>>>, data: &[i16]) { +fn write_samples_i16( + writer: &Arc>>>>, + data: &[i16], +) { if let Ok(mut writer) = writer.lock() { if let Some(writer) = writer.as_mut() { for sample in data { @@ -157,7 +163,10 @@ fn write_samples_i16(writer: &Arc>>>>, data: &[u16]) { +fn write_samples_u16( + writer: &Arc>>>>, + data: &[u16], +) { if let Ok(mut writer) = writer.lock() { if let Some(writer) = writer.as_mut() { for sample in data { @@ -213,8 +222,8 @@ mod tests { #[test] fn whisper_cli_transcript_path_round_trip() { - let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../.tooling/models/ggml-tiny.en.bin"); + let model_path = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.tooling/models/ggml-tiny.en.bin"); let fixture_wav = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../input.wav"); let temp_wav = std::env::temp_dir().join(format!( "jarvis-speech-test-{}.wav", diff --git a/main.py b/main.py index e69de29..b41a0e2 100644 --- a/main.py +++ b/main.py @@ -0,0 +1,5 @@ +from voice_os.cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..46d81cb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +numpy>=2.2.0 +Pillow>=11.1.0 +PyAutoGUI>=0.9.54 +sounddevice>=0.5.1 +soundfile>=0.13.1 diff --git a/scripts/bootstrap_macos.sh b/scripts/bootstrap_macos.sh new file mode 100755 index 0000000..7aae1e3 --- /dev/null +++ b/scripts/bootstrap_macos.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TOOLING_DIR="$ROOT_DIR/.tooling" +MODEL_DIR="$TOOLING_DIR/models" +BIN_DIR="$TOOLING_DIR/bin" +VENV_DIR="$ROOT_DIR/.venv" +NODE_VERSION="v24.14.0" +NODE_DIST="node-${NODE_VERSION}-darwin-arm64" +NODE_TARBALL="$TOOLING_DIR/${NODE_DIST}.tar.gz" +NODE_DIR="$TOOLING_DIR/${NODE_DIST}" +NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.tar.gz" +QWEN_MODEL_PATH="$MODEL_DIR/qwen2.5-1.5b-instruct-q4_k_m.gguf" +QWEN_MODEL_URL="https://huggingface.co/bartowski/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf?download=true" +WHISPER_MODEL_PATH="$MODEL_DIR/ggml-tiny.en.bin" +WHISPER_MODEL_URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin" +CHROME_PATH="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" +OCR_BIN="$BIN_DIR/jarvis_ocr" + +log() { + printf '\n[%s] %s\n' "jarvis-bootstrap" "$1" +} + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Missing required command: $1" >&2 + exit 1 + fi +} + +need_cmd curl +need_cmd tar +need_cmd python3 +need_cmd swiftc + +mkdir -p "$TOOLING_DIR" "$MODEL_DIR" "$BIN_DIR" + +NODE_BIN="" +NPM_BIN="" + +if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then + NODE_BIN="$(command -v node)" + NPM_BIN="$(command -v npm)" + log "Using system Node at $NODE_BIN" +else + if [[ ! -x "$NODE_DIR/bin/node" ]]; then + log "Downloading local Node runtime into $NODE_DIR" + curl -L "$NODE_URL" -o "$NODE_TARBALL" + rm -rf "$NODE_DIR" + tar -xzf "$NODE_TARBALL" -C "$TOOLING_DIR" + fi + NODE_BIN="$NODE_DIR/bin/node" + NPM_BIN="$NODE_DIR/bin/npm" + log "Using local Node at $NODE_BIN" +fi + +export PATH="$(dirname "$NODE_BIN"):$PATH" + +if [[ ! -d "$VENV_DIR" ]]; then + log "Creating Python virtualenv" + python3 -m venv "$VENV_DIR" +fi + +log "Installing Python dependencies" +"$VENV_DIR/bin/python" -m pip install --upgrade pip +"$VENV_DIR/bin/python" -m pip install -r "$ROOT_DIR/requirements.txt" + +log "Installing browser sidecar dependencies" +"$NPM_BIN" ci --prefix "$ROOT_DIR/browser_sidecar" + +if [[ "${JARVIS_SETUP_LOCAL_MODELS:-1}" == "1" ]]; then + if [[ ! -f "$QWEN_MODEL_PATH" ]]; then + log "Downloading local Qwen planner model" + curl -L "$QWEN_MODEL_URL" -o "$QWEN_MODEL_PATH" + fi + + if [[ ! -f "$WHISPER_MODEL_PATH" ]]; then + log "Downloading local Whisper model" + curl -L "$WHISPER_MODEL_URL" -o "$WHISPER_MODEL_PATH" + fi +else + log "Skipping local model downloads because JARVIS_SETUP_LOCAL_MODELS=0" +fi + +log "Compiling OCR helper" +swiftc -framework Vision -framework AppKit "$ROOT_DIR/tools/ocr.swift" -o "$OCR_BIN" + +if [[ ! -x "$CHROME_PATH" ]]; then + echo "Google Chrome was not found at $CHROME_PATH" >&2 + echo "Install Chrome or set JARVIS_CHROME_EXECUTABLE before launching Jarvis." >&2 +fi + +cat <&2 + exit 1 +fi + +if [[ -d "$LOCAL_NODE_DIR" ]]; then + export PATH="$LOCAL_NODE_DIR:$PATH" + export JARVIS_NODE_BINARY="${JARVIS_NODE_BINARY:-$LOCAL_NODE_DIR/node}" +fi + +if [[ -f "$LOCAL_QWEN_MODEL" ]]; then + export JARVIS_PLANNER_MODEL_PATH="${JARVIS_PLANNER_MODEL_PATH:-$LOCAL_QWEN_MODEL}" + export JARVIS_WORKER_MODEL_PATH="${JARVIS_WORKER_MODEL_PATH:-$LOCAL_QWEN_MODEL}" +fi + +if [[ -f "$LOCAL_WHISPER_MODEL" ]]; then + export JARVIS_WHISPER_MODEL_PATH="${JARVIS_WHISPER_MODEL_PATH:-$LOCAL_WHISPER_MODEL}" +fi + +cd "$ROOT_DIR" +exec "$VENV_DIR/bin/python" main.py "$@" diff --git a/target/.rustc_info.json b/target/.rustc_info.json index f166dc1..55b347a 100644 --- a/target/.rustc_info.json +++ b/target/.rustc_info.json @@ -1 +1 @@ -{"rustc_fingerprint":2066474714701204283,"outputs":{"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.93.1 (01f6ddf75 2026-02-11)\nbinary: rustc\ncommit-hash: 01f6ddf7588f42ae2d7eb0a2f21d44e8e96674cf\ncommit-date: 2026-02-11\nhost: aarch64-apple-darwin\nrelease: 1.93.1\nLLVM version: 21.1.8\n","stderr":""},"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/Users/vishayagarwal/.rustup/toolchains/stable-aarch64-apple-darwin\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dit\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"dpb2\"\ntarget_feature=\"fcma\"\ntarget_feature=\"fhm\"\ntarget_feature=\"flagm\"\ntarget_feature=\"fp16\"\ntarget_feature=\"frintts\"\ntarget_feature=\"jsconv\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"paca\"\ntarget_feature=\"pacg\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rcpc2\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sb\"\ntarget_feature=\"sha2\"\ntarget_feature=\"sha3\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"apple\"\nunix\n","stderr":""}},"successes":{}} \ No newline at end of file +{"rustc_fingerprint":16998740634698707727,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/opt/homebrew/Cellar/rust/1.94.0\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dit\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"dpb2\"\ntarget_feature=\"fcma\"\ntarget_feature=\"fhm\"\ntarget_feature=\"flagm\"\ntarget_feature=\"fp16\"\ntarget_feature=\"frintts\"\ntarget_feature=\"jsconv\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"paca\"\ntarget_feature=\"pacg\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rcpc2\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sb\"\ntarget_feature=\"sha2\"\ntarget_feature=\"sha3\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"apple\"\nunix\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.94.0 (4a4ef493e 2026-03-02) (Homebrew)\nbinary: rustc\ncommit-hash: 4a4ef493e3a1488c6e321570238084b38948f6db\ncommit-date: 2026-03-02\nhost: aarch64-apple-darwin\nrelease: 1.94.0\nLLVM version: 21.1.8\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/tools/ocr.swift b/tools/ocr.swift new file mode 100644 index 0000000..f1d3844 --- /dev/null +++ b/tools/ocr.swift @@ -0,0 +1,85 @@ +import AppKit +import Foundation +import Vision + +struct OCRBox: Codable { + let text: String + let confidence: Float + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct OCRResult: Codable { + let width: Double + let height: Double + let fullText: String + let boxes: [OCRBox] +} + +enum OCRToolError: Error { + case invalidArguments + case imageLoadFailed +} + +func cgImage(from path: String) throws -> CGImage { + guard + let image = NSImage(contentsOfFile: path), + let tiff = image.tiffRepresentation, + let bitmap = NSBitmapImageRep(data: tiff), + let cgImage = bitmap.cgImage + else { + throw OCRToolError.imageLoadFailed + } + return cgImage +} + +func recognize(path: String) throws -> OCRResult { + let image = try cgImage(from: path) + let imageWidth = Double(image.width) + let imageHeight = Double(image.height) + + var observed: [OCRBox] = [] + let request = VNRecognizeTextRequest { request, _ in + guard let observations = request.results as? [VNRecognizedTextObservation] else { + return + } + for observation in observations { + guard let candidate = observation.topCandidates(1).first else { continue } + let rect = observation.boundingBox + observed.append( + OCRBox( + text: candidate.string, + confidence: candidate.confidence, + x: rect.origin.x * imageWidth, + y: (1.0 - rect.origin.y - rect.height) * imageHeight, + width: rect.width * imageWidth, + height: rect.height * imageHeight + ) + ) + } + } + request.recognitionLevel = .accurate + request.usesLanguageCorrection = true + + let handler = VNImageRequestHandler(cgImage: image) + try handler.perform([request]) + + let fullText = observed.map(\.text).joined(separator: "\n") + return OCRResult(width: imageWidth, height: imageHeight, fullText: fullText, boxes: observed) +} + +guard CommandLine.arguments.count == 2 else { + fputs("usage: jarvis_ocr \n", stderr) + exit(2) +} + +do { + let result = try recognize(path: CommandLine.arguments[1]) + let data = try JSONEncoder().encode(result) + FileHandle.standardOutput.write(data) +} catch { + fputs("OCR failure: \(error)\n", stderr) + exit(1) +} diff --git a/voice_os/__init__.py b/voice_os/__init__.py new file mode 100644 index 0000000..99d01b9 --- /dev/null +++ b/voice_os/__init__.py @@ -0,0 +1 @@ +"""Jarvis voice-first desktop agent.""" diff --git a/voice_os/agent.py b/voice_os/agent.py new file mode 100644 index 0000000..17c1a6b --- /dev/null +++ b/voice_os/agent.py @@ -0,0 +1,566 @@ +from __future__ import annotations + +import re +import time +from pathlib import Path + +from .browser import BrowserSidecarClient +from .config import JarvisConfig +from .control import GuiController +from .intent import IntentResolver +from .llm import LocalLlmClient +from .memory import MemoryStore +from .planner import Planner +from .runtime import ensure_dir, make_run_dir, write_json +from .schemas import ExecutionResult, Intent, Observation, PlanStep, StepRecord +from .verifier import Verifier +from .vision import VisionObserver +from .voice import Speaker, VoicePipeline + + +class DesktopAgent: + def __init__(self, config: JarvisConfig) -> None: + self.config = config + self.llm = LocalLlmClient(config) + self.intent_resolver = IntentResolver(self.llm) + self.planner = Planner() + self.controller = GuiController(config) + self.browser = BrowserSidecarClient(config) + self.observer = VisionObserver(config) + self.verifier = Verifier() + self.memory = MemoryStore(config.paths.memory_path) + self.voice = VoicePipeline(config.voice) + self.speaker = Speaker(config.voice) + self.last_spoken_text = "" + self.last_spoken_at = 0.0 + + def run_text_command(self, command: str, speak: bool = True) -> ExecutionResult: + run_dir = make_run_dir(self.config.paths.artifacts_dir, command) + ensure_dir(run_dir) + intent = self.intent_resolver.resolve(command) + plan = self.planner.create_plan(intent) + steps: list[StepRecord] = [] + + if intent.kind == "unsupported" or not plan.steps: + message = "I could not safely execute that request." + result = ExecutionResult( + command=command, + success=False, + message=message, + intent=intent, + plan=plan, + steps=steps, + artifacts_dir=run_dir, + ) + self._persist_result(result) + if speak: + self._speak(message) + return result + + if speak: + self._speak(intent.response_text or "Working on it.") + + success = True + message = plan.success_message + for index, step in enumerate(plan.steps, start=1): + record = self._execute_step(run_dir, index, step) + steps.append(record) + if not record.success: + success = False + message = record.summary + break + + if speak: + self._speak(message if success else f"I hit a problem. {message}") + + result = ExecutionResult( + command=command, + success=success, + message=message, + intent=intent, + plan=plan, + steps=steps, + artifacts_dir=run_dir, + ) + self._persist_result(result) + return result + + def run_voice_command( + self, + seconds: int | None = None, + speak: bool = True, + notify_on_empty: bool = True, + suppress_short_unsupported: bool = False, + ) -> ExecutionResult: + transcript = self.voice.listen_once(seconds=seconds).strip() + return self._run_transcript( + transcript, + speak=speak, + notify_on_empty=notify_on_empty, + suppress_short_unsupported=suppress_short_unsupported, + ) + + def run_audio_command( + self, + audio_path: Path, + speak: bool = False, + notify_on_empty: bool = False, + suppress_short_unsupported: bool = False, + ) -> ExecutionResult: + transcript = self.voice.transcribe(Path(audio_path)).strip() + return self._run_transcript( + transcript, + speak=speak, + notify_on_empty=notify_on_empty, + suppress_short_unsupported=suppress_short_unsupported, + ) + + def _run_transcript( + self, + transcript: str, + speak: bool, + notify_on_empty: bool, + suppress_short_unsupported: bool, + ) -> ExecutionResult: + if not transcript or transcript.upper() == "[BLANK_AUDIO]": + return self._voice_non_command_result( + label="blank-audio", + command="", + message="I didn't catch that. Please try again.", + speak=speak and notify_on_empty, + ) + if self._is_self_echo(transcript): + return self._voice_non_command_result( + label="assistant-echo", + command=transcript, + message="Ignoring assistant echo.", + speak=False, + ) + if suppress_short_unsupported: + intent = self.intent_resolver.resolve(transcript) + if intent.kind == "unsupported" and len(transcript.split()) <= 6: + return self._voice_non_command_result( + label="ignored-unsupported", + command=transcript, + message="Ignoring unsupported voice input.", + speak=False, + ) + return self.run_text_command(transcript, speak=speak) + + def _voice_non_command_result( + self, + label: str, + command: str, + message: str, + speak: bool, + ) -> ExecutionResult: + run_dir = make_run_dir(self.config.paths.artifacts_dir, label) + ensure_dir(run_dir) + intent = Intent(kind="unsupported", original_text=command, confidence=1.0) + plan = self.planner.create_plan(intent) + result = ExecutionResult( + command=command, + success=False, + message=message, + intent=intent, + plan=plan, + steps=[], + artifacts_dir=run_dir, + ) + self._persist_result(result) + if speak: + self._speak(message) + return result + + def _speak(self, text: str) -> None: + self.last_spoken_text = text + self.last_spoken_at = time.time() + self.speaker.speak(text) + + def _is_self_echo(self, transcript: str) -> bool: + if not self.last_spoken_text: + return False + if time.time() - self.last_spoken_at > 15: + return False + heard = self._normalize_phrase(transcript) + spoken = self._normalize_phrase(self.last_spoken_text) + if not heard or not spoken: + return False + return heard == spoken or heard in spoken or spoken in heard + + @staticmethod + def _normalize_phrase(text: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", text.lower()).strip() + + def _execute_step(self, run_dir, index: int, step: PlanStep) -> StepRecord: + label = f"step-{index:02d}" + attempts = 0 + last_summary = "Unknown failure" + last_verification: dict = {} + last_details: dict = {} + before_path = None + after_path = None + + while attempts < self.config.max_step_retries: + attempts += 1 + before_obs = self.observer.observe(run_dir, f"{label}-before-{attempts}") + before_path = str(before_obs.screenshot_path) + try: + action_details = self._perform_action(step, before_obs) + except Exception as exc: + last_summary = f"{step.description} raised {exc}" + last_details = {"error": str(exc)} + continue + time.sleep(self.config.settle_delay_seconds) + after_obs = self.observer.observe(run_dir, f"{label}-after-{attempts}") + after_path = str(after_obs.screenshot_path) + + success, verification = self._verify_step(step, after_obs, action_details) + if success: + return StepRecord( + description=step.description, + success=True, + summary="Verified", + before_screenshot=before_path, + after_screenshot=after_path, + verification=verification, + details=action_details, + ) + last_summary = f"{step.description} did not verify after attempt {attempts}" + last_verification = verification + last_details = action_details + + return StepRecord( + description=step.description, + success=False, + summary=last_summary, + before_screenshot=before_path, + after_screenshot=after_path, + verification=last_verification, + details=last_details, + ) + + def _perform_action(self, step: PlanStep, before_obs: Observation) -> dict: + if step.kind == "open_app": + self.controller.open_app(step.params["app_name"]) + return {"tool": "open_app", "app_name": step.params["app_name"]} + if step.kind == "hide_app": + self.controller.hide_app(step.params["app_name"]) + return {"tool": "hide_app", "app_name": step.params["app_name"]} + if step.kind == "textedit_new_document": + self.controller.new_textedit_document() + return {"tool": "textedit_new_document"} + if step.kind == "textedit_set_text": + return { + "tool": "textedit_set_text", + "text": step.params["text"], + "textedit_document_text": self.controller.set_textedit_text(step.params["text"]), + } + if step.kind == "press_hotkey": + if step.verification.get("frontmost_app"): + self.controller.activate_app(step.verification["frontmost_app"]) + self.controller.hotkey(*step.params["keys"]) + return {"tool": "press_hotkey", "keys": step.params["keys"]} + if step.kind == "type_text": + if step.verification.get("frontmost_app"): + self.controller.activate_app(step.verification["frontmost_app"]) + self.controller.type_text(step.params["text"]) + return { + "tool": "type_text", + "text": step.params["text"], + "captured_text": self.controller.copy_frontmost_text(), + } + if step.kind == "browser_attach": + state = self.browser.post("/browser/attach_or_launch") + return {"tool": "browser_attach", "browser_state": state} + if step.kind == "browser_open": + state = self.browser.post("/browser/open", {"url": step.params["url"]}) + return {"tool": "browser_open", "browser_state": state, "url": step.params["url"]} + if step.kind == "browser_assert": + state = self.browser.post("/browser/assert", step.verification) + return {"tool": "browser_assert", "browser_state": state} + if step.kind == "gmail_send_email": + return self._execute_gmail_send(step, before_obs) + if step.kind == "messages_send": + return self._execute_messages_send(step) + if step.kind == "spotify_play": + return self._execute_spotify_play(step, before_obs) + raise RuntimeError(f"Unsupported step kind: {step.kind}") + + def _verify_step(self, step: PlanStep, after_obs: Observation, action_details: dict) -> tuple[bool, dict]: + if step.kind.startswith("browser_"): + browser_state = action_details.get("browser_state") or self.browser.post("/browser/snapshot") + ok, details = self.verifier.verify_browser(browser_state, step.verification) + return ok, {"browser": details} + if step.kind == "gmail_send_email": + ok = bool(action_details.get("gmail_sent")) + details = { + "gmail_sent": ok, + "recipient_email": action_details.get("recipient_email"), + "subject": action_details.get("subject"), + "message_sent_detected": action_details.get("message_sent_detected"), + } + return ok, details + if step.kind == "messages_send": + ok = bool(action_details.get("message_sent")) + details = { + "message_sent": ok, + "recipient_handle": action_details.get("recipient_handle"), + "recipient_name": action_details.get("recipient_name"), + } + return ok, details + if step.kind == "spotify_play": + player = action_details.get("player_state") or {} + query = str(step.verification.get("query", "")) + combined = " ".join( + [ + str(player.get("state", "")), + str(player.get("track", "")), + str(player.get("artist", "")), + ] + ).lower() + ok = "playing" in combined and self._query_matches_player_state(query, player) + details = {"player_state": player, "query": query, "matched": ok} + return ok, details + ok, details = self.verifier.verify(after_obs, step.verification, action_details=action_details) + return ok, details + + def _execute_gmail_send(self, step: PlanStep, before_obs: Observation) -> dict: + recipient = self._resolve_contact(step.params) + recipient_email = step.params.get("recipient_email") or recipient.get("email") or "" + recipient_name = step.params.get("recipient_name") or recipient.get("name") or "your contact" + if not recipient_email: + raise RuntimeError(f"Could not resolve an email address for {recipient_name}.") + + subject, body = self._draft_message( + channel="email", + recipient_name=recipient_name, + source_command=str(step.params.get("source_command", "")), + subject=step.params.get("subject"), + body=step.params.get("body"), + ) + compose_url = self.controller.gmail_compose_url( + to_email=recipient_email, + subject=subject, + body=body, + account_hint=step.params.get("account_hint"), + ) + tab_info = self.controller.chrome_open_url(compose_url) + time.sleep(3) + + run_dir = before_obs.screenshot_path.parent + compose_obs = self.observer.observe(run_dir, f"gmail-compose-{int(time.time())}") + send_box = self.observer.find_text(compose_obs, "Send") + ocr_text = self._observation_text(compose_obs) + if send_box is None: + raise RuntimeError("Gmail compose view did not expose a Send button.") + compose_ready = recipient_email.lower() in compose_url.lower() and "mail.google.com" in str( + tab_info.get("url", "") + ).lower() + if not compose_ready and recipient_email.lower() not in ocr_text.lower(): + raise RuntimeError("Gmail compose view did not show the resolved recipient.") + self.controller.click_point(*send_box.center) + time.sleep(3) + sent_obs = self.observer.observe(run_dir, f"gmail-sent-{int(time.time())}") + sent_text = self._observation_text(sent_obs) + sent_detected = "message sent" in sent_text.lower() + return { + "tool": "gmail_send_email", + "recipient_name": recipient_name, + "recipient_email": recipient_email, + "subject": subject, + "body": body, + "compose_url": compose_url, + "tab_info": tab_info, + "compose_text": ocr_text, + "sent_text": sent_text, + "message_sent_detected": sent_detected, + "gmail_sent": sent_detected, + } + + def _execute_messages_send(self, step: PlanStep) -> dict: + recipient = self._resolve_contact(step.params) + recipient_name = step.params.get("recipient_name") or recipient.get("name") or "your contact" + recipient_handle = ( + step.params.get("recipient_phone") + or recipient.get("phone") + or recipient.get("email") + or "" + ) + if not recipient_handle: + raise RuntimeError(f"Could not resolve a Messages handle for {recipient_name}.") + + _subject, body = self._draft_message( + channel="message", + recipient_name=recipient_name, + source_command=str(step.params.get("source_command", "")), + body=step.params.get("body"), + ) + send_result = self.controller.send_imessage(recipient_handle, body) + return { + "tool": "messages_send", + "recipient_name": recipient_name, + "recipient_handle": send_result.get("handle") or recipient_handle, + "body": body, + "message_sent": bool(send_result.get("handle")), + } + + def _execute_spotify_play(self, step: PlanStep, before_obs: Observation) -> dict: + query = str(step.params.get("query", "")).strip() + if not query: + raise RuntimeError("Spotify command is missing a query.") + + self.controller.spotify_open_search(query) + time.sleep(2) + run_dir = before_obs.screenshot_path.parent + search_obs = self.observer.observe(run_dir, f"spotify-search-{int(time.time())}") + primary_target = self._spotify_primary_target(query) + result_box = self.observer.find_text(search_obs, primary_target) or self.observer.find_text( + search_obs, query + ) + if result_box is not None: + self.controller.click_point(*result_box.center) + time.sleep(0.5) + self.controller.hotkey("enter") + player_state = self.controller.spotify_play() + return { + "tool": "spotify_play", + "query": query, + "selected_target": primary_target, + "player_state": player_state, + } + + def _resolve_contact(self, params: dict) -> dict: + resolved: dict[str, str] = {} + if params.get("recipient_name"): + resolved = self.controller.resolve_contact(str(params["recipient_name"])) or {} + if params.get("recipient_email"): + resolved["email"] = str(params["recipient_email"]) + if params.get("recipient_phone"): + resolved["phone"] = str(params["recipient_phone"]) + if params.get("recipient_name"): + resolved.setdefault("name", str(params["recipient_name"])) + return resolved + + def _draft_message( + self, + channel: str, + recipient_name: str, + source_command: str, + subject: str | None = None, + body: str | None = None, + ) -> tuple[str, str]: + cleaned_subject = (subject or "").strip() + cleaned_body = (body or "").strip() + if cleaned_body and cleaned_body.lower() not in {"something nice", "say something nice"}: + return cleaned_subject or f"Note for {recipient_name}", cleaned_body + + llm_payload = self.llm.json_completion( + system_prompt=( + "Return only JSON with keys subject and body. " + "Write a short, warm, specific message for the named recipient. " + "Do not mention AI, automation, or that this is a test unless the command says so. " + "Do not invent birthdays, trips, deadlines, or other context that the user did not mention. " + "Keep subject under 8 words and body under 60 words." + ), + user_prompt=( + f"Channel: {channel}\n" + f"Recipient: {recipient_name}\n" + f"User command: {source_command}\n" + "Generate a ready-to-send message." + ), + ) + if llm_payload: + drafted_subject = str(llm_payload.get("subject") or cleaned_subject or f"Note for {recipient_name}").strip() + drafted_body = str(llm_payload.get("body") or "").strip() + if drafted_body: + return drafted_subject, drafted_body + + fallback_body = ( + f"Hey {recipient_name}, I just wanted to say I appreciate you and I hope your day is going really well." + ) + return cleaned_subject or f"Hello {recipient_name}", cleaned_body or fallback_body + + @staticmethod + def _observation_text(observation: Observation) -> str: + combined = [observation.screen_text] + combined.extend(box.text for box in observation.ocr_boxes) + return " ".join(part for part in combined if part).strip() + + @staticmethod + def _spotify_primary_target(query: str) -> str: + normalized = re.sub(r"\s+by\s+.+$", "", query, flags=re.IGNORECASE).strip() + return normalized or query + + @staticmethod + def _query_matches_player_state(query: str, player_state: dict) -> bool: + normalized_query = query.lower() + combined = " ".join( + [ + str(player_state.get("track", "")), + str(player_state.get("artist", "")), + ] + ).lower() + if not normalized_query: + return False + if normalized_query in combined: + return True + terms = [term for term in re.split(r"[^a-z0-9]+", normalized_query) if term and term not in {"play", "song", "spotify", "by"}] + if not terms: + return False + return sum(term in combined for term in terms) >= max(1, len(terms) - 1) + + def _persist_result(self, result: ExecutionResult) -> None: + payload = { + "command": result.command, + "success": result.success, + "message": result.message, + "intent": { + "kind": result.intent.kind, + "app_name": result.intent.app_name, + "query": result.intent.query, + "target_text": result.intent.target_text, + "target_app": result.intent.target_app, + "recipient_name": result.intent.recipient_name, + "recipient_email": result.intent.recipient_email, + "recipient_phone": result.intent.recipient_phone, + "subject": result.intent.subject, + "body": result.intent.body, + "account_hint": result.intent.account_hint, + "confidence": result.intent.confidence, + "used_llm": result.intent.used_llm, + "metadata": result.intent.metadata, + }, + "plan": { + "goal": result.plan.goal, + "success_message": result.plan.success_message, + "steps": [ + { + "kind": step.kind, + "description": step.description, + "params": step.params, + "verification": step.verification, + } + for step in result.plan.steps + ], + }, + "steps": [ + { + "description": step.description, + "success": step.success, + "summary": step.summary, + "before_screenshot": step.before_screenshot, + "after_screenshot": step.after_screenshot, + "verification": step.verification, + "details": step.details, + } + for step in result.steps + ], + } + write_json(result.artifacts_dir / "result.json", payload) + self.memory.append_execution( + command=result.command, + success=result.success, + message=result.message, + intent_kind=result.intent.kind, + ) diff --git a/voice_os/browser.py b/voice_os/browser.py new file mode 100644 index 0000000..794b65f --- /dev/null +++ b/voice_os/browser.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path +from urllib import error, request + +from .config import JarvisConfig + + +class BrowserSidecarClient: + def __init__(self, config: JarvisConfig) -> None: + self.config = config + self._process: subprocess.Popen[str] | None = None + + @property + def health_url(self) -> str: + return f"{self.config.browser.sidecar_endpoint}/health" + + def ensure_running(self, timeout_seconds: float = 20.0) -> None: + if self._healthy(): + return + env = { + **dict(__import__("os").environ), + "JARVIS_BROWSER_HOST": self.config.browser.sidecar_host, + "JARVIS_BROWSER_PORT": str(self.config.browser.sidecar_port), + "JARVIS_CHROME_EXECUTABLE": str(self.config.browser.chrome_executable), + "JARVIS_CHROME_DEBUG_PORT": str(self.config.browser.chrome_debug_port), + "JARVIS_CHROME_ATTACH_URL": self.config.browser.chrome_attach_url, + } + log_path = self.config.paths.artifacts_dir / "browser-sidecar.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + log_file = log_path.open("a", encoding="utf-8") + self._process = subprocess.Popen( + [str(self.config.browser.node_binary), str(self.config.browser.sidecar_entry)], + cwd=str(self.config.browser.sidecar_dir), + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + ) + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if self._healthy(): + return + time.sleep(0.5) + raise RuntimeError("Browser sidecar did not become healthy.") + + def _healthy(self) -> bool: + try: + with request.urlopen(self.health_url, timeout=2) as response: + return response.status == 200 + except Exception: + return False + + def post(self, route: str, payload: dict | None = None) -> dict: + self.ensure_running() + req = request.Request( + f"{self.config.browser.sidecar_endpoint}{route}", + data=json.dumps(payload or {}).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(req, timeout=45) as response: + return json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + if exc.code >= 500 and route != "/browser/reset": + reset_req = request.Request( + f"{self.config.browser.sidecar_endpoint}/browser/reset", + data=b"{}", + headers={"Content-Type": "application/json"}, + method="POST", + ) + with request.urlopen(reset_req, timeout=45): + pass + with request.urlopen(req, timeout=45) as response: + return json.loads(response.read().decode("utf-8")) + raise diff --git a/voice_os/cli.py b/voice_os/cli.py new file mode 100644 index 0000000..3f301ae --- /dev/null +++ b/voice_os/cli.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import argparse +import json +import sys +import time + +from .agent import DesktopAgent +from .config import JarvisConfig +from .runtime import make_run_dir, write_json + + +SCENARIO_COMMANDS = [ + "Open Google Chrome", + "Search YouTube for NBA highlights", + "Type hello world in a text editor", + "Open my personal account in Google Chrome and send an email to Vishay saying something nice", + "Send a text message to Vishay saying have a great day", + "Play Sticky by Drake on Spotify", +] + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Jarvis Voice OS") + subparsers = parser.add_subparsers(dest="command", required=False) + + run_parser = subparsers.add_parser("run", help="Run a single command") + run_parser.add_argument("--text", help="Text command to execute") + run_parser.add_argument("--voice", action="store_true", help="Capture a voice command") + run_parser.add_argument("--audio-file", help="Transcribe and execute a WAV command") + run_parser.add_argument("--seconds", type=int, default=None, help="Voice capture length") + run_parser.add_argument("--mute", action="store_true", help="Disable spoken responses") + + loop_parser = subparsers.add_parser("voice-loop", help="Continuous voice loop") + loop_parser.add_argument("--seconds", type=int, default=None, help="Voice capture length") + loop_parser.add_argument("--mute", action="store_true", help="Disable spoken responses") + + subparsers.add_parser("scenario-test", help="Run the required local test flows") + subparsers.add_parser( + "voice-scenario-test", + help="Run the required local test flows through synthesized spoken audio", + ) + return parser + + +def run_scenarios(agent: DesktopAgent, mute: bool = True) -> int: + results = [] + for command in SCENARIO_COMMANDS: + agent.controller.hide_app("jarvis_rs") + result = agent.run_text_command(command, speak=not mute) + print(f"[{'PASS' if result.success else 'FAIL'}] {command} -> {result.message}") + results.append( + { + "command": command, + "success": result.success, + "message": result.message, + "artifacts_dir": str(result.artifacts_dir), + } + ) + report_dir = make_run_dir(agent.config.paths.artifacts_dir, "scenario-test") + write_json(report_dir / "scenario-report.json", {"results": results}) + failures = [item for item in results if not item["success"]] + return 0 if not failures else 1 + + +def run_voice_scenarios(agent: DesktopAgent) -> int: + report_dir = make_run_dir(agent.config.paths.artifacts_dir, "voice-scenario-test") + audio_dir = report_dir / "audio" + audio_dir.mkdir(parents=True, exist_ok=True) + results = [] + + for index, command in enumerate(SCENARIO_COMMANDS, start=1): + agent.controller.hide_app("jarvis_rs") + audio_path = agent.voice.synthesize_test_wav(command, audio_dir / f"{index:02d}.wav") + result = agent.run_audio_command(audio_path, speak=False, notify_on_empty=False) + print(f"[{'PASS' if result.success else 'FAIL'}] {command} -> {result.message}") + results.append( + { + "command": command, + "audio_file": str(audio_path), + "success": result.success, + "message": result.message, + "artifacts_dir": str(result.artifacts_dir), + } + ) + + echo_audio = agent.voice.synthesize_test_wav( + "I could not safely execute that request.", + audio_dir / "echo.wav", + ) + agent.last_spoken_text = "I could not safely execute that request." + agent.last_spoken_at = time.time() + echo_result = agent.run_audio_command(echo_audio, speak=False, notify_on_empty=False) + results.append( + { + "command": "assistant echo suppression", + "audio_file": str(echo_audio), + "success": echo_result.message == "Ignoring assistant echo.", + "message": echo_result.message, + "artifacts_dir": str(echo_result.artifacts_dir), + } + ) + print( + f"[{'PASS' if echo_result.message == 'Ignoring assistant echo.' else 'FAIL'}] " + f"Assistant echo suppression -> {echo_result.message}" + ) + + write_json(report_dir / "voice-scenario-report.json", {"results": results}) + failures = [item for item in results if not item["success"]] + return 0 if not failures else 1 + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + config = JarvisConfig.from_env() + agent = DesktopAgent(config) + + if args.command in (None, "run"): + if getattr(args, "voice", False): + result = agent.run_voice_command( + seconds=args.seconds, + speak=not args.mute, + ) + elif getattr(args, "audio_file", None): + result = agent.run_audio_command( + audio_path=args.audio_file, + speak=not args.mute, + notify_on_empty=False, + ) + elif getattr(args, "text", None): + result = agent.run_text_command(args.text, speak=not args.mute) + else: + print("Use `run --text ...`, `run --voice`, or `run --audio-file ...`.", file=sys.stderr) + return 2 + print(json.dumps({"success": result.success, "message": result.message, "artifacts_dir": str(result.artifacts_dir)}, indent=2)) + return 0 if result.success else 1 + + if args.command == "voice-loop": + print("Jarvis voice loop is live. Press Ctrl+C to stop.") + try: + while True: + result = agent.run_voice_command( + seconds=args.seconds, + speak=not args.mute, + notify_on_empty=False, + suppress_short_unsupported=True, + ) + if result.message in { + "I didn't catch that. Please try again.", + "Ignoring assistant echo.", + "Ignoring unsupported voice input.", + }: + continue + print( + json.dumps( + { + "success": result.success, + "message": result.message, + "artifacts_dir": str(result.artifacts_dir), + } + ) + ) + time.sleep(0.5) + except KeyboardInterrupt: + return 0 + + if args.command == "scenario-test": + return run_scenarios(agent, mute=True) + + if args.command == "voice-scenario-test": + return run_voice_scenarios(agent) + + return 0 diff --git a/voice_os/config.py b/voice_os/config.py new file mode 100644 index 0000000..a15e496 --- /dev/null +++ b/voice_os/config.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path + + +def repo_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def _env_path(name: str, default: Path) -> Path: + value = os.environ.get(name) + return Path(value).expanduser() if value else default + + +@dataclass(slots=True) +class BrowserConfig: + sidecar_host: str + sidecar_port: int + sidecar_endpoint: str + sidecar_dir: Path + sidecar_entry: Path + node_binary: Path + chrome_executable: Path + chrome_debug_port: int + chrome_attach_url: str + + +@dataclass(slots=True) +class VoiceConfig: + whisper_model_path: Path | None + recording_path: Path + sample_rate: int + chunk_seconds: int + voice_name: str + elevenlabs_api_key: str | None + elevenlabs_voice_id: str | None + + +@dataclass(slots=True) +class ModelConfig: + enabled: bool + llama_server_binary: str + model_path: Path | None + endpoint: str + port: int + model_name: str + context_size: int + + +@dataclass(slots=True) +class PathsConfig: + artifacts_dir: Path + memory_path: Path + ocr_source: Path + ocr_binary: Path + + +@dataclass(slots=True) +class JarvisConfig: + root_dir: Path + browser: BrowserConfig + voice: VoiceConfig + model: ModelConfig + paths: PathsConfig + max_step_retries: int + action_delay_seconds: float + settle_delay_seconds: float + + @classmethod + def from_env(cls) -> "JarvisConfig": + root = repo_root() + tooling = root / ".tooling" + node_root = tooling / "node-v24.14.0-darwin-arm64" / "bin" + models_dir = tooling / "models" + artifacts_dir = _env_path("JARVIS_ARTIFACTS_DIR", root / "artifacts") + ocr_bin = tooling / "bin" / "jarvis_ocr" + sidecar_host = os.environ.get("JARVIS_BROWSER_HOST", "127.0.0.1") + sidecar_port = int(os.environ.get("JARVIS_BROWSER_PORT", "4317")) + model_port = int(os.environ.get("JARVIS_LLM_PORT", "8012")) + + browser = BrowserConfig( + sidecar_host=sidecar_host, + sidecar_port=sidecar_port, + sidecar_endpoint=os.environ.get( + "JARVIS_BROWSER_SIDECAR_ENDPOINT", + f"http://{sidecar_host}:{sidecar_port}", + ), + sidecar_dir=_env_path("JARVIS_BROWSER_SIDECAR_DIR", root / "browser_sidecar"), + sidecar_entry=_env_path( + "JARVIS_BROWSER_SIDECAR_ENTRY", root / "browser_sidecar" / "server.mjs" + ), + node_binary=_env_path("JARVIS_NODE_BINARY", node_root / "node"), + chrome_executable=_env_path( + "JARVIS_CHROME_EXECUTABLE", + Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), + ), + chrome_debug_port=int(os.environ.get("JARVIS_CHROME_DEBUG_PORT", "9222")), + chrome_attach_url=os.environ.get( + "JARVIS_CHROME_ATTACH_URL", "http://127.0.0.1:9222" + ), + ) + + whisper_path = os.environ.get("JARVIS_WHISPER_MODEL_PATH") + local_whisper = models_dir / "ggml-tiny.en.bin" + voice = VoiceConfig( + whisper_model_path=Path(whisper_path).expanduser() + if whisper_path + else (local_whisper if local_whisper.exists() else None), + recording_path=_env_path( + "JARVIS_RECORDING_PATH", Path(tempfile.gettempdir()) / "jarvis-input.wav" + ), + sample_rate=int(os.environ.get("JARVIS_SAMPLE_RATE", "16000")), + chunk_seconds=int(os.environ.get("JARVIS_CAPTURE_SECONDS", "4")), + voice_name=os.environ.get("JARVIS_VOICE_NAME", "Samantha"), + elevenlabs_api_key=os.environ.get("ELEVENLABS_API_KEY"), + elevenlabs_voice_id=os.environ.get("ELEVENLABS_VOICE_ID"), + ) + + model_path = os.environ.get("JARVIS_PLANNER_MODEL_PATH") or os.environ.get( + "JARVIS_WORKER_MODEL_PATH" + ) + local_model = models_dir / "qwen2.5-1.5b-instruct-q4_k_m.gguf" + resolved_model = ( + Path(model_path).expanduser() + if model_path + else (local_model if local_model.exists() else None) + ) + llm_enabled = os.environ.get("JARVIS_DISABLE_LLM", "0") != "1" + model = ModelConfig( + enabled=llm_enabled, + llama_server_binary=os.environ.get("JARVIS_LLAMA_SERVER_BINARY", "llama-server"), + model_path=resolved_model, + endpoint=os.environ.get("JARVIS_LLM_ENDPOINT", f"http://127.0.0.1:{model_port}/v1"), + port=model_port, + model_name=os.environ.get("JARVIS_LLM_MODEL", "jarvis-local"), + context_size=int(os.environ.get("JARVIS_MODEL_CONTEXT_SIZE", "4096")), + ) + + paths = PathsConfig( + artifacts_dir=artifacts_dir, + memory_path=artifacts_dir / "memory.json", + ocr_source=root / "tools" / "ocr.swift", + ocr_binary=ocr_bin, + ) + + return cls( + root_dir=root, + browser=browser, + voice=voice, + model=model, + paths=paths, + max_step_retries=int(os.environ.get("JARVIS_MAX_STEP_RETRIES", "3")), + action_delay_seconds=float(os.environ.get("JARVIS_ACTION_DELAY_SECONDS", "0.35")), + settle_delay_seconds=float(os.environ.get("JARVIS_SETTLE_DELAY_SECONDS", "1.0")), + ) diff --git a/voice_os/control.py b/voice_os/control.py new file mode 100644 index 0000000..9ac1803 --- /dev/null +++ b/voice_os/control.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path +from urllib.parse import quote + +import pyautogui + +from .config import JarvisConfig + +pyautogui.FAILSAFE = True +pyautogui.PAUSE = 0.1 + + +class GuiController: + def __init__(self, config: JarvisConfig) -> None: + self.config = config + + def _run_osascript(self, script: str, check: bool = True) -> str: + completed = subprocess.run( + ["osascript", "-e", script], + check=check, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + def open_app(self, app_name: str) -> None: + for _ in range(4): + try: + self.activate_app(app_name) + except Exception: + subprocess.run(["open", "-a", app_name], check=True) + if self.frontmost_app() and app_name.lower() in self.frontmost_app().lower(): + time.sleep(self.config.settle_delay_seconds) + return + time.sleep(0.4) + subprocess.run(["open", "-a", app_name], check=True) + self.activate_app(app_name) + time.sleep(self.config.settle_delay_seconds) + + def activate_app(self, app_name: str) -> None: + script = f'tell application "{app_name}" to activate' + self._run_osascript(script) + time.sleep(self.config.action_delay_seconds) + + def hide_app(self, app_name: str) -> None: + script = f'tell application "{app_name}" to set visible to false' + self._run_osascript(script, check=False) + time.sleep(self.config.action_delay_seconds) + + def frontmost_app(self) -> str | None: + script = ( + 'tell application "System Events" to get name of first application process ' + "whose frontmost is true" + ) + try: + return self._run_osascript(script) or None + except Exception: + return None + + def chrome_open_url(self, url: str) -> dict[str, str]: + escaped_url = url.replace("\\", "\\\\").replace('"', '\\"') + script = f''' + tell application "Google Chrome" + activate + if (count of windows) = 0 then + make new window + end if + set URL of active tab of front window to "{escaped_url}" + delay 2 + return "{{\\"title\\":\\"" & (title of active tab of front window) & "\\",\\"url\\":\\"" & (URL of active tab of front window) & "\\"}}" + end tell + ''' + raw = self._run_osascript(script) + time.sleep(self.config.settle_delay_seconds) + try: + return json.loads(raw) + except json.JSONDecodeError: + return {"title": "", "url": url} + + def chrome_active_tab(self) -> dict[str, str]: + script = ''' + tell application "Google Chrome" + if (count of windows) = 0 then + return "{\\"title\\":\\"\\",\\"url\\":\\"\\"}" + end if + return "{\\"title\\":\\"" & (title of active tab of front window) & "\\",\\"url\\":\\"" & (URL of active tab of front window) & "\\"}" + end tell + ''' + raw = self._run_osascript(script, check=False) + try: + return json.loads(raw) + except json.JSONDecodeError: + return {"title": "", "url": ""} + + def click_point(self, x: int, y: int) -> None: + pyautogui.moveTo(x, y, duration=0.15) + pyautogui.click() + time.sleep(self.config.action_delay_seconds) + + def hotkey(self, *keys: str) -> None: + try: + pyautogui.hotkey(*keys) + except Exception: + modifiers = [] + mapping = {"command": "command down", "shift": "shift down", "option": "option down"} + for key in keys[:-1]: + if key in mapping: + modifiers.append(mapping[key]) + final_key = keys[-1] + script = f'tell application "System Events" to keystroke "{final_key}" using {{{", ".join(modifiers)}}}' + subprocess.run(["osascript", "-e", script], check=True) + time.sleep(self.config.action_delay_seconds) + + def type_text(self, text: str) -> None: + try: + pyautogui.write(text, interval=0.02) + except Exception: + subprocess.run(["pbcopy"], input=text.encode("utf-8"), check=True) + script = ( + 'tell application "System Events" to keystroke "v" using {command down}' + ) + subprocess.run(["osascript", "-e", script], check=True) + time.sleep(self.config.action_delay_seconds) + + def new_textedit_document(self) -> None: + script = """ + tell application "TextEdit" + activate + make new document + end tell + """ + subprocess.run( + ["osascript", "-e", script], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + time.sleep(self.config.settle_delay_seconds) + + def set_textedit_text(self, text: str) -> str: + escaped = text.replace("\\", "\\\\").replace('"', '\\"') + script = f''' + tell application "TextEdit" + activate + if not (exists document 1) then + make new document + end if + set text of front document to "{escaped}" + return text of front document + end tell + ''' + output = subprocess.check_output( + ["osascript", "-e", script], + text=True, + ).strip() + time.sleep(self.config.settle_delay_seconds) + return output + + def copy_frontmost_text(self) -> str | None: + original_clipboard = None + try: + original_clipboard = subprocess.check_output(["pbpaste"]) + except Exception: + original_clipboard = None + + self.hotkey("command", "a") + self.hotkey("command", "c") + time.sleep(0.2) + try: + copied = subprocess.check_output(["pbpaste"], text=True) + except Exception: + copied = None + + if original_clipboard is not None: + try: + subprocess.run(["pbcopy"], input=original_clipboard, check=True) + except Exception: + pass + return copied + + def resolve_contact(self, query: str) -> dict[str, str] | None: + escaped_query = query.replace("\\", "\\\\").replace('"', '\\"') + script = f''' + tell application "Contacts" + set matches to every person whose name contains "{escaped_query}" + if (count of matches) = 0 then + return "" + end if + set targetPerson to item 1 of matches + set emailValue to "" + try + set emailValue to value of first email of targetPerson + end try + set phoneValue to "" + try + set phoneValue to value of first phone of targetPerson + end try + return "{{\\"name\\":\\"" & (name of targetPerson) & "\\",\\"email\\":\\"" & emailValue & "\\",\\"phone\\":\\"" & phoneValue & "\\"}}" + end tell + ''' + raw = self._run_osascript(script, check=False) + if not raw: + return None + try: + data = json.loads(raw) + except json.JSONDecodeError: + return None + if not any(data.values()): + return None + return data + + def gmail_compose_url( + self, + to_email: str, + subject: str, + body: str, + account_hint: str | None = None, + ) -> str: + account_index = "0" if (account_hint or "").lower() != "schoology" else "1" + return ( + f"https://mail.google.com/mail/u/{account_index}/?view=cm&fs=1" + f"&to={quote(to_email)}&su={quote(subject)}&body={quote(body)}" + ) + + def send_imessage(self, handle: str, text: str) -> dict[str, str]: + escaped_handle = handle.replace("\\", "\\\\").replace('"', '\\"') + escaped_text = text.replace("\\", "\\\\").replace('"', '\\"') + script = f''' + tell application "Messages" + set targetService to 1st service whose service type = iMessage + set targetBuddy to buddy "{escaped_handle}" of targetService + send "{escaped_text}" to targetBuddy + return "{{\\"handle\\":\\"" & (handle of targetBuddy) & "\\",\\"name\\":\\"" & (full name of targetBuddy) & "\\"}}" + end tell + ''' + raw = self._run_osascript(script) + time.sleep(self.config.settle_delay_seconds) + return json.loads(raw) + + def spotify_open_search(self, query: str) -> None: + subprocess.run([ "open", f"spotify:search:{quote(query)}"], check=True) + self.activate_app("Spotify") + time.sleep(self.config.settle_delay_seconds) + + def spotify_player_state(self) -> dict[str, str]: + script = ''' + tell application "Spotify" + set currentState to player state as text + set currentName to "" + set currentArtist to "" + try + set currentName to name of current track + set currentArtist to artist of current track + end try + return "{\\"state\\":\\"" & currentState & "\\",\\"track\\":\\"" & currentName & "\\",\\"artist\\":\\"" & currentArtist & "\\"}" + end tell + ''' + raw = self._run_osascript(script, check=False) + try: + return json.loads(raw) + except json.JSONDecodeError: + return {"state": "", "track": "", "artist": ""} + + def spotify_play(self) -> dict[str, str]: + script = ''' + tell application "Spotify" + play + delay 2 + set currentState to player state as text + set currentName to "" + set currentArtist to "" + try + set currentName to name of current track + set currentArtist to artist of current track + end try + return "{\\"state\\":\\"" & currentState & "\\",\\"track\\":\\"" & currentName & "\\",\\"artist\\":\\"" & currentArtist & "\\"}" + end tell + ''' + raw = self._run_osascript(script, check=False) + try: + return json.loads(raw) + except json.JSONDecodeError: + return {"state": "", "track": "", "artist": ""} diff --git a/voice_os/intent.py b/voice_os/intent.py new file mode 100644 index 0000000..57820a0 --- /dev/null +++ b/voice_os/intent.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import re + +from .llm import LocalLlmClient +from .schemas import Intent + + +class IntentResolver: + def __init__(self, llm: LocalLlmClient) -> None: + self.llm = llm + + def resolve(self, text: str) -> Intent: + normalized = " ".join(text.strip().split()) + heuristic = self._resolve_heuristic(normalized) + if heuristic.kind != "unsupported" and heuristic.confidence >= 0.95: + return heuristic + + llm_result = self._resolve_llm(normalized) + if llm_result is not None and llm_result.kind != "unsupported": + return llm_result + + return heuristic + + def _resolve_llm(self, text: str) -> Intent | None: + payload = self.llm.json_completion( + system_prompt=( + "You convert voice commands for a desktop agent into compact JSON. " + "Return only JSON with keys: kind, app_name, query, target_text, target_app, " + "recipient_name, recipient_email, recipient_phone, subject, body, account_hint, " + "response_text, confidence, metadata. " + "Supported kinds are open_app, browser_search, type_text, send_email, send_message, " + "play_spotify, workflow, unsupported. " + "For send_email and send_message, infer the recipient name if the user supplied one. " + "If the user asks to say something nice, draft a natural message body instead of repeating " + "the literal phrase 'something nice'. " + "If the user requests more than one distinct task in one sentence, return kind=workflow and " + "put an actions array inside metadata. Each action item should include its own kind and fields." + ), + user_prompt=text, + ) + if not payload: + return None + kind = str(payload.get("kind", "unsupported")) + if kind not in { + "open_app", + "browser_search", + "type_text", + "send_email", + "send_message", + "play_spotify", + "workflow", + "unsupported", + }: + kind = "unsupported" + return Intent( + kind=kind, + original_text=text, + app_name=payload.get("app_name"), + query=payload.get("query"), + target_text=payload.get("target_text"), + target_app=payload.get("target_app"), + recipient_name=payload.get("recipient_name"), + recipient_email=payload.get("recipient_email"), + recipient_phone=payload.get("recipient_phone"), + subject=payload.get("subject"), + body=payload.get("body"), + account_hint=payload.get("account_hint"), + response_text=payload.get("response_text"), + confidence=float(payload.get("confidence") or 0.35), + used_llm=True, + metadata=payload.get("metadata") or {}, + ) + + def _resolve_heuristic(self, text: str) -> Intent: + lower = text.lower() + + match = re.search(r"\bopen\s+(?:google\s+)?chrome\b", lower) + if match: + return Intent( + kind="open_app", + original_text=text, + app_name="Google Chrome", + response_text="Opening Google Chrome.", + confidence=0.99, + ) + + match = re.search(r"\bsearch\s+youtube\s+for\s+(.+)$", lower) + if match: + query = text[match.start(1) :].strip() + return Intent( + kind="browser_search", + original_text=text, + app_name="Google Chrome", + query=query, + target_app="YouTube", + response_text=f"Searching YouTube for {query}.", + confidence=0.99, + ) + + match = re.search(r"\btype\s+(.+?)\s+in\s+(?:a\s+)?text\s+editor\b", lower) + if match: + start, end = match.span(1) + target_text = text[start:end].strip().strip("\"'") + return Intent( + kind="type_text", + original_text=text, + app_name="TextEdit", + target_text=target_text, + target_app="TextEdit", + response_text=f"Typing {target_text} in TextEdit.", + confidence=0.99, + ) + + if any(token in lower for token in {"email", "gmail", "compose"}): + recipient_name = self._extract_recipient_name(text, lower) + if not recipient_name: + return Intent(kind="unsupported", original_text=text, confidence=0.0) + body = self._extract_message_body(text) or "something nice" + return Intent( + kind="send_email", + original_text=text, + app_name="Google Chrome", + target_app="Gmail", + recipient_name=recipient_name, + body=body, + account_hint=self._extract_account_hint(lower), + response_text=f"Sending an email to {recipient_name}.", + confidence=0.97, + ) + + if ("message" in lower or "text" in lower): + recipient_name = self._extract_recipient_name(text, lower) + if not recipient_name: + return Intent(kind="unsupported", original_text=text, confidence=0.0) + body = self._extract_message_body(text) or f"Hey {recipient_name}, hope you're having a great day." + return Intent( + kind="send_message", + original_text=text, + app_name="Messages", + target_app="Messages", + recipient_name=recipient_name, + body=body, + response_text=f"Sending a message to {recipient_name}.", + confidence=0.97, + ) + + match = re.search(r"\b(?:play|place)\s+(.+?)\s+on\s+spotify\b", text, re.IGNORECASE) + if match: + query = match.group(1).strip().strip("\"'") + return Intent( + kind="play_spotify", + original_text=text, + app_name="Spotify", + target_app="Spotify", + query=query, + response_text=f"Playing {query} on Spotify.", + confidence=0.98, + ) + + return Intent(kind="unsupported", original_text=text, confidence=0.0) + + @staticmethod + def _extract_recipient_name(text: str, lower: str) -> str | None: + match = re.search( + r"\bto\s+([a-zA-Z][a-zA-Z0-9 .'-]+?)(?=\s+(?:saying|about|with|from|using|in|on)\b|$)", + text, + re.IGNORECASE, + ) + if match: + return match.group(1).strip().strip(",.") + if "vishay" in lower: + return "Vishay" + return None + + @staticmethod + def _extract_message_body(text: str) -> str | None: + match = re.search( + r"\b(?:saying|say|that says|with the message)\s+(.+)$", + text, + re.IGNORECASE, + ) + if not match: + return None + body = match.group(1).strip().strip("\"'.") + body = re.sub(r"\s+from\s+my\s+(?:personal|school(?:ogy)?)\s+g?mail\b.*$", "", body, flags=re.IGNORECASE) + return body.strip() + + @staticmethod + def _extract_account_hint(lower: str) -> str | None: + if "schoology" in lower or "school" in lower: + return "schoology" + if "personal" in lower: + return "personal" + return None diff --git a/voice_os/llm.py b/voice_os/llm.py new file mode 100644 index 0000000..e4727be --- /dev/null +++ b/voice_os/llm.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path +from urllib import error, request + +from .config import JarvisConfig + + +class LocalLlmClient: + def __init__(self, config: JarvisConfig) -> None: + self.config = config + self._process: subprocess.Popen[str] | None = None + + def available(self) -> bool: + return ( + self.config.model.enabled + and self.config.model.model_path is not None + and self.config.model.model_path.exists() + ) + + def ensure_server(self, timeout_seconds: float = 20.0) -> bool: + if not self.available(): + return False + if self._healthcheck(): + return True + + cmd = [ + self.config.model.llama_server_binary, + "--host", + "127.0.0.1", + "--port", + str(self.config.model.port), + "-m", + str(self.config.model.model_path), + "-c", + str(self.config.model.context_size), + ] + log_path = self.config.paths.artifacts_dir / "llama-server.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + log_file = log_path.open("a", encoding="utf-8") + self._process = subprocess.Popen( + cmd, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + ) + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if self._healthcheck(): + return True + time.sleep(0.5) + return False + + def _healthcheck(self) -> bool: + try: + with request.urlopen( + f"{self.config.model.endpoint}/models", timeout=2 + ) as response: + return response.status == 200 + except Exception: + return False + + def json_completion(self, system_prompt: str, user_prompt: str) -> dict | None: + content = self._chat_completion(system_prompt=system_prompt, user_prompt=user_prompt) + if not content: + return None + try: + start = content.find("{") + end = content.rfind("}") + if start == -1 or end == -1: + return None + return json.loads(content[start : end + 1]) + except Exception: + return None + + def text_completion(self, system_prompt: str, user_prompt: str) -> str | None: + return self._chat_completion(system_prompt=system_prompt, user_prompt=user_prompt) + + def _chat_completion(self, system_prompt: str, user_prompt: str) -> str | None: + if not self.ensure_server(): + return None + + payload = { + "model": self.config.model.model_name, + "temperature": 0.1, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + } + req = request.Request( + f"{self.config.model.endpoint}/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(req, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + except (error.URLError, TimeoutError, json.JSONDecodeError): + return None + + try: + return str(body["choices"][0]["message"]["content"]).strip() + except Exception: + return None diff --git a/voice_os/memory.py b/voice_os/memory.py new file mode 100644 index 0000000..efa2e5a --- /dev/null +++ b/voice_os/memory.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from .runtime import ensure_dir, read_json, write_json + + +class MemoryStore: + def __init__(self, path: Path) -> None: + self.path = path + ensure_dir(path.parent) + + def append_run(self, record: dict[str, Any]) -> None: + payload = read_json(self.path, {"runs": []}) + payload.setdefault("runs", []).append(record) + write_json(self.path, payload) + + def append_execution( + self, command: str, success: bool, message: str, intent_kind: str + ) -> None: + self.append_run( + { + "command": command, + "success": success, + "message": message, + "intent_kind": intent_kind, + } + ) diff --git a/voice_os/planner.py b/voice_os/planner.py new file mode 100644 index 0000000..6a607b6 --- /dev/null +++ b/voice_os/planner.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from urllib.parse import quote_plus + +from .schemas import Intent, Plan, PlanStep + + +class Planner: + def create_plan(self, intent: Intent) -> Plan: + if intent.kind == "workflow": + actions = intent.metadata.get("actions") or [] + steps: list[PlanStep] = [] + messages: list[str] = [] + for index, action in enumerate(actions, start=1): + sub_intent = self._intent_from_action(action, intent.original_text) + sub_plan = self.create_plan(sub_intent) + for step in sub_plan.steps: + if not step.description.lower().startswith(f"task {index}:"): + step.description = f"Task {index}: {step.description}" + steps.extend(sub_plan.steps) + if sub_plan.success_message: + messages.append(sub_plan.success_message) + return Plan( + goal=intent.original_text, + success_message=" ".join(messages) if messages else "Completed the requested workflow.", + steps=steps, + ) + + if intent.kind == "open_app": + return Plan( + goal=intent.original_text, + success_message=f"{intent.app_name} is open and active.", + steps=[ + PlanStep( + kind="open_app", + description=f"Open {intent.app_name}", + params={"app_name": intent.app_name}, + verification={"frontmost_app": intent.app_name}, + ) + ], + ) + + if intent.kind == "browser_search": + url = f"https://www.youtube.com/results?search_query={quote_plus(intent.query or '')}" + return Plan( + goal=intent.original_text, + success_message=f"YouTube results for {intent.query} are visible.", + steps=[ + PlanStep( + kind="browser_attach", + description="Ensure browser session is ready", + verification={"browser_ready": True}, + ), + PlanStep( + kind="browser_open", + description=f"Open YouTube results for {intent.query}", + params={"url": url}, + verification={ + "url_contains": "youtube.com/results", + "title_contains": "youtube", + "text_contains": intent.query, + }, + ), + ], + ) + + if intent.kind == "type_text": + return Plan( + goal=intent.original_text, + success_message=f"Typed {intent.target_text} in TextEdit.", + steps=[ + PlanStep( + kind="hide_app", + description="Hide Google Chrome to prevent focus steal", + params={"app_name": "Google Chrome"}, + verification={}, + ), + PlanStep( + kind="textedit_new_document", + description="Create a new document", + params={}, + verification={}, + ), + PlanStep( + kind="textedit_set_text", + description=f"Set TextEdit document text to {intent.target_text}", + params={"text": intent.target_text or ""}, + verification={ + "textedit_document_contains": intent.target_text or "", + }, + ), + ], + ) + + if intent.kind == "send_email": + recipient = intent.recipient_name or intent.recipient_email or "your contact" + return Plan( + goal=intent.original_text, + success_message=f"Sent an email to {recipient}.", + steps=[ + PlanStep( + kind="gmail_send_email", + description=f"Send a Gmail message to {recipient}", + params={ + "recipient_name": intent.recipient_name, + "recipient_email": intent.recipient_email, + "subject": intent.subject, + "body": intent.body, + "account_hint": intent.account_hint, + "source_command": intent.original_text, + }, + verification={"gmail_sent": True}, + ) + ], + ) + + if intent.kind == "send_message": + recipient = intent.recipient_name or intent.recipient_phone or "your contact" + return Plan( + goal=intent.original_text, + success_message=f"Sent a message to {recipient}.", + steps=[ + PlanStep( + kind="messages_send", + description=f"Send a Messages text to {recipient}", + params={ + "recipient_name": intent.recipient_name, + "recipient_phone": intent.recipient_phone, + "body": intent.body, + "source_command": intent.original_text, + }, + verification={"message_sent": True}, + ) + ], + ) + + if intent.kind == "play_spotify": + return Plan( + goal=intent.original_text, + success_message=f"Started Spotify playback for {intent.query}.", + steps=[ + PlanStep( + kind="spotify_play", + description=f"Play {intent.query} on Spotify", + params={"query": intent.query or ""}, + verification={"spotify_playing": True, "query": intent.query or ""}, + ) + ], + ) + + return Plan( + goal=intent.original_text, + success_message="I could not safely map that command to a supported action.", + steps=[], + ) + + @staticmethod + def _intent_from_action(action: dict, original_text: str) -> Intent: + return Intent( + kind=action.get("kind", "unsupported"), + original_text=original_text, + app_name=action.get("app_name"), + query=action.get("query"), + target_text=action.get("target_text"), + target_app=action.get("target_app"), + recipient_name=action.get("recipient_name"), + recipient_email=action.get("recipient_email"), + recipient_phone=action.get("recipient_phone"), + subject=action.get("subject"), + body=action.get("body"), + account_hint=action.get("account_hint"), + response_text=action.get("response_text"), + confidence=float(action.get("confidence") or 0.6), + used_llm=True, + metadata=action.get("metadata") or {}, + ) diff --git a/voice_os/runtime.py b/voice_os/runtime.py new file mode 100644 index 0000000..7792b32 --- /dev/null +++ b/voice_os/runtime.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import json +import re +from datetime import datetime +from pathlib import Path +from typing import Any + + +def slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")[:80] or "run" + + +def utc_stamp() -> str: + return datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + + +def ensure_dir(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def make_run_dir(base: Path, label: str) -> Path: + return ensure_dir(base / f"{utc_stamp()}-{slugify(label)}") + + +def write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def read_json(path: Path, default: Any) -> Any: + if not path.exists(): + return default + return json.loads(path.read_text(encoding="utf-8")) diff --git a/voice_os/schemas.py b/voice_os/schemas.py new file mode 100644 index 0000000..5e17f97 --- /dev/null +++ b/voice_os/schemas.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + + +IntentKind = Literal[ + "open_app", + "browser_search", + "type_text", + "send_email", + "send_message", + "play_spotify", + "workflow", + "unsupported", +] +StepKind = Literal[ + "open_app", + "hide_app", + "textedit_new_document", + "textedit_set_text", + "browser_attach", + "browser_open", + "browser_assert", + "press_hotkey", + "type_text", + "chrome_open_url", + "gmail_send_email", + "messages_send", + "spotify_play", +] + + +@dataclass(slots=True) +class OcrBox: + text: str + confidence: float + x: float + y: float + width: float + height: float + + @property + def center(self) -> tuple[int, int]: + return ( + int(round(self.x + (self.width / 2))), + int(round(self.y + (self.height / 2))), + ) + + +@dataclass(slots=True) +class Observation: + screenshot_path: Path + frontmost_app: str | None + window_title: str | None + screen_text: str + ocr_boxes: list[OcrBox] = field(default_factory=list) + browser_state: dict[str, Any] | None = None + + +@dataclass(slots=True) +class Intent: + kind: IntentKind + original_text: str + app_name: str | None = None + query: str | None = None + target_text: str | None = None + target_app: str | None = None + recipient_name: str | None = None + recipient_email: str | None = None + recipient_phone: str | None = None + subject: str | None = None + body: str | None = None + account_hint: str | None = None + response_text: str | None = None + confidence: float = 0.0 + used_llm: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class PlanStep: + kind: StepKind + description: str + params: dict[str, Any] = field(default_factory=dict) + verification: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class Plan: + goal: str + success_message: str + steps: list[PlanStep] + + +@dataclass(slots=True) +class StepRecord: + description: str + success: bool + summary: str + before_screenshot: str | None = None + after_screenshot: str | None = None + verification: dict[str, Any] = field(default_factory=dict) + details: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class ExecutionResult: + command: str + success: bool + message: str + intent: Intent + plan: Plan + steps: list[StepRecord] + artifacts_dir: Path diff --git a/voice_os/verifier.py b/voice_os/verifier.py new file mode 100644 index 0000000..dd2e48e --- /dev/null +++ b/voice_os/verifier.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from .schemas import Observation + + +class Verifier: + def verify(self, observation: Observation, rules: dict, action_details: dict | None = None) -> tuple[bool, dict]: + details: dict[str, object] = {} + + if "frontmost_app" in rules: + expected = str(rules["frontmost_app"]).lower() + actual = (observation.frontmost_app or "").lower() + matched = expected in actual or actual in expected + details["frontmost_app"] = { + "expected": rules["frontmost_app"], + "actual": observation.frontmost_app, + "matched": matched, + } + if not matched: + return False, details + + if "screen_text_contains" in rules: + expected = str(rules["screen_text_contains"]).lower() + haystack = observation.screen_text.lower() + matched = expected in haystack + details["screen_text_contains"] = { + "expected": rules["screen_text_contains"], + "matched": matched, + } + if not matched: + return False, details + + if "captured_text_contains" in rules: + captured_text = str((action_details or {}).get("captured_text", "")) + expected = str(rules["captured_text_contains"]).lower() + matched = expected in captured_text.lower() + details["captured_text_contains"] = { + "expected": rules["captured_text_contains"], + "captured_text": captured_text, + "matched": matched, + } + if not matched: + return False, details + + if "textedit_document_contains" in rules: + captured_text = str((action_details or {}).get("textedit_document_text", "")) + expected = str(rules["textedit_document_contains"]).lower() + matched = expected in captured_text.lower() + details["textedit_document_contains"] = { + "expected": rules["textedit_document_contains"], + "captured_text": captured_text, + "matched": matched, + } + if not matched: + return False, details + + return True, details + + def verify_browser(self, browser_state: dict, rules: dict) -> tuple[bool, dict]: + url = str(browser_state.get("url", "")) + title = str(browser_state.get("title", "")) + text = str(browser_state.get("visibleText", "")) + details = {"url": url, "title": title} + if "browser_ready" in rules and not browser_state.get("ready"): + return False, {**details, "ready": browser_state.get("ready")} + if "url_contains" in rules and str(rules["url_contains"]).lower() not in url.lower(): + return False, details + if "title_contains" in rules and str(rules["title_contains"]).lower() not in title.lower(): + return False, details + if "text_contains" in rules: + expected = str(rules["text_contains"]).lower() + combined = " ".join([url.lower(), title.lower(), text.lower()]) + tokens = [token for token in expected.split() if token] + matched = expected in combined or ( + tokens and sum(token in combined for token in tokens) >= max(1, len(tokens) - 1) + ) + details["text_contains"] = { + "expected": rules["text_contains"], + "matched": matched, + } + if not matched: + return False, details + return True, details diff --git a/voice_os/vision.py b/voice_os/vision.py new file mode 100644 index 0000000..3927e33 --- /dev/null +++ b/voice_os/vision.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from difflib import SequenceMatcher +import json +import subprocess +from pathlib import Path + +from .config import JarvisConfig +from .runtime import ensure_dir +from .schemas import Observation, OcrBox + + +class VisionObserver: + def __init__(self, config: JarvisConfig) -> None: + self.config = config + + def ensure_ocr_binary(self) -> None: + binary = self.config.paths.ocr_binary + if binary.exists(): + return + ensure_dir(binary.parent) + subprocess.run( + [ + "swiftc", + "-framework", + "Vision", + "-framework", + "AppKit", + str(self.config.paths.ocr_source), + "-o", + str(binary), + ], + check=True, + ) + + def observe(self, run_dir: Path, label: str) -> Observation: + self.ensure_ocr_binary() + screenshot_path = run_dir / f"{label}.png" + subprocess.run(["screencapture", "-x", str(screenshot_path)], check=True) + ocr_payload = subprocess.check_output( + [str(self.config.paths.ocr_binary), str(screenshot_path)], + text=True, + ) + data = json.loads(ocr_payload) + boxes = [ + OcrBox( + text=item["text"], + confidence=float(item["confidence"]), + x=float(item["x"]), + y=float(item["y"]), + width=float(item["width"]), + height=float(item["height"]), + ) + for item in data.get("boxes", []) + ] + return Observation( + screenshot_path=screenshot_path, + frontmost_app=self.frontmost_app(), + window_title=self.front_window_title(), + screen_text=data.get("full_text", ""), + ocr_boxes=boxes, + ) + + def frontmost_app(self) -> str | None: + script = ( + 'tell application "System Events" to get name of first application process ' + "whose frontmost is true" + ) + try: + return ( + subprocess.check_output(["osascript", "-e", script], text=True).strip() or None + ) + except Exception: + return None + + def front_window_title(self) -> str | None: + script = """ + tell application "System Events" + try + set frontApp to first application process whose frontmost is true + tell front window of frontApp + return value of attribute "AXTitle" + end tell + on error + return "" + end try + end tell + """ + try: + title = subprocess.check_output(["osascript", "-e", script], text=True).strip() + return title or None + except Exception: + return None + + def find_text(self, observation: Observation, query: str) -> OcrBox | None: + normalized_query = " ".join(query.lower().split()) + best_box: OcrBox | None = None + best_score = 0.0 + for box in observation.ocr_boxes: + candidate = " ".join(box.text.lower().split()) + if not candidate: + continue + score = self._score_candidate(candidate, normalized_query) + if score > best_score: + best_score = score + best_box = box + return best_box if best_score >= 0.55 else None + + @staticmethod + def _score_candidate(candidate: str, query: str) -> float: + if candidate == query: + return 1.0 + if query in candidate: + return 0.92 + if candidate in query and len(candidate) >= 4: + return 0.78 + query_tokens = set(query.split()) + candidate_tokens = set(candidate.split()) + if not query_tokens: + return 0.0 + overlap = len(query_tokens & candidate_tokens) / len(query_tokens) + ratio = SequenceMatcher(None, candidate, query).ratio() + return max(overlap, ratio * 0.9) diff --git a/voice_os/voice.py b/voice_os/voice.py new file mode 100644 index 0000000..7bfc1c3 --- /dev/null +++ b/voice_os/voice.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +import subprocess +import tempfile +import wave +from pathlib import Path +from urllib import request + +import numpy +import sounddevice + +from .config import VoiceConfig + + +class VoicePipeline: + def __init__(self, config: VoiceConfig) -> None: + self.config = config + + def record_wav(self, seconds: int | None = None) -> Path: + seconds = seconds or self.config.chunk_seconds + frames = int(seconds * self.config.sample_rate) + audio = sounddevice.rec( + frames, + samplerate=self.config.sample_rate, + channels=1, + dtype="int16", + ) + sounddevice.wait() + path = self.config.recording_path + path.parent.mkdir(parents=True, exist_ok=True) + with wave.open(str(path), "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(self.config.sample_rate) + wav_file.writeframes(numpy.asarray(audio).tobytes()) + return path + + def transcribe(self, wav_path: Path) -> str: + if self.config.whisper_model_path is None: + raise RuntimeError("Whisper model is not configured.") + prefix = wav_path.with_suffix("") + txt_path = prefix.with_suffix(".txt") + if txt_path.exists(): + txt_path.unlink() + cmd = [ + "whisper-cli", + "-m", + str(self.config.whisper_model_path), + "-f", + str(wav_path), + "-ng", + "-nt", + "-otxt", + "-of", + str(prefix), + ] + subprocess.run( + cmd, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return txt_path.read_text(encoding="utf-8").strip() + + def listen_once(self, seconds: int | None = None) -> str: + wav_path = self.record_wav(seconds=seconds) + return self.transcribe(wav_path) + + def synthesize_test_wav(self, text: str, output_path: Path) -> Path: + output_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory() as temp_dir: + aiff_path = Path(temp_dir) / "prompt.aiff" + subprocess.run( + ["/usr/bin/say", "-v", self.config.voice_name, "-o", str(aiff_path), text], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + subprocess.run( + [ + "/usr/bin/afconvert", + "-f", + "WAVE", + "-d", + "LEI16@16000", + "-c", + "1", + str(aiff_path), + str(output_path), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return output_path + + +class Speaker: + def __init__(self, config: VoiceConfig) -> None: + self.config = config + + def speak(self, text: str) -> None: + if self.config.elevenlabs_api_key and self.config.elevenlabs_voice_id: + if self._speak_elevenlabs(text): + return + self._speak_macos(text) + + def _speak_elevenlabs(self, text: str) -> bool: + payload = json.dumps( + { + "text": text, + "model_id": "eleven_flash_v2_5", + "voice_settings": {"stability": 0.35, "similarity_boost": 0.75}, + } + ).encode("utf-8") + req = request.Request( + f"https://api.elevenlabs.io/v1/text-to-speech/{self.config.elevenlabs_voice_id}", + data=payload, + headers={ + "Content-Type": "application/json", + "xi-api-key": self.config.elevenlabs_api_key, + "Accept": "audio/mpeg", + }, + method="POST", + ) + out_path = Path("/tmp/jarvis-tts.mp3") + try: + with request.urlopen(req, timeout=20) as response: + out_path.write_bytes(response.read()) + subprocess.run(["afplay", str(out_path)], check=True) + return True + except Exception: + return False + + def _speak_macos(self, text: str) -> None: + subprocess.run( + ["/usr/bin/say", "-v", self.config.voice_name, text], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + )