refactor for demo actions (spotify, gmail, texting)

.dockerignore

@@ -0,0 +1,6 @@
+.git
+target
+Downloads
+scenario_artifacts
+.tooling/browser-profile
+jarvis_rs/target

.env.jarvis.example

@@ -0,0 +1,6 @@
+JARVIS_FALLBACK_BASE_URL=https://api.openai.com/v1
+JARVIS_FALLBACK_MODEL=your-model-name
+JARVIS_FALLBACK_API_KEY=your-api-key
+ELEVENLABS_API_KEY=your-elevenlabs-api-key
+ELEVENLABS_VOICE_ID=your-elevenlabs-voice-id
+JARVIS_DISABLE_LLM=0

.gitignore

@@ -1,5 +1,9 @@
 /target
 .tooling/
+.venv/
+__pycache__/
+artifacts/
+scenario_artifacts/
 *.wav
 transcript.txt
 browser_sidecar/node_modules/

Dockerfile

@@ -0,0 +1,66 @@
+FROM rust:1.86-bookworm AS builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    libasound2-dev \
+    libgl1-mesa-dev \
+    libglu1-mesa-dev \
+    libudev-dev \
+    libwayland-dev \
+    libx11-dev \
+    libxcursor-dev \
+    libxinerama-dev \
+    libxkbcommon-dev \
+    libxrandr-dev \
+    nodejs \
+    npm \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY Cargo.toml Cargo.lock ./
+COPY jarvis_rs/Cargo.toml jarvis_rs/Cargo.toml
+COPY browser_sidecar/package.json browser_sidecar/package-lock.json browser_sidecar/
+RUN mkdir -p jarvis_rs/src && printf "fn main() {}\n" > jarvis_rs/src/main.rs
+RUN printf "pub fn placeholder() {}\n" > jarvis_rs/src/lib.rs
+RUN cargo build --release --bin agent_core_server || true
+
+COPY . .
+RUN npm ci --prefix browser_sidecar
+RUN cargo build --release --bin agent_core_server --bin scenario_runner
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    libasound2 \
+    libgl1 \
+    libglu1-mesa \
+    libudev1 \
+    libwayland-client0 \
+    libx11-6 \
+    libxcursor1 \
+    libxinerama1 \
+    libxkbcommon0 \
+    libxrandr2 \
+    nodejs \
+    npm \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY --from=builder /app/target/release/agent_core_server /usr/local/bin/agent_core_server
+COPY --from=builder /app/target/release/scenario_runner /usr/local/bin/scenario_runner
+COPY --from=builder /app/browser_sidecar /app/browser_sidecar
+
+ENV JARVIS_AGENT_CORE_BIND=0.0.0.0:8788
+
+EXPOSE 8788
+
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -fsS http://127.0.0.1:8788/health || exit 1
+
+CMD ["/usr/local/bin/agent_core_server"]

README.md

@@ -1 +1,67 @@
-*in progress*
+# Jarvis Voice OS
+
+Voice-first desktop agent for macOS with:
+
+- local Whisper STT via `whisper-cli`
+- browser control through a Playwright sidecar
+- screen observation through screenshot + Vision OCR
+- grounded desktop actions via `pyautogui`
+- planner/executor/verifier loop with deterministic demo-safe planning and optional local Qwen intent help
+
+## Quickstart
+
+Use the real repo root:
+
+```bash
+cd /Users/agarwal/coding/jarvis/jarvis
+./scripts/bootstrap_macos.sh
+./scripts/run_jarvis.sh run --voice
+```
+
+For a text-triggered command:
+
+```bash
+./scripts/run_jarvis.sh run --text "Search YouTube for NBA highlights"
+```
+
+For continuous voice capture:
+
+```bash
+./scripts/run_jarvis.sh voice-loop
+```
+
+To run the local required test suite:
+
+```bash
+./scripts/doctor.sh
+```
+
+## Architecture
+
+```text
+Microphone / text input
+  -> Whisper STT
+  -> Intent resolver (heuristics first, local LLM fallback)
+  -> Planner
+  -> Observe screen/browser
+  -> Execute tool
+  -> Re-observe
+  -> Verify
+  -> TTS response
+
+Tools:
+  open_app
+  browser_attach_or_launch
+  browser_open
+  browser_assert
+  press_hotkey
+  type_text
+  read_screen
+```
+
+## Notes
+
+- `bootstrap_macos.sh` creates `.venv`, installs Python and browser sidecar dependencies, compiles the OCR helper, and downloads local models if needed.
+- Grant Accessibility, Screen Recording, Automation, and Microphone permissions when macOS prompts.
+- Google Chrome is the default browser target. Set `JARVIS_CHROME_EXECUTABLE` if Chrome lives somewhere else.
+- If `ELEVENLABS_API_KEY` and `ELEVENLABS_VOICE_ID` are set, Jarvis will use ElevenLabs TTS; otherwise it falls back to macOS `say`.

browser_sidecar/server.mjs

@@ -15,6 +15,48 @@ let context = null;
 let page = null;
 let mode = 'uninitialized';
 
+function isRecoverableSessionError(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const normalized = message.toLowerCase();
+  return normalized.includes('target page, context or browser has been closed')
+    || normalized.includes('browser has been closed')
+    || normalized.includes('browser context closed')
+    || normalized.includes('context has been closed')
+    || normalized.includes('target closed')
+    || normalized.includes('connection closed')
+    || normalized.includes('browser disconnected')
+    || normalized.includes('has been collected to prevent unbounded heap growth');
+}
+
+async function clearSessionState() {
+  page = null;
+  context = null;
+  browser = null;
+  mode = 'uninitialized';
+}
+
+async function closeSessionState() {
+  const activeContext = context;
+  const activeBrowser = browser;
+  await clearSessionState();
+
+  if (activeContext) {
+    try {
+      await activeContext.close();
+    } catch (_error) {
+      // Best-effort cleanup so we can relaunch a fresh session.
+    }
+  }
+
+  if (activeBrowser && typeof activeBrowser.isConnected === 'function' && activeBrowser.isConnected()) {
+    try {
+      await activeBrowser.close();
+    } catch (_error) {
+      // CDP-attached browsers may reject close; keep cleanup best-effort.
+    }
+  }
+}
+
 function jsonResponse(res, statusCode, payload) {
   res.writeHead(statusCode, { 'content-type': 'application/json' });
   res.end(JSON.stringify(payload));
@@ -33,49 +75,72 @@ async function readBody(req) {
 }
 
 async function getActivePage() {
-  if (page && !page.isClosed()) {
-    return page;
-  }
+  try {
+    if (page && !page.isClosed()) {
+      return page;
+    }
 
-  if (!context) {
-    return null;
-  }
+    if (!context) {
+      return null;
+    }
+
+    const pages = context.pages();
+    if (pages.length > 0) {
+      page = pages[0];
+      return page;
+    }
 
-  const pages = context.pages();
-  if (pages.length > 0) {
-    page = pages[0];
+    page = await context.newPage();
     return page;
+  } catch (error) {
+    if (isRecoverableSessionError(error)) {
+      await clearSessionState();
+      return null;
+    }
+    throw error;
   }
-
-  page = await context.newPage();
-  return page;
 }
 
 async function describePage() {
-  const activePage = await getActivePage();
-  if (!activePage) {
+  try {
+    const activePage = await getActivePage();
+    if (!activePage) {
+      return {
+        ok: true,
+        ready: false,
+        mode,
+        browserName: 'Google Chrome',
+      };
+    }
+
+    const visibleText = await activePage.evaluate(() => {
+      const text = document.body ? document.body.innerText || '' : '';
+      return text.replace(/\s+/g, ' ').trim().slice(0, 6000);
+    });
+
     return {
       ok: true,
-      ready: false,
+      ready: true,
       mode,
       browserName: 'Google Chrome',
+      url: activePage.url(),
+      title: await activePage.title(),
+      visibleText,
     };
+  } catch (error) {
+    if (isRecoverableSessionError(error)) {
+      await clearSessionState();
+      return {
+        ok: false,
+        ready: false,
+        recoverable: true,
+        error: error instanceof Error ? error.message : String(error),
+        mode,
+        browserName: 'Google Chrome',
+      };
+    }
+    throw error;
   }
-
-  const visibleText = await activePage.evaluate(() => {
-    const text = document.body ? document.body.innerText || '' : '';
-    return text.replace(/\s+/g, ' ').trim().slice(0, 6000);
-  });
-
-  return {
-    ok: true,
-    ready: true,
-    mode,
-    browserName: 'Google Chrome',
-    url: activePage.url(),
-    title: await activePage.title(),
-    visibleText,
-  };
 }
 
 async function tryAttach() {
@@ -93,6 +158,7 @@ async function tryAttach() {
     page = await getActivePage();
     return true;
   } catch (_error) {
+    await clearSessionState();
     return false;
   }
 }
@@ -114,9 +180,11 @@ async function launchDedicatedProfile() {
 }
 
 async function ensureSession() {
-  if (context) {
-    await getActivePage();
-    return describePage();
+  if (context || browser || page) {
+    const existingPage = await getActivePage();
+    if (existingPage) {
+      return describePage();
+    }
   }
 
   if (!fs.existsSync(chromeExecutable)) {
@@ -131,6 +199,11 @@ async function ensureSession() {
   return describePage();
 }
 
+async function recoverSession() {
+  await closeSessionState();
+  return ensureSession();
+}
+
 async function browserOpen(payload) {
   const targetUrl = payload.url;
   if (!targetUrl) {
@@ -213,6 +286,29 @@ async function browserExtractText(payload) {
   };
 }
 
+async function browserEval(payload) {
+  const activePage = await getActivePage();
+  if (!payload.script) {
+    throw new Error('browser_eval requires script');
+  }
+
+  const evaluation = await activePage.evaluate(
+    ({ script, arg }) => {
+      const fn = new Function('arg', script);
+      return fn(arg ?? null);
+    },
+    {
+      script: String(payload.script),
+      arg: payload.arg ?? null,
+    }
+  );
+
+  return {
+    ...(await describePage()),
+    evaluation,
+  };
+}
+
 async function browserAssert(payload) {
   const snapshot = await describePage();
   const url = String(snapshot.url || '').toLowerCase();
@@ -246,6 +342,9 @@ async function dispatch(pathname, payload) {
   if (pathname === '/browser/attach_or_launch') {
     return ensureSession();
   }
+  if (pathname === '/browser/reset') {
+    return recoverSession();
+  }
   await ensureSession();
   if (pathname === '/browser/open') {
     return browserOpen(payload);
@@ -265,6 +364,9 @@ async function dispatch(pathname, payload) {
   if (pathname === '/browser/extract_text') {
     return browserExtractText(payload);
   }
+  if (pathname === '/browser/eval') {
+    return browserEval(payload);
+  }
   if (pathname === '/browser/assert') {
     return browserAssert(payload);
   }