From ea54b1185c4de80f898bcc95993b5b874281c2ff Mon Sep 17 00:00:00 2001 From: DiegoSGbr Date: Sat, 24 Oct 2026 13:03:31 -0300 Subject: [PATCH 01/25] feat(browser): open a session browser the agent drives A card menu item opens one Chromium window for that session. The agent drives the same page with the browser_* tools and lich browser, and a navigation returns once the document is interactive instead of waiting out the load event. Co-authored-by: Cursor --- CHANGELOG.md | 5 + README.es.md | 1 + README.ja.md | 1 + README.ko.md | 1 + README.md | 1 + README.pt-BR.md | 1 + README.ru.md | 1 + README.zh-CN.md | 1 + api/browser.go | 112 +++++++ api/testdata/contract.golden | 97 +++++- docs/browser-tab.md | 39 +++ docs/ceilings.md | 19 ++ docs/cli.md | 26 ++ go.mod | 7 + go.sum | 18 + internal/browser/actions.go | 180 ++++++++++ internal/browser/browser.go | 294 +++++++++++++++++ internal/browser/browser_test.go | 380 ++++++++++++++++++++++ internal/browser/chrome.go | 260 +++++++++++++++ internal/browser/chrome_test.go | 32 ++ internal/browser/fake_test.go | 156 +++++++++ internal/browser/keys.go | 68 ++++ internal/browser/keys_test.go | 41 +++ internal/browser/page.go | 82 +++++ internal/browser/pageinfo.go | 83 +++++ internal/browser/pageinfo_test.go | 51 +++ internal/browser/rpc.go | 119 +++++++ internal/browser/url.go | 34 ++ internal/browser/url_test.go | 37 +++ internal/chromium/candidates_darwin.go | 13 + internal/chromium/candidates_unix.go | 13 + internal/chromium/candidates_unix_test.go | 42 +++ internal/chromium/candidates_windows.go | 13 + internal/chromium/find.go | 102 ++++++ internal/chromium/find_test.go | 125 +++++++ internal/cli/browser.go | 283 ++++++++++++++++ internal/cli/browser_test.go | 104 ++++++ internal/cli/cli.go | 2 + internal/cli/help.go | 7 + internal/cli/mcp.go | 7 +- internal/cli/mcp_browser.go | 332 +++++++++++++++++++ internal/cli/mcp_test.go | 48 ++- internal/prompt/en.go | 4 +- internal/prompt/es.go | 4 +- internal/prompt/ja.go | 4 +- internal/prompt/ko.go | 4 +- internal/prompt/pt_br.go | 4 +- internal/prompt/ru.go | 4 +- internal/prompt/zh_cn.go | 4 +- internal/routes/browser.go | 27 ++ internal/routes/routes_test.go | 7 + main.go | 22 +- native/backend/project.go | 7 + native/cmd/lich-native/cardmenu.go | 2 + native/cmd/lich-native/cardmenu_test.go | 2 +- native/cmd/lich-native/main.go | 14 + native/cmd/lich-native/sidebar.go | 15 +- native/i18n/locales/en/sidebar.json | 2 + native/i18n/locales/es/sidebar.json | 2 + native/i18n/locales/ja/sidebar.json | 2 + native/i18n/locales/ko/sidebar.json | 2 + native/i18n/locales/pt-br/sidebar.json | 2 + native/i18n/locales/ru/sidebar.json | 2 + native/i18n/locales/zh-cn/sidebar.json | 2 + 64 files changed, 3333 insertions(+), 43 deletions(-) create mode 100644 api/browser.go create mode 100644 docs/browser-tab.md create mode 100644 internal/browser/actions.go create mode 100644 internal/browser/browser.go create mode 100644 internal/browser/browser_test.go create mode 100644 internal/browser/chrome.go create mode 100644 internal/browser/chrome_test.go create mode 100644 internal/browser/fake_test.go create mode 100644 internal/browser/keys.go create mode 100644 internal/browser/keys_test.go create mode 100644 internal/browser/page.go create mode 100644 internal/browser/pageinfo.go create mode 100644 internal/browser/pageinfo_test.go create mode 100644 internal/browser/rpc.go create mode 100644 internal/browser/url.go create mode 100644 internal/browser/url_test.go create mode 100644 internal/chromium/candidates_darwin.go create mode 100644 internal/chromium/candidates_unix.go create mode 100644 internal/chromium/candidates_unix_test.go create mode 100644 internal/chromium/candidates_windows.go create mode 100644 internal/chromium/find.go create mode 100644 internal/chromium/find_test.go create mode 100644 internal/cli/browser.go create mode 100644 internal/cli/browser_test.go create mode 100644 internal/cli/mcp_browser.go create mode 100644 internal/routes/browser.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 432af6c7c..aa85c83c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- A session card's menu opens a browser for that session, above Open in. It is a Chromium window of its own, and the session's agent drives the same page: open a URL, read it, click, type, press a key, go back and forward, scroll and screenshot, from `browser_*` tools or from `lich browser`. You watch and use that window while the agent does. Closing the card closes the browser. opencode's agent reaches it through `lich browser`; its own tool list does not include `browser_*`. + ### Fixed +- A session browser's open, reload, back and forward return once the page is interactive, instead of waiting for every image and tracker to finish. That wait held the agent's next action for as long as the load event took, up to 30 seconds, after the page was already on screen. A screenshot is the viewport being watched. - Branch, changes and pull request on the cards and the footer catch up within a second of a commit, checkout or edit made outside a turn, where they could lag up to three seconds, and selecting a card reads its checkout at once instead of showing what the last read found. - Hiding the pull request list in Pulls slides it away at once, where the list stayed on screen until the mouse moved or Pulls next checked GitHub. Showing it again no longer waits a frame either. - Kiro CLI sessions follow `KIRO_HOME`. With it set, lich read Kiro's conversations, cost and context from `~/.kiro` while Kiro kept them under `KIRO_HOME`, so they showed nothing and History listed none; the agent that carries lich's hooks was installed where Kiro never looked, and a sandboxed session was not given the directory it writes into. diff --git a/README.es.md b/README.es.md index 8a39cd786..c5afc6141 100644 --- a/README.es.md +++ b/README.es.md @@ -147,6 +147,7 @@ porque lee lo que cada CLI deja escrito, y no hay dos que registren lo mismo. | Una tarea que entregas a otra sesión informa "sin respuesta" en cuanto ese agente termina su turno sin responder | sí | sí | sí | sí | sí | no, espera al plazo | sí | sí | | Los mensajes propios de lich al agente (tareas entregadas, resultados listos, avisos) en el idioma de prompt que elijas | sí | sí | sí | sí | sí | sí | sí | sí | | Se le indica al iniciarse, en el idioma de prompt, que reparta el trabajo como sesiones de lich en vez de subagentes ocultos | sí | no | no | no | sí | no | no | no | +| La tarjeta de la sesión abre un navegador que el agente conduce, en la misma ventana que ves | sí | sí | sí | la tarjeta lo abre; el agente usa `lich browser`, no las tools `browser_*` | sí | sí | sí | sí | Ninguna de estas diferencias es un descuido, y en ningún caso lich se guarda algo que el CLI informe; [`docs/ceilings.md`](docs/ceilings.md) cuenta qué se diff --git a/README.ja.md b/README.ja.md index 2f9ed6319..553807c29 100644 --- a/README.ja.md +++ b/README.ja.md @@ -136,6 +136,7 @@ lich は各 CLI が書き出す情報を読んでいて、何を書き出すか | 別のセッションに渡したタスクは、相手のエージェントが返信せずにターンを終えた時点で、すぐに「unanswered」として戻ってくる | あり | あり | あり | あり | あり | なし（期限まで待つ） | あり | あり | | lich からエージェントへの独自メッセージ（渡されたタスク、結果の準備完了、通知）を、選択したプロンプト言語で送る | あり | あり | あり | あり | あり | あり | あり | あり | | 起動時に、見えないサブエージェントではなく lich セッションに作業を振り分けるよう、プロンプト言語で指示される | あり | なし | なし | なし | あり | なし | なし | なし | +| セッションのカードが、エージェントが操作するブラウザを開く。見ているのは同じウィンドウ | あり | あり | あり | カードは開く。エージェントは `browser_*` ではなく `lich browser` を使う | あり | あり | あり | あり | 「なし」はどれも意図したもので、CLI が出している情報を lich が隠しているわけではありません。 それぞれについて何を検証したかは [`docs/ceilings.md`](docs/ceilings.md) にまとめています。 diff --git a/README.ko.md b/README.ko.md index 488614bcd..b8779e7c5 100644 --- a/README.ko.md +++ b/README.ko.md @@ -136,6 +136,7 @@ lich로 할 수 있는 일: | 다른 세션에 넘긴 작업은 그 에이전트가 답하지 않고 턴을 끝내는 즉시 "unanswered"로 보고됨 | 예 | 예 | 예 | 예 | 예 | 아니요, 마감 시각까지 기다림 | 예 | 예 | | lich가 에이전트에게 보내는 메시지(넘긴 작업, 준비된 결과, 알림)가 선택한 프롬프트 언어로 전달됨 | 예 | 예 | 예 | 예 | 예 | 예 | 예 | 예 | | 세션을 시작할 때 숨은 서브에이전트 대신 lich 세션으로 작업을 나누라는 안내를 프롬프트 언어로 받음 | 예 | 아니요 | 아니요 | 아니요 | 예 | 아니요 | 아니요 | 아니요 | +| 세션 카드가 에이전트가 조작하는 브라우저를 연다. 보는 창은 같다 | 예 | 예 | 예 | 카드는 연다. 에이전트는 `browser_*`가 아니라 `lich browser`를 쓴다 | 예 | 예 | 예 | 예 | 빈칸은 모두 의도된 것이며, CLI가 보고하는 정보를 lich가 숨기는 경우는 없습니다. 빈칸마다 무엇을 측정했는지는 [`docs/ceilings.md`](docs/ceilings.md)에 정리되어 있습니다. diff --git a/README.md b/README.md index 7a7935ebb..2340abaab 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ reads what each CLI writes down and no two of them write down the same things. | A task you hand another session reports back "unanswered" as soon as that agent ends its turn without replying | yes | yes | yes | yes | yes | no, waits for the deadline | yes | yes | | lich's own messages to the agent (handed tasks, results ready, notices) in the prompt language you pick | yes | yes | yes | yes | yes | yes | yes | yes | | Told at spawn, in the prompt language, to fan work out as lich sessions rather than hidden subagents | yes | no | no | no | yes | no | no | no | +| The session card opens a browser its agent drives, in the same window you watch | yes | yes | yes | the card opens it; the agent uses `lich browser`, not `browser_*` | yes | yes | yes | yes | Every gap is deliberate, none of them is lich withholding something the CLI reports, and [`docs/ceilings.md`](docs/ceilings.md) says what was measured behind diff --git a/README.pt-BR.md b/README.pt-BR.md index e38c0cfaf..4be6d4649 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -148,6 +148,7 @@ mesmas coisas. | Uma tarefa passada para outra sessão volta como "sem resposta" assim que aquele agente encerra o turno sem responder | sim | sim | sim | sim | sim | não, espera o prazo | sim | sim | | As mensagens do próprio lich para o agente (tarefas delegadas, resultados prontos, avisos) no idioma de prompt que você escolher | sim | sim | sim | sim | sim | sim | sim | sim | | Instruído na criação, no idioma de prompt, a distribuir o trabalho em sessões do lich em vez de subagentes ocultos | sim | não | não | não | sim | não | não | não | +| O card da sessão abre um navegador que o agente dirige, na mesma janela que você acompanha | sim | sim | sim | o card abre; o agente usa `lich browser`, não as tools `browser_*` | sim | sim | sim | sim | Toda lacuna é proposital: nenhuma é o lich escondendo algo que a CLI informa, e o [`docs/ceilings.md`](docs/ceilings.md) diz o que foi medido em cada uma. diff --git a/README.ru.md b/README.ru.md index 5269a6fc8..b6babe62e 100644 --- a/README.ru.md +++ b/README.ru.md @@ -147,6 +147,7 @@ | Задача, переданная другой сессии, сразу возвращается как «без ответа», если тот агент завершил ход и не ответил | да | да | да | да | да | нет, ждёт истечения срока | да | да | | Сообщения самого lich агенту (переданные задачи, готовые результаты, уведомления) на выбранном вами языке промптов | да | да | да | да | да | да | да | да | | При запуске агент получает указание на языке промптов раздавать работу сессиям lich, а не скрытым субагентам | да | нет | нет | нет | да | нет | нет | нет | +| Карточка сессии открывает браузер, которым управляет агент, в том же окне, которое вы видите | да | да | да | карточка открывает; агент вызывает `lich browser`, а не инструменты `browser_*` | да | да | да | да | Все пробелы намеренные: ни в одном из них lich не скрывает того, что сообщает CLI, а что именно измерялось за каждым, описано в diff --git a/README.zh-CN.md b/README.zh-CN.md index e3516a816..7ada61fed 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -124,6 +124,7 @@ | 交给另一个会话的任务，对方一轮结束却没回复时，立即报告“未回复” | 有 | 有 | 有 | 有 | 有 | 无，要等到截止时间 | 有 | 有 | | lich 发给智能体的消息（派出的任务、结果已就绪、通知）使用你选的提示词语言 | 有 | 有 | 有 | 有 | 有 | 有 | 有 | 有 | | 启动时用提示词语言告诉智能体，要把工作分派成 lich 会话，而不是看不见的子智能体 | 有 | 无 | 无 | 无 | 有 | 无 | 无 | 无 | +| 会话卡片打开一个由智能体操作的浏览器，你看着的是同一个窗口 | 有 | 有 | 有 | 卡片会打开；智能体用 `lich browser`，而不是 `browser_*` 工具 | 有 | 有 | 有 | 有 | 每一处差异都是有意为之，没有一处是 CLI 报告了而 lich 不显示；每一处背后实测了什么，见 [`docs/ceilings.md`](docs/ceilings.md)。 diff --git a/api/browser.go b/api/browser.go new file mode 100644 index 000000000..a030372f4 --- /dev/null +++ b/api/browser.go @@ -0,0 +1,112 @@ +package api + +var ( + BrowserShow = define[SessionIDOptions, BrowserHandle]("browser.Show") + BrowserOpen = define[BrowserOpenOptions, BrowserHandle]("browser.Open") + BrowserInfo = define[SessionIDOptions, BrowserPageInfo]("browser.Info") + BrowserClick = define[BrowserClickOptions, None]("browser.Click") + BrowserType = define[BrowserTypeOptions, None]("browser.Type") + BrowserPress = define[BrowserPressOptions, None]("browser.Press") + BrowserScreenshot = define[BrowserScreenshotOptions, string]("browser.Screenshot") + BrowserNavigate = define[BrowserOpenOptions, BrowserHandle]("browser.Navigate") + BrowserReload = define[SessionIDOptions, None]("browser.Reload") + BrowserBack = define[SessionIDOptions, None]("browser.Back") + BrowserForward = define[SessionIDOptions, None]("browser.Forward") + BrowserScroll = define[BrowserScrollOptions, None]("browser.Scroll") + BrowserList = define[SessionIDOptions, []BrowserHandle]("browser.List") + BrowserClose = define[SessionIDOptions, None]("browser.Close") +) + +// BrowserHandle is one session's browser: the id later calls take, and enough +// for a tool result to say which window it is. +type BrowserHandle struct { + ID string `json:"id"` + URL string `json:"url"` + Title string `json:"title"` + Owner string `json:"owner"` + Headed bool `json:"headed"` +} + +// BrowserOpenOptions loads a URL in the session's browser, creating a headless +// one when the card has not opened a window yet. +type BrowserOpenOptions struct { + ID string `json:"id"` + URL string `json:"url"` +} + +// BrowserTarget picks an element. Index is the one page-info stamped; +// selector is a CSS query; x/y are viewport coordinates. Coordinates win when +// set, then selector, then index. +type BrowserTarget struct { + Index *int `json:"index,omitempty"` + Selector string `json:"selector,omitempty"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` +} + +// BrowserClickOptions clicks in the session's browser. +type BrowserClickOptions struct { + ID string `json:"id"` + BrowserTarget +} + +// BrowserTypeOptions types into the session's browser. An empty target types +// into whatever is focused. +type BrowserTypeOptions struct { + ID string `json:"id"` + Text string `json:"text"` + Clear bool `json:"clear"` + BrowserTarget +} + +// BrowserPressOptions presses one named key. An empty target presses into +// whatever is focused. +type BrowserPressOptions struct { + ID string `json:"id"` + Key string `json:"key"` + BrowserTarget +} + +// BrowserScreenshotOptions captures the page to Path, or to a file lich +// chooses when Path is empty. +type BrowserScreenshotOptions struct { + ID string `json:"id"` + Path string `json:"path"` +} + +// BrowserScrollOptions scrolls the page by DY CSS pixels. Positive is down. +type BrowserScrollOptions struct { + ID string `json:"id"` + DY int `json:"dy"` +} + +// BrowserRect is an element's viewport box, integer pixels. +type BrowserRect struct { + X int `json:"x"` + Y int `json:"y"` + W int `json:"w"` + H int `json:"h"` +} + +// BrowserInteractive is one clickable or fillable node. It never carries a +// control's value: filled and checked say that typing landed. +type BrowserInteractive struct { + Index int `json:"index"` + Tag string `json:"tag"` + Type string `json:"type,omitempty"` + Role string `json:"role,omitempty"` + Text string `json:"text"` + Href string `json:"href,omitempty"` + Rect BrowserRect `json:"rect"` + Filled *bool `json:"filled,omitempty"` + Checked *bool `json:"checked,omitempty"` +} + +// BrowserPageInfo is what the agent reads instead of a screenshot. +type BrowserPageInfo struct { + URL string `json:"url"` + Title string `json:"title"` + Text string `json:"text"` + Interactive []BrowserInteractive `json:"interactive"` + Truncated bool `json:"truncated"` +} diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index e9f87f8f8..660e6c2e6 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -3,6 +3,20 @@ agentplugin.Status(None) []PluginStatus agentplugin.Update(PluginProviderOptions) None appupdate.Apply(None) None appupdate.Status(None) AppUpdateStatus +browser.Back(SessionIDOptions) None +browser.Click(BrowserClickOptions) None +browser.Close(SessionIDOptions) None +browser.Forward(SessionIDOptions) None +browser.Info(SessionIDOptions) BrowserPageInfo +browser.List(SessionIDOptions) []BrowserHandle +browser.Navigate(BrowserOpenOptions) BrowserHandle +browser.Open(BrowserOpenOptions) BrowserHandle +browser.Press(BrowserPressOptions) None +browser.Reload(SessionIDOptions) None +browser.Screenshot(BrowserScreenshotOptions) string +browser.Scroll(BrowserScrollOptions) None +browser.Show(SessionIDOptions) BrowserHandle +browser.Type(BrowserTypeOptions) None claudeprofile.CancelLogin(ProfileSlugOptions) None claudeprofile.CreateProfile(CreateProfileOptions) Profile claudeprofile.DeleteProfile(ProfileSlugOptions) None @@ -265,6 +279,59 @@ AppUpdateStatus { installCommand string } +SessionIDOptions { + id string +} + +BrowserClickOptions { + id string + ...BrowserTarget +} + +BrowserPageInfo { + url string + title string + text string + interactive []BrowserInteractive + truncated bool +} + +BrowserHandle { + id string + url string + title string + owner string + headed bool +} + +BrowserOpenOptions { + id string + url string +} + +BrowserPressOptions { + id string + key string + ...BrowserTarget +} + +BrowserScreenshotOptions { + id string + path string +} + +BrowserScrollOptions { + id string + dy int +} + +BrowserTypeOptions { + id string + text string + clear bool + ...BrowserTarget +} + ProfileSlugOptions { slug string } @@ -1031,10 +1098,6 @@ ExternalSession { profile,omitempty string } -SessionIDOptions { - id string -} - GetSettingOptions { key string projectId string @@ -1507,6 +1570,25 @@ SessionExit { code,omitempty *int } +BrowserTarget { + index,omitempty *int + selector,omitempty string + x,omitempty *float64 + y,omitempty *float64 +} + +BrowserInteractive { + index int + tag string + type,omitempty string + role,omitempty string + text string + href,omitempty string + rect BrowserRect + filled,omitempty *bool + checked,omitempty *bool +} + ProfileUpdate { name,omitempty string color,omitempty *string @@ -1664,6 +1746,13 @@ PluginProfileFailure { error string } +BrowserRect { + x int + y int + w int + h int +} + PRThreadComment { id int64 author string diff --git a/docs/browser-tab.md b/docs/browser-tab.md new file mode 100644 index 000000000..132672aa8 --- /dev/null +++ b/docs/browser-tab.md @@ -0,0 +1,39 @@ +# Decision: the session browser is a sidecar window + +**Status: the card's Browser item opens it.** Right-click a session card → +**Browser**, above Open in. That opens a Chromium window owned by that session. +The agent drives the same page through `lich browser` and the `browser_*` MCP +tools. + +## Why not a page inside lich + +lich's window is native (`docs/native-window.md`). It does not embed Chromium. +The product is a **second Chromium process** on `chromium.FindBrowser` (system +Chrome, Edge, Chromium, Brave or Vivaldi), with its own user-data directory. +Closing that window does not quit lich. Closing the session closes the sidecar. + +## One browser per card + +- none yet → headed window at `about:blank` +- already headed → focus that window +- headless only (the agent opened first) → replace it with a headed window at + the same URL. chromedp cannot show a headless process in place + +The agent without the menu uses `browser_open`: headless if nobody asked for a +window, and the headed one if the person already did. MCP and `lich browser` +target `LICH_SESSION_ID`. + +## What the agent may load + +`http`, `https`, `about:blank`. Not `file:`, `javascript:`, `data:`, `chrome:`. +Page info never includes form values. Screenshots are files. Click, type and +press are real input events. `press` is named keys only. Arbitrary JavaScript +in the page is not a tool. + +## Providers + +Claude Code and Codex receive the tools at spawn. Antigravity, oh-my-pi, +Crush, Cursor CLI and Kiro receive them from the lich MCP server the plugin +registers. opencode's plugin defines its own tool list, so it does not list +`browser_*`; `lich browser` still runs in that session. The gap is in +`docs/ceilings.md`. diff --git a/docs/ceilings.md b/docs/ceilings.md index d71dab17c..35a1a9d50 100644 --- a/docs/ceilings.md +++ b/docs/ceilings.md @@ -910,6 +910,25 @@ work when nobody knows it and that the call site never shows. The mechanism and up to that late: observing `AppleInterfaceThemeChangedNotification` needs an Objective-C shim no one could compile or run without a Mac. The BSDs read no scheme and draw dark. +- **A session's browser is a second Chromium, one per card** (`internal/browser`, + `docs/browser-tab.md`): the card menu opens a headed window of the system + Chrome, Edge, Chromium, Brave or Vivaldi, never a page inside lich. The agent + drives that same window. A headless one the agent opened first is replaced by + a headed window at the same URL, because chromedp cannot show a headless + process. Closing the card, or the session's process ending, closes it; closing + the browser window does not close the session. It loads only http, https and + about:blank. Page info never includes a form's value. Screenshots are files. + There is no tool that runs JavaScript in the page. It is not a sandbox: the + page runs as the user, in an ephemeral profile. +- **opencode does not list `browser_*`** (`internal/agentplugin`, + `internal/cli/mcp_browser.go`): Claude Code and Codex are handed lich's MCP + server at spawn, and Antigravity, oh-my-pi, Crush, Cursor CLI and Kiro get + that same server from the plugin install, so they list the browser tools with + the rest. opencode's plugin defines its own tool list and does not ask `lich + mcp` for one, so a tool added here does not appear there until lich-plugin + ships it. The card still opens the window, and `lich browser` still drives it + from that session. + ## Languages - **The prompt language reaches a running session only through lich itself** (`internal/prompt`): text lich diff --git a/docs/cli.md b/docs/cli.md index 5575181ee..e82908d4a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -927,6 +927,32 @@ transcript)` pair cost and never which model each priced turn ran under, and the model on a session row is the one selected now — so a per-model total would have to be a second accounting path, not a filter on this one. +### `lich browser [args]` + +Drives the browser owned by the session this command runs in — the same +Chromium window the card's Browser item opens. It has to run inside a session +(`LICH_SESSION_ID`); from anywhere else it refuses without calling lich. + +``` +lich browser open [url] +lich browser info +lich browser click --index | --selector | --x --y +lich browser type [--clear] [--index ] +lich browser press [--index ] +lich browser screenshot [path] +lich browser navigate +lich browser reload | back | forward +lich browser scroll [--dy ] +lich browser list +lich browser close +``` + +`open` with no URL loads `about:blank`. A browser the card already showed is +reused; one the agent opened headless stays headless until the card asks for +the window. `info` describes the page and its clickable nodes and never +includes what was typed into a field. `screenshot` prints the PNG's path. +URLs are http, https or `about:blank`. + ### `lich mcp` Serves the session commands above as MCP tools over stdio — `cost` is not one diff --git a/go.mod b/go.mod index 8432b00fe..f5b347b2a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.27.0 require ( fyne.io/systray v1.11.0 github.com/UserExistsError/conpty v0.1.4 + github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f + github.com/chromedp/chromedp v0.16.0 github.com/coder/websocket v1.8.14 github.com/creack/pty v1.1.24 github.com/lichdotdev/godemirror v0.8.0 @@ -17,8 +19,13 @@ require ( require ( github.com/akavel/rsrc v0.10.2 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect github.com/dchest/jsmin v0.0.0-20220218165748-59f39799265f // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/josephspurrier/goversioninfo v1.4.1 // indirect diff --git a/go.sum b/go.sum index 21cd1c3c5..2e71c382a 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,12 @@ github.com/UserExistsError/conpty v0.1.4 h1:+3FhJhiqhyEJa+K5qaK3/w6w+sN3Nh9O9VbJ github.com/UserExistsError/conpty v0.1.4/go.mod h1:PDglKIkX3O/2xVk0MV9a6bCWxRmPVfxqZoTG/5sSd9I= github.com/akavel/rsrc v0.10.2 h1:Zxm8V5eI1hW4gGaYsJQUhxpjkENuG91ki8B4zCrvEsw= github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0= +github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk= +github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= @@ -15,6 +21,14 @@ github.com/dchest/jsmin v0.0.0-20220218165748-59f39799265f h1:OGqDDftRTwrvUoL6pO github.com/dchest/jsmin v0.0.0-20220218165748-59f39799265f/go.mod h1:Dv9D0NUlAsaQcGQZa5kc5mqR9ua72SmA8VXi4cd+cBw= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= @@ -25,6 +39,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/josephspurrier/goversioninfo v1.4.1 h1:5LvrkP+n0tg91J9yTkoVnt/QgNnrI1t4uSsWjIonrqY= github.com/josephspurrier/goversioninfo v1.4.1/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lichdotdev/godemirror v0.8.0 h1:sPnCNZ1JygGr6J4IoiT1Oe2tbv1HmrTbPXUDhC81T8I= github.com/lichdotdev/godemirror v0.8.0/go.mod h1:xCdFPbos7+4/bCymJIx7FnL4EpsuMpebRCbGM5Un4fg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -33,6 +49,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/zenity v0.10.14 h1:OBFl7qfXcvsdo1NUEGxTlZvAakgWMqz9nG38TuiaGLI= github.com/ncruces/zenity v0.10.14/go.mod h1:ZBW7uVe/Di3IcRYH0Br8X59pi+O6EPnNIOU66YHpOO4= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/randall77/makefat v0.0.0-20210315173500-7ddd0e42c844 h1:GranzK4hv1/pqTIhMTXt2X8MmMOuH3hMeUR0o9SP5yc= diff --git a/internal/browser/actions.go b/internal/browser/actions.go new file mode 100644 index 000000000..e823e7f26 --- /dev/null +++ b/internal/browser/actions.go @@ -0,0 +1,180 @@ +package browser + +import ( + "context" + "fmt" + "path/filepath" +) + +var errNoBrowser = fmt.Errorf("no browser for this session — open one first") + +func (s *Service) live(id string) (*session, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.reapLocked() + sess := s.lookupLocked(id) + if sess == nil { + return nil, errNoBrowser + } + return sess, nil +} + +func (s *Service) touch(sess *session) { + s.mu.Lock() + defer s.mu.Unlock() + if s.byID[sess.id] == sess { + sess.lastUsed = s.now() + } +} + +func (s *Service) drop(sess *session) { + s.mu.Lock() + defer s.mu.Unlock() + if s.byID[sess.id] == sess { + _ = s.destroyLocked(sess) + } +} + +func (s *Service) remember(sess *session, url, title string) { + s.mu.Lock() + defer s.mu.Unlock() + if s.byID[sess.id] != sess { + return + } + if url != "" { + sess.url = url + } + sess.title = title + sess.lastUsed = s.now() +} + +// Info returns the page the agent is looking at. Never includes form values. +func (s *Service) Info(id string) (PageInfo, error) { + sess, err := s.live(id) + if err != nil { + return PageInfo{}, err + } + var info PageInfo + err = runPage(func(ctx context.Context) error { + var e error + info, e = sess.page.Info(ctx) + return e + }) + if err != nil { + s.drop(sess) + return PageInfo{}, err + } + s.remember(sess, info.URL, info.Title) + return info, nil +} + +// Click sends a real mouse event at the target, not element.click(). +func (s *Service) Click(id string, t Target) error { + if t.empty() { + return fmt.Errorf("click needs an index, a selector, or x and y") + } + return s.act(id, func(p Page, ctx context.Context) error { + return p.Click(ctx, t) + }) +} + +// Type sends keystrokes. clear selects-and-replaces; the typed text never +// comes back through Info. +func (s *Service) Type(id, text string, clear bool, t Target) error { + return s.act(id, func(p Page, ctx context.Context) error { + return p.Type(ctx, text, clear, t) + }) +} + +// Press sends one named key (Enter, Tab, Escape, …) as a real key event. +// An optional target is focused first, the same way Type focuses before typing. +func (s *Service) Press(id, key string, t Target) error { + if _, err := keySequence(key); err != nil { + return err + } + return s.act(id, func(p Page, ctx context.Context) error { + return p.Press(ctx, key, t) + }) +} + +// Screenshot writes a PNG to dest (created if empty: inside the profile dir) +// and returns that path. The bytes stay off the transcript. +func (s *Service) Screenshot(id, dest string) (string, error) { + sess, err := s.live(id) + if err != nil { + return "", err + } + if dest == "" { + dest = filepath.Join(sess.dir, "screenshot.png") + } else { + abs, err := filepath.Abs(dest) + if err != nil { + return "", err + } + dest = abs + } + err = runPage(func(ctx context.Context) error { + return sess.page.Screenshot(ctx, dest) + }) + if err != nil { + s.drop(sess) + return "", err + } + s.touch(sess) + return dest, nil +} + +// Navigate loads url in this session's browser. +func (s *Service) Navigate(id, rawURL string) (Handle, error) { + u, err := allowURL(rawURL) + if err != nil { + return Handle{}, err + } + sess, err := s.live(id) + if err != nil { + return Handle{}, err + } + err = runPage(func(ctx context.Context) error { + return sess.page.Navigate(ctx, u) + }) + if err != nil { + s.drop(sess) + return Handle{}, err + } + s.remember(sess, u, "") + return sess.handle(), nil +} + +// Reload reloads the current page. +func (s *Service) Reload(id string) error { + return s.act(id, func(p Page, ctx context.Context) error { return p.Reload(ctx) }) +} + +// Back goes back in history. +func (s *Service) Back(id string) error { + return s.act(id, func(p Page, ctx context.Context) error { return p.Back(ctx) }) +} + +// Forward goes forward in history. +func (s *Service) Forward(id string) error { + return s.act(id, func(p Page, ctx context.Context) error { return p.Forward(ctx) }) +} + +// Scroll moves the viewport by dy CSS pixels (positive is down). +func (s *Service) Scroll(id string, dy int) error { + return s.act(id, func(p Page, ctx context.Context) error { return p.Scroll(ctx, dy) }) +} + +func (s *Service) act(id string, fn func(Page, context.Context) error) error { + sess, err := s.live(id) + if err != nil { + return err + } + err = runPage(func(ctx context.Context) error { return fn(sess.page, ctx) }) + if err != nil { + s.drop(sess) + return err + } + s.touch(sess) + return nil +} diff --git a/internal/browser/browser.go b/internal/browser/browser.go new file mode 100644 index 000000000..604fdcfff --- /dev/null +++ b/internal/browser/browser.go @@ -0,0 +1,294 @@ +package browser + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "sync" + "time" +) + +// idleAfter is how long a session's browser sits unused before the reaper +// closes it. Five minutes matches the "walked away from the card" window +// without holding a Chromium for a session that finished hours ago. +const idleAfter = 5 * time.Minute + +// reapEvery is how often the reaper looks. Shorter than idleAfter so a +// browser that went idle is gone within a fraction of the idle window, not a +// full extra one. +const reapEvery = 30 * time.Second + +// pageWait bounds one CDP action. Under the 90 seconds an MCP tool call may +// block for, long enough for a cold page, short enough that a hung Chrome +// does not take the tool with it. +const pageWait = 30 * time.Second + +var errNoOwner = errors.New("browser needs a lich session id") + +// Service is one Chromium context per lich session: the window Browser tab +// opens and the page the agent drives are the same one. Headless until +// someone asks for a window; headed after OpenVisible. +type Service struct { + mu sync.Mutex + launch Launcher + mkTmp func(dir, pattern string) (string, error) + now func() time.Time + idle time.Duration + byOwner map[string]*session + byID map[string]*session + stop chan struct{} + stopped bool +} + +type session struct { + id, owner, dir string + url, title string + headed bool + page Page + lastUsed time.Time +} + +// New starts a service that launches the system Chromium (chromium.FindBrowser) +// into an ephemeral profile, never the lich --app one. +func New() *Service { + s := newService(chromeLaunch, os.MkdirTemp, time.Now, idleAfter) + s.startReaper() + return s +} + +func newService(launch Launcher, mkTmp func(string, string) (string, error), now func() time.Time, idle time.Duration) *Service { + return &Service{ + launch: launch, + mkTmp: mkTmp, + now: now, + idle: idle, + byOwner: make(map[string]*session), + byID: make(map[string]*session), + stop: make(chan struct{}), + } +} + +// Open navigates this session's browser to url, creating a headless one if +// none exists. A headed window the user already opened is reused, not replaced. +func (s *Service) Open(owner, rawURL string) (Handle, error) { + if owner == "" { + return Handle{}, errNoOwner + } + u, err := allowURL(rawURL) + if err != nil { + return Handle{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + s.reapLocked() + if sess := s.liveByOwner(owner); sess != nil { + if err := runPage(func(ctx context.Context) error { + return sess.page.Navigate(ctx, u) + }); err != nil { + s.destroyLocked(sess) + return Handle{}, err + } + sess.url = u + sess.lastUsed = s.now() + return sess.handle(), nil + } + return s.launchLocked(owner, u, false) +} + +// OpenVisible brings this session's browser on screen. No browser yet: a +// headed window at about:blank. Already headed: focus it. Headless only: +// replace it with a headed window at the same URL — chromedp cannot promote +// a headless process in place. +func (s *Service) OpenVisible(owner string) (Handle, error) { + if owner == "" { + return Handle{}, errNoOwner + } + s.mu.Lock() + defer s.mu.Unlock() + s.reapLocked() + if sess := s.liveByOwner(owner); sess != nil { + if sess.headed { + _ = runPage(sess.page.Focus) + sess.lastUsed = s.now() + return sess.handle(), nil + } + u := sess.url + if u == "" { + u = "about:blank" + } + s.destroyLocked(sess) + return s.launchLocked(owner, u, true) + } + return s.launchLocked(owner, "about:blank", true) +} + +// List returns this session's browser, or an empty list. Never null: a +// script should not have to tell those apart. +func (s *Service) List(owner string) ([]Handle, error) { + if owner == "" { + return nil, errNoOwner + } + s.mu.Lock() + defer s.mu.Unlock() + s.reapLocked() + out := []Handle{} + if sess := s.liveByOwner(owner); sess != nil { + out = append(out, sess.handle()) + } + return out, nil +} + +// Close tears down one browser, by handle id or by the lich session that owns it. +func (s *Service) Close(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + sess := s.lookupLocked(id) + if sess == nil { + return nil + } + return s.destroyLocked(sess) +} + +// CloseOwnedBy tears down the browser of a lich session that is going away. +// Missing is not an error: most sessions never opened one. +func (s *Service) CloseOwnedBy(owner string) error { + return s.Close(owner) +} + +// Cleanup closes every browser this process started. Wired from main so a +// window-close exit does not leave Chromiums behind. +func (s *Service) Cleanup() error { + s.mu.Lock() + defer s.mu.Unlock() + if !s.stopped { + s.stopped = true + close(s.stop) + } + var errs []error + for _, sess := range s.byID { + if err := s.destroyLocked(sess); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *Service) launchLocked(owner, u string, headed bool) (Handle, error) { + dir, err := s.mkTmp("", "lich-browser-") + if err != nil { + return Handle{}, err + } + page, err := s.launch(dir, headed) + if err != nil { + _ = os.RemoveAll(dir) + return Handle{}, err + } + if err := runPage(func(ctx context.Context) error { + return page.Navigate(ctx, u) + }); err != nil { + _ = page.Close() + _ = os.RemoveAll(dir) + return Handle{}, err + } + sess := &session{ + id: newID(), owner: owner, dir: dir, url: u, headed: headed, page: page, lastUsed: s.now(), + } + s.byOwner[owner] = sess + s.byID[sess.id] = sess + return sess.handle(), nil +} + +func (s *Service) liveByOwner(owner string) *session { + sess := s.byOwner[owner] + if sess == nil || !sess.alive() { + if sess != nil { + s.destroyLocked(sess) + } + return nil + } + return sess +} + +func (s *Service) lookupLocked(id string) *session { + if sess := s.byID[id]; sess != nil { + if sess.alive() { + return sess + } + s.destroyLocked(sess) + } + if sess := s.byOwner[id]; sess != nil { + if sess.alive() { + return sess + } + s.destroyLocked(sess) + } + return nil +} + +func (s *Service) destroyLocked(sess *session) error { + delete(s.byID, sess.id) + if s.byOwner[sess.owner] == sess { + delete(s.byOwner, sess.owner) + } + err := sess.page.Close() + if rmErr := os.RemoveAll(sess.dir); err == nil { + err = rmErr + } + return err +} + +func (s *session) alive() bool { + return s != nil && s.page != nil && s.page.Alive() +} + +func (s *session) handle() Handle { + return Handle{ID: s.id, URL: s.url, Title: s.title, Owner: s.owner, Headed: s.headed} +} + +func (s *Service) startReaper() { + go func() { + tick := time.NewTicker(reapEvery) + defer tick.Stop() + for { + select { + case <-s.stop: + return + case <-tick.C: + s.Reap() + } + } + }() +} + +// Reap closes browsers that have sat idle past the idle window. +func (s *Service) Reap() { + s.mu.Lock() + defer s.mu.Unlock() + s.reapLocked() +} + +func (s *Service) reapLocked() { + cutoff := s.now().Add(-s.idle) + for _, sess := range s.byID { + if sess.lastUsed.Before(cutoff) { + _ = s.destroyLocked(sess) + } + } +} + +func runPage(fn func(context.Context) error) error { + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + return fn(ctx) +} + +func newID() string { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("%d", time.Now().UnixNano()) + } + return hex.EncodeToString(b[:]) +} diff --git a/internal/browser/browser_test.go b/internal/browser/browser_test.go new file mode 100644 index 000000000..a25a910a5 --- /dev/null +++ b/internal/browser/browser_test.go @@ -0,0 +1,380 @@ +package browser + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func testService(t *testing.T, launch *fakeLauncher) *Service { + t.Helper() + now := time.Date(2026, 8, 16, 12, 0, 0, 0, time.UTC) + s := newService(launch.launch, func(string, string) (string, error) { + return t.TempDir(), nil + }, func() time.Time { return now }, 5*time.Minute) + s.now = func() time.Time { return now } + t.Cleanup(func() { _ = s.Cleanup() }) + return s +} + +func TestOpenCreatesOneHeadlessPerOwner(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + + a, err := s.Open("sess-a", "https://example.com") + if err != nil { + t.Fatal(err) + } + if a.Headed || a.Owner != "sess-a" || a.URL != "https://example.com" { + t.Fatalf("handle = %+v", a) + } + b, err := s.Open("sess-a", "https://example.com/next") + if err != nil { + t.Fatal(err) + } + if a.ID != b.ID { + t.Fatalf("second Open minted a new browser: %s then %s", a.ID, b.ID) + } + if len(launch.pages) != 1 { + t.Fatalf("launched %d, want 1", len(launch.pages)) + } + if got := launch.pages[0].navigations; len(got) != 2 { + t.Fatalf("navigations = %v", got) + } + c, err := s.Open("sess-b", "https://other.example") + if err != nil { + t.Fatal(err) + } + if c.ID == a.ID { + t.Fatal("a second lich session shared the first's browser") + } + if len(launch.pages) != 2 { + t.Fatalf("launched %d, want 2", len(launch.pages)) + } +} + +func TestOpenRejectsBlockedURL(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("s", "file:///etc/passwd"); err == nil { + t.Fatal("file: URL was allowed") + } + list, err := s.List("s") + if err != nil { + t.Fatal(err) + } + if len(list) != 0 { + t.Fatalf("a rejected Open still left a browser: %v", list) + } +} + +func TestOpenVisibleReusesHeadedAndReplacesHeadless(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + + first, err := s.OpenVisible("s") + if err != nil { + t.Fatal(err) + } + if !first.Headed { + t.Fatal("OpenVisible launched headless") + } + second, err := s.OpenVisible("s") + if err != nil { + t.Fatal(err) + } + if first.ID != second.ID { + t.Fatal("second OpenVisible replaced a headed window") + } + if launch.pages[0].focused == 0 { + t.Fatal("reuse did not focus the existing window") + } + + h, err := s.Open("other", "https://example.com/page") + if err != nil { + t.Fatal(err) + } + if h.Headed { + t.Fatal("Open without a window should be headless") + } + vis, err := s.OpenVisible("other") + if err != nil { + t.Fatal(err) + } + if vis.ID == h.ID { + t.Fatal("OpenVisible kept the headless page instead of replacing it") + } + if !vis.Headed || vis.URL != "https://example.com/page" { + t.Fatalf("replaced handle = %+v", vis) + } + if !launch.pages[1].closed { + t.Fatal("headless page was not closed when promoting") + } +} + +func TestCloseOwnedByAndCleanup(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Open("a", "https://a.example"); err != nil { + t.Fatal(err) + } + if _, err := s.Open("b", "https://b.example"); err != nil { + t.Fatal(err) + } + if err := s.CloseOwnedBy("a"); err != nil { + t.Fatal(err) + } + if !launch.pages[0].closed { + t.Fatal("CloseOwnedBy left the page open") + } + list, _ := s.List("a") + if len(list) != 0 { + t.Fatal("owner a still has a browser") + } + if err := s.Cleanup(); err != nil { + t.Fatal(err) + } + if !launch.pages[1].closed { + t.Fatal("Cleanup left a page open") + } +} + +func TestActionsNeedABrowserAndRecordOnThePage(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Info("missing"); !errors.Is(err, errNoBrowser) { + t.Fatalf("Info with none: %v", err) + } + + h, err := s.Open("s", "https://example.com") + if err != nil { + t.Fatal(err) + } + idx := 3 + if err := s.Click(h.ID, Target{Index: &idx}); err != nil { + t.Fatal(err) + } + if err := s.Type("s", "hello", true, Target{}); err != nil { + t.Fatal(err) + } + if err := s.Press("s", "Enter", Target{}); err != nil { + t.Fatal(err) + } + dest, err := s.Screenshot("s", "") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dest); err != nil { + t.Fatalf("screenshot: %v", err) + } + if filepath.Base(dest) != "screenshot.png" { + t.Fatalf("screenshot path = %s", dest) + } + if err := s.Reload("s"); err != nil { + t.Fatal(err) + } + if err := s.Back("s"); err != nil { + t.Fatal(err) + } + if err := s.Forward("s"); err != nil { + t.Fatal(err) + } + if err := s.Scroll("s", 400); err != nil { + t.Fatal(err) + } + p := launch.pages[0] + if len(p.clicks) != 1 || p.clicks[0].Index == nil || *p.clicks[0].Index != 3 { + t.Fatalf("clicks = %+v", p.clicks) + } + if len(p.typed) != 1 || p.typed[0].text != "hello" || !p.typed[0].clear { + t.Fatalf("typed = %+v", p.typed) + } + if len(p.pressed) != 1 || p.pressed[0].key != "Enter" { + t.Fatalf("pressed = %+v", p.pressed) + } + if p.reloads != 1 || p.backs != 1 || p.forwards != 1 { + t.Fatalf("nav = reload %d back %d forward %d", p.reloads, p.backs, p.forwards) + } + if len(p.scrolled) != 1 || p.scrolled[0] != 400 { + t.Fatalf("scrolled = %v", p.scrolled) + } +} + +func TestClickRequiresATarget(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("s", ""); err != nil { + t.Fatal(err) + } + if err := s.Click("s", Target{}); err == nil { + t.Fatal("click with no target succeeded") + } +} + +func TestPressRejectsUnknownKey(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("s", ""); err != nil { + t.Fatal(err) + } + if err := s.Press("s", "Ctrl+A", Target{}); err == nil { + t.Fatal("unknown key was allowed") + } + if err := s.Press("s", "", Target{}); err == nil { + t.Fatal("empty key was allowed") + } +} + +func TestNavigateRejectsBlockedURL(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("s", "https://ok.example"); err != nil { + t.Fatal(err) + } + if _, err := s.Navigate("s", "javascript:alert(1)"); err == nil { + t.Fatal("navigate allowed javascript:") + } +} + +func TestIdleReaperClosesUnused(t *testing.T) { + launch := &fakeLauncher{} + now := time.Date(2026, 8, 16, 12, 0, 0, 0, time.UTC) + s := newService(launch.launch, func(string, string) (string, error) { + return t.TempDir(), nil + }, func() time.Time { return now }, time.Minute) + t.Cleanup(func() { _ = s.Cleanup() }) + + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + s.Reap() + if launch.pages[0].closed { + t.Fatal("reaper closed a browser that was just used") + } + now = now.Add(time.Minute + time.Second) + s.now = func() time.Time { return now } + s.Reap() + if !launch.pages[0].closed { + t.Fatal("reaper left an idle browser running") + } +} + +func TestOpenRequiresOwner(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("", "https://example.com"); !errors.Is(err, errNoOwner) { + t.Fatalf("Open with no owner: %v", err) + } + if _, err := s.OpenVisible(""); !errors.Is(err, errNoOwner) { + t.Fatalf("OpenVisible with no owner: %v", err) + } +} + +func TestNavigateUpdatesHandle(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("s", "https://a.example"); err != nil { + t.Fatal(err) + } + h, err := s.Navigate("s", "https://b.example") + if err != nil { + t.Fatal(err) + } + if h.URL != "https://b.example" { + t.Fatalf("handle url = %q", h.URL) + } +} + +func TestScreenshotWritesTheAskedPath(t *testing.T) { + s := testService(t, &fakeLauncher{}) + if _, err := s.Open("s", ""); err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), "shot.png") + got, err := s.Screenshot("s", dest) + if err != nil { + t.Fatal(err) + } + if got != dest { + t.Fatalf("path = %q, want %q", got, dest) + } +} + +func TestFailedInfoDropsTheBrowser(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + launch.pages[0].infoErr = errors.New("chrome gone") + if _, err := s.Info("s"); err == nil { + t.Fatal("Info succeeded on a dead page") + } + list, _ := s.List("s") + if len(list) != 0 { + t.Fatal("dead page was not dropped") + } +} + +func TestOpenDropsWhenReuseNavigateFails(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Open("s", "https://a.example"); err != nil { + t.Fatal(err) + } + launch.pages[0].navErr = errors.New("gone") + if _, err := s.Open("s", "https://b.example"); err == nil { + t.Fatal("Open reused a page that could not navigate") + } + if !launch.pages[0].closed { + t.Fatal("failed reuse left the page open") + } +} + +func TestLaunchFailureLeavesNoBrowser(t *testing.T) { + s := newService(func(string, bool) (Page, error) { + return nil, errors.New("no chrome") + }, func(string, string) (string, error) { + return t.TempDir(), nil + }, time.Now, time.Minute) + t.Cleanup(func() { _ = s.Cleanup() }) + if _, err := s.Open("s", "https://example.com"); err == nil { + t.Fatal("Open succeeded without a launcher") + } + list, _ := s.List("s") + if len(list) != 0 { + t.Fatal("a failed launch left a handle") + } +} + +func TestCloseByHandleID(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + h, err := s.Open("s", "https://example.com") + if err != nil { + t.Fatal(err) + } + if err := s.Close(h.ID); err != nil { + t.Fatal(err) + } + if !launch.pages[0].closed { + t.Fatal("Close(id) left the page open") + } +} + +func TestInfoReturnsPageInfoWithoutDroppingTheBrowser(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + launch.pages[0].title = "Example" + launch.pages[0].info = PageInfo{Title: "Example", Text: "hello", Interactive: []Interactive{{Tag: "a", Text: "Home"}}} + info, err := s.Info("s") + if err != nil { + t.Fatal(err) + } + if info.Title != "Example" || info.Text != "hello" { + t.Fatalf("info = %+v", info) + } + list, _ := s.List("s") + if len(list) != 1 { + t.Fatal("Info dropped the browser") + } +} diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go new file mode 100644 index 000000000..53bc73160 --- /dev/null +++ b/internal/browser/chrome.go @@ -0,0 +1,260 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "time" + + "github.com/chromedp/cdproto/input" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" + "github.com/lichdotdev/lich/internal/chromium" +) + +// domSettle is how long a navigation waits for the document to become +// interactive. chromedp.Navigate waits for the load event instead, which is +// every image and tracker: the page is already on screen and the tool is +// still held, up to pageWait. Past this, the agent reads whatever is there. +const domSettle = 1500 * time.Millisecond + +// chromeLaunch starts a Chromium-family browser via CDP. ExecPath is +// chromium.FindBrowser (system Chrome/Edge/… — never lich-shell); UserDataDir +// is this ephemeral dir, never lich/chromium-profile. Attaching to the UI +// window would expose the page and the session token; this process must stay +// a sidecar. +func chromeLaunch(dir string, headed bool) (Page, error) { + exe, err := chromium.FindBrowser(exec.LookPath) + if err != nil { + return nil, err + } + opts := append([]chromedp.ExecAllocatorOption{}, chromedp.DefaultExecAllocatorOptions[:]...) + opts = append(opts, chromedp.ExecPath(exe), chromedp.UserDataDir(dir)) + if headed { + opts = append(opts, chromedp.Flag("headless", false)) + } + allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...) + ctx, cancel := chromedp.NewContext(allocCtx) + if err := chromedp.Run(ctx); err != nil { + cancel() + allocCancel() + return nil, err + } + return &chromePage{ctx: ctx, cancel: cancel, allocCancel: allocCancel}, nil +} + +type chromePage struct { + ctx context.Context + cancel context.CancelFunc + allocCancel context.CancelFunc +} + +func (p *chromePage) Alive() bool { return p.ctx.Err() == nil } + +func (p *chromePage) Close() error { + p.cancel() + p.allocCancel() + return nil +} + +// run drives CDP on the allocator context chromedp attached, applying wait's +// deadline. Service callers pass a timeout derived from Background; chromedp.Run +// on that context is "invalid context" and the headed window is then closed. +func (p *chromePage) run(wait context.Context, actions ...chromedp.Action) error { + ctx, cancel := withPageDeadline(p.ctx, wait) + defer cancel() + return chromedp.Run(ctx, actions...) +} + +func (p *chromePage) Navigate(ctx context.Context, url string) error { + return p.run(ctx, settleAfter(func(ctx context.Context) error { + _, _, errorText, _, err := page.Navigate(url).Do(ctx) + if err != nil { + return err + } + if errorText != "" { + return fmt.Errorf("page load error %s", errorText) + } + return nil + })) +} + +func (p *chromePage) Info(ctx context.Context) (PageInfo, error) { + var info PageInfo + if err := p.run(ctx, chromedp.Evaluate(pageInfoScript, &info)); err != nil { + return PageInfo{}, err + } + return info, nil +} + +func (p *chromePage) Click(ctx context.Context, t Target) error { + pt, err := p.resolve(ctx, t) + if err != nil { + return err + } + return p.run(ctx, chromedp.MouseClickXY(pt.X, pt.Y)) +} + +func (p *chromePage) Type(ctx context.Context, text string, clear bool, t Target) error { + if !t.empty() { + if err := p.Click(ctx, t); err != nil { + return err + } + } + if clear { + err := p.run(ctx, chromedp.Evaluate(`(() => { + const el = document.activeElement; + if (el && 'value' in el) el.value = ''; + })()`, nil)) + if err != nil { + return err + } + } + return p.run(ctx, chromedp.ActionFunc(func(ctx context.Context) error { + return input.InsertText(text).Do(ctx) + })) +} + +func (p *chromePage) Press(ctx context.Context, key string, t Target) error { + seq, err := keySequence(key) + if err != nil { + return err + } + if !t.empty() { + if err := p.Click(ctx, t); err != nil { + return err + } + } + return p.run(ctx, chromedp.KeyEvent(seq)) +} + +func (p *chromePage) Screenshot(ctx context.Context, dest string) error { + var buf []byte + if err := p.run(ctx, chromedp.CaptureScreenshot(&buf)); err != nil { + return err + } + return os.WriteFile(dest, buf, 0o600) +} + +func (p *chromePage) Reload(ctx context.Context) error { + return p.run(ctx, settleAfter(func(ctx context.Context) error { + return page.Reload().Do(ctx) + })) +} + +func (p *chromePage) Back(ctx context.Context) error { + return p.run(ctx, history(-1)) +} + +func (p *chromePage) Forward(ctx context.Context) error { + return p.run(ctx, history(1)) +} + +// history moves one step. chromedp.NavigateBack waits for the load event, the +// same wait Navigate used to. +func history(delta int) chromedp.Action { + return settleAfter(func(ctx context.Context) error { + cur, entries, err := page.GetNavigationHistory().Do(ctx) + if err != nil { + return err + } + next := int(cur) + delta + if next < 0 || next >= len(entries) { + return fmt.Errorf("no history in that direction") + } + return page.NavigateToHistoryEntry(entries[next].ID).Do(ctx) + }) +} + +// settleAfter runs goThere, then returns once the document is interactive or +// domSettle has passed, whichever comes first. +func settleAfter(goThere func(context.Context) error) chromedp.Action { + return chromedp.ActionFunc(func(ctx context.Context) error { + if err := goThere(ctx); err != nil { + return err + } + deadline := time.Now().Add(domSettle) + for { + var state string + if err := chromedp.Evaluate(`document.readyState`, &state).Do(ctx); err == nil && + (state == "interactive" || state == "complete") { + return nil + } + if time.Now().After(deadline) { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + t := time.NewTimer(40 * time.Millisecond) + select { + case <-ctx.Done(): + t.Stop() + return ctx.Err() + case <-t.C: + } + } + }) +} + +func (p *chromePage) Scroll(ctx context.Context, dy int) error { + return p.run(ctx, chromedp.Evaluate(fmt.Sprintf("window.scrollBy(0, %d)", dy), nil)) +} + +func (p *chromePage) Focus(ctx context.Context) error { + return p.run(ctx, chromedp.Evaluate("window.focus()", nil)) +} + +// withPageDeadline hangs wait's timeout off the chromedp page context. A +// timeout from context.Background has no CDP target; wrapping it the other +// way around is how OpenVisible returned "invalid context" and closed the +// window it had just opened. +func withPageDeadline(pageCtx, waitCtx context.Context) (context.Context, context.CancelFunc) { + timeout := pageWait + if waitCtx != nil { + if d, ok := waitCtx.Deadline(); ok { + timeout = time.Until(d) + if timeout < 0 { + timeout = 0 + } + } + } + return context.WithTimeout(pageCtx, timeout) +} + +type point struct { + X float64 `json:"x"` + Y float64 `json:"y"` +} + +const resolveJS = `(function(index, selector, x, y) { + if (typeof x === 'number' && typeof y === 'number') { + return {x: x, y: y}; + } + var el = null; + if (selector) el = document.querySelector(selector); + else if (typeof index === 'number') { + el = document.querySelector('[data-lich-idx="' + index + '"]'); + } + if (!el) return null; + var r = el.getBoundingClientRect(); + return {x: r.left + r.width / 2, y: r.top + r.height / 2}; +})` + +func (p *chromePage) resolve(ctx context.Context, t Target) (point, error) { + args, err := json.Marshal([]any{t.Index, t.Selector, t.X, t.Y}) + if err != nil { + return point{}, err + } + expr := resolveJS + "(" + string(args[1:len(args)-1]) + ")" + var pt *point + if err := p.run(ctx, chromedp.Evaluate(expr, &pt)); err != nil { + return point{}, err + } + if pt == nil { + return point{}, fmt.Errorf("no element matches that target") + } + return *pt, nil +} diff --git a/internal/browser/chrome_test.go b/internal/browser/chrome_test.go new file mode 100644 index 000000000..d57645b62 --- /dev/null +++ b/internal/browser/chrome_test.go @@ -0,0 +1,32 @@ +package browser + +import ( + "context" + "testing" + "time" + + "github.com/chromedp/chromedp" +) + +func TestWithPageDeadlineKeepsChromedpTarget(t *testing.T) { + pageCtx, cancel := chromedp.NewContext(context.Background()) + defer cancel() + if chromedp.FromContext(pageCtx) == nil { + t.Fatal("NewContext produced no chromedp target") + } + + wait, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + if chromedp.FromContext(wait) != nil { + t.Fatal("a Background timeout unexpectedly carried a chromedp target") + } + + got, done := withPageDeadline(pageCtx, wait) + defer done() + if chromedp.FromContext(got) == nil { + t.Fatal("deadline wrapper dropped the chromedp target; CDP would return invalid context") + } + if _, ok := got.Deadline(); !ok { + t.Fatal("deadline wrapper dropped the wait deadline") + } +} diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go new file mode 100644 index 000000000..c72b18519 --- /dev/null +++ b/internal/browser/fake_test.go @@ -0,0 +1,156 @@ +package browser + +import ( + "context" + "os" + "sync" +) + +// fakePage is a Page that never talks to Chrome. Tests inject it through +// Launcher so the suite stays off the OS boundary. +type fakePage struct { + mu sync.Mutex + url, title string + closed bool + navigations []string + clicks []Target + typed []typed + pressed []pressed + reloads int + backs int + forwards int + scrolled []int + focused int + info PageInfo + shotErr error + navErr error + infoErr error +} + +type typed struct { + text string + clear bool + t Target +} + +type pressed struct { + key string + t Target +} + +func (p *fakePage) Alive() bool { + p.mu.Lock() + defer p.mu.Unlock() + return !p.closed +} + +func (p *fakePage) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + p.closed = true + return nil +} + +func (p *fakePage) Navigate(_ context.Context, url string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.navErr != nil { + return p.navErr + } + p.url = url + p.navigations = append(p.navigations, url) + return nil +} + +func (p *fakePage) Info(context.Context) (PageInfo, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.infoErr != nil { + return PageInfo{}, p.infoErr + } + info := p.info + if info.URL == "" { + info.URL = p.url + } + if info.Title == "" { + info.Title = p.title + } + return info, nil +} + +func (p *fakePage) Click(_ context.Context, t Target) error { + p.mu.Lock() + defer p.mu.Unlock() + p.clicks = append(p.clicks, t) + return nil +} + +func (p *fakePage) Type(_ context.Context, text string, clear bool, t Target) error { + p.mu.Lock() + defer p.mu.Unlock() + p.typed = append(p.typed, typed{text: text, clear: clear, t: t}) + return nil +} + +func (p *fakePage) Press(_ context.Context, key string, t Target) error { + p.mu.Lock() + defer p.mu.Unlock() + p.pressed = append(p.pressed, pressed{key: key, t: t}) + return nil +} + +func (p *fakePage) Screenshot(_ context.Context, dest string) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.shotErr != nil { + return p.shotErr + } + return os.WriteFile(dest, []byte("png"), 0o600) +} + +func (p *fakePage) Reload(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + p.reloads++ + return nil +} + +func (p *fakePage) Back(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + p.backs++ + return nil +} + +func (p *fakePage) Forward(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + p.forwards++ + return nil +} + +func (p *fakePage) Scroll(_ context.Context, dy int) error { + p.mu.Lock() + defer p.mu.Unlock() + p.scrolled = append(p.scrolled, dy) + return nil +} + +func (p *fakePage) Focus(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + p.focused++ + return nil +} + +type fakeLauncher struct { + pages []*fakePage + headed []bool +} + +func (f *fakeLauncher) launch(_ string, headed bool) (Page, error) { + p := &fakePage{url: "about:blank"} + f.pages = append(f.pages, p) + f.headed = append(f.headed, headed) + return p, nil +} diff --git a/internal/browser/keys.go b/internal/browser/keys.go new file mode 100644 index 000000000..da5d67bfe --- /dev/null +++ b/internal/browser/keys.go @@ -0,0 +1,68 @@ +package browser + +import ( + "fmt" + "strings" + + "github.com/chromedp/chromedp/kb" +) + +// namedKeys maps the names agents and `lich browser press` use to the +// single-rune sequences chromedp's KeyEvent understands (kb package). +// Aliases are folded in normalizeKey; this table is the canonical spellings. +var namedKeys = map[string]string{ + "enter": kb.Enter, + "tab": kb.Tab, + "escape": kb.Escape, + "backspace": kb.Backspace, + "delete": kb.Delete, + "space": " ", + "arrowup": kb.ArrowUp, + "arrowdown": kb.ArrowDown, + "arrowleft": kb.ArrowLeft, + "arrowright": kb.ArrowRight, + "home": kb.Home, + "end": kb.End, + "pageup": kb.PageUp, + "pagedown": kb.PageDown, +} + +// keyAliases fold common short names onto the table above. +var keyAliases = map[string]string{ + "return": "enter", + "esc": "escape", + "bs": "backspace", + "del": "delete", + "up": "arrowup", + "down": "arrowdown", + "left": "arrowleft", + "right": "arrowright", + "pgup": "pageup", + "pgdn": "pagedown", +} + +// keySequence turns a named key into the string chromedp.KeyEvent expects. +// Unknown names are refused so agents cannot invent chords or paste text here — +// that is what Type is for. +func keySequence(name string) (string, error) { + n := normalizeKey(name) + if n == "" { + return "", fmt.Errorf("press needs a key name (Enter, Tab, Escape, …)") + } + seq, ok := namedKeys[n] + if !ok { + return "", fmt.Errorf("unknown key %q — use Enter, Tab, Escape, Backspace, Delete, Space, ArrowUp/Down/Left/Right, Home, End, PageUp, PageDown", name) + } + return seq, nil +} + +func normalizeKey(name string) string { + n := strings.ToLower(strings.TrimSpace(name)) + n = strings.ReplaceAll(n, "-", "") + n = strings.ReplaceAll(n, "_", "") + n = strings.ReplaceAll(n, " ", "") + if alias, ok := keyAliases[n]; ok { + return alias + } + return n +} diff --git a/internal/browser/keys_test.go b/internal/browser/keys_test.go new file mode 100644 index 000000000..0d88093ff --- /dev/null +++ b/internal/browser/keys_test.go @@ -0,0 +1,41 @@ +package browser + +import "testing" + +func TestKeySequenceAcceptsNamedKeysAndAliases(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"Enter", "\r"}, + {"return", "\r"}, + {"Tab", "\t"}, + {"Esc", "\u001b"}, + {"Escape", "\u001b"}, + {"Arrow-Down", "\u0301"}, + {"arrow_up", "\u0304"}, + {"Space", " "}, + {" ENTER ", "\r"}, + } + for _, c := range cases { + got, err := keySequence(c.in) + if err != nil { + t.Fatalf("%q: %v", c.in, err) + } + if got != c.want { + t.Fatalf("%q → %q, want %q", c.in, got, c.want) + } + } +} + +func TestKeySequenceRejectsUnknownAndEmpty(t *testing.T) { + if _, err := keySequence(""); err == nil { + t.Fatal("empty key was allowed") + } + if _, err := keySequence("Ctrl+C"); err == nil { + t.Fatal("chord was allowed") + } + if _, err := keySequence("hello"); err == nil { + t.Fatal("text was allowed through press") + } +} diff --git a/internal/browser/page.go b/internal/browser/page.go new file mode 100644 index 000000000..5e866bde8 --- /dev/null +++ b/internal/browser/page.go @@ -0,0 +1,82 @@ +package browser + +import "context" + +// Handle is what Open and OpenVisible return: the id later calls take, and +// enough for a card or a tool result to say which window it is. +type Handle struct { + ID string `json:"id"` + URL string `json:"url"` + Title string `json:"title"` + Owner string `json:"owner"` + Headed bool `json:"headed"` +} + +// Target picks an element for click and type. Index is the one page-info +// stamped (data-lich-idx); selector is a CSS query; x/y are viewport +// coordinates. Coordinates win when set, then selector, then index. +type Target struct { + Index *int `json:"index,omitempty"` + Selector string `json:"selector,omitempty"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` +} + +func (t Target) empty() bool { + return t.Index == nil && t.Selector == "" && (t.X == nil || t.Y == nil) +} + +// Rect is an element's viewport box, integer pixels. +type Rect struct { + X int `json:"x"` + Y int `json:"y"` + W int `json:"w"` + H int `json:"h"` +} + +// Interactive is one clickable or fillable node from page-info. It never +// carries a control's value — filled/checked say that typing landed without +// putting the content in the transcript. +type Interactive struct { + Index int `json:"index"` + Tag string `json:"tag"` + Type string `json:"type,omitempty"` + Role string `json:"role,omitempty"` + Text string `json:"text"` + Href string `json:"href,omitempty"` + Rect Rect `json:"rect"` + Filled *bool `json:"filled,omitempty"` + Checked *bool `json:"checked,omitempty"` +} + +// PageInfo is what the agent reads instead of a screenshot: the URL, the +// visible text, and the interactive nodes it can click or type into. +type PageInfo struct { + URL string `json:"url"` + Title string `json:"title"` + Text string `json:"text"` + Interactive []Interactive `json:"interactive"` + Truncated bool `json:"truncated"` +} + +// Page is one Chromium tab the service drives. chrome.go is the real one; +// tests inject a fake. +type Page interface { + Navigate(ctx context.Context, url string) error + Info(ctx context.Context) (PageInfo, error) + Click(ctx context.Context, t Target) error + Type(ctx context.Context, text string, clear bool, t Target) error + Press(ctx context.Context, key string, t Target) error + Screenshot(ctx context.Context, dest string) error + Reload(ctx context.Context) error + Back(ctx context.Context) error + Forward(ctx context.Context) error + Scroll(ctx context.Context, dy int) error + Focus(ctx context.Context) error + Alive() bool + Close() error +} + +// Launcher starts a Chromium with its own user-data-dir. headed is a real +// window; false is headless. The directory is the caller's to delete. +type Launcher func(dir string, headed bool) (Page, error) diff --git a/internal/browser/pageinfo.go b/internal/browser/pageinfo.go new file mode 100644 index 000000000..f38fe37d5 --- /dev/null +++ b/internal/browser/pageinfo.go @@ -0,0 +1,83 @@ +package browser + +// pageInfoScript is injected into the page the agent is driving. It stamps +// data-lich-idx so a later click/type can find the same node, and it never +// reads a form control's value into the result. +// +// A password input has no innerText, so an innerText || value label chain +// hands the live credential to the agent and into the session transcript. +// Rather than classify which fields are sensitive — type === 'password' is +// defeated by a show-password toggle, and an ordinary text input can still +// hold a recovery phrase — no control value is serialized at all. Agents get +// filled / checked instead, which is enough to confirm that typing landed +// without the content leaving the page. +// +// A textarea is covered by the same rule: its content is its default value, so +// innerText on a textarea is a value read. The single exception is a +// push-button input (submit / button / reset), whose value is the author-written +// caption and the only label it has. A user cannot type into one. +const pageInfoScript = `(() => { + const sel = 'a[href], button, input, textarea, select, [role=button], [role=link], [onclick], [contenteditable=""], [contenteditable="true"]'; + const isVisible = (el) => { + const r = el.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return false; + if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth) return false; + const s = getComputedStyle(el); + return s.visibility !== 'hidden' && s.display !== 'none'; + }; + const CAPTION_TYPES = ['submit', 'button', 'reset']; + const attr = (el, name) => el.getAttribute(name) || ''; + const isCaptionInput = (tag, type) => + tag === 'input' && CAPTION_TYPES.indexOf(type) !== -1; + const labelFor = (el, tag, type) => { + if (tag === 'select') { + const opt = el.options ? el.options[el.selectedIndex] : null; + const optText = opt ? (opt.textContent || '').trim() : ''; + if (optText) return optText; + } else if (isCaptionInput(tag, type)) { + const caption = (el.value || '').trim(); + if (caption) return caption; + } else if (tag !== 'input' && tag !== 'textarea') { + const own = (el.innerText || '').trim(); + if (own) return own; + } + const labelled = el.labels && el.labels[0]; + const labelText = labelled ? (labelled.innerText || labelled.textContent || '').trim() : ''; + if (labelText) return labelText; + return ( + attr(el, 'aria-label') || + attr(el, 'title') || + attr(el, 'placeholder') || + attr(el, 'name') + ).trim(); + }; + const els = [...document.querySelectorAll(sel)].filter(isVisible); + const interactive = els.slice(0, 200).map((el, i) => { + el.setAttribute('data-lich-idx', String(i)); + const r = el.getBoundingClientRect(); + const tag = el.tagName.toLowerCase(); + const type = (el.getAttribute('type') || '').toLowerCase(); + const entry = { + index: i, + tag: tag, + type: el.getAttribute('type') || undefined, + role: el.getAttribute('role') || undefined, + text: labelFor(el, tag, type).slice(0, 120), + href: el.getAttribute('href') || undefined, + rect: { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height) }, + }; + if (type === 'checkbox' || type === 'radio') { + entry.checked = el.checked === true; + } else if (tag === 'textarea' || (tag === 'input' && !isCaptionInput(tag, type))) { + entry.filled = typeof el.value === 'string' && el.value.length > 0; + } + return entry; + }); + return { + url: location.href, + title: document.title, + text: (document.body ? document.body.innerText : '').trim().slice(0, 1200), + interactive: interactive, + truncated: els.length > 200, + }; +})()` diff --git a/internal/browser/pageinfo_test.go b/internal/browser/pageinfo_test.go new file mode 100644 index 000000000..ad0637de2 --- /dev/null +++ b/internal/browser/pageinfo_test.go @@ -0,0 +1,51 @@ +package browser + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestPageInfoScriptNeverSerializesControlValues(t *testing.T) { + if !strings.Contains(pageInfoScript, "data-lich-idx") { + t.Fatal("page-info must stamp data-lich-idx so click/type can find the node") + } + if strings.Contains(pageInfoScript, "data-nim-idx") { + t.Fatal("page-info still names the other product's attribute") + } + if strings.Contains(pageInfoScript, "innerText || el.value") || strings.Contains(pageInfoScript, "innerText || value") { + t.Fatal("innerText || value is how a password reaches the transcript") + } + if !strings.Contains(pageInfoScript, "entry.filled") { + t.Fatal("agents need filled, not the value, to know typing landed") + } + if !strings.Contains(pageInfoScript, "entry.checked") { + t.Fatal("agents need checked, not the value, for radios and checkboxes") + } +} + +func TestInteractiveJSONHasNoValueField(t *testing.T) { + filled, checked := true, false + el := Interactive{ + Index: 0, Tag: "input", Type: "password", Text: "Password", + Filled: &filled, Checked: &checked, + } + raw, err := json.Marshal(el) + if err != nil { + t.Fatal(err) + } + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + if _, ok := keys["value"]; ok { + t.Fatalf("interactive JSON leaked a value field: %s", raw) + } + body, err := json.Marshal(PageInfo{URL: "https://ex", Interactive: []Interactive{el}}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(body), `"value"`) { + t.Fatalf("page-info JSON leaked a value key: %s", body) + } +} diff --git a/internal/browser/rpc.go b/internal/browser/rpc.go new file mode 100644 index 000000000..78b162e96 --- /dev/null +++ b/internal/browser/rpc.go @@ -0,0 +1,119 @@ +package browser + +import ( + "context" + + "github.com/lichdotdev/lich/api" +) + +// The methods below are the RPC surface (api.Browser*). The ones they call +// keep the shape the package tests. Neither Cleanup nor CloseOwnedBy is +// routed: one ends every browser, the other is the session-close hook. + +// Show brings this session's browser on screen. +func (s *Service) Show(_ context.Context, opts api.SessionIDOptions) (api.BrowserHandle, error) { + h, err := s.OpenVisible(opts.ID) + return handleAPI(h), err +} + +// OpenURL navigates this session's browser, creating a headless one when the +// card has not opened a window. +func (s *Service) OpenURL(_ context.Context, opts api.BrowserOpenOptions) (api.BrowserHandle, error) { + h, err := s.Open(opts.ID, opts.URL) + return handleAPI(h), err +} + +// Read returns the page the agent is looking at. +func (s *Service) Read(_ context.Context, opts api.SessionIDOptions) (api.BrowserPageInfo, error) { + info, err := s.Info(opts.ID) + return infoAPI(info), err +} + +// ClickAt clicks in this session's browser. +func (s *Service) ClickAt(_ context.Context, opts api.BrowserClickOptions) (api.None, error) { + return api.None{}, s.Click(opts.ID, targetOf(opts.BrowserTarget)) +} + +// TypeText types into this session's browser. +func (s *Service) TypeText(_ context.Context, opts api.BrowserTypeOptions) (api.None, error) { + return api.None{}, s.Type(opts.ID, opts.Text, opts.Clear, targetOf(opts.BrowserTarget)) +} + +// PressKey presses one named key in this session's browser. +func (s *Service) PressKey(_ context.Context, opts api.BrowserPressOptions) (api.None, error) { + return api.None{}, s.Press(opts.ID, opts.Key, targetOf(opts.BrowserTarget)) +} + +// Capture writes a PNG of this session's page and returns its path. +func (s *Service) Capture(_ context.Context, opts api.BrowserScreenshotOptions) (string, error) { + return s.Screenshot(opts.ID, opts.Path) +} + +// Go loads a URL in this session's already-open browser. +func (s *Service) Go(_ context.Context, opts api.BrowserOpenOptions) (api.BrowserHandle, error) { + h, err := s.Navigate(opts.ID, opts.URL) + return handleAPI(h), err +} + +// ReloadPage reloads this session's page. +func (s *Service) ReloadPage(_ context.Context, opts api.SessionIDOptions) (api.None, error) { + return api.None{}, s.Reload(opts.ID) +} + +// GoBack goes back in this session's history. +func (s *Service) GoBack(_ context.Context, opts api.SessionIDOptions) (api.None, error) { + return api.None{}, s.Back(opts.ID) +} + +// GoForward goes forward in this session's history. +func (s *Service) GoForward(_ context.Context, opts api.SessionIDOptions) (api.None, error) { + return api.None{}, s.Forward(opts.ID) +} + +// ScrollBy scrolls this session's page. +func (s *Service) ScrollBy(_ context.Context, opts api.BrowserScrollOptions) (api.None, error) { + return api.None{}, s.Scroll(opts.ID, opts.DY) +} + +// Handles returns this session's browser, or an empty list. +func (s *Service) Handles(_ context.Context, opts api.SessionIDOptions) ([]api.BrowserHandle, error) { + list, err := s.List(opts.ID) + if err != nil { + return nil, err + } + out := make([]api.BrowserHandle, len(list)) + for i, h := range list { + out[i] = handleAPI(h) + } + return out, nil +} + +// CloseOne tears down this session's browser. +func (s *Service) CloseOne(_ context.Context, opts api.SessionIDOptions) (api.None, error) { + return api.None{}, s.Close(opts.ID) +} + +func handleAPI(h Handle) api.BrowserHandle { + return api.BrowserHandle{ID: h.ID, URL: h.URL, Title: h.Title, Owner: h.Owner, Headed: h.Headed} +} + +func infoAPI(p PageInfo) api.BrowserPageInfo { + out := api.BrowserPageInfo{URL: p.URL, Title: p.Title, Text: p.Text, Truncated: p.Truncated} + if p.Interactive == nil { + return out + } + out.Interactive = make([]api.BrowserInteractive, len(p.Interactive)) + for i, n := range p.Interactive { + out.Interactive[i] = api.BrowserInteractive{ + Index: n.Index, Tag: n.Tag, Type: n.Type, Role: n.Role, Text: n.Text, Href: n.Href, + Rect: api.BrowserRect{X: n.Rect.X, Y: n.Rect.Y, W: n.Rect.W, H: n.Rect.H}, + Filled: n.Filled, + Checked: n.Checked, + } + } + return out +} + +func targetOf(t api.BrowserTarget) Target { + return Target{Index: t.Index, Selector: t.Selector, X: t.X, Y: t.Y} +} diff --git a/internal/browser/url.go b/internal/browser/url.go new file mode 100644 index 000000000..47c692576 --- /dev/null +++ b/internal/browser/url.go @@ -0,0 +1,34 @@ +package browser + +import ( + "fmt" + "net/url" + "strings" +) + +// allowURL is the only URL the agent browser will load. http, https and +// about:blank are in; file, javascript, data, chrome and everything else are +// out — a file: URL would hand the agent the machine, and javascript: is just +// evaluate by another name. +func allowURL(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" || strings.EqualFold(raw, "about:blank") { + return "about:blank", nil + } + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("invalid url: %w", err) + } + switch strings.ToLower(u.Scheme) { + case "http", "https": + if u.Host == "" { + return "", fmt.Errorf("url %q has no host", raw) + } + return u.String(), nil + default: + if u.Scheme == "" { + return "", fmt.Errorf("url %q is missing a scheme", raw) + } + return "", fmt.Errorf("blocked url scheme %q", u.Scheme) + } +} diff --git a/internal/browser/url_test.go b/internal/browser/url_test.go new file mode 100644 index 000000000..57ded7d21 --- /dev/null +++ b/internal/browser/url_test.go @@ -0,0 +1,37 @@ +package browser + +import "testing" + +func TestAllowURL(t *testing.T) { + cases := []struct { + in, want, err string + }{ + {"", "about:blank", ""}, + {"about:blank", "about:blank", ""}, + {" ABOUT:BLANK ", "about:blank", ""}, + {"https://example.com/path", "https://example.com/path", ""}, + {"http://localhost:3000", "http://localhost:3000", ""}, + {"file:///etc/passwd", "", `blocked url scheme "file"`}, + {"javascript:alert(1)", "", `blocked url scheme "javascript"`}, + {"data:text/html,hi", "", `blocked url scheme "data"`}, + {"chrome://settings", "", `blocked url scheme "chrome"`}, + {"example.com", "", `url "example.com" is missing a scheme`}, + {"https://", "", `url "https://" has no host`}, + } + for _, c := range cases { + got, err := allowURL(c.in) + if c.err != "" { + if err == nil || err.Error() != c.err { + t.Errorf("allowURL(%q) err = %v, want %s", c.in, err, c.err) + } + continue + } + if err != nil { + t.Errorf("allowURL(%q): %v", c.in, err) + continue + } + if got != c.want { + t.Errorf("allowURL(%q) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/chromium/candidates_darwin.go b/internal/chromium/candidates_darwin.go new file mode 100644 index 000000000..ba05c5f83 --- /dev/null +++ b/internal/chromium/candidates_darwin.go @@ -0,0 +1,13 @@ +//go:build darwin + +package chromium + +import "os" + +// browserCandidates lists candidate browsers in preference order. macOS +// installs land in .app bundles under /Applications, not on PATH, so the list +// is absolute paths built from the install roots (darwinBrowserCandidates in +// find.go, where the logic stays testable from any OS). +func browserCandidates() []string { + return darwinBrowserCandidates(os.Getenv) +} diff --git a/internal/chromium/candidates_unix.go b/internal/chromium/candidates_unix.go new file mode 100644 index 000000000..f5077773c --- /dev/null +++ b/internal/chromium/candidates_unix.go @@ -0,0 +1,13 @@ +//go:build !windows && !darwin + +package chromium + +// browserCandidates lists candidate binaries in preference order — the shared +// chromiumNames list (find.go), since every Linux/BSD install lands on PATH +// under one of those names. Any Chromium gives the same CDP surface; the +// preference only picks the most conventional install. macOS keeps browsers in +// .app bundles off PATH and Windows under versioned install roots, so both have +// their own list. +func browserCandidates() []string { + return chromiumNames +} diff --git a/internal/chromium/candidates_unix_test.go b/internal/chromium/candidates_unix_test.go new file mode 100644 index 000000000..033776cef --- /dev/null +++ b/internal/chromium/candidates_unix_test.go @@ -0,0 +1,42 @@ +//go:build !windows && !darwin + +package chromium + +import ( + "slices" + "testing" +) + +// TestUnixBrowserCandidates pins the exact Linux/BSD candidate list in +// preference order: bare PATH names (Linux/BSD installs live on PATH), Chromium +// first. It mirrors the exact-list style of TestWindowsBrowserCandidates / +// TestDarwinBrowserCandidates — a membership check would miss an order +// regression or a stray entry — and it spells the list out rather than reading +// chromiumNames, which is the thing under test. +// +// The build tag matches candidates_unix.go (`!windows && !darwin`) on purpose: +// under a bare `!windows` this test also compiles into the darwin build, where +// browserCandidates() returns the .app list and none of these bare names, so +// the assertion would fail the macOS release suite. +func TestUnixBrowserCandidates(t *testing.T) { + want := []string{ + "chromium", + "chromium-browser", + "google-chrome", + "google-chrome-stable", + "google-chrome-beta", + "helium-browser", + "brave", + "brave-browser", + "vivaldi", + "vivaldi-stable", + "microsoft-edge", + "microsoft-edge-stable", + "thorium-browser", + "ungoogled-chromium", + "chrome", + } + if got := browserCandidates(); !slices.Equal(got, want) { + t.Fatalf("browserCandidates() = %v, want %v", got, want) + } +} diff --git a/internal/chromium/candidates_windows.go b/internal/chromium/candidates_windows.go new file mode 100644 index 000000000..a5c5df4f3 --- /dev/null +++ b/internal/chromium/candidates_windows.go @@ -0,0 +1,13 @@ +//go:build windows + +package chromium + +import "os" + +// browserCandidates lists candidate browsers in preference order. Windows +// installs don't land on PATH, so the list is mostly absolute paths built +// from the install-root environment variables (windowsBrowserCandidates in +// find.go, where the logic stays testable from any OS). +func browserCandidates() []string { + return windowsBrowserCandidates(os.Getenv) +} diff --git a/internal/chromium/find.go b/internal/chromium/find.go new file mode 100644 index 000000000..697c75f48 --- /dev/null +++ b/internal/chromium/find.go @@ -0,0 +1,102 @@ +package chromium + +import ( + "errors" + "fmt" +) + +// ErrNoBrowser is a machine with no Chromium-family browser the agent sidecar +// can drive. Distinct from ErrNoShell: the UI window is lich-shell; this is +// Chrome/Edge/Chromium/… for CDP (internal/browser). +var ErrNoBrowser = errors.New( + "no chromium-family browser found — install chromium, chrome, brave, vivaldi or edge") + +// FindBrowser returns the first Chromium-family binary that resolves, trying +// this OS's candidates (candidates_*.go) in preference order. lookPath is +// injectable for tests (production passes exec.LookPath, which also accepts +// the absolute paths the Windows and macOS lists use). +// +// This is the agent-browser sidecar only — never the lich UI window, which +// Resolve pins to LICH_SHELL / lich-shell (#570). +func FindBrowser(lookPath func(name string) (string, error)) (string, error) { + candidates := browserCandidates() + for _, name := range candidates { + if path, err := lookPath(name); err == nil { + return path, nil + } + } + return "", fmt.Errorf("%w (tried %v)", ErrNoBrowser, candidates) +} + +// chromiumNames are the Chromium-family executables on Linux/BSD PATH, in +// preference order. candidates_unix.go returns this list; Windows and macOS +// build absolute paths under the install roots instead. +var chromiumNames = []string{ + "chromium", + "chromium-browser", + "google-chrome", + "google-chrome-stable", + "google-chrome-beta", + "helium-browser", + "brave", + "brave-browser", + "vivaldi", + "vivaldi-stable", + "microsoft-edge", + "microsoft-edge-stable", + "thorium-browser", + "ungoogled-chromium", + "chrome", +} + +// windowsBrowserCandidates builds the Windows candidate list: chrome, then +// edge (present on every Windows), then brave and vivaldi, each under the +// install roots Windows exposes as environment variables, with bare PATH names +// last. Paths are joined with a literal backslash so the pure logic tests the +// same on any OS. Kept out of the build-tagged file for exactly that reason. +func windowsBrowserCandidates(getenv func(string) string) []string { + roots := []struct{ env, rel string }{ + {"ProgramFiles", `Google\Chrome\Application\chrome.exe`}, + {"ProgramFiles(x86)", `Google\Chrome\Application\chrome.exe`}, + {"LocalAppData", `Google\Chrome\Application\chrome.exe`}, + {"ProgramFiles(x86)", `Microsoft\Edge\Application\msedge.exe`}, + {"ProgramFiles", `Microsoft\Edge\Application\msedge.exe`}, + {"ProgramFiles", `BraveSoftware\Brave-Browser\Application\brave.exe`}, + {"LocalAppData", `BraveSoftware\Brave-Browser\Application\brave.exe`}, + {"ProgramFiles", `Vivaldi\Application\vivaldi.exe`}, + {"LocalAppData", `Vivaldi\Application\vivaldi.exe`}, + } + var out []string + for _, r := range roots { + if root := getenv(r.env); root != "" { + out = append(out, root+`\`+r.rel) + } + } + return append(out, "chrome", "msedge") +} + +// darwinBrowserCandidates builds the macOS candidate list: chrome, then +// chromium, then edge, then brave, then vivaldi, each as its .app executable +// under the system (/Applications) and per-user (~/Applications) install roots, +// with bare PATH names last for a Homebrew-formula install. Paths are joined +// with a literal slash so the pure logic tests the same on any OS. +func darwinBrowserCandidates(getenv func(string) string) []string { + apps := []string{ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Chromium.app/Contents/MacOS/Chromium", + "Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "Brave Browser.app/Contents/MacOS/Brave Browser", + "Vivaldi.app/Contents/MacOS/Vivaldi", + } + roots := []string{"/Applications"} + if home := getenv("HOME"); home != "" { + roots = append(roots, home+"/Applications") + } + var out []string + for _, app := range apps { + for _, root := range roots { + out = append(out, root+"/"+app) + } + } + return append(out, "chromium", "google-chrome") +} diff --git a/internal/chromium/find_test.go b/internal/chromium/find_test.go new file mode 100644 index 000000000..4707347ea --- /dev/null +++ b/internal/chromium/find_test.go @@ -0,0 +1,125 @@ +package chromium + +import ( + "errors" + "slices" + "testing" +) + +// TestFindBrowserPicksFirstHit proves missing candidates are skipped and +// preference order decides among the installed ones — against whichever +// candidate list this OS compiles in. +func TestFindBrowserPicksFirstHit(t *testing.T) { + candidates := browserCandidates() + if len(candidates) < 2 { + t.Fatal("candidate list too short to prove ordering") + } + hits := map[string]bool{candidates[1]: true, candidates[len(candidates)-1]: true} + lookPath := func(name string) (string, error) { + if hits[name] { + return "/resolved/" + name, nil + } + return "", errors.New("not found") + } + got, err := FindBrowser(lookPath) + if err != nil { + t.Fatalf("FindBrowser: %v", err) + } + if want := "/resolved/" + candidates[1]; got != want { + t.Fatalf("FindBrowser = %q, want the earliest installed candidate %q", got, want) + } +} + +func TestFindBrowserErrorsWhenNoneInstalled(t *testing.T) { + lookPath := func(string) (string, error) { return "", errors.New("not found") } + if _, err := FindBrowser(lookPath); err == nil { + t.Fatal("want error when no browser is on PATH") + } +} + +// TestWindowsBrowserCandidates proves the Windows list expands only the +// install roots present in the environment, prefers chrome > edge > brave > +// vivaldi, and always ends with the bare PATH names as a last resort. +func TestWindowsBrowserCandidates(t *testing.T) { + env := map[string]string{ + "ProgramFiles": `C:\Program Files`, + "ProgramFiles(x86)": `C:\Program Files (x86)`, + "LocalAppData": `C:\Users\u\AppData\Local`, + } + got := windowsBrowserCandidates(func(k string) string { return env[k] }) + + want := []string{ + `C:\Program Files\Google\Chrome\Application\chrome.exe`, + `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`, + `C:\Users\u\AppData\Local\Google\Chrome\Application\chrome.exe`, + `C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe`, + `C:\Program Files\Microsoft\Edge\Application\msedge.exe`, + `C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe`, + `C:\Users\u\AppData\Local\BraveSoftware\Brave-Browser\Application\brave.exe`, + `C:\Program Files\Vivaldi\Application\vivaldi.exe`, + `C:\Users\u\AppData\Local\Vivaldi\Application\vivaldi.exe`, + "chrome", + "msedge", + } + if !slices.Equal(got, want) { + t.Fatalf("candidates = %v, want %v", got, want) + } +} + +// TestWindowsBrowserCandidatesEmptyEnv proves a bare environment still leaves +// the PATH names, so FindBrowser never iterates an empty list. +func TestWindowsBrowserCandidatesEmptyEnv(t *testing.T) { + got := windowsBrowserCandidates(func(string) string { return "" }) + if !slices.Equal(got, []string{"chrome", "msedge"}) { + t.Fatalf("candidates = %v, want PATH names only", got) + } +} + +// TestDarwinBrowserCandidates proves the macOS list expands each browser under +// both the system and per-user Applications roots, prefers chrome > chromium > +// edge > brave > vivaldi, and ends with the bare PATH names. +func TestDarwinBrowserCandidates(t *testing.T) { + got := darwinBrowserCandidates(func(k string) string { + if k == "HOME" { + return "/Users/u" + } + return "" + }) + + want := []string{ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Users/u/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "/Users/u/Applications/Chromium.app/Contents/MacOS/Chromium", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Users/u/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", + "/Users/u/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", + "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi", + "/Users/u/Applications/Vivaldi.app/Contents/MacOS/Vivaldi", + "chromium", + "google-chrome", + } + if !slices.Equal(got, want) { + t.Fatalf("candidates = %v, want %v", got, want) + } +} + +// TestDarwinBrowserCandidatesNoHome proves a missing HOME drops the per-user +// root but still leaves the system paths and PATH names, so FindBrowser never +// iterates an empty list. +func TestDarwinBrowserCandidatesNoHome(t *testing.T) { + got := darwinBrowserCandidates(func(string) string { return "" }) + want := []string{ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", + "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi", + "chromium", + "google-chrome", + } + if !slices.Equal(got, want) { + t.Fatalf("candidates = %v, want %v", got, want) + } +} diff --git a/internal/cli/browser.go b/internal/cli/browser.go new file mode 100644 index 000000000..5159737db --- /dev/null +++ b/internal/cli/browser.go @@ -0,0 +1,283 @@ +package cli + +import ( + "context" + "flag" + "fmt" + "strconv" + "strings" + + "github.com/lichdotdev/lich/api" +) + +func (c *client) browser(args []string) error { + if len(args) == 0 { + return usageError("browser") + } + if args[0] == "-h" || args[0] == "--help" { + printHelp(c.stdout, newFlagSet("browser")) + return errHelpShown + } + switch args[0] { + case "open": + return c.browserOpen(args[1:]) + case "info": + return c.browserInfo(args[1:]) + case "click": + return c.browserClick(args[1:]) + case "type": + return c.browserType(args[1:]) + case "press": + return c.browserPress(args[1:]) + case "screenshot": + return c.browserScreenshot(args[1:]) + case "navigate": + return c.browserNavigate(args[1:]) + case "reload": + return c.browserID(api.BrowserReload, args[1:]) + case "back": + return c.browserID(api.BrowserBack, args[1:]) + case "forward": + return c.browserID(api.BrowserForward, args[1:]) + case "scroll": + return c.browserScroll(args[1:]) + case "list": + return c.browserList(args[1:]) + case "close": + return c.browserID(api.BrowserClose, args[1:]) + default: + return usageError("browser") + } +} + +func (c *client) browserOpen(args []string) error { + flags := newFlagSet("browser") + if err := c.parse(flags, args); err != nil { + return err + } + id, err := c.browserOwner() + if err != nil { + return err + } + h, err := callMethod(context.Background(), c, api.BrowserOpen, api.BrowserOpenOptions{ + ID: id, URL: strings.Join(flags.Args(), " "), + }, openCall) + if err != nil { + return err + } + fmt.Fprintln(c.stdout, h.URL) + return nil +} + +func (c *client) browserInfo(args []string) error { + id, err := c.browserOwnerArgs(args) + if err != nil { + return err + } + info, err := callMethod(context.Background(), c, api.BrowserInfo, api.SessionIDOptions{ID: id}, openCall) + if err != nil { + return err + } + return c.emit(info) +} + +func (c *client) browserClick(args []string) error { + flags := newFlagSet("browser") + tf := registerTarget(flags) + if err := c.parse(flags, args); err != nil { + return err + } + t := tf.target() + if t.Index == nil && t.Selector == "" && (t.X == nil || t.Y == nil) { + return fmt.Errorf("click needs --index, --selector, or --x and --y") + } + id, err := c.browserOwner() + if err != nil { + return err + } + _, err = callMethod(context.Background(), c, api.BrowserClick, api.BrowserClickOptions{ID: id, BrowserTarget: t}, shortCall) + return err +} + +func (c *client) browserType(args []string) error { + flags := newFlagSet("browser") + clear := flags.Bool("clear", false, "replace the field before typing") + tf := registerTarget(flags) + if err := c.parse(flags, args); err != nil { + return err + } + text := strings.Join(flags.Args(), " ") + if text == "" { + return usageError("browser") + } + id, err := c.browserOwner() + if err != nil { + return err + } + _, err = callMethod(context.Background(), c, api.BrowserType, api.BrowserTypeOptions{ + ID: id, Text: text, Clear: *clear, BrowserTarget: tf.target(), + }, shortCall) + return err +} + +func (c *client) browserPress(args []string) error { + flags := newFlagSet("browser") + tf := registerTarget(flags) + if err := c.parse(flags, args); err != nil { + return err + } + key := strings.Join(flags.Args(), " ") + if key == "" { + return usageError("browser") + } + id, err := c.browserOwner() + if err != nil { + return err + } + _, err = callMethod(context.Background(), c, api.BrowserPress, api.BrowserPressOptions{ + ID: id, Key: key, BrowserTarget: tf.target(), + }, shortCall) + return err +} + +func (c *client) browserScreenshot(args []string) error { + flags := newFlagSet("browser") + if err := c.parse(flags, args); err != nil { + return err + } + id, err := c.browserOwner() + if err != nil { + return err + } + dest := "" + if flags.NArg() > 0 { + dest = flags.Arg(0) + } + path, err := callMethod(context.Background(), c, api.BrowserScreenshot, api.BrowserScreenshotOptions{ID: id, Path: dest}, openCall) + if err != nil { + return err + } + fmt.Fprintln(c.stdout, path) + return nil +} + +func (c *client) browserNavigate(args []string) error { + flags := newFlagSet("browser") + if err := c.parse(flags, args); err != nil { + return err + } + raw := strings.Join(flags.Args(), " ") + if raw == "" { + return usageError("browser") + } + id, err := c.browserOwner() + if err != nil { + return err + } + h, err := callMethod(context.Background(), c, api.BrowserNavigate, api.BrowserOpenOptions{ID: id, URL: raw}, openCall) + if err != nil { + return err + } + fmt.Fprintln(c.stdout, h.URL) + return nil +} + +func (c *client) browserScroll(args []string) error { + flags := newFlagSet("browser") + dy := flags.Int("dy", 400, "pixels to scroll (positive is down)") + if err := c.parse(flags, args); err != nil { + return err + } + id, err := c.browserOwner() + if err != nil { + return err + } + _, err = callMethod(context.Background(), c, api.BrowserScroll, api.BrowserScrollOptions{ID: id, DY: *dy}, shortCall) + return err +} + +func (c *client) browserList(args []string) error { + id, err := c.browserOwnerArgs(args) + if err != nil { + return err + } + list, err := callMethod(context.Background(), c, api.BrowserList, api.SessionIDOptions{ID: id}, shortCall) + if err != nil { + return err + } + if list == nil { + list = []api.BrowserHandle{} + } + return c.emit(list) +} + +func (c *client) browserID(method api.Method[api.SessionIDOptions, api.None], args []string) error { + id, err := c.browserOwnerArgs(args) + if err != nil { + return err + } + _, err = callMethod(context.Background(), c, method, api.SessionIDOptions{ID: id}, shortCall) + return err +} + +func (c *client) browserOwnerArgs(args []string) (string, error) { + if err := c.browserNoArgs(args); err != nil { + return "", err + } + return c.browserOwner() +} + +func (c *client) browserNoArgs(args []string) error { + flags := newFlagSet("browser") + if err := c.parse(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return usageError("browser") + } + return nil +} + +type targetFlags struct { + index *int + selector *string + x, y *float64 +} + +func registerTarget(flags *flag.FlagSet) *targetFlags { + t := &targetFlags{} + t.selector = flags.String("selector", "", "CSS selector") + flags.Func("index", "element index from browser info", func(s string) error { + n, err := strconv.Atoi(s) + if err != nil { + return err + } + t.index = &n + return nil + }) + flags.Func("x", "viewport X", func(s string) error { + n, err := strconv.ParseFloat(s, 64) + if err != nil { + return err + } + t.x = &n + return nil + }) + flags.Func("y", "viewport Y", func(s string) error { + n, err := strconv.ParseFloat(s, 64) + if err != nil { + return err + } + t.y = &n + return nil + }) + return t +} + +func (t *targetFlags) target() api.BrowserTarget { + sel := "" + if t.selector != nil { + sel = *t.selector + } + return api.BrowserTarget{Index: t.index, Selector: sel, X: t.x, Y: t.y} +} diff --git a/internal/cli/browser_test.go b/internal/cli/browser_test.go new file mode 100644 index 000000000..7e8099d99 --- /dev/null +++ b/internal/cli/browser_test.go @@ -0,0 +1,104 @@ +package cli + +import ( + "bytes" + "strings" + "testing" + + "github.com/lichdotdev/lich/api" +) + +func TestBrowserOpenCallsThisSessionsBrowser(t *testing.T) { + f := newFakeLich(t, `{"id":"b1","url":"https://example.com","owner":"s1","headed":false}`) + + code, stdout, stderr := run(t, f, "browser", "open", "https://example.com") + if code != 0 { + t.Fatalf("exit = %d, stderr = %q", code, stderr) + } + opts := optionsOf[api.BrowserOpenOptions](t, f.only(t)) + if opts.ID != "s1" || opts.URL != "https://example.com" { + t.Errorf("options = %+v", opts) + } + if !strings.Contains(stdout, "https://example.com") { + t.Errorf("stdout = %q", stdout) + } +} + +func TestBrowserClickSendsATarget(t *testing.T) { + f := newFakeLich(t, `null`) + + code, _, stderr := run(t, f, "browser", "click", "--index", "3") + if code != 0 { + t.Fatalf("exit = %d, stderr = %q", code, stderr) + } + opts := optionsOf[api.BrowserClickOptions](t, f.only(t)) + if opts.ID != "s1" || opts.Index == nil || *opts.Index != 3 { + t.Errorf("options = %+v", opts) + } +} + +func TestBrowserPressSendsKeyAndOptionalTarget(t *testing.T) { + f := newFakeLich(t, `null`) + + code, _, stderr := run(t, f, "browser", "press", "--index", "2", "Enter") + if code != 0 { + t.Fatalf("exit = %d, stderr = %q", code, stderr) + } + opts := optionsOf[api.BrowserPressOptions](t, f.only(t)) + if opts.ID != "s1" || opts.Key != "Enter" || opts.Index == nil || *opts.Index != 2 { + t.Errorf("options = %+v", opts) + } +} + +func TestMCPBrowserPressCallsPress(t *testing.T) { + f := newFakeLich(t, `null`) + + replies := speak(t, f, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params": + {"name":"browser_press","arguments":{"key":"Tab"}}}`) + text, failed := textOf(t, replies[0]) + if failed { + t.Fatalf("tool reported a failure: %s", text) + } + opts := optionsOf[api.BrowserPressOptions](t, f.only(t)) + if opts.ID != "s1" || opts.Key != "Tab" { + t.Errorf("options = %+v", opts) + } +} + +func TestBrowserRefusesWithoutASession(t *testing.T) { + f := newFakeLich(t, `null`) + var stdout, stderr bytes.Buffer + code := dispatch([]string{"browser", "open", "https://example.com"}, &client{ + env: func(key string) string { + if key == "LICH_SESSION_ID" { + return "" + } + return f.env(key) + }, + version: testVersion, stdout: &stdout, stderr: &stderr, running: noInstance, + }) + if code != 1 { + t.Fatalf("exit = %d, want 1", code) + } + if !strings.Contains(stderr.String(), "inside a lich session") { + t.Errorf("stderr = %q", stderr.String()) + } + if len(f.calls) != 0 { + t.Errorf("reached the app without a session: %+v", f.calls) + } +} + +func TestMCPBrowserOpenUsesTheCaller(t *testing.T) { + f := newFakeLich(t, `{"id":"b1","url":"https://example.com","owner":"s1","headed":false}`) + + replies := speak(t, f, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params": + {"name":"browser_open","arguments":{"url":"https://example.com"}}}`) + text, failed := textOf(t, replies[0]) + if failed { + t.Fatalf("tool reported a failure: %s", text) + } + opts := optionsOf[api.BrowserOpenOptions](t, f.only(t)) + if opts.ID != "s1" || opts.URL != "https://example.com" { + t.Errorf("options = %+v", opts) + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index adf9fa608..95840f11c 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -189,6 +189,8 @@ func dispatch(args []string, c *client) int { return c.run(c.quit, args[1:]) case "rage": return c.run(c.rage, args[1:]) + case "browser": + return c.run(c.browser, args[1:]) case "doctor": // Not through run: this command's failure is the report it just // printed, and a second "lich: …" line on stderr would only say it diff --git a/internal/cli/help.go b/internal/cli/help.go index ee8a1ff53..c6c8a679c 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -132,6 +132,13 @@ var commands = []command{ "to 90 seconds; an answer that comes later is dropped.\n" + "Experimental: may change outside semver.", }, + { + name: "browser", + args: " [args]", + about: "Drive this session's browser — the same window the card's Browser item opens.\n" + + "Verbs: open [url], info, click, type, press , screenshot [path], navigate ,\n" + + "reload, back, forward, scroll, list, close.", + }, { name: "worktrees", args: "[--project ] [--json]", diff --git a/internal/cli/mcp.go b/internal/cli/mcp.go index f4b0c3e4a..9316aa85f 100644 --- a/internal/cli/mcp.go +++ b/internal/cli/mcp.go @@ -465,8 +465,9 @@ func property(kind, description string) map[string]any { } // mcpTools is the whole tool surface. Adding one is an entry here, or in -// folderTools, controlTools or askTools. That is why registering MCP once buys every -// tool lich grows later, on every provider, with no further per-provider work. +// folderTools, controlTools, askTools or browserTools. That is why registering +// MCP once buys every tool lich grows later, on every provider that took the +// registration, with no further per-provider work. var mcpTools = append([]mcpTool{ { Name: "list_sessions", @@ -754,7 +755,7 @@ var mcpTools = append([]mcpTool{ return "Answer sent.", nil }, }, -}, append(append(folderTools, controlTools...), askTools...)...) +}, append(append(append(folderTools, controlTools...), askTools...), browserTools...)...) // handOver gives a just-opened session its first task, wording the outcome the // way send_to_session words it — the caller should not have to learn two diff --git a/internal/cli/mcp_browser.go b/internal/cli/mcp_browser.go new file mode 100644 index 000000000..10570a15e --- /dev/null +++ b/internal/cli/mcp_browser.go @@ -0,0 +1,332 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/lichdotdev/lich/api" +) + +// browserTools are this session's Chromium window — the same one the card's +// Browser item opens. Appended to mcpTools, so tools/list cannot drift from +// the commands. +var browserTools = []mcpTool{ + { + Name: "browser_open", + Description: "Open a page in this lich session's browser. Creates the browser if " + + "this session has none (headless unless the user already opened Browser on " + + "this card). The URL must be http, https, or omitted (about:blank). file, " + + "javascript and data URLs are refused. This is not the lich window.", + Schema: schema(map[string]any{ + "url": property("string", "Page to load. Omit for about:blank."), + }), + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + h, err := callMethod(ctx, c, api.BrowserOpen, api.BrowserOpenOptions{ID: id, URL: a.text("url")}, openCall) + if err != nil { + return "", err + } + return encodeJSON(h) + }, + }, + { + Name: "browser_info", + Description: "Read this session's current page: URL, title, the text in the viewport, and " + + "interactive elements there (index, tag, label, href, box). Form values are never " + + "included — filled/checked say that typing landed. Call this before click/type " + + "so indexes match. Read-only.", + Schema: schema(map[string]any{}), + ReadOnly: true, + Run: mcpBrowserInfo, + }, + { + Name: "browser_click", + Description: "Click in this session's browser. Prefer index from browser_info, or a " + + "CSS selector, or viewport x and y. Sends a real mouse event, not element.click().", + Schema: schema(map[string]any{ + "index": property("integer", "Index from the last browser_info."), + "selector": property("string", "CSS selector."), + "x": property("number", "Viewport X."), + "y": property("number", "Viewport Y."), + }), + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + return mcpBrowserClick(ctx, c, a) + }, + }, + { + Name: "browser_type", + Description: "Type into this session's browser. Target as for browser_click; omit it " + + "to type into whatever is focused. clear replaces the field first. The text never " + + "comes back through browser_info.", + Schema: schema(map[string]any{ + "text": property("string", "Keys to send."), + "clear": property("boolean", "Replace the field before typing."), + "index": property("integer", "Index from the last browser_info."), + "selector": property("string", "CSS selector."), + "x": property("number", "Viewport X."), + "y": property("number", "Viewport Y."), + }, "text"), + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + _, err = callMethod(ctx, c, api.BrowserType, api.BrowserTypeOptions{ + ID: id, Text: a.text("text"), Clear: a.flag("clear"), BrowserTarget: mcpTarget(a), + }, shortCall) + if err != nil { + return "", err + } + return "ok", nil + }, + }, + { + Name: "browser_press", + Description: "Press one named key in this session's browser (Enter, Tab, Escape, " + + "Backspace, Delete, Space, ArrowUp/Down/Left/Right, Home, End, PageUp, PageDown). " + + "Real key event, not typed text — use browser_type for that. Optional target " + + "focuses first, as for browser_type.", + Schema: schema(map[string]any{ + "key": property("string", "Named key: Enter, Tab, Escape, …"), + "index": property("integer", "Index from the last browser_info."), + "selector": property("string", "CSS selector."), + "x": property("number", "Viewport X."), + "y": property("number", "Viewport Y."), + }, "key"), + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + _, err = callMethod(ctx, c, api.BrowserPress, api.BrowserPressOptions{ + ID: id, Key: a.text("key"), BrowserTarget: mcpTarget(a), + }, shortCall) + if err != nil { + return "", err + } + return "ok", nil + }, + }, + { + Name: "browser_screenshot", + Description: "Capture this session's page to a PNG file and return the path. Do not " + + "inline the bytes. Optional path; otherwise lich writes one next to the browser profile.", + Schema: schema(map[string]any{ + "path": property("string", "Destination PNG. Omit to let lich choose."), + }), + ReadOnly: true, + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + return callMethod(ctx, c, api.BrowserScreenshot, api.BrowserScreenshotOptions{ID: id, Path: a.text("path")}, openCall) + }, + }, + { + Name: "browser_navigate", + Description: "Load a URL in this session's already-open browser. Same allowlist as browser_open.", + Schema: schema(map[string]any{ + "url": property("string", "http(s) URL, or about:blank."), + }, "url"), + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + h, err := callMethod(ctx, c, api.BrowserNavigate, api.BrowserOpenOptions{ID: id, URL: a.text("url")}, openCall) + if err != nil { + return "", err + } + return encodeJSON(h) + }, + }, + { + Name: "browser_reload", + Description: "Reload this session's current page.", + Schema: schema(map[string]any{}), + Run: func(ctx context.Context, c *client, _ mcpArgs) (string, error) { + return mcpBrowserID(ctx, c, api.BrowserReload) + }, + }, + { + Name: "browser_back", + Description: "Go back in this session's browser history.", + Schema: schema(map[string]any{}), + Run: func(ctx context.Context, c *client, _ mcpArgs) (string, error) { + return mcpBrowserID(ctx, c, api.BrowserBack) + }, + }, + { + Name: "browser_forward", + Description: "Go forward in this session's browser history.", + Schema: schema(map[string]any{}), + Run: func(ctx context.Context, c *client, _ mcpArgs) (string, error) { + return mcpBrowserID(ctx, c, api.BrowserForward) + }, + }, + { + Name: "browser_scroll", + Description: "Scroll this session's page. dy is CSS pixels; positive is down. Default 400.", + Schema: schema(map[string]any{ + "dy": property("integer", "Pixels to scroll. Default 400."), + }), + Run: func(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + dy := 400 + if n := a.optionalInt("dy"); n != nil { + dy = *n + } + _, err = callMethod(ctx, c, api.BrowserScroll, api.BrowserScrollOptions{ID: id, DY: dy}, shortCall) + if err != nil { + return "", err + } + return "ok", nil + }, + }, + { + Name: "browser_list", + Description: "This session's browser, or an empty list. Read-only.", + Schema: schema(map[string]any{}), + ReadOnly: true, + Run: mcpBrowserList, + }, + { + Name: "browser_close", + Description: "Close this session's browser. The lich window is untouched. Closing the " + + "session card also closes it.", + Schema: schema(map[string]any{}), + Run: func(ctx context.Context, c *client, _ mcpArgs) (string, error) { + return mcpBrowserID(ctx, c, api.BrowserClose) + }, + }, +} + +func (c *client) browserOwner() (string, error) { + id := c.sessionID() + if id == "" { + return "", fmt.Errorf("browser tools run inside a lich session") + } + return id, nil +} + +func mcpTarget(a mcpArgs) api.BrowserTarget { + return api.BrowserTarget{ + Selector: a.text("selector"), + Index: a.optionalInt("index"), + X: a.optionalFloat("x"), + Y: a.optionalFloat("y"), + } +} + +func mcpBrowserClick(ctx context.Context, c *client, a mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + _, err = callMethod(ctx, c, api.BrowserClick, api.BrowserClickOptions{ID: id, BrowserTarget: mcpTarget(a)}, shortCall) + if err != nil { + return "", err + } + return "ok", nil +} + +func mcpBrowserID(ctx context.Context, c *client, method api.Method[api.SessionIDOptions, api.None]) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + if _, err := callMethod(ctx, c, method, api.SessionIDOptions{ID: id}, shortCall); err != nil { + return "", err + } + return "ok", nil +} + +func encodeJSON(v any) (string, error) { + b, err := json.Marshal(v) + if err != nil { + return "", err + } + return string(b), nil +} + +func mcpBrowserInfo(ctx context.Context, c *client, _ mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + info, err := callMethod(ctx, c, api.BrowserInfo, api.SessionIDOptions{ID: id}, openCall) + if err != nil { + return "", err + } + return encodeJSON(info) +} + +func mcpBrowserList(ctx context.Context, c *client, _ mcpArgs) (string, error) { + id, err := c.browserOwner() + if err != nil { + return "", err + } + list, err := callMethod(ctx, c, api.BrowserList, api.SessionIDOptions{ID: id}, shortCall) + if err != nil { + return "", err + } + if list == nil { + list = []api.BrowserHandle{} + } + return encodeJSON(list) +} + +// optionalInt reads a number that may be absent. Missing, empty, or a type +// this cannot parse is nil rather than zero: zero is a valid element index. +func (a mcpArgs) optionalInt(key string) *int { + v, ok := a[key] + if !ok || v == nil { + return nil + } + n := 0 + switch x := v.(type) { + case float64: + n = int(x) + case string: + if x == "" { + return nil + } + if _, err := fmt.Sscanf(x, "%d", &n); err != nil { + return nil + } + default: + return nil + } + return &n +} + +// optionalFloat is optionalInt for viewport coordinates. +func (a mcpArgs) optionalFloat(key string) *float64 { + v, ok := a[key] + if !ok || v == nil { + return nil + } + n := 0.0 + switch x := v.(type) { + case float64: + n = x + case string: + if x == "" { + return nil + } + if _, err := fmt.Sscanf(x, "%f", &n); err != nil { + return nil + } + default: + return nil + } + return &n +} diff --git a/internal/cli/mcp_test.go b/internal/cli/mcp_test.go index b888daeec..9eb100a09 100644 --- a/internal/cli/mcp_test.go +++ b/internal/cli/mcp_test.go @@ -107,7 +107,7 @@ func TestMCPHandshakeAdvertisesTools(t *testing.T) { // the one thing they get wrong on their own: that fanning work out means // these sessions and not the subagents their own harness offers. for _, want := range []string{ - "open_session", "send_to_session", "wait_for_answer", "worktree", "subagent", + "open_session", "send_to_session", "wait_for_answer", "worktree", "subagent", "browser_", } { if !strings.Contains(result.Instructions, want) { t.Errorf("instructions are missing %q:\n%s", want, result.Instructions) @@ -228,24 +228,40 @@ func TestMCPListsEveryTool(t *testing.T) { resultOf(t, replies[0], &result) want := map[string][]string{ - "list_sessions": {}, - "open_session": {}, - "close_session": {"session"}, - "rename_session": {"label"}, - "list_worktrees": {}, - "send_to_session": {"session", "prompt"}, - "wait_for_answer": {}, - "reply_to_session": {"answer"}, - "list_folders": {}, - "file_session": {"folder"}, - "rename_folder": {"folder", "to"}, - "color_folder": {"folder", "color"}, - "control_session": {"session", "action"}, - "ask_session": {"session", "question"}, + "list_sessions": {}, + "open_session": {}, + "close_session": {"session"}, + "rename_session": {"label"}, + "list_worktrees": {}, + "send_to_session": {"session", "prompt"}, + "wait_for_answer": {}, + "reply_to_session": {"answer"}, + "list_folders": {}, + "file_session": {"folder"}, + "rename_folder": {"folder", "to"}, + "color_folder": {"folder", "color"}, + "control_session": {"session", "action"}, + "ask_session": {"session", "question"}, + "browser_open": {}, + "browser_info": {}, + "browser_click": {}, + "browser_type": {"text"}, + "browser_press": {"key"}, + "browser_screenshot": {}, + "browser_navigate": {"url"}, + "browser_reload": {}, + "browser_back": {}, + "browser_forward": {}, + "browser_scroll": {}, + "browser_list": {}, + "browser_close": {}, } // The read-only ones, which a client may auto-allow. wait_for_answer is // not among them: collecting drains the inbox. - readOnly := map[string]bool{"list_sessions": true, "list_worktrees": true, "list_folders": true} + readOnly := map[string]bool{ + "list_sessions": true, "list_worktrees": true, "list_folders": true, + "browser_info": true, "browser_list": true, "browser_screenshot": true, + } if len(result.Tools) != len(want) { t.Fatalf("got %d tools, want %d", len(result.Tools), len(want)) } diff --git a/internal/prompt/en.go b/internal/prompt/en.go index 3c5b5d732..52de2dec6 100644 --- a/internal/prompt/en.go +++ b/internal/prompt/en.go @@ -77,7 +77,9 @@ Never poll for results. A send that outlives its wait hands back a ticket and yo A subagent or workflow step running inside this session is not its agent, and lich cannot tell them apart: it sends and opens with private, then waits on its own tickets. The [lich] note and the no-ticket wait_for_answer belong to the session's own agent. -When a [lich] message carrying a ticket arrives at YOUR prompt, you are the worker: do the task, then answer with %s — a concise report (what was done, where, what remains), never a transcript.`, +When a [lich] message carrying a ticket arrives at YOUR prompt, you are the worker: do the task, then answer with %s — a concise report (what was done, where, what remains), never a transcript. + +This session has a browser of its own (browser_* tools): the same Chromium window the card's Browser item opens, not the lich window. Open it, read the page, click, type, press named keys, and screenshot. Screenshots are files. Never paste their bytes into a reply.`, HandOverFailed: "The task did not reach it: %[1]v. The session is open, so hand it the task with " + "%[2]s once whatever that says is dealt with.", SendToSessionPrivate: "send_to_session with private", diff --git a/internal/prompt/es.go b/internal/prompt/es.go index 872a75b35..13052e961 100644 --- a/internal/prompt/es.go +++ b/internal/prompt/es.go @@ -76,7 +76,9 @@ Nunca hagas polling de resultados. Un envío que supera su espera devuelve un ti Un subagente o paso de workflow que corre dentro de esta sesión no es su agente, y lich no puede distinguirlos: envía y abre con private, y luego espera sus propios tickets. La nota [lich] y el wait_for_answer sin ticket pertenecen al agente de la propia sesión. -Cuando un mensaje [lich] con ticket llega a TU prompt, eres la sesión de trabajo: haz la tarea y responde con %s, un informe conciso (qué se hizo, dónde, qué falta), nunca una transcripción.`, +Cuando un mensaje [lich] con ticket llega a TU prompt, eres la sesión de trabajo: haz la tarea y responde con %s, un informe conciso (qué se hizo, dónde, qué falta), nunca una transcripción. + +Esta sesión tiene un navegador propio (herramientas browser_*): la misma ventana Chromium que abre el elemento Navegador de la tarjeta, no la ventana de lich. Ábrelo, lee la página, haz clic, escribe, pulsa teclas con nombre y captura la pantalla. Las capturas son archivos. Nunca pegues sus bytes en la respuesta.`, HandOverFailed: "La tarea no le llegó: %[1]v. La sesión está abierta, así que entrégale la tarea con " + "%[2]s cuando resuelvas lo que eso dice.", SendToSessionPrivate: "send_to_session con private", diff --git a/internal/prompt/ja.go b/internal/prompt/ja.go index 449cff02c..d190f599a 100644 --- a/internal/prompt/ja.go +++ b/internal/prompt/ja.go @@ -76,7 +76,9 @@ var ja = Catalog{ このセッションの中で動いているサブエージェントやワークフローのステップは、このセッションのエージェントではなく、lich はそれらを区別できません。それらは private で送信し、開き、そのあと自分のチケットを待ちます。[lich] の通知とチケットなしの wait_for_answer は、このセッション自身のエージェントのものです。 -チケット付きの [lich] メッセージがあなたのプロンプトに届いたら、あなたがワーカーです。タスクを実行し、%s で回答してください。回答は簡潔なレポート（何をしたか、どこで、何が残っているか）にし、やり取りの記録をそのまま貼らないでください。`, +チケット付きの [lich] メッセージがあなたのプロンプトに届いたら、あなたがワーカーです。タスクを実行し、%s で回答してください。回答は簡潔なレポート（何をしたか、どこで、何が残っているか）にし、やり取りの記録をそのまま貼らないでください。 + +このセッションには専用のブラウザがあります（browser_* ツール）。カードのブラウザ項目が開く Chromium のウィンドウであり、lich のウィンドウではありません。開いて、ページを読み、クリックし、入力し、名前のあるキーを押し、スクリーンショットを撮ってください。スクリーンショットはファイルです。そのバイト列を返信に貼らないでください。`, HandOverFailed: "タスクは届きませんでした：%[1]v。セッションは開いているので、その問題が片付いたら " + "%[2]s でタスクを渡してください。", SendToSessionPrivate: "private を指定した send_to_session", diff --git a/internal/prompt/ko.go b/internal/prompt/ko.go index 580de8baf..3987026d9 100644 --- a/internal/prompt/ko.go +++ b/internal/prompt/ko.go @@ -72,7 +72,9 @@ var ko = Catalog{ 이 세션 안에서 실행되는 서브에이전트나 워크플로 단계는 이 세션의 에이전트가 아니며, lich는 둘을 구분하지 못합니다. 그들은 private로 보내고 열며, 자기 티켓을 기다립니다. [lich] 메모와 티켓 없는 wait_for_answer는 이 세션 자체의 에이전트 몫입니다. -티켓이 붙은 [lich] 메시지가 당신의 프롬프트에 도착하면, 당신이 워커입니다. 작업을 하고 %s로 답하세요. 간결한 보고서(무엇을 했는지, 어디서, 무엇이 남았는지)로 답하고, 대화 기록은 보내지 마세요.`, +티켓이 붙은 [lich] 메시지가 당신의 프롬프트에 도착하면, 당신이 워커입니다. 작업을 하고 %s로 답하세요. 간결한 보고서(무엇을 했는지, 어디서, 무엇이 남았는지)로 답하고, 대화 기록은 보내지 마세요. + +이 세션에는 자기 브라우저가 있습니다 (browser_* 도구). 카드의 브라우저 항목이 여는 Chromium 창이며, lich 창이 아닙니다. 열고, 페이지를 읽고, 클릭하고, 입력하고, 이름이 있는 키를 누르고, 스크린샷을 찍으세요. 스크린샷은 파일입니다. 그 바이트를 답변에 붙이지 마세요.`, HandOverFailed: "작업이 전달되지 않았습니다: %[1]v. 세션은 열려 있으니, 그 내용이 해결되면 " + "%[2]s로 작업을 넘기세요.", SendToSessionPrivate: "private를 지정한 send_to_session", diff --git a/internal/prompt/pt_br.go b/internal/prompt/pt_br.go index c9f1082bf..081c6d31e 100644 --- a/internal/prompt/pt_br.go +++ b/internal/prompt/pt_br.go @@ -76,7 +76,9 @@ Nunca faça polling por resultados. Um envio que passa do tempo de espera devolv Um subagente ou etapa de workflow rodando dentro desta sessão não é o agente dela, e o lich não consegue distingui-los: ele envia e abre com private, e depois espera pelos próprios tickets. A nota [lich] e o wait_for_answer sem ticket pertencem ao agente da própria sessão. -Quando uma mensagem [lich] com ticket chega ao SEU prompt, você é a sessão de trabalho: faça a tarefa e responda com %s, um relatório conciso (o que foi feito, onde, o que falta), nunca uma transcrição.`, +Quando uma mensagem [lich] com ticket chega ao SEU prompt, você é a sessão de trabalho: faça a tarefa e responda com %s, um relatório conciso (o que foi feito, onde, o que falta), nunca uma transcrição. + +Esta sessão tem um navegador próprio (ferramentas browser_*): a mesma janela Chromium que o item Navegador do card abre, não a janela do lich. Abra, leia a página, clique, digite, pressione teclas nomeadas e capture a tela. Capturas são arquivos. Nunca cole os bytes na resposta.`, HandOverFailed: "A tarefa não chegou a ela: %[1]v. A sessão está aberta, então entregue a tarefa com " + "%[2]s depois de resolver o que isso diz.", SendToSessionPrivate: "send_to_session com private", diff --git a/internal/prompt/ru.go b/internal/prompt/ru.go index ab336f2e2..fec373a04 100644 --- a/internal/prompt/ru.go +++ b/internal/prompt/ru.go @@ -85,7 +85,9 @@ var ru = Catalog{ Субагент или шаг workflow, работающий внутри этой сессии, не является её агентом, и lich не может их различить: он отправляет и открывает с private, а затем ждёт своих тикетов. Заметка [lich] и wait_for_answer без тикета принадлежат собственному агенту сессии. -Когда в ВАШ промпт приходит сообщение [lich] с тикетом, рабочая сессия это вы: выполните задачу, затем ответьте через %s кратким отчётом (что сделано, где, что осталось), а не стенограммой.`, +Когда в ВАШ промпт приходит сообщение [lich] с тикетом, рабочая сессия это вы: выполните задачу, затем ответьте через %s кратким отчётом (что сделано, где, что осталось), а не стенограммой. + +У этой сессии есть свой браузер (инструменты browser_*): то же окно Chromium, которое открывает пункт Браузер на карточке, а не окно lich. Откройте его, прочитайте страницу, нажмите, введите текст, нажмите именованную клавишу и сделайте снимок. Снимки — это файлы. Не вставляйте их байты в ответ.`, HandOverFailed: "Задача до неё не дошла: %[1]v. Сессия открыта, поэтому передайте ей задачу через " + "%[2]s, как только разберётесь с тем, о чём говорит ошибка.", SendToSessionPrivate: "send_to_session с private", diff --git a/internal/prompt/zh_cn.go b/internal/prompt/zh_cn.go index 0c6794a45..0b06628ac 100644 --- a/internal/prompt/zh_cn.go +++ b/internal/prompt/zh_cn.go @@ -71,7 +71,9 @@ var zhCN = Catalog{ 在本会话内部运行的子智能体或工作流步骤不是它自己的智能体，lich 无法区分它们：它以 private 发送和打开，然后等待它自己的工单。[lich] 通知和不带工单的 wait_for_answer 属于本会话自己的智能体。 -当一条带工单的 [lich] 消息出现在你的提示处时，你就是工作者：完成任务，然后用 %s 回答，内容是简明的报告（做了什么、在哪里、还剩什么），不要贴完整记录。`, +当一条带工单的 [lich] 消息出现在你的提示处时，你就是工作者：完成任务，然后用 %s 回答，内容是简明的报告（做了什么、在哪里、还剩什么），不要贴完整记录。 + +本会话有自己的浏览器（browser_* 工具）：卡片上的浏览器项打开的是同一个 Chromium 窗口，不是 lich 的窗口。打开它，阅读页面，点击，输入，按有名字的键，并截图。截图是文件。不要把字节贴进回复。`, HandOverFailed: "任务没有送达：%[1]v。会话已打开，等上述问题处理完后，" + "请用 %[2]s 把任务交给它。", SendToSessionPrivate: "带 private 的 send_to_session", diff --git a/internal/routes/browser.go b/internal/routes/browser.go new file mode 100644 index 000000000..ad7e4f7af --- /dev/null +++ b/internal/routes/browser.go @@ -0,0 +1,27 @@ +package routes + +import ( + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/internal/browser" + "github.com/lichdotdev/lich/internal/rpc" +) + +// Browser exposes the session browser as api declares it. Cleanup and +// CloseOwnedBy stay out: one ends every browser this process started, the +// other runs when the session that owns one is closed. +func Browser(d *rpc.Handler, s *browser.Service) { + rpc.Handle(d, api.BrowserShow, s.Show) + rpc.Handle(d, api.BrowserOpen, s.OpenURL) + rpc.Handle(d, api.BrowserInfo, s.Read) + rpc.Handle(d, api.BrowserClick, s.ClickAt) + rpc.Handle(d, api.BrowserType, s.TypeText) + rpc.Handle(d, api.BrowserPress, s.PressKey) + rpc.Handle(d, api.BrowserScreenshot, s.Capture) + rpc.Handle(d, api.BrowserNavigate, s.Go) + rpc.Handle(d, api.BrowserReload, s.ReloadPage) + rpc.Handle(d, api.BrowserBack, s.GoBack) + rpc.Handle(d, api.BrowserForward, s.GoForward) + rpc.Handle(d, api.BrowserScroll, s.ScrollBy) + rpc.Handle(d, api.BrowserList, s.Handles) + rpc.Handle(d, api.BrowserClose, s.CloseOne) +} diff --git a/internal/routes/routes_test.go b/internal/routes/routes_test.go index 46977b079..55969dc95 100644 --- a/internal/routes/routes_test.go +++ b/internal/routes/routes_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/internal/browser" "github.com/lichdotdev/lich/internal/claudeprofile" "github.com/lichdotdev/lich/internal/codexprofile" "github.com/lichdotdev/lich/internal/ompprofile" @@ -29,6 +30,12 @@ func TestHandleRelayCoversEveryRelayMethod(t *testing.T) { assertHandled(t, d, "relay.") } +func TestHandleBrowserCoversEveryBrowserMethod(t *testing.T) { + d := rpc.New() + Browser(d, (*browser.Service)(nil)) + assertHandled(t, d, "browser.") +} + func TestHandleSystemCoversEverySystemMethod(t *testing.T) { d := rpc.New() System(d, nil) diff --git a/main.go b/main.go index 996ad228c..1e349df05 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/lichdotdev/lich/api" "github.com/lichdotdev/lich/internal/agentplugin" "github.com/lichdotdev/lich/internal/appupdate" + "github.com/lichdotdev/lich/internal/browser" "github.com/lichdotdev/lich/internal/claudeprofile" "github.com/lichdotdev/lich/internal/cli" "github.com/lichdotdev/lich/internal/codexprofile" @@ -146,7 +147,8 @@ func main() { // it (internal/terminal.handsOn). defer term.FlushHandsOn() coord := newCoordinator() - sys := registerServices(db, term, hub, configDir, logPath, env, launchEnv, coord) + sys, closeBrowsers := registerServices(db, term, hub, configDir, logPath, env, launchEnv, coord) + defer closeBrowsers() coord.SetWindowOpen(hub.Attached) term.SetRestart(coord.Do) @@ -202,7 +204,10 @@ func resolveEnv() (launchEnv, env []string) { // to exist first. It is lifted out of main only for its length; nothing here // runs anywhere else. func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub, - configDir, logPath string, env, launchEnv []string, coord *restart.Coordinator) *system.Service { + configDir, logPath string, env, launchEnv []string, coord *restart.Coordinator) (*system.Service, func()) { + // One Chromium per session, closed with the card and with the process. + // Never the lich window: this is a second browser the agent drives. + brows := browser.New() proj := project.New(project.ZenityPicker{}) // gh has one active account per host; a project can name a different one. proj.SetAccounts(db.GHAccountForPath) @@ -268,6 +273,7 @@ func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub return proj.PickFile(context.Background(), api.PickFileOptions{Title: title}) }) routes.Terminal(dispatcher, term) + routes.Browser(dispatcher, brows) routes.Drop(dispatcher, drops) routes.Project(dispatcher, proj) plugins := agentplugin.New(db) @@ -409,8 +415,14 @@ func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub // without a word to its caller. term.SetWorkerAnswer(rl.WorkerAnswered) term.SetWorkerUnanswered(rl.WorkerUnanswered) - term.SetSessionClosed(rl.SessionClosed) - term.SetSessionExited(rl.SessionExited) + term.SetSessionClosed(func(id, closerID string) { + rl.SessionClosed(id, closerID) + _ = brows.CloseOwnedBy(id) + }) + term.SetSessionExited(func(id string) { + rl.SessionExited(id) + _ = brows.CloseOwnedBy(id) + }) term.SetErrandStatus(rl.Status) // Both questions the relay asks about a provider — whether its sessions // report at all, and whether they can answer with a tool — are about what @@ -454,7 +466,7 @@ func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub term.Mount("/drop", http.HandlerFunc(drops.Upload)) term.Mount("/blob", http.HandlerFunc(proj.ServeBlob)) term.Mount("/events", hub) - return sys + return sys, func() { _ = brows.Cleanup() } } // run serves the loopback listener, opens the window on it (the native one, diff --git a/native/backend/project.go b/native/backend/project.go index 20cc863d9..51e6cba6e 100644 --- a/native/backend/project.go +++ b/native/backend/project.go @@ -35,6 +35,13 @@ func (c *Client) OpenFolderInEditor(ctx context.Context, dir string) (string, er return callMethod(ctx, c, api.SystemOpenFolderInEditor, api.OpenFolderOptions{Dir: dir}) } +// ShowBrowser opens the Chromium window owned by session id, creating it +// when the session has none yet. +func (c *Client) ShowBrowser(ctx context.Context, id string) error { + _, err := callMethod(ctx, c, api.BrowserShow, api.SessionIDOptions{ID: id}) + return err +} + // OpenFolder opens dir in the file manager. func (c *Client) OpenFolder(ctx context.Context, dir string) error { _, err := callMethod(ctx, c, api.SystemOpenFolder, api.OpenFolderOptions{Dir: dir}) diff --git a/native/cmd/lich-native/cardmenu.go b/native/cmd/lich-native/cardmenu.go index 0210af4cb..1364be6c5 100644 --- a/native/cmd/lich-native/cardmenu.go +++ b/native/cmd/lich-native/cardmenu.go @@ -14,6 +14,7 @@ var ( icMenuPin = icons.Lucide("pin") icMenuUnpin = icons.Lucide("pin-off") icOpenIn = icons.Lucide("folder-open") + icBrowser = icons.Lucide("globe") icInTerminal = icons.Lucide("terminal") icInEditor = icons.Lucide("folder-code") icMenuPR = icons.Lucide("git-pull-request-arrow") @@ -41,6 +42,7 @@ func cardMenu(sv *sessionView, folders []string, act *sidebarAction, startRename act.color, act.colorTo = sv.ID, name })}, {Kind: ui.MenuSubTrigger, Label: i18n.T("sidebar.sessionCard.moveToFolder"), Icon: icFolder, Sub: cardFolderMenu(sv, folders, act, newFolder)}, + {Label: i18n.T("sidebar.sessionCard.browser"), Icon: icBrowser, OnClick: func() { act.browser = sv.ID }}, {Kind: ui.MenuSubTrigger, Label: i18n.T("sidebar.sessionCard.openIn"), Icon: icOpenIn, Sub: openInMenu(sv, act)}, pullRequestItem(sv, act), } diff --git a/native/cmd/lich-native/cardmenu_test.go b/native/cmd/lich-native/cardmenu_test.go index 74c35c1c1..28c86a8e3 100644 --- a/native/cmd/lich-native/cardmenu_test.go +++ b/native/cmd/lich-native/cardmenu_test.go @@ -31,7 +31,7 @@ func TestCardMenu(t *testing.T) { sv := &sessionView{Session: backend.Session{ID: "s1", Kind: "claude", Color: "teal"}, shown: "/repo"} menu := cardMenu(sv, nil, &act, func() { renamed = true }, func() {}) - if want := []string{"Rename", "Pin", "Color", "Move to folder", "Open in", "Pull request", "", "Close session"}; !slices.Equal(labels(menu), want) { + if want := []string{"Rename", "Pin", "Color", "Move to folder", "Browser", "Open in", "Pull request", "", "Close session"}; !slices.Equal(labels(menu), want) { t.Fatalf("menu %v, want %v", labels(menu), want) } entry(t, menu, "Rename").OnClick() diff --git a/native/cmd/lich-native/main.go b/native/cmd/lich-native/main.go index 2c67de508..07a784506 100644 --- a/native/cmd/lich-native/main.go +++ b/native/cmd/lich-native/main.go @@ -863,6 +863,7 @@ func (u *window) apply(act sidebarAction) { go func() { u.logErr("pin session", u.client.SetSessionPinned(u.ctx, id, pinned)) }() } u.applyOpenIn(act) + u.applyBrowser(act) u.applyContinueIn(act) if pick := act.profilePick; pick.key != "" { u.settings.writeProviderKey(pick.key, pick.projectID, pick.value, nil) @@ -887,6 +888,19 @@ func renameSession(ctx context.Context, c *backend.Client, show func(ui.Toast) s } } +// applyBrowser opens the session's Chromium, the window its agent drives. +func (u *window) applyBrowser(act sidebarAction) { + id := act.browser + if id == "" { + return + } + go func() { + if err := u.client.ShowBrowser(u.ctx, id); err != nil { + u.toaster.Show(ui.Toast{Variant: ui.ToastError, Title: i18n.T("sidebar.sessionCard.browserFailed", i18n.Args{"error": backend.ErrorText(err)})}) + } + }() +} + // applyOpenIn carries out the card menu's Open in: a shell session at the // card's directory, the directory in the editor, or in the file manager. func (u *window) applyOpenIn(act sidebarAction) { diff --git a/native/cmd/lich-native/sidebar.go b/native/cmd/lich-native/sidebar.go index 6ae3cebc9..8ec094408 100644 --- a/native/cmd/lich-native/sidebar.go +++ b/native/cmd/lich-native/sidebar.go @@ -122,13 +122,13 @@ type sidebarAction struct { openPR int // a pull request to show on the Pulls screen launch launchRequest // Open in, from the card menu: a directory for a shell session, the - // editor or the file manager. - terminalAt, editorAt, folderAt string - rename sessionRename - color, colorTo string // session id, and its new tint ("" for the theme) - continueIn continueRequest - profilePick profilePick - worktree bool // the + menu asked for a worktree session + // editor or the file manager. browser is the session whose Chromium to show. + terminalAt, editorAt, folderAt, browser string + rename sessionRename + color, colorTo string // session id, and its new tint ("" for the theme) + continueIn continueRequest + profilePick profilePick + worktree bool // the + menu asked for a worktree session // The session to fork, in its own checkout or into a new worktree. forkHere, forkWorktree string sidebarWrites @@ -192,6 +192,7 @@ func (a *sidebarAction) takeCard(c sidebarAction) { pick(&a.terminalAt, c.terminalAt) pick(&a.editorAt, c.editorAt) pick(&a.folderAt, c.folderAt) + pick(&a.browser, c.browser) pick(&a.forkHere, c.forkHere) pick(&a.forkWorktree, c.forkWorktree) if c.pin != "" { diff --git a/native/i18n/locales/en/sidebar.json b/native/i18n/locales/en/sidebar.json index 71314842a..1b18cdfb9 100644 --- a/native/i18n/locales/en/sidebar.json +++ b/native/i18n/locales/en/sidebar.json @@ -33,6 +33,8 @@ "moveToFolder": "Move to folder", "takeOutOf": "Take out of {folder}", "newFolder": "New folder…", + "browser": "Browser", + "browserFailed": "Could not open the browser: {error}", "openIn": "Open in", "terminal": "Terminal", "editor": "Editor", diff --git a/native/i18n/locales/es/sidebar.json b/native/i18n/locales/es/sidebar.json index 9c6f610f5..498e0a674 100644 --- a/native/i18n/locales/es/sidebar.json +++ b/native/i18n/locales/es/sidebar.json @@ -33,6 +33,8 @@ "moveToFolder": "Mover a una carpeta", "takeOutOf": "Sacar de {folder}", "newFolder": "Nueva carpeta…", + "browser": "Navegador", + "browserFailed": "No se pudo abrir el navegador: {error}", "openIn": "Abrir en", "terminal": "Terminal", "editor": "Editor", diff --git a/native/i18n/locales/ja/sidebar.json b/native/i18n/locales/ja/sidebar.json index 9fdba4106..42ac1680b 100644 --- a/native/i18n/locales/ja/sidebar.json +++ b/native/i18n/locales/ja/sidebar.json @@ -33,6 +33,8 @@ "moveToFolder": "フォルダーに移動", "takeOutOf": "{folder} から出す", "newFolder": "新しいフォルダー…", + "browser": "ブラウザ", + "browserFailed": "ブラウザを開けませんでした: {error}", "openIn": "次で開く", "terminal": "ターミナル", "editor": "エディター", diff --git a/native/i18n/locales/ko/sidebar.json b/native/i18n/locales/ko/sidebar.json index 18a93f7bb..6a9d7e26c 100644 --- a/native/i18n/locales/ko/sidebar.json +++ b/native/i18n/locales/ko/sidebar.json @@ -32,6 +32,8 @@ "moveToFolder": "폴더로 이동", "takeOutOf": "{folder}에서 빼기", "newFolder": "새 폴더…", + "browser": "브라우저", + "browserFailed": "브라우저를 열 수 없습니다: {error}", "openIn": "다음에서 열기", "terminal": "터미널", "editor": "에디터", diff --git a/native/i18n/locales/pt-br/sidebar.json b/native/i18n/locales/pt-br/sidebar.json index 3fe68128f..98201e351 100644 --- a/native/i18n/locales/pt-br/sidebar.json +++ b/native/i18n/locales/pt-br/sidebar.json @@ -33,6 +33,8 @@ "moveToFolder": "Mover para pasta", "takeOutOf": "Tirar de {folder}", "newFolder": "Nova pasta…", + "browser": "Navegador", + "browserFailed": "Não foi possível abrir o navegador: {error}", "openIn": "Abrir em", "terminal": "Terminal", "editor": "Editor", diff --git a/native/i18n/locales/ru/sidebar.json b/native/i18n/locales/ru/sidebar.json index 7da6e67a7..a0917c6f9 100644 --- a/native/i18n/locales/ru/sidebar.json +++ b/native/i18n/locales/ru/sidebar.json @@ -35,6 +35,8 @@ "moveToFolder": "В папку", "takeOutOf": "Убрать из {folder}", "newFolder": "Новая папка…", + "browser": "Браузер", + "browserFailed": "Не удалось открыть браузер: {error}", "openIn": "Открыть в", "terminal": "Терминал", "editor": "Редактор", diff --git a/native/i18n/locales/zh-cn/sidebar.json b/native/i18n/locales/zh-cn/sidebar.json index f7ea6f555..a7b8bfe6d 100644 --- a/native/i18n/locales/zh-cn/sidebar.json +++ b/native/i18n/locales/zh-cn/sidebar.json @@ -33,6 +33,8 @@ "moveToFolder": "移到文件夹", "takeOutOf": "从 {folder} 中移出", "newFolder": "新建文件夹…", + "browser": "浏览器", + "browserFailed": "无法打开浏览器：{error}", "openIn": "打开方式", "terminal": "终端", "editor": "编辑器", From f9e06f4c48f5d42532f7a89107a2caa454c5af7b Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 19:16:30 -0300 Subject: [PATCH 02/25] fix(browser): keep the session window and its profile, and stop one launch from holding every card The reaper closed a headed window after five idle minutes while a person was still using it; only headless browsers are reaped now. Promoting a headless browser to a window reopens it on the same profile and carries every cookie, session ones included, instead of starting from an empty one. Focus raises the window with Page.bringToFront, which window.focus() from the page cannot. A clear before typing goes through the editing commands, so the field's input listeners see it. Chrome launches and closes happen outside the service lock, and close errors are logged instead of dropped. --- docs/browser-tab.md | 9 +- docs/ceilings.md | 10 +- internal/browser/actions.go | 8 +- internal/browser/browser.go | 243 +++++++++++++++++++-------- internal/browser/browser_test.go | 88 ++++++++++ internal/browser/chrome.go | 67 +++++++- internal/browser/chrome_live_test.go | 129 ++++++++++++++ internal/browser/fake_test.go | 30 +++- internal/browser/page.go | 10 +- main.go | 16 +- 10 files changed, 518 insertions(+), 92 deletions(-) create mode 100644 internal/browser/chrome_live_test.go diff --git a/docs/browser-tab.md b/docs/browser-tab.md index 132672aa8..8301fa046 100644 --- a/docs/browser-tab.md +++ b/docs/browser-tab.md @@ -16,8 +16,13 @@ Closing that window does not quit lich. Closing the session closes the sidecar. - none yet → headed window at `about:blank` - already headed → focus that window -- headless only (the agent opened first) → replace it with a headed window at - the same URL. chromedp cannot show a headless process in place +- headless only (the agent opened first) → reopen it as a headed window at + the same URL, on the same profile, with its cookies (session ones + included). chromedp cannot show a headless process in place, so what lives + only in the page, a half-filled form or sessionStorage, does not survive + +A headless browser nobody used for five minutes is closed. A headed window is +not: lich cannot see a person using it, so only the card closing ends it. The agent without the menu uses `browser_open`: headless if nobody asked for a window, and the headed one if the person already did. MCP and `lich browser` diff --git a/docs/ceilings.md b/docs/ceilings.md index 35a1a9d50..a67f2e4f6 100644 --- a/docs/ceilings.md +++ b/docs/ceilings.md @@ -913,10 +913,12 @@ work when nobody knows it and that the call site never shows. The mechanism and - **A session's browser is a second Chromium, one per card** (`internal/browser`, `docs/browser-tab.md`): the card menu opens a headed window of the system Chrome, Edge, Chromium, Brave or Vivaldi, never a page inside lich. The agent - drives that same window. A headless one the agent opened first is replaced by - a headed window at the same URL, because chromedp cannot show a headless - process. Closing the card, or the session's process ending, closes it; closing - the browser window does not close the session. It loads only http, https and + drives that same window. A headless one the agent opened first is reopened + as a headed window on the same profile, carrying the URL and every cookie, + because chromedp cannot show a headless process; a half-filled form or + sessionStorage is lost on the way. Closing the card, or the session's process + ending, closes it; closing the browser window does not close the session. A + headless browser idle for five minutes is closed, a headed window never is. It loads only http, https and about:blank. Page info never includes a form's value. Screenshots are files. There is no tool that runs JavaScript in the page. It is not a sandbox: the page runs as the user, in an ephemeral profile. diff --git a/internal/browser/actions.go b/internal/browser/actions.go index e823e7f26..671984144 100644 --- a/internal/browser/actions.go +++ b/internal/browser/actions.go @@ -10,7 +10,7 @@ var errNoBrowser = fmt.Errorf("no browser for this session — open one first") func (s *Service) live(id string) (*session, error) { s.mu.Lock() - defer s.mu.Unlock() + defer s.unlock() s.reapLocked() sess := s.lookupLocked(id) if sess == nil { @@ -29,9 +29,9 @@ func (s *Service) touch(sess *session) { func (s *Service) drop(sess *session) { s.mu.Lock() - defer s.mu.Unlock() + defer s.unlock() if s.byID[sess.id] == sess { - _ = s.destroyLocked(sess) + s.buryLocked(sess) } } @@ -142,7 +142,7 @@ func (s *Service) Navigate(id, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") - return sess.handle(), nil + return s.handleOf(sess), nil } // Reload reloads the current page. diff --git a/internal/browser/browser.go b/internal/browser/browser.go index 604fdcfff..497dea89c 100644 --- a/internal/browser/browser.go +++ b/internal/browser/browser.go @@ -6,14 +6,19 @@ import ( "encoding/hex" "errors" "fmt" + "log/slog" "os" "sync" "time" + + "github.com/chromedp/cdproto/network" ) -// idleAfter is how long a session's browser sits unused before the reaper +// idleAfter is how long a headless browser sits unused before the reaper // closes it. Five minutes matches the "walked away from the card" window -// without holding a Chromium for a session that finished hours ago. +// without holding a Chromium for a session that finished hours ago. A headed +// window is never reaped: lich cannot see a person using it, so only the card +// closing ends it. const idleAfter = 5 * time.Minute // reapEvery is how often the reaper looks. Shorter than idleAfter so a @@ -26,7 +31,10 @@ const reapEvery = 30 * time.Second // does not take the tool with it. const pageWait = 30 * time.Second -var errNoOwner = errors.New("browser needs a lich session id") +var ( + errNoOwner = errors.New("browser needs a lich session id") + errStopped = errors.New("browser service is shutting down") +) // Service is one Chromium context per lich session: the window Browser tab // opens and the page the agent drives are the same one. Headless until @@ -39,6 +47,12 @@ type Service struct { idle time.Duration byOwner map[string]*session byID map[string]*session + // owners serializes one session's launch, promotion and reuse, so a slow + // Chrome start holds that card alone and mu is never held across CDP. + owners map[string]*sync.Mutex + // dead holds what the locked code took out of the maps; unlock closes it + // after mu is released, because closing waits for Chrome to exit. + dead []*session stop chan struct{} stopped bool } @@ -67,6 +81,7 @@ func newService(launch Launcher, mkTmp func(string, string) (string, error), now idle: idle, byOwner: make(map[string]*session), byID: make(map[string]*session), + owners: make(map[string]*sync.Mutex), stop: make(chan struct{}), } } @@ -81,48 +96,73 @@ func (s *Service) Open(owner, rawURL string) (Handle, error) { if err != nil { return Handle{}, err } - s.mu.Lock() - defer s.mu.Unlock() - s.reapLocked() - if sess := s.liveByOwner(owner); sess != nil { - if err := runPage(func(ctx context.Context) error { - return sess.page.Navigate(ctx, u) - }); err != nil { - s.destroyLocked(sess) - return Handle{}, err - } - sess.url = u - sess.lastUsed = s.now() - return sess.handle(), nil + lock := s.ownerLock(owner) + lock.Lock() + defer lock.Unlock() + sess := s.current(owner) + if sess == nil { + return s.launchFresh(owner, u, false) + } + if err := runPage(func(ctx context.Context) error { + return sess.page.Navigate(ctx, u) + }); err != nil { + s.drop(sess) + return Handle{}, err } - return s.launchLocked(owner, u, false) + s.remember(sess, u, "") + return s.handleOf(sess), nil } // OpenVisible brings this session's browser on screen. No browser yet: a // headed window at about:blank. Already headed: focus it. Headless only: -// replace it with a headed window at the same URL — chromedp cannot promote -// a headless process in place. +// reopen it as a window (promote). func (s *Service) OpenVisible(owner string) (Handle, error) { if owner == "" { return Handle{}, errNoOwner } + lock := s.ownerLock(owner) + lock.Lock() + defer lock.Unlock() + sess := s.current(owner) + switch { + case sess == nil: + return s.launchFresh(owner, "about:blank", true) + case !sess.headed: + return s.promote(sess) + } + if err := runPage(sess.page.Focus); err != nil { + s.drop(sess) + return Handle{}, err + } + s.touch(sess) + return s.handleOf(sess), nil +} + +// promote reopens a headless browser as a window on the same profile and +// carries its cookies across: chromedp cannot show a headless process in +// place, and a session cookie lives only in the process. What else lives only +// in the page (a half-filled form, sessionStorage) does not survive. +func (s *Service) promote(sess *session) (Handle, error) { + var cookies []*network.CookieParam + if err := runPage(func(ctx context.Context) error { + var err error + cookies, err = sess.page.Cookies(ctx) + return err + }); err != nil { + s.drop(sess) + return Handle{}, err + } s.mu.Lock() - defer s.mu.Unlock() - s.reapLocked() - if sess := s.liveByOwner(owner); sess != nil { - if sess.headed { - _ = runPage(sess.page.Focus) - sess.lastUsed = s.now() - return sess.handle(), nil - } - u := sess.url - if u == "" { - u = "about:blank" - } - s.destroyLocked(sess) - return s.launchLocked(owner, u, true) + s.detachLocked(sess) + s.unlock() + if err := sess.page.Close(); err != nil { + return Handle{}, errors.Join(err, os.RemoveAll(sess.dir)) + } + u := sess.url + if u == "" { + u = "about:blank" } - return s.launchLocked(owner, "about:blank", true) + return s.start(sess.owner, u, true, sess.dir, cookies) } // List returns this session's browser, or an empty list. Never null: a @@ -131,12 +171,9 @@ func (s *Service) List(owner string) ([]Handle, error) { if owner == "" { return nil, errNoOwner } - s.mu.Lock() - defer s.mu.Unlock() - s.reapLocked() out := []Handle{} - if sess := s.liveByOwner(owner); sess != nil { - out = append(out, sess.handle()) + if sess := s.current(owner); sess != nil { + out = append(out, s.handleOf(sess)) } return out, nil } @@ -144,12 +181,15 @@ func (s *Service) List(owner string) ([]Handle, error) { // Close tears down one browser, by handle id or by the lich session that owns it. func (s *Service) Close(id string) error { s.mu.Lock() - defer s.mu.Unlock() sess := s.lookupLocked(id) + if sess != nil { + s.detachLocked(sess) + } + s.unlock() if sess == nil { return nil } - return s.destroyLocked(sess) + return sess.destroy() } // CloseOwnedBy tears down the browser of a lich session that is going away. @@ -162,82 +202,141 @@ func (s *Service) CloseOwnedBy(owner string) error { // window-close exit does not leave Chromiums behind. func (s *Service) Cleanup() error { s.mu.Lock() - defer s.mu.Unlock() if !s.stopped { s.stopped = true close(s.stop) } - var errs []error + all := append([]*session{}, s.dead...) + s.dead = nil for _, sess := range s.byID { - if err := s.destroyLocked(sess); err != nil { - errs = append(errs, err) - } + s.detachLocked(sess) + all = append(all, sess) + } + s.mu.Unlock() + var errs []error + for _, sess := range all { + errs = append(errs, sess.destroy()) } return errors.Join(errs...) } -func (s *Service) launchLocked(owner, u string, headed bool) (Handle, error) { +func (s *Service) launchFresh(owner, u string, headed bool) (Handle, error) { dir, err := s.mkTmp("", "lich-browser-") if err != nil { return Handle{}, err } + return s.start(owner, u, headed, dir, nil) +} + +// start launches a browser on dir, loads cookies and then u, and registers it. +// The caller holds owner's lock; mu is taken only to register. +func (s *Service) start(owner, u string, headed bool, dir string, cookies []*network.CookieParam) (Handle, error) { page, err := s.launch(dir, headed) if err != nil { - _ = os.RemoveAll(dir) - return Handle{}, err + return Handle{}, errors.Join(err, os.RemoveAll(dir)) } if err := runPage(func(ctx context.Context) error { + if len(cookies) > 0 { + if err := page.SetCookies(ctx, cookies); err != nil { + return err + } + } return page.Navigate(ctx, u) }); err != nil { - _ = page.Close() - _ = os.RemoveAll(dir) - return Handle{}, err + return Handle{}, errors.Join(err, page.Close(), os.RemoveAll(dir)) } sess := &session{ id: newID(), owner: owner, dir: dir, url: u, headed: headed, page: page, lastUsed: s.now(), } + s.mu.Lock() + defer s.unlock() + if s.stopped { + s.dead = append(s.dead, sess) + return Handle{}, errStopped + } s.byOwner[owner] = sess s.byID[sess.id] = sess return sess.handle(), nil } +func (s *Service) ownerLock(owner string) *sync.Mutex { + s.mu.Lock() + defer s.mu.Unlock() + lock := s.owners[owner] + if lock == nil { + lock = &sync.Mutex{} + s.owners[owner] = lock + } + return lock +} + +// current is owner's live browser after a reap, or nil. +func (s *Service) current(owner string) *session { + s.mu.Lock() + defer s.unlock() + s.reapLocked() + return s.liveByOwner(owner) +} + +func (s *Service) handleOf(sess *session) Handle { + s.mu.Lock() + defer s.mu.Unlock() + return sess.handle() +} + func (s *Service) liveByOwner(owner string) *session { sess := s.byOwner[owner] - if sess == nil || !sess.alive() { - if sess != nil { - s.destroyLocked(sess) - } + if sess == nil { + return nil + } + if !sess.alive() { + s.buryLocked(sess) return nil } return sess } func (s *Service) lookupLocked(id string) *session { - if sess := s.byID[id]; sess != nil { - if sess.alive() { - return sess + for _, sess := range []*session{s.byID[id], s.byOwner[id]} { + if sess == nil { + continue } - s.destroyLocked(sess) - } - if sess := s.byOwner[id]; sess != nil { if sess.alive() { return sess } - s.destroyLocked(sess) + s.buryLocked(sess) } return nil } -func (s *Service) destroyLocked(sess *session) error { +func (s *Service) detachLocked(sess *session) { delete(s.byID, sess.id) if s.byOwner[sess.owner] == sess { delete(s.byOwner, sess.owner) } - err := sess.page.Close() - if rmErr := os.RemoveAll(sess.dir); err == nil { - err = rmErr +} + +// buryLocked takes sess out of the maps and leaves it for unlock to close. +func (s *Service) buryLocked(sess *session) { + s.detachLocked(sess) + s.dead = append(s.dead, sess) +} + +// unlock releases mu, then closes what the locked section buried. Nobody waits +// on these closes, so a failure is logged rather than returned. +func (s *Service) unlock() { + dead := s.dead + s.dead = nil + s.mu.Unlock() + for _, sess := range dead { + if err := sess.destroy(); err != nil { + slog.Warn("close a session browser", "owner", sess.owner, "err", err) + } } - return err +} + +func (s *session) destroy() error { + return errors.Join(s.page.Close(), os.RemoveAll(s.dir)) } func (s *session) alive() bool { @@ -263,18 +362,18 @@ func (s *Service) startReaper() { }() } -// Reap closes browsers that have sat idle past the idle window. +// Reap closes headless browsers that have sat idle past the idle window. func (s *Service) Reap() { s.mu.Lock() - defer s.mu.Unlock() + defer s.unlock() s.reapLocked() } func (s *Service) reapLocked() { cutoff := s.now().Add(-s.idle) for _, sess := range s.byID { - if sess.lastUsed.Before(cutoff) { - _ = s.destroyLocked(sess) + if !sess.headed && sess.lastUsed.Before(cutoff) { + s.buryLocked(sess) } } } diff --git a/internal/browser/browser_test.go b/internal/browser/browser_test.go index a25a910a5..ffb06069b 100644 --- a/internal/browser/browser_test.go +++ b/internal/browser/browser_test.go @@ -4,8 +4,11 @@ import ( "errors" "os" "path/filepath" + "sync" "testing" "time" + + "github.com/chromedp/cdproto/network" ) func testService(t *testing.T, launch *fakeLauncher) *Service { @@ -378,3 +381,88 @@ func TestInfoReturnsPageInfoWithoutDroppingTheBrowser(t *testing.T) { t.Fatal("Info dropped the browser") } } + +func TestIdleReaperKeepsHeadedWindow(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + + if _, err := s.OpenVisible("s"); err != nil { + t.Fatal(err) + } + later := time.Date(2026, 8, 16, 13, 0, 0, 0, time.UTC) + s.now = func() time.Time { return later } + s.Reap() + if launch.pages[0].closed { + t.Fatal("reaper closed a window a person may be using; only the card closes it") + } +} + +func TestOpenVisibleKeepsProfileAndCookies(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + + if _, err := s.Open("s", "https://example.com/app"); err != nil { + t.Fatal(err) + } + sid := &network.CookieParam{Name: "sid", Value: "1", Domain: "example.com", Path: "/"} + launch.pages[0].cookies = []*network.CookieParam{sid} + + if _, err := s.OpenVisible("s"); err != nil { + t.Fatal(err) + } + if launch.dirs[1] != launch.dirs[0] { + t.Fatalf("window launched on profile %q, headless was on %q", launch.dirs[1], launch.dirs[0]) + } + if _, err := os.Stat(launch.dirs[0]); err != nil { + t.Fatalf("profile dir gone after promotion: %v", err) + } + got := launch.pages[1].setCookies + if len(got) != 1 || got[0].Name != "sid" || got[0].Value != "1" { + t.Fatalf("window cookies = %+v, want the headless session's sid", got) + } + if nav := launch.pages[1].navigations; len(nav) != 1 || nav[0] != "https://example.com/app" { + t.Fatalf("window navigations = %v", nav) + } +} + +func TestLaunchDoesNotHoldOtherSessions(t *testing.T) { + release := make(chan struct{}) + entered := make(chan struct{}) + var once sync.Once + launch := &fakeLauncher{} + launch.gate = func() { + first := false + once.Do(func() { first = true }) + if first { + close(entered) + <-release + } + } + s := testService(t, launch) + + slow := make(chan error, 1) + go func() { + _, err := s.Open("slow", "https://example.com") + slow <- err + }() + <-entered + + done := make(chan error, 1) + go func() { + _, err := s.List("other") + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + close(release) + t.Fatal("one session's Chrome launch held every other session's browser call") + } + close(release) + if err := <-slow; err != nil { + t.Fatal(err) + } +} diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go index 53bc73160..dcb33db01 100644 --- a/internal/browser/chrome.go +++ b/internal/browser/chrome.go @@ -8,8 +8,11 @@ import ( "os/exec" "time" + "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/input" + "github.com/chromedp/cdproto/network" "github.com/chromedp/cdproto/page" + "github.com/chromedp/cdproto/storage" "github.com/chromedp/chromedp" "github.com/lichdotdev/lich/internal/chromium" ) @@ -20,6 +23,10 @@ import ( // still held, up to pageWait. Past this, the agent reads whatever is there. const domSettle = 1500 * time.Millisecond +// closeWait bounds a graceful Browser.close: long enough for Chrome to flush +// the profile a promotion reopens, short enough that a hung one is killed. +const closeWait = 5 * time.Second + // chromeLaunch starts a Chromium-family browser via CDP. ExecPath is // chromium.FindBrowser (system Chrome/Edge/… — never lich-shell); UserDataDir // is this ephemeral dir, never lich/chromium-profile. Attaching to the UI @@ -53,10 +60,17 @@ type chromePage struct { func (p *chromePage) Alive() bool { return p.ctx.Err() == nil } +// Close asks Chrome to quit instead of killing it, so cookies and storage +// reach the profile on disk before promote relaunches on it. func (p *chromePage) Close() error { - p.cancel() - p.allocCancel() - return nil + defer p.allocCancel() + defer p.cancel() + if p.ctx.Err() != nil { + return nil + } + ctx, cancel := context.WithTimeout(p.ctx, closeWait) + defer cancel() + return chromedp.Cancel(ctx) } // run drives CDP on the allocator context chromedp attached, applying wait's @@ -104,9 +118,14 @@ func (p *chromePage) Type(ctx context.Context, text string, clear bool, t Target } } if clear { + // Assigning el.value fires no input event, so a framework that owns the + // field keeps its old value; a delete through the editing commands does. err := p.run(ctx, chromedp.Evaluate(`(() => { const el = document.activeElement; - if (el && 'value' in el) el.value = ''; + if (!el) return; + if (typeof el.select === 'function') el.select(); + else document.execCommand('selectAll'); + document.execCommand('delete'); })()`, nil)) if err != nil { return err @@ -203,8 +222,46 @@ func (p *chromePage) Scroll(ctx context.Context, dy int) error { return p.run(ctx, chromedp.Evaluate(fmt.Sprintf("window.scrollBy(0, %d)", dy), nil)) } +// Focus raises the window. window.focus() from the page does not: the OS +// only lets the browser itself bring a window forward. func (p *chromePage) Focus(ctx context.Context) error { - return p.run(ctx, chromedp.Evaluate("window.focus()", nil)) + return p.run(ctx, page.BringToFront()) +} + +func (p *chromePage) Cookies(ctx context.Context) ([]*network.CookieParam, error) { + var cookies []*network.Cookie + err := p.run(ctx, chromedp.ActionFunc(func(ctx context.Context) error { + var err error + cookies, err = storage.GetCookies().Do(ctx) + return err + })) + if err != nil { + return nil, err + } + out := make([]*network.CookieParam, 0, len(cookies)) + for _, c := range cookies { + out = append(out, cookieParam(c)) + } + return out, nil +} + +func (p *chromePage) SetCookies(ctx context.Context, cookies []*network.CookieParam) error { + return p.run(ctx, storage.SetCookies(cookies)) +} + +// cookieParam turns a cookie read back into one Chrome accepts. A session +// cookie keeps no expiry, so it stays a session cookie in the new process. +func cookieParam(c *network.Cookie) *network.CookieParam { + param := &network.CookieParam{ + Name: c.Name, Value: c.Value, Domain: c.Domain, Path: c.Path, + Secure: c.Secure, HTTPOnly: c.HTTPOnly, SameSite: c.SameSite, Priority: c.Priority, + SourceScheme: c.SourceScheme, SourcePort: c.SourcePort, PartitionKey: c.PartitionKey, + } + if !c.Session { + expires := cdp.TimeSinceEpoch(time.Unix(0, int64(c.Expires*float64(time.Second)))) + param.Expires = &expires + } + return param } // withPageDeadline hangs wait's timeout off the chromedp page context. A diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go new file mode 100644 index 000000000..865d2decc --- /dev/null +++ b/internal/browser/chrome_live_test.go @@ -0,0 +1,129 @@ +package browser + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os/exec" + "testing" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" + "github.com/lichdotdev/lich/internal/chromium" +) + +func liveChrome(t *testing.T) (*chromePage, *httptest.Server) { + t.Helper() + if _, err := chromium.FindBrowser(exec.LookPath); err != nil { + t.Skip("no Chromium-family browser on this machine") + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, ``) + })) + t.Cleanup(srv.Close) + p, err := chromeLaunch(t.TempDir(), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := p.Close(); err != nil { + t.Error(err) + } + }) + return p.(*chromePage), srv +} + +func TestChromeClearReachesTheFieldsListener(t *testing.T) { + p, srv := liveChrome(t) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + field := Target{Selector: "#f"} + if err := p.Type(ctx, "abc", false, field); err != nil { + t.Fatal(err) + } + if err := p.Type(ctx, "", true, field); err != nil { + t.Fatal(err) + } + var seen string + if err := p.run(ctx, chromedp.Evaluate(`document.body.dataset.seen`, &seen)); err != nil { + t.Fatal(err) + } + if seen != "" { + t.Fatalf("the page's input listener last saw %q; clearing fired no input event", seen) + } +} + +func TestChromeCookiesKeepASessionCookie(t *testing.T) { + p, srv := liveChrome(t) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + in := []*network.CookieParam{{Name: "sid", Value: "1", URL: srv.URL}} + if err := p.SetCookies(ctx, in); err != nil { + t.Fatal(err) + } + got, err := p.Cookies(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range got { + if c.Name == "sid" { + if c.Value != "1" || c.Expires != nil { + t.Fatalf("sid read back as %+v; a session cookie must stay one", c) + } + return + } + } + t.Fatalf("sid missing from %d cookies", len(got)) +} + +func TestPromoteRelaunchesOnTheProfileWithItsCookies(t *testing.T) { + _, srv := liveChrome(t) + // headless both times: what is under test is the relaunch on the same + // profile, and a headed window needs a display the suite may not have. + launch := func(dir string, _ bool) (Page, error) { return chromeLaunch(dir, false) } + s := newService(launch, func(string, string) (string, error) { return t.TempDir(), nil }, time.Now, idleAfter) + t.Cleanup(func() { + if err := s.Cleanup(); err != nil { + t.Error(err) + } + }) + if _, err := s.Open("s", srv.URL); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + in := []*network.CookieParam{{Name: "sid", Value: "1", URL: srv.URL}} + if err := s.byOwner["s"].page.SetCookies(ctx, in); err != nil { + t.Fatal(err) + } + if _, err := s.OpenVisible("s"); err != nil { + t.Fatal(err) + } + got, err := s.byOwner["s"].page.Cookies(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range got { + if c.Name == "sid" && c.Value == "1" { + return + } + } + t.Fatalf("session cookie lost in the promotion: %d cookies", len(got)) +} + +func TestCookieParamGivesAPersistentCookieItsExpiry(t *testing.T) { + when := time.Date(2027, 1, 2, 3, 4, 5, 0, time.UTC) + got := cookieParam(&network.Cookie{Name: "k", Expires: float64(when.Unix())}) + if got.Expires == nil || !got.Expires.Time().Equal(when) { + t.Fatalf("expires = %v, want %v", got.Expires, when) + } +} diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index c72b18519..86104528c 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -4,6 +4,8 @@ import ( "context" "os" "sync" + + "github.com/chromedp/cdproto/network" ) // fakePage is a Page that never talks to Chrome. Tests inject it through @@ -21,6 +23,8 @@ type fakePage struct { forwards int scrolled []int focused int + cookies []*network.CookieParam + setCookies []*network.CookieParam info PageInfo shotErr error navErr error @@ -143,14 +147,38 @@ func (p *fakePage) Focus(context.Context) error { return nil } +func (p *fakePage) Cookies(context.Context) ([]*network.CookieParam, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.cookies, nil +} + +func (p *fakePage) SetCookies(_ context.Context, cookies []*network.CookieParam) error { + p.mu.Lock() + defer p.mu.Unlock() + p.setCookies = append(p.setCookies, cookies...) + return nil +} + type fakeLauncher struct { + mu sync.Mutex pages []*fakePage headed []bool + dirs []string + // gate, when set, runs before the page exists: a test holds a launch open + // with it to see what else the service lets through meanwhile. + gate func() } -func (f *fakeLauncher) launch(_ string, headed bool) (Page, error) { +func (f *fakeLauncher) launch(dir string, headed bool) (Page, error) { + if f.gate != nil { + f.gate() + } + f.mu.Lock() + defer f.mu.Unlock() p := &fakePage{url: "about:blank"} f.pages = append(f.pages, p) f.headed = append(f.headed, headed) + f.dirs = append(f.dirs, dir) return p, nil } diff --git a/internal/browser/page.go b/internal/browser/page.go index 5e866bde8..cdb8db905 100644 --- a/internal/browser/page.go +++ b/internal/browser/page.go @@ -1,6 +1,10 @@ package browser -import "context" +import ( + "context" + + "github.com/chromedp/cdproto/network" +) // Handle is what Open and OpenVisible return: the id later calls take, and // enough for a card or a tool result to say which window it is. @@ -73,6 +77,10 @@ type Page interface { Forward(ctx context.Context) error Scroll(ctx context.Context, dy int) error Focus(ctx context.Context) error + // Cookies and SetCookies carry a browser's cookies, session ones included, + // into the process that replaces it. + Cookies(ctx context.Context) ([]*network.CookieParam, error) + SetCookies(ctx context.Context, cookies []*network.CookieParam) error Alive() bool Close() error } diff --git a/main.go b/main.go index 1e349df05..e0b9dd761 100644 --- a/main.go +++ b/main.go @@ -417,11 +417,11 @@ func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub term.SetWorkerUnanswered(rl.WorkerUnanswered) term.SetSessionClosed(func(id, closerID string) { rl.SessionClosed(id, closerID) - _ = brows.CloseOwnedBy(id) + closeSessionBrowser(brows, id) }) term.SetSessionExited(func(id string) { rl.SessionExited(id) - _ = brows.CloseOwnedBy(id) + closeSessionBrowser(brows, id) }) term.SetErrandStatus(rl.Status) // Both questions the relay asks about a provider — whether its sessions @@ -466,7 +466,17 @@ func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub term.Mount("/drop", http.HandlerFunc(drops.Upload)) term.Mount("/blob", http.HandlerFunc(proj.ServeBlob)) term.Mount("/events", hub) - return sys, func() { _ = brows.Cleanup() } + return sys, func() { + if err := brows.Cleanup(); err != nil { + slog.Warn("close the session browsers on exit", "err", err) + } + } +} + +func closeSessionBrowser(brows *browser.Service, id string) { + if err := brows.CloseOwnedBy(id); err != nil { + slog.Warn("close a session's browser", "session", id, "err", err) + } } // run serves the loopback listener, opens the window on it (the native one, From ef180ce502297f3718079193027ed5f4938e5464 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:13:09 -0300 Subject: [PATCH 03/25] feat(browser): stream a session's page to a window pane and take its input browser.Attach starts a screencast of the session's headless browser sized to the pane (starting one when there is none), and the frames reach the window as a per-session event family, acked only once sent so a slow window slows the stream. The pane's mouse, keys, committed text and toolbar go through their own methods, so only the agent's actions light up browser-agent. browser-state tells the window when a browser starts, closes, moves to a window or loads another page; browser.Running is the catch-up read. A pane showing a browser keeps the reaper away from it. --- api/browser.go | 97 ++++++++++++++ api/testdata/contract.golden | 75 ++++++++++- internal/browser/actions.go | 37 +++++- internal/browser/browser.go | 95 +++++++++----- internal/browser/chrome.go | 11 +- internal/browser/chrome_live_test.go | 81 +++++++++++- internal/browser/chrome_view.go | 164 +++++++++++++++++++++++ internal/browser/fake_test.go | 99 +++++++++++++- internal/browser/page.go | 23 +++- internal/browser/rpc.go | 48 +++++++ internal/browser/view.go | 141 ++++++++++++++++++++ internal/browser/view_test.go | 188 +++++++++++++++++++++++++++ internal/routes/browser.go | 8 ++ main.go | 2 +- 14 files changed, 1016 insertions(+), 53 deletions(-) create mode 100644 internal/browser/chrome_view.go create mode 100644 internal/browser/view.go create mode 100644 internal/browser/view_test.go diff --git a/api/browser.go b/api/browser.go index a030372f4..4297c9513 100644 --- a/api/browser.go +++ b/api/browser.go @@ -15,8 +15,105 @@ var ( BrowserScroll = define[BrowserScrollOptions, None]("browser.Scroll") BrowserList = define[SessionIDOptions, []BrowserHandle]("browser.List") BrowserClose = define[SessionIDOptions, None]("browser.Close") + + // The window's side: a pane that shows a session's page and passes the + // person's input to it, beside the agent driving the same page. + BrowserRunning = define[None, []BrowserState]("browser.Running") + BrowserAttach = define[BrowserViewOptions, BrowserHandle]("browser.Attach") + BrowserDetach = define[SessionIDOptions, None]("browser.Detach") + BrowserResize = define[BrowserViewOptions, None]("browser.Resize") + BrowserMouse = define[BrowserMouseOptions, None]("browser.Mouse") + BrowserKey = define[BrowserKeyOptions, None]("browser.Key") + BrowserText = define[BrowserTextOptions, None]("browser.Text") + BrowserNav = define[BrowserNavOptions, None]("browser.Nav") +) + +var ( + // BrowserStateEvent is emitted whenever a session's browser starts, closes, + // moves to a window or loads another page, so the one store the window + // keeps of them stays true. + BrowserStateEvent = defineEvent[BrowserState]("browser-state") + + // BrowserFrameEvents carries a base64 JPEG of a session's page while a pane + // shows it: per session, like terminal output, so only that pane listens. + BrowserFrameEvents = defineEventFamily[string]("browser:frame:") + + // BrowserAgentEvent is emitted each time the agent acts on its page, so the + // pane can show that the page moved because of it, and where it clicked. + BrowserAgentEvent = defineEvent[BrowserAgentAct]("browser-agent") ) +// BrowserState is a session's browser as the window shows it. Open false means +// it is gone; ByAgent says the agent started it rather than the window. +type BrowserState struct { + Owner string `json:"owner"` + Open bool `json:"open"` + Headed bool `json:"headed"` + ByAgent bool `json:"byAgent"` + URL string `json:"url"` + Title string `json:"title"` +} + +// BrowserAgentAct is one agent action on its page. X and Y are where it +// clicked, in CSS pixels, when the action was a click. +type BrowserAgentAct struct { + Owner string `json:"owner"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` +} + +// BrowserViewOptions sizes a session's page to the pane showing it: W and H in +// CSS pixels, Scale the device pixels per CSS pixel. +type BrowserViewOptions struct { + ID string `json:"id"` + W int `json:"w"` + H int `json:"h"` + Scale float64 `json:"scale"` +} + +// BrowserMouseOptions is one pointer event over the pane, in CSS pixels. +// Type is press, release, move or wheel; Button is left, middle, right or +// none; DX and DY are a wheel's scroll in CSS pixels. +type BrowserMouseOptions struct { + ID string `json:"id"` + Type string `json:"type"` + X float64 `json:"x"` + Y float64 `json:"y"` + Button string `json:"button,omitempty"` + Clicks int `json:"clicks,omitempty"` + DX float64 `json:"dx,omitempty"` + DY float64 `json:"dy,omitempty"` + Modifiers int `json:"modifiers,omitempty"` +} + +// BrowserKeyOptions is one key going down or up, for keys that do not type +// text (Enter, arrows, shortcuts): text goes through BrowserText. Key and Code +// are the DOM's KeyboardEvent.key and .code; Modifiers is CDP's bit set (Alt +// 1, Ctrl 2, Meta 4, Shift 8). +type BrowserKeyOptions struct { + ID string `json:"id"` + Type string `json:"type"` + Key string `json:"key"` + Code string `json:"code,omitempty"` + KeyCode int `json:"keyCode,omitempty"` + Modifiers int `json:"modifiers,omitempty"` +} + +// BrowserNavOptions is the pane's toolbar: Action is back, forward, reload or +// go, and URL is where go loads. +type BrowserNavOptions struct { + ID string `json:"id"` + Action string `json:"action"` + URL string `json:"url,omitempty"` +} + +// BrowserTextOptions types text into whatever has focus, the way an input +// method commits it. +type BrowserTextOptions struct { + ID string `json:"id"` + Text string `json:"text"` +} + // BrowserHandle is one session's browser: the id later calls take, and enough // for a tool result to say which window it is. type BrowserHandle struct { diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index 660e6c2e6..1721a9ab5 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -3,19 +3,27 @@ agentplugin.Status(None) []PluginStatus agentplugin.Update(PluginProviderOptions) None appupdate.Apply(None) None appupdate.Status(None) AppUpdateStatus +browser.Attach(BrowserViewOptions) BrowserHandle browser.Back(SessionIDOptions) None browser.Click(BrowserClickOptions) None browser.Close(SessionIDOptions) None +browser.Detach(SessionIDOptions) None browser.Forward(SessionIDOptions) None browser.Info(SessionIDOptions) BrowserPageInfo +browser.Key(BrowserKeyOptions) None browser.List(SessionIDOptions) []BrowserHandle +browser.Mouse(BrowserMouseOptions) None +browser.Nav(BrowserNavOptions) None browser.Navigate(BrowserOpenOptions) BrowserHandle browser.Open(BrowserOpenOptions) BrowserHandle browser.Press(BrowserPressOptions) None browser.Reload(SessionIDOptions) None +browser.Resize(BrowserViewOptions) None +browser.Running(None) []BrowserState browser.Screenshot(BrowserScreenshotOptions) string browser.Scroll(BrowserScrollOptions) None browser.Show(SessionIDOptions) BrowserHandle +browser.Text(BrowserTextOptions) None browser.Type(BrowserTypeOptions) None claudeprofile.CancelLogin(ProfileSlugOptions) None claudeprofile.CreateProfile(CreateProfileOptions) Profile @@ -213,6 +221,9 @@ themes.UpdateFromGit(UpdateThemeOptions) ThemeGitResult tray.SetLabels(TrayLabels) None event appupdate-progress AppUpdateProgress +event browser-agent BrowserAgentAct +event browser-state BrowserState +event browser:frame: string event claude-profile-login ProfileLogin event codex-profile-login ProfileLogin event lich-plugin-profiles PluginProfiles @@ -279,6 +290,21 @@ AppUpdateStatus { installCommand string } +BrowserViewOptions { + id string + w int + h int + scale float64 +} + +BrowserHandle { + id string + url string + title string + owner string + headed bool +} + SessionIDOptions { id string } @@ -296,12 +322,31 @@ BrowserPageInfo { truncated bool } -BrowserHandle { +BrowserKeyOptions { id string - url string - title string - owner string - headed bool + type string + key string + code,omitempty string + keyCode,omitempty int + modifiers,omitempty int +} + +BrowserMouseOptions { + id string + type string + x float64 + y float64 + button,omitempty string + clicks,omitempty int + dx,omitempty float64 + dy,omitempty float64 + modifiers,omitempty int +} + +BrowserNavOptions { + id string + action string + url,omitempty string } BrowserOpenOptions { @@ -315,6 +360,15 @@ BrowserPressOptions { ...BrowserTarget } +BrowserState { + owner string + open bool + headed bool + byAgent bool + url string + title string +} + BrowserScreenshotOptions { id string path string @@ -325,6 +379,11 @@ BrowserScrollOptions { dy int } +BrowserTextOptions { + id string + text string +} + BrowserTypeOptions { id string text string @@ -1389,6 +1448,12 @@ AppUpdateProgress { total int64 } +BrowserAgentAct { + owner string + x,omitempty *float64 + y,omitempty *float64 +} + ProfileLogin { profile string output,omitempty string diff --git a/internal/browser/actions.go b/internal/browser/actions.go index 671984144..f0d9e2078 100644 --- a/internal/browser/actions.go +++ b/internal/browser/actions.go @@ -65,6 +65,7 @@ func (s *Service) Info(id string) (PageInfo, error) { return PageInfo{}, err } s.remember(sess, info.URL, info.Title) + s.agentActed(sess, nil, nil) return info, nil } @@ -73,9 +74,23 @@ func (s *Service) Click(id string, t Target) error { if t.empty() { return fmt.Errorf("click needs an index, a selector, or x and y") } - return s.act(id, func(p Page, ctx context.Context) error { - return p.Click(ctx, t) + sess, err := s.live(id) + if err != nil { + return err + } + var x, y float64 + err = runPage(func(ctx context.Context) error { + var e error + x, y, e = sess.page.Click(ctx, t) + return e }) + if err != nil { + s.drop(sess) + return err + } + s.touch(sess) + s.agentActed(sess, &x, &y) + return nil } // Type sends keystrokes. clear selects-and-replaces; the typed text never @@ -121,6 +136,7 @@ func (s *Service) Screenshot(id, dest string) (string, error) { return "", err } s.touch(sess) + s.agentActed(sess, nil, nil) return dest, nil } @@ -142,6 +158,7 @@ func (s *Service) Navigate(id, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") + s.agentActed(sess, nil, nil) return s.handleOf(sess), nil } @@ -165,16 +182,26 @@ func (s *Service) Scroll(id string, dy int) error { return s.act(id, func(p Page, ctx context.Context) error { return p.Scroll(ctx, dy) }) } +// act runs one agent action on its page; drive is the same for the window's +// own input, which lights nothing up. func (s *Service) act(id string, fn func(Page, context.Context) error) error { + sess, err := s.drive(id, fn) + if err == nil { + s.agentActed(sess, nil, nil) + } + return err +} + +func (s *Service) drive(id string, fn func(Page, context.Context) error) (*session, error) { sess, err := s.live(id) if err != nil { - return err + return nil, err } err = runPage(func(ctx context.Context) error { return fn(sess.page, ctx) }) if err != nil { s.drop(sess) - return err + return nil, err } s.touch(sess) - return nil + return sess, nil } diff --git a/internal/browser/browser.go b/internal/browser/browser.go index 497dea89c..67564ad52 100644 --- a/internal/browser/browser.go +++ b/internal/browser/browser.go @@ -14,11 +14,11 @@ import ( "github.com/chromedp/cdproto/network" ) -// idleAfter is how long a headless browser sits unused before the reaper -// closes it. Five minutes matches the "walked away from the card" window -// without holding a Chromium for a session that finished hours ago. A headed -// window is never reaped: lich cannot see a person using it, so only the card -// closing ends it. +// idleAfter is how long a headless browser no pane shows sits unused before +// the reaper closes it. Five minutes matches the "walked away from the card" +// window without holding a Chromium for a session that finished hours ago. A +// headed window is never reaped: lich cannot see a person using it, so only +// the card closing ends it. const idleAfter = 5 * time.Minute // reapEvery is how often the reaper looks. Shorter than idleAfter so a @@ -55,20 +55,27 @@ type Service struct { dead []*session stop chan struct{} stopped bool + // emit pushes to the window's /events (events.Hub.Emit). + emit func(name string, data any) } type session struct { id, owner, dir string url, title string headed bool - page Page - lastUsed time.Time + // byAgent: the agent started it, not the window. watching: a pane shows + // it, so the reaper leaves it alone like a headed window. + byAgent, watching bool + page Page + lastUsed time.Time } // New starts a service that launches the system Chromium (chromium.FindBrowser) -// into an ephemeral profile, never the lich --app one. -func New() *Service { +// into an ephemeral profile, never the lich --app one, and tells the window +// about each browser through emit. +func New(emit func(name string, data any)) *Service { s := newService(chromeLaunch, os.MkdirTemp, time.Now, idleAfter) + s.emit = emit s.startReaper() return s } @@ -83,6 +90,7 @@ func newService(launch Launcher, mkTmp func(string, string) (string, error), now byID: make(map[string]*session), owners: make(map[string]*sync.Mutex), stop: make(chan struct{}), + emit: func(string, any) {}, } } @@ -101,7 +109,7 @@ func (s *Service) Open(owner, rawURL string) (Handle, error) { defer lock.Unlock() sess := s.current(owner) if sess == nil { - return s.launchFresh(owner, u, false) + return s.launchFresh(owner, u, false, true) } if err := runPage(func(ctx context.Context) error { return sess.page.Navigate(ctx, u) @@ -110,6 +118,7 @@ func (s *Service) Open(owner, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") + s.agentActed(sess, nil, nil) return s.handleOf(sess), nil } @@ -126,7 +135,7 @@ func (s *Service) OpenVisible(owner string) (Handle, error) { sess := s.current(owner) switch { case sess == nil: - return s.launchFresh(owner, "about:blank", true) + return s.launchFresh(owner, "about:blank", true, false) case !sess.headed: return s.promote(sess) } @@ -155,14 +164,16 @@ func (s *Service) promote(sess *session) (Handle, error) { s.mu.Lock() s.detachLocked(sess) s.unlock() - if err := sess.page.Close(); err != nil { + err := sess.page.Close() + s.emitState(sess, false) + if err != nil { return Handle{}, errors.Join(err, os.RemoveAll(sess.dir)) } u := sess.url if u == "" { u = "about:blank" } - return s.start(sess.owner, u, true, sess.dir, cookies) + return s.start(launchSpec{owner: sess.owner, url: u, headed: true, byAgent: sess.byAgent, dir: sess.dir, cookies: cookies}) } // List returns this session's browser, or an empty list. Never null: a @@ -189,7 +200,9 @@ func (s *Service) Close(id string) error { if sess == nil { return nil } - return sess.destroy() + err := sess.destroy() + s.emitState(sess, false) + return err } // CloseOwnedBy tears down the browser of a lich session that is going away. @@ -220,43 +233,57 @@ func (s *Service) Cleanup() error { return errors.Join(errs...) } -func (s *Service) launchFresh(owner, u string, headed bool) (Handle, error) { +func (s *Service) launchFresh(owner, u string, headed, byAgent bool) (Handle, error) { dir, err := s.mkTmp("", "lich-browser-") if err != nil { return Handle{}, err } - return s.start(owner, u, headed, dir, nil) + return s.start(launchSpec{owner: owner, url: u, headed: headed, byAgent: byAgent, dir: dir}) +} + +// launchSpec is one browser start: whose, where it opens, how, on which +// profile, and the cookies it carries over from the one it replaces. +type launchSpec struct { + owner, url, dir string + headed, byAgent bool + cookies []*network.CookieParam } -// start launches a browser on dir, loads cookies and then u, and registers it. -// The caller holds owner's lock; mu is taken only to register. -func (s *Service) start(owner, u string, headed bool, dir string, cookies []*network.CookieParam) (Handle, error) { - page, err := s.launch(dir, headed) +// start launches a browser on spec.dir, loads the cookies and then the URL, +// and registers it. The caller holds the owner's lock; mu is taken only to +// register. +func (s *Service) start(spec launchSpec) (Handle, error) { + page, err := s.launch(spec.dir, spec.headed) if err != nil { - return Handle{}, errors.Join(err, os.RemoveAll(dir)) + return Handle{}, errors.Join(err, os.RemoveAll(spec.dir)) + } + sess := &session{ + id: newID(), owner: spec.owner, dir: spec.dir, url: spec.url, headed: spec.headed, + byAgent: spec.byAgent, page: page, lastUsed: s.now(), } + page.Listen(s.frameSink(sess), s.infoSink(sess)) if err := runPage(func(ctx context.Context) error { - if len(cookies) > 0 { - if err := page.SetCookies(ctx, cookies); err != nil { + if len(spec.cookies) > 0 { + if err := page.SetCookies(ctx, spec.cookies); err != nil { return err } } - return page.Navigate(ctx, u) + return page.Navigate(ctx, spec.url) }); err != nil { - return Handle{}, errors.Join(err, page.Close(), os.RemoveAll(dir)) - } - sess := &session{ - id: newID(), owner: owner, dir: dir, url: u, headed: headed, page: page, lastUsed: s.now(), + return Handle{}, errors.Join(err, page.Close(), os.RemoveAll(spec.dir)) } s.mu.Lock() - defer s.unlock() if s.stopped { s.dead = append(s.dead, sess) + s.unlock() return Handle{}, errStopped } - s.byOwner[owner] = sess + s.byOwner[spec.owner] = sess s.byID[sess.id] = sess - return sess.handle(), nil + h := sess.handle() + s.unlock() + s.emitState(sess, true) + return h, nil } func (s *Service) ownerLock(owner string) *sync.Mutex { @@ -332,6 +359,7 @@ func (s *Service) unlock() { if err := sess.destroy(); err != nil { slog.Warn("close a session browser", "owner", sess.owner, "err", err) } + s.emitState(sess, false) } } @@ -362,7 +390,8 @@ func (s *Service) startReaper() { }() } -// Reap closes headless browsers that have sat idle past the idle window. +// Reap closes headless browsers no pane shows that have sat idle past the +// idle window. func (s *Service) Reap() { s.mu.Lock() defer s.unlock() @@ -372,7 +401,7 @@ func (s *Service) Reap() { func (s *Service) reapLocked() { cutoff := s.now().Add(-s.idle) for _, sess := range s.byID { - if !sess.headed && sess.lastUsed.Before(cutoff) { + if !sess.headed && !sess.watching && sess.lastUsed.Before(cutoff) { s.buryLocked(sess) } } diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go index dcb33db01..50adbff80 100644 --- a/internal/browser/chrome.go +++ b/internal/browser/chrome.go @@ -56,6 +56,7 @@ type chromePage struct { ctx context.Context cancel context.CancelFunc allocCancel context.CancelFunc + listen listeners } func (p *chromePage) Alive() bool { return p.ctx.Err() == nil } @@ -103,17 +104,17 @@ func (p *chromePage) Info(ctx context.Context) (PageInfo, error) { return info, nil } -func (p *chromePage) Click(ctx context.Context, t Target) error { +func (p *chromePage) Click(ctx context.Context, t Target) (x, y float64, err error) { pt, err := p.resolve(ctx, t) if err != nil { - return err + return 0, 0, err } - return p.run(ctx, chromedp.MouseClickXY(pt.X, pt.Y)) + return pt.X, pt.Y, p.run(ctx, chromedp.MouseClickXY(pt.X, pt.Y)) } func (p *chromePage) Type(ctx context.Context, text string, clear bool, t Target) error { if !t.empty() { - if err := p.Click(ctx, t); err != nil { + if _, _, err := p.Click(ctx, t); err != nil { return err } } @@ -142,7 +143,7 @@ func (p *chromePage) Press(ctx context.Context, key string, t Target) error { return err } if !t.empty() { - if err := p.Click(ctx, t); err != nil { + if _, _, err := p.Click(ctx, t); err != nil { return err } } diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index 865d2decc..3674c834b 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -2,6 +2,7 @@ package browser import ( "context" + "encoding/base64" "fmt" "net/http" "net/http/httptest" @@ -11,6 +12,7 @@ import ( "github.com/chromedp/cdproto/network" "github.com/chromedp/chromedp" + "github.com/lichdotdev/lich/api" "github.com/lichdotdev/lich/internal/chromium" ) @@ -21,7 +23,10 @@ func liveChrome(t *testing.T) (*chromePage, *httptest.Server) { } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "text/html") - fmt.Fprint(w, ``) + fmt.Fprint(w, `Probe +
+
`) })) t.Cleanup(srv.Close) p, err := chromeLaunch(t.TempDir(), false) @@ -127,3 +132,77 @@ func TestCookieParamGivesAPersistentCookieItsExpiry(t *testing.T) { t.Fatalf("expires = %v, want %v", got.Expires, when) } } + +func TestChromeScreencastSendsFramesAndPageInfo(t *testing.T) { + p, srv := liveChrome(t) + frames := make(chan string, 4) + titles := make(chan string, 4) + p.Listen(func(data string) { + select { + case frames <- data: + default: + } + }, func(_, title string) { + select { + case titles <- title: + default: + } + }) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + if err := p.Screencast(ctx, Viewport{W: 400, H: 300, Scale: 1}); err != nil { + t.Fatal(err) + } + select { + case data := <-frames: + img, err := base64.StdEncoding.DecodeString(data) + if err != nil || len(img) < 2 || img[0] != 0xFF || img[1] != 0xD8 { + t.Fatalf("frame is not a JPEG (err %v, %d bytes)", err, len(img)) + } + case <-time.After(10 * time.Second): + t.Fatal("no screencast frame") + } + deadline := time.After(10 * time.Second) + for { + select { + case title := <-titles: + if title == "Probe" { + return + } + case <-deadline: + t.Fatal("no page info with the document's title") + } + } +} + +func TestChromeWindowInputTypesAndSubmits(t *testing.T) { + p, srv := liveChrome(t) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"press", "release"} { + if err := p.Mouse(ctx, api.BrowserMouseOptions{Type: kind, X: 50, Y: 25, Button: "left", Clicks: 1}); err != nil { + t.Fatal(err) + } + } + if err := p.InsertText(ctx, "ação"); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"down", "up"} { + if err := p.Key(ctx, api.BrowserKeyOptions{Type: kind, Key: "Enter", Code: "Enter", KeyCode: 13}); err != nil { + t.Fatal(err) + } + } + var sent string + if err := p.run(ctx, chromedp.Evaluate(`document.body.dataset.sent || ""`, &sent)); err != nil { + t.Fatal(err) + } + if sent != "ação" { + t.Fatalf("form sent %q, want the clicked field's typed text", sent) + } +} diff --git a/internal/browser/chrome_view.go b/internal/browser/chrome_view.go new file mode 100644 index 000000000..53c22edd0 --- /dev/null +++ b/internal/browser/chrome_view.go @@ -0,0 +1,164 @@ +package browser + +import ( + "context" + "log/slog" + "math" + "sync" + + "github.com/chromedp/cdproto/emulation" + "github.com/chromedp/cdproto/input" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" + "github.com/lichdotdev/lich/api" +) + +// screencastQuality is the JPEG quality of a pane's frames: legible text at a +// fraction of PNG's size, since every frame crosses /events as base64. +const screencastQuality = 70 + +// listeners are what a chromePage hands its CDP events to; the service sets +// them once, right after the launch. +type listeners struct { + mu sync.Mutex + onFrame func(data string) + onInfo func(url, title string) +} + +func (p *chromePage) Listen(onFrame func(data string), onInfo func(url, title string)) { + p.listen.mu.Lock() + p.listen.onFrame, p.listen.onInfo = onFrame, onInfo + p.listen.mu.Unlock() + chromedp.ListenTarget(p.ctx, func(ev any) { + switch e := ev.(type) { + case *page.EventScreencastFrame: + go p.deliver(e) + case *page.EventFrameNavigated: + if e.Frame.ParentID == "" { + go p.reportInfo() + } + case *page.EventNavigatedWithinDocument, *page.EventDomContentEventFired: + go p.reportInfo() + } + }) +} + +// reportInfo reads the URL and title after a navigation: a link followed, a +// history step, a single-page app's pushState, or the document's title +// arriving with its DOM. +func (p *chromePage) reportInfo() { + p.listen.mu.Lock() + onInfo := p.listen.onInfo + p.listen.mu.Unlock() + if onInfo == nil { + return + } + var info struct { + URL string `json:"url"` + Title string `json:"title"` + } + if err := p.run(nil, chromedp.Evaluate(`({url: location.href, title: document.title})`, &info)); err != nil { + if p.Alive() { + slog.Warn("browser: read the page's URL and title", "err", err) + } + return + } + onInfo(info.URL, info.Title) +} + +// deliver hands one frame on, then acks it. Chrome sends no next frame until +// the ack, so a window slow to take frames slows the stream instead of +// piling it up. +func (p *chromePage) deliver(frame *page.EventScreencastFrame) { + p.listen.mu.Lock() + onFrame := p.listen.onFrame + p.listen.mu.Unlock() + if onFrame != nil { + onFrame(frame.Data) + } + ack := chromedp.ActionFunc(func(ctx context.Context) error { + return page.ScreencastFrameAck(frame.SessionID).Do(ctx) + }) + if err := p.run(nil, ack); err != nil && p.Alive() { + slog.Warn("browser: ack a screencast frame", "err", err) + } +} + +func (p *chromePage) Screencast(ctx context.Context, v Viewport) error { + if err := p.Resize(ctx, v); err != nil { + return err + } + w, h := v.devicePixels() + return p.run(ctx, page.StartScreencast(). + WithFormat(page.ScreencastFormatJpeg). + WithQuality(screencastQuality). + WithMaxWidth(w). + WithMaxHeight(h)) +} + +func (p *chromePage) StopScreencast(ctx context.Context) error { + return p.run(ctx, page.StopScreencast()) +} + +// Resize lays the page out at the pane's size, as a window of that size would. +func (p *chromePage) Resize(ctx context.Context, v Viewport) error { + return p.run(ctx, emulation.SetDeviceMetricsOverride(int64(v.W), int64(v.H), v.Scale, false)) +} + +func (p *chromePage) Mouse(ctx context.Context, m api.BrowserMouseOptions) error { + kind, ok := mouseTypes[m.Type] + if !ok { + return errUnknownInput + } + ev := input.DispatchMouseEvent(kind, m.X, m.Y).WithModifiers(input.Modifier(m.Modifiers)) + switch kind { + case input.MouseWheel: + ev = ev.WithDeltaX(m.DX).WithDeltaY(m.DY) + case input.MousePressed, input.MouseReleased: + ev = ev.WithButton(input.MouseButton(m.Button)).WithClickCount(int64(max(m.Clicks, 1))) + } + if kind != input.MouseReleased { + // buttons is what a drag reads: which button is still held as it moves. + ev = ev.WithButtons(mouseButtons[m.Button]) + } + return p.run(ctx, ev) +} + +// Key sends a key that types no text. Enter is the exception CDP needs: its +// default action (submit, newline) runs only on a keyDown carrying "\r". +func (p *chromePage) Key(ctx context.Context, k api.BrowserKeyOptions) error { + var ev *input.DispatchKeyEventParams + switch { + case k.Type == "up": + ev = input.DispatchKeyEvent(input.KeyUp) + case k.Type != "down": + return errUnknownInput + case k.Key == "Enter": + ev = input.DispatchKeyEvent(input.KeyDown).WithText("\r") + default: + ev = input.DispatchKeyEvent(input.KeyRawDown) + } + ev = ev.WithKey(k.Key).WithCode(k.Code). + WithWindowsVirtualKeyCode(int64(k.KeyCode)). + WithNativeVirtualKeyCode(int64(k.KeyCode)). + WithModifiers(input.Modifier(k.Modifiers)) + return p.run(ctx, ev) +} + +func (p *chromePage) InsertText(ctx context.Context, text string) error { + return p.run(ctx, input.InsertText(text)) +} + +var mouseTypes = map[string]input.MouseType{ + "press": input.MousePressed, + "release": input.MouseReleased, + "move": input.MouseMoved, + "wheel": input.MouseWheel, +} + +// mouseButtons is CDP's buttons bit set: left 1, right 2, middle 4. +var mouseButtons = map[string]int64{"left": 1, "right": 2, "middle": 4} + +func (v Viewport) devicePixels() (w, h int64) { + return int64(math.Round(float64(v.W) * v.Scale)), int64(math.Round(float64(v.H) * v.Scale)) +} diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index 86104528c..5df80a37a 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/chromedp/cdproto/network" + "github.com/lichdotdev/lich/api" ) // fakePage is a Page that never talks to Chrome. Tests inject it through @@ -25,6 +26,14 @@ type fakePage struct { focused int cookies []*network.CookieParam setCookies []*network.CookieParam + clickAt [2]float64 + onFrame func(string) + onInfo func(url, title string) + casting bool + viewports []Viewport + mice []api.BrowserMouseOptions + keys []api.BrowserKeyOptions + texts []string info PageInfo shotErr error navErr error @@ -82,11 +91,11 @@ func (p *fakePage) Info(context.Context) (PageInfo, error) { return info, nil } -func (p *fakePage) Click(_ context.Context, t Target) error { +func (p *fakePage) Click(_ context.Context, t Target) (x, y float64, err error) { p.mu.Lock() defer p.mu.Unlock() p.clicks = append(p.clicks, t) - return nil + return p.clickAt[0], p.clickAt[1], nil } func (p *fakePage) Type(_ context.Context, text string, clear bool, t Target) error { @@ -160,6 +169,92 @@ func (p *fakePage) SetCookies(_ context.Context, cookies []*network.CookieParam) return nil } +func (p *fakePage) Listen(onFrame func(string), onInfo func(url, title string)) { + p.mu.Lock() + defer p.mu.Unlock() + p.onFrame, p.onInfo = onFrame, onInfo +} + +func (p *fakePage) Screencast(_ context.Context, v Viewport) error { + p.mu.Lock() + defer p.mu.Unlock() + p.casting = true + p.viewports = append(p.viewports, v) + return nil +} + +func (p *fakePage) StopScreencast(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + p.casting = false + return nil +} + +func (p *fakePage) Resize(_ context.Context, v Viewport) error { + p.mu.Lock() + defer p.mu.Unlock() + p.viewports = append(p.viewports, v) + return nil +} + +func (p *fakePage) Mouse(_ context.Context, m api.BrowserMouseOptions) error { + p.mu.Lock() + defer p.mu.Unlock() + p.mice = append(p.mice, m) + return nil +} + +func (p *fakePage) Key(_ context.Context, k api.BrowserKeyOptions) error { + p.mu.Lock() + defer p.mu.Unlock() + p.keys = append(p.keys, k) + return nil +} + +func (p *fakePage) InsertText(_ context.Context, text string) error { + p.mu.Lock() + defer p.mu.Unlock() + p.texts = append(p.texts, text) + return nil +} + +// emitted records what the service pushed to the window, by event name. +type emitted struct { + mu sync.Mutex + sent []sentEvent +} + +type sentEvent struct { + name string + data any +} + +func (e *emitted) emit(name string, data any) { + e.mu.Lock() + defer e.mu.Unlock() + e.sent = append(e.sent, sentEvent{name, data}) +} + +func (e *emitted) named(name string) []any { + e.mu.Lock() + defer e.mu.Unlock() + var out []any + for _, ev := range e.sent { + if ev.name == name { + out = append(out, ev.data) + } + } + return out +} + +func (e *emitted) states() []api.BrowserState { + var out []api.BrowserState + for _, d := range e.named(api.BrowserStateEvent.Name) { + out = append(out, d.(api.BrowserState)) + } + return out +} + type fakeLauncher struct { mu sync.Mutex pages []*fakePage diff --git a/internal/browser/page.go b/internal/browser/page.go index cdb8db905..0b1a5ab0f 100644 --- a/internal/browser/page.go +++ b/internal/browser/page.go @@ -2,10 +2,21 @@ package browser import ( "context" + "errors" "github.com/chromedp/cdproto/network" + "github.com/lichdotdev/lich/api" ) +var errUnknownInput = errors.New("unknown input type") + +// Viewport is the pane a page is drawn into: W and H in CSS pixels, Scale the +// device pixels per CSS pixel. +type Viewport struct { + W, H int + Scale float64 +} + // Handle is what Open and OpenVisible return: the id later calls take, and // enough for a card or a tool result to say which window it is. type Handle struct { @@ -68,7 +79,8 @@ type PageInfo struct { type Page interface { Navigate(ctx context.Context, url string) error Info(ctx context.Context) (PageInfo, error) - Click(ctx context.Context, t Target) error + // Click returns where it clicked, in CSS pixels. + Click(ctx context.Context, t Target) (x, y float64, err error) Type(ctx context.Context, text string, clear bool, t Target) error Press(ctx context.Context, key string, t Target) error Screenshot(ctx context.Context, dest string) error @@ -81,6 +93,15 @@ type Page interface { // into the process that replaces it. Cookies(ctx context.Context) ([]*network.CookieParam, error) SetCookies(ctx context.Context, cookies []*network.CookieParam) error + // Listen hands the page what to call with each screencast frame (base64 + // JPEG) and with its URL and title whenever they change. + Listen(onFrame func(data string), onInfo func(url, title string)) + Screencast(ctx context.Context, v Viewport) error + StopScreencast(ctx context.Context) error + Resize(ctx context.Context, v Viewport) error + Mouse(ctx context.Context, m api.BrowserMouseOptions) error + Key(ctx context.Context, k api.BrowserKeyOptions) error + InsertText(ctx context.Context, text string) error Alive() bool Close() error } diff --git a/internal/browser/rpc.go b/internal/browser/rpc.go index 78b162e96..217c42f45 100644 --- a/internal/browser/rpc.go +++ b/internal/browser/rpc.go @@ -117,3 +117,51 @@ func infoAPI(p PageInfo) api.BrowserPageInfo { func targetOf(t api.BrowserTarget) Target { return Target{Index: t.Index, Selector: t.Selector, X: t.X, Y: t.Y} } + +// RunningStates is every browser this process runs, for a window catching up. +func (s *Service) RunningStates(_ context.Context, _ api.None) ([]api.BrowserState, error) { + return s.Running(), nil +} + +// AttachView shows this session's page in a pane. +func (s *Service) AttachView(_ context.Context, opts api.BrowserViewOptions) (api.BrowserHandle, error) { + h, err := s.Attach(opts.ID, viewportOf(opts)) + return handleAPI(h), err +} + +// DetachView stops a pane's frames. +func (s *Service) DetachView(_ context.Context, opts api.SessionIDOptions) (api.None, error) { + return api.None{}, s.Detach(opts.ID) +} + +// ResizeView lays the page out at the pane's new size. +func (s *Service) ResizeView(_ context.Context, opts api.BrowserViewOptions) (api.None, error) { + return api.None{}, s.Resize(opts.ID, viewportOf(opts)) +} + +// MouseInput passes one pointer event from the pane. +func (s *Service) MouseInput(_ context.Context, opts api.BrowserMouseOptions) (api.None, error) { + _, err := s.drive(opts.ID, func(p Page, ctx context.Context) error { return p.Mouse(ctx, opts) }) + return api.None{}, err +} + +// KeyInput passes one key from the pane. +func (s *Service) KeyInput(_ context.Context, opts api.BrowserKeyOptions) (api.None, error) { + _, err := s.drive(opts.ID, func(p Page, ctx context.Context) error { return p.Key(ctx, opts) }) + return api.None{}, err +} + +// TextInput types text the pane's input method committed. +func (s *Service) TextInput(_ context.Context, opts api.BrowserTextOptions) (api.None, error) { + _, err := s.drive(opts.ID, func(p Page, ctx context.Context) error { return p.InsertText(ctx, opts.Text) }) + return api.None{}, err +} + +// NavView runs the pane's toolbar. +func (s *Service) NavView(_ context.Context, opts api.BrowserNavOptions) (api.None, error) { + return api.None{}, s.Nav(opts.ID, opts.Action, opts.URL) +} + +func viewportOf(opts api.BrowserViewOptions) Viewport { + return Viewport{W: opts.W, H: opts.H, Scale: opts.Scale} +} diff --git a/internal/browser/view.go b/internal/browser/view.go new file mode 100644 index 000000000..ab5697209 --- /dev/null +++ b/internal/browser/view.go @@ -0,0 +1,141 @@ +package browser + +import ( + "context" + "errors" + "fmt" + + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/internal/events" +) + +// errInWindow: the pane draws a headless browser, and this one moved to its +// own window, where the person already sees it. +var errInWindow = errors.New("this browser is open in its own window") + +// Attach shows this session's browser in a pane sized v, starting a headless +// one at about:blank when there is none. The page the agent left stays as it +// is: the pane only starts watching it. +func (s *Service) Attach(owner string, v Viewport) (Handle, error) { + if owner == "" { + return Handle{}, errNoOwner + } + lock := s.ownerLock(owner) + lock.Lock() + defer lock.Unlock() + if s.current(owner) == nil { + if _, err := s.launchFresh(owner, "about:blank", false, false); err != nil { + return Handle{}, err + } + } + sess := s.current(owner) + if sess == nil { + return Handle{}, errNoBrowser + } + if sess.headed { + return Handle{}, errInWindow + } + if err := runPage(func(ctx context.Context) error { return sess.page.Screencast(ctx, v) }); err != nil { + s.drop(sess) + return Handle{}, err + } + s.mu.Lock() + sess.watching = true + s.mu.Unlock() + return s.handleOf(sess), nil +} + +// Detach stops the frames of a pane that closed or went off screen. The +// browser keeps running for the agent. +func (s *Service) Detach(owner string) error { + sess, err := s.drive(owner, func(p Page, ctx context.Context) error { return p.StopScreencast(ctx) }) + if err != nil { + return err + } + s.mu.Lock() + sess.watching = false + s.mu.Unlock() + return nil +} + +// Resize lays the page out again at the pane's new size. +func (s *Service) Resize(owner string, v Viewport) error { + _, err := s.drive(owner, func(p Page, ctx context.Context) error { return p.Resize(ctx, v) }) + return err +} + +// Nav runs the pane's toolbar: back, forward, reload, or go to url. +func (s *Service) Nav(owner, action, rawURL string) error { + var fn func(Page, context.Context) error + switch action { + case "back": + fn = func(p Page, ctx context.Context) error { return p.Back(ctx) } + case "forward": + fn = func(p Page, ctx context.Context) error { return p.Forward(ctx) } + case "reload": + fn = func(p Page, ctx context.Context) error { return p.Reload(ctx) } + case "go": + u, err := allowURL(rawURL) + if err != nil { + return err + } + fn = func(p Page, ctx context.Context) error { return p.Navigate(ctx, u) } + default: + return fmt.Errorf("unknown browser action %q", action) + } + _, err := s.drive(owner, fn) + return err +} + +// Running is every browser this process runs, for a window catching up. +func (s *Service) Running() []api.BrowserState { + s.mu.Lock() + defer s.mu.Unlock() + out := []api.BrowserState{} + for _, sess := range s.byID { + if sess.alive() { + out = append(out, sess.stateLocked(true)) + } + } + return out +} + +func (s *Service) frameSink(sess *session) func(string) { + frames := api.BrowserFrameEvents.Of(sess.owner) + return func(data string) { events.Send(s.emit, frames, data) } +} + +// infoSink records the page's URL and title as it navigates on its own (a +// link the person or the agent clicked), and tells the window once the +// browser is registered. +func (s *Service) infoSink(sess *session) func(url, title string) { + return func(url, title string) { + s.mu.Lock() + if url != "" { + sess.url = url + } + sess.title = title + registered := s.byID[sess.id] == sess + s.mu.Unlock() + if registered { + s.emitState(sess, true) + } + } +} + +func (s *Service) emitState(sess *session, open bool) { + s.mu.Lock() + state := sess.stateLocked(open) + s.mu.Unlock() + events.Send(s.emit, api.BrowserStateEvent, state) +} + +func (s *Service) agentActed(sess *session, x, y *float64) { + events.Send(s.emit, api.BrowserAgentEvent, api.BrowserAgentAct{Owner: sess.owner, X: x, Y: y}) +} + +func (s *session) stateLocked(open bool) api.BrowserState { + return api.BrowserState{ + Owner: s.owner, Open: open, Headed: s.headed, ByAgent: s.byAgent, URL: s.url, Title: s.title, + } +} diff --git a/internal/browser/view_test.go b/internal/browser/view_test.go new file mode 100644 index 000000000..ede0ba9d8 --- /dev/null +++ b/internal/browser/view_test.go @@ -0,0 +1,188 @@ +package browser + +import ( + "testing" + "time" + + "github.com/lichdotdev/lich/api" +) + +func viewService(t *testing.T) (*Service, *fakeLauncher, *emitted) { + t.Helper() + launch := &fakeLauncher{} + s := testService(t, launch) + out := &emitted{} + s.emit = out.emit + return s, launch, out +} + +func TestAgentOpenTellsTheWindowItWasTheAgent(t *testing.T) { + s, _, out := viewService(t) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + states := out.states() + if len(states) != 1 || !states[0].Open || !states[0].ByAgent || states[0].Owner != "s" { + t.Fatalf("states = %+v, want one open state by the agent", states) + } +} + +func TestAttachWatchesTheAgentsPageWithoutRelaunching(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Open("s", "https://example.com/app"); err != nil { + t.Fatal(err) + } + v := Viewport{W: 640, H: 480, Scale: 1.5} + h, err := s.Attach("s", v) + if err != nil { + t.Fatal(err) + } + if len(launch.pages) != 1 { + t.Fatalf("attach launched %d browsers, want the agent's one", len(launch.pages)) + } + if h.URL != "https://example.com/app" { + t.Fatalf("attached handle = %+v", h) + } + page := launch.pages[0] + if !page.casting || page.viewports[0] != v { + t.Fatalf("screencast not started at %+v: casting=%v viewports=%v", v, page.casting, page.viewports) + } + page.onFrame("anBlZw==") + frames := out.named(api.BrowserFrameEvents.Of("s").Name) + if len(frames) != 1 || frames[0] != "anBlZw==" { + t.Fatalf("frames = %v", frames) + } +} + +func TestAttachWithoutABrowserStartsOneTheWindowOwns(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + if len(launch.pages) != 1 || launch.headed[0] { + t.Fatalf("attach launched headed=%v, want one headless", launch.headed) + } + if st := out.states(); len(st) != 1 || st[0].ByAgent { + t.Fatalf("states = %+v, want one opened by the window", st) + } +} + +func TestAttachRefusesABrowserInItsOwnWindow(t *testing.T) { + s, _, _ := viewService(t) + if _, err := s.OpenVisible("s"); err != nil { + t.Fatal(err) + } + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != errInWindow { + t.Fatalf("attach on a headed browser = %v, want errInWindow", err) + } +} + +func TestReaperLeavesAWatchedBrowserUntilDetach(t *testing.T) { + s, launch, _ := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + later := time.Date(2026, 8, 16, 13, 0, 0, 0, time.UTC) + s.now = func() time.Time { return later } + s.Reap() + if launch.pages[0].closed { + t.Fatal("reaper closed a browser a pane is showing") + } + if err := s.Detach("s"); err != nil { + t.Fatal(err) + } + if launch.pages[0].casting { + t.Fatal("detach left the screencast running") + } + s.now = func() time.Time { return later.Add(time.Hour) } + s.Reap() + if !launch.pages[0].closed { + t.Fatal("reaper kept an unwatched idle browser") + } +} + +func TestPageNavigatingOnItsOwnUpdatesTheWindow(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + launch.pages[0].onInfo("https://example.com/next", "Next") + states := out.states() + last := states[len(states)-1] + if last.URL != "https://example.com/next" || last.Title != "Next" || !last.Open { + t.Fatalf("last state = %+v", last) + } + if run := s.Running(); len(run) != 1 || run[0].Title != "Next" { + t.Fatalf("running = %+v", run) + } +} + +func TestCloseTellsTheWindow(t *testing.T) { + s, _, out := viewService(t) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + if err := s.CloseOwnedBy("s"); err != nil { + t.Fatal(err) + } + states := out.states() + if last := states[len(states)-1]; last.Open || last.Owner != "s" { + t.Fatalf("last state = %+v, want closed", last) + } + if run := s.Running(); len(run) != 0 { + t.Fatalf("running after close = %+v", run) + } +} + +func TestAgentClickReportsWhereAndWindowInputDoesNot(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + launch.pages[0].clickAt = [2]float64{120, 48} + sel := "#go" + if err := s.Click("s", Target{Selector: sel}); err != nil { + t.Fatal(err) + } + acts := out.named(api.BrowserAgentEvent.Name) + if len(acts) != 1 { + t.Fatalf("agent acts = %v", acts) + } + act := acts[0].(api.BrowserAgentAct) + if act.X == nil || *act.X != 120 || *act.Y != 48 { + t.Fatalf("agent click at %+v, want 120,48", act) + } + + if err := s.Nav("s", "reload", ""); err != nil { + t.Fatal(err) + } + if _, err := s.MouseInput(t.Context(), api.BrowserMouseOptions{ID: "s", Type: "press", X: 1, Y: 2, Button: "left"}); err != nil { + t.Fatal(err) + } + if _, err := s.KeyInput(t.Context(), api.BrowserKeyOptions{ID: "s", Type: "down", Key: "Enter"}); err != nil { + t.Fatal(err) + } + if _, err := s.TextInput(t.Context(), api.BrowserTextOptions{ID: "s", Text: "ção"}); err != nil { + t.Fatal(err) + } + if n := len(out.named(api.BrowserAgentEvent.Name)); n != 1 { + t.Fatalf("window input lit the agent chip: %d agent acts", n) + } + p := launch.pages[0] + if p.reloads != 1 || len(p.mice) != 1 || len(p.keys) != 1 || len(p.texts) != 1 || p.texts[0] != "ção" { + t.Fatalf("page got reloads=%d mice=%v keys=%v texts=%v", p.reloads, p.mice, p.keys, p.texts) + } +} + +func TestNavRejectsBlockedURLAndUnknownAction(t *testing.T) { + s, _, _ := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + if err := s.Nav("s", "go", "file:///etc/passwd"); err == nil { + t.Fatal("nav loaded a file: URL") + } + if err := s.Nav("s", "jump", ""); err == nil { + t.Fatal("nav accepted an unknown action") + } +} diff --git a/internal/routes/browser.go b/internal/routes/browser.go index ad7e4f7af..d30f94a05 100644 --- a/internal/routes/browser.go +++ b/internal/routes/browser.go @@ -24,4 +24,12 @@ func Browser(d *rpc.Handler, s *browser.Service) { rpc.Handle(d, api.BrowserScroll, s.ScrollBy) rpc.Handle(d, api.BrowserList, s.Handles) rpc.Handle(d, api.BrowserClose, s.CloseOne) + rpc.Handle(d, api.BrowserRunning, s.RunningStates) + rpc.Handle(d, api.BrowserAttach, s.AttachView) + rpc.Handle(d, api.BrowserDetach, s.DetachView) + rpc.Handle(d, api.BrowserResize, s.ResizeView) + rpc.Handle(d, api.BrowserMouse, s.MouseInput) + rpc.Handle(d, api.BrowserKey, s.KeyInput) + rpc.Handle(d, api.BrowserText, s.TextInput) + rpc.Handle(d, api.BrowserNav, s.NavView) } diff --git a/main.go b/main.go index e0b9dd761..828d07c76 100644 --- a/main.go +++ b/main.go @@ -207,7 +207,7 @@ func registerServices(db *store.Service, term *terminal.Service, hub *events.Hub configDir, logPath string, env, launchEnv []string, coord *restart.Coordinator) (*system.Service, func()) { // One Chromium per session, closed with the card and with the process. // Never the lich window: this is a second browser the agent drives. - brows := browser.New() + brows := browser.New(hub.Emit) proj := project.New(project.ZenityPicker{}) // gh has one active account per host; a project can name a different one. proj.SetAccounts(db.GHAccountForPath) From 811dd7f1716315bf2c159134a90864bc794f2487 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:22:12 -0300 Subject: [PATCH 04/25] feat(native): show a session's browser in a tab beside it The card's Browser item, and the toast that offers a browser the agent opened, put the session in a tab of its own beside its page: a layout whose second pane is the page, so the divider, rotating and zoom are the layout's. The tab takes no other session, and a session in another layout is asked about before it moves. The page pane draws the backend's frames and passes on the pointer, keys, committed text and pasted text; its toolbar goes back, forward, reloads, loads a typed address, or moves the browser to its own window. A card with a browser shows a globe. --- native/backend/browser.go | 68 +++ native/backend/project.go | 7 - native/cmd/lich-native/browser_keys.go | 103 +++++ native/cmd/lich-native/browser_keys_test.go | 45 ++ native/cmd/lich-native/browser_model.go | 179 +++++++ native/cmd/lich-native/browser_model_test.go | 129 ++++++ native/cmd/lich-native/browser_pane.go | 437 ++++++++++++++++++ native/cmd/lich-native/browser_window.go | 66 +++ native/cmd/lich-native/layouts.go | 26 +- native/cmd/lich-native/layouts_browser.go | 97 ++++ .../cmd/lich-native/layouts_browser_test.go | 91 ++++ native/cmd/lich-native/layouts_window.go | 28 +- native/cmd/lich-native/main.go | 24 +- native/cmd/lich-native/model.go | 25 +- native/cmd/lich-native/model_delegates.go | 4 + native/cmd/lich-native/model_layouts.go | 43 +- native/cmd/lich-native/sidebar_card.go | 3 + native/cmd/lich-native/stage_layout.go | 11 +- native/i18n/locales/en/layouts.json | 15 + native/i18n/locales/es/layouts.json | 15 + native/i18n/locales/ja/layouts.json | 15 + native/i18n/locales/ko/layouts.json | 15 + native/i18n/locales/pt-br/layouts.json | 15 + native/i18n/locales/ru/layouts.json | 15 + native/i18n/locales/zh-cn/layouts.json | 15 + 25 files changed, 1432 insertions(+), 59 deletions(-) create mode 100644 native/backend/browser.go create mode 100644 native/cmd/lich-native/browser_keys.go create mode 100644 native/cmd/lich-native/browser_keys_test.go create mode 100644 native/cmd/lich-native/browser_model.go create mode 100644 native/cmd/lich-native/browser_model_test.go create mode 100644 native/cmd/lich-native/browser_pane.go create mode 100644 native/cmd/lich-native/browser_window.go create mode 100644 native/cmd/lich-native/layouts_browser.go create mode 100644 native/cmd/lich-native/layouts_browser_test.go diff --git a/native/backend/browser.go b/native/backend/browser.go new file mode 100644 index 000000000..9fbbde918 --- /dev/null +++ b/native/backend/browser.go @@ -0,0 +1,68 @@ +package backend + +import ( + "context" + + "github.com/lichdotdev/lich/api" +) + +// ShowBrowser moves session id's browser to its own Chromium window, or +// brings that window forward when it is there already. +func (c *Client) ShowBrowser(ctx context.Context, id string) error { + _, err := callMethod(ctx, c, api.BrowserShow, api.SessionIDOptions{ID: id}) + return err +} + +// CloseBrowser ends session id's browser. +func (c *Client) CloseBrowser(ctx context.Context, id string) error { + _, err := callMethod(ctx, c, api.BrowserClose, api.SessionIDOptions{ID: id}) + return err +} + +// RunningBrowsers is every session browser the backend runs. +func (c *Client) RunningBrowsers(ctx context.Context) ([]api.BrowserState, error) { + return callMethod(ctx, c, api.BrowserRunning, api.None{}) +} + +// AttachBrowser starts the frames of session id's page at the pane's size, +// starting a browser when the session has none. +func (c *Client) AttachBrowser(ctx context.Context, v api.BrowserViewOptions) error { + _, err := callMethod(ctx, c, api.BrowserAttach, v) + return err +} + +// DetachBrowser stops the frames of session id's page. +func (c *Client) DetachBrowser(ctx context.Context, id string) error { + _, err := callMethod(ctx, c, api.BrowserDetach, api.SessionIDOptions{ID: id}) + return err +} + +// ResizeBrowser lays session id's page out at the pane's new size. +func (c *Client) ResizeBrowser(ctx context.Context, v api.BrowserViewOptions) error { + _, err := callMethod(ctx, c, api.BrowserResize, v) + return err +} + +// BrowserMouse passes one pointer event to the page. +func (c *Client) BrowserMouse(ctx context.Context, m api.BrowserMouseOptions) error { + _, err := callMethod(ctx, c, api.BrowserMouse, m) + return err +} + +// BrowserKey passes one key to the page. +func (c *Client) BrowserKey(ctx context.Context, k api.BrowserKeyOptions) error { + _, err := callMethod(ctx, c, api.BrowserKey, k) + return err +} + +// BrowserText types committed text into the page. +func (c *Client) BrowserText(ctx context.Context, id, text string) error { + _, err := callMethod(ctx, c, api.BrowserText, api.BrowserTextOptions{ID: id, Text: text}) + return err +} + +// BrowserNav runs the pane's toolbar: back, forward, reload, or go to url. +func (c *Client) BrowserNav(ctx context.Context, id, action, url string) error { + _, err := callMethod(ctx, c, api.BrowserNav, api.BrowserNavOptions{ID: id, Action: action, URL: url}) + return err +} diff --git a/native/backend/project.go b/native/backend/project.go index 51e6cba6e..20cc863d9 100644 --- a/native/backend/project.go +++ b/native/backend/project.go @@ -35,13 +35,6 @@ func (c *Client) OpenFolderInEditor(ctx context.Context, dir string) (string, er return callMethod(ctx, c, api.SystemOpenFolderInEditor, api.OpenFolderOptions{Dir: dir}) } -// ShowBrowser opens the Chromium window owned by session id, creating it -// when the session has none yet. -func (c *Client) ShowBrowser(ctx context.Context, id string) error { - _, err := callMethod(ctx, c, api.BrowserShow, api.SessionIDOptions{ID: id}) - return err -} - // OpenFolder opens dir in the file manager. func (c *Client) OpenFolder(ctx context.Context, dir string) error { _, err := callMethod(ctx, c, api.SystemOpenFolder, api.OpenFolderOptions{Dir: dir}) diff --git a/native/cmd/lich-native/browser_keys.go b/native/cmd/lich-native/browser_keys.go new file mode 100644 index 000000000..0c7eb695b --- /dev/null +++ b/native/cmd/lich-native/browser_keys.go @@ -0,0 +1,103 @@ +package main + +import ( + "strings" + + "gioui.org/io/key" + "gioui.org/io/pointer" + + "github.com/lichdotdev/lich/api" +) + +// domKey is a key as a page's KeyboardEvent reads it: key, code, and the +// Windows virtual key code Chrome derives the rest from. +type domKey struct { + key, code string + keyCode int +} + +// browserNamedKeys are the keys that type no text, by Gio's name. +var browserNamedKeys = map[key.Name]domKey{ + key.NameReturn: {"Enter", "Enter", 13}, + key.NameEnter: {"Enter", "NumpadEnter", 13}, + key.NameTab: {"Tab", "Tab", 9}, + key.NameDeleteBackward: {"Backspace", "Backspace", 8}, + key.NameDeleteForward: {"Delete", "Delete", 46}, + key.NameEscape: {"Escape", "Escape", 27}, + key.NameLeftArrow: {"ArrowLeft", "ArrowLeft", 37}, + key.NameUpArrow: {"ArrowUp", "ArrowUp", 38}, + key.NameRightArrow: {"ArrowRight", "ArrowRight", 39}, + key.NameDownArrow: {"ArrowDown", "ArrowDown", 40}, + key.NameHome: {"Home", "Home", 36}, + key.NameEnd: {"End", "End", 35}, + key.NamePageUp: {"PageUp", "PageUp", 33}, + key.NamePageDown: {"PageDown", "PageDown", 34}, + key.NameSpace: {" ", "Space", 32}, +} + +// browserKey is the page's key for a Gio key event, false for one whose text +// reaches the page as committed text instead. +func browserKey(id string, e key.Event) (api.BrowserKeyOptions, bool) { + k, ok := browserNamedKeys[e.Name] + if !ok { + k, ok = characterDomKey(e.Name, e.Modifiers) + } + if !ok { + return api.BrowserKeyOptions{}, false + } + kind := "down" + if e.State == key.Release { + kind = "up" + } + return api.BrowserKeyOptions{ID: id, Type: kind, Key: k.key, Code: k.code, KeyCode: k.keyCode, Modifiers: cdpModifiers(e.Modifiers)}, true +} + +// characterDomKey is a letter or digit typed with Ctrl, Alt or Command, +// which carries no EditEvent: the shortcuts a page handles itself. +func characterDomKey(name key.Name, mods key.Modifiers) (domKey, bool) { + if mods&(key.ModCtrl|key.ModAlt|key.ModCommand) == 0 || len(name) != 1 { + return domKey{}, false + } + c := name[0] + switch { + case 'A' <= c && c <= 'Z': + letter := strings.ToLower(string(c)) + if mods.Contain(key.ModShift) { + letter = string(c) + } + return domKey{letter, "Key" + string(c), int(c)}, true + case '0' <= c && c <= '9': + return domKey{string(c), "Digit" + string(c), int(c)}, true + } + return domKey{}, false +} + +// cdpModifiers is CDP's modifier bit set: Alt 1, Ctrl 2, Meta 4, Shift 8. +func cdpModifiers(m key.Modifiers) int { + out := 0 + for _, bit := range []struct { + gio key.Modifiers + cdp int + }{{key.ModAlt, 1}, {key.ModCtrl, 2}, {key.ModSuper, 4}, {key.ModShift, 8}} { + if m.Contain(bit.gio) { + out |= bit.cdp + } + } + if key.ModShortcut == key.ModCommand && m.Contain(key.ModCommand) { + out |= 4 + } + return out +} + +// browserButton is CDP's name for a Gio pointer button, "" for none. +func browserButton(b pointer.Buttons) string { + switch { + case b.Contain(pointer.ButtonPrimary): + return "left" + case b.Contain(pointer.ButtonSecondary): + return "right" + case b.Contain(pointer.ButtonTertiary): + return "middle" + } + return "" +} diff --git a/native/cmd/lich-native/browser_keys_test.go b/native/cmd/lich-native/browser_keys_test.go new file mode 100644 index 000000000..fc66e58a6 --- /dev/null +++ b/native/cmd/lich-native/browser_keys_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "testing" + + "gioui.org/io/key" + "gioui.org/io/pointer" +) + +func TestBrowserKeyNamesWhatTypesNoText(t *testing.T) { + cases := []struct { + e key.Event + key string + code string + kc int + mods int + kind string + }{ + {key.Event{Name: key.NameReturn, State: key.Press}, "Enter", "Enter", 13, 0, "down"}, + {key.Event{Name: key.NameDeleteBackward, State: key.Release}, "Backspace", "Backspace", 8, 0, "up"}, + {key.Event{Name: key.NameLeftArrow, Modifiers: key.ModShift, State: key.Press}, "ArrowLeft", "ArrowLeft", 37, 8, "down"}, + {key.Event{Name: "A", Modifiers: key.ModCtrl, State: key.Press}, "a", "KeyA", 65, 2, "down"}, + {key.Event{Name: "Z", Modifiers: key.ModCtrl | key.ModShift, State: key.Press}, "Z", "KeyZ", 90, 10, "down"}, + {key.Event{Name: "1", Modifiers: key.ModAlt, State: key.Press}, "1", "Digit1", 49, 1, "down"}, + } + for _, c := range cases { + got, ok := browserKey("s", c.e) + if !ok || got.Key != c.key || got.Code != c.code || got.KeyCode != c.kc || got.Modifiers != c.mods || got.Type != c.kind || got.ID != "s" { + t.Errorf("%v: got %+v, %v", c.e, got, ok) + } + } +} + +func TestBrowserKeyLeavesTypedTextToTheEditEvent(t *testing.T) { + if got, ok := browserKey("s", key.Event{Name: "A", State: key.Press}); ok { + t.Fatalf("a plain letter became a key event: %+v", got) + } +} + +func TestBrowserButton(t *testing.T) { + if browserButton(pointer.ButtonPrimary) != "left" || browserButton(pointer.ButtonSecondary) != "right" || + browserButton(pointer.ButtonTertiary) != "middle" || browserButton(0) != "" { + t.Fatal("browserButton") + } +} diff --git a/native/cmd/lich-native/browser_model.go b/native/cmd/lich-native/browser_model.go new file mode 100644 index 000000000..5de721a94 --- /dev/null +++ b/native/cmd/lich-native/browser_model.go @@ -0,0 +1,179 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "image" + "image/jpeg" + "time" + + "gioui.org/op/paint" + + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/backend" +) + +// The model's store of session browsers: which sessions have one and what it +// shows, fed by browser.Running once and browser-state after; the page's last +// frame; and when the agent last acted on it. + +// browserFrame is a page's last screencast frame, ready to draw. +type browserFrame struct { + img paint.ImageOp + size image.Point +} + +// browserMark is the agent's last action on its page, and where it clicked. +type browserMark struct { + at time.Time + x, y *float64 +} + +type browserStore struct { + states map[string]api.BrowserState + frames map[string]browserFrame + agent map[string]browserMark +} + +func newBrowserStore() browserStore { + return browserStore{states: map[string]api.BrowserState{}, frames: map[string]browserFrame{}, agent: map[string]browserMark{}} +} + +// hydrateBrowsers reads the browsers running now, and takes out of the +// layouts every page whose browser did not survive (a backend restarted, or +// a browser moved to its own window while this one was away). +func (m *model) hydrateBrowsers(ctx context.Context) error { + running, err := m.client.RunningBrowsers(ctx) + if err != nil { + return err + } + m.mu.Lock() + for _, st := range running { + m.browsers.states[st.Owner] = st + } + pruned := false + for _, t := range append([]layoutTab(nil), m.layouts.tabs...) { + if owner, ok := t.browserOf(); ok && !m.browserShowableLocked(owner) { + pruned = m.layouts.dropBrowser(owner) || pruned + } + } + m.mu.Unlock() + if pruned { + m.saveLayouts() + } + m.invalidate() + return nil +} + +// browserShowableLocked reports whether session's browser can be drawn in a +// pane: it runs, and not in its own window. Callers hold m.mu. +func (m *model) browserShowableLocked(session string) bool { + st, ok := m.browsers.states[session] + return ok && st.Open && !st.Headed +} + +// browserState is session's browser, false for none. +func (m *model) browserState(session string) (api.BrowserState, bool) { + m.mu.Lock() + defer m.mu.Unlock() + st, ok := m.browsers.states[session] + return st, ok +} + +// applyBrowserState takes a browser-state event: a page that can no longer +// be drawn leaves its tab, and a browser the agent just started is offered +// to the person (browserOpened). +func (m *model) applyBrowserState(ctx context.Context, st api.BrowserState) { + m.mu.Lock() + _, known := m.browsers.states[st.Owner] + if st.Open { + m.browsers.states[st.Owner] = st + } else { + delete(m.browsers.states, st.Owner) + delete(m.browsers.frames, st.Owner) + delete(m.browsers.agent, st.Owner) + } + shownTab := "" + if i := m.layouts.browserTab(st.Owner); i >= 0 && m.shownLayout == m.layouts.tabs[i].ID { + shownTab = m.shownLayout + } + dropped := !m.browserShowableLocked(st.Owner) && m.layouts.dropBrowser(st.Owner) + m.mu.Unlock() + if dropped { + m.saveLayouts() + if shownTab != "" { + m.setActive(ctx, st.Owner) + } + } + if st.Open && st.ByAgent && !known && m.browserOpened != nil { + m.browserOpened(st) + } +} + +// applyBrowserFrame decodes one frame of owner's page off the frame +// goroutine, so the pane only draws it. +func (m *model) applyBrowserFrame(owner string, ev backend.Event) error { + data, err := backend.Decode(ev, api.BrowserFrameEvents.Of(owner)) + if err != nil { + return err + } + raw, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return fmt.Errorf("%s: %w", ev.Name, err) + } + img, err := jpeg.Decode(bytes.NewReader(raw)) + if err != nil { + return fmt.Errorf("%s: %w", ev.Name, err) + } + frame := browserFrame{img: paint.NewImageOp(img), size: img.Bounds().Size()} + m.mu.Lock() + m.browsers.frames[owner] = frame + m.mu.Unlock() + return nil +} + +func (m *model) applyBrowserAgent(a api.BrowserAgentAct, now time.Time) { + m.mu.Lock() + defer m.mu.Unlock() + m.browsers.agent[a.Owner] = browserMark{at: now, x: a.X, y: a.Y} +} + +// browserView is what session's page pane draws: its state, its last frame +// and the agent's last action. +func (m *model) browserView(session string) (api.BrowserState, browserFrame, browserMark) { + m.mu.Lock() + defer m.mu.Unlock() + return m.browsers.states[session], m.browsers.frames[session], m.browsers.agent[session] +} + +// openBrowserTab puts session's browser tab on screen, making it when there +// is none. A session in another layout stays there unless move is set; it +// reports false when nothing opened. +func (m *model) openBrowserTab(ctx context.Context, session string, move bool) bool { + m.mu.Lock() + if !m.liveLocked(session) { + m.mu.Unlock() + return false + } + if i := m.layouts.browserTab(session); i >= 0 { + m.mu.Unlock() + m.focusPane(ctx, browserPaneOf(session)) + return true + } + if m.layouts.of(session) >= 0 { + if !move { + m.mu.Unlock() + return false + } + m.layouts.take(session) + } + p, i := m.sessionAt(session) + m.layouts.openBrowser(p.Sessions[i].Label, session) + m.mu.Unlock() + m.setActive(ctx, session) + m.focusPane(ctx, browserPaneOf(session)) + m.saveLayouts() + return true +} diff --git a/native/cmd/lich-native/browser_model_test.go b/native/cmd/lich-native/browser_model_test.go new file mode 100644 index 000000000..036cd1ca9 --- /dev/null +++ b/native/cmd/lich-native/browser_model_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "image" + "image/jpeg" + "testing" + + "github.com/lichdotdev/lich/api" +) + +func TestOpenBrowserTabShowsTheSessionBesideItsPage(t *testing.T) { + m, _ := newLayoutsModel(t) + if !m.openBrowserTab(context.Background(), "orch", false) { + t.Fatal("refused a session in no layout") + } + id, project, active := layoutScreen(t, m) + m.mu.Lock() + defer m.mu.Unlock() + tab := m.layouts.byID(id) + if tab == nil || paneShape(tab.Root) != "row(orch,browser:orch)" || tab.Name != "orchestrator" { + t.Fatalf("shown tab %+v", tab) + } + if project != "lich" || active != "orch" || tab.Focus != "browser:orch" { + t.Fatalf("project %q active %q focus %q", project, active, tab.Focus) + } +} + +func TestOpenBrowserTabAsksBeforeTakingTheSessionFromALayout(t *testing.T) { + m, _ := newLayoutsModel(t) + ctx := context.Background() + m.splitBeside(ctx, "orch", "wa", edgeRight, false) + if m.openBrowserTab(ctx, "wa", false) { + t.Fatal("took wa out of its layout without move") + } + if !m.openBrowserTab(ctx, "wa", true) { + t.Fatal("refused the move it was told to make") + } + m.mu.Lock() + defer m.mu.Unlock() + if len(m.layouts.tabs) != 1 || paneShape(m.layouts.tabs[0].Root) != "row(wa,browser:wa)" { + t.Fatalf("got %+v", m.layouts.tabs) + } +} + +func TestBrowserTabTakesNoOneElse(t *testing.T) { + m, _ := newLayoutsModel(t) + ctx := context.Background() + m.openBrowserTab(ctx, "orch", false) + if m.splitBeside(ctx, "orch", "notes", edgeBottom, false) { + t.Fatal("split another session into a browser tab") + } + if m.showDelegates(ctx, "orch", []string{"wa"}) { + t.Fatal("laid delegates out in a browser tab") + } +} + +func TestClosedBrowserLeavesItsTabAndShowsTheSessionAlone(t *testing.T) { + m, _ := newLayoutsModel(t) + ctx := context.Background() + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "orch", Open: true}) + m.openBrowserTab(ctx, "orch", false) + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "orch", Open: false}) + if id, project, active := layoutScreen(t, m); id != "" || project != "lich" || active != "orch" { + t.Fatalf("layout %q project %q active %q, want the session alone", id, project, active) + } + if _, ok := m.browserState("orch"); ok { + t.Fatal("kept a closed browser") + } +} + +func TestBrowserMovedToAWindowLeavesItsTab(t *testing.T) { + m, _ := newLayoutsModel(t) + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "orch", Open: true}) + m.openBrowserTab(context.Background(), "orch", false) + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "orch", Open: true, Headed: true}) + m.mu.Lock() + defer m.mu.Unlock() + if len(m.layouts.tabs) != 0 { + t.Fatalf("tabs %+v, want the page gone with its browser in a window", m.layouts.tabs) + } +} + +func TestOnlyANewAgentBrowserIsOffered(t *testing.T) { + m, _ := newLayoutsModel(t) + var offered []string + m.browserOpened = func(st api.BrowserState) { offered = append(offered, st.Owner) } + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "orch", Open: true}) + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "wa", Open: true, ByAgent: true}) + send(m, api.BrowserStateEvent.Name, api.BrowserState{Owner: "wa", Open: true, ByAgent: true, URL: "https://example.com"}) + if len(offered) != 1 || offered[0] != "wa" { + t.Fatalf("offered %v, want the agent's new browser once", offered) + } +} + +func TestBrowserFrameIsDecodedForThePane(t *testing.T) { + m, _ := newLayoutsModel(t) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 8, 6)), nil); err != nil { + t.Fatal(err) + } + send(m, api.BrowserFrameEvents.Of("orch").Name, base64.StdEncoding.EncodeToString(buf.Bytes())) + if _, frame, _ := m.browserView("orch"); frame.size != image.Pt(8, 6) { + t.Fatalf("frame size %v", frame.size) + } +} + +func TestTypedURL(t *testing.T) { + cases := map[string]string{ + "": "about:blank", + "localhost:3000/a": "http://localhost:3000/a", + "app.localhost": "http://app.localhost", + "127.0.0.1:8080": "http://127.0.0.1:8080", + "example.com": "https://example.com", + "http://example.com/x": "http://example.com/x", + "https://example.com": "https://example.com", + "example.com:8443/login": "https://example.com:8443/login", + } + for in, want := range cases { + if got := typedURL(in); got != want { + t.Errorf("typedURL(%q) = %q, want %q", in, got, want) + } + } + if displayURL("about:blank") != "" || displayURL("https://x.dev") != "https://x.dev" { + t.Fatal("displayURL") + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go new file mode 100644 index 000000000..c87508dae --- /dev/null +++ b/native/cmd/lich-native/browser_pane.go @@ -0,0 +1,437 @@ +package main + +import ( + "context" + "image" + "log" + "runtime" + "strings" + "time" + + "gioui.org/f32" + "gioui.org/font" + "gioui.org/io/clipboard" + "gioui.org/io/event" + "gioui.org/io/key" + "gioui.org/io/pointer" + "gioui.org/io/transfer" + "gioui.org/layout" + "gioui.org/op" + "gioui.org/op/clip" + "gioui.org/op/paint" + "gioui.org/widget" + + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/gioterm" + "github.com/lichdotdev/lich/native/i18n" + "github.com/lichdotdev/lich/native/ui" + "github.com/lichdotdev/lich/native/ui/icons" +) + +// A page pane: the browser of a session drawn in its tab beside it, from the +// frames the backend streams, with the person's pointer and keys passed on. + +var ( + icBrowserBack = icons.Lucide("arrow-left") + icBrowserForward = icons.Lucide("arrow-right") + icBrowserReload = icons.Lucide("rotate-cw") + icBrowserWindow = icons.Lucide("external-link") + icBrowserAgent = icons.Lucide("bot") +) + +// agentGlow is how long the agent chip and its click mark stay up after the +// agent's last action on the page. +const agentGlow = 2 * time.Second + +// doubleClick is the window two presses of a button count as one double +// click in, as most desktops set it. +const doubleClick = 400 * time.Millisecond + +// browserPaneUI is one page pane's widgets and what it last told the +// backend, kept across frames by session. +type browserPaneUI struct { + back, forward, reload, window iconActionState + zoom, close widget.Clickable + zoomTip, closeTip ui.Tooltip + url widget.Editor + // shownURL is the page's URL the field last showed: a navigation + // replaces the field's text unless the person is editing it. + shownURL string + tag int + composition gioterm.Composition + // view is the size the backend lays the page out at; zero until the + // pane is attached. + view api.BrowserViewOptions + // held is the button the pointer holds down, for a drag's moves. + held string + lastPress time.Time + presses int + input chan func() + onScreen, wasOnScreen bool +} + +func (u *window) browserPane(session string) *browserPaneUI { + if u.browserPanes == nil { + u.browserPanes = map[string]*browserPaneUI{} + } + p, ok := u.browserPanes[session] + if !ok { + p = &browserPaneUI{input: make(chan func(), 256)} + // One goroutine sends a pane's input, so a release never overtakes + // its press. + go func() { + for send := range p.input { + send() + } + }() + u.browserPanes[session] = p + } + return p +} + +// sendBrowser queues one call to the backend for session's page, logging +// what fails: the page answers on screen, not here. +func (u *window) sendBrowser(session, what string, call func(context.Context) error) { + p := u.browserPane(session) + send := func() { + if err := call(u.ctx); err != nil { + log.Printf("browser %s of %s: %v", what, session, err) + } + } + select { + case p.input <- send: + default: + log.Printf("browser %s of %s: input queue full, dropped", what, session) + } +} + +// browserPaneAt draws session's page pane in r, the way paneAt draws a +// session's: a header, the toolbar, and the page. +func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Rectangle, several bool) { + pane := browserPaneOf(session) + p := u.browserPane(session) + p.onScreen = true + st, frame, mark := u.model.browserView(session) + defer op.Offset(r.Min).Push(gtx.Ops).Pop() + gtx.Constraints = layout.Exact(r.Size()) + size := r.Size() + focused := pane == v.layout.Focus + th := u.theme + shape := clip.UniformRRect(image.Rectangle{Max: size}, gtx.Dp(paneRadius)) + paint.FillShape(gtx.Ops, th.Background, shape.Op(gtx.Ops)) + area := shape.Push(gtx.Ops) + head, bar := gtx.Dp(paneHeader), gtx.Dp(ui.Space(9)) + header := gtx + header.Constraints = layout.Exact(image.Pt(size.X, head)) + u.browserHeader(header, p, pane, focused, gtx.Now.Sub(mark.at) < agentGlow) + line := th.Over(th.Border) + paint.FillShape(gtx.Ops, line, clip.Rect{Min: image.Pt(0, head-gtx.Dp(1)), Max: image.Pt(size.X, head)}.Op()) + tool := op.Offset(image.Pt(0, head)).Push(gtx.Ops) + toolbar := gtx + toolbar.Constraints = layout.Exact(image.Pt(size.X, bar)) + u.browserToolbar(toolbar, p, session, st) + tool.Pop() + paint.FillShape(gtx.Ops, line, clip.Rect{Min: image.Pt(0, head+bar-gtx.Dp(1)), Max: image.Pt(size.X, head+bar)}.Op()) + page := gtx + page.Constraints = layout.Exact(image.Pt(size.X, max(size.Y-head-bar, 0))) + off := op.Offset(image.Pt(0, head+bar)).Push(gtx.Ops) + u.browserPage(page, p, session, frame, mark, focused) + if several && !focused { + paint.FillShape(gtx.Ops, ui.Alpha(th.Background, paneDim), clip.Rect{Max: page.Constraints.Max}.Op()) + } + off.Pop() + u.paneFocusTarget(gtx, pane) + area.Pop() + u.lockedDropRefusal(gtx, session, image.Rectangle{Max: size}, r.Add(u.islandAt)) + border, width := line, float32(gtx.Dp(1)) + if several && focused { + border, width = th.Ring, float32(gtx.Dp(1.5)) + } + paint.FillShape(gtx.Ops, border, clip.Stroke{Path: shape.Path(gtx.Ops), Width: width}.Op()) + if gtx.Now.Sub(mark.at) < agentGlow { + gtx.Execute(op.InvalidateCmd{At: mark.at.Add(agentGlow)}) + } +} + +// browserHeader is the page pane's top row: its name, the agent chip while +// the agent acts on the page, and zoom and close. +func (u *window) browserHeader(gtx C, p *browserPaneUI, pane string, focused, agent bool) { + th := u.theme + if p.zoom.Clicked(gtx) { + u.layoutStage.focus, u.layoutStage.zoom = pane, true + } + if p.close.Clicked(gtx) { + session := paneSession(pane) + u.sendBrowser(session, "close", func(ctx context.Context) error { return u.client.CloseBrowser(ctx, session) }) + } + fg, weight := th.MutedForeground, font.Normal + if focused { + fg, weight = th.Foreground, font.Medium + } + layout.Inset{Left: ui.Space(2.5), Right: ui.Space(2)}.Layout(gtx, func(gtx C) D { + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(func(gtx C) D { return icBrowser.Layout(gtx, ui.Space(3.5), th.MutedForeground) }), + ui.Gap(2), + layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.title")).In(fg).Weight(weight).Layout), + ui.Gap(2), + layout.Rigid(func(gtx C) D { + if !agent { + return D{} + } + return pill(gtx, th, func(gtx C) D { + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(func(gtx C) D { return icBrowserAgent.Layout(gtx, ui.Space(3), th.Foreground) }), + ui.Gap(1), + layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.agent")).Layout), + ) + }, ui.Space(2), ui.Space(2)) + }), + layout.Flexed(1, func(gtx C) D { return D{Size: image.Pt(gtx.Constraints.Max.X, 0)} }), + layout.Rigid(func(gtx C) D { + zoom := th.Tooltip(&u.root, &p.zoomTip, i18n.T("layouts.pane.zoom")) + return zoom.Layout(gtx, func(gtx C) D { return cornerButton(gtx, th, &p.zoom, icPaneZoom, false) }) + }), + ui.Gap(1.5), + layout.Rigid(func(gtx C) D { + closer := th.Tooltip(&u.root, &p.closeTip, i18n.T("layouts.browser.close")) + return closer.Layout(gtx, func(gtx C) D { return cornerButton(gtx, th, &p.close, icPaneUnsplit, false) }) + }), + ) + }) +} + +// browserToolbar is back, forward, reload, the address and Open in window. +func (u *window) browserToolbar(gtx C, p *browserPaneUI, session string, st api.BrowserState) { + th := u.theme + nav := func(action string) { + u.sendBrowser(session, action, func(ctx context.Context) error { return u.client.BrowserNav(ctx, session, action, "") }) + } + if p.back.click.Clicked(gtx) { + nav("back") + } + if p.forward.click.Clicked(gtx) { + nav("forward") + } + if p.reload.click.Clicked(gtx) { + nav("reload") + } + if p.window.click.Clicked(gtx) { + u.sendBrowser(session, "show", func(ctx context.Context) error { return u.client.ShowBrowser(ctx, session) }) + } + for { + ev, ok := p.url.Update(gtx) + if !ok { + break + } + if submit, ok := ev.(widget.SubmitEvent); ok { + target := typedURL(submit.Text) + u.sendBrowser(session, "go", func(ctx context.Context) error { return u.client.BrowserNav(ctx, session, "go", target) }) + gtx.Execute(key.FocusCmd{Tag: &p.tag}) + } + } + if st.URL != p.shownURL && !gtx.Focused(&p.url) { + p.shownURL = st.URL + p.url.SetText(displayURL(st.URL)) + } + p.url.Submit = true + field := th.Input(&p.url, i18n.T("layouts.browser.address")) + field.Height, field.TextSize, field.Mono = ui.Space(6), ui.TextXS, true + layout.Inset{Left: ui.Space(1.5), Right: ui.Space(1.5)}.Layout(gtx, func(gtx C) D { + gtx.Constraints.Min.Y = gtx.Constraints.Max.Y + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(func(gtx C) D { + return iconAction(gtx, th, &u.root, &p.back, icBrowserBack, i18n.T("layouts.browser.back"), false) + }), + layout.Rigid(func(gtx C) D { + return iconAction(gtx, th, &u.root, &p.forward, icBrowserForward, i18n.T("layouts.browser.forward"), false) + }), + layout.Rigid(func(gtx C) D { + return iconAction(gtx, th, &u.root, &p.reload, icBrowserReload, i18n.T("layouts.browser.reload"), false) + }), + ui.Gap(1.5), + layout.Flexed(1, field.Layout), + ui.Gap(1.5), + layout.Rigid(func(gtx C) D { + return iconAction(gtx, th, &u.root, &p.window, icBrowserWindow, i18n.T("layouts.browser.openInWindow"), false) + }), + ) + }) +} + +// browserPage draws the page's last frame over gtx's area, keeps the +// backend's page the area's size, and passes the pointer and keys on. +func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame browserFrame, mark browserMark, focused bool) { + size := gtx.Constraints.Max + scale := gtx.Metric.PxPerDp + want := api.BrowserViewOptions{ID: session, W: int(float32(size.X) / scale), H: int(float32(size.Y) / scale), Scale: float64(scale)} + if want.W > 0 && want.H > 0 && want != p.view { + attach := p.view.ID == "" + p.view = want + u.sendBrowser(session, "attach", func(ctx context.Context) error { + if attach { + return u.client.AttachBrowser(ctx, want) + } + return u.client.ResizeBrowser(ctx, want) + }) + } + paint.FillShape(gtx.Ops, u.theme.Background, clip.Rect{Max: size}.Op()) + if frame.size.X > 0 { + // A frame is the page at the size it was last laid out at, which + // lags a resize by a frame or two: it is drawn scaled to the area. + sx, sy := float32(size.X)/float32(frame.size.X), float32(size.Y)/float32(frame.size.Y) + s := min(sx, sy) + t := op.Affine(f32.Affine2D{}.Scale(f32.Point{}, f32.Pt(s, s))).Push(gtx.Ops) + frame.img.Add(gtx.Ops) + paint.PaintOp{}.Add(gtx.Ops) + t.Pop() + } + if gtx.Now.Sub(mark.at) < agentGlow && mark.x != nil && mark.y != nil { + at := image.Pt(int(float32(*mark.x)*scale), int(float32(*mark.y)*scale)) + r := gtx.Dp(ui.Space(2)) + dot := clip.Ellipse{Min: at.Sub(image.Pt(r, r)), Max: at.Add(image.Pt(r, r))} + paint.FillShape(gtx.Ops, ui.Alpha(u.theme.Ring, 0.35), dot.Op(gtx.Ops)) + paint.FillShape(gtx.Ops, u.theme.Ring, clip.Stroke{Path: dot.Path(gtx.Ops), Width: float32(gtx.Dp(1.5))}.Op()) + } + u.browserInput(gtx, p, session, scale, focused) + defer clip.Rect{Max: size}.Push(gtx.Ops).Pop() + event.Op(gtx.Ops, &p.tag) +} + +// browserInput passes the pane's pointer and keys to the page. +func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale float32, focused bool) { + filters := append(gioterm.KeyFilters(&p.tag), + pointer.Filter{Target: &p.tag, Kinds: pointer.Scroll | pointer.Press | pointer.Release | pointer.Drag | pointer.Move, + ScrollX: pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20}, ScrollY: pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20}}, + key.Filter{Focus: &p.tag, Name: "V", Required: key.ModCommand}, + transfer.TargetFilter{Target: &p.tag, Type: "application/text"}) + if focused && !gtx.Focused(&p.url) && !gtx.Focused(&p.tag) { + gtx.Execute(key.FocusCmd{Tag: &p.tag}) + } + for { + ev, ok := gtx.Event(filters...) + if !ok { + return + } + switch e := ev.(type) { + case key.EditEvent, key.CompositionEvent, key.SelectionEvent, key.FocusEvent: + if text := p.composition.Update(e, &p.tag, gtx.Execute); text != "" { + u.sendBrowser(session, "text", func(ctx context.Context) error { return u.client.BrowserText(ctx, session, text) }) + } + case key.Event: + if e.State == key.Press && isPasteChord(e, runtime.GOOS) { + gtx.Execute(clipboard.ReadCmd{Tag: &p.tag}) + break + } + if k, ok := browserKey(session, e); ok { + u.sendBrowser(session, "key", func(ctx context.Context) error { return u.client.BrowserKey(ctx, k) }) + } + case transfer.DataEvent: + text, err := readTransfer(e) + if err != nil { + log.Printf("paste into the browser of %s: %v", session, err) + break + } + u.sendBrowser(session, "paste", func(ctx context.Context) error { return u.client.BrowserText(ctx, session, text) }) + case pointer.Event: + u.browserPointer(gtx, p, session, scale, e) + } + } +} + +func (u *window) browserPointer(gtx C, p *browserPaneUI, session string, scale float32, e pointer.Event) { + m := api.BrowserMouseOptions{ID: session, X: float64(e.Position.X / scale), Y: float64(e.Position.Y / scale), Modifiers: cdpModifiers(e.Modifiers)} + switch e.Kind { + case pointer.Press: + gtx.Execute(key.FocusCmd{Tag: &p.tag}) + u.layoutStage.focus = browserPaneOf(session) + if gtx.Now.Sub(p.lastPress) < doubleClick { + p.presses++ + } else { + p.presses = 1 + } + p.lastPress = gtx.Now + p.held = browserButton(e.Buttons) + m.Type, m.Button, m.Clicks = "press", p.held, p.presses + case pointer.Release: + m.Type, m.Button, m.Clicks = "release", p.held, p.presses + p.held = "" + case pointer.Move, pointer.Drag: + m.Type, m.Button = "move", p.held + case pointer.Scroll: + m.Type, m.DX, m.DY = "wheel", float64(e.Scroll.X/scale), float64(e.Scroll.Y/scale) + default: + return + } + u.sendBrowser(session, m.Type, func(ctx context.Context) error { return u.client.BrowserMouse(ctx, m) }) +} + +// settleBrowserPanes stops the frames of every page pane that was on screen +// last frame and is not on this one, after the frame laid everything out. +func (u *window) settleBrowserPanes() { + for session, p := range u.browserPanes { + gone := p.wasOnScreen && !p.onScreen + p.wasOnScreen, p.onScreen = p.onScreen, false + if !gone || p.view.ID == "" { + continue + } + p.view = api.BrowserViewOptions{} + u.sendBrowser(session, "detach", func(ctx context.Context) error { return u.client.DetachBrowser(ctx, session) }) + } +} + +// lockedDropRefusal tells a drag carried over a browser tab's pane that the +// tab takes no one else. +func (u *window) lockedDropRefusal(gtx C, session string, local, at image.Rectangle) { + carried := u.sidebar.carry.Carrying() + if carried == "" || carried == session || !u.layoutStage.pointer.at.In(at) { + return + } + u.dropHighlight(gtx, local, i18n.T("layouts.browser.locked", i18n.Args{"label": u.model.sessionLabel(session)})) +} + +// displayURL is the page's URL as the address field shows it: about:blank +// as an empty field. +func displayURL(raw string) string { + if raw == "about:blank" { + return "" + } + return raw +} + +// typedURL is what the address field's text loads: a URL as typed, and a +// bare host (localhost:3000, example.com) over http for a local one and https +// for any other. +func typedURL(text string) string { + switch { + case text == "": + return "about:blank" + case hasScheme(text): + return text + case isLocalHost(text): + return "http://" + text + } + return "https://" + text +} + +func hasScheme(text string) bool { + for i, c := range text { + switch { + case c == ':': + return i > 0 && len(text) > i+2 && text[i+1:i+3] == "//" + case !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || i > 0 && ('0' <= c && c <= '9' || c == '+' || c == '-' || c == '.')): + return false + } + } + return false +} + +func isLocalHost(text string) bool { + host := text + for i, c := range text { + if c == ':' || c == '/' { + host = text[:i] + break + } + } + return host == "localhost" || host == "127.0.0.1" || host == "[::1]" || strings.HasSuffix(host, ".localhost") +} diff --git a/native/cmd/lich-native/browser_window.go b/native/cmd/lich-native/browser_window.go new file mode 100644 index 000000000..f16ed35eb --- /dev/null +++ b/native/cmd/lich-native/browser_window.go @@ -0,0 +1,66 @@ +package main + +import ( + "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/backend" + "github.com/lichdotdev/lich/native/i18n" + "github.com/lichdotdev/lich/native/ui" +) + +// The window's side of a session browser: the card's Browser item, and the +// toast that offers a browser the agent opened. + +// applyBrowser opens the browser of the card the menu was opened on. +func (u *window) applyBrowser(act sidebarAction) { + if act.browser != "" { + u.openBrowser(act.browser) + } +} + +// openBrowser puts session's browser on screen: in its tab, asked about +// first when the session sits in another layout, or, for a browser moved to +// its own window, by bringing that window forward. +func (u *window) openBrowser(session string) { + if st, ok := u.model.browserState(session); ok && st.Open && st.Headed { + go func() { + if err := u.client.ShowBrowser(u.ctx, session); err != nil { + u.toaster.Show(ui.Toast{Variant: ui.ToastError, Title: i18n.T("sidebar.sessionCard.browserFailed", i18n.Args{"error": backend.ErrorText(err)})}) + } + }() + return + } + if from := u.model.layoutOf(session); from != "" && u.model.browserTabOf(session) == "" { + u.askMoveToBrowser(session, from) + return + } + u.model.openBrowserTab(u.ctx, session, false) + u.model.invalidate() +} + +// offerBrowser tells the person the agent of a session opened a browser, +// with Open to put it on screen. +func (u *window) offerBrowser(st api.BrowserState) { + label := u.model.sessionLabel(st.Owner) + if label == "" { + return + } + session := st.Owner + u.duplicates.Show(ui.Toast{ + ID: "browser:" + session, + Title: i18n.T("layouts.browser.opened", i18n.Args{"label": label}), + Description: displayURL(st.URL), + Icon: icBrowser, + Cancel: &ui.ToastButton{Label: i18n.T("layouts.browser.dismiss")}, + Action: &ui.ToastButton{Label: i18n.T("layouts.browser.open"), OnClick: func() { u.openBrowser(session) }}, + }) +} + +// browserTabOf is the id of session's browser tab, "" for none. +func (m *model) browserTabOf(session string) string { + m.mu.Lock() + defer m.mu.Unlock() + if i := m.layouts.browserTab(session); i >= 0 { + return m.layouts.tabs[i].ID + } + return "" +} diff --git a/native/cmd/lich-native/layouts.go b/native/cmd/lich-native/layouts.go index 5d62fcde7..3afb487ac 100644 --- a/native/cmd/lich-native/layouts.go +++ b/native/cmd/lich-native/layouts.go @@ -64,20 +64,12 @@ type layoutTab struct { } // sessions are the tab's sessions in reading order, left to right and top to -// bottom. +// bottom; a browser tab's page is a pane, not a session (panes). func (t *layoutTab) sessions() []string { - var out []string - var walk func(n *paneNode) - walk = func(n *paneNode) { - if n.leaf() { - out = append(out, n.Session) - return - } - walk(n.First) - walk(n.Second) - } - walk(t.Root) - return out + return slices.DeleteFunc(t.panes(), func(pane string) bool { + _, page := browserOwner(pane) + return page + }) } func (t *layoutTab) holds(id string) bool { return slices.Contains(t.sessions(), id) } @@ -290,7 +282,8 @@ func (ls *layoutTabs) openBeside(name, left string, column []string) string { // take removes session from whichever layout holds it, reporting whether // one did. A layout left with a single pane is no layout any more: it goes, -// and left is the session that pane showed, "" while the layout stays. +// and left is the session that pane showed, "" while the layout stays or +// when what is left is a page, whose session just went. func (ls *layoutTabs) take(session string) (left string, ok bool) { i := ls.of(session) if i < 0 { @@ -300,6 +293,9 @@ func (ls *layoutTabs) take(session string) (left string, ok bool) { t.remove(session) if t.Root.leaf() { left = t.Root.Session + if _, page := browserOwner(left); page { + left = "" + } ls.tabs = slices.Delete(ls.tabs, i, i+1) } return left, true @@ -356,7 +352,7 @@ func parseLayouts(raw string) (layoutTabs, error) { } seen[id] = true } - if t.Root != nil && !t.holds(t.Focus) { + if t.Root != nil && !slices.Contains(t.panes(), t.Focus) { t.Focus = t.sessions()[0] } } diff --git a/native/cmd/lich-native/layouts_browser.go b/native/cmd/lich-native/layouts_browser.go new file mode 100644 index 000000000..59a0f52c4 --- /dev/null +++ b/native/cmd/lich-native/layouts_browser.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "slices" + "strings" +) + +// A browser tab is a layout of two panes, a session and the page its browser +// shows, and nothing else joins it. The page's pane is a leaf like any other, +// named after its session with browserPanePrefix, so rotating, dragging the +// divider, zooming and storing the tab are the layout's own. + +const browserPanePrefix = "browser:" + +func browserPaneOf(session string) string { return browserPanePrefix + session } + +// browserOwner is the session whose page pane is, false for a session's pane. +func browserOwner(pane string) (string, bool) { + return strings.CutPrefix(pane, browserPanePrefix) +} + +// paneSession is the session a pane belongs to: itself, or a page's owner. +func paneSession(pane string) string { + if owner, ok := browserOwner(pane); ok { + return owner + } + return pane +} + +// panes are every leaf of the tab in reading order, pages included. +func (t *layoutTab) panes() []string { + var out []string + var walk func(n *paneNode) + walk = func(n *paneNode) { + if n.leaf() { + out = append(out, n.Session) + return + } + walk(n.First) + walk(n.Second) + } + walk(t.Root) + return out +} + +// browserOf is the session whose page this tab shows, false for a tab +// without one. +func (t *layoutTab) browserOf() (string, bool) { + for _, pane := range t.panes() { + if owner, ok := browserOwner(pane); ok { + return owner, true + } + } + return "", false +} + +// openBrowser makes session's browser tab, its terminal at the left and its +// page at the right, named name and focused on the page, and answers its id. +// session may not be in a layout already: moving it out is the caller's to +// ask about. +func (ls *layoutTabs) openBrowser(name, session string) string { + ls.seq++ + page := browserPaneOf(session) + t := layoutTab{ID: fmt.Sprintf("layout-%d", ls.seq), Name: name, Root: &paneNode{Session: session}, Focus: page} + t.split(session, page, edgeRight) + ls.tabs = append(ls.tabs, t) + return t.ID +} + +// browserTab is the index of session's browser tab, -1 for none. +func (ls *layoutTabs) browserTab(session string) int { + return slices.IndexFunc(ls.tabs, func(t layoutTab) bool { + return findLeaf(t.Root, browserPaneOf(session)) != nil + }) +} + +// dropBrowser takes session's page out of its tab, which leaves the session +// alone and so dissolves the tab. It reports whether there was one. +func (ls *layoutTabs) dropBrowser(session string) bool { + i := ls.browserTab(session) + if i < 0 { + return false + } + ls.tabs = slices.Delete(ls.tabs, i, i+1) + return true +} + +// locked reports whether layout i takes no pane from outside it: a browser +// tab holds its session and that session's page only. +func (ls *layoutTabs) locked(i int) bool { + if i < 0 { + return false + } + _, ok := ls.tabs[i].browserOf() + return ok +} diff --git a/native/cmd/lich-native/layouts_browser_test.go b/native/cmd/lich-native/layouts_browser_test.go new file mode 100644 index 000000000..76132e83c --- /dev/null +++ b/native/cmd/lich-native/layouts_browser_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "reflect" + "testing" +) + +func TestOpenBrowserPutsThePageRightOfItsSession(t *testing.T) { + var ls layoutTabs + tab := ls.byID(ls.openBrowser("checkout", "s")) + if got, want := paneShape(tab.Root), "row(s,browser:s)"; got != want { + t.Fatalf("got %s, want %s", got, want) + } + if tab.Focus != "browser:s" { + t.Fatalf("focus %q, want the page", tab.Focus) + } + if got := tab.sessions(); !reflect.DeepEqual(got, []string{"s"}) { + t.Fatalf("sessions %v, want the session alone", got) + } + if owner, ok := tab.browserOf(); !ok || owner != "s" { + t.Fatalf("browserOf = %q, %v", owner, ok) + } + if ls.of("s") != 0 || !ls.locked(0) { + t.Fatal("the session's tab is not found or not locked") + } +} + +func TestBrowserTabRotatesLikeAnyLayout(t *testing.T) { + var ls layoutTabs + tab := ls.byID(ls.openBrowser("checkout", "s")) + if !tab.rotate("s") { + t.Fatal("rotate refused the browser tab") + } + if got, want := paneShape(tab.Root), "column(s,browser:s)"; got != want { + t.Fatalf("got %s, want %s", got, want) + } +} + +func TestTakingTheSessionDissolvesItsBrowserTabLeavingNoPage(t *testing.T) { + var ls layoutTabs + ls.openBrowser("checkout", "s") + left, ok := ls.take("s") + if !ok || left != "" || len(ls.tabs) != 0 { + t.Fatalf("take = %q, %v with %d tabs; want the tab gone and no page left on screen", left, ok, len(ls.tabs)) + } +} + +func TestDropBrowserDissolvesOnlyThatTab(t *testing.T) { + var ls layoutTabs + ls.open("pair", "a", "b", edgeRight) + ls.openBrowser("checkout", "s") + if ls.dropBrowser("a") { + t.Fatal("dropped a browser a session without one") + } + if !ls.dropBrowser("s") || len(ls.tabs) != 1 || ls.locked(0) { + t.Fatalf("tabs after drop: %+v", ls.tabs) + } +} + +func TestPruneLeavesAPageWhoseSessionLives(t *testing.T) { + var ls layoutTabs + ls.openBrowser("checkout", "s") + if ls.prune(func(id string) bool { return id == "s" }) { + t.Fatal("prune took a page whose session is live") + } + if !ls.prune(func(string) bool { return false }) || len(ls.tabs) != 0 { + t.Fatal("prune kept the browser tab of a closed session") + } +} + +func TestBrowserTabRoundTrips(t *testing.T) { + var ls layoutTabs + ls.openBrowser("checkout", "s") + raw, err := ls.marshal() + if err != nil { + t.Fatal(err) + } + back, err := parseLayouts(raw) + if err != nil { + t.Fatal(err) + } + if len(back.tabs) != 1 || paneShape(back.tabs[0].Root) != "row(s,browser:s)" || back.tabs[0].Focus != "browser:s" { + t.Fatalf("round trip: %+v", back.tabs) + } +} + +func TestPaneSessionIsThePagesOwner(t *testing.T) { + if paneSession("browser:s") != "s" || paneSession("s") != "s" { + t.Fatal("paneSession") + } +} diff --git a/native/cmd/lich-native/layouts_window.go b/native/cmd/lich-native/layouts_window.go index fe929c46e..ce55e00a1 100644 --- a/native/cmd/lich-native/layouts_window.go +++ b/native/cmd/lich-native/layouts_window.go @@ -81,19 +81,28 @@ type moveConfirm struct { orchestrator string delegates []string away int - cancel widget.Clickable - confirm widget.Clickable + // browser is a move into session's browser tab instead. + browser bool + cancel widget.Clickable + confirm widget.Clickable } func (u *window) askMove(r besideRequest, id, from string) { c := &u.layoutStage.move c.open, c.req, c.session, c.from = true, r, id, u.model.layoutName(from) - c.label, c.delegates = u.model.sessionLabel(id), nil + c.label, c.delegates, c.browser = u.model.sessionLabel(id), nil, false +} + +func (u *window) askMoveToBrowser(id, from string) { + c := &u.layoutStage.move + c.open, c.session, c.from = true, id, u.model.layoutName(from) + c.label, c.delegates, c.browser = u.model.sessionLabel(id), nil, true + u.model.invalidate() } func (u *window) askMoveDelegates(orchestrator string, ids []string, away int) { c := &u.layoutStage.move - c.open, c.orchestrator, c.delegates, c.away = true, orchestrator, ids, away + c.open, c.orchestrator, c.delegates, c.away, c.browser = true, orchestrator, ids, away, false // The question was laid out at the top of this frame, closed. u.model.invalidate() } @@ -104,9 +113,12 @@ func (u *window) layoutMoveConfirm(gtx C, th *ui.Theme) { c := &u.layoutStage.move if c.confirm.Clicked(gtx) && c.open { c.open = false - if c.delegates != nil { + switch { + case c.browser: + u.model.openBrowserTab(u.ctx, c.session, true) + case c.delegates != nil: u.model.showDelegates(u.ctx, c.orchestrator, c.delegates) - } else { + default: u.model.splitBeside(u.ctx, c.req.target, c.session, c.req.edge, true) } u.model.invalidate() @@ -127,6 +139,10 @@ func (u *window) layoutMoveConfirm(gtx C, th *ui.Theme) { } func (c *moveConfirm) question() (title, body string) { + if c.browser { + return i18n.T("layouts.move.title", i18n.Args{"label": c.label}), + i18n.T("layouts.browser.moveBody", i18n.Args{"label": c.label, "layout": c.from}) + } if c.delegates != nil { return i18n.T("layouts.delegates.moveTitle", i18n.Args{"count": len(c.delegates)}), i18n.T("layouts.delegates.moveBody", i18n.Args{"count": c.away}) diff --git a/native/cmd/lich-native/main.go b/native/cmd/lich-native/main.go index 07a784506..e98af7734 100644 --- a/native/cmd/lich-native/main.go +++ b/native/cmd/lich-native/main.go @@ -120,8 +120,8 @@ type window struct { root ui.Root toaster ui.Toaster // duplicates holds the warnings about a conversation resumed in two - // cards and the questions of cards whose process exited, in the - // bottom-right corner, apart from every other toast. + // cards, the questions of cards whose process exited and the browsers + // agents opened, in the bottom-right corner, apart from every other toast. duplicates ui.Toaster // caption is where the topbar draws the window's buttons, chrome what // the window manager last said about the window. @@ -169,6 +169,8 @@ type window struct { islandAt image.Point // layoutStage is the panes' state while a layout is on screen. layoutStage layoutStage + // browserPanes are the page panes' widgets, by session (browser_pane.go). + browserPanes map[string]*browserPaneUI // hotkeyLanguage is the language the hotkeys' labels were last put in. hotkeyLanguage i18n.Locale // fontsLanguage is the language the interface's CJK faces were last @@ -228,6 +230,7 @@ func run(w *app.Window, rt backend.Runtime) error { u.model.themeSelection = u.appearance.applySelection u.model.exited = u.sessionExited u.model.duplicate = u.warnDuplicate + u.model.browserOpened = u.offerBrowser u.model.pluginIncompatible = func() { go u.checkPlugin() } u.model.appUpdateProgress = u.updateProgressed u.palette.closed = u.model.closed @@ -365,6 +368,9 @@ func (u *window) connect() error { u.logErr("load layouts", err) u.toaster.Show(ui.Toast{Variant: ui.ToastError, Title: i18n.T("layouts.load.failed", i18n.Args{"error": backend.ErrorText(err)})}) } + if err := u.model.hydrateBrowsers(u.ctx); err != nil { + u.logErr("read the session browsers", err) + } if _, ok := u.model.project(); !ok && *flagProject != "" { path, err := filepath.Abs(*flagProject) if err != nil { @@ -527,6 +533,7 @@ func (u *window) layout(gtx C) error { u.apply(act) u.applyTopbar(top) u.applyLayoutStage() + u.settleBrowserPanes() u.runHotkeys(gtx, fired) return contentErr } @@ -888,19 +895,6 @@ func renameSession(ctx context.Context, c *backend.Client, show func(ui.Toast) s } } -// applyBrowser opens the session's Chromium, the window its agent drives. -func (u *window) applyBrowser(act sidebarAction) { - id := act.browser - if id == "" { - return - } - go func() { - if err := u.client.ShowBrowser(u.ctx, id); err != nil { - u.toaster.Show(ui.Toast{Variant: ui.ToastError, Title: i18n.T("sidebar.sessionCard.browserFailed", i18n.Args{"error": backend.ErrorText(err)})}) - } - }() -} - // applyOpenIn carries out the card menu's Open in: a shell session at the // card's directory, the directory in the editor, or in the file manager. func (u *window) applyOpenIn(act sidebarAction) { diff --git a/native/cmd/lich-native/model.go b/native/cmd/lich-native/model.go index 7cd45f7f2..8aa3fd200 100644 --- a/native/cmd/lich-native/model.go +++ b/native/cmd/lich-native/model.go @@ -111,6 +111,10 @@ type model struct { // screen, "" for a project's session alone. layouts layoutTabs shownLayout string + // browsers are the session browsers (browser_model.go); browserOpened + // hears one the agent started. + browsers browserStore + browserOpened func(api.BrowserState) } type sessionCwd = api.SessionCwd @@ -125,7 +129,8 @@ func newModel(client *backend.Client, invalidate func()) *model { ompProfiles: newProfileBook(ompProfileProvider, client, invalidate, quota), xdgProfiles: xdgProfileBooks(client, invalidate, quota), providers: newProviderBook(client, invalidate), - closed: newClosedProjects(client, invalidate)} + closed: newClosedProjects(client, invalidate), + browsers: newBrowserStore()} } func (m *model) reload(ctx context.Context) error { @@ -266,6 +271,13 @@ func (m *model) hydrateTurnStates(ctx context.Context) error { // serves one client and a newer one replaces it, or lich is gone. func (m *model) follow(ctx context.Context, events <-chan backend.Event) { for ev := range events { + if owner, frame := api.BrowserFrameEvents.ID(ev.Name); frame { + if err := m.applyBrowserFrame(owner, ev); err != nil { + log.Print(err) + } + m.invalidate() + continue + } if id, exited := api.TerminalExitEvents.ID(ev.Name); exited { m.mu.Lock() m.status.exited(id) @@ -342,6 +354,14 @@ func (m *model) follow(ctx context.Context, events <-chan backend.Event) { } case api.SessionTouchedEvent.Name: m.sessionTouched(ctx, ev) + case api.BrowserStateEvent.Name: + if st, err := backend.Decode(ev, api.BrowserStateEvent); err == nil { + m.applyBrowserState(ctx, st) + } + case api.BrowserAgentEvent.Name: + if a, err := backend.Decode(ev, api.BrowserAgentEvent); err == nil { + m.applyBrowserAgent(a, time.Now()) + } case api.SessionFocusEvent.Name: m.sessionFocus(ev) case api.WindowRaiseEvent.Name: @@ -499,6 +519,7 @@ type sessionView struct { aged bool // status has a clock (busy, compacting, waiting) agent string // provider CLI live in the PTY, "" for none sandbox sandboxEvent + browser bool // a browser of this session runs relay sessionRelay // Direction "" for none tool sessionTool // Name "" outside a tool call reason string // what a waiting session is blocked on, "" for unsaid @@ -527,7 +548,7 @@ func (m *model) viewOf(p backend.Project, s backend.Session, now time.Time) sess status: m.status.status(s.ID), unread: m.status.unread(s.ID), age: age, aged: aged, agent: m.agent[s.ID], sandbox: m.sandbox[s.ID], git: m.git[shown], base: m.base[shown], pr: m.prs[shown].pr, - relay: m.relay[s.ID], tool: tool, reason: m.status.reason(s.ID), + relay: m.relay[s.ID], tool: tool, reason: m.status.reason(s.ID), browser: m.browsers.states[s.ID].Open, profileName: m.profileNameOf(s.Kind, s.Profile), continueIn: m.continueTargets(s.Kind, s.Profile), layout: m.layoutNameOfLocked(s.ID), running: usage.Model, } diff --git a/native/cmd/lich-native/model_delegates.go b/native/cmd/lich-native/model_delegates.go index 1dd835c0f..5d1a9c4f1 100644 --- a/native/cmd/lich-native/model_delegates.go +++ b/native/cmd/lich-native/model_delegates.go @@ -59,6 +59,10 @@ func (m *model) showDelegates(ctx context.Context, orchestrator string, ids []st m.mu.Unlock() return false } + if m.layouts.locked(m.layouts.of(orchestrator)) { + m.mu.Unlock() + return false + } target, below := m.delegateSlotLocked(orchestrator, m.delegatesLocked(orchestrator)) for _, id := range ids { m.layouts.take(id) diff --git a/native/cmd/lich-native/model_layouts.go b/native/cmd/lich-native/model_layouts.go index 901aa96bc..7802847fc 100644 --- a/native/cmd/lich-native/model_layouts.go +++ b/native/cmd/lich-native/model_layouts.go @@ -94,8 +94,8 @@ func (m *model) refocusLayoutLocked() bool { m.shownLayout = "" return false } - m.active = t.Focus - if p, _ := m.sessionAt(t.Focus); p != nil { + m.active = paneSession(t.Focus) + if p, _ := m.sessionAt(m.active); p != nil { m.activeProject = p.ID } return true @@ -112,7 +112,7 @@ func (m *model) showLayout(ctx context.Context, id string) bool { } focus := t.Focus m.mu.Unlock() - m.setActive(ctx, focus) + m.setActive(ctx, paneSession(focus)) return true } @@ -167,10 +167,11 @@ func (m *model) layoutOf(id string) string { // splitBeside puts session against target's edge and brings it into focus: // in the layout holding target, else in a new layout of the two named after // target. A session that is in another layout already stays where it is -// unless move is set; it reports false when nothing was split. +// unless move is set, and a browser tab takes no one; it reports false when +// nothing was split. func (m *model) splitBeside(ctx context.Context, target, session string, edge paneEdge, move bool) bool { m.mu.Lock() - if target == session || !m.liveLocked(target) || !m.liveLocked(session) { + if target == session || !m.liveLocked(target) || !m.liveLocked(session) || m.layouts.locked(m.layouts.of(target)) { m.mu.Unlock() return false } @@ -207,7 +208,12 @@ func (m *model) unsplit(ctx context.Context, session string) { left, _ := m.layouts.take(session) next := left if t := m.layouts.byID(id); t != nil { - next = t.Focus + next = paneSession(t.Focus) + } + if next == "" { + // The session left its browser tab, which goes with it: it is what + // stays on screen. + next = session } m.mu.Unlock() m.setActive(ctx, next) @@ -223,7 +229,7 @@ func (m *model) closeLayout(ctx context.Context, id string) { m.mu.Unlock() return } - focus, shown := t.Focus, m.shownLayout == id + focus, shown := paneSession(t.Focus), m.shownLayout == id m.layouts.close(id) m.mu.Unlock() if shown { @@ -253,14 +259,33 @@ func (m *model) cyclePane(ctx context.Context, step int) { if !ok { return } - ids := t.sessions() + ids := t.panes() at := 0 for i, id := range ids { if id == t.Focus { at = i } } - m.setActive(ctx, ids[(at+step+len(ids))%len(ids)]) + m.focusPane(ctx, ids[(at+step+len(ids))%len(ids)]) +} + +// focusPane puts pane in focus: its session on screen, and in the layout +// holding it, the pane itself, which for a page is not its session's. +func (m *model) focusPane(ctx context.Context, pane string) { + session := paneSession(pane) + if active, ok := m.activeSession(); !ok || active.ID != session { + m.setActive(ctx, session) + } + m.mu.Lock() + changed := false + if i := m.layouts.of(session); i >= 0 && m.layouts.tabs[i].Focus != pane { + m.layouts.tabs[i].Focus = pane + changed = true + } + m.mu.Unlock() + if changed { + m.saveLayouts() + } } // setRatio stores a divider's new ratio once its drag ends. diff --git a/native/cmd/lich-native/sidebar_card.go b/native/cmd/lich-native/sidebar_card.go index c0112e915..bb5e4a696 100644 --- a/native/cmd/lich-native/sidebar_card.go +++ b/native/cmd/lich-native/sidebar_card.go @@ -221,6 +221,9 @@ func (s *sidebar) titleRow(th *ui.Theme, sv *sessionView) []layout.FlexChild { if sv.Forked { row = append(row, layout.Rigid(func(gtx C) D { return icMenuFork.Layout(gtx, 12, th.MutedForeground) }), ui.Gap(1.5)) } + if sv.browser { + row = append(row, layout.Rigid(func(gtx C) D { return icBrowser.Layout(gtx, 12, th.MutedForeground) }), ui.Gap(1.5)) + } if s.renaming(sv.ID) { return append(row, layout.Flexed(1, func(gtx C) D { return s.renameField(gtx, th, sv.ID, s.renames[sv.ID]) })) } diff --git a/native/cmd/lich-native/stage_layout.go b/native/cmd/lich-native/stage_layout.go index 2c4467452..5ab5ff20e 100644 --- a/native/cmd/lich-native/stage_layout.go +++ b/native/cmd/lich-native/stage_layout.go @@ -113,14 +113,13 @@ func (u *window) layoutIsland(gtx C, v stageView) error { // pointer; the Sortable only carries the pane. st.sort.Update(gtx) lt := v.layout - ids := lt.sessions() root := lt.Root if lt.Zoomed { root = &paneNode{Session: lt.Focus} } u.terms.onScreen(onScreenOf(root)...) in := image.Rectangle{Max: gtx.Constraints.Max}.Inset(gtx.Dp(gutter)) - return u.paneTree(gtx, v, root, "", in, len(ids) > 1) + return u.paneTree(gtx, v, root, "", in, len(lt.panes()) > 1) } func onScreenOf(n *paneNode) []string { @@ -132,6 +131,10 @@ func onScreenOf(n *paneNode) []string { // root, then one digit per step, 0 first and 1 second. func (u *window) paneTree(gtx C, v stageView, n *paneNode, path string, r image.Rectangle, several bool) error { if n.leaf() { + if owner, page := browserOwner(n.Session); page { + u.browserPaneAt(gtx, v, owner, r, several) + return nil + } return u.paneAt(gtx, v, n.Session, r, several) } first, second, gap := u.splitRects(gtx, v, n, path, r) @@ -427,9 +430,7 @@ func (u *window) applyLayoutStage() { u.layDelegatesOut(id) } if focus != "" { - if active, ok := u.model.activeSession(); !ok || active.ID != focus { - u.model.setActive(u.ctx, focus) - } + u.model.focusPane(u.ctx, focus) } if zoom { u.model.toggleZoom() diff --git a/native/i18n/locales/en/layouts.json b/native/i18n/locales/en/layouts.json index b73dd7ede..245d2849e 100644 --- a/native/i18n/locales/en/layouts.json +++ b/native/i18n/locales/en/layouts.json @@ -15,6 +15,21 @@ "other": "{count} more · {waiting} waiting" } }, + "browser": { + "title": "Browser", + "agent": "agent", + "close": "Close the browser", + "address": "Type a URL", + "back": "Back", + "forward": "Forward", + "reload": "Reload", + "openInWindow": "Open in a window", + "locked": "This tab holds only {label} and its browser", + "opened": "{label} opened a browser", + "dismiss": "Dismiss", + "open": "Open", + "moveBody": "It leaves \"{layout}\". A browser tab holds only {label} and its browser." + }, "split": { "noRoom": "No room for another pane here. Make the window wider or take a pane out first." }, diff --git a/native/i18n/locales/es/layouts.json b/native/i18n/locales/es/layouts.json index 869c13950..9c54414dd 100644 --- a/native/i18n/locales/es/layouts.json +++ b/native/i18n/locales/es/layouts.json @@ -15,6 +15,21 @@ "other": "{count} más · {waiting} esperando" } }, + "browser": { + "title": "Navegador", + "agent": "agente", + "close": "Cerrar el navegador", + "address": "Escribe una URL", + "back": "Atrás", + "forward": "Adelante", + "reload": "Recargar", + "openInWindow": "Abrir en una ventana", + "locked": "Esta pestaña es solo de {label} y su navegador", + "opened": "{label} abrió un navegador", + "dismiss": "Descartar", + "open": "Abrir", + "moveBody": "Sale de \"{layout}\". La pestaña del navegador solo admite {label} y su navegador." + }, "split": { "noRoom": "No hay espacio para otro panel aquí. Ensancha la ventana o saca antes un panel." }, diff --git a/native/i18n/locales/ja/layouts.json b/native/i18n/locales/ja/layouts.json index f55e9f8ca..8df3eb23a 100644 --- a/native/i18n/locales/ja/layouts.json +++ b/native/i18n/locales/ja/layouts.json @@ -13,6 +13,21 @@ "other": "ほか {count} 件 · {waiting} 件待機中" } }, + "browser": { + "title": "ブラウザ", + "agent": "エージェント", + "close": "ブラウザを閉じる", + "address": "URL を入力", + "back": "戻る", + "forward": "進む", + "reload": "再読み込み", + "openInWindow": "ウィンドウで開く", + "locked": "このタブには {label} とそのブラウザしか置けません", + "opened": "{label} がブラウザを開きました", + "dismiss": "閉じる", + "open": "開く", + "moveBody": "「{layout}」から外れます。ブラウザのタブには {label} とそのブラウザしか置けません。" + }, "split": { "noRoom": "ここには別のペインを置く余裕がありません。ウィンドウを広げるか、先にペインを外してください。" }, diff --git a/native/i18n/locales/ko/layouts.json b/native/i18n/locales/ko/layouts.json index a442721c5..b59dc600a 100644 --- a/native/i18n/locales/ko/layouts.json +++ b/native/i18n/locales/ko/layouts.json @@ -13,6 +13,21 @@ "other": "{count}개 더 · {waiting}개 대기 중" } }, + "browser": { + "title": "브라우저", + "agent": "에이전트", + "close": "브라우저 닫기", + "address": "URL 입력", + "back": "뒤로", + "forward": "앞으로", + "reload": "새로고침", + "openInWindow": "창에서 열기", + "locked": "이 탭에는 {label}와(과) 그 브라우저만 둘 수 있습니다", + "opened": "{label}이(가) 브라우저를 열었습니다", + "dismiss": "닫기", + "open": "열기", + "moveBody": "\"{layout}\"에서 빠집니다. 브라우저 탭에는 {label}와(과) 그 브라우저만 둘 수 있습니다." + }, "split": { "noRoom": "여기에는 창을 하나 더 놓을 공간이 없습니다. 창을 넓히거나 먼저 창 하나를 빼세요." }, diff --git a/native/i18n/locales/pt-br/layouts.json b/native/i18n/locales/pt-br/layouts.json index d17799adb..9bad37a17 100644 --- a/native/i18n/locales/pt-br/layouts.json +++ b/native/i18n/locales/pt-br/layouts.json @@ -15,6 +15,21 @@ "other": "mais {count} · {waiting} esperando" } }, + "browser": { + "title": "Browser", + "agent": "agente", + "close": "Fechar o browser", + "address": "Digite uma URL", + "back": "Voltar", + "forward": "Avançar", + "reload": "Recarregar", + "openInWindow": "Abrir em janela", + "locked": "Esta aba é só de {label} e do browser dela", + "opened": "{label} abriu um browser", + "dismiss": "Dispensar", + "open": "Abrir", + "moveBody": "Ela sai de \"{layout}\". A aba do browser só comporta {label} e o browser dela." + }, "split": { "noRoom": "Não há espaço para outro painel aqui. Alargue a janela ou tire um painel antes." }, diff --git a/native/i18n/locales/ru/layouts.json b/native/i18n/locales/ru/layouts.json index 97a79400e..21984156b 100644 --- a/native/i18n/locales/ru/layouts.json +++ b/native/i18n/locales/ru/layouts.json @@ -19,6 +19,21 @@ "other": "ещё {count} · ждут: {waiting}" } }, + "browser": { + "title": "Браузер", + "agent": "агент", + "close": "Закрыть браузер", + "address": "Введите URL", + "back": "Назад", + "forward": "Вперёд", + "reload": "Обновить", + "openInWindow": "Открыть в окне", + "locked": "В этой вкладке только {label} и её браузер", + "opened": "{label} открыла браузер", + "dismiss": "Скрыть", + "open": "Открыть", + "moveBody": "Она уйдёт из «{layout}». Во вкладке браузера помещаются только {label} и её браузер." + }, "split": { "noRoom": "Здесь нет места для ещё одной панели. Сделайте окно шире или сначала уберите панель." }, diff --git a/native/i18n/locales/zh-cn/layouts.json b/native/i18n/locales/zh-cn/layouts.json index 00d2f4f14..a7639266b 100644 --- a/native/i18n/locales/zh-cn/layouts.json +++ b/native/i18n/locales/zh-cn/layouts.json @@ -13,6 +13,21 @@ "other": "另外 {count} 个 · {waiting} 个等待中" } }, + "browser": { + "title": "浏览器", + "agent": "智能体", + "close": "关闭浏览器", + "address": "输入网址", + "back": "后退", + "forward": "前进", + "reload": "重新加载", + "openInWindow": "在窗口中打开", + "locked": "此标签页只容纳 {label} 及其浏览器", + "opened": "{label} 打开了浏览器", + "dismiss": "忽略", + "open": "打开", + "moveBody": "它将离开“{layout}”。浏览器标签页只容纳 {label} 及其浏览器。" + }, "split": { "noRoom": "这里没有空间再放一个窗格。请加宽窗口，或先移出一个窗格。" }, From d0b3a13de9e588a860cae596ffb883c2fff6e5cc Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:23:04 -0300 Subject: [PATCH 05/25] docs(browser): the session browser is a tab of frames beside its session --- CHANGELOG.md | 2 +- README.es.md | 2 +- README.ja.md | 2 +- README.ko.md | 2 +- README.md | 2 +- README.pt-BR.md | 2 +- README.ru.md | 2 +- README.zh-CN.md | 2 +- docs/browser-tab.md | 59 +++++++++++++++++++++++++++------------------ docs/ceilings.md | 25 ++++++++++--------- 10 files changed, 56 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa85c83c7..bba72bad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- A session card's menu opens a browser for that session, above Open in. It is a Chromium window of its own, and the session's agent drives the same page: open a URL, read it, click, type, press a key, go back and forward, scroll and screenshot, from `browser_*` tools or from `lich browser`. You watch and use that window while the agent does. Closing the card closes the browser. opencode's agent reaches it through `lich browser`; its own tool list does not include `browser_*`. +- A session card's menu opens a browser for that session, above Open in, in a tab beside the session: its terminal on one side and the page on the other, with the divider, rotate and zoom of a split. The session's agent drives the same page: open a URL, read it, click, type, press a key, go back and forward, scroll and screenshot, from `browser_*` tools or from `lich browser`, and the pane marks what the agent does. You click, type and paste in the page while it does. When the agent opens a browser on its own, a toast in the bottom-right corner offers to show it. Open in a window moves the page to a Chromium window of its own, cookies and all. Closing the card closes the browser. The browser is the Chrome, Edge, Chromium, Brave, Vivaldi or Helium already installed. opencode's agent reaches it through `lich browser`; its own tool list does not include `browser_*`. ### Fixed diff --git a/README.es.md b/README.es.md index c5afc6141..3759e89a9 100644 --- a/README.es.md +++ b/README.es.md @@ -147,7 +147,7 @@ porque lee lo que cada CLI deja escrito, y no hay dos que registren lo mismo. | Una tarea que entregas a otra sesión informa "sin respuesta" en cuanto ese agente termina su turno sin responder | sí | sí | sí | sí | sí | no, espera al plazo | sí | sí | | Los mensajes propios de lich al agente (tareas entregadas, resultados listos, avisos) en el idioma de prompt que elijas | sí | sí | sí | sí | sí | sí | sí | sí | | Se le indica al iniciarse, en el idioma de prompt, que reparta el trabajo como sesiones de lich en vez de subagentes ocultos | sí | no | no | no | sí | no | no | no | -| La tarjeta de la sesión abre un navegador que el agente conduce, en la misma ventana que ves | sí | sí | sí | la tarjeta lo abre; el agente usa `lich browser`, no las tools `browser_*` | sí | sí | sí | sí | +| La tarjeta de la sesión abre un navegador que el agente conduce, en una pestaña junto a la sesión | sí | sí | sí | la tarjeta lo abre; el agente usa `lich browser`, no las tools `browser_*` | sí | sí | sí | sí | Ninguna de estas diferencias es un descuido, y en ningún caso lich se guarda algo que el CLI informe; [`docs/ceilings.md`](docs/ceilings.md) cuenta qué se diff --git a/README.ja.md b/README.ja.md index 553807c29..46557754b 100644 --- a/README.ja.md +++ b/README.ja.md @@ -136,7 +136,7 @@ lich は各 CLI が書き出す情報を読んでいて、何を書き出すか | 別のセッションに渡したタスクは、相手のエージェントが返信せずにターンを終えた時点で、すぐに「unanswered」として戻ってくる | あり | あり | あり | あり | あり | なし（期限まで待つ） | あり | あり | | lich からエージェントへの独自メッセージ（渡されたタスク、結果の準備完了、通知）を、選択したプロンプト言語で送る | あり | あり | あり | あり | あり | あり | あり | あり | | 起動時に、見えないサブエージェントではなく lich セッションに作業を振り分けるよう、プロンプト言語で指示される | あり | なし | なし | なし | あり | なし | なし | なし | -| セッションのカードが、エージェントが操作するブラウザを開く。見ているのは同じウィンドウ | あり | あり | あり | カードは開く。エージェントは `browser_*` ではなく `lich browser` を使う | あり | あり | あり | あり | +| セッションのカードが、エージェントが操作するブラウザをセッションの隣のタブに開く | あり | あり | あり | カードは開く。エージェントは `browser_*` ではなく `lich browser` を使う | あり | あり | あり | あり | 「なし」はどれも意図したもので、CLI が出している情報を lich が隠しているわけではありません。 それぞれについて何を検証したかは [`docs/ceilings.md`](docs/ceilings.md) にまとめています。 diff --git a/README.ko.md b/README.ko.md index b8779e7c5..4ba40259d 100644 --- a/README.ko.md +++ b/README.ko.md @@ -136,7 +136,7 @@ lich로 할 수 있는 일: | 다른 세션에 넘긴 작업은 그 에이전트가 답하지 않고 턴을 끝내는 즉시 "unanswered"로 보고됨 | 예 | 예 | 예 | 예 | 예 | 아니요, 마감 시각까지 기다림 | 예 | 예 | | lich가 에이전트에게 보내는 메시지(넘긴 작업, 준비된 결과, 알림)가 선택한 프롬프트 언어로 전달됨 | 예 | 예 | 예 | 예 | 예 | 예 | 예 | 예 | | 세션을 시작할 때 숨은 서브에이전트 대신 lich 세션으로 작업을 나누라는 안내를 프롬프트 언어로 받음 | 예 | 아니요 | 아니요 | 아니요 | 예 | 아니요 | 아니요 | 아니요 | -| 세션 카드가 에이전트가 조작하는 브라우저를 연다. 보는 창은 같다 | 예 | 예 | 예 | 카드는 연다. 에이전트는 `browser_*`가 아니라 `lich browser`를 쓴다 | 예 | 예 | 예 | 예 | +| 세션 카드가 에이전트가 조작하는 브라우저를 세션 옆 탭에 연다 | 예 | 예 | 예 | 카드는 연다. 에이전트는 `browser_*`가 아니라 `lich browser`를 쓴다 | 예 | 예 | 예 | 예 | 빈칸은 모두 의도된 것이며, CLI가 보고하는 정보를 lich가 숨기는 경우는 없습니다. 빈칸마다 무엇을 측정했는지는 [`docs/ceilings.md`](docs/ceilings.md)에 정리되어 있습니다. diff --git a/README.md b/README.md index 2340abaab..dceadb73e 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ reads what each CLI writes down and no two of them write down the same things. | A task you hand another session reports back "unanswered" as soon as that agent ends its turn without replying | yes | yes | yes | yes | yes | no, waits for the deadline | yes | yes | | lich's own messages to the agent (handed tasks, results ready, notices) in the prompt language you pick | yes | yes | yes | yes | yes | yes | yes | yes | | Told at spawn, in the prompt language, to fan work out as lich sessions rather than hidden subagents | yes | no | no | no | yes | no | no | no | -| The session card opens a browser its agent drives, in the same window you watch | yes | yes | yes | the card opens it; the agent uses `lich browser`, not `browser_*` | yes | yes | yes | yes | +| The session card opens a browser its agent drives, in a tab beside the session | yes | yes | yes | the card opens it; the agent uses `lich browser`, not `browser_*` | yes | yes | yes | yes | Every gap is deliberate, none of them is lich withholding something the CLI reports, and [`docs/ceilings.md`](docs/ceilings.md) says what was measured behind diff --git a/README.pt-BR.md b/README.pt-BR.md index 4be6d4649..2a080c7aa 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -148,7 +148,7 @@ mesmas coisas. | Uma tarefa passada para outra sessão volta como "sem resposta" assim que aquele agente encerra o turno sem responder | sim | sim | sim | sim | sim | não, espera o prazo | sim | sim | | As mensagens do próprio lich para o agente (tarefas delegadas, resultados prontos, avisos) no idioma de prompt que você escolher | sim | sim | sim | sim | sim | sim | sim | sim | | Instruído na criação, no idioma de prompt, a distribuir o trabalho em sessões do lich em vez de subagentes ocultos | sim | não | não | não | sim | não | não | não | -| O card da sessão abre um navegador que o agente dirige, na mesma janela que você acompanha | sim | sim | sim | o card abre; o agente usa `lich browser`, não as tools `browser_*` | sim | sim | sim | sim | +| O card da sessão abre um navegador que o agente dirige, numa aba ao lado da sessão | sim | sim | sim | o card abre; o agente usa `lich browser`, não as tools `browser_*` | sim | sim | sim | sim | Toda lacuna é proposital: nenhuma é o lich escondendo algo que a CLI informa, e o [`docs/ceilings.md`](docs/ceilings.md) diz o que foi medido em cada uma. diff --git a/README.ru.md b/README.ru.md index b6babe62e..087dbf51d 100644 --- a/README.ru.md +++ b/README.ru.md @@ -147,7 +147,7 @@ | Задача, переданная другой сессии, сразу возвращается как «без ответа», если тот агент завершил ход и не ответил | да | да | да | да | да | нет, ждёт истечения срока | да | да | | Сообщения самого lich агенту (переданные задачи, готовые результаты, уведомления) на выбранном вами языке промптов | да | да | да | да | да | да | да | да | | При запуске агент получает указание на языке промптов раздавать работу сессиям lich, а не скрытым субагентам | да | нет | нет | нет | да | нет | нет | нет | -| Карточка сессии открывает браузер, которым управляет агент, в том же окне, которое вы видите | да | да | да | карточка открывает; агент вызывает `lich browser`, а не инструменты `browser_*` | да | да | да | да | +| Карточка сессии открывает браузер, которым управляет агент, во вкладке рядом с сессией | да | да | да | карточка открывает; агент вызывает `lich browser`, а не инструменты `browser_*` | да | да | да | да | Все пробелы намеренные: ни в одном из них lich не скрывает того, что сообщает CLI, а что именно измерялось за каждым, описано в diff --git a/README.zh-CN.md b/README.zh-CN.md index 7ada61fed..9c14599be 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -124,7 +124,7 @@ | 交给另一个会话的任务，对方一轮结束却没回复时，立即报告“未回复” | 有 | 有 | 有 | 有 | 有 | 无，要等到截止时间 | 有 | 有 | | lich 发给智能体的消息（派出的任务、结果已就绪、通知）使用你选的提示词语言 | 有 | 有 | 有 | 有 | 有 | 有 | 有 | 有 | | 启动时用提示词语言告诉智能体，要把工作分派成 lich 会话，而不是看不见的子智能体 | 有 | 无 | 无 | 无 | 有 | 无 | 无 | 无 | -| 会话卡片打开一个由智能体操作的浏览器，你看着的是同一个窗口 | 有 | 有 | 有 | 卡片会打开；智能体用 `lich browser`，而不是 `browser_*` 工具 | 有 | 有 | 有 | 有 | +| 会话卡片在会话旁的标签页中打开一个由智能体操作的浏览器 | 有 | 有 | 有 | 卡片会打开；智能体用 `lich browser`，而不是 `browser_*` 工具 | 有 | 有 | 有 | 有 | 每一处差异都是有意为之，没有一处是 CLI 报告了而 lich 不显示；每一处背后实测了什么，见 [`docs/ceilings.md`](docs/ceilings.md)。 diff --git a/docs/browser-tab.md b/docs/browser-tab.md index 8301fa046..e19c78fa0 100644 --- a/docs/browser-tab.md +++ b/docs/browser-tab.md @@ -1,32 +1,43 @@ -# Decision: the session browser is a sidecar window +# Decision: the session browser is a tab of frames **Status: the card's Browser item opens it.** Right-click a session card → -**Browser**, above Open in. That opens a Chromium window owned by that session. -The agent drives the same page through `lich browser` and the `browser_*` MCP -tools. - -## Why not a page inside lich - -lich's window is native (`docs/native-window.md`). It does not embed Chromium. -The product is a **second Chromium process** on `chromium.FindBrowser` (system -Chrome, Edge, Chromium, Brave or Vivaldi), with its own user-data directory. -Closing that window does not quit lich. Closing the session closes the sidecar. +**Browser**, above Open in. That opens a tab beside the session: its terminal +and its page, split as any layout is. The agent drives the same page through +`lich browser` and the `browser_*` MCP tools. + +## Why frames and not a webview + +lich's window is native (`docs/native-window.md`). On Linux it speaks Wayland +directly, without GTK, so no system webview fits inside it; Windows and macOS +would take one, Linux would not. The page is a **headless Chromium** on +`chromium.FindBrowser` (the system's Chrome, Edge, Chromium, Brave, Vivaldi +or Helium), with its own user-data directory. `browser.Attach` starts its +screencast at the pane's size; each frame reaches the window as +`browser:frame:` and is acked only once sent, so a window slow to +draw slows the stream. The pane's pointer, keys, committed text (an input +method's or a paste) and toolbar go back through `browser.Mouse`, `.Key`, +`.Text` and `.Nav`, which light nothing up: only the agent's actions emit +`browser-agent`, which the pane shows as the agent chip and a mark where it +clicked. ## One browser per card -- none yet → headed window at `about:blank` -- already headed → focus that window -- headless only (the agent opened first) → reopen it as a headed window at - the same URL, on the same profile, with its cookies (session ones - included). chromedp cannot show a headless process in place, so what lives - only in the page, a half-filled form or sessionStorage, does not survive - -A headless browser nobody used for five minutes is closed. A headed window is -not: lich cannot see a person using it, so only the card closing ends it. - -The agent without the menu uses `browser_open`: headless if nobody asked for a -window, and the headed one if the person already did. MCP and `lich browser` -target `LICH_SESSION_ID`. +- the card's Browser item → its tab, starting a headless browser at + `about:blank` when there is none. A session in another layout is asked + about before it moves. +- the agent's `browser_open` first → a headless browser, and a toast in the + bottom-right corner whose Open shows that same page in the tab +- **Open in a window** → the page reopens as a headed window on the same + profile, with its cookies (session ones included); its tab goes. chromedp + cannot show a headless process in place, so what lives only in the page, a + half-filled form or sessionStorage, does not survive. Browser on the card + then brings that window forward. + +`browser-state` tells the window when a browser starts, closes, moves to a +window or loads another page; `browser.Running` is what a window reads when +it attaches. A browser no pane shows and no one used for five minutes is +closed; one a pane shows, or a window, is not. The tab takes no other +session, and closing its page closes the browser. ## What the agent may load diff --git a/docs/ceilings.md b/docs/ceilings.md index a67f2e4f6..9df20fbbd 100644 --- a/docs/ceilings.md +++ b/docs/ceilings.md @@ -910,18 +910,19 @@ work when nobody knows it and that the call site never shows. The mechanism and up to that late: observing `AppleInterfaceThemeChangedNotification` needs an Objective-C shim no one could compile or run without a Mac. The BSDs read no scheme and draw dark. -- **A session's browser is a second Chromium, one per card** (`internal/browser`, - `docs/browser-tab.md`): the card menu opens a headed window of the system - Chrome, Edge, Chromium, Brave or Vivaldi, never a page inside lich. The agent - drives that same window. A headless one the agent opened first is reopened - as a headed window on the same profile, carrying the URL and every cookie, - because chromedp cannot show a headless process; a half-filled form or - sessionStorage is lost on the way. Closing the card, or the session's process - ending, closes it; closing the browser window does not close the session. A - headless browser idle for five minutes is closed, a headed window never is. It loads only http, https and - about:blank. Page info never includes a form's value. Screenshots are files. - There is no tool that runs JavaScript in the page. It is not a sandbox: the - page runs as the user, in an ephemeral profile. +- **A session's browser is a second Chromium, one per card, drawn from frames** (`internal/browser`, + `docs/browser-tab.md`): a headless Chrome, Edge, Chromium, Brave, Vivaldi or Helium already installed, + whose page reaches the session's browser tab as a JPEG screencast over `/events`, at the pace the window + takes frames. The native window cannot hold a webview: on Linux it speaks Wayland without GTK. So the pane + has no DevTools, shows the arrow whatever cursor the page asks for, does not copy the page's selection to + the system clipboard (paste works), and leaves popups, downloads and the page's own dialogs out; Open in + a window moves the page to a real Chromium window for those, reopening it on the same profile with every + cookie, since chromedp cannot show a headless process in place. A half-filled form or sessionStorage is + lost on that move. Closing the card, or the session's process ending, closes the browser; closing that + window does not close the session. A headless browser no pane shows and no one used for five minutes is + closed; one a pane shows, or a window, never is. It loads only http, https and about:blank. Page info never + includes a form's value. Screenshots are files. There is no tool that runs JavaScript in the page. It is + not a sandbox: the page runs as the user, in an ephemeral profile. - **opencode does not list `browser_*`** (`internal/agentplugin`, `internal/cli/mcp_browser.go`): Claude Code and Codex are handed lich's MCP server at spawn, and Antigravity, oh-my-pi, Crush, Cursor CLI and Kiro get From 9851b950edc380a6504942e818c980f74e487424 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:25:03 -0300 Subject: [PATCH 06/25] fix(native): show a layout that holds a Home session A session of the project at the home directory routes with Home's flag set, and the layout on screen was hidden whenever that flag was, so a split or a browser tab holding one stayed off screen. showHome already clears the layout it leaves, so the flag added nothing but that. --- native/cmd/lich-native/browser_model_test.go | 20 ++++++++++++++++++++ native/cmd/lich-native/model_layouts.go | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/native/cmd/lich-native/browser_model_test.go b/native/cmd/lich-native/browser_model_test.go index 036cd1ca9..f59c45a9e 100644 --- a/native/cmd/lich-native/browser_model_test.go +++ b/native/cmd/lich-native/browser_model_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/backend" ) func TestOpenBrowserTabShowsTheSessionBesideItsPage(t *testing.T) { @@ -127,3 +128,22 @@ func TestTypedURL(t *testing.T) { t.Fatal("displayURL") } } + +func TestALayoutOfHomeSessionsComesOnScreen(t *testing.T) { + m, _ := newLayoutsModel(t) + m.projects = append(m.projects, backend.Project{ID: "home", Name: "~", Path: "/home/u", Sessions: []backend.Session{{ID: "h1", Label: "home one"}, {ID: "h2", Label: "home two"}}}) + m.homeID = "home" + ctx := context.Background() + if !m.openBrowserTab(ctx, "h1", false) { + t.Fatal("refused a Home session") + } + if id, _, active := layoutScreen(t, m); id == "" || active != "h1" { + t.Fatalf("layout %q active %q: a Home session's browser tab stayed off screen", id, active) + } + if !m.splitBeside(ctx, "h2", "notes", edgeRight, false) { + t.Fatal("split refused") + } + if id, _, _ := layoutScreen(t, m); id == "" { + t.Fatal("a split holding a Home session stayed off screen") + } +} diff --git a/native/cmd/lich-native/model_layouts.go b/native/cmd/lich-native/model_layouts.go index 7802847fc..2f1fb1729 100644 --- a/native/cmd/lich-native/model_layouts.go +++ b/native/cmd/lich-native/model_layouts.go @@ -124,7 +124,7 @@ func (m *model) shownLayoutTab() (layoutTab, bool) { } func (m *model) shownLayoutLocked() (layoutTab, bool) { - if m.shownLayout == "" || m.home { + if m.shownLayout == "" { return layoutTab{}, false } t := m.layouts.byID(m.shownLayout) @@ -243,7 +243,7 @@ func (m *model) closeLayout(ctx context.Context, id string) { func (m *model) toggleZoom() { m.mu.Lock() t := m.layouts.byID(m.shownLayout) - if t == nil || m.home { + if t == nil { m.mu.Unlock() return } @@ -459,7 +459,7 @@ func (m *model) movePane(ctx context.Context, source, target string, edge paneEd func (m *model) rotateFocused() bool { m.mu.Lock() t := m.layouts.byID(m.shownLayout) - if t == nil || m.home || !t.rotate(t.Focus) { + if t == nil || !t.rotate(t.Focus) { m.mu.Unlock() return false } From be0268b6fdad41f9ecf7fd9aa552d48267313ce3 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:30:48 -0300 Subject: [PATCH 07/25] fix(browser): keep the pane's frames at its size and its input from piling up A pane that grew (zoom, a dragged divider) got frames at the size its screencast started with, stretched over it: resizing a watched page now restarts the screencast at the new size. Chrome answers each pointer event in about 15ms, slower than a touchpad fires them, so the pane's queue merged nothing and lagged further behind with every event; moves and resizes that pile up behind a call in flight now keep the last one, and scrolls add up. Frames go out at JPEG quality 85, where text shows no ringing. --- internal/browser/chrome_view.go | 12 ++- internal/browser/view.go | 18 +++- internal/browser/view_test.go | 15 ++++ native/cmd/lich-native/browser_pane.go | 45 ++++------ native/cmd/lich-native/browser_queue.go | 88 ++++++++++++++++++++ native/cmd/lich-native/browser_queue_test.go | 53 ++++++++++++ 6 files changed, 198 insertions(+), 33 deletions(-) create mode 100644 native/cmd/lich-native/browser_queue.go create mode 100644 native/cmd/lich-native/browser_queue_test.go diff --git a/internal/browser/chrome_view.go b/internal/browser/chrome_view.go index 53c22edd0..22eeace63 100644 --- a/internal/browser/chrome_view.go +++ b/internal/browser/chrome_view.go @@ -13,9 +13,10 @@ import ( "github.com/lichdotdev/lich/api" ) -// screencastQuality is the JPEG quality of a pane's frames: legible text at a -// fraction of PNG's size, since every frame crosses /events as base64. -const screencastQuality = 70 +// screencastQuality is the JPEG quality of a pane's frames: text without +// visible ringing at a fraction of PNG's size, since every frame crosses +// /events as base64. +const screencastQuality = 85 // listeners are what a chromePage hands its CDP events to; the service sets // them once, right after the launch. @@ -84,7 +85,12 @@ func (p *chromePage) deliver(frame *page.EventScreencastFrame) { } } +// Screencast starts the frames at v, or restarts them there: Chrome keeps +// the size a screencast started with. func (p *chromePage) Screencast(ctx context.Context, v Viewport) error { + if err := p.run(ctx, page.StopScreencast()); err != nil { + return err + } if err := p.Resize(ctx, v); err != nil { return err } diff --git a/internal/browser/view.go b/internal/browser/view.go index ab5697209..e43400ba3 100644 --- a/internal/browser/view.go +++ b/internal/browser/view.go @@ -58,9 +58,23 @@ func (s *Service) Detach(owner string) error { return nil } -// Resize lays the page out again at the pane's new size. +// Resize lays the page out again at the pane's new size. A pane showing it +// gets its frames at that size too: the screencast keeps the size it started +// at, and the window would stretch the smaller frames over the larger pane. func (s *Service) Resize(owner string, v Viewport) error { - _, err := s.drive(owner, func(p Page, ctx context.Context) error { return p.Resize(ctx, v) }) + sess, err := s.live(owner) + if err != nil { + return err + } + s.mu.Lock() + watching := sess.watching + s.mu.Unlock() + _, err = s.drive(owner, func(p Page, ctx context.Context) error { + if watching { + return p.Screencast(ctx, v) + } + return p.Resize(ctx, v) + }) return err } diff --git a/internal/browser/view_test.go b/internal/browser/view_test.go index ede0ba9d8..7a7622d65 100644 --- a/internal/browser/view_test.go +++ b/internal/browser/view_test.go @@ -186,3 +186,18 @@ func TestNavRejectsBlockedURLAndUnknownAction(t *testing.T) { t.Fatal("nav accepted an unknown action") } } + +func TestResizeRestartsTheFramesOfAWatchedPage(t *testing.T) { + s, launch, _ := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + bigger := Viewport{W: 1600, H: 900, Scale: 1} + if err := s.Resize("s", bigger); err != nil { + t.Fatal(err) + } + p := launch.pages[0] + if !p.casting || p.viewports[len(p.viewports)-1] != bigger { + t.Fatalf("after resize casting=%v viewports=%v, want frames at %v", p.casting, p.viewports, bigger) + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index c87508dae..840ea581a 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "image" "log" "runtime" @@ -66,7 +67,7 @@ type browserPaneUI struct { held string lastPress time.Time presses int - input chan func() + queue *browserQueue onScreen, wasOnScreen bool } @@ -76,33 +77,22 @@ func (u *window) browserPane(session string) *browserPaneUI { } p, ok := u.browserPanes[session] if !ok { - p = &browserPaneUI{input: make(chan func(), 256)} - // One goroutine sends a pane's input, so a release never overtakes - // its press. - go func() { - for send := range p.input { - send() - } - }() + p = &browserPaneUI{queue: newBrowserQueue()} + go p.queue.run(u.ctx, func(err error) { log.Printf("browser of %s: %v", session, err) }) u.browserPanes[session] = p } return p } -// sendBrowser queues one call to the backend for session's page, logging -// what fails: the page answers on screen, not here. +// sendBrowser queues one call to the backend for session's page; a failure +// is logged, since the page answers on screen. func (u *window) sendBrowser(session, what string, call func(context.Context) error) { - p := u.browserPane(session) - send := func() { - if err := call(u.ctx); err != nil { - log.Printf("browser %s of %s: %v", what, session, err) + u.browserPane(session).queue.push(browserCall{send: func(ctx context.Context) error { + if err := call(ctx); err != nil { + return fmt.Errorf("%s: %w", what, err) } - } - select { - case p.input <- send: - default: - log.Printf("browser %s of %s: input queue full, dropped", what, session) - } + return nil + }}) } // browserPaneAt draws session's page pane in r, the way paneAt draws a @@ -267,12 +257,11 @@ func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame brow if want.W > 0 && want.H > 0 && want != p.view { attach := p.view.ID == "" p.view = want - u.sendBrowser(session, "attach", func(ctx context.Context) error { - if attach { - return u.client.AttachBrowser(ctx, want) - } - return u.client.ResizeBrowser(ctx, want) - }) + if attach { + u.sendBrowser(session, "attach", func(ctx context.Context) error { return u.client.AttachBrowser(ctx, want) }) + } else { + u.browserPane(session).queue.push(browserCall{resize: true, send: func(ctx context.Context) error { return u.client.ResizeBrowser(ctx, want) }}) + } } paint.FillShape(gtx.Ops, u.theme.Background, clip.Rect{Max: size}.Op()) if frame.size.X > 0 { @@ -362,7 +351,7 @@ func (u *window) browserPointer(gtx C, p *browserPaneUI, session string, scale f default: return } - u.sendBrowser(session, m.Type, func(ctx context.Context) error { return u.client.BrowserMouse(ctx, m) }) + u.browserPane(session).queue.push(browserCall{mouse: &m, send: func(ctx context.Context) error { return u.client.BrowserMouse(ctx, m) }}) } // settleBrowserPanes stops the frames of every page pane that was on screen diff --git a/native/cmd/lich-native/browser_queue.go b/native/cmd/lich-native/browser_queue.go new file mode 100644 index 000000000..1cc21dd84 --- /dev/null +++ b/native/cmd/lich-native/browser_queue.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "sync" + + "github.com/lichdotdev/lich/api" +) + +// browserQueue sends one pane's input to the backend in order, one call at +// a time. Chrome takes ~15ms to answer each pointer event (measured +// 2026-10-26), less often than a touchpad or a moving mouse fires them, so +// what piles up behind a call in flight is merged: moves and resizes keep +// the last one and scrolls add up. Presses, releases, keys and text are never +// merged. +type browserQueue struct { + mu sync.Mutex + calls []browserCall + wake chan struct{} +} + +// browserCall is one queued call: a pointer event to merge (mouse set), a +// resize to merge (resize set), or any other call. +type browserCall struct { + mouse *api.BrowserMouseOptions + resize bool + send func(context.Context) error +} + +func newBrowserQueue() *browserQueue { + return &browserQueue{wake: make(chan struct{}, 1)} +} + +func (q *browserQueue) push(c browserCall) { + q.mu.Lock() + if n := len(q.calls); n > 0 && c.resize && q.calls[n-1].resize { + q.calls[n-1] = c + q.mu.Unlock() + return + } + if n := len(q.calls); n > 0 && c.mouse != nil && q.calls[n-1].mouse != nil { + last := q.calls[n-1].mouse + switch { + case c.mouse.Type == "move" && last.Type == "move": + q.calls[n-1] = c + q.mu.Unlock() + return + case c.mouse.Type == "wheel" && last.Type == "wheel": + last.DX, last.DY = last.DX+c.mouse.DX, last.DY+c.mouse.DY + q.mu.Unlock() + return + } + } + q.calls = append(q.calls, c) + q.mu.Unlock() + select { + case q.wake <- struct{}{}: + default: + } +} + +// next takes the oldest call, false when none is queued. +func (q *browserQueue) next() (browserCall, bool) { + q.mu.Lock() + defer q.mu.Unlock() + if len(q.calls) == 0 { + return browserCall{}, false + } + c := q.calls[0] + q.calls = q.calls[1:] + return c, true +} + +// run sends the queued calls until ctx ends, reporting each failure to fail. +func (q *browserQueue) run(ctx context.Context, fail func(error)) { + for { + select { + case <-ctx.Done(): + return + case <-q.wake: + } + for c, ok := q.next(); ok; c, ok = q.next() { + if err := c.send(ctx); err != nil { + fail(err) + } + } + } +} diff --git a/native/cmd/lich-native/browser_queue_test.go b/native/cmd/lich-native/browser_queue_test.go new file mode 100644 index 000000000..3dad861cb --- /dev/null +++ b/native/cmd/lich-native/browser_queue_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "reflect" + "testing" + + "github.com/lichdotdev/lich/api" +) + +func TestBrowserQueueMergesWhatPilesUpBehindACall(t *testing.T) { + q := newBrowserQueue() + var sent []string + call := func(m api.BrowserMouseOptions) browserCall { + return browserCall{mouse: &m, send: func(context.Context) error { + sent = append(sent, m.Type) + return nil + }} + } + q.push(call(api.BrowserMouseOptions{Type: "move", X: 1})) + q.push(call(api.BrowserMouseOptions{Type: "move", X: 2})) + q.push(call(api.BrowserMouseOptions{Type: "press"})) + q.push(call(api.BrowserMouseOptions{Type: "wheel", DY: 10})) + q.push(call(api.BrowserMouseOptions{Type: "wheel", DY: 15})) + q.push(browserCall{send: func(context.Context) error { sent = append(sent, "key"); return nil }}) + q.push(call(api.BrowserMouseOptions{Type: "move", X: 3})) + + if len(q.calls) != 5 || q.calls[0].mouse.X != 2 || q.calls[2].mouse.DY != 25 { + t.Fatalf("queued %d calls: first move x=%v wheel dy=%v", len(q.calls), q.calls[0].mouse.X, q.calls[2].mouse.DY) + } + for c, ok := q.next(); ok; c, ok = q.next() { + if err := c.send(context.Background()); err != nil { + t.Fatal(err) + } + } + if want := []string{"move", "press", "wheel", "key", "move"}; !reflect.DeepEqual(sent, want) { + t.Fatalf("sent %v, want %v", sent, want) + } +} + +func TestBrowserQueueKeepsOnlyTheLastPendingResize(t *testing.T) { + q := newBrowserQueue() + var sizes []int + for w := 1; w <= 3; w++ { + q.push(browserCall{resize: true, send: func(context.Context) error { sizes = append(sizes, w); return nil }}) + } + for c, ok := q.next(); ok; c, ok = q.next() { + _ = c.send(context.Background()) + } + if !reflect.DeepEqual(sizes, []int{3}) { + t.Fatalf("sent sizes %v, want the last one", sizes) + } +} From 73fa72160c5d0eb57179c86067e7408d15b17a59 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:37:12 -0300 Subject: [PATCH 08/25] fix(browser): an input event Chrome leaves unanswered no longer closes the page The pane's mouse, keys, text, resize and toolbar waited up to 30 seconds on Chrome and, on any failure, dropped the browser as an agent action does. An input event dispatched while a navigation swaps the document can go unanswered, so one click on a link froze the pane for 30 seconds and then closed it. The window's calls now wait 2 seconds for input and leave the browser running unless its process is gone. --- internal/browser/actions.go | 5 +++-- internal/browser/fake_test.go | 4 ++++ internal/browser/rpc.go | 6 +++--- internal/browser/view.go | 33 ++++++++++++++++++++++++++++++--- internal/browser/view_test.go | 18 ++++++++++++++++++ 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/internal/browser/actions.go b/internal/browser/actions.go index f0d9e2078..6710058ac 100644 --- a/internal/browser/actions.go +++ b/internal/browser/actions.go @@ -182,8 +182,9 @@ func (s *Service) Scroll(id string, dy int) error { return s.act(id, func(p Page, ctx context.Context) error { return p.Scroll(ctx, dy) }) } -// act runs one agent action on its page; drive is the same for the window's -// own input, which lights nothing up. +// act runs one agent action on its page and tells the window the agent +// acted; drive runs it, dropping a browser whose page failed it. The +// window's own calls go through steer instead. func (s *Service) act(id string, fn func(Page, context.Context) error) error { sess, err := s.drive(id, fn) if err == nil { diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index 5df80a37a..111d02f39 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -32,6 +32,7 @@ type fakePage struct { casting bool viewports []Viewport mice []api.BrowserMouseOptions + mouseErr error keys []api.BrowserKeyOptions texts []string info PageInfo @@ -200,6 +201,9 @@ func (p *fakePage) Resize(_ context.Context, v Viewport) error { func (p *fakePage) Mouse(_ context.Context, m api.BrowserMouseOptions) error { p.mu.Lock() defer p.mu.Unlock() + if p.mouseErr != nil { + return p.mouseErr + } p.mice = append(p.mice, m) return nil } diff --git a/internal/browser/rpc.go b/internal/browser/rpc.go index 217c42f45..4e31e7a92 100644 --- a/internal/browser/rpc.go +++ b/internal/browser/rpc.go @@ -141,19 +141,19 @@ func (s *Service) ResizeView(_ context.Context, opts api.BrowserViewOptions) (ap // MouseInput passes one pointer event from the pane. func (s *Service) MouseInput(_ context.Context, opts api.BrowserMouseOptions) (api.None, error) { - _, err := s.drive(opts.ID, func(p Page, ctx context.Context) error { return p.Mouse(ctx, opts) }) + _, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error { return p.Mouse(ctx, opts) }) return api.None{}, err } // KeyInput passes one key from the pane. func (s *Service) KeyInput(_ context.Context, opts api.BrowserKeyOptions) (api.None, error) { - _, err := s.drive(opts.ID, func(p Page, ctx context.Context) error { return p.Key(ctx, opts) }) + _, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error { return p.Key(ctx, opts) }) return api.None{}, err } // TextInput types text the pane's input method committed. func (s *Service) TextInput(_ context.Context, opts api.BrowserTextOptions) (api.None, error) { - _, err := s.drive(opts.ID, func(p Page, ctx context.Context) error { return p.InsertText(ctx, opts.Text) }) + _, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error { return p.InsertText(ctx, opts.Text) }) return api.None{}, err } diff --git a/internal/browser/view.go b/internal/browser/view.go index e43400ba3..dbd74b5af 100644 --- a/internal/browser/view.go +++ b/internal/browser/view.go @@ -4,11 +4,18 @@ import ( "context" "errors" "fmt" + "time" "github.com/lichdotdev/lich/api" "github.com/lichdotdev/lich/internal/events" ) +// inputWait bounds one of the pane's input events. Chrome answers one in +// ~15ms, but an event dispatched while a navigation swaps the document can go +// unanswered (as Puppeteer has seen); the pane's queue waits on it, so it +// must give up long before pageWait. +const inputWait = 2 * time.Second + // errInWindow: the pane draws a headless browser, and this one moved to its // own window, where the person already sees it. var errInWindow = errors.New("this browser is open in its own window") @@ -48,7 +55,7 @@ func (s *Service) Attach(owner string, v Viewport) (Handle, error) { // Detach stops the frames of a pane that closed or went off screen. The // browser keeps running for the agent. func (s *Service) Detach(owner string) error { - sess, err := s.drive(owner, func(p Page, ctx context.Context) error { return p.StopScreencast(ctx) }) + sess, err := s.steer(owner, inputWait, func(p Page, ctx context.Context) error { return p.StopScreencast(ctx) }) if err != nil { return err } @@ -69,7 +76,7 @@ func (s *Service) Resize(owner string, v Viewport) error { s.mu.Lock() watching := sess.watching s.mu.Unlock() - _, err = s.drive(owner, func(p Page, ctx context.Context) error { + _, err = s.steer(owner, inputWait, func(p Page, ctx context.Context) error { if watching { return p.Screencast(ctx, v) } @@ -97,10 +104,30 @@ func (s *Service) Nav(owner, action, rawURL string) error { default: return fmt.Errorf("unknown browser action %q", action) } - _, err := s.drive(owner, fn) + _, err := s.steer(owner, pageWait, fn) return err } +// steer runs one of the window's own calls on owner's page, bounded by wait. +// Unlike an agent action, a call that fails leaves the browser running +// unless its process is gone: the person is still looking at the page. +func (s *Service) steer(owner string, wait time.Duration, fn func(Page, context.Context) error) (*session, error) { + sess, err := s.live(owner) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), wait) + defer cancel() + if err := fn(sess.page, ctx); err != nil { + if !sess.alive() { + s.drop(sess) + } + return nil, err + } + s.touch(sess) + return sess, nil +} + // Running is every browser this process runs, for a window catching up. func (s *Service) Running() []api.BrowserState { s.mu.Lock() diff --git a/internal/browser/view_test.go b/internal/browser/view_test.go index 7a7622d65..559ea1c40 100644 --- a/internal/browser/view_test.go +++ b/internal/browser/view_test.go @@ -1,6 +1,7 @@ package browser import ( + "context" "testing" "time" @@ -201,3 +202,20 @@ func TestResizeRestartsTheFramesOfAWatchedPage(t *testing.T) { t.Fatalf("after resize casting=%v viewports=%v, want frames at %v", p.casting, p.viewports, bigger) } } + +func TestWindowInputThatTimesOutKeepsTheBrowser(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + launch.pages[0].mouseErr = context.DeadlineExceeded + if _, err := s.MouseInput(t.Context(), api.BrowserMouseOptions{ID: "s", Type: "release", Button: "left"}); err == nil { + t.Fatal("a mouse event Chrome never answered reported success") + } + if launch.pages[0].closed { + t.Fatal("a page that missed one input event was closed") + } + if st := out.states(); !st[len(st)-1].Open { + t.Fatalf("last state %+v, want the browser still open", st[len(st)-1]) + } +} From 3a6f5349d9cf4bf943b92b252574c06be75e20d7 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:40:51 -0300 Subject: [PATCH 09/25] fix(browser): load a link's new tab in the page instead of freezing it A link with target=_blank, like GitHub's About link, opened a second tab in the headless browser, which put the pane's page in the background: there it takes no input and draws no frames, so every click after timed out. The page now watches the tabs it opens, closes each one once it has a URL, and loads that URL in itself. --- docs/ceilings.md | 2 +- internal/browser/chrome_live_test.go | 48 ++++++++++++++++++++++ internal/browser/chrome_view.go | 61 +++++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/docs/ceilings.md b/docs/ceilings.md index 9df20fbbd..a39a10c26 100644 --- a/docs/ceilings.md +++ b/docs/ceilings.md @@ -915,7 +915,7 @@ work when nobody knows it and that the call site never shows. The mechanism and whose page reaches the session's browser tab as a JPEG screencast over `/events`, at the pace the window takes frames. The native window cannot hold a webview: on Linux it speaks Wayland without GTK. So the pane has no DevTools, shows the arrow whatever cursor the page asks for, does not copy the page's selection to - the system clipboard (paste works), and leaves popups, downloads and the page's own dialogs out; Open in + the system clipboard (paste works), loads what the page opens in a new tab or window in the page itself, and leaves downloads and the page's own dialogs out; Open in a window moves the page to a real Chromium window for those, reopening it on the same profile with every cookie, since chromedp cannot show a headless process in place. A half-filled form or sessionStorage is lost on that move. Closing the card, or the session's process ending, closes the browser; closing that diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index 3674c834b..6f162b14c 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "os/exec" + "strings" "testing" "time" @@ -206,3 +207,50 @@ func TestChromeWindowInputTypesAndSubmits(t *testing.T) { t.Fatalf("form sent %q, want the clicked field's typed text", sent) } } + +func TestChromeLinkToANewTabLoadsInThePageAndKeepsItResponding(t *testing.T) { + pg, err := chromeLaunch(t.TempDir(), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := pg.Close(); err != nil { + t.Error(err) + } + }) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprintf(w, `%saway`, r.URL.Path) + })) + t.Cleanup(srv.Close) + p := pg.(*chromePage) + urls := make(chan string, 8) + p.Listen(func(string) {}, func(url, _ string) { urls <- url }) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL+"/"); err != nil { + t.Fatal(err) + } + if err := p.Screencast(ctx, Viewport{W: 400, H: 300, Scale: 1}); err != nil { + t.Fatal(err) + } + for _, kind := range []string{"press", "release"} { + if err := p.Mouse(ctx, api.BrowserMouseOptions{Type: kind, X: 20, Y: 20, Button: "left", Clicks: 1}); err != nil { + t.Fatal(err) + } + } + deadline := time.After(10 * time.Second) + for loaded := false; !loaded; { + select { + case u := <-urls: + loaded = strings.HasSuffix(u, "/elsewhere") + case <-deadline: + t.Fatal("the link's new tab never loaded in the page") + } + } + quick, stop := context.WithTimeout(ctx, inputWait) + defer stop() + if err := p.Mouse(quick, api.BrowserMouseOptions{Type: "move", X: 50, Y: 50}); err != nil { + t.Fatalf("the page stopped taking input after the link: %v", err) + } +} diff --git a/internal/browser/chrome_view.go b/internal/browser/chrome_view.go index 22eeace63..49ef95467 100644 --- a/internal/browser/chrome_view.go +++ b/internal/browser/chrome_view.go @@ -6,9 +6,11 @@ import ( "math" "sync" + "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/emulation" "github.com/chromedp/cdproto/input" "github.com/chromedp/cdproto/page" + "github.com/chromedp/cdproto/target" "github.com/chromedp/chromedp" "github.com/lichdotdev/lich/api" ) @@ -21,7 +23,9 @@ const screencastQuality = 85 // listeners are what a chromePage hands its CDP events to; the service sets // them once, right after the launch. type listeners struct { - mu sync.Mutex + mu sync.Mutex + // taken are the tabs this page opened that keepOneTab already took over. + taken sync.Map onFrame func(data string) onInfo func(url, title string) } @@ -42,6 +46,61 @@ func (p *chromePage) Listen(onFrame func(data string), onInfo func(url, title st go p.reportInfo() } }) + p.keepOneTab() +} + +// keepOneTab loads, in this page, what a link or a script opens in a new tab +// or window, and closes that tab. The pane shows only this page, and Chrome +// puts the one a new tab covers in the background, where it takes no input +// and draws no frames: one click on a target=_blank link froze the pane. +func (p *chromePage) keepOneTab() { + c := chromedp.FromContext(p.ctx) + self := c.Target.TargetID + discover := chromedp.ActionFunc(func(ctx context.Context) error { + return target.SetDiscoverTargets(true).Do(cdp.WithExecutor(ctx, c.Browser)) + }) + if err := p.run(nil, discover); err != nil { + slog.Warn("browser: watch for new tabs", "err", err) + return + } + chromedp.ListenBrowser(p.ctx, func(ev any) { + var info *target.Info + switch e := ev.(type) { + case *target.EventTargetCreated: + info = e.TargetInfo + case *target.EventTargetInfoChanged: + info = e.TargetInfo + } + // A new tab is created at about:blank and gets its URL after. + if info == nil || info.OpenerID != self || info.Type != "page" || info.URL == "" || info.URL == "about:blank" { + return + } + // Both the created and the info-changed event can name the tab. + if _, taken := p.listen.taken.LoadOrStore(info.TargetID, true); !taken { + go p.takeOver(info.TargetID, info.URL) + } + }) +} + +// takeOver closes the tab id this page opened and loads its url here. +func (p *chromePage) takeOver(id target.ID, url string) { + c := chromedp.FromContext(p.ctx) + closeTab := chromedp.ActionFunc(func(ctx context.Context) error { + return target.CloseTarget(id).Do(cdp.WithExecutor(ctx, c.Browser)) + }) + if err := p.run(nil, closeTab); err != nil { + slog.Warn("browser: close a tab the page opened", "url", url, "err", err) + } + if err := p.run(nil, page.BringToFront()); err != nil { + slog.Warn("browser: bring the page back to the front", "err", err) + } + if _, err := allowURL(url); err != nil { + slog.Warn("browser: a new tab's URL is not one the page may load", "url", url, "err", err) + return + } + if err := p.Navigate(context.Background(), url); err != nil && p.Alive() { + slog.Warn("browser: load a new tab's URL in the page", "url", url, "err", err) + } } // reportInfo reads the URL and title after a navigation: a link followed, a From 61f93299d7128bf9e614e916da456d3e6eba6c88 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:42:52 -0300 Subject: [PATCH 10/25] test(browser): back returns to the page a new-tab link left --- internal/browser/chrome_live_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index 6f162b14c..1f4743ace 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -253,4 +253,14 @@ func TestChromeLinkToANewTabLoadsInThePageAndKeepsItResponding(t *testing.T) { if err := p.Mouse(quick, api.BrowserMouseOptions{Type: "move", X: 50, Y: 50}); err != nil { t.Fatalf("the page stopped taking input after the link: %v", err) } + if err := p.Back(ctx); err != nil { + t.Fatalf("back after the link: %v", err) + } + var href string + if err := p.run(ctx, chromedp.Evaluate(`location.href`, &href)); err != nil { + t.Fatal(err) + } + if href != srv.URL+"/" { + t.Fatalf("back went to %q, want the page that held the link", href) + } } From 3f6272a215e9986ff04bac3c69d2eabb578755a2 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 20:49:38 -0300 Subject: [PATCH 11/25] fix(browser): tell the agent its browser shows in a tab, not a window browser_open said the browser was headless unless the user opened Browser on the card, and the MCP instructions called it the Chromium window the card opens, so an agent that read headed:false told the user no window would appear and asked them to open one. The tool, the instructions in every prompt language, lich browser's help and docs/cli.md now say the page shows in a tab beside the session, which lich offers to open. The pt-BR pane says Navegador, as the card item does. --- docs/cli.md | 10 +++++----- internal/cli/help.go | 2 +- internal/cli/mcp_browser.go | 19 ++++++++++--------- internal/prompt/en.go | 2 +- internal/prompt/es.go | 2 +- internal/prompt/ja.go | 2 +- internal/prompt/ko.go | 2 +- internal/prompt/pt_br.go | 2 +- internal/prompt/ru.go | 2 +- internal/prompt/zh_cn.go | 2 +- native/i18n/locales/pt-br/layouts.json | 10 +++++----- 11 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index e82908d4a..d8673adf6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -929,8 +929,8 @@ have to be a second accounting path, not a filter on this one. ### `lich browser [args]` -Drives the browser owned by the session this command runs in — the same -Chromium window the card's Browser item opens. It has to run inside a session +Drives the browser owned by the session this command runs in — the page its +tab in the lich window shows. It has to run inside a session (`LICH_SESSION_ID`); from anywhere else it refuses without calling lich. ``` @@ -947,9 +947,9 @@ lich browser list lich browser close ``` -`open` with no URL loads `about:blank`. A browser the card already showed is -reused; one the agent opened headless stays headless until the card asks for -the window. `info` describes the page and its clickable nodes and never +`open` with no URL loads `about:blank`. The browser runs headless and the user +sees its page in a tab beside the session; one that **Open in a window** moved +to a Chromium window of its own is reused there. `info` describes the page and its clickable nodes and never includes what was typed into a field. `screenshot` prints the PNG's path. URLs are http, https or `about:blank`. diff --git a/internal/cli/help.go b/internal/cli/help.go index c6c8a679c..20de3182b 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -135,7 +135,7 @@ var commands = []command{ { name: "browser", args: " [args]", - about: "Drive this session's browser — the same window the card's Browser item opens.\n" + + about: "Drive this session's browser — the page its tab in the lich window shows.\n" + "Verbs: open [url], info, click, type, press , screenshot [path], navigate ,\n" + "reload, back, forward, scroll, list, close.", }, diff --git a/internal/cli/mcp_browser.go b/internal/cli/mcp_browser.go index 10570a15e..708b51bf5 100644 --- a/internal/cli/mcp_browser.go +++ b/internal/cli/mcp_browser.go @@ -8,16 +8,17 @@ import ( "github.com/lichdotdev/lich/api" ) -// browserTools are this session's Chromium window — the same one the card's -// Browser item opens. Appended to mcpTools, so tools/list cannot drift from -// the commands. +// browserTools drive this session's browser, the page its tab in the lich +// window shows. Appended to mcpTools, so tools/list cannot drift from the +// commands. var browserTools = []mcpTool{ { Name: "browser_open", - Description: "Open a page in this lich session's browser. Creates the browser if " + - "this session has none (headless unless the user already opened Browser on " + - "this card). The URL must be http, https, or omitted (about:blank). file, " + - "javascript and data URLs are refused. This is not the lich window.", + Description: "Open a page in this lich session's browser, creating the browser if " + + "this session has none. It runs headless: the user sees the same page in a tab " + + "beside this session, which lich offers to open, so no window of its own is needed " + + "(headed is false, and that is expected). The URL must be http, https, or omitted " + + "(about:blank). file, javascript and data URLs are refused.", Schema: schema(map[string]any{ "url": property("string", "Page to load. Omit for about:blank."), }), @@ -200,8 +201,8 @@ var browserTools = []mcpTool{ }, { Name: "browser_close", - Description: "Close this session's browser. The lich window is untouched. Closing the " + - "session card also closes it.", + Description: "Close this session's browser, and with it the page's tab in the lich window. " + + "Closing the session card also closes it.", Schema: schema(map[string]any{}), Run: func(ctx context.Context, c *client, _ mcpArgs) (string, error) { return mcpBrowserID(ctx, c, api.BrowserClose) diff --git a/internal/prompt/en.go b/internal/prompt/en.go index 52de2dec6..3022be4ad 100644 --- a/internal/prompt/en.go +++ b/internal/prompt/en.go @@ -79,7 +79,7 @@ A subagent or workflow step running inside this session is not its agent, and li When a [lich] message carrying a ticket arrives at YOUR prompt, you are the worker: do the task, then answer with %s — a concise report (what was done, where, what remains), never a transcript. -This session has a browser of its own (browser_* tools): the same Chromium window the card's Browser item opens, not the lich window. Open it, read the page, click, type, press named keys, and screenshot. Screenshots are files. Never paste their bytes into a reply.`, +This session has a browser of its own (browser_* tools). It runs headless inside lich, and the user sees the same page in a tab beside this session: the card's Browser item opens it, and lich offers it with a notice when you open the browser. No window of its own appears, and none is needed: do not ask the user to open one. Open it, read the page, click, type, press named keys, and screenshot. Screenshots are files. Never paste their bytes into a reply.`, HandOverFailed: "The task did not reach it: %[1]v. The session is open, so hand it the task with " + "%[2]s once whatever that says is dealt with.", SendToSessionPrivate: "send_to_session with private", diff --git a/internal/prompt/es.go b/internal/prompt/es.go index 13052e961..00454dd09 100644 --- a/internal/prompt/es.go +++ b/internal/prompt/es.go @@ -78,7 +78,7 @@ Un subagente o paso de workflow que corre dentro de esta sesión no es su agente Cuando un mensaje [lich] con ticket llega a TU prompt, eres la sesión de trabajo: haz la tarea y responde con %s, un informe conciso (qué se hizo, dónde, qué falta), nunca una transcripción. -Esta sesión tiene un navegador propio (herramientas browser_*): la misma ventana Chromium que abre el elemento Navegador de la tarjeta, no la ventana de lich. Ábrelo, lee la página, haz clic, escribe, pulsa teclas con nombre y captura la pantalla. Las capturas son archivos. Nunca pegues sus bytes en la respuesta.`, +Esta sesión tiene un navegador propio (herramientas browser_*). Corre sin ventana dentro de lich, y el usuario ve la misma página en una pestaña junto a esta sesión: la abre el elemento Navegador de la tarjeta, y lich se la ofrece con un aviso cuando abres el navegador. No aparece ninguna ventana propia y no hace falta: no le pidas al usuario que abra una. Ábrelo, lee la página, haz clic, escribe, pulsa teclas con nombre y captura la pantalla. Las capturas son archivos. Nunca pegues sus bytes en la respuesta.`, HandOverFailed: "La tarea no le llegó: %[1]v. La sesión está abierta, así que entrégale la tarea con " + "%[2]s cuando resuelvas lo que eso dice.", SendToSessionPrivate: "send_to_session con private", diff --git a/internal/prompt/ja.go b/internal/prompt/ja.go index d190f599a..0fb36ff0c 100644 --- a/internal/prompt/ja.go +++ b/internal/prompt/ja.go @@ -78,7 +78,7 @@ var ja = Catalog{ チケット付きの [lich] メッセージがあなたのプロンプトに届いたら、あなたがワーカーです。タスクを実行し、%s で回答してください。回答は簡潔なレポート（何をしたか、どこで、何が残っているか）にし、やり取りの記録をそのまま貼らないでください。 -このセッションには専用のブラウザがあります（browser_* ツール）。カードのブラウザ項目が開く Chromium のウィンドウであり、lich のウィンドウではありません。開いて、ページを読み、クリックし、入力し、名前のあるキーを押し、スクリーンショットを撮ってください。スクリーンショットはファイルです。そのバイト列を返信に貼らないでください。`, +このセッションには専用のブラウザがあります（browser_* ツール）。lich の中でウィンドウなしで動き、ユーザーは同じページをこのセッションの隣のタブで見ます。カードのブラウザ項目がそのタブを開き、あなたがブラウザを開くと lich が通知でそれを勧めます。専用のウィンドウは現れず、必要もありません。ユーザーにウィンドウを開くよう頼まないでください。開いて、ページを読み、クリックし、入力し、名前のあるキーを押し、スクリーンショットを撮ってください。スクリーンショットはファイルです。そのバイト列を返信に貼らないでください。`, HandOverFailed: "タスクは届きませんでした：%[1]v。セッションは開いているので、その問題が片付いたら " + "%[2]s でタスクを渡してください。", SendToSessionPrivate: "private を指定した send_to_session", diff --git a/internal/prompt/ko.go b/internal/prompt/ko.go index 3987026d9..332f3e51a 100644 --- a/internal/prompt/ko.go +++ b/internal/prompt/ko.go @@ -74,7 +74,7 @@ var ko = Catalog{ 티켓이 붙은 [lich] 메시지가 당신의 프롬프트에 도착하면, 당신이 워커입니다. 작업을 하고 %s로 답하세요. 간결한 보고서(무엇을 했는지, 어디서, 무엇이 남았는지)로 답하고, 대화 기록은 보내지 마세요. -이 세션에는 자기 브라우저가 있습니다 (browser_* 도구). 카드의 브라우저 항목이 여는 Chromium 창이며, lich 창이 아닙니다. 열고, 페이지를 읽고, 클릭하고, 입력하고, 이름이 있는 키를 누르고, 스크린샷을 찍으세요. 스크린샷은 파일입니다. 그 바이트를 답변에 붙이지 마세요.`, +이 세션에는 자기 브라우저가 있습니다 (browser_* 도구). lich 안에서 창 없이 실행되고, 사용자는 같은 페이지를 이 세션 옆 탭에서 봅니다. 카드의 브라우저 항목이 그 탭을 열고, 당신이 브라우저를 열면 lich가 알림으로 보여 주기를 제안합니다. 별도의 창은 나타나지 않으며 필요하지도 않습니다. 사용자에게 창을 열어 달라고 하지 마세요. 열고, 페이지를 읽고, 클릭하고, 입력하고, 이름이 있는 키를 누르고, 스크린샷을 찍으세요. 스크린샷은 파일입니다. 그 바이트를 답변에 붙이지 마세요.`, HandOverFailed: "작업이 전달되지 않았습니다: %[1]v. 세션은 열려 있으니, 그 내용이 해결되면 " + "%[2]s로 작업을 넘기세요.", SendToSessionPrivate: "private를 지정한 send_to_session", diff --git a/internal/prompt/pt_br.go b/internal/prompt/pt_br.go index 081c6d31e..55393f417 100644 --- a/internal/prompt/pt_br.go +++ b/internal/prompt/pt_br.go @@ -78,7 +78,7 @@ Um subagente ou etapa de workflow rodando dentro desta sessão não é o agente Quando uma mensagem [lich] com ticket chega ao SEU prompt, você é a sessão de trabalho: faça a tarefa e responda com %s, um relatório conciso (o que foi feito, onde, o que falta), nunca uma transcrição. -Esta sessão tem um navegador próprio (ferramentas browser_*): a mesma janela Chromium que o item Navegador do card abre, não a janela do lich. Abra, leia a página, clique, digite, pressione teclas nomeadas e capture a tela. Capturas são arquivos. Nunca cole os bytes na resposta.`, +Esta sessão tem um navegador próprio (ferramentas browser_*). Ele roda sem janela dentro do lich, e o usuário vê a mesma página numa aba ao lado desta sessão: o item Navegador do card abre essa aba, e o lich oferece com um aviso quando você abre o navegador. Nenhuma janela própria aparece, e nenhuma é necessária: não peça ao usuário para abrir uma. Abra, leia a página, clique, digite, pressione teclas nomeadas e capture a tela. Capturas são arquivos. Nunca cole os bytes na resposta.`, HandOverFailed: "A tarefa não chegou a ela: %[1]v. A sessão está aberta, então entregue a tarefa com " + "%[2]s depois de resolver o que isso diz.", SendToSessionPrivate: "send_to_session com private", diff --git a/internal/prompt/ru.go b/internal/prompt/ru.go index fec373a04..47e6d4e03 100644 --- a/internal/prompt/ru.go +++ b/internal/prompt/ru.go @@ -87,7 +87,7 @@ var ru = Catalog{ Когда в ВАШ промпт приходит сообщение [lich] с тикетом, рабочая сессия это вы: выполните задачу, затем ответьте через %s кратким отчётом (что сделано, где, что осталось), а не стенограммой. -У этой сессии есть свой браузер (инструменты browser_*): то же окно Chromium, которое открывает пункт Браузер на карточке, а не окно lich. Откройте его, прочитайте страницу, нажмите, введите текст, нажмите именованную клавишу и сделайте снимок. Снимки — это файлы. Не вставляйте их байты в ответ.`, +У этой сессии есть свой браузер (инструменты browser_*). Он работает без окна внутри lich, а пользователь видит ту же страницу во вкладке рядом с этой сессией: её открывает пункт Браузер на карточке, а когда вы открываете браузер, lich предлагает её уведомлением. Отдельное окно не появляется и не нужно: не просите пользователя его открыть. Откройте браузер, прочитайте страницу, нажмите, введите текст, нажмите именованную клавишу и сделайте снимок. Снимки — это файлы. Не вставляйте их байты в ответ.`, HandOverFailed: "Задача до неё не дошла: %[1]v. Сессия открыта, поэтому передайте ей задачу через " + "%[2]s, как только разберётесь с тем, о чём говорит ошибка.", SendToSessionPrivate: "send_to_session с private", diff --git a/internal/prompt/zh_cn.go b/internal/prompt/zh_cn.go index 0b06628ac..55cbae39b 100644 --- a/internal/prompt/zh_cn.go +++ b/internal/prompt/zh_cn.go @@ -73,7 +73,7 @@ var zhCN = Catalog{ 当一条带工单的 [lich] 消息出现在你的提示处时，你就是工作者：完成任务，然后用 %s 回答，内容是简明的报告（做了什么、在哪里、还剩什么），不要贴完整记录。 -本会话有自己的浏览器（browser_* 工具）：卡片上的浏览器项打开的是同一个 Chromium 窗口，不是 lich 的窗口。打开它，阅读页面，点击，输入，按有名字的键，并截图。截图是文件。不要把字节贴进回复。`, +本会话有自己的浏览器（browser_* 工具）。它在 lich 内部无窗口运行，用户在本会话旁边的标签页中看到同一个页面：卡片上的浏览器项会打开该标签页，你打开浏览器时 lich 也会用通知提示。不会出现单独的窗口，也不需要：不要让用户去打开窗口。打开它，阅读页面，点击，输入，按有名字的键，并截图。截图是文件。不要把字节贴进回复。`, HandOverFailed: "任务没有送达：%[1]v。会话已打开，等上述问题处理完后，" + "请用 %[2]s 把任务交给它。", SendToSessionPrivate: "带 private 的 send_to_session", diff --git a/native/i18n/locales/pt-br/layouts.json b/native/i18n/locales/pt-br/layouts.json index 9bad37a17..1e019e6fe 100644 --- a/native/i18n/locales/pt-br/layouts.json +++ b/native/i18n/locales/pt-br/layouts.json @@ -16,19 +16,19 @@ } }, "browser": { - "title": "Browser", + "title": "Navegador", "agent": "agente", - "close": "Fechar o browser", + "close": "Fechar o navegador", "address": "Digite uma URL", "back": "Voltar", "forward": "Avançar", "reload": "Recarregar", "openInWindow": "Abrir em janela", - "locked": "Esta aba é só de {label} e do browser dela", - "opened": "{label} abriu um browser", + "locked": "Esta aba é só de {label} e do navegador dela", + "opened": "{label} abriu um navegador", "dismiss": "Dispensar", "open": "Abrir", - "moveBody": "Ela sai de \"{layout}\". A aba do browser só comporta {label} e o browser dela." + "moveBody": "Ela sai de \"{layout}\". A aba do navegador só comporta {label} e o navegador dela." }, "split": { "noRoom": "Não há espaço para outro painel aqui. Alargue a janela ou tire um painel antes." From 788ecfd2b741975e8ef6c143541ebcedc785329f Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:15:18 -0300 Subject: [PATCH 12/25] feat(native): show the agent's clicks with lich's own cursor The pane marked an agent's click with a small dot that was easy to miss. lich's own arrow now glides to each click, dips and rings, and fades two seconds after the agent's last action, with a badge naming the provider that clicked. It takes the session's card color, or the theme's primary for a card without one, outlined in primary-foreground so it reads over any page. The header's agent chip goes: the badge says who is acting. --- native/cmd/lich-native/browser_cursor.go | 212 ++++++++++++++++++ native/cmd/lich-native/browser_cursor_test.go | 111 +++++++++ native/cmd/lich-native/browser_pane.go | 66 +++--- native/i18n/locales/en/layouts.json | 1 - native/i18n/locales/es/layouts.json | 1 - native/i18n/locales/ja/layouts.json | 1 - native/i18n/locales/ko/layouts.json | 1 - native/i18n/locales/pt-br/layouts.json | 1 - native/i18n/locales/ru/layouts.json | 1 - native/i18n/locales/zh-cn/layouts.json | 1 - 10 files changed, 358 insertions(+), 38 deletions(-) create mode 100644 native/cmd/lich-native/browser_cursor.go create mode 100644 native/cmd/lich-native/browser_cursor_test.go diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go new file mode 100644 index 000000000..eddacc3d0 --- /dev/null +++ b/native/cmd/lich-native/browser_cursor.go @@ -0,0 +1,212 @@ +package main + +import ( + "image" + "image/color" + "math" + "time" + + "gioui.org/f32" + "gioui.org/font" + "gioui.org/layout" + "gioui.org/op" + "gioui.org/op/clip" + "gioui.org/op/paint" + + "github.com/lichdotdev/lich/native/ui" + "github.com/lichdotdev/lich/native/ui/icons" +) + +// The agent's pointer over its page: lich's own arrow, in the session's card +// color or the theme's primary, outlined in primary-foreground so it reads +// over any page, with a badge naming the provider. It glides to each click, +// dips, rings, and fades once the agent goes quiet. Timings from the approved +// mockup (2026-10-26). +const ( + cursorGlide = 280 * time.Millisecond + cursorDip = 170 * time.Millisecond + cursorRing = 520 * time.Millisecond + cursorFade = 300 * time.Millisecond + // cursorDipScale is how far the arrow shrinks at the bottom of its dip. + cursorDipScale = 0.14 +) + +// cursorShadow is the drop shadow under the arrow, the mockup's 35% black. +var cursorShadow = color.NRGBA{A: 90} + +// cursorEase is the glide's curve, CSS's cubic-bezier(.2,.8,.2,1). +var cursorEase = ui.CubicBezier(0.2, 0.8, 0.2, 1) + +// agentCursor is where the agent's pointer is, in CSS pixels of the page. +type agentCursor struct { + shown bool + from, to f32.Point + // moved is when the glide to to began; last is the agent's last action, + // which the fade counts from; seen is the last mark taken. + moved, last, seen time.Time +} + +// take moves the cursor for a new mark: a click glides it to its point, any +// other action only keeps it up. +func (c *agentCursor) take(m browserMark) { + if m.at.IsZero() || !m.at.After(c.seen) { + return + } + c.seen, c.last = m.at, m.at + if m.x == nil || m.y == nil { + return + } + to := f32.Pt(float32(*m.x), float32(*m.y)) + from := to + if c.shown { + from = c.position(m.at) + } + c.from, c.to, c.moved, c.shown = from, to, m.at, true +} + +func (c *agentCursor) position(now time.Time) f32.Point { + t := cursorEase(progress(now.Sub(c.moved), cursorGlide)) + return c.from.Add(c.to.Sub(c.from).Mul(t)) +} + +// alpha is the cursor's opacity at now: whole until agentGlow after the last +// action, then fading out over cursorFade. +func (c *agentCursor) alpha(now time.Time) float32 { + if !c.shown { + return 0 + } + return 1 - progress(now.Sub(c.last)-agentGlow, cursorFade) +} + +// landed is how long ago the glide reached the point, negative while it +// still moves. +func (c *agentCursor) landed(now time.Time) time.Duration { + return now.Sub(c.moved) - cursorGlide +} + +// settled is when nothing about the cursor changes any more until it fades. +func (c *agentCursor) settled() time.Time { + return c.moved.Add(cursorGlide + cursorRing) +} + +func progress(d, span time.Duration) float32 { + return float32(min(max(float64(d)/float64(span), 0), 1)) +} + +// cursorArrow is lich's arrow in a 24-unit box, its tip at cursorTip: rounder +// and heavier than the system's, so it is not taken for the person's own. +var cursorArrow = []f32.Point{ + f32.Pt(4.2, 2.6), f32.Pt(19.4, 10.4), f32.Pt(20.6, 11), f32.Pt(19.3, 11.6), + f32.Pt(13.2, 13.6), f32.Pt(10.6, 19.6), f32.Pt(10, 20.9), f32.Pt(9.4, 19.6), +} + +var cursorTip = f32.Pt(4, 3) + +func arrowPath(ops *op.Ops, unit float32) clip.PathSpec { + p := func(i int) f32.Point { return cursorArrow[i].Sub(cursorTip).Mul(unit) } + var path clip.Path + path.Begin(ops) + path.MoveTo(p(0)) + path.LineTo(p(1)) + path.QuadTo(p(2), p(3)) + path.LineTo(p(4)) + path.LineTo(p(5)) + path.QuadTo(p(6), p(7)) + path.Close() + return path.End() +} + +// drawAgentCursor draws c over the page, scale being device pixels per CSS +// pixel, in tint with outline around it, the badge naming provider with its +// icon. It asks for the frames its motion needs. +func (u *window) drawAgentCursor(gtx C, c *agentCursor, scale float32, tint, outline color.NRGBA, provider string, icon *icons.Icon) { + now := gtx.Now + a := c.alpha(now) + if a <= 0 { + c.shown = false + return + } + at := c.position(now).Mul(scale) + landed := c.landed(now) + defer paint.PushOpacity(gtx.Ops, a).Pop() + if landed >= 0 && landed < cursorRing { + p := progress(landed, cursorRing) + r := float32(gtx.Dp(17)) * cursorEase(p) + ring := clip.Ellipse{Min: image.Pt(int(at.X-r), int(at.Y-r)), Max: image.Pt(int(at.X+r), int(at.Y+r))} + paint.FillShape(gtx.Ops, ui.Alpha(tint, 0.95*(1-p)), clip.Stroke{Path: ring.Path(gtx.Ops), Width: float32(gtx.Dp(2))}.Op()) + } + s := float32(1) + if landed >= 0 && landed < cursorDip { + s -= cursorDipScale * float32(math.Sin(math.Pi*float64(progress(landed, cursorDip)))) + } + unit := float32(gtx.Dp(1)) + place := op.Affine(f32.Affine2D{}.Scale(f32.Point{}, f32.Pt(s, s)).Offset(at)).Push(gtx.Ops) + shadow := op.Offset(image.Pt(0, gtx.Dp(1))).Push(gtx.Ops) + paint.FillShape(gtx.Ops, cursorShadow, clip.Outline{Path: arrowPath(gtx.Ops, unit)}.Op()) + shadow.Pop() + paint.FillShape(gtx.Ops, tint, clip.Outline{Path: arrowPath(gtx.Ops, unit)}.Op()) + paint.FillShape(gtx.Ops, outline, clip.Stroke{Path: arrowPath(gtx.Ops, unit), Width: 1.6 * unit}.Op()) + place.Pop() + badge := op.Offset(image.Pt(int(at.X)+gtx.Dp(12), int(at.Y)+gtx.Dp(15))).Push(gtx.Ops) + u.cursorBadge(gtx, tint, outline, provider, icon) + badge.Pop() + switch { + case now.Before(c.settled()) || c.last.Add(agentGlow).Before(now): + gtx.Execute(op.InvalidateCmd{}) + default: + gtx.Execute(op.InvalidateCmd{At: c.last.Add(agentGlow)}) + } +} + +// cursorBadge is the pill beside the arrow: the provider's icon and name in +// ink on tint, outlined like the arrow. +func (u *window) cursorBadge(gtx C, tint, outline color.NRGBA, provider string, icon *icons.Icon) { + th := u.theme.On(tint) + fg := inkOn(tint, outline) + gtx.Constraints.Min = image.Point{} + rec := op.Record(gtx.Ops) + d := layout.Inset{Left: ui.Space(1.25), Right: ui.Space(2)}.Layout(gtx, func(gtx C) D { + gtx.Constraints.Min.Y = gtx.Dp(20) + return layout.Flex{Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(func(gtx C) D { return icon.Layout(gtx, ui.Space(3), fg) }), + ui.Gap(1), + layout.Rigid(th.Text(ui.TextXS, provider).In(fg).Weight(font.Medium).Layout), + ) + }) + call := rec.Stop() + pill := clip.UniformRRect(image.Rectangle{Max: d.Size}, d.Size.Y/2) + paint.FillShape(gtx.Ops, tint, pill.Op(gtx.Ops)) + paint.FillShape(gtx.Ops, outline, clip.Stroke{Path: pill.Path(gtx.Ops), Width: float32(gtx.Dp(1.5))}.Op()) + call.Add(gtx.Ops) +} + +// inkOn is the text color on tint: the theme's primary-foreground, which is +// made for the theme's primary, unless tint is a card color it does not read +// on, where black or white is, by luminance. +func inkOn(tint, themed color.NRGBA) color.NRGBA { + if contrast(tint, themed) >= 3 { + return themed + } + if luminance(tint) > 0.4 { + return color.NRGBA{A: 255} + } + return color.NRGBA{R: 255, G: 255, B: 255, A: 255} +} + +// contrast is WCAG's contrast ratio of two opaque colors. +func contrast(a, b color.NRGBA) float64 { + la, lb := luminance(a), luminance(b) + return (max(la, lb) + 0.05) / (min(la, lb) + 0.05) +} + +// luminance is WCAG's relative luminance of an opaque sRGB color. +func luminance(c color.NRGBA) float64 { + lin := func(v uint8) float64 { + s := float64(v) / 255 + if s <= 0.04045 { + return s / 12.92 + } + return math.Pow((s+0.055)/1.055, 2.4) + } + return 0.2126*lin(c.R) + 0.7152*lin(c.G) + 0.0722*lin(c.B) +} diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go new file mode 100644 index 000000000..1e52425a4 --- /dev/null +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "image" + "image/color" + "testing" + "time" + + "gioui.org/f32" + "gioui.org/font/gofont" + "gioui.org/layout" + "gioui.org/op" + "gioui.org/text" + "gioui.org/unit" + + "github.com/lichdotdev/lich/native/ui" +) + +func clickMark(at time.Time, x, y float64) browserMark { return browserMark{at: at, x: &x, y: &y} } + +func TestAgentCursorAppearsAtItsFirstClickAndGlidesToTheNext(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + c.take(clickMark(t0, 50, 40)) + if !c.shown || c.position(t0) != f32.Pt(50, 40) { + t.Fatalf("first click at %v, shown %v", c.position(t0), c.shown) + } + t1 := t0.Add(time.Second) + c.take(clickMark(t1, 150, 40)) + mid := c.position(t1.Add(cursorGlide / 2)) + if mid.X <= 50 || mid.X >= 150 { + t.Fatalf("halfway through the glide at %v, want between the two clicks", mid) + } + if got := c.position(t1.Add(cursorGlide)); got != f32.Pt(150, 40) { + t.Fatalf("after the glide at %v, want the new click", got) + } +} + +func TestAgentCursorGlidesFromWhereItIsWhenAClickInterruptsAGlide(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + c.take(clickMark(t0, 0, 0)) + t1 := t0.Add(time.Second) + c.take(clickMark(t1, 100, 0)) + t2 := t1.Add(cursorGlide / 2) + was := c.position(t2) + c.take(clickMark(t2, 100, 100)) + if c.from != was { + t.Fatalf("new glide starts at %v, want where the cursor was, %v", c.from, was) + } +} + +func TestAgentCursorStaysUpWhileTheAgentActsAndThenFades(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + c.take(clickMark(t0, 10, 10)) + c.take(browserMark{at: t0.Add(time.Second)}) + if a := c.alpha(t0.Add(time.Second + agentGlow - time.Millisecond)); a != 1 { + t.Fatalf("alpha %v before the glow ends, want 1: an action without a point keeps it up", a) + } + if a := c.alpha(t0.Add(time.Second + agentGlow + cursorFade)); a != 0 { + t.Fatalf("alpha %v after the fade, want 0", a) + } + if c.position(t0.Add(2*time.Second)) != f32.Pt(10, 10) { + t.Fatal("an action without a point moved the cursor") + } +} + +func TestAgentCursorTakesEachMarkOnce(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + m := clickMark(t0, 10, 10) + c.take(m) + c.take(clickMark(t0.Add(time.Second), 90, 90)) + c.take(m) + if c.to != f32.Pt(90, 90) { + t.Fatalf("an old mark taken again moved the cursor to %v", c.to) + } +} + +func TestInkOnKeepsTheThemesPairAndReadsOnCardColors(t *testing.T) { + white, black := color.NRGBA{R: 255, G: 255, B: 255, A: 255}, color.NRGBA{A: 255} + if got := inkOn(ui.Dark.Primary, ui.Dark.PrimaryForeground); got != ui.Dark.PrimaryForeground { + t.Fatalf("on the dark theme's primary, ink %v, want its primary-foreground", got) + } + for name, c := range cardColors { + ink := inkOn(c, ui.Dark.PrimaryForeground) + if ink != white && ink != black && ink != ui.Dark.PrimaryForeground { + t.Fatalf("%s: ink %v", name, ink) + } + if contrast(c, ink) < 3 { + t.Errorf("%s: badge text contrast %.2f, want at least 3", name, contrast(c, ink)) + } + } +} + +func TestDrawAgentCursorLaysOutWithoutAPage(t *testing.T) { + th := &ui.Theme{Palette: ui.Dark, Shaper: text.NewShaper(text.WithCollection(gofont.Collection())), Surface: ui.Dark.Canvas} + u := &window{theme: th} + var c agentCursor + t0 := time.Unix(100, 0) + c.take(clickMark(t0, 40, 30)) + for _, d := range []time.Duration{0, cursorGlide, cursorGlide + cursorDip/2, agentGlow + cursorFade/2, agentGlow + cursorFade} { + gtx := layout.Context{Ops: new(op.Ops), Now: t0.Add(d), Metric: unit.Metric{PxPerDp: 1, PxPerSp: 1}, + Constraints: layout.Exact(image.Pt(400, 300))} + u.drawAgentCursor(gtx, &c, 1, th.Primary, th.PrimaryForeground, "Claude Code", providerIcon("claude")) + } + if c.shown { + t.Fatal("cursor still shown after it faded out") + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index 840ea581a..72bbdd057 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "image" + "image/color" "log" "runtime" "strings" @@ -37,11 +38,10 @@ var ( icBrowserForward = icons.Lucide("arrow-right") icBrowserReload = icons.Lucide("rotate-cw") icBrowserWindow = icons.Lucide("external-link") - icBrowserAgent = icons.Lucide("bot") ) -// agentGlow is how long the agent chip and its click mark stay up after the -// agent's last action on the page. +// agentGlow is how long the agent's cursor stays up after its last action on +// the page, before it fades. const agentGlow = 2 * time.Second // doubleClick is the window two presses of a button count as one double @@ -68,6 +68,7 @@ type browserPaneUI struct { lastPress time.Time presses int queue *browserQueue + cursor agentCursor onScreen, wasOnScreen bool } @@ -113,7 +114,7 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta head, bar := gtx.Dp(paneHeader), gtx.Dp(ui.Space(9)) header := gtx header.Constraints = layout.Exact(image.Pt(size.X, head)) - u.browserHeader(header, p, pane, focused, gtx.Now.Sub(mark.at) < agentGlow) + u.browserHeader(header, p, pane, focused) line := th.Over(th.Border) paint.FillShape(gtx.Ops, line, clip.Rect{Min: image.Pt(0, head-gtx.Dp(1)), Max: image.Pt(size.X, head)}.Op()) tool := op.Offset(image.Pt(0, head)).Push(gtx.Ops) @@ -125,7 +126,8 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta page := gtx page.Constraints = layout.Exact(image.Pt(size.X, max(size.Y-head-bar, 0))) off := op.Offset(image.Pt(0, head+bar)).Push(gtx.Ops) - u.browserPage(page, p, session, frame, mark, focused) + p.cursor.take(mark) + u.browserPage(page, p, session, frame, focused) if several && !focused { paint.FillShape(gtx.Ops, ui.Alpha(th.Background, paneDim), clip.Rect{Max: page.Constraints.Max}.Op()) } @@ -138,14 +140,10 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta border, width = th.Ring, float32(gtx.Dp(1.5)) } paint.FillShape(gtx.Ops, border, clip.Stroke{Path: shape.Path(gtx.Ops), Width: width}.Op()) - if gtx.Now.Sub(mark.at) < agentGlow { - gtx.Execute(op.InvalidateCmd{At: mark.at.Add(agentGlow)}) - } } -// browserHeader is the page pane's top row: its name, the agent chip while -// the agent acts on the page, and zoom and close. -func (u *window) browserHeader(gtx C, p *browserPaneUI, pane string, focused, agent bool) { +// browserHeader is the page pane's top row: its name, zoom and close. +func (u *window) browserHeader(gtx C, p *browserPaneUI, pane string, focused bool) { th := u.theme if p.zoom.Clicked(gtx) { u.layoutStage.focus, u.layoutStage.zoom = pane, true @@ -163,19 +161,6 @@ func (u *window) browserHeader(gtx C, p *browserPaneUI, pane string, focused, ag layout.Rigid(func(gtx C) D { return icBrowser.Layout(gtx, ui.Space(3.5), th.MutedForeground) }), ui.Gap(2), layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.title")).In(fg).Weight(weight).Layout), - ui.Gap(2), - layout.Rigid(func(gtx C) D { - if !agent { - return D{} - } - return pill(gtx, th, func(gtx C) D { - return layout.Flex{Alignment: layout.Middle}.Layout(gtx, - layout.Rigid(func(gtx C) D { return icBrowserAgent.Layout(gtx, ui.Space(3), th.Foreground) }), - ui.Gap(1), - layout.Rigid(th.Text(ui.TextXS, i18n.T("layouts.browser.agent")).Layout), - ) - }, ui.Space(2), ui.Space(2)) - }), layout.Flexed(1, func(gtx C) D { return D{Size: image.Pt(gtx.Constraints.Max.X, 0)} }), layout.Rigid(func(gtx C) D { zoom := th.Tooltip(&u.root, &p.zoomTip, i18n.T("layouts.pane.zoom")) @@ -250,7 +235,7 @@ func (u *window) browserToolbar(gtx C, p *browserPaneUI, session string, st api. // browserPage draws the page's last frame over gtx's area, keeps the // backend's page the area's size, and passes the pointer and keys on. -func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame browserFrame, mark browserMark, focused bool) { +func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame browserFrame, focused bool) { size := gtx.Constraints.Max scale := gtx.Metric.PxPerDp want := api.BrowserViewOptions{ID: session, W: int(float32(size.X) / scale), H: int(float32(size.Y) / scale), Scale: float64(scale)} @@ -274,12 +259,9 @@ func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame brow paint.PaintOp{}.Add(gtx.Ops) t.Pop() } - if gtx.Now.Sub(mark.at) < agentGlow && mark.x != nil && mark.y != nil { - at := image.Pt(int(float32(*mark.x)*scale), int(float32(*mark.y)*scale)) - r := gtx.Dp(ui.Space(2)) - dot := clip.Ellipse{Min: at.Sub(image.Pt(r, r)), Max: at.Add(image.Pt(r, r))} - paint.FillShape(gtx.Ops, ui.Alpha(u.theme.Ring, 0.35), dot.Op(gtx.Ops)) - paint.FillShape(gtx.Ops, u.theme.Ring, clip.Stroke{Path: dot.Path(gtx.Ops), Width: float32(gtx.Dp(1.5))}.Op()) + if p.cursor.shown { + tint, provider, icon := u.agentLook(session) + u.drawAgentCursor(gtx, &p.cursor, scale, tint, u.theme.PrimaryForeground, provider, icon) } u.browserInput(gtx, p, session, scale, focused) defer clip.Rect{Max: size}.Push(gtx.Ops).Pop() @@ -424,3 +406,25 @@ func isLocalHost(text string) bool { } return host == "localhost" || host == "127.0.0.1" || host == "[::1]" || strings.HasSuffix(host, ".localhost") } + +// agentLook is how session's agent cursor is drawn: in its card color, or the +// theme's primary for a card without one, naming the provider that runs in it. +func (u *window) agentLook(session string) (tint color.NRGBA, provider string, icon *icons.Icon) { + tint = u.theme.Primary + sv, ok := u.model.sessionViewOf(session) + if !ok { + return tint, "", providerIcon("") + } + if c, tinted := cardColors[sv.Color]; tinted { + tint = c + } + kind := sv.Kind + if sv.agent != "" { + kind = sv.agent + } + provider = kind + if info, found := u.model.providers.find(kind); found && info.name != "" { + provider = info.name + } + return tint, provider, providerIcon(kind) +} diff --git a/native/i18n/locales/en/layouts.json b/native/i18n/locales/en/layouts.json index 245d2849e..7c74696e6 100644 --- a/native/i18n/locales/en/layouts.json +++ b/native/i18n/locales/en/layouts.json @@ -17,7 +17,6 @@ }, "browser": { "title": "Browser", - "agent": "agent", "close": "Close the browser", "address": "Type a URL", "back": "Back", diff --git a/native/i18n/locales/es/layouts.json b/native/i18n/locales/es/layouts.json index 9c54414dd..69a04a06a 100644 --- a/native/i18n/locales/es/layouts.json +++ b/native/i18n/locales/es/layouts.json @@ -17,7 +17,6 @@ }, "browser": { "title": "Navegador", - "agent": "agente", "close": "Cerrar el navegador", "address": "Escribe una URL", "back": "Atrás", diff --git a/native/i18n/locales/ja/layouts.json b/native/i18n/locales/ja/layouts.json index 8df3eb23a..d9cb24ffe 100644 --- a/native/i18n/locales/ja/layouts.json +++ b/native/i18n/locales/ja/layouts.json @@ -15,7 +15,6 @@ }, "browser": { "title": "ブラウザ", - "agent": "エージェント", "close": "ブラウザを閉じる", "address": "URL を入力", "back": "戻る", diff --git a/native/i18n/locales/ko/layouts.json b/native/i18n/locales/ko/layouts.json index b59dc600a..93ec15d37 100644 --- a/native/i18n/locales/ko/layouts.json +++ b/native/i18n/locales/ko/layouts.json @@ -15,7 +15,6 @@ }, "browser": { "title": "브라우저", - "agent": "에이전트", "close": "브라우저 닫기", "address": "URL 입력", "back": "뒤로", diff --git a/native/i18n/locales/pt-br/layouts.json b/native/i18n/locales/pt-br/layouts.json index 1e019e6fe..90669962d 100644 --- a/native/i18n/locales/pt-br/layouts.json +++ b/native/i18n/locales/pt-br/layouts.json @@ -17,7 +17,6 @@ }, "browser": { "title": "Navegador", - "agent": "agente", "close": "Fechar o navegador", "address": "Digite uma URL", "back": "Voltar", diff --git a/native/i18n/locales/ru/layouts.json b/native/i18n/locales/ru/layouts.json index 21984156b..b7349d599 100644 --- a/native/i18n/locales/ru/layouts.json +++ b/native/i18n/locales/ru/layouts.json @@ -21,7 +21,6 @@ }, "browser": { "title": "Браузер", - "agent": "агент", "close": "Закрыть браузер", "address": "Введите URL", "back": "Назад", diff --git a/native/i18n/locales/zh-cn/layouts.json b/native/i18n/locales/zh-cn/layouts.json index a7639266b..bc9a4cfda 100644 --- a/native/i18n/locales/zh-cn/layouts.json +++ b/native/i18n/locales/zh-cn/layouts.json @@ -15,7 +15,6 @@ }, "browser": { "title": "浏览器", - "agent": "智能体", "close": "关闭浏览器", "address": "输入网址", "back": "后退", From 3493d82fb52f4f758bd61ecefa7b9e714281d740 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:26:35 -0300 Subject: [PATCH 13/25] feat(browser): show where the agent scrolls, types and presses An agent's scroll was window.scrollBy, with no point to show; it is now a wheel turn where the agent last acted, or the viewport's middle before it acted anywhere, which also scrolls what is under that point the way a person's wheel does. Typing into or pressing on an element reports the element it clicked first. browser-agent carries the action and a scroll's distance, and the pane's cursor glides to each of them: a click, a type or a press dips and rings, a scroll slides two chevrons in its direction. --- api/browser.go | 14 ++-- api/testdata/contract.golden | 2 + internal/browser/actions.go | 69 ++++++++++++++++--- internal/browser/browser.go | 9 ++- internal/browser/chrome.go | 13 +++- internal/browser/chrome_live_test.go | 35 ++++++++++ internal/browser/fake_test.go | 9 ++- internal/browser/page.go | 4 +- internal/browser/view.go | 25 ++++++- internal/browser/view_test.go | 61 ++++++++++++++++ native/cmd/lich-native/browser_cursor.go | 54 +++++++++++++-- native/cmd/lich-native/browser_cursor_test.go | 14 ++++ native/cmd/lich-native/browser_model.go | 11 +-- 13 files changed, 286 insertions(+), 34 deletions(-) diff --git a/api/browser.go b/api/browser.go index 4297c9513..6cdd9ccea 100644 --- a/api/browser.go +++ b/api/browser.go @@ -54,12 +54,16 @@ type BrowserState struct { Title string `json:"title"` } -// BrowserAgentAct is one agent action on its page. X and Y are where it -// clicked, in CSS pixels, when the action was a click. +// BrowserAgentAct is one agent action on its page. Action is click, type, +// press or scroll, "" for one that touches no point (reading the page, a +// screenshot, a navigation). X and Y are where it acted, in CSS pixels, when +// it acted at a point; DY is a scroll's distance, positive down. type BrowserAgentAct struct { - Owner string `json:"owner"` - X *float64 `json:"x,omitempty"` - Y *float64 `json:"y,omitempty"` + Owner string `json:"owner"` + Action string `json:"action,omitempty"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` + DY int `json:"dy,omitempty"` } // BrowserViewOptions sizes a session's page to the pane showing it: W and H in diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index ab96f4231..782da23eb 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -1507,8 +1507,10 @@ BranchPullRequest { BrowserAgentAct { owner string + action,omitempty string x,omitempty *float64 y,omitempty *float64 + dy,omitempty int } ProfileLogin { diff --git a/internal/browser/actions.go b/internal/browser/actions.go index 6710058ac..7025b8557 100644 --- a/internal/browser/actions.go +++ b/internal/browser/actions.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "path/filepath" + + "github.com/lichdotdev/lich/api" ) var errNoBrowser = fmt.Errorf("no browser for this session — open one first") @@ -65,7 +67,7 @@ func (s *Service) Info(id string) (PageInfo, error) { return PageInfo{}, err } s.remember(sess, info.URL, info.Title) - s.agentActed(sess, nil, nil) + s.agentActed(sess, api.BrowserAgentAct{}) return info, nil } @@ -89,15 +91,19 @@ func (s *Service) Click(id string, t Target) error { return err } s.touch(sess) - s.agentActed(sess, &x, &y) + s.agentActed(sess, api.BrowserAgentAct{Action: "click", X: &x, Y: &y}) return nil } // Type sends keystrokes. clear selects-and-replaces; the typed text never // comes back through Info. func (s *Service) Type(id, text string, clear bool, t Target) error { - return s.act(id, func(p Page, ctx context.Context) error { - return p.Type(ctx, text, clear, t) + return s.actAt(id, api.BrowserAgentAct{Action: "type"}, func(p Page, ctx context.Context) (*point, error) { + at, err := clickTarget(ctx, p, t) + if err != nil { + return nil, err + } + return at, p.Type(ctx, text, clear, Target{}) }) } @@ -107,11 +113,29 @@ func (s *Service) Press(id, key string, t Target) error { if _, err := keySequence(key); err != nil { return err } - return s.act(id, func(p Page, ctx context.Context) error { - return p.Press(ctx, key, t) + return s.actAt(id, api.BrowserAgentAct{Action: "press"}, func(p Page, ctx context.Context) (*point, error) { + at, err := clickTarget(ctx, p, t) + if err != nil { + return nil, err + } + return at, p.Press(ctx, key, Target{}) }) } +// clickTarget clicks t, as typing into or pressing on an element does first, +// and returns where; nil for an empty target, which acts on whatever has +// focus. +func clickTarget(ctx context.Context, p Page, t Target) (*point, error) { + if t.empty() { + return nil, nil + } + x, y, err := p.Click(ctx, t) + if err != nil { + return nil, err + } + return &point{X: x, Y: y}, nil +} + // Screenshot writes a PNG to dest (created if empty: inside the profile dir) // and returns that path. The bytes stay off the transcript. func (s *Service) Screenshot(id, dest string) (string, error) { @@ -136,7 +160,7 @@ func (s *Service) Screenshot(id, dest string) (string, error) { return "", err } s.touch(sess) - s.agentActed(sess, nil, nil) + s.agentActed(sess, api.BrowserAgentAct{}) return dest, nil } @@ -158,7 +182,7 @@ func (s *Service) Navigate(id, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") - s.agentActed(sess, nil, nil) + s.agentActed(sess, api.BrowserAgentAct{}) return s.handleOf(sess), nil } @@ -178,8 +202,15 @@ func (s *Service) Forward(id string) error { } // Scroll moves the viewport by dy CSS pixels (positive is down). +// It turns the wheel where the agent last acted, else in the middle. func (s *Service) Scroll(id string, dy int) error { - return s.act(id, func(p Page, ctx context.Context) error { return p.Scroll(ctx, dy) }) + return s.actAt(id, api.BrowserAgentAct{Action: "scroll", DY: dy}, func(p Page, ctx context.Context) (*point, error) { + x, y, err := p.Scroll(ctx, s.agentPoint(id), dy) + if err != nil { + return nil, err + } + return &point{X: x, Y: y}, nil + }) } // act runs one agent action on its page and tells the window the agent @@ -188,11 +219,29 @@ func (s *Service) Scroll(id string, dy int) error { func (s *Service) act(id string, fn func(Page, context.Context) error) error { sess, err := s.drive(id, fn) if err == nil { - s.agentActed(sess, nil, nil) + s.agentActed(sess, api.BrowserAgentAct{}) } return err } +// actAt is act for act, made at the point fn returns (nil for none). +func (s *Service) actAt(id string, act api.BrowserAgentAct, fn func(Page, context.Context) (*point, error)) error { + var at *point + sess, err := s.drive(id, func(p Page, ctx context.Context) error { + var err error + at, err = fn(p, ctx) + return err + }) + if err != nil { + return err + } + if at != nil { + act.X, act.Y = &at.X, &at.Y + } + s.agentActed(sess, act) + return nil +} + func (s *Service) drive(id string, fn func(Page, context.Context) error) (*session, error) { sess, err := s.live(id) if err != nil { diff --git a/internal/browser/browser.go b/internal/browser/browser.go index 67564ad52..33c657370 100644 --- a/internal/browser/browser.go +++ b/internal/browser/browser.go @@ -12,6 +12,7 @@ import ( "time" "github.com/chromedp/cdproto/network" + "github.com/lichdotdev/lich/api" ) // idleAfter is how long a headless browser no pane shows sits unused before @@ -66,8 +67,10 @@ type session struct { // byAgent: the agent started it, not the window. watching: a pane shows // it, so the reaper leaves it alone like a headed window. byAgent, watching bool - page Page - lastUsed time.Time + // agentAt is where the agent last acted, where its next scroll turns. + agentAt *point + page Page + lastUsed time.Time } // New starts a service that launches the system Chromium (chromium.FindBrowser) @@ -118,7 +121,7 @@ func (s *Service) Open(owner, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") - s.agentActed(sess, nil, nil) + s.agentActed(sess, api.BrowserAgentAct{}) return s.handleOf(sess), nil } diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go index 50adbff80..3091ac3ea 100644 --- a/internal/browser/chrome.go +++ b/internal/browser/chrome.go @@ -219,8 +219,17 @@ func settleAfter(goThere func(context.Context) error) chromedp.Action { }) } -func (p *chromePage) Scroll(ctx context.Context, dy int) error { - return p.run(ctx, chromedp.Evaluate(fmt.Sprintf("window.scrollBy(0, %d)", dy), nil)) +// Scroll is a real wheel turn rather than window.scrollBy, so it scrolls what +// is under the point (a list with its own scroll) as a person's would. +func (p *chromePage) Scroll(ctx context.Context, at *point, dy int) (x, y float64, err error) { + if at == nil { + at = new(point) + if err := p.run(ctx, chromedp.Evaluate(`({x: innerWidth / 2, y: innerHeight / 2})`, at)); err != nil { + return 0, 0, err + } + } + wheel := input.DispatchMouseEvent(input.MouseWheel, at.X, at.Y).WithDeltaY(float64(dy)) + return at.X, at.Y, p.run(ctx, wheel) } // Focus raises the window. window.focus() from the page does not: the OS diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index 1f4743ace..9bd7c61c4 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -264,3 +264,38 @@ func TestChromeLinkToANewTabLoadsInThePageAndKeepsItResponding(t *testing.T) { t.Fatalf("back went to %q, want the page that held the link", href) } } + +func TestChromeScrollTurnsTheWheelAtThePoint(t *testing.T) { + p, _ := liveChrome(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `
tall
`) + })) + t.Cleanup(srv.Close) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + x, y, err := p.Scroll(ctx, nil, 400) + if err != nil { + t.Fatal(err) + } + if x <= 0 || y <= 0 { + t.Fatalf("scrolled at %v,%v, want the viewport's middle", x, y) + } + deadline := time.Now().Add(5 * time.Second) + for { + var scrolled float64 + if err := p.run(ctx, chromedp.Evaluate(`scrollY`, &scrolled)); err != nil { + t.Fatal(err) + } + if scrolled > 0 { + return + } + if time.Now().After(deadline) { + t.Fatal("the wheel turned and the page did not scroll") + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index 111d02f39..76ddb5ab7 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -23,6 +23,7 @@ type fakePage struct { backs int forwards int scrolled []int + scrolledAt []*point focused int cookies []*network.CookieParam setCookies []*network.CookieParam @@ -143,11 +144,15 @@ func (p *fakePage) Forward(context.Context) error { return nil } -func (p *fakePage) Scroll(_ context.Context, dy int) error { +func (p *fakePage) Scroll(_ context.Context, at *point, dy int) (x, y float64, err error) { p.mu.Lock() defer p.mu.Unlock() p.scrolled = append(p.scrolled, dy) - return nil + p.scrolledAt = append(p.scrolledAt, at) + if at == nil { + return 400, 300, nil + } + return at.X, at.Y, nil } func (p *fakePage) Focus(context.Context) error { diff --git a/internal/browser/page.go b/internal/browser/page.go index 0b1a5ab0f..9ea5dc179 100644 --- a/internal/browser/page.go +++ b/internal/browser/page.go @@ -87,7 +87,9 @@ type Page interface { Reload(ctx context.Context) error Back(ctx context.Context) error Forward(ctx context.Context) error - Scroll(ctx context.Context, dy int) error + // Scroll turns the wheel by dy CSS pixels at at, or at the viewport's + // middle when at is nil, and returns where it turned it. + Scroll(ctx context.Context, at *point, dy int) (x, y float64, err error) Focus(ctx context.Context) error // Cookies and SetCookies carry a browser's cookies, session ones included, // into the process that replaces it. diff --git a/internal/browser/view.go b/internal/browser/view.go index dbd74b5af..fcde4d6a3 100644 --- a/internal/browser/view.go +++ b/internal/browser/view.go @@ -171,8 +171,29 @@ func (s *Service) emitState(sess *session, open bool) { events.Send(s.emit, api.BrowserStateEvent, state) } -func (s *Service) agentActed(sess *session, x, y *float64) { - events.Send(s.emit, api.BrowserAgentEvent, api.BrowserAgentAct{Owner: sess.owner, X: x, Y: y}) +// agentActed tells the window the agent acted on sess's page, and keeps the +// point it acted at for its next scroll. +func (s *Service) agentActed(sess *session, act api.BrowserAgentAct) { + act.Owner = sess.owner + if act.X != nil && act.Y != nil { + s.mu.Lock() + sess.agentAt = &point{X: *act.X, Y: *act.Y} + s.mu.Unlock() + } + events.Send(s.emit, api.BrowserAgentEvent, act) +} + +// agentPoint is where the agent last acted on owner's page, nil before it +// acted at a point. +func (s *Service) agentPoint(owner string) *point { + s.mu.Lock() + defer s.unlock() + sess := s.lookupLocked(owner) + if sess == nil || sess.agentAt == nil { + return nil + } + at := *sess.agentAt + return &at } func (s *session) stateLocked(open bool) api.BrowserState { diff --git a/internal/browser/view_test.go b/internal/browser/view_test.go index 559ea1c40..42d8ed5f2 100644 --- a/internal/browser/view_test.go +++ b/internal/browser/view_test.go @@ -219,3 +219,64 @@ func TestWindowInputThatTimesOutKeepsTheBrowser(t *testing.T) { t.Fatalf("last state %+v, want the browser still open", st[len(st)-1]) } } + +func TestAgentScrollTurnsWhereItLastActedAndSaysWhere(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + p := launch.pages[0] + if err := s.Scroll("s", 300); err != nil { + t.Fatal(err) + } + p.clickAt = [2]float64{120, 48} + if err := s.Click("s", Target{Selector: "#go"}); err != nil { + t.Fatal(err) + } + if err := s.Scroll("s", -200); err != nil { + t.Fatal(err) + } + if p.scrolledAt[0] != nil { + t.Fatalf("first scroll turned at %+v, want the middle (nil) before any click", p.scrolledAt[0]) + } + if at := p.scrolledAt[1]; at == nil || at.X != 120 || at.Y != 48 { + t.Fatalf("scroll after a click turned at %+v, want the click's point", at) + } + var acts []api.BrowserAgentAct + for _, d := range out.named(api.BrowserAgentEvent.Name) { + acts = append(acts, d.(api.BrowserAgentAct)) + } + first, last := acts[0], acts[len(acts)-1] + if first.Action != "scroll" || first.DY != 300 || first.X == nil || *first.X != 400 { + t.Fatalf("first scroll reported %+v, want scroll 300 at the middle", first) + } + if last.Action != "scroll" || last.DY != -200 || *last.X != 120 || *last.Y != 48 { + t.Fatalf("second scroll reported %+v", last) + } +} + +func TestAgentTypeAndPressReportTheElementTheyClicked(t *testing.T) { + s, launch, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + launch.pages[0].clickAt = [2]float64{30, 40} + if err := s.Type("s", "hi", false, Target{Selector: "#q"}); err != nil { + t.Fatal(err) + } + if err := s.Press("s", "Enter", Target{}); err != nil { + t.Fatal(err) + } + acts := out.named(api.BrowserAgentEvent.Name) + typed, pressed := acts[0].(api.BrowserAgentAct), acts[1].(api.BrowserAgentAct) + if typed.Action != "type" || typed.X == nil || *typed.X != 30 || *typed.Y != 40 { + t.Fatalf("type reported %+v, want the field it clicked", typed) + } + if pressed.Action != "press" || pressed.X != nil { + t.Fatalf("press without a target reported %+v, want no point", pressed) + } + p := launch.pages[0] + if len(p.clicks) != 1 || len(p.typed) != 1 || !p.typed[0].t.empty() { + t.Fatalf("clicks %v typed %+v: type should click its target once, then type into what has focus", p.clicks, p.typed) + } +} diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go index eddacc3d0..d9eb6022d 100644 --- a/native/cmd/lich-native/browser_cursor.go +++ b/native/cmd/lich-native/browser_cursor.go @@ -27,6 +27,8 @@ const ( cursorDip = 170 * time.Millisecond cursorRing = 520 * time.Millisecond cursorFade = 300 * time.Millisecond + // cursorScroll is how long a scroll's chevrons take to slide past. + cursorScroll = 560 * time.Millisecond // cursorDipScale is how far the arrow shrinks at the bottom of its dip. cursorDipScale = 0.14 ) @@ -41,13 +43,17 @@ var cursorEase = ui.CubicBezier(0.2, 0.8, 0.2, 1) type agentCursor struct { shown bool from, to f32.Point + // scroll is the direction of the scroll the cursor glided for, 1 down + // and -1 up; 0 for a click, which dips and rings instead. + scroll int // moved is when the glide to to began; last is the agent's last action, // which the fade counts from; seen is the last mark taken. moved, last, seen time.Time } -// take moves the cursor for a new mark: a click glides it to its point, any -// other action only keeps it up. +// take moves the cursor for a new mark: an action at a point (a click, typing +// into or pressing on an element, a scroll) glides it there, any other only +// keeps it up. func (c *agentCursor) take(m browserMark) { if m.at.IsZero() || !m.at.After(c.seen) { return @@ -62,6 +68,13 @@ func (c *agentCursor) take(m browserMark) { from = c.position(m.at) } c.from, c.to, c.moved, c.shown = from, to, m.at, true + c.scroll = 0 + if m.action == "scroll" && m.dy != 0 { + c.scroll = 1 + if m.dy < 0 { + c.scroll = -1 + } + } } func (c *agentCursor) position(now time.Time) f32.Point { @@ -86,7 +99,7 @@ func (c *agentCursor) landed(now time.Time) time.Duration { // settled is when nothing about the cursor changes any more until it fades. func (c *agentCursor) settled() time.Time { - return c.moved.Add(cursorGlide + cursorRing) + return c.moved.Add(cursorGlide + max(cursorRing, cursorScroll)) } func progress(d, span time.Duration) float32 { @@ -129,14 +142,17 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, scale float32, tint, out at := c.position(now).Mul(scale) landed := c.landed(now) defer paint.PushOpacity(gtx.Ops, a).Pop() - if landed >= 0 && landed < cursorRing { + if c.scroll != 0 { + drawScrollChevrons(gtx, at, landed, c.scroll, tint, outline) + } + if c.scroll == 0 && landed >= 0 && landed < cursorRing { p := progress(landed, cursorRing) r := float32(gtx.Dp(17)) * cursorEase(p) ring := clip.Ellipse{Min: image.Pt(int(at.X-r), int(at.Y-r)), Max: image.Pt(int(at.X+r), int(at.Y+r))} paint.FillShape(gtx.Ops, ui.Alpha(tint, 0.95*(1-p)), clip.Stroke{Path: ring.Path(gtx.Ops), Width: float32(gtx.Dp(2))}.Op()) } s := float32(1) - if landed >= 0 && landed < cursorDip { + if c.scroll == 0 && landed >= 0 && landed < cursorDip { s -= cursorDipScale * float32(math.Sin(math.Pi*float64(progress(landed, cursorDip)))) } unit := float32(gtx.Dp(1)) @@ -158,6 +174,34 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, scale float32, tint, out } } +// drawScrollChevrons slides two chevrons under the arrow at at in the scroll's +// direction, fading in and out, for cursorScroll after the glide landed. +func drawScrollChevrons(gtx C, at f32.Point, landed time.Duration, dir int, tint, outline color.NRGBA) { + if landed < 0 || landed >= cursorScroll { + return + } + p := progress(landed, cursorScroll) + // in until a third of the way, then out, as the mockup's keyframes + alpha := min(p/0.35, (1-p)/0.65) + unit := float32(gtx.Dp(1)) + d := float32(dir) + shift := (-6 + 16*p) * d * unit + center := at.Add(f32.Pt(0, 30*unit+shift)) + chevron := func(dy float32) clip.PathSpec { + var path clip.Path + path.Begin(gtx.Ops) + path.MoveTo(center.Add(f32.Pt(-6*unit, (dy-3)*d*unit))) + path.LineTo(center.Add(f32.Pt(0, (dy+3)*d*unit))) + path.LineTo(center.Add(f32.Pt(6*unit, (dy-3)*d*unit))) + return path.End() + } + defer paint.PushOpacity(gtx.Ops, alpha).Pop() + for _, dy := range []float32{-5, 5} { + paint.FillShape(gtx.Ops, outline, clip.Stroke{Path: chevron(dy), Width: 4.4 * unit}.Op()) + paint.FillShape(gtx.Ops, tint, clip.Stroke{Path: chevron(dy), Width: 2.6 * unit}.Op()) + } +} + // cursorBadge is the pill beside the arrow: the provider's icon and name in // ink on tint, outlined like the arrow. func (u *window) cursorBadge(gtx C, tint, outline color.NRGBA, provider string, icon *icons.Icon) { diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go index 1e52425a4..121d3cdfa 100644 --- a/native/cmd/lich-native/browser_cursor_test.go +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -109,3 +109,17 @@ func TestDrawAgentCursorLaysOutWithoutAPage(t *testing.T) { t.Fatal("cursor still shown after it faded out") } } + +func TestAgentCursorScrollShowsItsDirectionInsteadOfAClick(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + x, y := 200.0, 150.0 + c.take(browserMark{at: t0, x: &x, y: &y, action: "scroll", dy: -300}) + if c.scroll != -1 || c.to != f32.Pt(200, 150) { + t.Fatalf("scroll up: scroll %d to %v", c.scroll, c.to) + } + c.take(clickMark(t0.Add(time.Second), 10, 10)) + if c.scroll != 0 { + t.Fatalf("a click after a scroll kept scroll %d", c.scroll) + } +} diff --git a/native/cmd/lich-native/browser_model.go b/native/cmd/lich-native/browser_model.go index 5de721a94..75ebe1d9b 100644 --- a/native/cmd/lich-native/browser_model.go +++ b/native/cmd/lich-native/browser_model.go @@ -25,10 +25,13 @@ type browserFrame struct { size image.Point } -// browserMark is the agent's last action on its page, and where it clicked. +// browserMark is the agent's last action on its page: which one, where, and +// for a scroll how far (positive down). type browserMark struct { - at time.Time - x, y *float64 + action string + dy int + at time.Time + x, y *float64 } type browserStore struct { @@ -137,7 +140,7 @@ func (m *model) applyBrowserFrame(owner string, ev backend.Event) error { func (m *model) applyBrowserAgent(a api.BrowserAgentAct, now time.Time) { m.mu.Lock() defer m.mu.Unlock() - m.browsers.agent[a.Owner] = browserMark{at: now, x: a.X, y: a.Y} + m.browsers.agent[a.Owner] = browserMark{at: now, x: a.X, y: a.Y, action: a.Action, dy: a.DY} } // browserView is what session's page pane draws: its state, its last frame From 3bba0537556e1a8a4e98ca4239254a5f15d387a2 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:31:13 -0300 Subject: [PATCH 14/25] fix(native): keep the agent's cursor through its turn, and the page undimmed The cursor left two seconds after each action, so between two tool calls of one turn it vanished and came back, reading as a stutter. It now stays while the agent's turn runs and five seconds after its last action or the turn's end. The page pane is no longer dimmed out of focus as a terminal is: the session's prompt keeps the focus nearly always, and the person watches the page beside it. --- native/cmd/lich-native/browser_cursor.go | 29 ++++++++++++++----- native/cmd/lich-native/browser_cursor_test.go | 25 ++++++++++++++-- native/cmd/lich-native/browser_pane.go | 25 +++++++++------- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go index d9eb6022d..4d40556f6 100644 --- a/native/cmd/lich-native/browser_cursor.go +++ b/native/cmd/lich-native/browser_cursor.go @@ -49,6 +49,8 @@ type agentCursor struct { // moved is when the glide to to began; last is the agent's last action, // which the fade counts from; seen is the last mark taken. moved, last, seen time.Time + // busy is whether the agent's turn ran at the last frame. + busy bool } // take moves the cursor for a new mark: an action at a point (a click, typing @@ -82,13 +84,25 @@ func (c *agentCursor) position(now time.Time) f32.Point { return c.from.Add(c.to.Sub(c.from).Mul(t)) } -// alpha is the cursor's opacity at now: whole until agentGlow after the last -// action, then fading out over cursorFade. +// observe notes whether the agent's turn runs at now: its end counts as the +// agent's last activity, which the hold counts from. +func (c *agentCursor) observe(now time.Time, busy bool) { + if c.busy && !busy && now.After(c.last) { + c.last = now + } + c.busy = busy +} + +// alpha is the cursor's opacity at now: whole while the agent's turn runs and +// for agentHold after its last activity, then fading out over cursorFade. func (c *agentCursor) alpha(now time.Time) float32 { - if !c.shown { + switch { + case !c.shown: return 0 + case c.busy: + return 1 } - return 1 - progress(now.Sub(c.last)-agentGlow, cursorFade) + return 1 - progress(now.Sub(c.last)-agentHold, cursorFade) } // landed is how long ago the glide reached the point, negative while it @@ -166,11 +180,12 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, scale float32, tint, out badge := op.Offset(image.Pt(int(at.X)+gtx.Dp(12), int(at.Y)+gtx.Dp(15))).Push(gtx.Ops) u.cursorBadge(gtx, tint, outline, provider, icon) badge.Pop() + // A busy cursor waits for the turn's end, whose status event redraws. switch { - case now.Before(c.settled()) || c.last.Add(agentGlow).Before(now): + case now.Before(c.settled()) || !c.busy && c.last.Add(agentHold).Before(now): gtx.Execute(op.InvalidateCmd{}) - default: - gtx.Execute(op.InvalidateCmd{At: c.last.Add(agentGlow)}) + case !c.busy: + gtx.Execute(op.InvalidateCmd{At: c.last.Add(agentHold)}) } } diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go index 121d3cdfa..d9cfbb4b4 100644 --- a/native/cmd/lich-native/browser_cursor_test.go +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -55,10 +55,10 @@ func TestAgentCursorStaysUpWhileTheAgentActsAndThenFades(t *testing.T) { t0 := time.Unix(100, 0) c.take(clickMark(t0, 10, 10)) c.take(browserMark{at: t0.Add(time.Second)}) - if a := c.alpha(t0.Add(time.Second + agentGlow - time.Millisecond)); a != 1 { + if a := c.alpha(t0.Add(time.Second + agentHold - time.Millisecond)); a != 1 { t.Fatalf("alpha %v before the glow ends, want 1: an action without a point keeps it up", a) } - if a := c.alpha(t0.Add(time.Second + agentGlow + cursorFade)); a != 0 { + if a := c.alpha(t0.Add(time.Second + agentHold + cursorFade)); a != 0 { t.Fatalf("alpha %v after the fade, want 0", a) } if c.position(t0.Add(2*time.Second)) != f32.Pt(10, 10) { @@ -100,7 +100,7 @@ func TestDrawAgentCursorLaysOutWithoutAPage(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) c.take(clickMark(t0, 40, 30)) - for _, d := range []time.Duration{0, cursorGlide, cursorGlide + cursorDip/2, agentGlow + cursorFade/2, agentGlow + cursorFade} { + for _, d := range []time.Duration{0, cursorGlide, cursorGlide + cursorDip/2, agentHold + cursorFade/2, agentHold + cursorFade} { gtx := layout.Context{Ops: new(op.Ops), Now: t0.Add(d), Metric: unit.Metric{PxPerDp: 1, PxPerSp: 1}, Constraints: layout.Exact(image.Pt(400, 300))} u.drawAgentCursor(gtx, &c, 1, th.Primary, th.PrimaryForeground, "Claude Code", providerIcon("claude")) @@ -123,3 +123,22 @@ func TestAgentCursorScrollShowsItsDirectionInsteadOfAClick(t *testing.T) { t.Fatalf("a click after a scroll kept scroll %d", c.scroll) } } + +func TestAgentCursorStaysWhileTheTurnRunsAndHoldsAfterItEnds(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + c.take(clickMark(t0, 10, 10)) + thinking := t0.Add(time.Minute) + c.observe(thinking, true) + if a := c.alpha(thinking); a != 1 { + t.Fatalf("alpha %v a minute into the turn, want 1: the agent is thinking between tools", a) + } + ended := thinking.Add(time.Second) + c.observe(ended, false) + if a := c.alpha(ended.Add(agentHold - time.Millisecond)); a != 1 { + t.Fatalf("alpha %v within the hold after the turn ended, want 1", a) + } + if a := c.alpha(ended.Add(agentHold + cursorFade)); a != 0 { + t.Fatalf("alpha %v after the hold and the fade, want 0", a) + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index 72bbdd057..f6a5f403f 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -40,9 +40,11 @@ var ( icBrowserWindow = icons.Lucide("external-link") ) -// agentGlow is how long the agent's cursor stays up after its last action on -// the page, before it fades. -const agentGlow = 2 * time.Second +// agentHold is how long the agent's cursor stays up after the agent's last +// action on the page or the end of its turn, before it fades. While its turn +// runs the cursor stays: between two tool calls the agent is thinking, not +// gone, and a cursor that left in between made it look stuttering. +const agentHold = 5 * time.Second // doubleClick is the window two presses of a button count as one double // click in, as most desktops set it. @@ -127,10 +129,9 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta page.Constraints = layout.Exact(image.Pt(size.X, max(size.Y-head-bar, 0))) off := op.Offset(image.Pt(0, head+bar)).Push(gtx.Ops) p.cursor.take(mark) + // Unlike a terminal, the page is not dimmed out of focus: the person keeps + // the session's prompt in focus nearly always, and watches the page. u.browserPage(page, p, session, frame, focused) - if several && !focused { - paint.FillShape(gtx.Ops, ui.Alpha(th.Background, paneDim), clip.Rect{Max: page.Constraints.Max}.Op()) - } off.Pop() u.paneFocusTarget(gtx, pane) area.Pop() @@ -260,7 +261,8 @@ func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame brow t.Pop() } if p.cursor.shown { - tint, provider, icon := u.agentLook(session) + tint, provider, icon, busy := u.agentLook(session) + p.cursor.observe(gtx.Now, busy) u.drawAgentCursor(gtx, &p.cursor, scale, tint, u.theme.PrimaryForeground, provider, icon) } u.browserInput(gtx, p, session, scale, focused) @@ -408,12 +410,13 @@ func isLocalHost(text string) bool { } // agentLook is how session's agent cursor is drawn: in its card color, or the -// theme's primary for a card without one, naming the provider that runs in it. -func (u *window) agentLook(session string) (tint color.NRGBA, provider string, icon *icons.Icon) { +// theme's primary for a card without one, naming the provider that runs in it; +// busy is whether the agent's turn is running. +func (u *window) agentLook(session string) (tint color.NRGBA, provider string, icon *icons.Icon, busy bool) { tint = u.theme.Primary sv, ok := u.model.sessionViewOf(session) if !ok { - return tint, "", providerIcon("") + return tint, "", providerIcon(""), false } if c, tinted := cardColors[sv.Color]; tinted { tint = c @@ -426,5 +429,5 @@ func (u *window) agentLook(session string) (tint color.NRGBA, provider string, i if info, found := u.model.providers.find(kind); found && info.name != "" { provider = info.name } - return tint, provider, providerIcon(kind) + return tint, provider, providerIcon(kind), sv.status == statusBusy } From 90455f1da74837a2136b9928c6f47cc99ecbf13a Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:37:50 -0300 Subject: [PATCH 15/25] fix(native): play every agent action on the cursor, in turn An agent that calls tools in parallel sends a click and a scroll within the same moment. The store kept only a session's last action, so a click followed by a scroll between two frames was never drawn, and one that was drawn had its ring cut short by the next glide. The store now keeps a session's latest actions and the cursor plays each one whole, in order, keeping the latest three when the agent outruns it. --- native/cmd/lich-native/browser_cursor.go | 59 +++++++--- native/cmd/lich-native/browser_cursor_test.go | 109 ++++++++++++------ native/cmd/lich-native/browser_model.go | 27 +++-- native/cmd/lich-native/browser_model_test.go | 25 ++++ native/cmd/lich-native/browser_pane.go | 5 +- 5 files changed, 163 insertions(+), 62 deletions(-) diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go index 4d40556f6..a3b5a06d4 100644 --- a/native/cmd/lich-native/browser_cursor.go +++ b/native/cmd/lich-native/browser_cursor.go @@ -47,29 +47,52 @@ type agentCursor struct { // and -1 up; 0 for a click, which dips and rings instead. scroll int // moved is when the glide to to began; last is the agent's last action, - // which the fade counts from; seen is the last mark taken. - moved, last, seen time.Time + // which the fade counts from. + moved, last time.Time + // seen is the last mark taken; pending are the actions at a point still + // to play, each played whole before the next. + seen uint64 + pending []browserMark // busy is whether the agent's turn ran at the last frame. busy bool } -// take moves the cursor for a new mark: an action at a point (a click, typing -// into or pressing on an element, a scroll) glides it there, any other only -// keeps it up. -func (c *agentCursor) take(m browserMark) { - if m.at.IsZero() || !m.at.After(c.seen) { - return +// cursorPending is how many actions the cursor keeps waiting to play: past it +// the oldest go, so a fast agent leaves the cursor a beat behind, not a replay. +const cursorPending = 3 + +// take queues the marks not seen yet: an action at a point (a click, typing +// into or pressing on an element, a scroll) is played by advance, any other +// only keeps the cursor up. +func (c *agentCursor) take(marks []browserMark) { + for _, m := range marks { + if m.seq <= c.seen { + continue + } + c.seen = m.seq + if m.at.After(c.last) { + c.last = m.at + } + if m.x != nil && m.y != nil { + c.pending = append(c.pending, m) + } } - c.seen, c.last = m.at, m.at - if m.x == nil || m.y == nil { + c.pending = c.pending[max(0, len(c.pending)-cursorPending):] +} + +// advance starts the next queued action once the one playing has finished. +func (c *agentCursor) advance(now time.Time) { + if len(c.pending) == 0 || c.shown && now.Before(c.played()) { return } + m := c.pending[0] + c.pending = c.pending[1:] to := f32.Pt(float32(*m.x), float32(*m.y)) from := to if c.shown { - from = c.position(m.at) + from = c.to } - c.from, c.to, c.moved, c.shown = from, to, m.at, true + c.from, c.to, c.moved, c.shown = from, to, now, true c.scroll = 0 if m.action == "scroll" && m.dy != 0 { c.scroll = 1 @@ -111,9 +134,13 @@ func (c *agentCursor) landed(now time.Time) time.Duration { return now.Sub(c.moved) - cursorGlide } -// settled is when nothing about the cursor changes any more until it fades. -func (c *agentCursor) settled() time.Time { - return c.moved.Add(cursorGlide + max(cursorRing, cursorScroll)) +// played is when the action playing has finished: its glide, then its ring +// or its chevrons. +func (c *agentCursor) played() time.Time { + if c.scroll != 0 { + return c.moved.Add(cursorGlide + cursorScroll) + } + return c.moved.Add(cursorGlide + cursorRing) } func progress(d, span time.Duration) float32 { @@ -182,7 +209,7 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, scale float32, tint, out badge.Pop() // A busy cursor waits for the turn's end, whose status event redraws. switch { - case now.Before(c.settled()) || !c.busy && c.last.Add(agentHold).Before(now): + case now.Before(c.played()) || len(c.pending) > 0 || !c.busy && c.last.Add(agentHold).Before(now): gtx.Execute(op.InvalidateCmd{}) case !c.busy: gtx.Execute(op.InvalidateCmd{At: c.last.Add(agentHold)}) diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go index d9cfbb4b4..c836a3f5a 100644 --- a/native/cmd/lich-native/browser_cursor_test.go +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -16,17 +16,34 @@ import ( "github.com/lichdotdev/lich/native/ui" ) -func clickMark(at time.Time, x, y float64) browserMark { return browserMark{at: at, x: &x, y: &y} } +var markSeq uint64 + +func clickMark(at time.Time, x, y float64) browserMark { + markSeq++ + return browserMark{seq: markSeq, at: at, x: &x, y: &y, action: "click"} +} + +func scrollMark(at time.Time, x, y float64, dy int) browserMark { + m := clickMark(at, x, y) + m.action, m.dy = "scroll", dy + return m +} + +// play feeds marks to c at now and lets it start what it can. +func play(c *agentCursor, now time.Time, marks ...browserMark) { + c.take(marks) + c.advance(now) +} func TestAgentCursorAppearsAtItsFirstClickAndGlidesToTheNext(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) - c.take(clickMark(t0, 50, 40)) + play(&c, t0, clickMark(t0, 50, 40)) if !c.shown || c.position(t0) != f32.Pt(50, 40) { t.Fatalf("first click at %v, shown %v", c.position(t0), c.shown) } t1 := t0.Add(time.Second) - c.take(clickMark(t1, 150, 40)) + play(&c, t1, clickMark(t1, 150, 40)) mid := c.position(t1.Add(cursorGlide / 2)) if mid.X <= 50 || mid.X >= 150 { t.Fatalf("halfway through the glide at %v, want between the two clicks", mid) @@ -36,27 +53,46 @@ func TestAgentCursorAppearsAtItsFirstClickAndGlidesToTheNext(t *testing.T) { } } -func TestAgentCursorGlidesFromWhereItIsWhenAClickInterruptsAGlide(t *testing.T) { +func TestAgentCursorPlaysParallelActionsInTurn(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) - c.take(clickMark(t0, 0, 0)) - t1 := t0.Add(time.Second) - c.take(clickMark(t1, 100, 0)) - t2 := t1.Add(cursorGlide / 2) - was := c.position(t2) - c.take(clickMark(t2, 100, 100)) - if c.from != was { - t.Fatalf("new glide starts at %v, want where the cursor was, %v", c.from, was) + // An agent's click and scroll called in parallel reach the pane between + // two frames; the click still dips and rings before the scroll plays. + play(&c, t0, clickMark(t0, 40, 40), scrollMark(t0, 40, 40, 500)) + if c.scroll != 0 || c.to != f32.Pt(40, 40) { + t.Fatalf("playing scroll %d to %v first, want the click", c.scroll, c.to) + } + c.advance(t0.Add(cursorGlide)) + if c.scroll != 0 { + t.Fatal("the scroll cut the click's ring short") + } + c.advance(t0.Add(cursorGlide + cursorRing)) + if c.scroll != 1 { + t.Fatalf("after the click played, scroll %d, want the scroll down", c.scroll) + } +} + +func TestAgentCursorKeepsOnlyTheLatestPendingActions(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + marks := []browserMark{} + for i := range 6 { + marks = append(marks, clickMark(t0, float64(i), 0)) + } + play(&c, t0, marks...) + if len(c.pending) != cursorPending-1 || c.to != f32.Pt(6-cursorPending, 0) { + t.Fatalf("pending %d, playing %v; want the latest %d, the oldest of them playing", len(c.pending), c.to, cursorPending) } } func TestAgentCursorStaysUpWhileTheAgentActsAndThenFades(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) - c.take(clickMark(t0, 10, 10)) - c.take(browserMark{at: t0.Add(time.Second)}) + play(&c, t0, clickMark(t0, 10, 10)) + markSeq++ + play(&c, t0.Add(time.Second), browserMark{seq: markSeq, at: t0.Add(time.Second)}) if a := c.alpha(t0.Add(time.Second + agentHold - time.Millisecond)); a != 1 { - t.Fatalf("alpha %v before the glow ends, want 1: an action without a point keeps it up", a) + t.Fatalf("alpha %v before the hold ends, want 1: an action without a point keeps it up", a) } if a := c.alpha(t0.Add(time.Second + agentHold + cursorFade)); a != 0 { t.Fatalf("alpha %v after the fade, want 0", a) @@ -70,11 +106,26 @@ func TestAgentCursorTakesEachMarkOnce(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) m := clickMark(t0, 10, 10) - c.take(m) - c.take(clickMark(t0.Add(time.Second), 90, 90)) - c.take(m) - if c.to != f32.Pt(90, 90) { - t.Fatalf("an old mark taken again moved the cursor to %v", c.to) + play(&c, t0, m) + later := t0.Add(time.Second) + play(&c, later, clickMark(later, 90, 90)) + play(&c, later.Add(time.Second), m) + if c.to != f32.Pt(90, 90) || len(c.pending) != 0 { + t.Fatalf("an old mark taken again: playing %v, pending %d", c.to, len(c.pending)) + } +} + +func TestAgentCursorScrollShowsItsDirectionInsteadOfAClick(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + play(&c, t0, scrollMark(t0, 200, 150, -300)) + if c.scroll != -1 || c.to != f32.Pt(200, 150) { + t.Fatalf("scroll up: scroll %d to %v", c.scroll, c.to) + } + later := t0.Add(time.Second) + play(&c, later, clickMark(later, 10, 10)) + if c.scroll != 0 { + t.Fatalf("a click after a scroll kept scroll %d", c.scroll) } } @@ -99,7 +150,7 @@ func TestDrawAgentCursorLaysOutWithoutAPage(t *testing.T) { u := &window{theme: th} var c agentCursor t0 := time.Unix(100, 0) - c.take(clickMark(t0, 40, 30)) + play(&c, t0, clickMark(t0, 40, 30)) for _, d := range []time.Duration{0, cursorGlide, cursorGlide + cursorDip/2, agentHold + cursorFade/2, agentHold + cursorFade} { gtx := layout.Context{Ops: new(op.Ops), Now: t0.Add(d), Metric: unit.Metric{PxPerDp: 1, PxPerSp: 1}, Constraints: layout.Exact(image.Pt(400, 300))} @@ -110,24 +161,10 @@ func TestDrawAgentCursorLaysOutWithoutAPage(t *testing.T) { } } -func TestAgentCursorScrollShowsItsDirectionInsteadOfAClick(t *testing.T) { - var c agentCursor - t0 := time.Unix(100, 0) - x, y := 200.0, 150.0 - c.take(browserMark{at: t0, x: &x, y: &y, action: "scroll", dy: -300}) - if c.scroll != -1 || c.to != f32.Pt(200, 150) { - t.Fatalf("scroll up: scroll %d to %v", c.scroll, c.to) - } - c.take(clickMark(t0.Add(time.Second), 10, 10)) - if c.scroll != 0 { - t.Fatalf("a click after a scroll kept scroll %d", c.scroll) - } -} - func TestAgentCursorStaysWhileTheTurnRunsAndHoldsAfterItEnds(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) - c.take(clickMark(t0, 10, 10)) + play(&c, t0, clickMark(t0, 10, 10)) thinking := t0.Add(time.Minute) c.observe(thinking, true) if a := c.alpha(thinking); a != 1 { diff --git a/native/cmd/lich-native/browser_model.go b/native/cmd/lich-native/browser_model.go index 75ebe1d9b..6d4565f08 100644 --- a/native/cmd/lich-native/browser_model.go +++ b/native/cmd/lich-native/browser_model.go @@ -7,6 +7,7 @@ import ( "fmt" "image" "image/jpeg" + "slices" "time" "gioui.org/op/paint" @@ -25,23 +26,30 @@ type browserFrame struct { size image.Point } -// browserMark is the agent's last action on its page: which one, where, and -// for a scroll how far (positive down). +// browserMark is one agent action on its page: which one, where, and for a +// scroll how far (positive down). seq orders the marks of one store. type browserMark struct { + seq uint64 action string dy int at time.Time x, y *float64 } +// agentMarksKept is how many of a session's latest agent actions the store +// keeps for its pane: an agent that calls tools in parallel sends several +// between two frames, and the cursor plays them in turn. +const agentMarksKept = 8 + type browserStore struct { states map[string]api.BrowserState frames map[string]browserFrame - agent map[string]browserMark + agent map[string][]browserMark + seq uint64 } func newBrowserStore() browserStore { - return browserStore{states: map[string]api.BrowserState{}, frames: map[string]browserFrame{}, agent: map[string]browserMark{}} + return browserStore{states: map[string]api.BrowserState{}, frames: map[string]browserFrame{}, agent: map[string][]browserMark{}} } // hydrateBrowsers reads the browsers running now, and takes out of the @@ -140,15 +148,18 @@ func (m *model) applyBrowserFrame(owner string, ev backend.Event) error { func (m *model) applyBrowserAgent(a api.BrowserAgentAct, now time.Time) { m.mu.Lock() defer m.mu.Unlock() - m.browsers.agent[a.Owner] = browserMark{at: now, x: a.X, y: a.Y, action: a.Action, dy: a.DY} + m.browsers.seq++ + mark := browserMark{seq: m.browsers.seq, at: now, x: a.X, y: a.Y, action: a.Action, dy: a.DY} + marks := append(m.browsers.agent[a.Owner], mark) + m.browsers.agent[a.Owner] = marks[max(0, len(marks)-agentMarksKept):] } // browserView is what session's page pane draws: its state, its last frame -// and the agent's last action. -func (m *model) browserView(session string) (api.BrowserState, browserFrame, browserMark) { +// and the agent's latest actions, oldest first. +func (m *model) browserView(session string) (api.BrowserState, browserFrame, []browserMark) { m.mu.Lock() defer m.mu.Unlock() - return m.browsers.states[session], m.browsers.frames[session], m.browsers.agent[session] + return m.browsers.states[session], m.browsers.frames[session], slices.Clone(m.browsers.agent[session]) } // openBrowserTab puts session's browser tab on screen, making it when there diff --git a/native/cmd/lich-native/browser_model_test.go b/native/cmd/lich-native/browser_model_test.go index f59c45a9e..41821a88c 100644 --- a/native/cmd/lich-native/browser_model_test.go +++ b/native/cmd/lich-native/browser_model_test.go @@ -147,3 +147,28 @@ func TestALayoutOfHomeSessionsComesOnScreen(t *testing.T) { t.Fatal("a split holding a Home session stayed off screen") } } + +func TestAgentActionsBetweenTwoFramesAreAllKeptInOrder(t *testing.T) { + m, _ := newLayoutsModel(t) + x, y := 10.0, 20.0 + send(m, api.BrowserAgentEvent.Name, api.BrowserAgentAct{Owner: "orch", Action: "click", X: &x, Y: &y}) + send(m, api.BrowserAgentEvent.Name, api.BrowserAgentAct{Owner: "orch", Action: "scroll", X: &x, Y: &y, DY: 500}) + for range agentMarksKept { + send(m, api.BrowserAgentEvent.Name, api.BrowserAgentAct{Owner: "orch"}) + } + _, _, marks := m.browserView("orch") + if len(marks) != agentMarksKept { + t.Fatalf("kept %d marks, want the latest %d", len(marks), agentMarksKept) + } + for i := 1; i < len(marks); i++ { + if marks[i].seq <= marks[i-1].seq { + t.Fatalf("marks out of order: %+v", marks) + } + } + m2, _ := newLayoutsModel(t) + send(m2, api.BrowserAgentEvent.Name, api.BrowserAgentAct{Owner: "orch", Action: "click", X: &x, Y: &y}) + send(m2, api.BrowserAgentEvent.Name, api.BrowserAgentAct{Owner: "orch", Action: "scroll", X: &x, Y: &y, DY: 500}) + if _, _, two := m2.browserView("orch"); len(two) != 2 || two[0].action != "click" || two[1].action != "scroll" { + t.Fatalf("marks %+v, want the click then the scroll", two) + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index f6a5f403f..2884676dc 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -104,7 +104,7 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta pane := browserPaneOf(session) p := u.browserPane(session) p.onScreen = true - st, frame, mark := u.model.browserView(session) + st, frame, marks := u.model.browserView(session) defer op.Offset(r.Min).Push(gtx.Ops).Pop() gtx.Constraints = layout.Exact(r.Size()) size := r.Size() @@ -128,7 +128,8 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta page := gtx page.Constraints = layout.Exact(image.Pt(size.X, max(size.Y-head-bar, 0))) off := op.Offset(image.Pt(0, head+bar)).Push(gtx.Ops) - p.cursor.take(mark) + p.cursor.take(marks) + p.cursor.advance(gtx.Now) // Unlike a terminal, the page is not dimmed out of focus: the person keeps // the session's prompt in focus nearly always, and watches the page. u.browserPage(page, p, session, frame, focused) From 89cbed69a8ae49621f873c09e50f5742f6efeac2 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:46:29 -0300 Subject: [PATCH 16/25] feat(browser): show the agent going back, forward, reloading and typing an address An agent's back, forward, reload and navigation changed the page with nothing to show for it. browser-agent now names each of them, with the address a navigation went to, and the pane's cursor plays them on its toolbar: it glides to the button and presses it, the button lit as a person's click lights it, or to the address field, where the address is typed in. The cursor is drawn over the whole pane rather than the page. --- api/browser.go | 9 +- api/testdata/contract.golden | 1 + internal/browser/actions.go | 18 +-- internal/browser/browser.go | 2 +- internal/browser/view_test.go | 28 ++++ native/cmd/lich-native/browser_cursor.go | 41 +++--- native/cmd/lich-native/browser_cursor_test.go | 35 ++++- native/cmd/lich-native/browser_model.go | 3 +- native/cmd/lich-native/browser_pane.go | 125 ++++++++++++++++-- native/cmd/lich-native/browser_queue_test.go | 48 +++++++ 10 files changed, 266 insertions(+), 44 deletions(-) diff --git a/api/browser.go b/api/browser.go index 6cdd9ccea..5d0a406d7 100644 --- a/api/browser.go +++ b/api/browser.go @@ -55,15 +55,18 @@ type BrowserState struct { } // BrowserAgentAct is one agent action on its page. Action is click, type, -// press or scroll, "" for one that touches no point (reading the page, a -// screenshot, a navigation). X and Y are where it acted, in CSS pixels, when -// it acted at a point; DY is a scroll's distance, positive down. +// press or scroll at a point; back, forward, reload or navigate, which the +// pane's toolbar does; or "" for one that changes nothing on screen (reading +// the page, a screenshot). X and Y are where it acted, in CSS pixels, when it +// acted at a point; DY is a scroll's distance, positive down; URL is where a +// navigate went. type BrowserAgentAct struct { Owner string `json:"owner"` Action string `json:"action,omitempty"` X *float64 `json:"x,omitempty"` Y *float64 `json:"y,omitempty"` DY int `json:"dy,omitempty"` + URL string `json:"url,omitempty"` } // BrowserViewOptions sizes a session's page to the pane showing it: W and H in diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index 782da23eb..4b7ca4bd6 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -1511,6 +1511,7 @@ BrowserAgentAct { x,omitempty *float64 y,omitempty *float64 dy,omitempty int + url,omitempty string } ProfileLogin { diff --git a/internal/browser/actions.go b/internal/browser/actions.go index 7025b8557..d164b8379 100644 --- a/internal/browser/actions.go +++ b/internal/browser/actions.go @@ -182,23 +182,23 @@ func (s *Service) Navigate(id, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") - s.agentActed(sess, api.BrowserAgentAct{}) + s.agentActed(sess, api.BrowserAgentAct{Action: "navigate", URL: u}) return s.handleOf(sess), nil } // Reload reloads the current page. func (s *Service) Reload(id string) error { - return s.act(id, func(p Page, ctx context.Context) error { return p.Reload(ctx) }) + return s.act(id, api.BrowserAgentAct{Action: "reload"}, func(p Page, ctx context.Context) error { return p.Reload(ctx) }) } // Back goes back in history. func (s *Service) Back(id string) error { - return s.act(id, func(p Page, ctx context.Context) error { return p.Back(ctx) }) + return s.act(id, api.BrowserAgentAct{Action: "back"}, func(p Page, ctx context.Context) error { return p.Back(ctx) }) } // Forward goes forward in history. func (s *Service) Forward(id string) error { - return s.act(id, func(p Page, ctx context.Context) error { return p.Forward(ctx) }) + return s.act(id, api.BrowserAgentAct{Action: "forward"}, func(p Page, ctx context.Context) error { return p.Forward(ctx) }) } // Scroll moves the viewport by dy CSS pixels (positive is down). @@ -213,13 +213,13 @@ func (s *Service) Scroll(id string, dy int) error { }) } -// act runs one agent action on its page and tells the window the agent -// acted; drive runs it, dropping a browser whose page failed it. The -// window's own calls go through steer instead. -func (s *Service) act(id string, fn func(Page, context.Context) error) error { +// act runs one agent action on its page and tells the window the agent did +// act; drive runs it, dropping a browser whose page failed it. The window's +// own calls go through steer instead. +func (s *Service) act(id string, act api.BrowserAgentAct, fn func(Page, context.Context) error) error { sess, err := s.drive(id, fn) if err == nil { - s.agentActed(sess, api.BrowserAgentAct{}) + s.agentActed(sess, act) } return err } diff --git a/internal/browser/browser.go b/internal/browser/browser.go index 33c657370..621612a94 100644 --- a/internal/browser/browser.go +++ b/internal/browser/browser.go @@ -121,7 +121,7 @@ func (s *Service) Open(owner, rawURL string) (Handle, error) { return Handle{}, err } s.remember(sess, u, "") - s.agentActed(sess, api.BrowserAgentAct{}) + s.agentActed(sess, api.BrowserAgentAct{Action: "navigate", URL: u}) return s.handleOf(sess), nil } diff --git a/internal/browser/view_test.go b/internal/browser/view_test.go index 42d8ed5f2..ad5729d91 100644 --- a/internal/browser/view_test.go +++ b/internal/browser/view_test.go @@ -2,6 +2,7 @@ package browser import ( "context" + "strings" "testing" "time" @@ -280,3 +281,30 @@ func TestAgentTypeAndPressReportTheElementTheyClicked(t *testing.T) { t.Fatalf("clicks %v typed %+v: type should click its target once, then type into what has focus", p.clicks, p.typed) } } + +func TestAgentNavigationSaysWhichToolbarActionItWas(t *testing.T) { + s, _, out := viewService(t) + if _, err := s.Attach("s", Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + if _, err := s.Navigate("s", "https://example.com/a"); err != nil { + t.Fatal(err) + } + for _, step := range []func(string) error{s.Back, s.Forward, s.Reload} { + if err := step("s"); err != nil { + t.Fatal(err) + } + } + if _, err := s.Open("s", "https://example.com/b"); err != nil { + t.Fatal(err) + } + var got []string + for _, d := range out.named(api.BrowserAgentEvent.Name) { + a := d.(api.BrowserAgentAct) + got = append(got, a.Action+" "+a.URL) + } + want := []string{"navigate https://example.com/a", "back ", "forward ", "reload ", "navigate https://example.com/b"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("agent actions %q, want %q", got, want) + } +} diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go index a3b5a06d4..3df1d9903 100644 --- a/native/cmd/lich-native/browser_cursor.go +++ b/native/cmd/lich-native/browser_cursor.go @@ -39,10 +39,14 @@ var cursorShadow = color.NRGBA{A: 90} // cursorEase is the glide's curve, CSS's cubic-bezier(.2,.8,.2,1). var cursorEase = ui.CubicBezier(0.2, 0.8, 0.2, 1) -// agentCursor is where the agent's pointer is, in CSS pixels of the page. +// agentCursor is where the agent's pointer is, in pixels of its pane: over +// the page for what the agent does in it, over the toolbar for back, forward, +// reload and an address. type agentCursor struct { shown bool from, to f32.Point + // action is the one playing. + action string // scroll is the direction of the scroll the cursor glided for, 1 down // and -1 up; 0 for a click, which dips and rings instead. scroll int @@ -61,9 +65,13 @@ type agentCursor struct { // the oldest go, so a fast agent leaves the cursor a beat behind, not a replay. const cursorPending = 3 -// take queues the marks not seen yet: an action at a point (a click, typing -// into or pressing on an element, a scroll) is played by advance, any other -// only keeps the cursor up. +// toolbarActions are the agent's actions the cursor plays on the pane's +// toolbar rather than at a point of the page. +var toolbarActions = map[string]bool{"back": true, "forward": true, "reload": true, "navigate": true} + +// take queues the marks not seen yet: an action at a point of the page (a +// click, typing into or pressing on an element, a scroll) or on the toolbar +// is played by advance, any other only keeps the cursor up. func (c *agentCursor) take(marks []browserMark) { for _, m := range marks { if m.seq <= c.seen { @@ -73,26 +81,28 @@ func (c *agentCursor) take(marks []browserMark) { if m.at.After(c.last) { c.last = m.at } - if m.x != nil && m.y != nil { + if m.x != nil && m.y != nil || toolbarActions[m.action] { c.pending = append(c.pending, m) } } c.pending = c.pending[max(0, len(c.pending)-cursorPending):] } -// advance starts the next queued action once the one playing has finished. -func (c *agentCursor) advance(now time.Time) { +// advance starts the next queued action once the one playing has finished, +// at the point place puts it in the pane, and returns it; false when none +// started. +func (c *agentCursor) advance(now time.Time, place func(browserMark) f32.Point) (browserMark, bool) { if len(c.pending) == 0 || c.shown && now.Before(c.played()) { - return + return browserMark{}, false } m := c.pending[0] c.pending = c.pending[1:] - to := f32.Pt(float32(*m.x), float32(*m.y)) + to := place(m) from := to if c.shown { from = c.to } - c.from, c.to, c.moved, c.shown = from, to, now, true + c.from, c.to, c.moved, c.shown, c.action = from, to, now, true, m.action c.scroll = 0 if m.action == "scroll" && m.dy != 0 { c.scroll = 1 @@ -100,6 +110,7 @@ func (c *agentCursor) advance(now time.Time) { c.scroll = -1 } } + return m, true } func (c *agentCursor) position(now time.Time) f32.Point { @@ -170,17 +181,17 @@ func arrowPath(ops *op.Ops, unit float32) clip.PathSpec { return path.End() } -// drawAgentCursor draws c over the page, scale being device pixels per CSS -// pixel, in tint with outline around it, the badge naming provider with its -// icon. It asks for the frames its motion needs. -func (u *window) drawAgentCursor(gtx C, c *agentCursor, scale float32, tint, outline color.NRGBA, provider string, icon *icons.Icon) { +// drawAgentCursor draws c over its pane, in tint with outline around it, the +// badge naming provider with its icon. It asks for the frames its motion +// needs. +func (u *window) drawAgentCursor(gtx C, c *agentCursor, tint, outline color.NRGBA, provider string, icon *icons.Icon) { now := gtx.Now a := c.alpha(now) if a <= 0 { c.shown = false return } - at := c.position(now).Mul(scale) + at := c.position(now) landed := c.landed(now) defer paint.PushOpacity(gtx.Ops, a).Pop() if c.scroll != 0 { diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go index c836a3f5a..993627844 100644 --- a/native/cmd/lich-native/browser_cursor_test.go +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -29,10 +29,20 @@ func scrollMark(at time.Time, x, y float64, dy int) browserMark { return m } +// atMark places a page mark at its own point, and a toolbar one at toolbarAt. +func atMark(m browserMark) f32.Point { + if m.x == nil { + return toolbarAt + } + return f32.Pt(float32(*m.x), float32(*m.y)) +} + +var toolbarAt = f32.Pt(12, 40) + // play feeds marks to c at now and lets it start what it can. func play(c *agentCursor, now time.Time, marks ...browserMark) { c.take(marks) - c.advance(now) + c.advance(now, atMark) } func TestAgentCursorAppearsAtItsFirstClickAndGlidesToTheNext(t *testing.T) { @@ -62,11 +72,11 @@ func TestAgentCursorPlaysParallelActionsInTurn(t *testing.T) { if c.scroll != 0 || c.to != f32.Pt(40, 40) { t.Fatalf("playing scroll %d to %v first, want the click", c.scroll, c.to) } - c.advance(t0.Add(cursorGlide)) + c.advance(t0.Add(cursorGlide), atMark) if c.scroll != 0 { t.Fatal("the scroll cut the click's ring short") } - c.advance(t0.Add(cursorGlide + cursorRing)) + c.advance(t0.Add(cursorGlide+cursorRing), atMark) if c.scroll != 1 { t.Fatalf("after the click played, scroll %d, want the scroll down", c.scroll) } @@ -154,7 +164,7 @@ func TestDrawAgentCursorLaysOutWithoutAPage(t *testing.T) { for _, d := range []time.Duration{0, cursorGlide, cursorGlide + cursorDip/2, agentHold + cursorFade/2, agentHold + cursorFade} { gtx := layout.Context{Ops: new(op.Ops), Now: t0.Add(d), Metric: unit.Metric{PxPerDp: 1, PxPerSp: 1}, Constraints: layout.Exact(image.Pt(400, 300))} - u.drawAgentCursor(gtx, &c, 1, th.Primary, th.PrimaryForeground, "Claude Code", providerIcon("claude")) + u.drawAgentCursor(gtx, &c, th.Primary, th.PrimaryForeground, "Claude Code", providerIcon("claude")) } if c.shown { t.Fatal("cursor still shown after it faded out") @@ -179,3 +189,20 @@ func TestAgentCursorStaysWhileTheTurnRunsAndHoldsAfterItEnds(t *testing.T) { t.Fatalf("alpha %v after the hold and the fade, want 0", a) } } + +func TestAgentCursorPlaysNavigationOnTheToolbar(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + markSeq++ + back := browserMark{seq: markSeq, at: t0, action: "back"} + markSeq++ + shot := browserMark{seq: markSeq, at: t0, action: "screenshot"} + c.take([]browserMark{back, shot}) + m, started := c.advance(t0, atMark) + if !started || m.action != "back" || c.to != toolbarAt { + t.Fatalf("started %v %q at %v, want back on the toolbar", started, m.action, c.to) + } + if len(c.pending) != 0 { + t.Fatalf("pending %d: an action that changes nothing on screen was queued", len(c.pending)) + } +} diff --git a/native/cmd/lich-native/browser_model.go b/native/cmd/lich-native/browser_model.go index 6d4565f08..81d03476a 100644 --- a/native/cmd/lich-native/browser_model.go +++ b/native/cmd/lich-native/browser_model.go @@ -32,6 +32,7 @@ type browserMark struct { seq uint64 action string dy int + url string at time.Time x, y *float64 } @@ -149,7 +150,7 @@ func (m *model) applyBrowserAgent(a api.BrowserAgentAct, now time.Time) { m.mu.Lock() defer m.mu.Unlock() m.browsers.seq++ - mark := browserMark{seq: m.browsers.seq, at: now, x: a.X, y: a.Y, action: a.Action, dy: a.DY} + mark := browserMark{seq: m.browsers.seq, at: now, x: a.X, y: a.Y, action: a.Action, dy: a.DY, url: a.URL} marks := append(m.browsers.agent[a.Owner], mark) m.browsers.agent[a.Owner] = marks[max(0, len(marks)-agentMarksKept):] } diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index 2884676dc..ec6258281 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -72,6 +72,17 @@ type browserPaneUI struct { queue *browserQueue cursor agentCursor onScreen, wasOnScreen bool + // flash is the toolbar button the agent's cursor pressed, and when; + // typing is the address it types into the field, from when. + flash agentPress + typing agentPress +} + +// agentPress is what the agent's cursor does on the toolbar, from at: the +// button it presses (back, forward, reload) or the address it types. +type agentPress struct { + what string + at time.Time } func (u *window) browserPane(session string) *browserPaneUI { @@ -114,6 +125,7 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta paint.FillShape(gtx.Ops, th.Background, shape.Op(gtx.Ops)) area := shape.Push(gtx.Ops) head, bar := gtx.Dp(paneHeader), gtx.Dp(ui.Space(9)) + u.playAgentCursor(gtx, p, marks, head, bar) header := gtx header.Constraints = layout.Exact(image.Pt(size.X, head)) u.browserHeader(header, p, pane, focused) @@ -128,12 +140,11 @@ func (u *window) browserPaneAt(gtx C, v stageView, session string, r image.Recta page := gtx page.Constraints = layout.Exact(image.Pt(size.X, max(size.Y-head-bar, 0))) off := op.Offset(image.Pt(0, head+bar)).Push(gtx.Ops) - p.cursor.take(marks) - p.cursor.advance(gtx.Now) // Unlike a terminal, the page is not dimmed out of focus: the person keeps // the session's prompt in focus nearly always, and watches the page. u.browserPage(page, p, session, frame, focused) off.Pop() + u.paintAgentCursor(gtx, p, session) u.paneFocusTarget(gtx, pane) area.Pop() u.lockedDropRefusal(gtx, session, image.Rectangle{Max: size}, r.Add(u.islandAt)) @@ -206,7 +217,9 @@ func (u *window) browserToolbar(gtx C, p *browserPaneUI, session string, st api. gtx.Execute(key.FocusCmd{Tag: &p.tag}) } } - if st.URL != p.shownURL && !gtx.Focused(&p.url) { + if text, typing := p.typed(gtx); typing && !gtx.Focused(&p.url) { + p.url.SetText(text) + } else if st.URL != p.shownURL && !gtx.Focused(&p.url) { p.shownURL = st.URL p.url.SetText(displayURL(st.URL)) } @@ -217,13 +230,13 @@ func (u *window) browserToolbar(gtx C, p *browserPaneUI, session string, st api. gtx.Constraints.Min.Y = gtx.Constraints.Max.Y return layout.Flex{Alignment: layout.Middle}.Layout(gtx, layout.Rigid(func(gtx C) D { - return iconAction(gtx, th, &u.root, &p.back, icBrowserBack, i18n.T("layouts.browser.back"), false) + return iconAction(gtx, th, &u.root, &p.back, icBrowserBack, i18n.T("layouts.browser.back"), p.flashing(gtx, "back")) }), layout.Rigid(func(gtx C) D { - return iconAction(gtx, th, &u.root, &p.forward, icBrowserForward, i18n.T("layouts.browser.forward"), false) + return iconAction(gtx, th, &u.root, &p.forward, icBrowserForward, i18n.T("layouts.browser.forward"), p.flashing(gtx, "forward")) }), layout.Rigid(func(gtx C) D { - return iconAction(gtx, th, &u.root, &p.reload, icBrowserReload, i18n.T("layouts.browser.reload"), false) + return iconAction(gtx, th, &u.root, &p.reload, icBrowserReload, i18n.T("layouts.browser.reload"), p.flashing(gtx, "reload")) }), ui.Gap(1.5), layout.Flexed(1, field.Layout), @@ -261,11 +274,6 @@ func (u *window) browserPage(gtx C, p *browserPaneUI, session string, frame brow paint.PaintOp{}.Add(gtx.Ops) t.Pop() } - if p.cursor.shown { - tint, provider, icon, busy := u.agentLook(session) - p.cursor.observe(gtx.Now, busy) - u.drawAgentCursor(gtx, &p.cursor, scale, tint, u.theme.PrimaryForeground, provider, icon) - } u.browserInput(gtx, p, session, scale, focused) defer clip.Rect{Max: size}.Push(gtx.Ops).Pop() event.Op(gtx.Ops, &p.tag) @@ -432,3 +440,98 @@ func (u *window) agentLook(session string) (tint color.NRGBA, provider string, i } return tint, provider, providerIcon(kind), sv.status == statusBusy } + +// toolbarFlash is how long a toolbar button the agent's cursor pressed stays +// lit, as a click's would. +const toolbarFlash = 220 * time.Millisecond + +// typingStep and typingMost pace the address the agent's cursor types into +// the field: a key every typingStep, the whole address within typingMost. +const ( + typingStep = 28 * time.Millisecond + typingMost = 700 * time.Millisecond +) + +// playAgentCursor takes the agent's new actions and starts the next one, at +// its point of the page under the header and toolbar, or on the toolbar. +func (u *window) playAgentCursor(gtx C, p *browserPaneUI, marks []browserMark, head, bar int) { + p.cursor.take(marks) + scale := gtx.Metric.PxPerDp + place := func(m browserMark) f32.Point { + if toolbarActions[m.action] { + return toolbarPoint(gtx, m.action, head, bar) + } + return f32.Pt(float32(*m.x)*scale, float32(*m.y)*scale+float32(head+bar)) + } + m, started := p.cursor.advance(gtx.Now, place) + if !started { + return + } + landed := p.cursor.moved.Add(cursorGlide) + switch m.action { + case "navigate": + p.typing = agentPress{what: displayURL(m.url), at: landed} + case "back", "forward", "reload": + p.flash = agentPress{what: m.action, at: landed} + } +} + +// toolbarPoint is where on the toolbar the cursor plays action: the middle of +// its button, or into the address field. It follows browserToolbar's layout: +// an inset, then back, forward and reload of one iconAction side each, a gap, +// and the field. +func toolbarPoint(gtx C, action string, head, bar int) f32.Point { + inset, side, gap := gtx.Dp(ui.Space(1.5)), gtx.Dp(ui.Space(6)), gtx.Dp(ui.Space(1.5)) + y := float32(head + bar/2) + slot := map[string]int{"back": 0, "forward": 1, "reload": 2} + if i, ok := slot[action]; ok { + return f32.Pt(float32(inset+side*i+side/2), y) + } + return f32.Pt(float32(inset+3*side+gap+gtx.Dp(ui.Space(10))), y) +} + +// paintAgentCursor draws the agent's cursor over the whole pane. +func (u *window) paintAgentCursor(gtx C, p *browserPaneUI, session string) { + if !p.cursor.shown { + return + } + tint, provider, icon, busy := u.agentLook(session) + p.cursor.observe(gtx.Now, busy) + u.drawAgentCursor(gtx, &p.cursor, tint, u.theme.PrimaryForeground, provider, icon) +} + +// flashing reports whether the agent's cursor lit the toolbar button for +// action at now. +func (p *browserPaneUI) flashing(gtx C, action string) bool { + if p.flash.what != action || gtx.Now.Before(p.flash.at) { + return false + } + end := p.flash.at.Add(toolbarFlash) + if gtx.Now.Before(end) { + gtx.Execute(op.InvalidateCmd{At: end}) + return true + } + return false +} + +// typed is the part of the agent's address shown in the field at now, and +// whether the typing still runs; it waits for the cursor to land. +func (p *browserPaneUI) typed(gtx C) (string, bool) { + t := p.typing + if t.what == "" { + return "", false + } + runes := []rune(t.what) + step := min(typingStep, typingMost/time.Duration(max(len(runes), 1))) + if gtx.Now.Before(t.at) { + gtx.Execute(op.InvalidateCmd{}) + return "", true + } + n := int(gtx.Now.Sub(t.at)/step) + 1 + if n >= len(runes) { + p.typing = agentPress{} + return t.what, false + } + gtx.Execute(op.InvalidateCmd{}) + return string(runes[:n]), true +} diff --git a/native/cmd/lich-native/browser_queue_test.go b/native/cmd/lich-native/browser_queue_test.go index 3dad861cb..69f686ec0 100644 --- a/native/cmd/lich-native/browser_queue_test.go +++ b/native/cmd/lich-native/browser_queue_test.go @@ -4,8 +4,14 @@ import ( "context" "reflect" "testing" + "time" + + "gioui.org/layout" + "gioui.org/op" + "gioui.org/unit" "github.com/lichdotdev/lich/api" + "github.com/lichdotdev/lich/native/ui" ) func TestBrowserQueueMergesWhatPilesUpBehindACall(t *testing.T) { @@ -51,3 +57,45 @@ func TestBrowserQueueKeepsOnlyTheLastPendingResize(t *testing.T) { t.Fatalf("sent sizes %v, want the last one", sizes) } } + +func TestToolbarPointsFollowTheToolbarsLayout(t *testing.T) { + gtx := layout.Context{Metric: unit.Metric{PxPerDp: 1, PxPerSp: 1}} + back, fwd, reload, addr := toolbarPoint(gtx, "back", 30, 36), toolbarPoint(gtx, "forward", 30, 36), toolbarPoint(gtx, "reload", 30, 36), toolbarPoint(gtx, "navigate", 30, 36) + side := float32(gtx.Dp(ui.Space(6))) + if fwd.X-back.X != side || reload.X-fwd.X != side || addr.X <= reload.X+side/2 { + t.Fatalf("back %v forward %v reload %v address %v: buttons one side apart, the address after them", back, fwd, reload, addr) + } + if back.Y != 48 || addr.Y != 48 { + t.Fatalf("y %v, want the toolbar's middle under the header", back.Y) + } +} + +func TestTypedAddressGrowsAfterTheCursorLands(t *testing.T) { + t0 := time.Unix(100, 0) + p := &browserPaneUI{typing: agentPress{what: "example.com", at: t0}} + gtx := layout.Context{Ops: new(op.Ops), Now: t0.Add(-time.Millisecond)} + if text, typing := p.typed(gtx); text != "" || !typing { + t.Fatalf("before landing: %q %v", text, typing) + } + gtx.Now = t0.Add(3 * typingStep) + if text, _ := p.typed(gtx); text != "exam" { + t.Fatalf("three steps in: %q", text) + } + gtx.Now = t0.Add(typingMost) + if text, typing := p.typed(gtx); text != "example.com" || typing { + t.Fatalf("done: %q %v", text, typing) + } +} + +func TestToolbarButtonFlashesOnceTheCursorLands(t *testing.T) { + t0 := time.Unix(100, 0) + p := &browserPaneUI{flash: agentPress{what: "back", at: t0}} + gtx := layout.Context{Ops: new(op.Ops), Now: t0.Add(toolbarFlash / 2)} + if !p.flashing(gtx, "back") || p.flashing(gtx, "forward") { + t.Fatal("back should be lit, forward not") + } + gtx.Now = t0.Add(toolbarFlash) + if p.flashing(gtx, "back") { + t.Fatal("back still lit after the flash") + } +} From 5d4d2f66c3990dad674972203730d4d7e20bf995 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:53:44 -0300 Subject: [PATCH 17/25] fix(browser): copy and paste between the page and the system clipboard The page pane took the terminal's paste chord, Ctrl+Shift+V, so Ctrl+V sent the key to the headless browser, which pasted its own empty clipboard; and a copy in the page stayed in that clipboard too. Ctrl+V (Cmd+V on macOS) now types the system clipboard's text into the page, and Ctrl+C or Ctrl+X reads the page's selection through browser.Selection and writes it to the system clipboard, before the key reaches the page so a cut still has its text. A password field gives nothing to copy, as Chrome's own copy does. --- api/browser.go | 4 + api/testdata/contract.golden | 1 + internal/browser/chrome_live_test.go | 31 ++++++++ internal/browser/chrome_view.go | 16 ++++ internal/browser/fake_test.go | 7 ++ internal/browser/page.go | 2 + internal/browser/rpc.go | 11 +++ internal/routes/browser.go | 1 + native/backend/browser.go | 5 ++ native/cmd/lich-native/browser_clipboard.go | 74 +++++++++++++++++++ .../cmd/lich-native/browser_clipboard_test.go | 30 ++++++++ native/cmd/lich-native/browser_pane.go | 25 +++++-- 12 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 native/cmd/lich-native/browser_clipboard.go create mode 100644 native/cmd/lich-native/browser_clipboard_test.go diff --git a/api/browser.go b/api/browser.go index 5d0a406d7..7c22344d3 100644 --- a/api/browser.go +++ b/api/browser.go @@ -26,6 +26,10 @@ var ( BrowserKey = define[BrowserKeyOptions, None]("browser.Key") BrowserText = define[BrowserTextOptions, None]("browser.Text") BrowserNav = define[BrowserNavOptions, None]("browser.Nav") + // BrowserSelection is the text selected in the page, which the pane + // copies to the system clipboard: the headless browser's own clipboard + // is not the person's. + BrowserSelection = define[SessionIDOptions, string]("browser.Selection") ) var ( diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index 4b7ca4bd6..7813f0b34 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -22,6 +22,7 @@ browser.Resize(BrowserViewOptions) None browser.Running(None) []BrowserState browser.Screenshot(BrowserScreenshotOptions) string browser.Scroll(BrowserScrollOptions) None +browser.Selection(SessionIDOptions) string browser.Show(SessionIDOptions) BrowserHandle browser.Text(BrowserTextOptions) None browser.Type(BrowserTypeOptions) None diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index 9bd7c61c4..98d73aab9 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -299,3 +299,34 @@ func TestChromeScrollTurnsTheWheelAtThePoint(t *testing.T) { time.Sleep(50 * time.Millisecond) } } + +func TestChromeSelectionReadsAFieldAndNotAPassword(t *testing.T) { + p, _ := liveChrome(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `

plain text

`) + })) + t.Cleanup(srv.Close) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + cases := []struct{ js, want string }{ + {`q.focus(); q.setSelectionRange(6, 11)`, "world"}, + {`pw.focus(); pw.select()`, ""}, + {`document.activeElement.blur(); getSelection().selectAllChildren(document.getElementById('p'))`, "plain text"}, + } + for _, c := range cases { + if err := p.run(ctx, chromedp.Evaluate(c.js, nil)); err != nil { + t.Fatal(err) + } + got, err := p.Selection(ctx) + if err != nil { + t.Fatal(err) + } + if got != c.want { + t.Errorf("after %s: selection %q, want %q", c.js, got, c.want) + } + } +} diff --git a/internal/browser/chrome_view.go b/internal/browser/chrome_view.go index 49ef95467..26afc1396 100644 --- a/internal/browser/chrome_view.go +++ b/internal/browser/chrome_view.go @@ -214,6 +214,22 @@ func (p *chromePage) InsertText(ctx context.Context, text string) error { return p.run(ctx, input.InsertText(text)) } +// selectionScript reads the selection of a text field, which getSelection +// does not see, else of the document. A password field gives nothing, as +// Chrome refuses to copy one. +const selectionScript = `(() => { + const el = document.activeElement; + if (el && typeof el.selectionStart === 'number' && typeof el.value === 'string') { + return el.type === 'password' ? '' : el.value.slice(el.selectionStart, el.selectionEnd); + } + return String(getSelection()); +})()` + +func (p *chromePage) Selection(ctx context.Context) (string, error) { + var text string + return text, p.run(ctx, chromedp.Evaluate(selectionScript, &text)) +} + var mouseTypes = map[string]input.MouseType{ "press": input.MousePressed, "release": input.MouseReleased, diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index 76ddb5ab7..cf2cb4ef2 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -36,6 +36,7 @@ type fakePage struct { mouseErr error keys []api.BrowserKeyOptions texts []string + selection string info PageInfo shotErr error navErr error @@ -220,6 +221,12 @@ func (p *fakePage) Key(_ context.Context, k api.BrowserKeyOptions) error { return nil } +func (p *fakePage) Selection(context.Context) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.selection, nil +} + func (p *fakePage) InsertText(_ context.Context, text string) error { p.mu.Lock() defer p.mu.Unlock() diff --git a/internal/browser/page.go b/internal/browser/page.go index 9ea5dc179..08e8f4260 100644 --- a/internal/browser/page.go +++ b/internal/browser/page.go @@ -104,6 +104,8 @@ type Page interface { Mouse(ctx context.Context, m api.BrowserMouseOptions) error Key(ctx context.Context, k api.BrowserKeyOptions) error InsertText(ctx context.Context, text string) error + // Selection is the text selected in the page, "" in a password field. + Selection(ctx context.Context) (string, error) Alive() bool Close() error } diff --git a/internal/browser/rpc.go b/internal/browser/rpc.go index 4e31e7a92..500adf296 100644 --- a/internal/browser/rpc.go +++ b/internal/browser/rpc.go @@ -157,6 +157,17 @@ func (s *Service) TextInput(_ context.Context, opts api.BrowserTextOptions) (api return api.None{}, err } +// SelectedText is the text selected in this session's page. +func (s *Service) SelectedText(_ context.Context, opts api.SessionIDOptions) (string, error) { + var text string + _, err := s.steer(opts.ID, inputWait, func(p Page, ctx context.Context) error { + var err error + text, err = p.Selection(ctx) + return err + }) + return text, err +} + // NavView runs the pane's toolbar. func (s *Service) NavView(_ context.Context, opts api.BrowserNavOptions) (api.None, error) { return api.None{}, s.Nav(opts.ID, opts.Action, opts.URL) diff --git a/internal/routes/browser.go b/internal/routes/browser.go index d30f94a05..7608c2d5a 100644 --- a/internal/routes/browser.go +++ b/internal/routes/browser.go @@ -32,4 +32,5 @@ func Browser(d *rpc.Handler, s *browser.Service) { rpc.Handle(d, api.BrowserKey, s.KeyInput) rpc.Handle(d, api.BrowserText, s.TextInput) rpc.Handle(d, api.BrowserNav, s.NavView) + rpc.Handle(d, api.BrowserSelection, s.SelectedText) } diff --git a/native/backend/browser.go b/native/backend/browser.go index 9fbbde918..623689bc7 100644 --- a/native/backend/browser.go +++ b/native/backend/browser.go @@ -61,6 +61,11 @@ func (c *Client) BrowserText(ctx context.Context, id, text string) error { return err } +// BrowserSelection is the text selected in session id's page. +func (c *Client) BrowserSelection(ctx context.Context, id string) (string, error) { + return callMethod(ctx, c, api.BrowserSelection, api.SessionIDOptions{ID: id}) +} + // BrowserNav runs the pane's toolbar: back, forward, reload, or go to url. func (c *Client) BrowserNav(ctx context.Context, id, action, url string) error { _, err := callMethod(ctx, c, api.BrowserNav, api.BrowserNavOptions{ID: id, Action: action, URL: url}) diff --git a/native/cmd/lich-native/browser_clipboard.go b/native/cmd/lich-native/browser_clipboard.go new file mode 100644 index 000000000..ce613d39b --- /dev/null +++ b/native/cmd/lich-native/browser_clipboard.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "io" + "strings" + + "gioui.org/io/clipboard" + "gioui.org/io/key" +) + +// The page pane's clipboard. The headless browser has a clipboard of its own, +// so the page's copy and paste reach the person's only through the pane: a +// paste types the system clipboard's text into the page, a copy or a cut +// reads the page's selection and writes it to the system clipboard. + +// browserClipboardChord is the clipboard action a key asks of the page: +// paste, copy or cut, "" for none. They are a browser's chords (Ctrl+V, C and +// X, Cmd on macOS), not the terminal's Ctrl+Shift+V, which pastes too. +func browserClipboardChord(e key.Event, goos string) string { + if e.State != key.Press { + return "" + } + mod := key.ModCtrl + if goos == "darwin" { + mod = key.ModCommand + } + if !e.Modifiers.Contain(mod) { + return "" + } + switch e.Name { + case "V": + return "paste" + case "C": + if !e.Modifiers.Contain(key.ModShift) { + return "copy" + } + case "X": + return "cut" + } + return "" +} + +// copyPageSelection reads the page's selection and hands it to the next +// frame to write: a clipboard write runs on the frame's goroutine. +func (u *window) copyPageSelection(p *browserPaneUI, session string) { + u.sendBrowser(session, "copy", func(ctx context.Context) error { + text, err := u.client.BrowserSelection(ctx, session) + if err != nil || text == "" { + return err + } + select { + case p.copied <- text: + default: + // A copy not written yet is replaced by the newer one. + select { + case <-p.copied: + default: + } + p.copied <- text + } + u.model.invalidate() + return nil + }) +} + +// writeCopied writes a copied selection waiting for this frame. +func writeCopied(gtx C, p *browserPaneUI) { + select { + case text := <-p.copied: + gtx.Execute(clipboard.WriteCmd{Type: "application/text", Data: io.NopCloser(strings.NewReader(text))}) + default: + } +} diff --git a/native/cmd/lich-native/browser_clipboard_test.go b/native/cmd/lich-native/browser_clipboard_test.go new file mode 100644 index 000000000..8012e0bd4 --- /dev/null +++ b/native/cmd/lich-native/browser_clipboard_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + + "gioui.org/io/key" +) + +func TestBrowserClipboardChordsAreTheBrowsersNotTheTerminals(t *testing.T) { + cases := []struct { + e key.Event + goos string + want string + }{ + {key.Event{Name: "V", Modifiers: key.ModCtrl, State: key.Press}, "linux", "paste"}, + {key.Event{Name: "V", Modifiers: key.ModCtrl | key.ModShift, State: key.Press}, "linux", "paste"}, + {key.Event{Name: "C", Modifiers: key.ModCtrl, State: key.Press}, "linux", "copy"}, + {key.Event{Name: "X", Modifiers: key.ModCtrl, State: key.Press}, "windows", "cut"}, + {key.Event{Name: "V", Modifiers: key.ModCommand, State: key.Press}, "darwin", "paste"}, + {key.Event{Name: "C", Modifiers: key.ModCommand, State: key.Press}, "darwin", "copy"}, + {key.Event{Name: "C", Modifiers: key.ModCtrl, State: key.Press}, "darwin", ""}, + {key.Event{Name: "V", Modifiers: key.ModCtrl, State: key.Release}, "linux", ""}, + {key.Event{Name: "A", Modifiers: key.ModCtrl, State: key.Press}, "linux", ""}, + } + for _, c := range cases { + if got := browserClipboardChord(c.e, c.goos); got != c.want { + t.Errorf("%v on %s: %q, want %q", c.e, c.goos, got, c.want) + } + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index ec6258281..ad6a3ce17 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -66,10 +66,13 @@ type browserPaneUI struct { // pane is attached. view api.BrowserViewOptions // held is the button the pointer holds down, for a drag's moves. - held string - lastPress time.Time - presses int - queue *browserQueue + held string + lastPress time.Time + presses int + queue *browserQueue + // copied is the page selection a copy read, for the next frame to write + // to the system clipboard. + copied chan string cursor agentCursor onScreen, wasOnScreen bool // flash is the toolbar button the agent's cursor pressed, and when; @@ -91,7 +94,7 @@ func (u *window) browserPane(session string) *browserPaneUI { } p, ok := u.browserPanes[session] if !ok { - p = &browserPaneUI{queue: newBrowserQueue()} + p = &browserPaneUI{queue: newBrowserQueue(), copied: make(chan string, 1)} go p.queue.run(u.ctx, func(err error) { log.Printf("browser of %s: %v", session, err) }) u.browserPanes[session] = p } @@ -285,10 +288,13 @@ func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale flo pointer.Filter{Target: &p.tag, Kinds: pointer.Scroll | pointer.Press | pointer.Release | pointer.Drag | pointer.Move, ScrollX: pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20}, ScrollY: pointer.ScrollRange{Min: -1 << 20, Max: 1 << 20}}, key.Filter{Focus: &p.tag, Name: "V", Required: key.ModCommand}, + key.Filter{Focus: &p.tag, Name: "C", Required: key.ModCommand}, + key.Filter{Focus: &p.tag, Name: "X", Required: key.ModCommand}, transfer.TargetFilter{Target: &p.tag, Type: "application/text"}) if focused && !gtx.Focused(&p.url) && !gtx.Focused(&p.tag) { gtx.Execute(key.FocusCmd{Tag: &p.tag}) } + writeCopied(gtx, p) for { ev, ok := gtx.Event(filters...) if !ok { @@ -300,9 +306,14 @@ func (u *window) browserInput(gtx C, p *browserPaneUI, session string, scale flo u.sendBrowser(session, "text", func(ctx context.Context) error { return u.client.BrowserText(ctx, session, text) }) } case key.Event: - if e.State == key.Press && isPasteChord(e, runtime.GOOS) { + switch browserClipboardChord(e, runtime.GOOS) { + case "paste": gtx.Execute(clipboard.ReadCmd{Tag: &p.tag}) - break + continue + case "copy", "cut": + // Read before the key reaches the page: a cut takes the + // selection away. The queue keeps the two in order. + u.copyPageSelection(p, session) } if k, ok := browserKey(session, e); ok { u.sendBrowser(session, "key", func(ctx context.Context) error { return u.client.BrowserKey(ctx, k) }) From 68872e42cbfdfb61e1e9f5f730c5f51c4a15f2f9 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 21:56:50 -0300 Subject: [PATCH 18/25] feat(browser): answer the agent's page reads in half the text, and with every action browser_info answered with JSON, its element list 93% of a read: every element's keys and its box, four numbers an agent clicking by index has no use for. A page now reads as lines, one per element (index, tag, label, link), with links on the page's own origin cut to their path: half the text on a session's real reads. browser_click, browser_type and browser_press, and lich browser's click, type and press, answer with the page they left, read once a navigation they started has its document, so an agent no longer follows each action with a read of its own. A read that fails after an action that went through keeps the browser. --- api/browser.go | 14 +++--- api/testdata/contract.golden | 6 +-- docs/cli.md | 6 ++- internal/browser/chrome.go | 17 +++++++ internal/browser/fake_test.go | 8 ++++ internal/browser/page.go | 3 ++ internal/browser/rpc.go | 41 +++++++++++++--- internal/browser/view_test.go | 28 +++++++++++ internal/cli/browser.go | 33 +++++++++---- internal/cli/browser_page.go | 78 +++++++++++++++++++++++++++++++ internal/cli/browser_page_test.go | 51 ++++++++++++++++++++ internal/cli/mcp_browser.go | 37 ++++++++------- 12 files changed, 279 insertions(+), 43 deletions(-) create mode 100644 internal/cli/browser_page.go create mode 100644 internal/cli/browser_page_test.go diff --git a/api/browser.go b/api/browser.go index 7c22344d3..450adcf70 100644 --- a/api/browser.go +++ b/api/browser.go @@ -1,12 +1,14 @@ package api var ( - BrowserShow = define[SessionIDOptions, BrowserHandle]("browser.Show") - BrowserOpen = define[BrowserOpenOptions, BrowserHandle]("browser.Open") - BrowserInfo = define[SessionIDOptions, BrowserPageInfo]("browser.Info") - BrowserClick = define[BrowserClickOptions, None]("browser.Click") - BrowserType = define[BrowserTypeOptions, None]("browser.Type") - BrowserPress = define[BrowserPressOptions, None]("browser.Press") + BrowserShow = define[SessionIDOptions, BrowserHandle]("browser.Show") + BrowserOpen = define[BrowserOpenOptions, BrowserHandle]("browser.Open") + BrowserInfo = define[SessionIDOptions, BrowserPageInfo]("browser.Info") + // Click, Type and Press answer with the page as it stands after them, so + // an agent reads what its action did without a browser.Info of its own. + BrowserClick = define[BrowserClickOptions, BrowserPageInfo]("browser.Click") + BrowserType = define[BrowserTypeOptions, BrowserPageInfo]("browser.Type") + BrowserPress = define[BrowserPressOptions, BrowserPageInfo]("browser.Press") BrowserScreenshot = define[BrowserScreenshotOptions, string]("browser.Screenshot") BrowserNavigate = define[BrowserOpenOptions, BrowserHandle]("browser.Navigate") BrowserReload = define[SessionIDOptions, None]("browser.Reload") diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index 7813f0b34..a5758954e 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -5,7 +5,7 @@ appupdate.Apply(None) None appupdate.Status(None) AppUpdateStatus browser.Attach(BrowserViewOptions) BrowserHandle browser.Back(SessionIDOptions) None -browser.Click(BrowserClickOptions) None +browser.Click(BrowserClickOptions) BrowserPageInfo browser.Close(SessionIDOptions) None browser.Detach(SessionIDOptions) None browser.Forward(SessionIDOptions) None @@ -16,7 +16,7 @@ browser.Mouse(BrowserMouseOptions) None browser.Nav(BrowserNavOptions) None browser.Navigate(BrowserOpenOptions) BrowserHandle browser.Open(BrowserOpenOptions) BrowserHandle -browser.Press(BrowserPressOptions) None +browser.Press(BrowserPressOptions) BrowserPageInfo browser.Reload(SessionIDOptions) None browser.Resize(BrowserViewOptions) None browser.Running(None) []BrowserState @@ -25,7 +25,7 @@ browser.Scroll(BrowserScrollOptions) None browser.Selection(SessionIDOptions) string browser.Show(SessionIDOptions) BrowserHandle browser.Text(BrowserTextOptions) None -browser.Type(BrowserTypeOptions) None +browser.Type(BrowserTypeOptions) BrowserPageInfo claudeprofile.CancelLogin(ProfileSlugOptions) None claudeprofile.CreateProfile(CreateProfileOptions) Profile claudeprofile.DeleteProfile(ProfileSlugOptions) None diff --git a/docs/cli.md b/docs/cli.md index d8673adf6..11dd6a9fc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -949,8 +949,10 @@ lich browser close `open` with no URL loads `about:blank`. The browser runs headless and the user sees its page in a tab beside the session; one that **Open in a window** moved -to a Chromium window of its own is reused there. `info` describes the page and its clickable nodes and never -includes what was typed into a field. `screenshot` prints the PNG's path. +to a Chromium window of its own is reused there. `info` describes the page, one line per clickable node +(index, tag, label, link), and never includes what was typed into a field; +`click`, `type` and `press` print the same description of the page they left, +once a navigation they started has its document. `screenshot` prints the PNG's path. URLs are http, https or `about:blank`. ### `lich mcp` diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go index 3091ac3ea..1b417b8dd 100644 --- a/internal/browser/chrome.go +++ b/internal/browser/chrome.go @@ -96,6 +96,23 @@ func (p *chromePage) Navigate(ctx context.Context, url string) error { })) } +// navGrace is how long after an action a navigation it starts may take to +// replace the document: until then the old one reads as complete. +const navGrace = 150 * time.Millisecond + +func (p *chromePage) Settle(ctx context.Context) error { + return p.run(ctx, settleAfter(func(ctx context.Context) error { + t := time.NewTimer(navGrace) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } + })) +} + func (p *chromePage) Info(ctx context.Context) (PageInfo, error) { var info PageInfo if err := p.run(ctx, chromedp.Evaluate(pageInfoScript, &info)); err != nil { diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index cf2cb4ef2..e3c03d3bf 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -36,6 +36,7 @@ type fakePage struct { mouseErr error keys []api.BrowserKeyOptions texts []string + settles int selection string info PageInfo shotErr error @@ -78,6 +79,13 @@ func (p *fakePage) Navigate(_ context.Context, url string) error { return nil } +func (p *fakePage) Settle(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + p.settles++ + return nil +} + func (p *fakePage) Info(context.Context) (PageInfo, error) { p.mu.Lock() defer p.mu.Unlock() diff --git a/internal/browser/page.go b/internal/browser/page.go index 08e8f4260..ab1e6a3dc 100644 --- a/internal/browser/page.go +++ b/internal/browser/page.go @@ -79,6 +79,9 @@ type PageInfo struct { type Page interface { Navigate(ctx context.Context, url string) error Info(ctx context.Context) (PageInfo, error) + // Settle waits until a navigation the last action may have started has + // its document, bounded by domSettle. + Settle(ctx context.Context) error // Click returns where it clicked, in CSS pixels. Click(ctx context.Context, t Target) (x, y float64, err error) Type(ctx context.Context, text string, clear bool, t Target) error diff --git a/internal/browser/rpc.go b/internal/browser/rpc.go index 500adf296..640b2a65c 100644 --- a/internal/browser/rpc.go +++ b/internal/browser/rpc.go @@ -30,18 +30,47 @@ func (s *Service) Read(_ context.Context, opts api.SessionIDOptions) (api.Browse } // ClickAt clicks in this session's browser. -func (s *Service) ClickAt(_ context.Context, opts api.BrowserClickOptions) (api.None, error) { - return api.None{}, s.Click(opts.ID, targetOf(opts.BrowserTarget)) +func (s *Service) ClickAt(_ context.Context, opts api.BrowserClickOptions) (api.BrowserPageInfo, error) { + if err := s.Click(opts.ID, targetOf(opts.BrowserTarget)); err != nil { + return api.BrowserPageInfo{}, err + } + return s.pageAfter(opts.ID) } // TypeText types into this session's browser. -func (s *Service) TypeText(_ context.Context, opts api.BrowserTypeOptions) (api.None, error) { - return api.None{}, s.Type(opts.ID, opts.Text, opts.Clear, targetOf(opts.BrowserTarget)) +func (s *Service) TypeText(_ context.Context, opts api.BrowserTypeOptions) (api.BrowserPageInfo, error) { + if err := s.Type(opts.ID, opts.Text, opts.Clear, targetOf(opts.BrowserTarget)); err != nil { + return api.BrowserPageInfo{}, err + } + return s.pageAfter(opts.ID) } // PressKey presses one named key in this session's browser. -func (s *Service) PressKey(_ context.Context, opts api.BrowserPressOptions) (api.None, error) { - return api.None{}, s.Press(opts.ID, opts.Key, targetOf(opts.BrowserTarget)) +func (s *Service) PressKey(_ context.Context, opts api.BrowserPressOptions) (api.BrowserPageInfo, error) { + if err := s.Press(opts.ID, opts.Key, targetOf(opts.BrowserTarget)); err != nil { + return api.BrowserPageInfo{}, err + } + return s.pageAfter(opts.ID) +} + +// pageAfter is the page an action left, read once a navigation it started has +// its document. A read that fails keeps the browser: the action itself went +// through, and the page may be between two documents. +func (s *Service) pageAfter(id string) (api.BrowserPageInfo, error) { + var info PageInfo + sess, err := s.steer(id, pageWait, func(p Page, ctx context.Context) error { + if err := p.Settle(ctx); err != nil { + return err + } + var err error + info, err = p.Info(ctx) + return err + }) + if err != nil { + return api.BrowserPageInfo{}, err + } + s.remember(sess, info.URL, info.Title) + return infoAPI(info), nil } // Capture writes a PNG of this session's page and returns its path. diff --git a/internal/browser/view_test.go b/internal/browser/view_test.go index ad5729d91..b608b1090 100644 --- a/internal/browser/view_test.go +++ b/internal/browser/view_test.go @@ -2,6 +2,7 @@ package browser import ( "context" + "errors" "strings" "testing" "time" @@ -308,3 +309,30 @@ func TestAgentNavigationSaysWhichToolbarActionItWas(t *testing.T) { t.Fatalf("agent actions %q, want %q", got, want) } } + +func TestAgentClickAnswersWithThePageItLeft(t *testing.T) { + s, launch, _ := viewService(t) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + p := launch.pages[0] + p.info = PageInfo{URL: "https://example.com/next", Title: "Next", Interactive: []Interactive{{Index: 0, Tag: "a", Text: "Home"}}} + idx := 0 + info, err := s.ClickAt(t.Context(), api.BrowserClickOptions{ID: "s", BrowserTarget: api.BrowserTarget{Index: &idx}}) + if err != nil { + t.Fatal(err) + } + if p.settles != 1 || info.URL != "https://example.com/next" || len(info.Interactive) != 1 { + t.Fatalf("settles %d, info %+v: the click should settle and read the page", p.settles, info) + } + if run := s.Running(); run[0].Title != "Next" { + t.Fatalf("running %+v: the page read after the click should update the window's title", run) + } + p.infoErr = errors.New("context destroyed by a navigation") + if _, err := s.PressKey(t.Context(), api.BrowserPressOptions{ID: "s", Key: "Enter"}); err == nil { + t.Fatal("a failed read after the press reported success") + } + if p.closed { + t.Fatal("a failed read after a press that went through closed the browser") + } +} diff --git a/internal/cli/browser.go b/internal/cli/browser.go index 5159737db..de364df39 100644 --- a/internal/cli/browser.go +++ b/internal/cli/browser.go @@ -78,7 +78,13 @@ func (c *client) browserInfo(args []string) error { if err != nil { return err } - return c.emit(info) + return c.printPage(info) +} + +// printPage prints a page the way the browser_* tools answer with it. +func (c *client) printPage(info api.BrowserPageInfo) error { + _, err := fmt.Fprintln(c.stdout, pageText(info)) + return err } func (c *client) browserClick(args []string) error { @@ -95,8 +101,11 @@ func (c *client) browserClick(args []string) error { if err != nil { return err } - _, err = callMethod(context.Background(), c, api.BrowserClick, api.BrowserClickOptions{ID: id, BrowserTarget: t}, shortCall) - return err + info, err := callMethod(context.Background(), c, api.BrowserClick, api.BrowserClickOptions{ID: id, BrowserTarget: t}, openCall) + if err != nil { + return err + } + return c.printPage(info) } func (c *client) browserType(args []string) error { @@ -114,10 +123,13 @@ func (c *client) browserType(args []string) error { if err != nil { return err } - _, err = callMethod(context.Background(), c, api.BrowserType, api.BrowserTypeOptions{ + info, err := callMethod(context.Background(), c, api.BrowserType, api.BrowserTypeOptions{ ID: id, Text: text, Clear: *clear, BrowserTarget: tf.target(), - }, shortCall) - return err + }, openCall) + if err != nil { + return err + } + return c.printPage(info) } func (c *client) browserPress(args []string) error { @@ -134,10 +146,13 @@ func (c *client) browserPress(args []string) error { if err != nil { return err } - _, err = callMethod(context.Background(), c, api.BrowserPress, api.BrowserPressOptions{ + info, err := callMethod(context.Background(), c, api.BrowserPress, api.BrowserPressOptions{ ID: id, Key: key, BrowserTarget: tf.target(), - }, shortCall) - return err + }, openCall) + if err != nil { + return err + } + return c.printPage(info) } func (c *client) browserScreenshot(args []string) error { diff --git a/internal/cli/browser_page.go b/internal/cli/browser_page.go new file mode 100644 index 000000000..38b78485e --- /dev/null +++ b/internal/cli/browser_page.go @@ -0,0 +1,78 @@ +package cli + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/lichdotdev/lich/api" +) + +// pageText is a page as an agent reads it: one line per interactive element +// rather than JSON, whose keys and boxes cost about twice the tokens for the +// same page (a session reading the Notacil app spent 3 to 4 thousand per +// read, 93% of it the element list; measured 2026-10-26). Positions are left +// out: an agent clicks by index, and a box is a screenshot's job. +func pageText(info api.BrowserPageInfo) string { + var b strings.Builder + fmt.Fprintf(&b, "url: %s\ntitle: %s\n", info.URL, info.Title) + if info.Text != "" { + fmt.Fprintf(&b, "text: %s\n", info.Text) + } + if len(info.Interactive) > 0 { + b.WriteString("interactive (index tag \"label\" -> link):\n") + } + base, _ := url.Parse(info.URL) + for _, n := range info.Interactive { + b.WriteString(elementLine(n, base)) + b.WriteByte('\n') + } + if info.Truncated { + b.WriteString("(more elements below, scroll to reach them)\n") + } + return strings.TrimSuffix(b.String(), "\n") +} + +func elementLine(n api.BrowserInteractive, base *url.URL) string { + parts := []string{strconv.Itoa(n.Index), elementKind(n)} + if n.Text != "" { + parts = append(parts, strconv.Quote(n.Text)) + } + if n.Href != "" { + parts = append(parts, "-> "+shortHref(n.Href, base)) + } + if n.Filled != nil && *n.Filled { + parts = append(parts, "(filled)") + } + if n.Checked != nil && *n.Checked { + parts = append(parts, "(checked)") + } + return strings.Join(parts, " ") +} + +// elementKind is the tag, with an input's type and a role the tag does not +// already say: input:email, div[button]. +func elementKind(n api.BrowserInteractive) string { + kind := n.Tag + if n.Type != "" && n.Tag == "input" { + kind += ":" + n.Type + } + if n.Role != "" && n.Role != n.Tag && !(n.Role == "link" && n.Tag == "a") { + kind += "[" + n.Role + "]" + } + return kind +} + +// shortHref drops the page's own origin from a link, which every link on the +// page would otherwise repeat. +func shortHref(href string, base *url.URL) string { + if base == nil || base.Host == "" { + return href + } + origin := base.Scheme + "://" + base.Host + if rest, ok := strings.CutPrefix(href, origin); ok && strings.HasPrefix(rest, "/") { + return rest + } + return href +} diff --git a/internal/cli/browser_page_test.go b/internal/cli/browser_page_test.go new file mode 100644 index 000000000..5e945f4a2 --- /dev/null +++ b/internal/cli/browser_page_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/lichdotdev/lich/api" +) + +func TestPageTextIsOneLinePerElement(t *testing.T) { + yes := true + info := api.BrowserPageInfo{ + URL: "https://app.notacil.com.br/invoices", Title: "Notacil", Text: "Notas fiscais", + Interactive: []api.BrowserInteractive{ + {Index: 0, Tag: "button", Type: "button", Text: "Toggle Sidebar", Rect: api.BrowserRect{X: 207, Y: 21, W: 28, H: 28}}, + {Index: 1, Tag: "a", Role: "link", Text: "Dashboard", Href: "https://app.notacil.com.br/dashboard"}, + {Index: 2, Tag: "a", Text: "Docs", Href: "https://docs.example.com/x"}, + {Index: 3, Tag: "input", Type: "email", Text: "E-mail", Filled: &yes}, + {Index: 4, Tag: "div", Role: "button", Text: "Emitir nota \"já\""}, + {Index: 5, Tag: "input", Type: "checkbox", Text: "Lembrar", Checked: &yes}, + }, + Truncated: true, + } + want := strings.Join([]string{ + "url: https://app.notacil.com.br/invoices", + "title: Notacil", + "text: Notas fiscais", + `interactive (index tag "label" -> link):`, + `0 button "Toggle Sidebar"`, + `1 a "Dashboard" -> /dashboard`, + `2 a "Docs" -> https://docs.example.com/x`, + `3 input:email "E-mail" (filled)`, + `4 div[button] "Emitir nota \"já\""`, + `5 input:checkbox "Lembrar" (checked)`, + "(more elements below, scroll to reach them)", + }, "\n") + if got := pageText(info); got != want { + t.Fatalf("got\n%s\nwant\n%s", got, want) + } + raw, _ := json.Marshal(info) + if len(pageText(info)) >= len(raw)*2/3 { + t.Fatalf("text %d chars against %d for the JSON: not smaller enough", len(pageText(info)), len(raw)) + } +} + +func TestPageTextOfAnEmptyPage(t *testing.T) { + if got := pageText(api.BrowserPageInfo{URL: "about:blank"}); got != "url: about:blank\ntitle: " { + t.Fatalf("got %q", got) + } +} diff --git a/internal/cli/mcp_browser.go b/internal/cli/mcp_browser.go index 708b51bf5..d45b8aca7 100644 --- a/internal/cli/mcp_browser.go +++ b/internal/cli/mcp_browser.go @@ -37,17 +37,19 @@ var browserTools = []mcpTool{ { Name: "browser_info", Description: "Read this session's current page: URL, title, the text in the viewport, and " + - "interactive elements there (index, tag, label, href, box). Form values are never " + - "included — filled/checked say that typing landed. Call this before click/type " + - "so indexes match. Read-only.", + "one line per interactive element there: index, tag, label and link. Form values are " + + "never included — (filled)/(checked) say that typing landed. click, type and press " + + "already answer with this, so call it only to look again without acting. Read-only.", Schema: schema(map[string]any{}), ReadOnly: true, Run: mcpBrowserInfo, }, { Name: "browser_click", - Description: "Click in this session's browser. Prefer index from browser_info, or a " + - "CSS selector, or viewport x and y. Sends a real mouse event, not element.click().", + Description: "Click in this session's browser. Prefer index from the last page read, or a " + + "CSS selector, or viewport x and y. Sends a real mouse event, not element.click(). " + + "Answers with the page afterwards, as browser_info reads it, once a navigation the " + + "click started has its document.", Schema: schema(map[string]any{ "index": property("integer", "Index from the last browser_info."), "selector": property("string", "CSS selector."), @@ -61,8 +63,8 @@ var browserTools = []mcpTool{ { Name: "browser_type", Description: "Type into this session's browser. Target as for browser_click; omit it " + - "to type into whatever is focused. clear replaces the field first. The text never " + - "comes back through browser_info.", + "to type into whatever is focused. clear replaces the field first. Answers with the " + + "page afterwards, which never includes the typed text.", Schema: schema(map[string]any{ "text": property("string", "Keys to send."), "clear": property("boolean", "Replace the field before typing."), @@ -76,13 +78,13 @@ var browserTools = []mcpTool{ if err != nil { return "", err } - _, err = callMethod(ctx, c, api.BrowserType, api.BrowserTypeOptions{ + info, err := callMethod(ctx, c, api.BrowserType, api.BrowserTypeOptions{ ID: id, Text: a.text("text"), Clear: a.flag("clear"), BrowserTarget: mcpTarget(a), - }, shortCall) + }, openCall) if err != nil { return "", err } - return "ok", nil + return pageText(info), nil }, }, { @@ -90,7 +92,8 @@ var browserTools = []mcpTool{ Description: "Press one named key in this session's browser (Enter, Tab, Escape, " + "Backspace, Delete, Space, ArrowUp/Down/Left/Right, Home, End, PageUp, PageDown). " + "Real key event, not typed text — use browser_type for that. Optional target " + - "focuses first, as for browser_type.", + "focuses first, as for browser_type. Answers with the page afterwards, once a " + + "navigation the key started (Enter on a form) has its document.", Schema: schema(map[string]any{ "key": property("string", "Named key: Enter, Tab, Escape, …"), "index": property("integer", "Index from the last browser_info."), @@ -103,13 +106,13 @@ var browserTools = []mcpTool{ if err != nil { return "", err } - _, err = callMethod(ctx, c, api.BrowserPress, api.BrowserPressOptions{ + info, err := callMethod(ctx, c, api.BrowserPress, api.BrowserPressOptions{ ID: id, Key: a.text("key"), BrowserTarget: mcpTarget(a), - }, shortCall) + }, openCall) if err != nil { return "", err } - return "ok", nil + return pageText(info), nil }, }, { @@ -232,11 +235,11 @@ func mcpBrowserClick(ctx context.Context, c *client, a mcpArgs) (string, error) if err != nil { return "", err } - _, err = callMethod(ctx, c, api.BrowserClick, api.BrowserClickOptions{ID: id, BrowserTarget: mcpTarget(a)}, shortCall) + info, err := callMethod(ctx, c, api.BrowserClick, api.BrowserClickOptions{ID: id, BrowserTarget: mcpTarget(a)}, openCall) if err != nil { return "", err } - return "ok", nil + return pageText(info), nil } func mcpBrowserID(ctx context.Context, c *client, method api.Method[api.SessionIDOptions, api.None]) (string, error) { @@ -267,7 +270,7 @@ func mcpBrowserInfo(ctx context.Context, c *client, _ mcpArgs) (string, error) { if err != nil { return "", err } - return encodeJSON(info) + return pageText(info), nil } func mcpBrowserList(ctx context.Context, c *client, _ mcpArgs) (string, error) { From 672bc8cee320e4591f7301bf2f7e16e83e3357f3 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:07:19 -0300 Subject: [PATCH 19/25] perf(browser): browser_type answers ok again instead of the page Typing into a field seldom changes the page, and answering each one with the page was a fifth of the page text a long session read. browser_type and lich browser type answer ok; browser_press Enter, which submits, still answers with the page, and browser_info reads it when typing did change it. --- api/browser.go | 9 ++++++--- api/testdata/contract.golden | 2 +- docs/cli.md | 5 +++-- internal/browser/rpc.go | 7 ++----- internal/cli/browser.go | 9 +++------ internal/cli/mcp_browser.go | 15 ++++++++------- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/api/browser.go b/api/browser.go index 450adcf70..109b351d8 100644 --- a/api/browser.go +++ b/api/browser.go @@ -4,10 +4,13 @@ var ( BrowserShow = define[SessionIDOptions, BrowserHandle]("browser.Show") BrowserOpen = define[BrowserOpenOptions, BrowserHandle]("browser.Open") BrowserInfo = define[SessionIDOptions, BrowserPageInfo]("browser.Info") - // Click, Type and Press answer with the page as it stands after them, so - // an agent reads what its action did without a browser.Info of its own. + // Click and Press answer with the page as it stands after them, so an + // agent reads what its action did without a browser.Info of its own. Type + // does not: typing into a field seldom changes the page, and a page per + // field filled cost a fifth of a session's page text (measured + // 2026-10-26). BrowserClick = define[BrowserClickOptions, BrowserPageInfo]("browser.Click") - BrowserType = define[BrowserTypeOptions, BrowserPageInfo]("browser.Type") + BrowserType = define[BrowserTypeOptions, None]("browser.Type") BrowserPress = define[BrowserPressOptions, BrowserPageInfo]("browser.Press") BrowserScreenshot = define[BrowserScreenshotOptions, string]("browser.Screenshot") BrowserNavigate = define[BrowserOpenOptions, BrowserHandle]("browser.Navigate") diff --git a/api/testdata/contract.golden b/api/testdata/contract.golden index a5758954e..5b31dc886 100644 --- a/api/testdata/contract.golden +++ b/api/testdata/contract.golden @@ -25,7 +25,7 @@ browser.Scroll(BrowserScrollOptions) None browser.Selection(SessionIDOptions) string browser.Show(SessionIDOptions) BrowserHandle browser.Text(BrowserTextOptions) None -browser.Type(BrowserTypeOptions) BrowserPageInfo +browser.Type(BrowserTypeOptions) None claudeprofile.CancelLogin(ProfileSlugOptions) None claudeprofile.CreateProfile(CreateProfileOptions) Profile claudeprofile.DeleteProfile(ProfileSlugOptions) None diff --git a/docs/cli.md b/docs/cli.md index 11dd6a9fc..7641435d7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -951,8 +951,9 @@ lich browser close sees its page in a tab beside the session; one that **Open in a window** moved to a Chromium window of its own is reused there. `info` describes the page, one line per clickable node (index, tag, label, link), and never includes what was typed into a field; -`click`, `type` and `press` print the same description of the page they left, -once a navigation they started has its document. `screenshot` prints the PNG's path. +`click` and `press` print the same description of the page they left, once a +navigation they started has its document; `type` prints nothing, as a field +filled seldom changes the page. `screenshot` prints the PNG's path. URLs are http, https or `about:blank`. ### `lich mcp` diff --git a/internal/browser/rpc.go b/internal/browser/rpc.go index 640b2a65c..57709bf08 100644 --- a/internal/browser/rpc.go +++ b/internal/browser/rpc.go @@ -38,11 +38,8 @@ func (s *Service) ClickAt(_ context.Context, opts api.BrowserClickOptions) (api. } // TypeText types into this session's browser. -func (s *Service) TypeText(_ context.Context, opts api.BrowserTypeOptions) (api.BrowserPageInfo, error) { - if err := s.Type(opts.ID, opts.Text, opts.Clear, targetOf(opts.BrowserTarget)); err != nil { - return api.BrowserPageInfo{}, err - } - return s.pageAfter(opts.ID) +func (s *Service) TypeText(_ context.Context, opts api.BrowserTypeOptions) (api.None, error) { + return api.None{}, s.Type(opts.ID, opts.Text, opts.Clear, targetOf(opts.BrowserTarget)) } // PressKey presses one named key in this session's browser. diff --git a/internal/cli/browser.go b/internal/cli/browser.go index de364df39..c080fae36 100644 --- a/internal/cli/browser.go +++ b/internal/cli/browser.go @@ -123,13 +123,10 @@ func (c *client) browserType(args []string) error { if err != nil { return err } - info, err := callMethod(context.Background(), c, api.BrowserType, api.BrowserTypeOptions{ + _, err = callMethod(context.Background(), c, api.BrowserType, api.BrowserTypeOptions{ ID: id, Text: text, Clear: *clear, BrowserTarget: tf.target(), - }, openCall) - if err != nil { - return err - } - return c.printPage(info) + }, shortCall) + return err } func (c *client) browserPress(args []string) error { diff --git a/internal/cli/mcp_browser.go b/internal/cli/mcp_browser.go index d45b8aca7..855c39fe8 100644 --- a/internal/cli/mcp_browser.go +++ b/internal/cli/mcp_browser.go @@ -38,8 +38,8 @@ var browserTools = []mcpTool{ Name: "browser_info", Description: "Read this session's current page: URL, title, the text in the viewport, and " + "one line per interactive element there: index, tag, label and link. Form values are " + - "never included — (filled)/(checked) say that typing landed. click, type and press " + - "already answer with this, so call it only to look again without acting. Read-only.", + "never included — (filled)/(checked) say that typing landed. click and press already " + + "answer with this, so call it only to look again without acting. Read-only.", Schema: schema(map[string]any{}), ReadOnly: true, Run: mcpBrowserInfo, @@ -63,8 +63,9 @@ var browserTools = []mcpTool{ { Name: "browser_type", Description: "Type into this session's browser. Target as for browser_click; omit it " + - "to type into whatever is focused. clear replaces the field first. Answers with the " + - "page afterwards, which never includes the typed text.", + "to type into whatever is focused. clear replaces the field first. Answers ok, not " + + "the page: submit with browser_press Enter, which answers with the page, or call " + + "browser_info if typing changed it. The typed text never comes back.", Schema: schema(map[string]any{ "text": property("string", "Keys to send."), "clear": property("boolean", "Replace the field before typing."), @@ -78,13 +79,13 @@ var browserTools = []mcpTool{ if err != nil { return "", err } - info, err := callMethod(ctx, c, api.BrowserType, api.BrowserTypeOptions{ + _, err = callMethod(ctx, c, api.BrowserType, api.BrowserTypeOptions{ ID: id, Text: a.text("text"), Clear: a.flag("clear"), BrowserTarget: mcpTarget(a), - }, openCall) + }, shortCall) if err != nil { return "", err } - return pageText(info), nil + return "ok", nil }, }, { From 1c9f8bfcebeaaa259a8d21228578c2ee7653c294 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:20:50 -0300 Subject: [PATCH 20/25] fix(browser): click an element scrolled out of view instead of the void below it Typing into a form scrolls the field into view and can push its save button below the page's bottom; a click on the button, by index or selector, was sent at its box there, outside the viewport, where it hit nothing and carried the pane's cursor off the pane. An element out of view is now scrolled to the middle before its click, as Playwright does; a click at x and y outside the viewport is refused with the viewport's size; and the cursor stays inside the page whatever it is told. --- internal/browser/chrome.go | 31 +++++++++++-- internal/browser/chrome_live_test.go | 49 ++++++++++++++++++++ internal/cli/mcp_browser.go | 3 +- native/cmd/lich-native/browser_pane.go | 11 ++++- native/cmd/lich-native/browser_queue_test.go | 15 ++++++ 5 files changed, 103 insertions(+), 6 deletions(-) diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go index 1b417b8dd..6d22985e8 100644 --- a/internal/browser/chrome.go +++ b/internal/browser/chrome.go @@ -313,8 +313,14 @@ type point struct { Y float64 `json:"y"` } +// resolveJS finds a click's point in the viewport. An element out of view, +// which typing into a form scrolls the save button to, is scrolled to the +// middle first: a click at its old box lands outside the page and hits +// nothing. A point given outside the viewport is reported, not clicked. const resolveJS = `(function(index, selector, x, y) { + var inView = function(px, py) { return px >= 0 && py >= 0 && px < innerWidth && py < innerHeight; }; if (typeof x === 'number' && typeof y === 'number') { + if (!inView(x, y)) return {x: x, y: y, outside: true, w: innerWidth, h: innerHeight}; return {x: x, y: y}; } var el = null; @@ -324,21 +330,38 @@ const resolveJS = `(function(index, selector, x, y) { } if (!el) return null; var r = el.getBoundingClientRect(); + if (!inView(r.left + r.width / 2, r.top + r.height / 2)) { + el.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'}); + r = el.getBoundingClientRect(); + } return {x: r.left + r.width / 2, y: r.top + r.height / 2}; })` +// resolved is resolveJS's answer: the point, or a point outside the +// viewport of w by h. +type resolved struct { + point + Outside bool `json:"outside"` + W float64 `json:"w"` + H float64 `json:"h"` +} + func (p *chromePage) resolve(ctx context.Context, t Target) (point, error) { args, err := json.Marshal([]any{t.Index, t.Selector, t.X, t.Y}) if err != nil { return point{}, err } expr := resolveJS + "(" + string(args[1:len(args)-1]) + ")" - var pt *point - if err := p.run(ctx, chromedp.Evaluate(expr, &pt)); err != nil { + var r *resolved + if err := p.run(ctx, chromedp.Evaluate(expr, &r)); err != nil { return point{}, err } - if pt == nil { + if r == nil { return point{}, fmt.Errorf("no element matches that target") } - return *pt, nil + if r.Outside { + return point{}, fmt.Errorf("%g,%g is outside the %gx%g viewport: scroll first, or click by index or selector, which scroll the element into view", + r.X, r.Y, r.W, r.H) + } + return r.point, nil } diff --git a/internal/browser/chrome_live_test.go b/internal/browser/chrome_live_test.go index 98d73aab9..e6377eaa0 100644 --- a/internal/browser/chrome_live_test.go +++ b/internal/browser/chrome_live_test.go @@ -330,3 +330,52 @@ func TestChromeSelectionReadsAFieldAndNotAPassword(t *testing.T) { } } } + +func TestChromeClickReachesAnElementScrolledOutOfView(t *testing.T) { + p, _ := liveChrome(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `
form
+`) + })) + t.Cleanup(srv.Close) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + if err := p.Resize(ctx, Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + x, y, err := p.Click(ctx, Target{Selector: "#save"}) + if err != nil { + t.Fatal(err) + } + if y < 0 || y > 600 || x < 0 || x > 800 { + t.Fatalf("clicked at %v,%v, outside the 800x600 viewport", x, y) + } + var saved string + if err := p.run(ctx, chromedp.Evaluate(`document.body.dataset.saved || ""`, &saved)); err != nil { + t.Fatal(err) + } + if saved != "yes" { + t.Fatal("the button below the fold was not clicked") + } +} + +func TestChromeClickRefusesAPointOutsideTheViewport(t *testing.T) { + p, srv := liveChrome(t) + ctx, cancel := context.WithTimeout(context.Background(), pageWait) + defer cancel() + if err := p.Navigate(ctx, srv.URL); err != nil { + t.Fatal(err) + } + if err := p.Resize(ctx, Viewport{W: 800, H: 600, Scale: 1}); err != nil { + t.Fatal(err) + } + x, y := 400.0, 715.0 + _, _, err := p.Click(ctx, Target{X: &x, Y: &y}) + if err == nil || !strings.Contains(err.Error(), "800x600") { + t.Fatalf("click at 400,715 in an 800x600 viewport: %v, want an error naming the viewport", err) + } +} diff --git a/internal/cli/mcp_browser.go b/internal/cli/mcp_browser.go index 855c39fe8..f00956e7b 100644 --- a/internal/cli/mcp_browser.go +++ b/internal/cli/mcp_browser.go @@ -47,7 +47,8 @@ var browserTools = []mcpTool{ { Name: "browser_click", Description: "Click in this session's browser. Prefer index from the last page read, or a " + - "CSS selector, or viewport x and y. Sends a real mouse event, not element.click(). " + + "CSS selector: either scrolls the element into view first. x and y are viewport " + + "coordinates and must be inside it. Sends a real mouse event, not element.click(). " + "Answers with the page afterwards, as browser_info reads it, once a navigation the " + "click started has its document.", Schema: schema(map[string]any{ diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index ad6a3ce17..dd16ae397 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -472,7 +472,7 @@ func (u *window) playAgentCursor(gtx C, p *browserPaneUI, marks []browserMark, h if toolbarActions[m.action] { return toolbarPoint(gtx, m.action, head, bar) } - return f32.Pt(float32(*m.x)*scale, float32(*m.y)*scale+float32(head+bar)) + return pagePoint(*m.x, *m.y, scale, image.Rect(0, head+bar, gtx.Constraints.Max.X, gtx.Constraints.Max.Y)) } m, started := p.cursor.advance(gtx.Now, place) if !started { @@ -487,6 +487,15 @@ func (u *window) playAgentCursor(gtx C, p *browserPaneUI, marks []browserMark, h } } +// pagePoint is the pane point of the page's CSS point x,y, the page drawn in +// page, kept inside it: an action reported outside the viewport would carry +// the cursor off the pane. +func pagePoint(x, y float64, scale float32, page image.Rectangle) f32.Point { + px := min(max(float32(x)*scale, float32(page.Min.X)), float32(page.Max.X-1)) + py := min(max(float32(y)*scale+float32(page.Min.Y), float32(page.Min.Y)), float32(page.Max.Y-1)) + return f32.Pt(px, py) +} + // toolbarPoint is where on the toolbar the cursor plays action: the middle of // its button, or into the address field. It follows browserToolbar's layout: // an inset, then back, forward and reload of one iconAction side each, a gap, diff --git a/native/cmd/lich-native/browser_queue_test.go b/native/cmd/lich-native/browser_queue_test.go index 69f686ec0..be4091e00 100644 --- a/native/cmd/lich-native/browser_queue_test.go +++ b/native/cmd/lich-native/browser_queue_test.go @@ -2,10 +2,12 @@ package main import ( "context" + "image" "reflect" "testing" "time" + "gioui.org/f32" "gioui.org/layout" "gioui.org/op" "gioui.org/unit" @@ -99,3 +101,16 @@ func TestToolbarButtonFlashesOnceTheCursorLands(t *testing.T) { t.Fatal("back still lit after the flash") } } + +func TestPagePointStaysInsideThePage(t *testing.T) { + page := image.Rect(0, 66, 800, 666) + if got := pagePoint(100, 50, 1, page); got != f32.Pt(100, 116) { + t.Fatalf("inside: %v", got) + } + if got := pagePoint(475, 715, 1, page); got.Y != 665 { + t.Fatalf("below the page: %v, want kept at its bottom edge", got) + } + if got := pagePoint(-5, 10, 2, page); got.X != 0 || got.Y != 86 { + t.Fatalf("left of the page at scale 2: %v", got) + } +} From ef3f3f771b69b4b2961fb2b355a7de8894b96a5d Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:35:43 -0300 Subject: [PATCH 21/25] fix(native): let the agent's cursor catch up with a fast agent An agent filling a form types into a field every half second, and the cursor played each action whole, eight tenths of a second, so it fell seconds behind and glided to fields the page had since scrolled away from. An action now plays at four tenths of its time while more wait behind it, and only the last one plays whole. --- native/cmd/lich-native/browser_cursor.go | 53 ++++++++++++++----- native/cmd/lich-native/browser_cursor_test.go | 23 ++++++++ native/cmd/lich-native/browser_pane.go | 2 +- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go index 3df1d9903..2ff00eb1c 100644 --- a/native/cmd/lich-native/browser_cursor.go +++ b/native/cmd/lich-native/browser_cursor.go @@ -31,6 +31,11 @@ const ( cursorScroll = 560 * time.Millisecond // cursorDipScale is how far the arrow shrinks at the bottom of its dip. cursorDipScale = 0.14 + // cursorCatchUp is the share of its time an action plays in while more + // wait behind it: an agent filling a form fires a field every half second, + // and played whole the cursor fell seconds behind the page, pointing at + // where fields had been before the page scrolled. + cursorCatchUp = 0.4 ) // cursorShadow is the drop shadow under the arrow, the mockup's 35% black. @@ -45,8 +50,10 @@ var cursorEase = ui.CubicBezier(0.2, 0.8, 0.2, 1) type agentCursor struct { shown bool from, to f32.Point - // action is the one playing. + // action is the one playing; pace is the share of its time it plays in, + // cursorCatchUp while more actions wait behind it. action string + pace float64 // scroll is the direction of the scroll the cursor glided for, 1 down // and -1 up; 0 for a click, which dips and rings instead. scroll int @@ -103,6 +110,10 @@ func (c *agentCursor) advance(now time.Time, place func(browserMark) f32.Point) from = c.to } c.from, c.to, c.moved, c.shown, c.action = from, to, now, true, m.action + c.pace = 1 + if len(c.pending) > 0 { + c.pace = cursorCatchUp + } c.scroll = 0 if m.action == "scroll" && m.dy != 0 { c.scroll = 1 @@ -114,7 +125,7 @@ func (c *agentCursor) advance(now time.Time, place func(browserMark) f32.Point) } func (c *agentCursor) position(now time.Time) f32.Point { - t := cursorEase(progress(now.Sub(c.moved), cursorGlide)) + t := cursorEase(progress(now.Sub(c.moved), c.span(cursorGlide))) return c.from.Add(c.to.Sub(c.from).Mul(t)) } @@ -142,16 +153,29 @@ func (c *agentCursor) alpha(now time.Time) float32 { // landed is how long ago the glide reached the point, negative while it // still moves. func (c *agentCursor) landed(now time.Time) time.Duration { - return now.Sub(c.moved) - cursorGlide + return now.Sub(c.moved) - c.span(cursorGlide) +} + +// span is d at the pace the action playing goes at. +func (c *agentCursor) span(d time.Duration) time.Duration { + if c.pace <= 0 { + return d + } + return time.Duration(float64(d) * c.pace) +} + +// landing is when the glide of the action playing reaches its point. +func (c *agentCursor) landing() time.Time { + return c.moved.Add(c.span(cursorGlide)) } // played is when the action playing has finished: its glide, then its ring // or its chevrons. func (c *agentCursor) played() time.Time { if c.scroll != 0 { - return c.moved.Add(cursorGlide + cursorScroll) + return c.landing().Add(c.span(cursorScroll)) } - return c.moved.Add(cursorGlide + cursorRing) + return c.landing().Add(c.span(cursorRing)) } func progress(d, span time.Duration) float32 { @@ -195,17 +219,18 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, tint, outline color.NRGB landed := c.landed(now) defer paint.PushOpacity(gtx.Ops, a).Pop() if c.scroll != 0 { - drawScrollChevrons(gtx, at, landed, c.scroll, tint, outline) + drawScrollChevrons(gtx, at, landed, c.span(cursorScroll), c.scroll, tint, outline) } - if c.scroll == 0 && landed >= 0 && landed < cursorRing { - p := progress(landed, cursorRing) + ringFor, dipFor := c.span(cursorRing), c.span(cursorDip) + if c.scroll == 0 && landed >= 0 && landed < ringFor { + p := progress(landed, ringFor) r := float32(gtx.Dp(17)) * cursorEase(p) ring := clip.Ellipse{Min: image.Pt(int(at.X-r), int(at.Y-r)), Max: image.Pt(int(at.X+r), int(at.Y+r))} paint.FillShape(gtx.Ops, ui.Alpha(tint, 0.95*(1-p)), clip.Stroke{Path: ring.Path(gtx.Ops), Width: float32(gtx.Dp(2))}.Op()) } s := float32(1) - if c.scroll == 0 && landed >= 0 && landed < cursorDip { - s -= cursorDipScale * float32(math.Sin(math.Pi*float64(progress(landed, cursorDip)))) + if c.scroll == 0 && landed >= 0 && landed < dipFor { + s -= cursorDipScale * float32(math.Sin(math.Pi*float64(progress(landed, dipFor)))) } unit := float32(gtx.Dp(1)) place := op.Affine(f32.Affine2D{}.Scale(f32.Point{}, f32.Pt(s, s)).Offset(at)).Push(gtx.Ops) @@ -228,12 +253,12 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, tint, outline color.NRGB } // drawScrollChevrons slides two chevrons under the arrow at at in the scroll's -// direction, fading in and out, for cursorScroll after the glide landed. -func drawScrollChevrons(gtx C, at f32.Point, landed time.Duration, dir int, tint, outline color.NRGBA) { - if landed < 0 || landed >= cursorScroll { +// direction, fading in and out, for span after the glide landed. +func drawScrollChevrons(gtx C, at f32.Point, landed, span time.Duration, dir int, tint, outline color.NRGBA) { + if landed < 0 || landed >= span { return } - p := progress(landed, cursorScroll) + p := progress(landed, span) // in until a third of the way, then out, as the mockup's keyframes alpha := min(p/0.35, (1-p)/0.65) unit := float32(gtx.Dp(1)) diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go index 993627844..1fae8aa99 100644 --- a/native/cmd/lich-native/browser_cursor_test.go +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -206,3 +206,26 @@ func TestAgentCursorPlaysNavigationOnTheToolbar(t *testing.T) { t.Fatalf("pending %d: an action that changes nothing on screen was queued", len(c.pending)) } } + +func TestAgentCursorCatchesUpWithAFastAgent(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + // four fields typed into within a second and a half, as a fast agent fills + // a form + play(&c, t0, clickMark(t0, 10, 10), clickMark(t0, 10, 50), clickMark(t0, 10, 90)) + if c.pace != cursorCatchUp { + t.Fatalf("pace %v with two actions behind, want the catch-up pace", c.pace) + } + now := t0 + for len(c.pending) > 0 { + now = c.played() + c.advance(now, atMark) + } + if c.pace != 1 { + t.Fatalf("pace %v for the last action, want it played whole", c.pace) + } + whole := cursorGlide + cursorRing + if behind := now.Sub(t0); behind >= 2*whole { + t.Fatalf("the last action started %v after the first, want well under two whole plays (%v)", behind, 2*whole) + } +} diff --git a/native/cmd/lich-native/browser_pane.go b/native/cmd/lich-native/browser_pane.go index dd16ae397..199cb889d 100644 --- a/native/cmd/lich-native/browser_pane.go +++ b/native/cmd/lich-native/browser_pane.go @@ -478,7 +478,7 @@ func (u *window) playAgentCursor(gtx C, p *browserPaneUI, marks []browserMark, h if !started { return } - landed := p.cursor.moved.Add(cursorGlide) + landed := p.cursor.landing() switch m.action { case "navigate": p.typing = agentPress{what: displayURL(m.url), at: landed} From 6adfad638e5e2fdc10acbc9bf5c5901a0774f676 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:39:36 -0300 Subject: [PATCH 22/25] fix(native): move the agent's cursor on once a click has dipped An agent filling a form types into a field every half second, about when the cursor finishes a click's glide, and the cursor waited out each ring, eight tenths of a second, before the next: it fell further behind with every field. A click's ring now fades on where it was while the cursor glides to the next action, which it may do once the click has dipped. TestAgentCursorPlaysParallelActionsInTurn changes with that contract: the scroll after a click waits for the click's dip, not its ring, and the ring is still drawn. --- native/cmd/lich-native/browser_cursor.go | 62 ++++++++++++++++--- native/cmd/lich-native/browser_cursor_test.go | 32 ++++++++-- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/native/cmd/lich-native/browser_cursor.go b/native/cmd/lich-native/browser_cursor.go index 2ff00eb1c..712f5df55 100644 --- a/native/cmd/lich-native/browser_cursor.go +++ b/native/cmd/lich-native/browser_cursor.go @@ -4,6 +4,7 @@ import ( "image" "image/color" "math" + "slices" "time" "gioui.org/f32" @@ -66,6 +67,17 @@ type agentCursor struct { pending []browserMark // busy is whether the agent's turn ran at the last frame. busy bool + // rings are the rings of clicks the cursor already moved on from, left + // to fade where they were: the cursor goes on once a click has dipped, + // rather than waiting out its ring. + rings []cursorRingAt +} + +// cursorRingAt is one click's ring: where, from when, for how long. +type cursorRingAt struct { + at f32.Point + start time.Time + span time.Duration } // cursorPending is how many actions the cursor keeps waiting to play: past it @@ -104,6 +116,10 @@ func (c *agentCursor) advance(now time.Time, place func(browserMark) f32.Point) } m := c.pending[0] c.pending = c.pending[1:] + if c.shown && c.scroll == 0 { + c.rings = append(c.rings, cursorRingAt{at: c.to, start: c.landing(), span: c.span(cursorRing)}) + } + c.rings = slices.DeleteFunc(c.rings, func(r cursorRingAt) bool { return !now.Before(r.start.Add(r.span)) }) to := place(m) from := to if c.shown { @@ -169,13 +185,28 @@ func (c *agentCursor) landing() time.Time { return c.moved.Add(c.span(cursorGlide)) } -// played is when the action playing has finished: its glide, then its ring -// or its chevrons. +// played is when the cursor may go on to the next action: a click once it +// has dipped, its ring fading on behind it; a scroll once its chevrons, which +// ride the cursor, have slid past. func (c *agentCursor) played() time.Time { if c.scroll != 0 { return c.landing().Add(c.span(cursorScroll)) } - return c.landing().Add(c.span(cursorRing)) + return c.landing().Add(c.span(cursorDip)) +} + +// settled is when nothing about the cursor moves any more until it fades. +func (c *agentCursor) settled() time.Time { + end := c.played() + if c.scroll == 0 { + end = c.landing().Add(c.span(cursorRing)) + } + for _, r := range c.rings { + if r.start.Add(r.span).After(end) { + end = r.start.Add(r.span) + } + } + return end } func progress(d, span time.Duration) float32 { @@ -221,12 +252,12 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, tint, outline color.NRGB if c.scroll != 0 { drawScrollChevrons(gtx, at, landed, c.span(cursorScroll), c.scroll, tint, outline) } - ringFor, dipFor := c.span(cursorRing), c.span(cursorDip) - if c.scroll == 0 && landed >= 0 && landed < ringFor { - p := progress(landed, ringFor) - r := float32(gtx.Dp(17)) * cursorEase(p) - ring := clip.Ellipse{Min: image.Pt(int(at.X-r), int(at.Y-r)), Max: image.Pt(int(at.X+r), int(at.Y+r))} - paint.FillShape(gtx.Ops, ui.Alpha(tint, 0.95*(1-p)), clip.Stroke{Path: ring.Path(gtx.Ops), Width: float32(gtx.Dp(2))}.Op()) + for _, r := range c.rings { + drawCursorRing(gtx, r.at, now.Sub(r.start), r.span, tint) + } + dipFor := c.span(cursorDip) + if c.scroll == 0 { + drawCursorRing(gtx, at, landed, c.span(cursorRing), tint) } s := float32(1) if c.scroll == 0 && landed >= 0 && landed < dipFor { @@ -245,13 +276,24 @@ func (u *window) drawAgentCursor(gtx C, c *agentCursor, tint, outline color.NRGB badge.Pop() // A busy cursor waits for the turn's end, whose status event redraws. switch { - case now.Before(c.played()) || len(c.pending) > 0 || !c.busy && c.last.Add(agentHold).Before(now): + case now.Before(c.settled()) || len(c.pending) > 0 || !c.busy && c.last.Add(agentHold).Before(now): gtx.Execute(op.InvalidateCmd{}) case !c.busy: gtx.Execute(op.InvalidateCmd{At: c.last.Add(agentHold)}) } } +// drawCursorRing draws a click's ring at at, since into its span. +func drawCursorRing(gtx C, at f32.Point, since, span time.Duration, tint color.NRGBA) { + if since < 0 || since >= span { + return + } + p := progress(since, span) + r := float32(gtx.Dp(17)) * cursorEase(p) + ring := clip.Ellipse{Min: image.Pt(int(at.X-r), int(at.Y-r)), Max: image.Pt(int(at.X+r), int(at.Y+r))} + paint.FillShape(gtx.Ops, ui.Alpha(tint, 0.95*(1-p)), clip.Stroke{Path: ring.Path(gtx.Ops), Width: float32(gtx.Dp(2))}.Op()) +} + // drawScrollChevrons slides two chevrons under the arrow at at in the scroll's // direction, fading in and out, for span after the glide landed. func drawScrollChevrons(gtx C, at f32.Point, landed, span time.Duration, dir int, tint, outline color.NRGBA) { diff --git a/native/cmd/lich-native/browser_cursor_test.go b/native/cmd/lich-native/browser_cursor_test.go index 1fae8aa99..aae7d3199 100644 --- a/native/cmd/lich-native/browser_cursor_test.go +++ b/native/cmd/lich-native/browser_cursor_test.go @@ -67,18 +67,23 @@ func TestAgentCursorPlaysParallelActionsInTurn(t *testing.T) { var c agentCursor t0 := time.Unix(100, 0) // An agent's click and scroll called in parallel reach the pane between - // two frames; the click still dips and rings before the scroll plays. + // two frames; the click still dips before the scroll plays, and its ring + // fades on where it was. play(&c, t0, clickMark(t0, 40, 40), scrollMark(t0, 40, 40, 500)) if c.scroll != 0 || c.to != f32.Pt(40, 40) { t.Fatalf("playing scroll %d to %v first, want the click", c.scroll, c.to) } - c.advance(t0.Add(cursorGlide), atMark) + dipped := c.played() + c.advance(dipped.Add(-time.Millisecond), atMark) if c.scroll != 0 { - t.Fatal("the scroll cut the click's ring short") + t.Fatal("the scroll cut the click's dip short") } - c.advance(t0.Add(cursorGlide+cursorRing), atMark) + c.advance(dipped, atMark) if c.scroll != 1 { - t.Fatalf("after the click played, scroll %d, want the scroll down", c.scroll) + t.Fatalf("after the click dipped, scroll %d, want the scroll down", c.scroll) + } + if len(c.rings) != 1 { + t.Fatalf("rings %d, want the click's ring still fading", len(c.rings)) } } @@ -229,3 +234,20 @@ func TestAgentCursorCatchesUpWithAFastAgent(t *testing.T) { t.Fatalf("the last action started %v after the first, want well under two whole plays (%v)", behind, 2*whole) } } + +func TestAgentCursorGoesOnOnceAClickDippedAndLeavesItsRing(t *testing.T) { + var c agentCursor + t0 := time.Unix(100, 0) + play(&c, t0, clickMark(t0, 10, 10)) + t1 := t0.Add(cursorGlide + cursorDip) + play(&c, t1, clickMark(t1, 200, 10)) + if c.to != f32.Pt(200, 10) { + t.Fatalf("playing %v after the first click dipped, want the next click", c.to) + } + if len(c.rings) != 1 || c.rings[0].at != f32.Pt(10, 10) { + t.Fatalf("rings %+v, want the first click's ring left fading at its point", c.rings) + } + if !c.settled().After(c.played()) { + t.Fatal("settled before the rings faded") + } +} From 29e71c81a2025bb916dfa33e1285a4303ede5d08 Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:43:43 -0300 Subject: [PATCH 23/25] fix(browser): a failed agent action no longer closes a browser whose Chrome is alive Any agent action that failed dropped the browser, as if the page had died: a click on an index from an older read, or one that landed while the page was between documents, answered no element matches and closed the page the person was looking at, logged in and all. A failed call now drops the browser only when its Chrome process is gone, as the pane's own input already did. TestFailedInfoDropsTheBrowser now fails on a page whose Chrome died, and TestOpenDropsWhenReuseNavigateFails becomes TestOpenKeepsALiveBrowserWhenReuseNavigateFails: the contract changed for a navigation that fails on a live page. A missed target's error now tells the agent to read the page again. --- internal/browser/actions.go | 7 +++++ internal/browser/browser_test.go | 45 ++++++++++++++++++++++++++++---- internal/browser/chrome.go | 2 +- internal/browser/fake_test.go | 24 ++++++++++------- internal/browser/view.go | 4 +-- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/internal/browser/actions.go b/internal/browser/actions.go index d164b8379..7550e1d4a 100644 --- a/internal/browser/actions.go +++ b/internal/browser/actions.go @@ -29,7 +29,14 @@ func (s *Service) touch(sess *session) { } } +// drop takes out a browser a call failed on, if its Chrome is gone. One that +// is alive stays: a click on a stale index or on a page between documents +// fails and the next call works, and closing the browser lost the person's +// page and the agent's login with it. func (s *Service) drop(sess *session) { + if sess.alive() { + return + } s.mu.Lock() defer s.unlock() if s.byID[sess.id] == sess { diff --git a/internal/browser/browser_test.go b/internal/browser/browser_test.go index ffb06069b..f23fa6293 100644 --- a/internal/browser/browser_test.go +++ b/internal/browser/browser_test.go @@ -306,6 +306,7 @@ func TestFailedInfoDropsTheBrowser(t *testing.T) { t.Fatal(err) } launch.pages[0].infoErr = errors.New("chrome gone") + launch.pages[0].dead = true if _, err := s.Info("s"); err == nil { t.Fatal("Info succeeded on a dead page") } @@ -315,18 +316,33 @@ func TestFailedInfoDropsTheBrowser(t *testing.T) { } } -func TestOpenDropsWhenReuseNavigateFails(t *testing.T) { +func TestOpenKeepsALiveBrowserWhenReuseNavigateFails(t *testing.T) { launch := &fakeLauncher{} s := testService(t, launch) if _, err := s.Open("s", "https://a.example"); err != nil { t.Fatal(err) } - launch.pages[0].navErr = errors.New("gone") + launch.pages[0].navErr = errors.New("net::ERR_NAME_NOT_RESOLVED") if _, err := s.Open("s", "https://b.example"); err == nil { - t.Fatal("Open reused a page that could not navigate") + t.Fatal("Open reported success for a navigation that failed") } - if !launch.pages[0].closed { - t.Fatal("failed reuse left the page open") + if launch.pages[0].closed { + t.Fatal("a failed navigation closed a browser whose Chrome is alive") + } +} + +func TestOpenReplacesABrowserWhoseChromeDied(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Open("s", "https://a.example"); err != nil { + t.Fatal(err) + } + launch.pages[0].dead = true + if _, err := s.Open("s", "https://b.example"); err != nil { + t.Fatal(err) + } + if len(launch.pages) != 2 || launch.pages[1].url != "https://b.example" { + t.Fatalf("pages %d: a dead Chrome should be replaced by a fresh one at the new URL", len(launch.pages)) } } @@ -466,3 +482,22 @@ func TestLaunchDoesNotHoldOtherSessions(t *testing.T) { t.Fatal(err) } } + +func TestAFailedAgentActionKeepsALiveBrowser(t *testing.T) { + launch := &fakeLauncher{} + s := testService(t, launch) + if _, err := s.Open("s", "https://example.com"); err != nil { + t.Fatal(err) + } + launch.pages[0].clickErr = errors.New("no element matches that target") + idx := 9 + if err := s.Click("s", Target{Index: &idx}); err == nil { + t.Fatal("a click that missed its target reported success") + } + if launch.pages[0].closed { + t.Fatal("a click that missed its target closed the browser") + } + if list, _ := s.List("s"); len(list) != 1 { + t.Fatal("the browser left the list after a missed click") + } +} diff --git a/internal/browser/chrome.go b/internal/browser/chrome.go index 6d22985e8..2c0ce5d47 100644 --- a/internal/browser/chrome.go +++ b/internal/browser/chrome.go @@ -357,7 +357,7 @@ func (p *chromePage) resolve(ctx context.Context, t Target) (point, error) { return point{}, err } if r == nil { - return point{}, fmt.Errorf("no element matches that target") + return point{}, fmt.Errorf("no element matches that target: the page may have changed since it was read; read it again for fresh indexes") } if r.Outside { return point{}, fmt.Errorf("%g,%g is outside the %gx%g viewport: scroll first, or click by index or selector, which scroll the element into view", diff --git a/internal/browser/fake_test.go b/internal/browser/fake_test.go index e3c03d3bf..007d054a4 100644 --- a/internal/browser/fake_test.go +++ b/internal/browser/fake_test.go @@ -34,14 +34,17 @@ type fakePage struct { viewports []Viewport mice []api.BrowserMouseOptions mouseErr error - keys []api.BrowserKeyOptions - texts []string - settles int - selection string - info PageInfo - shotErr error - navErr error - infoErr error + clickErr error + // dead is a Chrome whose process ended under the page. + dead bool + keys []api.BrowserKeyOptions + texts []string + settles int + selection string + info PageInfo + shotErr error + navErr error + infoErr error } type typed struct { @@ -58,7 +61,7 @@ type pressed struct { func (p *fakePage) Alive() bool { p.mu.Lock() defer p.mu.Unlock() - return !p.closed + return !p.closed && !p.dead } func (p *fakePage) Close() error { @@ -105,6 +108,9 @@ func (p *fakePage) Info(context.Context) (PageInfo, error) { func (p *fakePage) Click(_ context.Context, t Target) (x, y float64, err error) { p.mu.Lock() defer p.mu.Unlock() + if p.clickErr != nil { + return 0, 0, p.clickErr + } p.clicks = append(p.clicks, t) return p.clickAt[0], p.clickAt[1], nil } diff --git a/internal/browser/view.go b/internal/browser/view.go index fcde4d6a3..2602df6e1 100644 --- a/internal/browser/view.go +++ b/internal/browser/view.go @@ -119,9 +119,7 @@ func (s *Service) steer(owner string, wait time.Duration, fn func(Page, context. ctx, cancel := context.WithTimeout(context.Background(), wait) defer cancel() if err := fn(sess.page, ctx); err != nil { - if !sess.alive() { - s.drop(sess) - } + s.drop(sess) return nil, err } s.touch(sess) From 0fda3edf35f04d461cc8b72f31c5fe130e03943d Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:48:21 -0300 Subject: [PATCH 24/25] docs(changelog): the session browser's agent cursor, clipboard and compact page reads --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6492962c..8dba82f81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- A session card's menu opens a browser for that session, above Open in, in a tab beside the session: its terminal on one side and the page on the other, with the divider, rotate and zoom of a split. The session's agent drives the same page: open a URL, read it, click, type, press a key, go back and forward, scroll and screenshot, from `browser_*` tools or from `lich browser`, and the pane marks what the agent does. You click, type and paste in the page while it does. When the agent opens a browser on its own, a toast in the bottom-right corner offers to show it. Open in a window moves the page to a Chromium window of its own, cookies and all. Closing the card closes the browser. The browser is the Chrome, Edge, Chromium, Brave, Vivaldi or Helium already installed. opencode's agent reaches it through `lich browser`; its own tool list does not include `browser_*`. +- A session card's menu opens a browser for that session, above Open in, in a tab beside the session: its terminal on one side and the page on the other, with the divider, rotate and zoom of a split. The session's agent drives the same page: open a URL, read it, click, type, press a key, go back and forward, scroll and screenshot, from `browser_*` tools or from `lich browser`. The agent's own cursor shows where it acts: lich's arrow, in the session's card color or the theme's, with a badge naming the provider, glides to each click, field and scroll on the page and to back, forward, reload and the address on the pane's toolbar, playing even an agent's parallel calls in turn. You click, type and copy and paste with your own clipboard in the page while it does. Each click and key the agent sends answers with the page it left, in a compact form, so a long session spends about half the tokens it would reading the page back as JSON. When the agent opens a browser on its own, a toast in the bottom-right corner offers to show it. Open in a window moves the page to a Chromium window of its own, cookies and all. Closing the card closes the browser. The browser is the Chrome, Edge, Chromium, Brave, Vivaldi or Helium already installed. opencode's agent reaches it through `lich browser`; its own tool list does not include `browser_*`. ### Fixed From 8c5fc857b4f25f83c971dcb3cd76872717a53c5b Mon Sep 17 00:00:00 2001 From: omartelo Date: Mon, 26 Oct 2026 22:57:42 -0300 Subject: [PATCH 25/25] ci: let the runner's Chromium start its sandbox for the browser live tests Ubuntu 24.04 runners restrict unprivileged user namespaces through AppArmor, and Chromium aborts with "No usable sandbox!" there, failing every internal/browser live test on linux while macOS and Windows pass. --- .github/workflows/ci.yml | 6 ++++++ .github/workflows/release.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1060a1d07..fe371f61b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,12 @@ jobs: exit 1 fi + # Ubuntu 24.04 refuses unprivileged user namespaces to binaries without an + # AppArmor profile, the runner's Chromium among them, and Chromium will not + # start without its sandbox: the internal/browser live tests need it. + - name: Allow Chromium's sandbox + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Backend tests run: bash .github/scripts/go-test-summary.sh linux diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c0d607170..8d7397a5e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,6 +139,12 @@ jobs: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + # Ubuntu 24.04 refuses unprivileged user namespaces to binaries without an + # AppArmor profile, the runner's Chromium among them, and Chromium will not + # start without its sandbox: the internal/browser live tests need it. + - name: Allow Chromium's sandbox + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + # The suite emits pass/fail counts + coverage into the job summary and # still fails the job on a red test (shared with ci.yml). - name: Backend tests