From fa4e521d338ea14aef6e82879ad81029d3c9835a Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Wed, 22 Jul 2026 09:35:35 -0700 Subject: [PATCH 1/4] feat(cloak): add -p cloak stealth-browser profile and auth-session support Second of two commits on this branch, on top of the kimi batch. Adds the CloakBrowser profile (#456) and the image/tag precedence fix it needs. - cloak profile: `-p cloak` runs agents in a CloakBrowser image (Dockerfile.cloak, extends :rust) -- source-patched stealth Chromium, headed on Xvfb :99 + openbox, binary baked at build (auto-update off). Ships the deva-cloak skill, make build-cloak/test-cloak, a paths-filtered cloak-image.yml, and release :cloak / :vX.Y.Z-cloak tags. - auth sessions: a persistent host profile mount (~/.config/deva/ cloak-profile -> userDataDir) so a login survives container removal. Interactive login over VNC is on-demand and direct-reach on OrbStack/Linux -- the agent starts x11vnc and you connect at the container IP; --cloak-vnc is the Docker-Desktop fallback that publishes a loopback port. --cloak-browser is an optional always-on cloak-browserd daemon the agent attaches to over CDP. - fix(precedence): env > config > -p > default, so an explicit -p beats a .deva DEVA_DOCKER_TAG (guarded by scripts/test-image-precedence.sh). Not built/smoke-tested locally (container network blocks apt); release CI builds the image on GitHub runners. Co-Authored-By: Claude Opus 4.8 (1M context) --- .dockerignore | 1 + .github/workflows/cloak-image.yml | 54 +++++++ .github/workflows/release.yml | 57 ++++++- AGENTS.md | 3 +- CHANGELOG.md | 36 +++++ DEV-LOGS.md | 37 ++++- Dockerfile.cloak | 88 +++++++++++ Makefile | 51 ++++++- README.md | 1 + deva.sh | 245 +++++++++++++++++++++++++++--- docker-entrypoint.sh | 42 +++++ docs/advanced-usage.md | 10 ++ docs/custom-images.md | 6 + llms.txt | 7 +- scripts/cloak-browserd.mjs | 57 +++++++ scripts/cloak-entrypoint.sh | 53 +++++++ scripts/test-image-precedence.sh | 70 +++++++++ scripts/update-version-pins.sh | 5 + scripts/version-pins.sh | 1 + skills/deva-cloak/SKILL.md | 223 +++++++++++++++++++++++++++ versions.env | 4 + 21 files changed, 1019 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/cloak-image.yml create mode 100644 Dockerfile.cloak create mode 100644 scripts/cloak-browserd.mjs create mode 100644 scripts/cloak-entrypoint.sh create mode 100644 scripts/test-image-precedence.sh create mode 100644 skills/deva-cloak/SKILL.md diff --git a/.dockerignore b/.dockerignore index 25f44aa..075b74a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -31,6 +31,7 @@ compose.yaml # References directory (don't need in image) references/ +reference/ # Claude trace logs .claude-trace/ diff --git a/.github/workflows/cloak-image.yml b/.github/workflows/cloak-image.yml new file mode 100644 index 0000000..fd890c7 --- /dev/null +++ b/.github/workflows/cloak-image.yml @@ -0,0 +1,54 @@ +name: Cloak Image + +# Paths-filtered on purpose: the cloak build bakes a ~200MB Chromium binary, +# too heavy for every PR. Nightly also skips cloak deliberately — it only +# refreshes agent CLI versions, none of which live in the cloak layer. +# Release builds and pushes the real :cloak tag. +on: + pull_request: + branches: [ main ] + paths: + - "Dockerfile.cloak" + - "scripts/cloak-entrypoint.sh" + - "versions.env" + - ".github/workflows/cloak-image.yml" + push: + branches: [ main ] + paths: + - "Dockerfile.cloak" + - "scripts/cloak-entrypoint.sh" + - "versions.env" + - ".github/workflows/cloak-image.yml" + +jobs: + build-and-test: + name: Build and Test Cloak Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load pinned versions + id: pins + shell: bash + run: | + set -euo pipefail + source ./scripts/version-pins.sh + load_version_pins + emit_github_outputs "$GITHUB_OUTPUT" + + # Build on top of the published rust image: this validates the cloak + # layer itself without rebuilding the whole base/rust stack in CI. + - name: Build cloak image + run: | + make build-cloak-image \ + IMAGE_NAME=deva-smoke \ + CLOAK_TAG=ci-cloak \ + RUST_IMAGE=ghcr.io/thevibeworks/deva:rust \ + CLOAKBROWSER_WRAPPER_VERSION="${{ steps.pins.outputs.cloakbrowser_wrapper_version }}" + + - name: Test cloak image + run: | + make test-cloak \ + IMAGE_NAME=deva-smoke \ + CLOAK_TAG=ci-cloak diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78191b8..76d9f26 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,6 +55,7 @@ jobs: ccx_version: ${{ steps.pins.outputs.ccx_version }} copilot_api_version: ${{ steps.pins.outputs.copilot_api_version }} playwright_version: ${{ steps.pins.outputs.playwright_version }} + cloakbrowser_wrapper_version: ${{ steps.pins.outputs.cloakbrowser_wrapper_version }} rust_toolchains: ${{ steps.pins.outputs.rust_toolchains }} rust_default_toolchain: ${{ steps.pins.outputs.rust_default_toolchain }} rust_targets: ${{ steps.pins.outputs.rust_targets }} @@ -216,10 +217,60 @@ jobs: RUST_DEFAULT_TOOLCHAIN=${{ needs.load-version-pins.outputs.rust_default_toolchain }} RUST_TARGETS=${{ needs.load-version-pins.outputs.rust_targets }} + build-and-push-cloak: + name: Build and Push Cloak Profile Image + runs-on: ubuntu-latest + needs: [prepare, load-version-pins, build-and-push-rust] + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ needs.prepare.outputs.release_tag }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for cloak profile + id: meta-cloak + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ needs.prepare.outputs.release_tag }}-cloak + type=raw,value=cloak + + - name: Build and push cloak image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.cloak + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta-cloak.outputs.tags }} + labels: ${{ steps.meta-cloak.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.release_tag }}-rust + CLOAKBROWSER_WRAPPER_VERSION=${{ needs.load-version-pins.outputs.cloakbrowser_wrapper_version }} + release: name: Create GitHub Release runs-on: ubuntu-latest - needs: [prepare, build-and-push, build-and-push-rust] + needs: [prepare, build-and-push, build-and-push-rust, build-and-push-cloak] permissions: contents: write steps: @@ -258,6 +309,10 @@ jobs: echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${RELEASE_TAG}-rust\`" >> release_notes.md echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:rust\`" >> release_notes.md echo "" >> release_notes.md + echo "**Cloak Profile (CloakBrowser stealth Chromium, headed):**" >> release_notes.md + echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${RELEASE_TAG}-cloak\`" >> release_notes.md + echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:cloak\`" >> release_notes.md + echo "" >> release_notes.md echo "## Supported Architectures" >> release_notes.md echo "" >> release_notes.md echo "- linux/amd64" >> release_notes.md diff --git a/AGENTS.md b/AGENTS.md index a30b3c8..982e6b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,6 @@ runtime is Linux. - $HOME is /home/deva (not /root). sudo works without password. - Pre-installed: Node.js, Python (use `uv`, not pip), Go, git, gh, make, curl. pip is NOT in PATH. -- Docker is available (socket mounted from host). -- System packages and build caches persist across sessions. +- Ephemeral container. Installed packages will not persist. - Container details are in DEVA_* environment variables. diff --git a/CHANGELOG.md b/CHANGELOG.md index df70396..177a39e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Makefile, Dockerfile(+rust), version-pins/upgrade/update scripts, ci/nightly/release workflows; tests added (release-utils registry, version-upgrade mock, install-tooling, and `test-kimi-auth.sh`). +- cloak authenticated-session support (#456): composable pieces for logging + into sites inside the stealth browser. The session persists via a host + profile dir (`~/.config/deva/cloak-profile` -> Chromium `userDataDir`, + auto-mounted for either path below) so a login done once survives container + removal. + - Interactive login over VNC: on OrbStack/native Linux the host reaches the + container directly, so the agent starts `x11vnc` on the `:99` display on + demand and you connect at the container IP -- no flag, no port publish. + - `--cloak-vnc` (`DEVA_CLOAK_VNC=1`): Docker-Desktop fallback that publishes + the VNC display on a host-loopback port (there the container IP is not + routable). Auto-generates a VNC password (printed once) unless + `DEVA_CLOAK_VNC_PASSWORD` is set; the published port is fixed at container + create. Independent of `--cloak-browser`. + - `--cloak-browser` (`DEVA_CLOAK_BROWSER=1`): an always-on CloakBrowser + daemon (`cloak-browserd`) holds one headed, persistent browser on the + cloak display with a loopback CDP endpoint. The agent drives it via + `connectOverCDP(CLOAK_CDP_ENDPOINT)` -- the same live browser, not a + throwaway. Starts post-UID-remap in docker-entrypoint so profile files + match the mounted host owner. + Documented in the `deva-cloak` skill. +- cloak profile (#456): `deva.sh -p cloak ` runs agents in a new + image (`Dockerfile.cloak`, extends `:rust`) with CloakBrowser stealth + Chromium, headed on Xvfb `:99` + openbox. The Chromium binary is baked + at build time (hermetic, auto-update off); `CLOAKBROWSER_WRAPPER_VERSION` + in versions.env pins the wrapper and thereby Chromium. Ships the + `deva-cloak` skill (keyed off `DEVA_CLOAK=1`), `make build-cloak` / + `test-cloak` targets, a paths-filtered `cloak-image.yml` CI workflow, + and release pushes `:cloak` / `:vX.Y.Z-cloak` tags. Nightly skips cloak + on purpose — that layer contains no agent CLIs. ### Fixed - `--trace` launch killed by `cp: cannot create regular file @@ -37,6 +66,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 test became the subshell's exit status and `set -e` killed every non-verbose traced launch right after the banner; the subshell now exits 0 explicitly. +- image/tag precedence: a `.deva` config file setting `DEVA_DOCKER_TAG` + (or `DEVA_DOCKER_IMAGE`) silently overrode the real environment and made + CLI `-p` a no-op — `deva.sh -p cloak` with a config `DEVA_DOCKER_TAG=rust` + launched rust, and even `DEVA_DOCKER_TAG=cloak deva.sh ...` was clobbered + back to the config value. Precedence is now environment > config files > + `-p` profile > default, with explicit CLI `-p` beating a config-file tag. + Guarded by `scripts/test-image-precedence.sh` in CI. ## [0.16.0] - 2026-07-14 diff --git a/DEV-LOGS.md b/DEV-LOGS.md index 422fa8b..8932701 100644 --- a/DEV-LOGS.md +++ b/DEV-LOGS.md @@ -13,11 +13,40 @@ - Minimal markdown markers, no unnecessary formatting, minimal emojis. - Reference issue numbers in the format `#` for easy linking. -# [2026-07-21] Dev Log: traced launches die silently in freshly built images #414 -- Why: after rebuilding images, every `--trace` launch died with "failed to launch ephemeral container" and no error. +# [2026-07-21] Dev Log: cloak always-on browser + VNC + persistent profile #456 +- Why: three asks for auth sites in the cloak browser — (1) VNC auto-secured with a password, (2) an always-on browser the agent can drive freely, (3) mount what preserves the whole session to the host. Verified the load-bearing assumption first in a throwaway cloak container: a persistent CloakBrowser launched with `--remote-debugging-port` exposes a CDP endpoint a SEPARATE Playwright client reaches via connectOverCDP, drives, and the browser survives that client disconnecting. So one daemon can back both the human (VNC) and the agent (CDP). - What: - - The #414 flock rework ended the CA-install subshell on `[ "$VERBOSE" = "true" ] && echo ...`; with VERBOSE unset the failed test became the subshell's exit status, and under `set -e` the entrypoint died right after the agent banner — silently, only in freshly built images (the entrypoint is baked). Fix: explicit `exit 0` before the subshell closes. Verified A/B: baked image dies, overlay-fixed image runs `deva.sh claude --trace --rm -Q -- --version` end-to-end (cctrace MITM up, trace file written). Rebuild images to pick this up. -- Result: traced launch verified live against the host daemon. + - scripts/cloak-browserd.mjs: daemon holding launchPersistentContext (headed :99, userDataDir, `--remote-debugging-port=9222 --remote-debugging-address=127.0.0.1`), relaunch-on-close, SIGTERM-clean. Absolute import of cloakbrowser (lives outside $CLOAK_APP_DIR). Verified end-to-end (daemon up → agent connectOverCDP drives it → persists after disconnect). + - docker-entrypoint.sh maybe_start_cloak_browser: starts the daemon AFTER setup_nonroot_user (post UID/GID remap) so the mounted profile is written as the host UID — starting it in cloak-entrypoint (pre-remap) would leave 1001-owned files the agent can't touch. Idempotent under flock + a CDP-port probe: boot (`tail -f /dev/null`) starts it once under PID 1's tree, later agent execs find it up and skip. Verified branch logic with mocks (gate off, port-up skip, start path). + - deva.sh: `--cloak-browser`/`DEVA_CLOAK_BROWSER` (env + CDP endpoint + auto-mount `~/.config/deva/cloak-profile`→`/home/deva/.cloak-profile`, first-writer-wins). `--cloak-vnc` implies it (blank display otherwise). setup_cloak_runtime = one image guard + one warning + correct order. VNC auto-password: generated when unset, printed once in the announce; the var name contains PASSWORD so deva's existing redaction masks it in --debug. announce_cloak_browser prints the CDP endpoint + host profile path. + - Bug caught in test: the VNC password gen `tr -dc ... config files > `-p` > default. scripts/test-image-precedence.sh pins all four cases and runs in CI; it `env -u`s the deva container's own `DEVA_DOCKER_TAG` so the harness env can't fake results — which is exactly how the bug hid in earlier dry-run smokes. +- Result: both repro commands (`deva -p cloak`, `DEVA_DOCKER_TAG=cloak deva -p cloak claude` with a rust-pinned config) now resolve `:cloak`; traced launch verified live against the host daemon. + +# [2026-07-20] Dev Log: cloak profile — CloakBrowser stealth Chromium image #456 +- Why: agents in the sandbox need a browser that passes bot detection (Cloudflare, FingerprintJS, reCAPTCHA scoring); plain Playwright headless in the rust image fails those checks. CloakBrowser is a source-patched Chromium with the same Playwright API — run it headed on a virtual display inside the container. +- What: + - Dockerfile.cloak extends :rust: apt layer (xvfb/openbox/xdotool + anti-bot font packages), wrapper+playwright-core installed into CLOAK_APP_DIR=/opt/cloak (not global — bare ESM specifiers don't resolve from a global install), `cloakbrowser install` bakes the Chromium binary at build time with auto-update off (hermetic; the free v146 binary stays on GitHub Releases forever). Kept tini as PID 1 and restated `CMD ["claude"]` — redeclaring ENTRYPOINT resets the inherited CMD. + - cloak-entrypoint.sh: clean stale X locks, Xvfb :99 (1920x1080x24), poll display-ready via xdotool (bounded ~10s), openbox for --start-maximized, then exec docker-entrypoint.sh. + - deva.sh: `-p cloak` profile, ghcr tag `cloak`, local-tag fallback cloak > rust > latest; skills/deva-cloak documents agent-side usage (DEVA_CLOAK=1 marker, run from $CLOAK_APP_DIR). + - Version pin CLOAKBROWSER_WRAPPER_VERSION (wrapper hardcodes per-arch Chromium versions, so it pins Chromium too) wired through versions.env, Makefile (+VERSION_QUERY_OVERRIDES), version-pins.sh VERSION_PIN_VARS, update-version-pins.sh heredoc+refresh — the first draft omitted the heredoc entry, so `make versions-pin` would have silently deleted the pin. + - CI: paths-filtered cloak-image.yml builds on ghcr :rust and runs make test-cloak; release pushes :cloak and :vX.Y.Z-cloak (BASE_IMAGE = the release's -rust tag). Nightly skips cloak: that layer has no agent CLIs, and the ~200MB Chromium bake would double nightly cost for nothing. +- Result: cloak rides feat/kimi-agent (#455) — bundling two features on one branch violates our one-branch-per-issue rule; noted for the PR split. Build verified via docker on the host socket; test-cloak asserts cloakbrowser info + display geometry. # [2026-07-20] Dev Log: add kimi agent (Moonshot Kimi Code CLI) #455 - Why: Moonshot shipped an official coding-agent CLI (@moonshot-ai/kimi-code, bin `kimi`); deva should launch it like claude/codex/gemini/grok. Traced the grok integration (commit 93a59e0) via ccx as the template. diff --git a/Dockerfile.cloak b/Dockerfile.cloak new file mode 100644 index 0000000..cc21081 --- /dev/null +++ b/Dockerfile.cloak @@ -0,0 +1,88 @@ +# syntax=docker/dockerfile:1.4 +# deva.sh - Cloak Browser Image +# Extends rust image with CloakBrowser stealth Chromium + headed display server + +ARG BASE_IMAGE=ghcr.io/thevibeworks/deva:rust +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="deva-cloak" +LABEL org.opencontainers.image.description="Stealth browser environment with CloakBrowser headed Chromium" + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Xvfb + openbox for headed mode; xdotool for display-ready polling. +# x11vnc exposes the Xvfb display so a human can drive the browser for +# interactive logins (MFA/captcha) -- started on demand, see cloak-entrypoint. +# CloakBrowser needs the same Chromium system deps already in the rust image, +# plus the font packages that anti-bot canvas checks require. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + xvfb xdotool openbox x11vnc \ + fonts-noto-color-emoji fonts-freefont-ttf fonts-unifont \ + fonts-ipafont-gothic fonts-wqy-zenhei fonts-tlwg-loma-otf + +# Hermetic image: the free Chromium binary is baked at build time, so disable +# runtime auto-update -- no surprise network fetch or binary drift on launch +# inside the sandbox. v146 stays free on GitHub Releases forever, so the pinned +# binary never disappears; bump CLOAKBROWSER_WRAPPER_VERSION to move it. +# DEVA_CLOAK=1 detection marker -- the deva-cloak skill and agents key off it. +# CLOAK_APP_DIR fixed dir holding the wrapper's node_modules, so a script run +# from here can `import 'cloakbrowser'` (bare ESM specifiers +# resolve from cwd; a global install is not importable). +ENV DEVA_CLOAK=1 \ + CLOAK_APP_DIR=/opt/cloak \ + DISPLAY=:99 \ + CLOAKBROWSER_CACHE_DIR=/opt/cloakbrowser \ + CLOAKBROWSER_AUTO_UPDATE=false + +RUN mkdir -p "$CLOAKBROWSER_CACHE_DIR" "$CLOAK_APP_DIR" && \ + chown -R "$DEVA_UID:$DEVA_GID" "$CLOAKBROWSER_CACHE_DIR" "$CLOAK_APP_DIR" + +USER $DEVA_USER + +# npm wrapper version. NOT the cloakbrowser CLOAKBROWSER_VERSION env var, which +# is a Chromium binary pin -- keep these names distinct. The wrapper hardcodes +# the per-arch free Chromium version, so pinning the wrapper pins Chromium too. +# Declared here, below the apt layer: an in-scope ARG is part of every later +# RUN's cache key, and a wrapper bump must not rebuild the apt/font layer. +ARG CLOAKBROWSER_WRAPPER_VERSION=0.4.12 + +LABEL org.opencontainers.image.cloakbrowser_wrapper_version=${CLOAKBROWSER_WRAPPER_VERSION} + +# Install the CloakBrowser JS wrapper + playwright-core into CLOAK_APP_DIR (not +# global -- ESM can't import a bare specifier from a global install), then +# pre-download the stealth Chromium binary during build so runtime needs no net. +# The npm cache lives in a build-cache mount, not the layer -- no clean needed. +RUN --mount=type=cache,target=/home/deva/.npm,uid=${DEVA_UID},gid=${DEVA_GID},sharing=locked \ + cd "$CLOAK_APP_DIR" && \ + npm init -y >/dev/null && \ + npm install --no-audit --no-fund \ + "cloakbrowser@${CLOAKBROWSER_WRAPPER_VERSION}" playwright-core && \ + ./node_modules/.bin/cloakbrowser install && \ + ./node_modules/.bin/cloakbrowser info + +USER root + +# Expose the CloakBrowser CLI globally (`cloakbrowser info|install|update`) +RUN ln -sf "$CLOAK_APP_DIR/node_modules/.bin/cloakbrowser" /usr/local/bin/cloakbrowser + +COPY --chmod=755 scripts/cloak-entrypoint.sh /usr/local/bin/cloak-entrypoint.sh + +# Always-on browser daemon: launched by docker-entrypoint when +# DEVA_CLOAK_BROWSER=1. Lives outside CLOAK_APP_DIR and imports cloakbrowser +# by absolute path, so it runs from anywhere. +COPY scripts/cloak-browserd.mjs /usr/local/lib/cloak/cloak-browserd.mjs + +WORKDIR /root + +# Start Xvfb + openbox, then chain to the standard entrypoint. Keep tini as +# PID 1 (same as base/rust) -- cloak-entrypoint backgrounds Xvfb and openbox, +# and their orphans need a reaper. Redeclaring ENTRYPOINT resets the +# inherited CMD, so restate the default agent. +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/cloak-entrypoint.sh"] + +CMD ["claude"] diff --git a/Makefile b/Makefile index d3b8b9a..f9d95e7 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,15 @@ VERSION_PINS_FILE ?= versions.env IMAGE_NAME := ghcr.io/thevibeworks/deva TAG := latest RUST_TAG := rust +CLOAK_TAG := cloak CORE_TAG := core DOCKERFILE := Dockerfile RUST_DOCKERFILE := Dockerfile.rust +CLOAK_DOCKERFILE := Dockerfile.cloak MULTI_ARCH_PLATFORMS := linux/amd64,linux/arm64 MAIN_IMAGE := $(IMAGE_NAME):$(TAG) RUST_IMAGE := $(IMAGE_NAME):$(RUST_TAG) +CLOAK_IMAGE := $(IMAGE_NAME):$(CLOAK_TAG) CORE_IMAGE := $(IMAGE_NAME):$(CORE_TAG) CONTAINER_NAME := deva-$(shell basename $(PWD))-$(shell date +%s) @@ -39,6 +42,7 @@ KIMI_CODE_VERSION ?= 0.28.0 CCX_VERSION ?= v0.1.4 COPILOT_API_VERSION ?= 0ea08febdd7e3e055b03dd298bf57e669500b5c1 PLAYWRIGHT_VERSION ?= 1.60.0 +CLOAKBROWSER_WRAPPER_VERSION ?= 0.4.12 RUST_TOOLCHAINS ?= stable RUST_DEFAULT_TOOLCHAIN ?= stable RUST_TARGETS ?= wasm32-unknown-unknown @@ -97,6 +101,9 @@ RUST_BUILD_ARGS := $(AGENT_BUILD_ARGS) \ --build-arg RUST_DEFAULT_TOOLCHAIN=$(RUST_DEFAULT_TOOLCHAIN) \ --build-arg RUST_TARGETS=$(RUST_TARGETS) +CLOAK_BUILD_ARGS := \ + --build-arg CLOAKBROWSER_WRAPPER_VERSION=$(CLOAKBROWSER_WRAPPER_VERSION) + VERSION_QUERY_OVERRIDES := \ $(if $(filter command line environment environment\ override override,$(origin NODE_MAJOR)),NODE_MAJOR=$(NODE_MAJOR)) \ $(if $(filter command line environment environment\ override override,$(origin GO_VERSION)),GO_VERSION=$(GO_VERSION)) \ @@ -113,6 +120,7 @@ VERSION_QUERY_OVERRIDES := \ $(if $(filter command line environment environment\ override override,$(origin CCX_VERSION)),CCX_VERSION=$(CCX_VERSION)) \ $(if $(filter command line environment environment\ override override,$(origin COPILOT_API_VERSION)),COPILOT_API_VERSION=$(COPILOT_API_VERSION)) \ $(if $(filter command line environment environment\ override override,$(origin PLAYWRIGHT_VERSION)),PLAYWRIGHT_VERSION=$(PLAYWRIGHT_VERSION)) \ + $(if $(filter command line environment environment\ override override,$(origin CLOAKBROWSER_WRAPPER_VERSION)),CLOAKBROWSER_WRAPPER_VERSION=$(CLOAKBROWSER_WRAPPER_VERSION)) \ $(if $(filter command line environment environment\ override override,$(origin RUST_TOOLCHAINS)),RUST_TOOLCHAINS=$(RUST_TOOLCHAINS)) \ $(if $(filter command line environment environment\ override override,$(origin RUST_DEFAULT_TOOLCHAIN)),RUST_DEFAULT_TOOLCHAIN=$(RUST_DEFAULT_TOOLCHAIN)) \ $(if $(filter command line environment environment\ override override,$(origin RUST_TARGETS)),RUST_TARGETS=$(RUST_TARGETS)) @@ -122,9 +130,9 @@ export VERSION_PINS_FILE .DEFAULT_GOAL := help -.PHONY: build build-network-check build-main rebuild build-core build-rust-image build-rust build-all -.PHONY: buildx buildx-multi buildx-multi-rust buildx-multi-local -.PHONY: versions-up versions versions-pin toolchains scripts commands clean clean-all shell test test-rust test-local +.PHONY: build build-network-check build-main rebuild build-core build-rust-image build-rust build-cloak-image build-cloak build-all +.PHONY: buildx buildx-multi buildx-multi-rust buildx-multi-cloak buildx-multi-local +.PHONY: versions-up versions versions-pin toolchains scripts commands clean clean-all shell test test-rust test-cloak test-local .PHONY: info push pull build-test dev context-size lint version-check .PHONY: release-patch release-minor release-major help @@ -216,11 +224,22 @@ build-rust-image: build-network-check build-rust: build-core build-rust-image +build-cloak-image: build-network-check + @echo "🔨 Building CloakBrowser Docker image..." + docker build $(DOCKER_BUILD_FLAGS) -f $(CLOAK_DOCKERFILE) \ + --build-arg BASE_IMAGE=$(RUST_IMAGE) \ + $(CLOAK_BUILD_ARGS) \ + -t $(CLOAK_IMAGE) . + @echo "✅ Cloak build completed: $(CLOAK_IMAGE)" + +build-cloak: build-rust build-cloak-image + build-all: @echo "🔨 Building all images with pins from $(VERSION_PINS_FILE)..." @$(MAKE) build-core @$(MAKE) build-main @$(MAKE) build-rust-image + @$(MAKE) build-cloak-image @echo "✅ All images built successfully" buildx: @@ -243,6 +262,14 @@ buildx-multi-rust: --push -t $(RUST_IMAGE) . @echo "✅ Multi-arch Rust build completed and pushed: $(RUST_IMAGE)" +buildx-multi-cloak: + @echo "🔨 Building multi-arch Cloak images for amd64 and arm64..." + docker buildx build $(DOCKER_BUILD_FLAGS) -f $(CLOAK_DOCKERFILE) --platform $(MULTI_ARCH_PLATFORMS) \ + --build-arg BASE_IMAGE=$(RUST_IMAGE) \ + $(CLOAK_BUILD_ARGS) \ + --push -t $(CLOAK_IMAGE) . + @echo "✅ Multi-arch Cloak build completed and pushed: $(CLOAK_IMAGE)" + buildx-multi-local: @echo "🔨 Building multi-arch images locally..." docker buildx build $(DOCKER_BUILD_FLAGS) -f $(DOCKERFILE) --platform $(MULTI_ARCH_PLATFORMS) \ @@ -342,6 +369,18 @@ test-rust: docker run --rm $(RUST_IMAGE) bash -c 'cargo-watch --version && wasm-pack --version' @echo "✅ Rust tests passed" +test-cloak: + @echo "🧪 Testing $(CLOAK_IMAGE)..." + @echo "Testing CloakBrowser binary..." + docker run --rm $(CLOAK_IMAGE) bash -c 'cloakbrowser info --quick' + @echo "Testing Xvfb display..." + docker run --rm $(CLOAK_IMAGE) bash -c 'xdotool getdisplaygeometry' + @echo "Testing x11vnc present (VNC login path)..." + docker run --rm $(CLOAK_IMAGE) bash -c 'command -v x11vnc' + @echo "Testing always-on browser daemon present..." + docker run --rm $(CLOAK_IMAGE) bash -c 'node --check /usr/local/lib/cloak/cloak-browserd.mjs' + @echo "✅ Cloak tests passed" + test-local: @echo "🧪 Testing $(MAIN_IMAGE) with local directory..." docker run --rm -it \ @@ -409,7 +448,8 @@ help: @echo " build-core Build stable core image only" @echo " build-main Build main Docker image only" @echo " build-rust Build Rust Docker image" - @echo " build-all Build all images (main + rust)" + @echo " build-cloak Build CloakBrowser Docker image" + @echo " build-all Build all images (main + rust + cloak)" @echo " rebuild Rebuild without cache" @echo " buildx Build with buildx" @echo " buildx-multi Build multi-arch and push" @@ -423,6 +463,7 @@ help: @echo " commands Alias for help" @echo " test Test main image" @echo " test-rust Test Rust image" + @echo " test-cloak Test CloakBrowser image" @echo " shell Open shell in container" @echo " clean Aggressive cleanup (unused containers/images/networks/cache)" @echo " clean-all NUCLEAR cleanup (ALL unused Docker resources + volumes)" @@ -452,6 +493,7 @@ help: @echo " KIMI_CODE_VERSION Kimi Code CLI version (default: $(KIMI_CODE_VERSION))" @echo " CCX_VERSION Atlas CLI version (default: $(CCX_VERSION))" @echo " PLAYWRIGHT_VERSION Playwright version (default: $(PLAYWRIGHT_VERSION))" + @echo " CLOAKBROWSER_WRAPPER_VERSION CloakBrowser npm wrapper version (default: $(CLOAKBROWSER_WRAPPER_VERSION))" @echo " RUST_TOOLCHAINS Rust toolchains to install (default: $(RUST_TOOLCHAINS))" @echo " RUST_DEFAULT_TOOLCHAIN Rust default toolchain (default: $(RUST_DEFAULT_TOOLCHAIN))" @echo "" @@ -460,6 +502,7 @@ help: @echo " make build-core # Build stable core image only" @echo " make build-main # Build main image only" @echo " make build-rust # Build Rust image only" + @echo " make build-cloak # Build CloakBrowser image only" @echo " make TAG=dev build # Build all with custom tag" @echo " make CLAUDE_CODE_VERSION=2.0.5 build # Override with specific version" @echo " make GEMINI_CLI_VERSION=0.18.0 build # Override gemini version" diff --git a/README.md b/README.md index 041f90b..ca7e973 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,7 @@ Stable release tags: - `ghcr.io/thevibeworks/deva:latest` - `ghcr.io/thevibeworks/deva:rust` +- `ghcr.io/thevibeworks/deva:cloak` (CloakBrowser stealth Chromium, headed; `deva.sh -p cloak`) Nightly refresh tags: diff --git a/deva.sh b/deva.sh index 63ec125..dbd603a 100755 --- a/deva.sh +++ b/deva.sh @@ -4,13 +4,21 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" AGENTS_DIR="$SCRIPT_DIR/agents" +# Image/tag precedence: environment > .deva config files > -p profile > default. +# *_ENV_SET means "explicitly set somewhere" (profile must not clobber it); +# *_SOURCE records where, so config files never override the real environment +# and an explicit CLI -p can still beat a config-file value (resolve_profile). DEVA_DOCKER_IMAGE_ENV_SET=false +DEVA_DOCKER_IMAGE_SOURCE="" if [ -n "${DEVA_DOCKER_IMAGE+x}" ]; then DEVA_DOCKER_IMAGE_ENV_SET=true + DEVA_DOCKER_IMAGE_SOURCE="env" fi DEVA_DOCKER_TAG_ENV_SET=false +DEVA_DOCKER_TAG_SOURCE="" if [ -n "${DEVA_DOCKER_TAG+x}" ]; then DEVA_DOCKER_TAG_ENV_SET=true + DEVA_DOCKER_TAG_SOURCE="env" fi VERSION="0.16.0" @@ -22,6 +30,7 @@ DEVA_CODEX_BROWSER_MCP_PACKAGE="${DEVA_CODEX_BROWSER_MCP_PACKAGE:-${DEVA_PLAYWRI DEVA_PLAYWRIGHT_MCP_PACKAGE="${DEVA_PLAYWRIGHT_MCP_PACKAGE:-$DEVA_CODEX_BROWSER_MCP_PACKAGE}" PROFILE="${DEVA_PROFILE:-${DEVA_IMAGE_PROFILE:-}}" +PROFILE_FROM_CLI=false CONFIG_ROOT="" @@ -135,7 +144,16 @@ Deva flags: Mount an alternate auth/config home into /home/deva -e VAR[=VALUE] Pass environment variable into the container (pulls from host when VALUE omitted) -p NAME, --profile NAME - Select profile: base (default), rust. Pulls tag, falls back to Dockerfile. + Select profile: base (default), rust, cloak. Pulls tag, falls back to Dockerfile. + --cloak-browser Start an always-on CloakBrowser the agent drives via + CLOAK_CDP_ENDPOINT; auto-mounts a persistent profile + (~/.config/deva/cloak-profile). Only with -p cloak. + --cloak-vnc Docker-Desktop fallback: publish the cloak VNC display + on a host loopback port for interactive login + (MFA/captcha). On OrbStack/Linux skip this -- the host + reaches the container directly; start x11vnc on demand. + Auto-generates a VNC password (or set + DEVA_CLOAK_VNC_PASSWORD). Port fixed at create. -p cloak. -Q, --quick Bare mode: no host config mounts, no .deva loading, no autolink, implies --rm. Like emacs -Q. Mutually exclusive with -c. --host-net Use host networking for the agent container @@ -364,7 +382,7 @@ default_config_home_for_agent() { validate_profile() { case "$1" in - base | rust | "") return 0 ;; + base | rust | cloak | "") return 0 ;; *) return 1 ;; esac } @@ -408,8 +426,8 @@ check_image() { # Digest-pinned refs are exact; tag fallback does not make sense there. local available_tags="" if [[ "$DEVA_DOCKER_IMAGE" != *@* ]]; then - # Check common profile tags (prefer rust as it's a superset of base) - for tag in rust latest; do + # Check common profile tags (prefer cloak > rust > latest) + for tag in cloak rust latest; do if [ "$tag" = "$DEVA_DOCKER_TAG" ]; then continue # Skip the one we already tried fi @@ -434,6 +452,9 @@ check_image() { rust) [ -f "${SCRIPT_DIR}/Dockerfile.rust" ] && df="${SCRIPT_DIR}/Dockerfile.rust" || df="" ;; + cloak) + [ -f "${SCRIPT_DIR}/Dockerfile.cloak" ] && df="${SCRIPT_DIR}/Dockerfile.cloak" || df="" + ;; esac echo "Docker image $image_ref not found locally" >&2 @@ -444,6 +465,10 @@ check_image() { echo "Build with: make build-rust" >&2 echo "Manual docker builds need explicit build args and BASE_IMAGE; see docs/custom-images.md" >&2 ;; + cloak) + echo "Build with: make build-cloak" >&2 + echo "Manual docker builds need explicit build args and BASE_IMAGE; see docs/custom-images.md" >&2 + ;; "" | base) echo "Build with: make build" >&2 echo "Manual docker builds need explicit build args; see docs/custom-images.md" >&2 @@ -797,7 +822,7 @@ prepare_browser_integration() { "" | base) PROFILE="rust" ;; - rust) + rust | cloak) ;; *) echo "warning: --browser-mcp assumes the selected image contains node/npx and browser runtime deps" >&2 @@ -2945,18 +2970,32 @@ process_var_config() { fi ;; DEVA_DOCKER_IMAGE) - DEVA_DOCKER_IMAGE="$value" - DEVA_DOCKER_IMAGE_ENV_SET=true - normalize_docker_image_parts - export DEVA_DOCKER_IMAGE - USER_ENVS+=("$name=$value") + # Real environment beats config files — a .deva default must not + # clobber an explicit `DEVA_DOCKER_IMAGE=... deva.sh ...`. + if [ "$DEVA_DOCKER_IMAGE_SOURCE" != "env" ]; then + local _tag_was_set="$DEVA_DOCKER_TAG_ENV_SET" + DEVA_DOCKER_IMAGE="$value" + DEVA_DOCKER_IMAGE_ENV_SET=true + DEVA_DOCKER_IMAGE_SOURCE="config" + normalize_docker_image_parts + # normalize may adopt an image-embedded tag: that tag came from + # this config value, so it is config-sourced too. + if [ "$_tag_was_set" = false ] && [ "$DEVA_DOCKER_TAG_ENV_SET" = true ]; then + DEVA_DOCKER_TAG_SOURCE="config" + fi + export DEVA_DOCKER_IMAGE + USER_ENVS+=("$name=$value") + fi ;; DEVA_DOCKER_TAG) - DEVA_DOCKER_TAG="$value" - DEVA_DOCKER_TAG_ENV_SET=true - normalize_docker_image_parts - export DEVA_DOCKER_TAG - USER_ENVS+=("$name=$value") + if [ "$DEVA_DOCKER_TAG_SOURCE" != "env" ]; then + DEVA_DOCKER_TAG="$value" + DEVA_DOCKER_TAG_ENV_SET=true + DEVA_DOCKER_TAG_SOURCE="config" + normalize_docker_image_parts + export DEVA_DOCKER_TAG + USER_ENVS+=("$name=$value") + fi ;; DEFAULT_AGENT) DEFAULT_AGENT="$value" @@ -3128,8 +3167,9 @@ parse_wrapper_args() { fi local prof="${incoming[$((i + 1))]}" PROFILE="$prof" + PROFILE_FROM_CLI=true if ! validate_profile "$prof"; then - echo "error: unknown profile '$prof'. Valid: base, rust." >&2 + echo "error: unknown profile '$prof'. Valid: base, rust, cloak." >&2 echo "hint: if you intended '-p' for the agent, place it after '--' (e.g., deva.sh claude -- -p 'task')" >&2 exit 1 fi @@ -3184,6 +3224,21 @@ parse_wrapper_args() { i=$((i + 1)) continue ;; + --cloak-vnc) + # Expose the cloak display over VNC for interactive logins: publishes + # a host-loopback port + mounts a persistent profile. The AGENT drives + # its own browser on the display -- no daemon. No-op outside -p cloak. + DEVA_CLOAK_VNC=1 + i=$((i + 1)) + continue + ;; + --cloak-browser) + # Start the always-on CloakBrowser daemon and mount a persistent + # profile; the agent drives it via CLOAK_CDP_ENDPOINT. -p cloak only. + DEVA_CLOAK_BROWSER=1 + i=$((i + 1)) + continue + ;; --verbose | --debug) DEBUG_MODE=true i=$((i + 1)) @@ -3218,6 +3273,17 @@ load_agent_module() { fi } +# A named profile fills in image/tag that nothing else set. One exception: +# an explicit CLI -p is this run's intent and beats a .deva config-file tag +# (else a config `DEVA_DOCKER_TAG=rust` makes `-p cloak` a silent no-op). +# The real environment still wins over both. +profile_may_set_tag() { + if [ "$DEVA_DOCKER_TAG_ENV_SET" = false ]; then + return 0 + fi + [ "$PROFILE_FROM_CLI" = true ] && [ "$DEVA_DOCKER_TAG_SOURCE" = "config" ] +} + resolve_profile() { local default_repo="ghcr.io/thevibeworks/deva" case "${PROFILE:-}" in @@ -3225,7 +3291,7 @@ resolve_profile() { if [ "$DEVA_DOCKER_IMAGE_ENV_SET" = false ]; then DEVA_DOCKER_IMAGE="$default_repo" fi - if [ "$DEVA_DOCKER_TAG_ENV_SET" = false ]; then + if profile_may_set_tag; then DEVA_DOCKER_TAG="latest" fi ;; @@ -3233,10 +3299,18 @@ resolve_profile() { if [ "$DEVA_DOCKER_IMAGE_ENV_SET" = false ]; then DEVA_DOCKER_IMAGE="$default_repo" fi - if [ "$DEVA_DOCKER_TAG_ENV_SET" = false ]; then + if profile_may_set_tag; then DEVA_DOCKER_TAG="rust" fi ;; + cloak) + if [ "$DEVA_DOCKER_IMAGE_ENV_SET" = false ]; then + DEVA_DOCKER_IMAGE="$default_repo" + fi + if profile_may_set_tag; then + DEVA_DOCKER_TAG="cloak" + fi + ;; *) if [ "$DEVA_DOCKER_IMAGE_ENV_SET" = false ] && [ -z "${DEVA_DOCKER_IMAGE:-}" ]; then DEVA_DOCKER_IMAGE="$default_repo" @@ -3247,6 +3321,134 @@ resolve_profile() { is_known_agent() { [ -f "$AGENTS_DIR/$1.sh" ]; } +# 0 if this run targets the cloak image (via -p cloak or a cloak tag). +is_cloak_image() { + [ "${PROFILE:-}" = "cloak" ] || [[ "${DEVA_DOCKER_TAG:-}" == *cloak* ]] +} + +# Cloak runtime knobs: the always-on browser daemon (--cloak-browser) and VNC +# (--cloak-vnc). Both opt-in and independent -- with --cloak-vnc the agent +# drives its own browser on the display; --cloak-browser holds a shared one on +# CDP. One image guard, one warning, correct ordering. +setup_cloak_runtime() { + { [ "${DEVA_CLOAK_VNC:-}" = "1" ] || [ "${DEVA_CLOAK_BROWSER:-}" = "1" ]; } || return 0 + if ! is_cloak_image; then + echo "warning: --cloak-vnc/--cloak-browser ignored -- they ship only in the cloak image (use -p cloak)" >&2 + DEVA_CLOAK_VNC="" + DEVA_CLOAK_BROWSER="" + return 0 + fi + # A persistent profile is what makes a login survive; mount it for either + # path. VNC and the always-on daemon are independent: with --cloak-vnc the + # AGENT drives its own browser on the display (no daemon); --cloak-browser is + # a separate opt-in that holds one always-on browser on CDP. + mount_cloak_profile + setup_cloak_vnc + setup_cloak_browser +} + +# Mount a host dir at the Chromium userDataDir so the whole session (cookies, +# localStorage, IndexedDB, history, cache) persists to the host across container +# removal. Shared by the VNC and daemon paths. First-writer-wins: a user mount +# at the same target wins. +mount_cloak_profile() { + local ctr_dir=/home/deva/.cloak-profile + DOCKER_ARGS+=("-e" "DEVA_CLOAK_PROFILE_DIR=${ctr_dir}") + if ! user_volumes_declares_target "$ctr_dir"; then + local host_dir="${DEVA_CLOAK_PROFILE_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/deva/cloak-profile}" + mkdir -p "$host_dir" 2>/dev/null || true + DOCKER_ARGS+=("-v" "${host_dir}:${ctr_dir}") + DEVA_CLOAK_PROFILE_HOST="$host_dir" + fi +} + +# Opt-in always-on CloakBrowser daemon: pass the daemon flag + CDP endpoint so +# the agent can attach (connectOverCDP) to one shared, always-on browser. The +# profile is mounted separately (mount_cloak_profile). This is the heavy path; +# the default/VNC path has the agent drive its own browser instead. +setup_cloak_browser() { + [ "${DEVA_CLOAK_BROWSER:-}" = "1" ] || return 0 + local cdp_port="${DEVA_CLOAK_CDP_PORT:-9222}" + DOCKER_ARGS+=("-e" "DEVA_CLOAK_BROWSER=1") + DOCKER_ARGS+=("-e" "DEVA_CLOAK_CDP_PORT=${cdp_port}") + DOCKER_ARGS+=("-e" "CLOAK_CDP_ENDPOINT=http://127.0.0.1:${cdp_port}") +} + +# Opt-in VNC helper for the Docker-Desktop case. On OrbStack/native Linux the +# host reaches the container directly, so you don't need this: the agent starts +# x11vnc on the :99 display on demand and you connect at the container IP -- no +# flag, no publish. --cloak-vnc is the Docker-Desktop fallback, where the +# container IP is NOT routable from the host: it publishes a HOST-loopback port +# -> container 5900 (probe up from 5900 for concurrent containers). That publish +# is fixed at container CREATE, so a warm container created without it cannot +# gain the mapping (announce_cloak_vnc warns). +DEVA_CLOAK_VNC_URL="" +DEVA_CLOAK_PROFILE_HOST="" +setup_cloak_vnc() { + [ "${DEVA_CLOAK_VNC:-}" = "1" ] || return 0 + local port=5900 tries=0 free_port="" + while [ "$tries" -lt 12 ]; do + if ! (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + free_port="$port" + break + fi + port=$((port + 1)) + tries=$((tries + 1)) + done + if [ -z "$free_port" ]; then + echo "warning: no free host port in 5900-5911; VNC will not be reachable from the host" >&2 + return 0 + fi + # Secure by default: if the user gave no password, generate one so VNC is + # never unauthenticated. VNC caps passwords at 8 chars, so generate 8. + DEVA_CLOAK_VNC_PASSWORD_EFFECTIVE="${DEVA_CLOAK_VNC_PASSWORD:-}" + DEVA_CLOAK_VNC_PASSWORD_GENERATED=false + if [ -z "$DEVA_CLOAK_VNC_PASSWORD_EFFECTIVE" ]; then + # Read a FINITE chunk of urandom, then filter+cut. Piping an unbounded + # `&2 + [ -n "$DEVA_CLOAK_PROFILE_HOST" ] && echo " profile persists to: $DEVA_CLOAK_PROFILE_HOST" >&2 +} + +# "new": trust the URL (container is about to be created with the port). +# "existing": the port was fixed at create; if this run asked for VNC but is +# attaching to a container created without it, say so instead of printing a +# dead URL. +announce_cloak_vnc() { + local phase="$1" + if [ "$phase" = "existing" ] && [ "${DEVA_CLOAK_VNC:-}" = "1" ] && [ -z "$DEVA_CLOAK_VNC_URL" ]; then + echo "note: DEVA_CLOAK_VNC=1 but this container was created without a VNC port; recreate it (--rm, or stop/remove) to enable VNC" >&2 + return 0 + fi + [ -n "$DEVA_CLOAK_VNC_URL" ] || return 0 + echo "Cloak VNC: $DEVA_CLOAK_VNC_URL (open in a VNC client to watch/drive the browser the agent opens)" >&2 + if [ "${DEVA_CLOAK_VNC_PASSWORD_GENERATED:-false}" = true ]; then + echo " VNC password (auto-generated): ${DEVA_CLOAK_VNC_PASSWORD_EFFECTIVE}" >&2 + else + echo " VNC password: (your DEVA_CLOAK_VNC_PASSWORD)" >&2 + fi + # The daemon announce already prints the profile path; only print it here + # when there is no daemon (the VNC-only path still mounts a profile). + [ "${DEVA_CLOAK_BROWSER:-}" != "1" ] && [ -n "$DEVA_CLOAK_PROFILE_HOST" ] && \ + echo " profile persists to: $DEVA_CLOAK_PROFILE_HOST" >&2 +} + ACTION="run" MANAGEMENT_MODE="launch" RAW_ARGS=("$@") @@ -3926,6 +4128,9 @@ _step "append_grok_update_guard" append_user_envs _step "append_user_envs" + +setup_cloak_runtime +_step "setup_cloak_runtime" validate_bind_mount_shape _step "validate_bind_mount_shape" @@ -4061,6 +4266,8 @@ if [ "$EPHEMERAL_MODE" = false ]; then # Trace UI reachability is fixed at container create (port publish); # attaching to a container created without it cannot gain the mapping. announce_trace_ui existing + announce_cloak_browser existing + announce_cloak_vnc existing if [ "$AUTH_PROVISION_MODE" = true ]; then docker exec -e "$_trace_env" "${DOCKER_TERMINAL_ARGS[@]}" "$CONTAINER_NAME" /usr/local/bin/docker-entrypoint.sh "${AGENT_COMMAND[@]}" || true @@ -4072,6 +4279,8 @@ else echo "Launching ${ACTIVE_AGENT} (ephemeral mode) via $(docker_image_ref)" write_session_file announce_trace_ui new + announce_cloak_browser new + announce_cloak_vnc new if [ "$AUTH_PROVISION_MODE" = true ]; then docker "${DOCKER_ARGS[@]}" "${AGENT_COMMAND[@]}" || true finish_auth_provision diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 53fa988..5f68460 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -397,6 +397,47 @@ setup_trace_ca() { return 0 } +# Start the always-on CloakBrowser daemon (DEVA_CLOAK_BROWSER=1, cloak image +# only). Runs AFTER setup_nonroot_user so the browser writes the mounted +# profile as the remapped host UID -- starting it in cloak-entrypoint (pre- +# remap) would leave root/1001-owned files the agent can't touch. Idempotent +# under flock + a CDP-port probe: the boot invocation (tail -f /dev/null) starts +# it once as a child of PID 1's tree, later agent execs find it up and skip. +maybe_start_cloak_browser() { + [ "${DEVA_CLOAK:-}" = "1" ] || return 0 + [ "${DEVA_CLOAK_BROWSER:-}" = "1" ] || return 0 + local daemon=/usr/local/lib/cloak/cloak-browserd.mjs + if ! command -v node >/dev/null 2>&1 || [ ! -f "$daemon" ]; then + echo "[cloak] browser daemon unavailable (node or $daemon missing); skipping" >&2 + return 0 + fi + + local port="${DEVA_CLOAK_CDP_PORT:-9222}" + local profile="${DEVA_CLOAK_PROFILE_DIR:-$DEVA_HOME/.cloak-profile}" + local lock=/run/lock/deva-cloak-browserd.lock + ( + if ! flock -x -w 30 9; then + echo "[cloak] browser daemon lock timeout; skipping" >&2 + exit 0 + fi + # Already listening -> a prior invocation started it; nothing to do. + if (exec 8<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + exit 0 + fi + mkdir -p "$profile" + chown "$DEVA_UID:$DEVA_GID" "$profile" 2>/dev/null || true + local log="$DEVA_HOME/.cloak-browserd.log" + # setsid+nohup so the daemon outlives the exec/boot process that spawns it. + setsid nohup gosu "$DEVA_USER" env \ + "HOME=$DEVA_HOME" "PATH=$PATH" "DISPLAY=${DISPLAY:-:99}" \ + "CLOAK_APP_DIR=${CLOAK_APP_DIR:-/opt/cloak}" \ + "DEVA_CLOAK_PROFILE_DIR=$profile" "DEVA_CLOAK_CDP_PORT=$port" \ + node "$daemon" >"$log" 2>&1 & + echo "[cloak] browser daemon starting (CDP 127.0.0.1:$port, profile $profile, log $log)" >&2 + ) 9>"$lock" + return 0 +} + ensure_agent_binaries() { case "$DEVA_AGENT" in claude) @@ -451,6 +492,7 @@ main() { fix_rust_permissions fix_docker_socket_permissions setup_trace_ca + maybe_start_cloak_browser ensure_agent_binaries if [ $# -eq 0 ]; then diff --git a/docs/advanced-usage.md b/docs/advanced-usage.md index 4f34c07..a2f0fa9 100644 --- a/docs/advanced-usage.md +++ b/docs/advanced-usage.md @@ -96,16 +96,26 @@ Supported profiles: - `base` -> `ghcr.io/thevibeworks/deva:latest` - `rust` -> `ghcr.io/thevibeworks/deva:rust` +- `cloak` -> `ghcr.io/thevibeworks/deva:cloak` Use them like this: ```bash deva.sh claude -p rust deva.sh codex -p rust +deva.sh claude -p cloak ``` If the image tag is missing locally, deva pulls it. If that fails and a matching Dockerfile exists, it points you at the build command. +The `cloak` profile extends the rust image with [CloakBrowser](https://www.npmjs.com/package/cloakbrowser) — +a stealth Chromium with source-level fingerprint patches, running headed on a +virtual display (Xvfb) inside the container. Use it when an agent needs a +browser that passes bot detection; the baked-in `deva-cloak` skill (keyed off +`DEVA_CLOAK=1` in the image) tells agents how to drive it. The Chromium +binary is baked at build time with auto-update off, so the container needs no +network to launch a browser. Build locally with `make build-cloak`. + If you want your own image entirely, read [Custom Images](custom-images.md). That covers local builds, private tags, per-project overrides, and personal-only setups. diff --git a/docs/custom-images.md b/docs/custom-images.md index 8ce9606..35daffb 100644 --- a/docs/custom-images.md +++ b/docs/custom-images.md @@ -42,6 +42,7 @@ Supported path: ```bash make build-main make build-rust +make build-cloak ``` If you only changed the late agent-install layer and want the fastest rebuild: @@ -99,6 +100,11 @@ docker build -f Dockerfile.rust -t deva-local:rust \ --build-arg RUST_DEFAULT_TOOLCHAIN="$RUST_DEFAULT_TOOLCHAIN" \ --build-arg RUST_TARGETS="$RUST_TARGETS" \ . + +docker build -f Dockerfile.cloak -t deva-local:cloak \ + --build-arg BASE_IMAGE=deva-local:rust \ + --build-arg CLOAKBROWSER_WRAPPER_VERSION="$CLOAKBROWSER_WRAPPER_VERSION" \ + . ``` Then run deva against it: diff --git a/llms.txt b/llms.txt index 5271e1a..4c53ab0 100644 --- a/llms.txt +++ b/llms.txt @@ -5,9 +5,9 @@ > project container serves all five agents. A bash script, not a framework. MIT. deva.sh is a single entry point installed on the host; agents run inside -`ghcr.io/thevibeworks/deva` images (`:latest`, `:rust` for stable, `:nightly`, -`:nightly-rust` refreshed daily). Agent-level permission prompts are disabled -by design — isolation comes from Docker, not permission theater: +`ghcr.io/thevibeworks/deva` images (`:latest`, `:rust`, `:cloak` for stable, +`:nightly`, `:nightly-rust` refreshed daily). Agent-level permission prompts +are disabled by design — isolation comes from Docker, not permission theater: `claude --dangerously-skip-permissions`, `codex --dangerously-bypass-approvals-and-sandbox`, `gemini --yolo`, `grok --always-approve`, and `kimi --yolo`. Auth @@ -29,6 +29,7 @@ deva.sh codex # same container shape, other agents deva.sh gemini deva.sh grok deva.sh kimi +deva.sh claude -p cloak # stealth-browser image (CloakBrowser, headed Xvfb) deva.sh claude --debug --dry-run # inspect the docker run before trusting it ``` diff --git a/scripts/cloak-browserd.mjs b/scripts/cloak-browserd.mjs new file mode 100644 index 0000000..4a1d5d0 --- /dev/null +++ b/scripts/cloak-browserd.mjs @@ -0,0 +1,57 @@ +// cloak-browserd - always-on CloakBrowser daemon. +// +// Holds ONE persistent, headed, stealth browser on the cloak display (:99) +// with a loopback CDP endpoint. The human watches and drives it over VNC; the +// agent attaches with Playwright `connectOverCDP(CLOAK_CDP_ENDPOINT)` and +// drives the SAME live browser + profile -- so a login done once (by the human +// over VNC, or the agent) is immediately usable by the other, and persists in +// the mounted userDataDir. Started by docker-entrypoint when +// DEVA_CLOAK_BROWSER=1; relaunches if the browser dies. +// +// Absolute import (not bare 'cloakbrowser'): this file lives outside +// $CLOAK_APP_DIR, so ESM bare-specifier resolution from cwd would miss it. +const appDir = process.env.CLOAK_APP_DIR || '/opt/cloak'; +const { launchPersistentContext } = await import( + appDir + '/node_modules/cloakbrowser/dist/index.js' +); + +const userDataDir = process.env.DEVA_CLOAK_PROFILE_DIR || '/home/deva/.cloak-profile'; +const port = Number(process.env.DEVA_CLOAK_CDP_PORT || 9222); + +let ctx = null; +let stopping = false; + +async function start() { + ctx = await launchPersistentContext({ + userDataDir, + headless: false, // visible on :99 for VNC; the whole point of the cloak image + // --remote-debugging-port opens the loopback CDP endpoint the agent attaches + // to; -address pins it to loopback so it never leaves the container. + args: [`--remote-debugging-port=${port}`, '--remote-debugging-address=127.0.0.1'], + }); + if (ctx.pages().length === 0) await ctx.newPage(); + console.log(`[cloak-browserd] up: CDP http://127.0.0.1:${port} profile=${userDataDir}`); + ctx.on('close', () => { + if (stopping) return; + console.log('[cloak-browserd] browser closed; relaunching in 1s'); + setTimeout(() => start().catch(fail), 1000); + }); +} + +function fail(e) { + console.error('[cloak-browserd] fatal:', (e && e.stack) || e); + process.exit(1); +} + +for (const sig of ['SIGTERM', 'SIGINT']) { + process.on(sig, async () => { + stopping = true; + try { await ctx?.close(); } catch { /* already gone */ } + process.exit(0); + }); +} + +// Keep the event loop alive across a relaunch gap even if all other handles clear. +setInterval(() => {}, 1 << 30); + +start().catch(fail); diff --git a/scripts/cloak-entrypoint.sh b/scripts/cloak-entrypoint.sh new file mode 100644 index 0000000..7d6307d --- /dev/null +++ b/scripts/cloak-entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# CloakBrowser entrypoint: start Xvfb + openbox, then chain to main entrypoint. +# Headed mode by default -- the whole point of the cloak image. + +set -e + +# Clean stale X lock from previous container instance (survives docker restart +# because /tmp is not tmpfs in this image). +rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 + +# Start virtual framebuffer (1920x1080, 24-bit color) +Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & + +# Wait for X server to accept connections before starting the window manager. +# Bounded to ~10s -- avoids the race where openbox starts before X is ready. +for _ in $(seq 1 50); do + DISPLAY=:99 xdotool getdisplaygeometry >/dev/null 2>&1 && break + sleep 0.2 +done + +# Window manager so --start-maximized is honored (bare Xvfb has no WM) +DISPLAY=:99 openbox & + +export DISPLAY=:99 + +# Opt-in VNC: expose the Xvfb display so a human can drive the browser for +# interactive logins (MFA/captcha). Off by default -- automated cloak runs +# don't need it and a live display server is needless attack surface. +# +# x11vnc binds 0.0.0.0:5900 inside the container ON PURPOSE: the security +# boundary is deva publishing that port to the HOST loopback only +# (-p 127.0.0.1:PORT:5900). -localhost would bind container loopback, which +# docker's port-proxy can't reach, so the tunnel would be dead. +# DEVA_CLOAK_VNC_PASSWORD adds VNC-level auth on top; without it the display +# is reachable by anything that already reached the host-loopback port. +if [ "${DEVA_CLOAK_VNC:-}" = "1" ]; then + vnc_auth=(-nopw) + if [ -n "${DEVA_CLOAK_VNC_PASSWORD:-}" ]; then + mkdir -p /home/deva/.vnc + x11vnc -storepasswd "$DEVA_CLOAK_VNC_PASSWORD" /home/deva/.vnc/passwd >/dev/null 2>&1 + chown -R "${DEVA_UID:-1001}:${DEVA_GID:-1001}" /home/deva/.vnc + vnc_auth=(-rfbauth /home/deva/.vnc/passwd) + else + echo "[cloak] WARNING: DEVA_CLOAK_VNC=1 without DEVA_CLOAK_VNC_PASSWORD -- VNC is unauthenticated (reachable only via the host-loopback port deva forwards). Set DEVA_CLOAK_VNC_PASSWORD for defense in depth." >&2 + fi + # -forever: survive client disconnects; -shared: allow reconnects. + x11vnc -display :99 -rfbport 5900 -forever -shared -quiet -bg "${vnc_auth[@]}" \ + >/dev/null 2>&1 || echo "[cloak] WARNING: x11vnc failed to start" >&2 + echo "[cloak] VNC server on :5900 (display :99) -- connect via the host port deva forwarded" >&2 +fi + +# Chain to the standard deva entrypoint +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/scripts/test-image-precedence.sh b/scripts/test-image-precedence.sh new file mode 100644 index 0000000..1f4c2c3 --- /dev/null +++ b/scripts/test-image-precedence.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# test-image-precedence.sh - image/tag resolution precedence +# Contract: environment > .deva config files > -p profile > default, +# with one exception: an explicit CLI -p beats a config-file tag +# (a .deva DEVA_DOCKER_TAG must not turn -p into a silent no-op). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +tmp_home="$(mktemp -d)" +cleanup() { rm -rf "$tmp_home"; } +trap cleanup EXIT + +mkdir -p "$tmp_home/.config/deva" +printf 'DEVA_DOCKER_IMAGE=ghcr.io/thevibeworks/deva\nDEVA_DOCKER_TAG=rust\n' \ + > "$tmp_home/.config/deva/.deva" + +fail=0 +resolved_image() { + ( + cd "$REPO_ROOT" + # env -u: the harness itself may run inside a deva container that + # exports DEVA_DOCKER_TAG/IMAGE; those must not leak into cases + # that model an unset environment. + env -u DEVA_DOCKER_IMAGE -u DEVA_DOCKER_TAG \ + HOME="$tmp_home" \ + XDG_CONFIG_HOME="$tmp_home/.config" \ + XDG_CACHE_HOME="$tmp_home/.cache" \ + DEVA_NO_DOCKER=1 \ + ${1+"$@"} ./deva.sh claude --dry-run + ) 2>&1 | grep -oE 'deva\.image=[^ ]*' | head -1 +} + +want() { + local desc="$1" expect="$2" got="$3" + if [ "$got" = "deva.image=$expect" ]; then + echo " PASS $desc" + else + echo " FAIL $desc" + echo " expected: deva.image=$expect" >&2 + echo " got: $got" >&2 + fail=1 + fi +} + +repo="ghcr.io/thevibeworks/deva" + +echo "=== image/tag precedence ===" +want "config file alone wins over default" "$repo:rust" \ + "$(resolved_image)" +want "CLI -p beats config-file tag" "$repo:cloak" \ + "$(cd "$REPO_ROOT"; env -u DEVA_DOCKER_IMAGE -u DEVA_DOCKER_TAG \ + HOME="$tmp_home" XDG_CONFIG_HOME="$tmp_home/.config" \ + XDG_CACHE_HOME="$tmp_home/.cache" DEVA_NO_DOCKER=1 \ + ./deva.sh claude -p cloak --dry-run 2>&1 | grep -oE 'deva\.image=[^ ]*' | head -1)" +want "environment beats config file" "$repo:cloak" \ + "$(resolved_image DEVA_DOCKER_TAG=cloak)" +want "environment beats CLI -p" "$repo:latest" \ + "$(cd "$REPO_ROOT"; env -u DEVA_DOCKER_IMAGE \ + DEVA_DOCKER_TAG=latest \ + HOME="$tmp_home" XDG_CONFIG_HOME="$tmp_home/.config" \ + XDG_CACHE_HOME="$tmp_home/.cache" DEVA_NO_DOCKER=1 \ + ./deva.sh claude -p cloak --dry-run 2>&1 | grep -oE 'deva\.image=[^ ]*' | head -1)" + +if [ "$fail" -ne 0 ]; then + echo "FAIL: image precedence" >&2 + exit 1 +fi +echo "OK: image precedence" diff --git a/scripts/update-version-pins.sh b/scripts/update-version-pins.sh index 099c19f..c169e7a 100644 --- a/scripts/update-version-pins.sh +++ b/scripts/update-version-pins.sh @@ -71,6 +71,10 @@ KIMI_CODE_VERSION=$KIMI_CODE_VERSION CCX_VERSION=$CCX_VERSION COPILOT_API_VERSION=$COPILOT_API_VERSION PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION +# CloakBrowser npm wrapper version. This also pins the Chromium binary: +# the wrapper hardcodes per-arch free-binary versions (linux-x64 146.x.x.5, +# linux-arm64 146.x.x.3), so bumping the wrapper is what moves Chromium. +CLOAKBROWSER_WRAPPER_VERSION=$CLOAKBROWSER_WRAPPER_VERSION RUST_TOOLCHAINS=$RUST_TOOLCHAINS RUST_DEFAULT_TOOLCHAIN=$RUST_DEFAULT_TOOLCHAIN @@ -110,6 +114,7 @@ main() { refresh_pin CCX_VERSION "$(fetch_latest_git_tag https://github.com/thevibeworks/ccx.git)" refresh_pin COPILOT_API_VERSION "$(fetch_latest_commit https://github.com/ericc-ch/copilot-api.git refs/heads/master)" refresh_pin PLAYWRIGHT_VERSION "$(fetch_npm_version playwright)" + refresh_pin CLOAKBROWSER_WRAPPER_VERSION "$(fetch_npm_version cloakbrowser)" if [[ $DRY_RUN -eq 1 ]]; then emit_version_pins diff --git a/scripts/version-pins.sh b/scripts/version-pins.sh index 7c8e460..1fa2c0e 100644 --- a/scripts/version-pins.sh +++ b/scripts/version-pins.sh @@ -22,6 +22,7 @@ VERSION_PIN_VARS=( CCX_VERSION COPILOT_API_VERSION PLAYWRIGHT_VERSION + CLOAKBROWSER_WRAPPER_VERSION RUST_TOOLCHAINS RUST_DEFAULT_TOOLCHAIN RUST_TARGETS diff --git a/skills/deva-cloak/SKILL.md b/skills/deva-cloak/SKILL.md new file mode 100644 index 0000000..9ce32e1 --- /dev/null +++ b/skills/deva-cloak/SKILL.md @@ -0,0 +1,223 @@ +--- +name: deva-cloak +description: Drive CloakBrowser stealth Chromium inside the deva cloak container - a headed, anti-detection browser for scraping, automation, or checking a site the way a real browser sees it. Use when the task needs a browser that bypasses bot detection (Cloudflare, FingerprintJS, reCAPTCHA scoring), or when the user asks to browse/scrape a protected site from inside deva. Only applies in the cloak image (verify with `command -v cloakbrowser`; DEVA_CLOAK=1 is the marker); the base/rust images have plain Playwright instead. +--- + +# deva-cloak: stealth browser in the deva cloak container + +CloakBrowser is a custom-compiled Chromium with source-level fingerprint +patches. Same Playwright API, different binary -- detection sites score it as +a real browser because it is one. This image runs it **headed by default** on a +virtual display, which passes checks that headless fails even with the patches. + +## Am I in the cloak context? + +This skill only applies when the browser is actually present. ALWAYS run +this check fresh -- never conclude from conversation context, a resumed +session, or `$DEVA_CLOAK` alone (env markers can be scrubbed or stale; +the binary is the ground truth): + +```bash +command -v cloakbrowser >/dev/null && cloakbrowser info --quick +``` + +If that fails, you are in the base or rust image -- there is no CloakBrowser +here. (`DEVA_CLOAK=1` is baked into the cloak image as a convenience marker, +but treat it as a hint, not the test.) Tell the user to relaunch with +`deva.sh -p cloak `, or use the plain Playwright install (rust image) +instead. Do not try to `npm install cloakbrowser` yourself; the binary is +baked into the cloak image and a runtime install would download ~200MB and +defeat the point. + +## What is baked in + +- Wrapper + `playwright-core` live in `$CLOAK_APP_DIR` (`/opt/cloak`). +- Chromium binary is pre-downloaded, cache at `/opt/cloakbrowser`, auto-update + off -- fully offline, no network needed to launch. +- Free binary = **Chromium 146** (58 fingerprint patches). No Pro license. + Good enough for most sites and for viewing your own apps; the newest + detection tricks (latest Cloudflare/DataDome) may still win. Pro (146->150, + 71 patches) needs `CLOAKBROWSER_LICENSE_KEY` at runtime -- no rebuild. +- Headed display: Xvfb on `:99` + openbox, started by the entrypoint. +- `x11vnc` for VNC into that display (interactive logins). Start it on demand + (`x11vnc -display :99 -rfbport 5900 -bg`); on OrbStack/Linux the Mac reaches + it directly at the container IP -- no port publish, no launch flag. +- `cloak-browserd` always-on browser daemon + (`/usr/local/lib/cloak/cloak-browserd.mjs`); OPT-IN, runs only with + `--cloak-browser`, exposes CDP at `CLOAK_CDP_ENDPOINT`. Independent of VNC. + +## Running a script + +CloakBrowser is ESM-only, so run the script **from `$CLOAK_APP_DIR`** where its +`node_modules` resolves. Write output to absolute workspace paths. + +```bash +cd "$CLOAK_APP_DIR" +node --input-type=module <<'EOF' +import { launch } from 'cloakbrowser'; + +// Headed is the whole point of this image -- keep headless:false for +// detection-heavy sites. Flip to true only for speed on trusted targets. +const browser = await launch({ headless: false }); +const page = await browser.newPage(); +await page.goto('https://example.com', { waitUntil: 'domcontentloaded' }); +console.log(await page.title()); +await browser.close(); +EOF +``` + +The returned object is a normal Playwright `Browser` -- `newPage`, +`newContext`, `screenshot`, etc. all work unchanged. + +## Options that matter + +Pass to `launch({ ... })`: + +- `headless: false` -- default posture here; some sites detect headless even + with the C++ patches. +- `humanize: true` -- human-like mouse curves, per-char typing, scroll. Turn on + for behavioral detection (deviceandbrowserinfo, reCAPTCHA v3 scoring). +- `proxy: 'http://user:pass@host:port'` or `'socks5://...'` -- residential IPs + pass reputation checks that datacenter IPs fail. Add `geoip: true` to match + timezone/locale to the proxy exit IP. +- `args: ['--fingerprint=12345']` -- fixed seed = a consistent returning-visitor + identity across runs (better for reCAPTCHA v3 than a fresh identity each time). + +Recommended stack for a hard anti-bot target: +`launch({ headless: false, humanize: true, proxy: '', geoip: true })`. + +## Authenticated sites: log in over VNC + +You cannot copy the host browser's cookie DB in -- Chrome encrypts cookie +values with an OS-keychain key the Linux container can't reach, so the blobs +are undecryptable here. Instead the human logs in once over VNC, watching a +browser YOU (the agent) drive on the cloak display -- same browser, same +profile, persisted to the host. + +**The human launches normally -- no VNC flag needed on OrbStack/Linux:** + +```bash +deva.sh -p cloak claude +``` + +**You (the agent) start VNC on demand when a login is needed.** OrbStack (and +native Linux Docker) route the container straight from the host, so a server +you start INSIDE the container is reachable from the Mac with no port publish +and no launch flag: + +```bash +x11vnc -display :99 -rfbport 5900 -forever -shared -bg # add -passwd to require a password +``` + +Then tell the human how to connect. The container is reachable from the Mac at +its OrbStack address -- `orb list` on the Mac gives the name/IP, then +`open vnc://:5900` (or `.orb.local:5900`), built-in Screen +Sharing. No `-p`, no recreate. + +(`--cloak-vnc` still exists, but only for plain Docker Desktop, where the +container IP is NOT routable from the host and a published loopback port is the +only way in. On OrbStack you don't need it -- and it does not have to be set at +launch.) + +**You (the agent) open a headed browser on the display and drive it.** Use a +persistent context on the mounted profile so the login survives: + +```bash +cd "$CLOAK_APP_DIR" +node --input-type=module <<'EOF' +import { launchPersistentContext } from 'cloakbrowser'; +const ctx = await launchPersistentContext({ + userDataDir: process.env.DEVA_CLOAK_PROFILE_DIR || '/home/deva/.cloak-profile', + headless: false, // renders on :99 so the human sees it over VNC +}); +const page = ctx.pages()[0] || await ctx.newPage(); +await page.goto('https://SITE/login'); +// Tell the human: open the printed vnc:// URL and log in (MFA/captcha). +// Wait for the post-login page, then continue -- you hold the same context. +await page.waitForURL('**/app**', { timeout: 300000 }); +console.log('logged in:', await page.title()); +await ctx.close(); // profile saved to the mounted host dir +EOF +``` + +The human opens `vnc://127.0.0.1:PORT` (macOS: `open vnc://127.0.0.1:5901`, +built-in Screen Sharing), sees your browser, and logs in. Because you hold the +same persistent context, the session is yours immediately and -- via the +mounted profile -- survives to the next run, even headless with no VNC. + +**Optional: always-on shared daemon (`--cloak-browser`).** For a browser that +stays up across agent runs (so the human can log in even when no agent is +active), launch with `--cloak-browser`. deva starts one persistent headed +browser and sets `CLOAK_CDP_ENDPOINT`; attach to it instead of launching your +own -- do NOT `launch()` a second browser, it would miss the login: + +```bash +import pw from 'playwright-core'; +const b = await pw.chromium.connectOverCDP(process.env.CLOAK_CDP_ENDPOINT); +const ctx = b.contexts()[0]; // the daemon's profile, NOT b.newContext() +const page = ctx.pages()[0] || await ctx.newPage(); +await page.goto('https://SITE/app'); // already logged in +await b.close(); // detaches; the daemon browser stays up +``` + +`CLOAK_CDP_ENDPOINT` is set only when the daemon runs (`--cloak-browser`); its +absence means there is no daemon and you should launch your own browser as +above. + +**Persistence / what to mount.** `--cloak-vnc` (and `--cloak-browser`) +auto-mounts `~/.config/deva/cloak-profile` (host) to the Chromium `userDataDir` +`/home/deva/.cloak-profile`. That one dir is the whole session -- cookies, +localStorage, IndexedDB, service workers, Cache, History, Login Data. It +survives container removal, so a login done once is reused by every later run +(even headless, no VNC). Override the host location with +`DEVA_CLOAK_PROFILE_DIR=/path deva.sh -p cloak --cloak-vnc ...`, or mount +your own dir at `/home/deva/.cloak-profile` (your mount wins). Do NOT mount +`/opt/cloakbrowser` -- that is the baked binary, not session data. + +Gotcha: only cookies with an `expires` survive a profile reopen -- session +cookies (no expiry) are memory-only. Real logins persist via the site's +long-lived "remember me" / refresh-token cookie, which carries an expiry, so +genuine sessions survive (verified). + +**storageState alternative** (no daemon/VNC): if you captured Playwright +storage state elsewhere (cookies + localStorage as plaintext JSON) and mounted +the file, load it into a one-off context -- note the nesting: + +```bash +node --input-type=module <<'EOF' +import { launchContext } from 'cloakbrowser'; +// storageState MUST be under contextOptions -- launchContext feeds it to +// newContext(). Top-level storageState is silently dropped and you get an +// UNauthenticated browser (verified against cloakbrowser 0.4.x). +const ctx = await launchContext({ headless: false, contextOptions: { storageState: '/home/deva/.cloak-auth/SITE.json' } }); +const page = await ctx.newPage(); +await page.goto('https://SITE/app'); +await ctx.close(); +EOF +``` + +SECURITY -- the profile dir (and any storage-state file) is a live bearer +credential: whoever holds it holds the session. This container runs the agent +with permissions disabled. VNC is auto-passworded; keep it that way. Only keep +the single site the task needs, and never print, log, copy, or write these +files back to a shared/primary account. Treat them as secrets. + +## CLI (no script needed) + +```bash +cloakbrowser info # what binary launches, license tier, env checks +cloakbrowser info --quick # skip the launch test +``` + +## Gotchas + +- Bare `import 'cloakbrowser'` only resolves when cwd is `$CLOAK_APP_DIR` + (Node ESM resolves bare specifiers from cwd for `--input-type=module`). From + elsewhere, import the absolute path: + `import(process.env.CLOAK_APP_DIR + '/node_modules/cloakbrowser/dist/index.js')`. +- Do not run `npx cloakbrowser ...` -- the wrapper is not global; use the + `cloakbrowser` command (symlinked) or the `$CLOAK_APP_DIR` binary. +- For reCAPTCHA scoring, use native `setTimeout`, not `page.waitForTimeout()` + (the latter sends CDP traffic reCAPTCHA detects); use `page.type()` not + `page.fill()` so keystroke events fire. +- CloakBrowser prevents CAPTCHAs; it does not solve them. Bring your own proxy. diff --git a/versions.env b/versions.env index 029154c..0183027 100644 --- a/versions.env +++ b/versions.env @@ -17,6 +17,10 @@ KIMI_CODE_VERSION=0.28.1 CCX_VERSION=v0.11.0 COPILOT_API_VERSION=0ea08febdd7e3e055b03dd298bf57e669500b5c1 PLAYWRIGHT_VERSION=1.61.1 +# CloakBrowser npm wrapper version. This also pins the Chromium binary: +# the wrapper hardcodes per-arch free-binary versions (linux-x64 146.x.x.5, +# linux-arm64 146.x.x.3), so bumping the wrapper is what moves Chromium. +CLOAKBROWSER_WRAPPER_VERSION=0.4.12 RUST_TOOLCHAINS=stable RUST_DEFAULT_TOOLCHAIN=stable From fe6782bbdab892f2c1621c7c86bf11716be8548b Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Mon, 27 Jul 2026 01:55:18 -0700 Subject: [PATCH 2/4] ci(cloak): stop the optional cloak image from gating releases - release job no longer needs build-and-push-cloak; the cloak job is continue-on-error. A ~200MB Chromium bake per arch (arm64 under QEMU) must not sit on the critical path of a GitHub Release, and a flake in an optional layer must not fail a release whose artifact is deva.sh - cloak holds no agent CLIs and only changes when Dockerfile.cloak, cloak-entrypoint.sh or CLOAKBROWSER_WRAPPER_VERSION change -- never on a version bump. Nightly already skipped it for the same reason - release notes: :cloak is the rolling tag, :vX.Y.Z-cloak may lag - skill: goto(data:,{domcontentloaded}) not setContent() -- setContent hangs on a fresh daemon page waiting for a load event that never fires - dev log records the live proofs that close the 07-21 unverified gap: RFB 003.008 on :5900, CDP on :9222, connectOverCDP drives the live browser and the daemon survives client disconnect Refs #456 --- .github/workflows/ci.yml | 6 ++++++ .github/workflows/release.yml | 13 ++++++++++--- CHANGELOG.md | 7 +++++-- DEV-LOGS.md | 8 ++++++++ skills/deva-cloak/SKILL.md | 4 ++++ 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 665d9e2..6b18a54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,6 +169,12 @@ jobs: set -euo pipefail ./scripts/test-install-agent-tooling.sh + - name: Smoke image/tag precedence + shell: bash + run: | + set -euo pipefail + ./scripts/test-image-precedence.sh + - name: Smoke kimi auth wiring shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76d9f26..ad546c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -217,10 +217,17 @@ jobs: RUST_DEFAULT_TOOLCHAIN=${{ needs.load-version-pins.outputs.rust_default_toolchain }} RUST_TARGETS=${{ needs.load-version-pins.outputs.rust_targets }} + # Optional, non-gating: the cloak layer bakes a ~200MB Chromium per arch + # (arm64 under QEMU), holds no agent CLIs, and changes only when + # Dockerfile.cloak / cloak-entrypoint.sh / CLOAKBROWSER_WRAPPER_VERSION move + # -- not on a deva.sh version bump. The GitHub Release must not be hostage to + # it, so `release` does not wait on this job and a failure here does not fail + # the release. Re-run this job alone to publish a missing -cloak tag. build-and-push-cloak: name: Build and Push Cloak Profile Image runs-on: ubuntu-latest needs: [prepare, load-version-pins, build-and-push-rust] + continue-on-error: true permissions: contents: read packages: write @@ -270,7 +277,7 @@ jobs: release: name: Create GitHub Release runs-on: ubuntu-latest - needs: [prepare, build-and-push, build-and-push-rust, build-and-push-cloak] + needs: [prepare, build-and-push, build-and-push-rust] permissions: contents: write steps: @@ -309,9 +316,9 @@ jobs: echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${RELEASE_TAG}-rust\`" >> release_notes.md echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:rust\`" >> release_notes.md echo "" >> release_notes.md - echo "**Cloak Profile (CloakBrowser stealth Chromium, headed):**" >> release_notes.md - echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${RELEASE_TAG}-cloak\`" >> release_notes.md + echo "**Cloak Profile (CloakBrowser stealth Chromium, headed) - optional:**" >> release_notes.md echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:cloak\`" >> release_notes.md + echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${RELEASE_TAG}-cloak\` (published when the non-gating cloak build finishes; it may lag this release)" >> release_notes.md echo "" >> release_notes.md echo "## Supported Architectures" >> release_notes.md echo "" >> release_notes.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 177a39e..9ec3718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,8 +50,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 in versions.env pins the wrapper and thereby Chromium. Ships the `deva-cloak` skill (keyed off `DEVA_CLOAK=1`), `make build-cloak` / `test-cloak` targets, a paths-filtered `cloak-image.yml` CI workflow, - and release pushes `:cloak` / `:vX.Y.Z-cloak` tags. Nightly skips cloak - on purpose — that layer contains no agent CLIs. + and release pushes `:cloak` / `:vX.Y.Z-cloak` tags. cloak is an auxiliary + image on its own cadence: nightly skips it (that layer holds no agent + CLIs) and the release cloak job is non-gating (`continue-on-error`, not + in the `release` job's `needs`) — a ~200MB Chromium bake per arch must + never hold the GitHub Release hostage. ### Fixed - `--trace` launch killed by `cp: cannot create regular file diff --git a/DEV-LOGS.md b/DEV-LOGS.md index 8932701..ac61666 100644 --- a/DEV-LOGS.md +++ b/DEV-LOGS.md @@ -13,6 +13,14 @@ - Minimal markdown markers, no unnecessary formatting, minimal emojis. - Reference issue numbers in the format `#` for easy linking. +# [2026-07-27] Dev Log: cloak release policy + closing the unverified VNC/daemon gap #456 +- Why: two loose ends before shipping. (1) The 2026-07-21 entry left "x11vnc serving RFB and a live daemon inside the built image" unverified — that env couldn't apt-install. (2) release.yml made the GitHub Release `needs` the cloak build, so a ~200MB Chromium bake per arch (arm64 under QEMU) sat on the critical path of every release, including patch releases that touch nothing in that layer. +- What: + - Live verification against the built `:cloak` image (wrapper 0.5.2, Chrome 146.0.7680.177): x11vnc answers the RFB handshake on :5900 with `RFB 003.008`; cloak-browserd serves CDP on 127.0.0.1:9222; a separate playwright-core client `connectOverCDP`s, drives a page, and the daemon survives that client's `close()`. Stealth confirmed live: UA spoofs `Windows NT 10.0`, `navigator.webdriver === false`. First probe raced the daemons — both are up ~4s after boot, so poll the ports, don't poll only the display. + - Agent-facing footgun found while proving it: `page.setContent()` on a fresh daemon page hangs waiting for a `load` event that never fires. `goto('data:text/html,...', {waitUntil:'domcontentloaded'})` is the working equivalent — documented in the skill. + - Release policy: cloak is an auxiliary image on its own cadence, NOT a general releasable profile. It holds no agent CLIs, and its content only moves when Dockerfile.cloak / cloak-entrypoint.sh / CLOAKBROWSER_WRAPPER_VERSION move — never on a deva.sh version bump. So the cloak job is now `continue-on-error: true` and is OUT of the `release` job's `needs`; release notes present `:cloak` as the rolling tag and mark `:vX.Y.Z-cloak` as possibly lagging. Nightly already skipped cloak for the same reason. +- Result: all four live proofs pass; `make test-cloak` green. The structural point: an optional 200MB capability layer must never be able to fail the mandatory release artifact. Follow-up filed for the profile registry — profile identity is currently duplicated across deva.sh, Makefile, and three workflows. + # [2026-07-21] Dev Log: cloak always-on browser + VNC + persistent profile #456 - Why: three asks for auth sites in the cloak browser — (1) VNC auto-secured with a password, (2) an always-on browser the agent can drive freely, (3) mount what preserves the whole session to the host. Verified the load-bearing assumption first in a throwaway cloak container: a persistent CloakBrowser launched with `--remote-debugging-port` exposes a CDP endpoint a SEPARATE Playwright client reaches via connectOverCDP, drives, and the browser survives that client disconnecting. So one daemon can back both the human (VNC) and the agent (CDP). - What: diff --git a/skills/deva-cloak/SKILL.md b/skills/deva-cloak/SKILL.md index 9ce32e1..18fd73e 100644 --- a/skills/deva-cloak/SKILL.md +++ b/skills/deva-cloak/SKILL.md @@ -164,6 +164,10 @@ await b.close(); // detaches; the daemon browser stays absence means there is no daemon and you should launch your own browser as above. +Use `goto()`, not `setContent()`: on a fresh daemon page `setContent` waits for +a `load` event that never fires and times out. `page.goto('data:text/html,...', +{waitUntil: 'domcontentloaded'})` is the working equivalent for injected markup. + **Persistence / what to mount.** `--cloak-vnc` (and `--cloak-browser`) auto-mounts `~/.config/deva/cloak-profile` (host) to the Chromium `userDataDir` `/home/deva/.cloak-profile`. That one dir is the whole session -- cookies, From a90ffc9304d955fb56e11fa6c7215ccb1b7e4032 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Mon, 27 Jul 2026 02:39:13 -0700 Subject: [PATCH 3/4] fix(ci): make test-image-precedence.sh executable CI invokes it as ./scripts/test-image-precedence.sh and got exit 126, Permission denied. Local runs used `bash