diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..fdf03ab --- /dev/null +++ b/.flake8 @@ -0,0 +1,20 @@ +[flake8] +max-line-length = 100 +extend-ignore = + E203, + E501, + W503, + E402, + # B008: false positive on FastAPI's Depends()/File()/Query()/Form() defaults + B008 +exclude = + .git, + __pycache__, + .venv, + build, + dist, + movensys_vlm/models, + movensys_sample/movensys_vlm, + movensys_sample/.venv +per-file-ignores = + movensys_sample/movensys_robopoly/tests/game/test_rules_m2.py:F821,F841 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..61ea07f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Lint + +on: + pull_request: + branches: [main, devel] + +defaults: + run: + shell: bash + +jobs: + lint: + name: flake8 (${{ matrix.python-version }} / ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + python-version: '3.10' + - os: ubuntu-24.04 + python-version: '3.12' + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install flake8 + run: | + python -m pip install --upgrade pip + python -m pip install flake8 flake8-bugbear + + - name: flake8 movensys_sample/movensys_robopoly + run: | + flake8 movensys_sample/movensys_robopoly \ + --exclude=.venv,__pycache__,build,dist + + - name: flake8 movensys_vlm + run: | + flake8 movensys_vlm \ + --exclude=.venv,__pycache__,models,static diff --git a/.github/workflows/movensys-intelligence.yml b/.github/workflows/movensys-intelligence.yml new file mode 100644 index 0000000..8be114f --- /dev/null +++ b/.github/workflows/movensys-intelligence.yml @@ -0,0 +1,65 @@ +name: CI - movensys_vlm + +on: + pull_request: + branches: [main, devel] + paths: + - 'movensys_vlm/**' + +jobs: + build-and-test: + strategy: + fail-fast: false + matrix: + include: + - ros_distro: jazzy + os: ubuntu-24.04 + - ros_distro: humble + os: ubuntu-22.04 + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Pre-install ros2-apt-source pinned to a known release so setup-ros + # skips its unauthenticated api.github.com call (rate-limited on + # shared runner IPs → empty version → 404 → curl exit 22). + - name: Pre-install ros2-apt-source + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends curl ca-certificates lsb-release + codename=$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME}}") + version=1.2.0 + deb="ros2-apt-source_${version}.${codename}_all.deb" + curl --fail --location --retry 5 --retry-delay 3 \ + -o "/tmp/${deb}" \ + "https://github.com/ros-infrastructure/ros-apt-source/releases/download/${version}/${deb}" + sudo dpkg -i "/tmp/${deb}" + rm -f "/tmp/${deb}" + sudo apt-get update + + - name: Setup ROS ${{ matrix.ros_distro }} + uses: ros-tooling/setup-ros@v0.7 + with: + required-ros-distributions: ${{ matrix.ros_distro }} + + - name: Install Python dependencies + run: | + if [ "${{ matrix.ros_distro }}" = "jazzy" ]; then + pip3 install --no-cache-dir --break-system-packages \ + fastapi "uvicorn[standard]" pydantic "openai>=1.30.0" Pillow + else + pip3 install --no-cache-dir \ + fastapi "uvicorn[standard]" pydantic "openai>=1.30.0" Pillow + fi + + - name: Test Python syntax + run: | + for f in main.py router.py ros2_node.py; do + echo "=== syntax check: $f ===" + python3 -m py_compile ${{ github.workspace }}/movensys_vlm/$f + echo " OK" + done diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000..f3f74b7 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,101 @@ +name: Unit Tests + +on: + pull_request: + branches: [main, devel] + +defaults: + run: + shell: bash + +jobs: + robopoly: + name: pytest movensys_robopoly (${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + python-version: '3.10' + - os: ubuntu-24.04 + python-version: '3.12' + + # Stub-mode invariant: external service URLs unset so adapters use stubs. + env: + STT_SERVICE_URL: '' + LLM_SERVICE_URL: '' + ROBOT_SERVICE_URL: '' + + defaults: + run: + working-directory: movensys_sample/movensys_robopoly + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then + python -m pip install -r requirements.txt + fi + python -m pip install pytest pytest-asyncio httpx + + - name: Python syntax check + run: | + find . -type f -name '*.py' \ + -not -path './.venv/*' \ + -not -path './build/*' \ + -not -path './__pycache__/*' \ + -print0 | xargs -0 -n1 python -m py_compile + + - name: pytest + run: | + # Exit code 5 = "no tests collected" is acceptable (all tests may be + # marked stale and skipped at module level). + rc=0 + python -m pytest -v tests/ --maxfail=10 || rc=$? + if [ "$rc" != "0" ] && [ "$rc" != "5" ]; then + exit "$rc" + fi + + vlm-syntax: + name: Python syntax check movensys_vlm (${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + python-version: '3.10' + - os: ubuntu-24.04 + python-version: '3.12' + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install runtime dependencies + run: | + python -m pip install --upgrade pip + python -m pip install fastapi 'uvicorn[standard]' pydantic 'openai>=1.30.0' Pillow + + - name: py_compile movensys_vlm Python files + run: | + for f in main.py router.py ros2_node.py memory_client.py vlm_client.py whisper_client.py whisper_server.py; do + if [ -f "movensys_vlm/$f" ]; then + echo "=== $f ===" + python -m py_compile "movensys_vlm/$f" + echo " OK" + fi + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1358417 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +movensys_vlm/models +movensys_vlm/vllm +.venv/ \ No newline at end of file diff --git a/README.md b/README.md index 284a8b4..35ce298 100644 --- a/README.md +++ b/README.md @@ -1 +1,154 @@ -# movensys-intellegence +# Movensys Intelligence + +Vision-language and speech intelligence layer for the +[`movensys-manipulator`](https://github.com/movensys/movensys-manipulator) +stack. Adds a FastAPI VLM service, a Whisper speech endpoint, a Qdrant +vector memory, optional Phoenix tracing, and sample applications that drive +the manipulator from natural language. + +## Overview + +This repository sits on top of the WMX ROS 2 manipulator stack and gives it +a higher-level reasoning layer: + +- **VLM service** — FastAPI server wrapping a vLLM-hosted Gemma 4 model + with image input, exposed as REST + WebSocket. It bridges to ROS 2 so it + can call manipulator services (`MovePose`, `MoveJoints`, `GetEefPose`, + etc.) directly. +- **Whisper service** — streaming speech-to-text used to issue commands by + voice. +- **Vector memory** — Qdrant-backed long-term memory for the VLM agent. +- **Sample apps** — `movensys_robopoly`, a board-game demo where the robot + picks and places pieces under VLM control, with a YOLO + AprilTag + perception pipeline and a dry-run mode that exercises the full stack + without moving the arm. + +The entire stack runs as a set of Docker compose services and supports +NVIDIA desktop GPUs, Jetson Thor, and Intel B60 / Panther Lake XPU. + +## Repository Layout + +``` +. +├── movensys_vlm/ +│ ├── main.py / router.py / ros2_node.py # FastAPI app + ROS 2 bridge +│ ├── vlm_client.py / whisper_client.py # vLLM + Whisper clients +│ ├── memory_client.py # Qdrant vector memory client +│ ├── models/ # Local model assets (Gemma, Whisper, embeddings) +│ ├── docker/ # Compose files: vllm, whisper, vectordb, vlm +│ └── doc/running.md # Step-by-step bring-up +└── movensys_sample/ + └── movensys_robopoly/ # Board-game demo (FastAPI + adapters) + ├── main.py / router.py + ├── pick_and_place.py + ├── adapters/ # robot, ros_image, stt, vlm + ├── game/ # rules, manager, decks, boards + ├── scripts/ # auto_play_dry_run, render helpers + └── docker/ # Compose stack +``` + +## Services and Ports + +| Service | Default port | Purpose | +|------------------------|--------------|------------------------------------------| +| `movensys_vlm` (FastAPI) | 8000 | VLM REST/WebSocket API + ROS 2 bridge | +| `vllm` | 9000 | vLLM OpenAI-compatible inference server | +| `whisper` | 9010 | Speech-to-text server | +| `vectordb` (Qdrant) | 6333 | Long-term vector memory | +| `movensys_robopoly` | 7999 | Robopoly demo UI/API | +| `phoenix` (optional) | 6006 | OpenTelemetry/LLM traces UI | + +## Requirements + +- Ubuntu 22.04 or 24.04 +- Docker with `docker compose` +- Hardware: NVIDIA GPU (desktop or Jetson Thor) or Intel XPU (B60 / Panther Lake) +- Local model weights placed under `movensys_vlm/models/` (Gemma 4 E2B/E4B, Whisper large-v3, embedding model) +- The [`movensys-manipulator`](https://github.com/movensys/movensys-manipulator) stack running (the VLM publishes/calls its ROS 2 services) + +## Quick Start + +### 1. Configure the host environment + +Add the following to your `~/.bashrc`: + +``` +export XPU_CORE=nvidia-gpu # {nvidia-gpu, intel-xpu} +export CPU_ARCH=amd64 # {amd64, arm64} +``` + +``` +source ~/.bashrc +``` + +### 2. Clone the repository + +``` +mkdir -p ~/workspaces +cd ~/workspaces +git clone https://github.com/movensys/movensys-intelligence.git +``` + +### 3. Start the VLM stack + +For Nvidia desktop, Jetson Thor, or Intel B60: + +``` +cd ~/workspaces/movensys-intelligence/movensys_vlm/docker +COMPOSE_PROFILES=$XPU_CORE docker compose -f vllm.yaml up -d --build +COMPOSE_PROFILES=$CPU_ARCH docker compose -f vectordb.yaml up -d --build +COMPOSE_PROFILES=$XPU_CORE docker compose -f whisper.yaml up -d --build +COMPOSE_PROFILES=$XPU_CORE docker compose -f movensys_vlm.yaml up -d --build +``` + +For Intel Panther Lake (vLLM uses a separate build path): + +``` +cd ~/workspaces/movensys-intelligence/movensys_vlm/docker +./vllm-intel-build.sh +./vllm-intel-run.sh +``` + +Wait for `application startup complete` in the vLLM logs before continuing. + +> On Jetson Thor or Intel Panther Lake, drop kernel caches between restarts +> if memory pressure builds up: +> `sync && sudo sysctl vm.drop_caches=3` + +Full bring-up, teardown, and Phoenix-tracing options are documented in +[`movensys_vlm/doc/running.md`](movensys_vlm/doc/running.md). + +### 4. Run a sample application + +The Robopoly board-game demo drives the manipulator via the VLM stack. With +the `movensys-manipulator` YOLO simulation example running (see +[`movensys-manipulator/doc/6a_yolo_simulation.md`](https://github.com/movensys/movensys-manipulator/blob/main/doc/6a_yolo_simulation.md)): + +``` +export MOVENSYS_PNP_DRY_RUN=0 # set to 1 to skip arm motion +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker +docker compose up -d --build +``` + +Open the UI on `http://localhost:7999/`, toggle `is_YOLO` on, and start a +game. Dry-run mode and the auto-play test script are described in +[`movensys_sample/doc/1a_robopoly_simulation.md`](movensys_sample/doc/1a_robopoly_simulation.md). + +### Pick-and-place from the command line + +``` +cd ~/workspaces/movensys-intelligence +python3 movensys_sample/movensys_robopoly/pick_and_place.py red_cube GO true 2>&1 | tee baseline.log +grep '\[timing\]' baseline.log +``` + +## Related Repositories + +- [movensys-manipulator](https://github.com/movensys/movensys-manipulator) — ROS 2 manipulator stack driven by this layer +- [movensys-simulation](https://github.com/movensys/movensys-simulation) — Isaac Sim scenes used by the demos +- [wmx-ros2](https://github.com/movensys/wmx-ros2) — Core WMX motion control packages +- [wmx-ros2-doc](https://github.com/movensys/wmx-ros2-doc) — WMX ROS 2 documentation site + +## License + +Released under the MIT License. diff --git a/movensys_sample/doc/1a_robopoly_simulation.md b/movensys_sample/doc/1a_robopoly_simulation.md new file mode 100644 index 0000000..150ddb1 --- /dev/null +++ b/movensys_sample/doc/1a_robopoly_simulation.md @@ -0,0 +1,51 @@ +# Running Robopoly Game +## Step 1: Movensys-manipulator +check `movensys-manipulator/doc` 1_ and 2_ +Run `movensys-manipulator/doc/6a_yolo_simulation.md` step 1-4 + +## Step 2: Run VLM package +Run `movensys_vlm/doc/running.md` + +## Step 3: Running movensys_robopoly +``` +export MOVENSYS_PNP_DRY_RUN=0 +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker +docker compose down +docker compose build +docker compose up -d +``` + +## Step 4: Enjoy the robopoly game +1. Click `Toggle is_YOLO` and check `is_YOLO` is set to ON. +2. Click Reset game and play the game. + + + + + + + + + +# Running Robopoly Game w/o moving robot arm +## Step 1: Running movensys_robopoly in DRY RUN mode +``` +export MOVENSYS_PNP_DRY_RUN=1 +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker +docker compose down +docker compose build +docker compose up -d +``` + +## Step 2: Enjoy the robopoly game +1. Click `Toggle is_YOLO` and check `is_YOLO` is set to OFF. +2. Set your microphone. +3. Click Reset game. +4. Press `Z` key and speak into microphone to request one of game action. +5. Press `X` key and speak to communicate game status, game strategies, etc. + +# Step 3: Auto dry run test (optional) +``` +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/ +python3 scripts/auto_play_dry_run.py +``` \ No newline at end of file diff --git a/movensys_sample/doc/1c_robopoly_real.md b/movensys_sample/doc/1c_robopoly_real.md new file mode 100644 index 0000000..a4318f2 --- /dev/null +++ b/movensys_sample/doc/1c_robopoly_real.md @@ -0,0 +1,68 @@ +# Running Robopoly Game +## 1. Execution Procedure + +### Step 1: Launch wmx-ros2 (Terminal 1) +```bash +cd ~/workspaces/movensys-intelligence/movensys_sample/doc +./run_robopoly.sh wmx-ros2 +``` + + + + +### Step 2a: Build containers on Nvidia env (Terminal 2) +```bash +./run_robopoly.sh build_nvidia +``` + +### Step 2b-1: Build vllm containers on Intel env (Terminal 2) +```bash +./run_robopoly.sh build_intel_vllm +``` + +### Step 2b-2: Build other containers (Terminal 3) +```bash +./run_robopoly.sh build_intel +``` + + +### Step 3. Run moveit, containers, yolo (Terminal 3) +- Make sure whether the build process is done. +```bash +docker logs -f movensys_manipulator_container +``` + +- Run the demo +```bash +./run_robopoly.sh run +``` + + + + + + +## 2. Debug tips (Optional) +### 2-1. vllm +``` +docker logs -f vllm_container +``` +### 2-2. movensys-manipulator +``` +docker logs -f movensys-manipulator +``` +### 2-3. moveit +``` +tmux a -t robopoly +``` + +### 2-4. tmux +1. excape tmux +- Sequentially press `Ctrl b` and `d` + +2. move screen +- Sequentially press `Ctrl b` and `` + +3. Visual mode +- Sequentially press `Ctrl b` and `[` +- Then, use `PgUp` or `PgDn`. \ No newline at end of file diff --git a/movensys_sample/doc/run_robopoly.sh b/movensys_sample/doc/run_robopoly.sh new file mode 100755 index 0000000..e1d567a --- /dev/null +++ b/movensys_sample/doc/run_robopoly.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +set -e + +MODE=${1:-} +case "$MODE" in + wmx-ros2|build_nvidia|build_intel_vllm|build_intel|run) ;; + *) + echo "Usage: $0 {wmx-ros2|build_nvidia|build_intel_vllm|build_intel|run}" >&2 + echo " wmx-ros2 Launch the wmx-ros2 manipulator driver (foreground, prompts for sudo)" >&2 + echo " build_nvidia Rebuild docker images and start persistent containers (NVIDIA GPU)" >&2 + echo " build_intel_vllm Down all + drop caches + build/run vllm only (Intel GPU)" >&2 + echo " build_intel Build remaining services: vectordb, vlm, whisper, manipulator, robopoly (Intel GPU)" >&2 + echo " run Start runtime containers + ROS launches in a tmux session" >&2 + echo "" >&2 + echo "Recommended order (each in its own terminal):" >&2 + echo " Terminal 1: $0 wmx-ros2" >&2 + echo " Terminal 2: $0 build_nvidia (NVIDIA GPU, only when code/images change)" >&2 + echo " Terminal 2: $0 build_intel_vllm (Intel GPU, only when code/images change)" >&2 + echo " Terminal 2: $0 build_intel (Intel GPU, after build_intel_vllm)" >&2 + echo " Terminal 3: $0 run" >&2 + exit 1 + ;; +esac + +# ============================================================================ +# BUILD MODE: 3-phase sequence mirrors movensys_vlm/doc/running.md +# Phase A — DOWN everything first +# Step 1: down movensys_vlm + vectordb + whisper +# Step 2: down vllm +# (+ manipulator + robopoly — local additions outside running.md) +# Phase B — DROP CACHES (Step 3) +# Phase C — BUILD + UP sequentially +# Step 4: vllm (4a nvidia/Thor/B60, 4b Intel Panther Lake) +# Step 5: vectordb + phoenix + movensys_vlm +# Step 6: whisper (en, --force-recreate) +# (+ manipulator + robopoly) +# +# Modes: +# build_nvidia Phase A + B + C (Step 4a vllm, then Steps 5-9) +# build_intel_vllm Phase A + B + C (Step 4b vllm only) +# build_intel Steps 5-9 only (run after build_intel_vllm) +# ============================================================================ + +# ----- Phase A: DOWN everything -------------------------------------------- +_build_down() { + echo "==> [Phase A] down all containers" + + echo "all of docker down" + cd "${MOVENSYS_MANIPULATOR_PACKAGES}/docker" + docker compose -f "${MOVENSYS_ROS_VERSION}.yaml" \ + -f "movensys_manipulator.${CPU_ARCH}.yaml" down + + cd ~/workspaces/movensys-intelligence/movensys_vlm/docker + COMPOSE_PROFILES=$XPU_CORE docker compose -f movensys_vlm.yaml down + COMPOSE_PROFILES=$CPU_ARCH docker compose -f vectordb.yaml down + COMPOSE_PROFILES=$XPU_CORE docker compose -f whisper.yaml down + COMPOSE_PROFILES=$XPU_CORE docker compose -f vllm.yaml down + + cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker + docker compose down +} + +# ----- Phase B: DROP CACHES ------------------------------------------------ +_build_drop_caches() { + echo "==> [Phase B] release memory caches" + sync && sudo sysctl vm.drop_caches=3 +} + +# ----- Phase C, Steps 5-9: remaining services build + up ------------------- +_build_services() { + cd ~/workspaces/movensys-intelligence/movensys_vlm/docker + + echo " -- Step 5: movensys_vlm build + up" + export PHOENIX_TRACING=0 + COMPOSE_PROFILES=$XPU_CORE docker compose -f movensys_vlm.yaml build + COMPOSE_PROFILES=$XPU_CORE docker compose -f movensys_vlm.yaml up -d --force-recreate + + echo " -- Step 6: whisper build + up (en, --force-recreate)" + COMPOSE_PROFILES=$XPU_CORE docker compose -f whisper.yaml build + WHISPER_DEFAULT_LANGUAGE=en COMPOSE_PROFILES=$XPU_CORE \ + docker compose -f whisper.yaml up -d --force-recreate + + echo " -- Step 7: movensys-manipulator build + up" + cd "${MOVENSYS_MANIPULATOR_PACKAGES}/docker" + docker compose -f "${MOVENSYS_ROS_VERSION}.yaml" \ + -f "movensys_manipulator.${CPU_ARCH}.yaml" build + docker compose -f "${MOVENSYS_ROS_VERSION}.yaml" \ + -f "movensys_manipulator.${CPU_ARCH}.yaml" up -d + + echo " -- Step 8: robopoly build + up" + export MOVENSYS_PNP_DRY_RUN=0 + cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker + docker compose build + docker compose up -d +} + +if [[ "$MODE" == "build_nvidia" ]]; then + _build_down + _build_drop_caches + + # ----- Phase C: BUILD + UP sequentially ---------------------------------- + echo "==> [Phase C] build + up sequentially" + cd ~/workspaces/movensys-intelligence/movensys_vlm/docker + echo " -- Step 4a: vllm build + up (Nvidia/Thor/B60)" + COMPOSE_PROFILES=$XPU_CORE docker compose -f vllm.yaml build + COMPOSE_PROFILES=$XPU_CORE docker compose -f vllm.yaml up -d + + _build_services + + echo "==> [build] done" + exit 0 +fi + +if [[ "$MODE" == "build_intel_vllm" ]]; then + _build_down + _build_drop_caches + + # ----- Phase C: vllm only ------------------------------------------------ + echo "==> [Phase C] vllm build + up" + cd ~/workspaces/movensys-intelligence/movensys_vlm/docker + echo " -- Step 4b: vllm build + run (Intel Panther Lake)" + ./vllm-intel-build.sh + ./vllm-intel-run.sh +fi + +if [[ "$MODE" == "build_intel" ]]; then + # Steps 5-9 only — run after build_intel_vllm has brought vllm up. + echo "==> [build_intel] build + up remaining services" + _build_services + + echo "==> [build] done" + exit 0 +fi + +# ============================================================================ +# WMX-ROS2 MODE: foreground manipulator driver, owns its own terminal + sudo +# ============================================================================ +if [[ "$MODE" == "wmx-ros2" ]]; then + echo "==> [wmx-ros2] launching manipulator driver" + exec sudo --preserve-env=PATH \ + --preserve-env=AMENT_PREFIX_PATH \ + --preserve-env=COLCON_PREFIX_PATH \ + --preserve-env=PYTHONPATH \ + --preserve-env=LD_LIBRARY_PATH \ + --preserve-env=ROS_DISTRO \ + --preserve-env=ROS_VERSION \ + --preserve-env=ROS_PYTHON_VERSION \ + --preserve-env=ROS_DOMAIN_ID \ + --preserve-env=RMW_IMPLEMENTATION \ + bash -c "source /opt/ros/${ROS_DISTRO}/setup.bash \ + && source ${HOME}/workspaces/movensys_ws/install/setup.bash \ + && ros2 launch wmx_ros2_package wmx_ros2_cr3a_manipulator.launch.py use_sim_time:=false" +fi + +# ============================================================================ +# RUN MODE: bring runtime containers up + launch ROS nodes in a tmux session. +# ============================================================================ + +SESSION=robopoly + +# Wipe any prior session so re-runs start clean +tmux kill-session -t "$SESSION" 2>/dev/null || true + +# --- Window 1: MoveIt2 ------------------------------------------------------- +tmux new-session -d -s "$SESSION" -n moveit +tmux send-keys -t "$SESSION:moveit" "\ +mros ros2 launch movensys_manipulator_moveit_config moveit.launch.py use_sim_time:=true\ +" Enter +sleep 3 + +# --- Window 2: YOLO cube detection ------------------------------------------- +tmux new-window -t "$SESSION" -n yolo +tmux send-keys -t "$SESSION:yolo" "\ +mros ros2 launch movensys_manipulator_perception yolo_dice_and_cube_detector.launch.py \ +" Enter +sleep 3 + +tmux select-window -t "$SESSION:moveit" +tmux attach -t "$SESSION" diff --git a/movensys_sample/movensys_robopoly/.gitignore b/movensys_sample/movensys_robopoly/.gitignore new file mode 100644 index 0000000..2c5ad3b --- /dev/null +++ b/movensys_sample/movensys_robopoly/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +.pytest_cache/ +.coverage +.venv/ +*.egg-info/ diff --git a/movensys_sample/movensys_robopoly/adapters/__init__.py b/movensys_sample/movensys_robopoly/adapters/__init__.py new file mode 100644 index 0000000..1c73595 --- /dev/null +++ b/movensys_sample/movensys_robopoly/adapters/__init__.py @@ -0,0 +1,6 @@ +from adapters.robot import RobotAdapter +from adapters.ros_image import RosImageSubscriber +from adapters.stt import STTAdapter +from adapters.vlm import VLMAdapter + +__all__ = ["RobotAdapter", "RosImageSubscriber", "STTAdapter", "VLMAdapter"] diff --git a/movensys_sample/movensys_robopoly/adapters/robot.py b/movensys_sample/movensys_robopoly/adapters/robot.py new file mode 100644 index 0000000..2f32851 --- /dev/null +++ b/movensys_sample/movensys_robopoly/adapters/robot.py @@ -0,0 +1,105 @@ +"""Robot motion adapter — routed through movensys_vlm_container. + +All robot motion goes through the orchestrator's HTTP API. The previous +`roll_dice` / `move_piece` / `base_position` methods modeled domain +concepts (Monopoly dice, horse moves) that the orchestrator doesn't +expose — those flows are implemented directly in robopoly's own +`/api/dice/roll_robot` and `/api/move/apply_robot` routes via the +`pick_and_place.py` subprocess, NOT through this adapter. + +This adapter now wraps the low-level motion / IO routes the orchestrator +actually offers: + - POST /api/services/gripper (open/close) + - GET /api/services/get_eef_pose + - POST /api/move/absolute_cartesian_base + - POST /api/move/relative_cartesian_base + +Stub when MOVENSYS_VLM_URL is empty: methods return success without +performing any motion. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Literal, Sequence + +import httpx + + +Mode = Literal["live", "stub"] + + +def _as_xyz(seq: Sequence[float], name: str) -> list[float]: + values = list(seq) + if len(values) != 3: + raise ValueError(f"{name} must have 3 elements, got {len(values)}") + return [float(v) for v in values] + + +@dataclass +class RobotAdapter: + base_url: str + timeout_s: float = 30.0 + + @classmethod + def from_env(cls) -> "RobotAdapter": + return cls(base_url=os.environ.get("MOVENSYS_VLM_URL", "").strip()) + + @property + def mode(self) -> Mode: + return "live" if self.base_url else "stub" + + def health(self) -> dict[str, Any]: + payload: dict[str, Any] = {"mode": self.mode, "endpoint": "/api/move/*"} + if self.base_url: + payload["url"] = self.base_url + return payload + + async def gripper(self, open_: bool) -> dict[str, Any]: + """POST /api/services/gripper {data: bool}.""" + if self.mode == "stub": + return {"success": True, "message": "stub"} + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + r = await client.post( + f"{self.base_url}/api/services/gripper", json={"data": open_} + ) + r.raise_for_status() + return r.json() + + async def get_eef_pose(self) -> dict[str, Any]: + """GET /api/services/get_eef_pose.""" + if self.mode == "stub": + return {"success": True, "pos": [0.0, 0.0, 0.0], "rpy": [0.0, 0.0, 0.0], "message": "stub"} + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + r = await client.get(f"{self.base_url}/api/services/get_eef_pose") + r.raise_for_status() + return r.json() + + async def move_absolute_cartesian_base( + self, pos: Sequence[float], ori: Sequence[float] + ) -> dict[str, Any]: + """POST /api/move/absolute_cartesian_base {pos:[x,y,z], ori:[roll,pitch,yaw]}.""" + body = {"pos": _as_xyz(pos, "pos"), "ori": _as_xyz(ori, "ori")} + if self.mode == "stub": + return {"success": True, "message": "stub", **body} + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + r = await client.post( + f"{self.base_url}/api/move/absolute_cartesian_base", json=body + ) + r.raise_for_status() + return r.json() + + async def move_relative_cartesian_base( + self, pos: Sequence[float], ori: Sequence[float] + ) -> dict[str, Any]: + """POST /api/move/relative_cartesian_base.""" + body = {"pos": _as_xyz(pos, "pos"), "ori": _as_xyz(ori, "ori")} + if self.mode == "stub": + return {"success": True, "message": "stub", **body} + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + r = await client.post( + f"{self.base_url}/api/move/relative_cartesian_base", json=body + ) + r.raise_for_status() + return r.json() diff --git a/movensys_sample/movensys_robopoly/adapters/ros_image.py b/movensys_sample/movensys_robopoly/adapters/ros_image.py new file mode 100644 index 0000000..e6f6adc --- /dev/null +++ b/movensys_sample/movensys_robopoly/adapters/ros_image.py @@ -0,0 +1,167 @@ +"""rclpy-based subscriber for YOLO debug image topics. + +Robopoly's original design route image data through the movensys_vlm +orchestrator (see doc/API.md note about no `rclpy` in robopoly), but the +operator's deploy flow only rebuilds the robopoly container — forcing a +VLM rebuild for the debug-image overlay was awkward in practice. This +module ships the subscription with robopoly so a single +`docker compose build` in movensys_robopoly/docker brings the feature up. + +The subscriber spins in a background thread (rclpy `MultiThreadedExecutor` +on a dedicated thread) so FastAPI's asyncio loop is untouched. + +Topics: +- /yolo_dice_detector/debug_image — dice detector overlay +- /yolo_cube_detector/debug_image — cube detector overlay + +Both publish `sensor_msgs/Image` with either `rgb8` or `bgr8` encoding. +We JPEG-encode the latest frame as it arrives and cache the base64 +payload so the FastAPI WS handler can serve it without copying every +poll. +""" +from __future__ import annotations + +import base64 +import logging +import threading +from typing import Optional + +log = logging.getLogger("monopoly.ros_image") + + +class RosImageSubscriber: + """Optional ROS subscriber. Falls back to None on import failure so the + server still boots in environments without rclpy installed (e.g. dev + machines outside the docker container).""" + + def __init__(self) -> None: + self.latest_dice_debug: Optional[dict] = None + self.latest_cube_debug: Optional[dict] = None + # Raw gripper-camera RGB frame for the chance-card overlay. Same + # encoding pipeline as the YOLO debug streams. + self.latest_hand_rgb: Optional[dict] = None + self._thread: Optional[threading.Thread] = None + self._executor = None + self._node = None + self._available = False + self._np = None + self._cv2 = None + + try: + import rclpy # noqa: F401 + from sensor_msgs.msg import Image # noqa: F401 + import numpy as np + import cv2 + except ImportError as exc: + log.warning( + "rclpy / sensor_msgs / numpy / cv2 unavailable — YOLO debug " + "image overlay disabled (%s). The dice and cube debug streams " + "will be empty; the rest of the game is unaffected.", + exc, + ) + return + + self._np = np + self._cv2 = cv2 + self._available = True + + def start(self) -> None: + if not self._available or self._thread is not None: + return + import rclpy + from rclpy.executors import MultiThreadedExecutor + from rclpy.callback_groups import ReentrantCallbackGroup + from rclpy.node import Node + from sensor_msgs.msg import Image + + if not rclpy.ok(): + try: + rclpy.init() + except Exception as exc: + log.warning("rclpy.init failed — disabling YOLO overlay: %s", exc) + self._available = False + return + + class _Node(Node): + def __init__(self, owner: "RosImageSubscriber") -> None: + super().__init__("movensys_monopoly_yolo_overlay") + self._owner = owner + cb = ReentrantCallbackGroup() + self.create_subscription( + Image, "/yolo_dice_detector/debug_image", + self._on_dice, 1, callback_group=cb, + ) + self.create_subscription( + Image, "/yolo_cube_detector/debug_image", + self._on_cube, 1, callback_group=cb, + ) + self.create_subscription( + Image, "/image_hand/rgb", + self._on_hand, 1, callback_group=cb, + ) + + def _on_dice(self, msg: Image) -> None: + self._owner.latest_dice_debug = self._owner._encode(msg) + + def _on_cube(self, msg: Image) -> None: + self._owner.latest_cube_debug = self._owner._encode(msg) + + def _on_hand(self, msg: Image) -> None: + self._owner.latest_hand_rgb = self._owner._encode(msg) + + self._node = _Node(self) + self._executor = MultiThreadedExecutor() + self._executor.add_node(self._node) + + def _spin() -> None: + try: + self._executor.spin() + except Exception: + log.exception("YOLO overlay executor crashed") + + self._thread = threading.Thread( + target=_spin, name="yolo-overlay-spin", daemon=True, + ) + self._thread.start() + log.info("YOLO debug-image subscriber started") + + def stop(self) -> None: + if self._executor is not None: + try: + self._executor.shutdown() + except Exception: + pass + if self._node is not None: + try: + self._node.destroy_node() + except Exception: + pass + try: + import rclpy + if rclpy.ok(): + rclpy.shutdown() + except Exception: + pass + + def _encode(self, msg) -> Optional[dict]: + try: + img = self._np.frombuffer(bytes(msg.data), dtype=self._np.uint8) + img = img.reshape(msg.height, msg.width, 3) + if msg.encoding == "rgb8": + bgr = self._cv2.cvtColor(img, self._cv2.COLOR_RGB2BGR) + else: + bgr = img + ok, buf = self._cv2.imencode( + ".jpg", bgr, [self._cv2.IMWRITE_JPEG_QUALITY, 80], + ) + if not ok: + return None + return { + "data": base64.b64encode(buf.tobytes()).decode("utf-8"), + "encoding": "jpeg", + "width": msg.width, + "height": msg.height, + } + except Exception as exc: + log.debug("encode failure: %s", exc) + return None diff --git a/movensys_sample/movensys_robopoly/adapters/stt.py b/movensys_sample/movensys_robopoly/adapters/stt.py new file mode 100644 index 0000000..d981959 --- /dev/null +++ b/movensys_sample/movensys_robopoly/adapters/stt.py @@ -0,0 +1,53 @@ +"""Whisper STT adapter — routed through movensys_vlm_container. + +All robopoly external comms go through the orchestrator's HTTP API. This +adapter posts audio to `POST {MOVENSYS_VLM_URL}/api/whisper/transcribe`. + +Stub when MOVENSYS_VLM_URL is empty: UI falls back to text input and +/api/debug/inject_utterance. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Literal + +import httpx + + +Mode = Literal["live", "stub"] + + +@dataclass +class STTAdapter: + base_url: str + timeout_s: float = 30.0 + + @classmethod + def from_env(cls) -> "STTAdapter": + return cls(base_url=os.environ.get("MOVENSYS_VLM_URL", "").strip()) + + @property + def mode(self) -> Mode: + return "live" if self.base_url else "stub" + + def health(self) -> dict[str, Any]: + payload: dict[str, Any] = {"mode": self.mode, "endpoint": "/api/whisper/transcribe"} + if self.base_url: + payload["url"] = self.base_url + return payload + + async def transcribe(self, audio: bytes, filename: str = "utterance.wav") -> str: + """POST audio -> {text}. Stub mode raises (caller should route to debug input).""" + if self.mode == "stub": + raise RuntimeError("STT adapter is in stub mode; use /api/debug/inject_utterance") + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + # The orchestrator expects the multipart field to be named `file`. + files = {"file": (filename, audio, "audio/wav")} + r = await client.post(f"{self.base_url}/api/whisper/transcribe", files=files) + r.raise_for_status() + data = r.json() + if data.get("error"): + raise RuntimeError(f"whisper error: {data['error']}") + return str(data.get("text") or "") diff --git a/movensys_sample/movensys_robopoly/adapters/vlm.py b/movensys_sample/movensys_robopoly/adapters/vlm.py new file mode 100644 index 0000000..4b1540f --- /dev/null +++ b/movensys_sample/movensys_robopoly/adapters/vlm.py @@ -0,0 +1,144 @@ +"""VLM intent adapter — routed through movensys_vlm_container. + +The orchestrator's /api/vlm/infer returns free-form text from the VLM, so +this adapter builds a strict-JSON system prompt to coerce a structured +{intent, args, confidence} response, then parses it. + +Intent vocabulary is owned by this package; non-vocabulary intents are +collapsed to `unknown`. + +Image grounding: the orchestrator pulls images directly from the ROS +camera identified by `camera` ("top" | "hand" | "none"). The legacy +`image_b64` parameter is accepted but ignored — there is no path to ship +raw bytes through the orchestrator's API today. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from dataclasses import dataclass +from typing import Any, Literal + +import httpx + + +log = logging.getLogger(__name__) + +Mode = Literal["live", "stub"] +Camera = Literal["top", "hand", "none"] + +INTENT_VOCABULARY = frozenset( + { + "start_game", + "roll_dice", + "end_turn", + "buy", + "buy_and_build", + "skip", + "jail_exit", + "mortgage", + "unmortgage", + "sell_building", + "board_query", + "unknown", + } +) + + +_CODE_FENCE_RE = re.compile(r"^\s*```(?:json)?\s*|\s*```\s*$", re.IGNORECASE) + + +def _strip_code_fence(text: str) -> str: + return _CODE_FENCE_RE.sub("", text).strip() + + +def _build_system_prompt() -> str: + vocab = ", ".join(sorted(INTENT_VOCABULARY)) + return ( + "You are an intent classifier for a Monopoly-style board game.\n" + "Read the user utterance (and any board context provided) and reply " + "with STRICT JSON in this shape:\n" + ' {"intent": "", ' + '"args": {}, ' + '"confidence": }\n' + "Return ONLY the JSON object. No prose, no markdown fences, no commentary." + ) + + +@dataclass +class VLMAdapter: + base_url: str + timeout_s: float = 30.0 + + @classmethod + def from_env(cls) -> "VLMAdapter": + return cls(base_url=os.environ.get("MOVENSYS_VLM_URL", "").strip()) + + @property + def mode(self) -> Mode: + return "live" if self.base_url else "stub" + + def health(self) -> dict[str, Any]: + payload: dict[str, Any] = {"mode": self.mode, "endpoint": "/api/vlm/infer"} + if self.base_url: + payload["url"] = self.base_url + return payload + + async def infer( + self, + *, + text: str | None = None, + image_b64: str | None = None, + context: dict[str, Any] | None = None, + camera: Camera = "none", + ) -> dict[str, Any]: + """POST /api/vlm/infer -> {intent, args, confidence?}. Stub raises.""" + if self.mode == "stub": + raise RuntimeError("LLM adapter is in stub mode; use /api/debug/simulate_llm_intent") + + if image_b64 is not None: + # The orchestrator pulls camera frames itself; raw bytes have no + # transport. Pass camera="top" or "hand" to ground on a live frame. + log.debug("VLMAdapter: image_b64 ignored — use camera='top'|'hand' instead") + + prompt_parts: list[str] = [] + if text: + prompt_parts.append(text) + if context: + prompt_parts.append(f"Context: {json.dumps(context, ensure_ascii=False)}") + user_prompt = "\n\n".join(prompt_parts) or "Classify the player's intent." + + body = { + "camera": camera, + "prompt": user_prompt, + "system_prompt": _build_system_prompt(), + "max_tokens": 256, + "temperature": 0.0, + } + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + r = await client.post(f"{self.base_url}/api/vlm/infer", json=body) + r.raise_for_status() + data = r.json() + + if data.get("error"): + log.warning("VLMAdapter: orchestrator returned error: %s", data["error"]) + return {"intent": "unknown", "args": {}, "confidence": None} + + raw = _strip_code_fence(str(data.get("response") or "")) + try: + obj = json.loads(raw) + except json.JSONDecodeError: + log.warning("VLMAdapter: response not valid JSON: %r", raw[:200]) + return {"intent": "unknown", "args": {}, "confidence": None} + + intent = obj.get("intent", "unknown") + if intent not in INTENT_VOCABULARY: + intent = "unknown" + return { + "intent": intent, + "args": obj.get("args") or {}, + "confidence": obj.get("confidence"), + } diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_down.png b/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_down.png new file mode 100644 index 0000000..866c5ac Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_down.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_middle.png b/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_middle.png new file mode 100644 index 0000000..8670df1 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_middle.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_up.png b/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_up.png new file mode 100644 index 0000000..181cfe7 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_korea/whiteboard_korea_up.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_down.png b/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_down.png new file mode 100644 index 0000000..c6cf8d2 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_down.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_middle.png b/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_middle.png new file mode 100644 index 0000000..4188519 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_middle.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_up.png b/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_up.png new file mode 100644 index 0000000..10669e8 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_number/whiteboard_w_number_up.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_korea.png b/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_korea.png new file mode 100644 index 0000000..673317b Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_korea.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_usa.png b/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_usa.png new file mode 100644 index 0000000..31b0ee9 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_usa.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_w_number.png b/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_w_number.png new file mode 100644 index 0000000..df9de0e Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_samples/whiteboard_w_number.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_down.png b/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_down.png new file mode 100644 index 0000000..25aa772 Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_down.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_middle.png b/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_middle.png new file mode 100644 index 0000000..e61d30f Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_middle.png differ diff --git a/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_up.png b/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_up.png new file mode 100644 index 0000000..5a80e9a Binary files /dev/null and b/movensys_sample/movensys_robopoly/boards/whiteboard_usa/whiteboard_usa_up.png differ diff --git a/movensys_sample/movensys_robopoly/doc/API.md b/movensys_sample/movensys_robopoly/doc/API.md new file mode 100644 index 0000000..8af55d3 --- /dev/null +++ b/movensys_sample/movensys_robopoly/doc/API.md @@ -0,0 +1,215 @@ +# API surface of `movensys_robopoly` + +This doc enumerates every API the robopoly stack exposes or consumes, and +how the pieces are wired together. + +robopoly is uvicorn-served on `:7999` (host-side, not Docker). The browser +makes two kinds of calls from `http://localhost:7999/`: + +1. **Same-origin** to robopoly's own backend at `:7999`. +2. **Cross-origin** to the `movensys_vlm` orchestrator at `:8000` (via + `VLM_BASE = http://localhost:8000` in `static/app.js`). + +All robot motion ultimately goes through the orchestrator — the +`pick_and_place.py` subprocess is itself an HTTP client of `:8000`, so the +"robopoly only talks to `movensys_vlm` via its API" rule holds end-to-end. + +--- + +## 1. Browser → robopoly backend (`:7999`) + +### Health / adapter mode + +- `GET /api/health` — liveness probe. +- `GET /api/robot/health` — reports `RobotAdapter` mode (`live` / `stub`, + derived from `MOVENSYS_VLM_URL` env). **No network call** — just reads + the env var. +- `GET /api/stt/health` — same for `STTAdapter`. +- `GET /api/vlm/health` — same for `VLMAdapter`. +- `GET /api/modes` — aggregates all three. + +> The `mode=live` badge only confirms `MOVENSYS_VLM_URL` is set, not that +> the orchestrator is reachable. The **Ask** button is the real +> reachability test. + +### Game state + +- `GET /api/game/state` — full FSM / positions / money / properties + snapshot. Read from in-memory `GameManager`. +- `POST /api/game/start` — start a board; emits `game_started` on the + event bus. +- `POST /api/game/end_turn` — flip turn; emits `fsm_transition`. +- `GET /api/game/winner` — current winner or `null`. +- `POST /api/game/config` — patch `RuntimeConfig` (`dice_source`, + `auctions_enabled`, `income_tax_mode`, `player_colors`, `is_YOLO`). +- `POST /api/game/save_state` — writes the current snapshot to + `saved_status.yaml` on disk. +- `POST /api/game/load_state` — reads `saved_status.yaml` and replaces + state. +- `GET /api/game/next_prompt` — M1 stub hint string. +- `GET /api/game/rules` — returns `doc/game_logic.md` verbatim as + `text/markdown`. Used by the VLM-player agent loop to seed the + system prompt with the authoritative spec on every page load + (see `vlm_as_player.md` §3). + +### Dice / move + +- `POST /api/dice/request` — mark dice-request (in-memory). +- `POST /api/dice/submit` — submit manual / RNG dice value (in-memory). +- `POST /api/dice/roll_robot` — spawns `pick_and_place.py dice GO ` + as a subprocess. Parses `DICE_NUMBER=N` from its stdout, then calls + `submit_dice(N, "robot")`. The subprocess itself drives the arm via the + orchestrator's HTTP API (see §3). +- `POST /api/move/apply` — apply move (in-memory). +- `POST /api/move/apply_robot` — spawns `pick_and_place.py + `, waits for it to finish, then calls `apply_move`. Subprocess + drives the arm via the orchestrator. + +### Properties + +- `GET /api/properties` +- `GET /api/properties/{pid}` +- `POST /api/properties/{pid}/decide` — accept / skip / build on arrival. +- `POST /api/properties/{pid}/buy` +- `POST /api/properties/{pid}/build` +- `POST /api/properties/{pid}/mortgage` +- `POST /api/properties/{pid}/unmortgage` +- `POST /api/properties/{pid}/sell_building` + +### Money + +- `GET /api/money` — balances snapshot. +- `GET /api/money/{player}` — single balance. + +### Effects (debug) + +- `POST /api/effects/{effect_type}` — apply a Chance / Community Chest + effect manually. + +### WebSocket event streams + +All four read from the in-memory `EventBus`: + +- `WS /api/stream/game` — every event. +- `WS /api/stream/board` — board events only (`move_applied`, + `lap_completed`, `fsm_transition`, `game_started`, `game_won`). +- `WS /api/stream/money` — money / property-payment events. +- `WS /api/stream/properties` — property-state events. + +--- + +## 2. Browser → `movensys_vlm` orchestrator (`:8000`) + +The Ask VLM widget, mic, memory counter, and system-prompt editor all +bypass robopoly's backend and hit the orchestrator directly. + +### VLM inference + +- `POST /api/vlm/infer` — fired by the **Ask** button and by the + VLM-player agent loop. Body includes `client: "robopoly"` so the + orchestrator reads the per-client system prompt slot. + Image source (one of): + - `image_b64: ""` (+ `camera: "none"`) — caller-supplied + image; the orchestrator skips ROS lookup entirely. The agent loop + sends a screenshot of the on-screen GAME BOARD block via this + path (see `vlm_as_player.md` §2.3). + - `camera: "top" | "hand"` — orchestrator grabs the latest ROS RGB + frame and uses that. + - `camera: "none"` (no `image_b64`) — text-only inference. + On the orchestrator side this call triggers: + - `memory_client.recall(prompt)` → TEI embedder `:9020` + Qdrant `:6333` + (search for related past Q&A pairs to inject into the system prompt). + - vLLM `:9000` — actual chat completion. + - `memory_client.store(Q+A)` → TEI `:9020` + Qdrant `:6333` (persist + this turn for future recall). + +### System prompt (per-client) + +The orchestrator keeps a separate prompt slot per `client` key +(`"robopoly"`, `"vlm"`, `"default"`). robopoly's UI always passes +`?client=robopoly` so its textarea is isolated from `/vlm`'s. + +- `GET /api/vlm/system_prompt?client=robopoly` — fired on page load to + populate the textarea. +- `PUT /api/vlm/system_prompt?client=robopoly` — **Save**. +- `DELETE /api/vlm/system_prompt?client=robopoly` — **Reset**. + +### Speech-to-text + +- `POST /api/whisper/transcribe` — multipart upload from the **Rec** + button; the orchestrator forwards audio to the Whisper service + `:9010` and returns `{"text": "..."}`. + +### Memory (vector DB) + +- `GET /api/vlm/memory` — counter polls this every 5 s and right after + every Ask. Returns `{count, enabled}` from Qdrant `:6333`. +- `DELETE /api/vlm/memory` — **Clear memory** button. Drops the Qdrant + collection. + +--- + +## 3. `pick_and_place.py` (subprocess spawned by §1) → orchestrator (`:8000`) + +The dice/move subprocess is an HTTP client of `movensys_vlm`. It does +not touch ROS2 directly. The base URL is hard-coded: + +```python +URL = "http://localhost:8000" +``` + +### Motion + +- `POST /api/move/absolute_cartesian_base` +- `POST /api/move/relative_cartesian_base` +- `POST /api/move/relative_cartesian_tool` +- `POST /api/move/absolute_joint_pose` +- `POST /api/move/joint_absolute` +- `POST /api/move/joint_relative` + +### Services / config + +- `POST /api/services/gripper` +- `GET /api/services/get_eef_pose` +- `POST /api/config/scales` + +### ROS topic snapshots (read-only) + +- `GET /api/topics/yolo_tf` — YOLO-detected piece poses. +- `GET /api/topics/{piece_1|piece_2|dice|…}` — per-object pose snapshot. +- `GET /api/topics/dice_number` — YOLO-detected dice face value + (`/yolo_dice_detector/dice_number`). This is what the subprocess + prints as `DICE_NUMBER=N` so robopoly can pick it up. + +--- + +## 4. Adapters in `adapters/` — wired but health-only + +`adapters/vlm.py`, `adapters/stt.py`, and `adapters/robot.py` all point +at `MOVENSYS_VLM_URL` and expose `.infer()` / `.transcribe()` / +`.gripper()` / etc. methods, but **only their `.health()` is currently +called** (from `/api/*/health` and `/api/modes`). No game-logic code path +calls the methods; the real motion path is the `pick_and_place.py` +subprocess described in §3. + +--- + +## 5. Service topology recap + +``` +browser ─ same-origin ─► robopoly :7999 ─► subprocess (pick_and_place.py) + │ │ + │ └─► orchestrator :8000 (HTTP) + │ + └─► saved_status.yaml on disk + +browser ─ cross-origin ─► orchestrator :8000 + ├─► vLLM :9000 + ├─► Whisper :9010 + ├─► TEI :9020 + ├─► Qdrant :6333 + └─► ROS2 (cameras, IK, gripper) via ros2_node.py +``` + +robopoly never imports `rclpy`; every ROS interaction is mediated by the +orchestrator's HTTP API. diff --git a/movensys_sample/movensys_robopoly/doc/PRD.md b/movensys_sample/movensys_robopoly/doc/PRD.md new file mode 100644 index 0000000..13d7854 --- /dev/null +++ b/movensys_sample/movensys_robopoly/doc/PRD.md @@ -0,0 +1,183 @@ +# Robopoly Voice-Chat UX — PRD + +Scope: frontend-only enhancements to the existing `static/index.html` + +`static/app.js` + `static/app.css` UI. No backend route changes; everything is +built on top of the existing endpoints. + +External services consumed by the frontend: + +| Concern | Host | Endpoint | +| -------------------- | ---------------- | ------------------------------------------ | +| Game state + actions | `localhost:7999` | `/api/*` (REST + `/api/stream/game` WS) | +| VLM inference | `localhost:8000` | `POST /api/vlm/infer` | +| Whisper STT | `localhost:8000` | `POST /api/whisper/transcribe` | +| Robot joint states | `localhost:8000` | WS `/api/stream/joint_states` | +| Robot EEF pose | `localhost:8000` | WS `/api/stream/eef_pose`, `/api/stream/eef_rpy` | + +## 1. Goals + +1. Treat the `Ask VLM → Query & response` panel as a live conversation, not + a single text-in / text-out widget. +2. Replace the on-screen `Rec` button workflow with two hold-to-talk hotkeys: + - **Z** — drives the VLM-as-player loop (rolls dice on your turn, picks a + property action while a decision modal is open). + - **X** — asks the VLM a free-form question about the *current state* of + the robot or the game. +3. Surface the “whose turn is it” banner and every user/assistant utterance + in the chat transcript so the operator can scroll back through the game. + +## 2. Non-goals + +- No backend changes. The existing `/api/whisper/transcribe`, + `/api/vlm/infer`, `/api/vlm/system_prompt`, and game/dice/move/decide + endpoints stay as-is. +- No change to the `System prompt` editor panel — only the + `Query & response` panel is reshaped. +- No automatic text-to-speech output. The bot replies stay text-only. +- No support for arbitrary new property-decision verbs beyond the existing + `buy / build / build_hotel / skip` JSON actions. + +## 3. Feature breakdown + +### 3.1 Chat-style Query & Response (features 2 + 4) + +- Replace `#vlm-response` (single block) with `#vlm-chat`, a scrolling + message column that renders each entry as a speech bubble: + - `me` (right-aligned, accent fill) — user STT transcripts, typed prompts. + - `bot` (left-aligned, panel fill) — VLM/agent replies. + - `sys` (centered, muted) — turn-change announcements, status notes. +- The existing text input + `Ask` button still work and append a `me` bubble + followed by a `bot` bubble (no behavioural change required for them; the + user explicitly said “leave the on-screen mic button alone for now”). +- “It’s ``’s turn” transitions emit a `sys` bubble each time the + active player flips. The original `
` block under + the board stays as-is — the chat panel mirrors the same string. +- Auto-scroll: when a new bubble appends, the chat column scrolls to bottom + unless the user has manually scrolled up >40 px. + +### 3.2 Z-hotkey: voice → immediate action (features 3 + 5) + +Holding **Z** anywhere on the page starts mic capture; releasing **Z** +stops capture and pipes the audio through `/api/whisper/transcribe`. + +- The transcript is appended to the chat as a `me` bubble. +- The transcript is then passed straight to `vlmPlayerAct(transcript)` — + the existing single-agent function that emits the JSON action. +- This is the only consumer; the on-screen `Ask` button is *not* invoked. + +State-dispatch rules (mirrors the existing `maybeAutoTriggerRobotTurn`): + +| FSM state | Behaviour | +| ------------------ | ------------------------------------------------------------------------------------------- | +| `TURN_START` | Treat the voice as the user’s “nudge”. `vlmPlayerAct` will return `roll_and_move`. | +| `AWAIT_DECISION` | Voice describes the buy intent (“buy land”, “skip this one”, “buy hotel”). `vlmPlayerAct` returns `decide`. | +| any other | Append a `sys` bubble “Ignored — wrong phase” and don’t dispatch. | + +Why route through `vlmPlayerAct`: it already knows how to encode the game +state into the prompt, parse the JSON action, and call the existing +end-turn-chained `Roll dice` button or `submitDecision`. We get free reuse +of the auto-liquidation / jail / lap-cap flow. + +Hotkey hygiene: + +- Ignored when focus is inside a text input or textarea (so typing `z` in + the `Ask` box still works). +- Ignored when the page is hidden (`document.hidden`). +- `keydown` auto-repeat is suppressed (we only act on the first press). + +### 3.3 X-hotkey: voice → contextual Q&A (feature 6) + +Holding **X** records, release transcribes via Whisper. The transcript is +appended as a `me` bubble. Then the frontend sends a `/api/vlm/infer` call +with **no JSON-action contract** — this is a free-form answer path, not the +agent path. + +Context bundle attached to the prompt: + +1. The full game-state snapshot (`buildVlmStateSummary(currentState)`). +2. A `robot` block sourced from the localhost:8000 WS streams: + - `latest_joint_states` — last frame from `/api/stream/joint_states`. + - `latest_eef_pose` — last frame from `/api/stream/eef_pose`. + - `latest_eef_rpy` — last frame from `/api/stream/eef_rpy`. +3. The user’s transcribed question. + +The frontend keeps three persistent WebSockets to the VLM server (opened on +boot, auto-reconnect with backoff). Each socket caches the most recent +frame in module-level variables — there is no polling, the X-hotkey just +reads the cached value when assembling the prompt. + +System prompt for X-questions is a separate `client="robopoly_qa"` slot so +it doesn’t fight the agent-mode prompt the existing code installs at boot: + +> You are a helpful, concise game-and-robot assistant for a Movensys-Monopoly +> demo. The user can ask about (a) the current state of the 6-DOF arm — +> joint angles in rad, EEF cartesian pose in m — and (b) the current +> game state, including why certain property decisions were made. Answer +> in plain prose (no JSON, no fences). Use 1–4 sentences. If the user asks +> about a property choice, ground your reasoning in the game state JSON +> (cash, owned properties, tier, distance, etc.) rather than guessing. + +VLM response is rendered as a `bot` bubble. + +### 3.4 Robot status WS subscriptions + +- On page boot, after `setupVlm()`, call `setupRobotStateStream()`. +- Three sockets are opened in parallel: `eef_pose`, `eef_rpy`, `joint_states`. +- Each socket’s `onmessage` parses `{ data, error }` and updates a module- + level cache only when `error == null` and `data != null`. +- `onclose` retries with exponential backoff (1 s → 2 s → 4 s, capped at 10 s). +- The cached values are exposed via `getRobotStateSnapshot()` for the X-key + Q&A prompt assembly. + +### 3.5 Interaction matrix + +| Trigger | Phase | Effect | +| -------------------- | ---------------- | ------------------------------------------- | +| Hold Z, release | `TURN_START` | Rolls + moves the active player. | +| Hold Z, release | `AWAIT_DECISION` | Buys / builds / skips current property. | +| Hold X, release | any | Logs my question + bot answer in the chat. | +| `Ask` button | any | Unchanged from today. | +| On-screen `Rec` mic | any | Unchanged from today. | + +## 4. UI changes (concrete) + +- `index.html`: replace `
` and the small meta line + underneath with a `
` plus a + hidden meta strip (kept for ms/timestamp display under each bot bubble). +- `app.css`: add `.vlm-chat`, `.vlm-msg.me`, `.vlm-msg.bot`, `.vlm-msg.sys`, + bubble shapes (rounded corners with the matching corner squared off), + and a key-hint legend (`Z = act · X = ask`) above the chat column. +- `app.js`: + - new module `chat` (functions `appendChat({role, text, meta})`, + `clearChat()`, `chatScrollToBottom()`). + - `announce()` keeps writing to the notification banner and *also* + appends a `sys` bubble for turn changes. + - new `setupHotkeys()` wiring Z + X with shared recorder helpers + (factored out of the existing `setupVlm` mic logic — same MediaRecorder + bootstrap, same transcribeBlob path). + - new `setupRobotStateStream()` with the three-socket cache. + - new `askVlmAboutState(question)` (X-key handler) that builds the + state+robot context bundle and calls `/api/vlm/infer`. + +## 5. Risks & open questions + +- **Mic permission UX**: Z-key’s first press triggers the browser’s mic + prompt and the user may release the key before granting permission. We + handle this by setting a “armed” flag — if the prompt resolves after the + key has already been released, we discard the recorder instead of + starting an aborted capture. +- **Whisper for one-syllable utterances** (e.g. just “buy”) may return an + empty string. Falling back to a `sys` bubble “Heard nothing — try again” + keeps the loop usable. +- **Agent confusion on buy decisions**: the existing system prompt + forbids prose, so voice transcripts like “buy hotel here” are folded + into the `Context:` line of `vlmPlayerAct` and rely on the model to + pick the matching `decide` JSON. This is the same risk that already + exists for typed user nudges; no change in posture. +- **WS availability**: if localhost:8000 isn’t reachable, the joint / + cartesian context will simply be `null`. The X-key Q&A still works + for game-state questions; the system prompt acknowledges that + robot fields may be missing. +- **Key collisions**: Z/X may collide with future keyboard shortcuts. + The hotkeys are gated on `!isInputFocused() && !e.repeat`; users can + always type in a textbox to escape capture. diff --git a/movensys_sample/movensys_robopoly/doc/game_logic.md b/movensys_sample/movensys_robopoly/doc/game_logic.md new file mode 100644 index 0000000..383e330 --- /dev/null +++ b/movensys_sample/movensys_robopoly/doc/game_logic.md @@ -0,0 +1,270 @@ +# Game logic + +This is the authoritative spec for the robopoly gameplay loop. Where the +spec and the current code disagree, the spec wins — flagged items at the +end are TODOs against the implementation, not contradictions. + +## 1. Players + +1.1. There are 2 players: **user** (red cube) and **robot** (green cube). +1.2. The first turn is the user's. Then turns alternate user → robot → user → … +1.3. All interactions go through buttons on http://localhost:7999/. No + keyboard shortcuts or text commands are required. + +## 2. Money + +2.1. Each player has two figures shown in the UI: + 2.1.1. **Liquid** — cash on hand. Starts at **$1000** for both + players (board JSON `seed_money`). All buys, builds, rents, + taxes, and chance payouts move this number. + 2.1.2. **Assets** — total value of owned tiers (sum of $100 / $200 + / $300 across all owned properties). Recomputed from the + property state, not stored separately. +2.2. Passing GO grants a **start bonus** of $100 (board JSON + `start_bonus`). +2.3. A player is bankrupt when their liquid would go negative and they + have no assets left to sell. On bankruptcy the other player wins. + +## 3. Turn flow (same for both players) + +A turn is driven by a **single chained flow** — roll → move → tile +resolution → end-turn — that runs automatically. The only human +interruption is the property-buy modal (§4.1.1 / §4.1.2); rent, tax, +chance, and auto-liquidation all resolve server-side without +prompting. + +The trigger differs by player (see `vlm_as_player.md` for the full +agent protocol): + +- **User turn**: the user types in the **Ask VLM** textbox; the + frontend routes that message through the VLM agent, which returns + a `roll_and_move` action and the page dispatches the chain. A + fallback *Roll dice* button on the Dice Status card runs the same + chain without the VLM. +- **Robot turn**: when the WebSocket reports `turn=robot, + fsm=TURN_START`, the frontend auto-prompts the VLM with the + current state; the VLM replies with `roll_and_move` and (if the + robot lands on a buyable tile) `decide`. No human input. + +3.1. **Roll the dice** — clicking *Roll dice* runs the YOLO robot + pipeline: + 3.1.1. The arm picks the dice and drops it in the rolling area + (`pick_and_place.py dice GO`). + 3.1.2. The YOLO dice detector publishes the face value on + `/yolo_dice_detector/dice_number`, which the server reads + via the orchestrator's `/api/topics/dice_number`. + 3.1.3. The detected value is submitted as the current player's + dice roll. +3.2. **Move (automatic)** — immediately after the dice value is + submitted, the client drives the player's cube to the computed + destination via `/api/move/apply_robot` (subprocess + `pick_and_place.py `). The on-screen piece + advances at the same moment. +3.3. **Resolve the tile (automatic)** — runs server-side as part of + the move (see §4): + 3.3.1. Property arrival on an unowned tile, or a self-owned tile + with an available upgrade, pops the **Buy modal** (the only + place the chain pauses for human input). + 3.3.2. Rent owed to the opponent is paid automatically. If the + payer cannot cover the rent, the auto-liquidation pathway + in §5.2 runs first; if still short, the payer goes + bankrupt and the opponent wins. + 3.3.3. Tax and chance cash deltas apply automatically; shortfalls + run the same auto-liquidation pathway. +3.4. **End turn (automatic)** — control flips to the other player + as soon as the tile resolution completes (or, when the Buy + modal was shown, as soon as the player's choice is submitted). + There is no *End turn* button. + +## 4. Arrival actions by tile kind + +### 4.1 Property tile (Suwon, Seoul, Jeonju, Daejeon, Gyeongju, Busan, Daegu, Bundang) + +All properties use a **uniform pricing scheme**, independent of which +tile it is: + +| Tier | Cost | Visual on the board | +|-------|---------|---------------------| +| Land | **$100** | 1 colored circle | +| House | **$200** | 2 colored circles | +| Hotel | **$300** | 3 colored circles | + +Red circles = user-owned, green = robot. + +4.1.1. **Unowned tile** — the player picks any tier to buy directly + (free choice). The Buy modal offers *Skip / Buy land / Buy + + house / Buy + hotel*. +4.1.2. **Already owned by the current player** — the modal offers the + upgrade path. The "delta" cost = $100 per tier crossed. + 4.1.2.1. Land → House: pay $100. + 4.1.2.2. House → Hotel: pay $100. + 4.1.2.3. Land → Hotel (skip house): pay $200. + 4.1.2.4. Hotel → nothing more to buy. +4.1.3. **Already owned by the opponent** — the current player **pays + rent** to the opponent and cannot buy. Rent equals the **total + price the opponent has paid in** for the current tier (i.e. + the cumulative buy cost from §4.1): + 4.1.3.1. Opponent owns land → rent = **$100**. + 4.1.3.2. Opponent owns house → rent = **$200**. + 4.1.3.3. Opponent owns hotel → rent = **$300**. +4.1.4. If the rent payment would bankrupt the payer, the game + auto-liquidates assets first (sell buildings, then mortgage + land). If still short, the payer goes bankrupt and the opponent + wins. + +### 4.2 Utility tile (Electric Company) + +4.2.1. Buyable for $100 (land tier only — no houses or hotels on + utilities). +4.2.2. Rent if owned by the opponent = **$100** (flat, same as the + land-tier rent in §4.1.3.1; the dice sum does not factor in). + +### 4.3 Tax tile (Non-Free Parking) + +4.3.1. Landing pays a flat **$100** to the bank. (The tile's + `amount` field in the board JSON is no longer used.) + Auto-liquidation rules in §4.1.4 apply on shortfall. + +### 4.4 Chance tile + +4.4.1. Drawing a card yields either **+$200** or **−$200** (from/to + the bank, equal probability). +4.4.2. No other effects (no jail-card draws, no move-to-tile cards) — + chance in this game is a coin flip of cash only. + +### 4.5 Go-to-jail tile + +4.5.1. The player's piece is teleported to the **IN_JAIL** tile + immediately. Their turn ends. robot arm need pick and place automatically for this. +4.5.2. While in jail, the player must either: + 4.5.2.1. Roll **6** on their next turn (any face counts — single + die) to escape and move 6 spaces from IN_JAIL, **or** + 4.5.2.2. Wait out **2 turns** in jail. On the third turn after + being jailed they are released automatically and roll + normally. +4.5.3. Properties owned during jail still collect rent from the + opponent. +4.5.4. **IN_JAIL is never landed on by dice movement.** Pieces only + end up on IN_JAIL via §4.5.1's GO_TO_JAIL teleport. During + normal dice movement, if the computed destination would be + IN_JAIL the piece advances one more tile (so a player at + Suwon (tile 1) rolling 2 ends on Electric Company (tile 4), + not on IN_JAIL (tile 3)). Passing over IN_JAIL is free as + always — the skip only kicks in when IN_JAIL itself would + be the stop. + +### 4.6 GO tile / IN_JAIL tile (jail visit, not jailed) / blank tiles + +4.6.1. The GO start bonus is granted whether the player **passes** + GO or **lands** on it. IN_JAIL (as a visitor, not jailed) and + blank tiles have no arrival effect. + +## 5. Out-of-money handling + +5.1. **No voluntary selling.** A player cannot choose to sell or + downgrade their own properties during normal play. Owned tiers + can only move down via the auto-liquidation pathway below. +5.2. **Auto-liquidation** kicks in only when the player owes a + payment (rent §4.1.3, utility rent §4.2.2, tax §4.3, chance loss + §4.4) that their current liquid cannot cover. It runs in this + fixed order until the debt is paid: + 5.2.1. Downgrade hotels to houses (refund $100 each, leaves 2 + circles). + 5.2.2. Downgrade houses to land (refund $100 each, leaves 1 + circle). + 5.2.3. Sell remaining land $100. + 5.2.4. If the debt is still not covered after step 5.2.3, the + player is **bankrupt** and the opponent wins per §6.1. +5.3. Auto-liquidation is the only way a player's circle count + decreases. There is just one "sell" event — each tier drop + (hotel→house, house→land, or land→unowned) emits the same + `tier_sold` event, surfaced in the notification banner as + "user sold hotel on Seoul (+$100)" etc. No separate + `mortgage` / `building_sold` distinction. + +## 6. Win condition + +The game ends as soon as **either** of these triggers: + +6.1. **Bankruptcy.** A player can no longer pay what they owe even + after auto-liquidation. The other player wins immediately. +6.2. **Lap limit.** A player has completed **5 full rotations of the + board** (= passed GO 5 times). Tracked per player by counting + `lap_completed` events. As soon as either player's counter + reaches 5, the game ends at the end of that turn. + 6.2.1. Winner = the player with the larger **accumulated + money** = `liquid + assets value`. Assets value uses the + same §2.1.2 sum ($100 × land tiers + $200 × house tiers + + $300 × hotel tiers). + 6.2.2. If both totals are equal the UI banner shows **"draw"**. + 6.2.3. The 5-lap counter and bankruptcy condition are checked + independently — bankruptcy still ends the game + immediately, even before any player reaches 5 laps. + +There is no other end condition — no "first to N dollars" and no +fixed cash threshold. + +## 7. UI surfaces relevant to gameplay + +7.1. **Game board** in the center, with the live cube positions and + ownership circles overlay. +7.2. **Game state card** (top-right): turn, last dice, both positions, + `is_YOLO` toggle, save/load buttons. +7.3. **Dice Status card**: dice face + two buttons — **Roll dice** + (manual fallback that drives the full §3 chain without the VLM) + and **Reset game** (restart the current board from turn 1). The + previous *Apply move* and *End turn* buttons are removed; both + actions are automatic. Under normal play, user turns are + triggered from the Ask VLM textbox and robot turns auto-trigger + — see `vlm_as_player.md`. +7.4. **Events** card: raw event log (debugging). +7.5. **Notification banner** under the board: human-readable + announcements ("It's robot's turn", "robot bought Seoul", "user + paid rent ($100)", "🏆 user wins!"). +7.6. **Money chips** under the board: user / robot liquid balances + (large monospace value, color-flashed on change). +7.7. **Ask VLM** sidebar (left): unrelated to gameplay rules — it's a + debug / interaction surface for the vision-language model. + +## 8. Save / load + +8.1. *SAVE* / *LOAD* buttons in the Game State card persist the + current snapshot to `saved_status.yaml` next to the server. Loading + replaces the entire game state. Memory in the vector DB is not + part of the save. +8.2. **Reset game** also wipes the VLM's Qdrant memory + (`DELETE :8000/api/vlm/memory`) so the next game starts from a + clean slate — past Q&A pairs from the previous game cannot bias + the agent's decisions. + +--- + +## Appendix — implementation notes + +All spec items above are now in code. Key locations: + +- Constants live at the top of `game/rules.py`: + `TIER_PRICE` / `LAND_PRICE` / `HOUSE_PRICE` / `HOTEL_PRICE` / + `TAX_AMOUNT` / `CHANCE_AMOUNT` / `LAPS_TO_WIN`. +- `tier_of(p)` + `_set_tier(p, target)` collapse the + `houses` / `has_hotel` representation into a 0–3 tier integer. +- `assets_value(state, player)` and `total_money(state, player)` are + the spec §2.1.2 / §6.2.1 helpers. The frontend computes the same + totals client-side from `state.properties`. +- `rules.sell_tier()` is the only sell path; `_auto_liquidate` walks + tier 3 → 2 → 1 and emits one `tier_sold` event per drop. +- Chance (§4.4) is a `random.choice([+200, -200])` inline in + `_resolve_once`. `chance.json` is kept as documentation only. +- Jail (§4.5) is set up by `effects.go_to_jail` (`in_jail = True`, + `jail_turns_left = 2`); `submit_dice` enforces "roll 6 to escape + or skip a turn", emits `jail_escaped` / `jail_skipped` / + `jail_released`. +- Lap cap (§6.2) lives in `rules.end_turn` — checks + `lap_count[p] >= LAPS_TO_WIN`, computes totals, sets + `state.winner` (or `None` on a tie). Manager publishes `game_won` + with `{winner, draw, reason: "lap_cap", totals}`. +- The voluntary mortgage / unmortgage / sell_building HTTP routes + are deleted; the manager helpers are also gone. +- Per-tile `price_buy` / `price_building` / `rent_table` / `amount` + fields in `static/assets/boards/board_final.json` are silently + ignored by the Pydantic model; cosmetic deletion only. diff --git a/movensys_sample/movensys_robopoly/doc/running.md b/movensys_sample/movensys_robopoly/doc/running.md new file mode 100644 index 0000000..adf9b09 --- /dev/null +++ b/movensys_sample/movensys_robopoly/doc/running.md @@ -0,0 +1,111 @@ +# Running movensys-monopoly + +## 1. Local development (uvicorn) + +Use a virtual environment so the project's pinned versions don't fight +with system Python packages: + +```bash +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +``` + +### Run + +```bash +python3 -m uvicorn main:app --host 127.0.0.1 --port 7999 +``` + +> Bind to `127.0.0.1` (not `0.0.0.0`). The browser requires a secure +> origin to grant `getUserMedia()` mic access — `http://localhost:*` +> and `http://127.0.0.1:*` qualify; `http://0.0.0.0:*` and +> `http://:*` do not. Open the UI at `http://localhost:7999/`. + +### Stop + +```bash +pkill -f 'uvicorn main:app' +``` + +Verify nothing is left: + +```bash +pgrep -fa 'uvicorn main:app' || echo "no uvicorn running" +``` + +--- + +## 2. Docker Compose + +The compose file bind-mounts the working-copy source (`main.py`, +`router.py`, `pick_and_place.py`, `adapters/`, `game/`, `static/`, +`doc/`, `saved_status.yaml`) into `/app` inside the container. That +means **edits on the host are picked up on the next container restart** +— you do **not** need to `docker compose build` for source changes. Only +rebuild when `requirements.txt` or the `Dockerfile` itself changes. + +### Run (full clean cycle) + +```bash +export MOVENSYS_PNP_DRY_RUN=1 # optional — skips real robot motion +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker +docker compose down +docker compose build # only needed when deps/Dockerfile change +docker compose up -d # foreground; Ctrl-C to stop +``` + +Detached variant (matches `run.sh`): + +```bash +docker compose up -d --force-recreate +``` + +### Logs + +```bash +docker compose logs -f +``` + +### Pick up source changes without rebuild + +```bash +docker compose restart # re-execs uvicorn against the mounted /app +``` + +--- + +## 3. Dry-run mode (no hardware) + +When `MOVENSYS_PNP_DRY_RUN` is truthy, +[`pick_and_place.py`](../pick_and_place.py) short-circuits every +interaction with the manipulator stack: + +- Testing the monopoly server flow on a workstation with no robot attached. +- Iterating on game logic / UI without burning robot cycles between attempts. +- CI / integration tests that exercise the rules engine and FastAPI surface end-to-end. + +### How to enable + +The flag is **off by default** — the compose file exposes it as a pass-through env var (`MOVENSYS_PNP_DRY_RUN=${MOVENSYS_PNP_DRY_RUN:-}`), +so just `export` it in the same shell that runs `docker compose`: + +```bash +export MOVENSYS_PNP_DRY_RUN=1 +cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker +docker compose down +docker compose build +docker compose up -d +``` + +```bash +amixer -c 1 cset numid=4 40 +alsamixer # hardare/driver layer +``` + +Or as a one-off, inline: + +```bash +MOVENSYS_PNP_DRY_RUN=1 docker compose -f docker/docker-compose.yml \ + up -d --force-recreate +``` diff --git a/movensys_sample/movensys_robopoly/doc/vlm_as_player.md b/movensys_sample/movensys_robopoly/doc/vlm_as_player.md new file mode 100644 index 0000000..ac62986 --- /dev/null +++ b/movensys_sample/movensys_robopoly/doc/vlm_as_player.md @@ -0,0 +1,282 @@ +# VLM as the robot player + +The robot side of robopoly is driven by the orchestrator's VLM +(`movensys_vlm`, `:8000`). The browser at `localhost:7999` does **not** +play moves on its own — it converts game state into a prompt, asks the +VLM what to do, parses a single JSON action from the reply, and +dispatches that action through the existing `/api` endpoints. + +No fine-tuning. The model's behavior is shaped entirely by the +**system prompt** the page installs once on first load (see §3). + +--- + +## 1. Scope — what runs where + +- `localhost:7999` (robopoly) — owns the game state, the robot arm + (via `pick_and_place.py`), the property modal, and the VLM-agent + loop in `static/app.js`. Nothing in this loop bypasses the rules + engine; it just clicks the same buttons a human would. +- `localhost:8000` (orchestrator) — hosts `POST /api/vlm/infer` and + the per-client system-prompt slot at `?client=robopoly`. Used as a + black box; the agent loop only depends on those two endpoints. + +## 2. Action protocol + +Every reply from the VLM is parsed as **exactly one** JSON object. +Markdown fences (```json … ```) are tolerated; surrounding prose is +dropped. The parser picks the first balanced `{ … }` block. + +Two actions are recognized: + +### 2.1 `roll_and_move` + +```json +{ "action": "roll_and_move", "player": "user" | "robot" } +``` + +Used when `state.fsm == "TURN_START"`. The frontend dispatches this +by clicking the Roll-dice button, which runs the spec §3 chain in +`game_logic.md`. The **dice step** branches on whose turn it is — +the move + end-turn steps are identical: + +1. **Dice value** — picked by `currentState.turn`: + - `state.turn == "user"` → `POST /api/dice/read_robot`. The human + has already rolled the die by hand; the arm only moves to the + dice scan pose so the camera has a clear view, then YOLO is + read. **No pickup, no drop.** Submitted as `source="manual"`. + - `state.turn == "robot"` → `POST /api/dice/roll_robot`. The arm + physically picks up, lifts, and drops the die, retreats to the + scan pose, then YOLO reads the rolled face. Submitted as + `source="robot"`. +2. `POST /api/move/apply_robot` — arm drives the named player's + cube to the destination tile; tile resolution runs server-side + (rent / tax / chance / auto-liquidation / auto-jail PnP). +3. `POST /api/game/end_turn` — automatic when no Buy modal pops. + +The VLM action itself does **not** change between turns — both +emit `roll_and_move` with the appropriate `player`. The frontend +picks read vs roll based on `currentState.turn`. + +### 2.2 `decide` + +```json +{ "action": "decide", "choice": "skip" | "buy" | "build" | "build_hotel" } +``` + +Used only when `state.decision_pending` is present (the player +landed on a buyable tile and the FSM is `AWAIT_DECISION`). + +- `buy` → buy land for $100 +- `build` → upgrade to house tier (delta = (2 − current_tier) × $100) +- `build_hotel` → upgrade to hotel tier (delta = (3 − current_tier) × $100) +- `skip` → pass on the purchase + +The decision goes through `POST /api/properties/{pid}/decide`, +followed by an automatic `POST /api/game/end_turn`. + +### 2.3 Prompt envelope + +Every call sends three things to the orchestrator's `/api/vlm/infer`: + +- `image_b64:` — a JPEG snapshot of the on-screen **GAME BOARD** block + captured client-side by `captureBoardImage()` in `static/app.js`. + The capture composites the board background PNG with the `#pieces` + SVG overlay (cubes + ownership circles) into a single canvas, then + exports as base64 JPEG at quality 0.8. The orchestrator passes this + string straight to `vlm_client.infer` without consulting any ROS + topic. + - `camera` is set to `"none"` in this case so the orchestrator + doesn't also try to grab a physical-camera frame. + - If the canvas capture fails (tainted canvas, no SVG element, + etc.), the call falls back to `camera: "top"` so the agent still + gets *some* visual grounding from the physical top-down camera. + If that camera isn't publishing either, the orchestrator + degrades to text-only and the JSON state alone drives the + decision. +- `client: "robopoly"` — selects the per-client system-prompt slot + (see §3). +- `prompt:` — the inlined action grammar (so the model can't drift + back to a generic "vision assistant" role), followed by a context + line and a JSON snapshot of game state: + +```text + + +State: +{ + "turn": "robot", + "fsm": "TURN_START", + "turn_number": 4, + "positions": { "user": 5, "robot": 0 }, + "balances": { "user": 1000, "robot": 1000 }, + "lap_count": { "user": 0, "robot": 0 }, + "last_dice": [3, 0], + "last_dice_sum": 3, + "properties_owned": { + "user": [ { "id": "boardfinal:seoul", "tile_index": 2, "tier": 1 } ], + "robot": [] + }, + "decision_pending": null +} +``` + +`decision_pending` is omitted unless the FSM is `AWAIT_DECISION`; in +that case it carries `property_id`, `current_tier`, `max_tier`. + +## 3. System prompt + +On page load the agent installs its system prompt at +`PUT :8000/api/vlm/system_prompt?client=robopoly`. The installed text +has **two parts**: + +1. A fixed agent preamble (`VLM_PLAYER_SYSTEM_PROMPT` in + `static/app.js`) — defines the action grammar, the "you are NOT a + vision assistant" rule, and example replies. +2. The **full authoritative game spec**, pulled from the robopoly + backend at `GET :7999/api/game/rules` (which serves + `doc/game_logic.md` verbatim as `text/markdown`). This way any + edit to the spec doc — jail flow, IN_JAIL skip, auto-liquidation + order, lap cap, etc. — auto-propagates into the agent's + knowledge without code changes. + +If `/api/game/rules` is unreachable (older server, stripped image), +the agent installs just the preamble plus a compact rules summary +inside it; the loop still works, only the verbose spec is missing. + +The install **overwrites** any previous prompt in the slot — a stale +"vision assistant" prompt could otherwise cause the VLM to refuse +with "I cannot physically roll dice for you" instead of emitting an +action. To customize after boot, edit the textarea in the Ask VLM +sidebar (the PUT from the sidebar wins over the auto-install for the +rest of the session). + +## 4. User turn (spec §4) + +4.1. The user physically rolls the die by hand onto the dice scan area. +4.2. The user types into the **Ask VLM** textbox (any message — + "I rolled" or "user just rolled the dice" both work) and presses + Enter / Ask. +4.3. The frontend detects `state.turn == "user"`, + `state.fsm == "TURN_START"` and routes the message through the + agent loop instead of the normal free-form Q&A path. +4.4. The VLM replies with `{"action": "roll_and_move", "player": "user"}`. + The frontend dispatches **the read-only dice path** + (`/api/dice/read_robot`): the arm moves to the dice scan pose so + the gripper is out of the camera's way, YOLO reads the face the + human threw, then the red cube is driven to `(from + dice) % size`. + The arm **never picks up or drops** the die on user turns. +4.5. If the arrival is a buyable tile, the Buy modal pops. The user + clicks **Skip / Buy land / Buy + house / Buy + hotel** in the + UI — *not* through the VLM. The user makes their own buy choices. +4.6. The turn ends automatically (either after auto-resolution, or + immediately after `submitDecision` if the modal was shown). + +> Outside `turn=user, fsm=TURN_START`, the Ask VLM textbox keeps its +> original free-form Q&A behavior — the agent loop does not steal +> non-turn messages. + +## 5. Robot turn (spec §5) + +5.1. When the WebSocket delivers any state event whose result is + `turn=robot, fsm=TURN_START`, the frontend auto-prompts the VLM + ("It's your turn (robot). Roll the dice and move your cube."). +5.2. The VLM replies with `{"action": "roll_and_move", "player": "robot"}`. + The frontend dispatches **the full roll path** (`/api/dice/roll_robot`): + the arm picks up the die at the scan pose, lifts it, drops it, + retreats to clear the camera, then YOLO reads the rolled face. + After that, the green cube is driven to `(from + dice) % size` — + same move chain as the user turn, just with a different cube. +5.3. If the robot lands on a buyable tile, the frontend re-prompts + the VLM with the decision_pending state. The VLM replies with a + `decide` action and the frontend dispatches it. No human input. +5.4. The chain ends the turn automatically. Control returns to the + user; the agent goes quiet until the next robot turn. + +## 6. Idempotency + +The auto-trigger is keyed by `(turn, fsm, turn_number, pending_pid)` +so a burst of WebSocket events during the robot's resolution does +not cause the VLM to be prompted multiple times for the same step. A +single in-flight guard (`vlmPlayerInFlight`) blocks reentrancy from +the user-turn path while the robot turn is mid-action and vice versa. + +## 7. Failure modes + +- VLM unreachable / 5xx → the prompt fails, the agent loop logs and + exits without dispatching. The user can still click the Roll-dice + button directly as a fallback (same chain, no VLM involvement). +- VLM emits unparseable text → `parseVlmAction` returns null; the + loop logs the raw response and exits. Same fallback applies. +- VLM emits a `decide` action while the FSM is `TURN_START` (or vice + versa) → the executor silently rejects mismatched actions because + the underlying buttons are disabled outside their valid FSM. +- **`btn-roll-dice` is disabled when the VLM action arrives** — most + common cause is `turnInFlight` stuck `true` from a prior failed + chain (the `finally` block resets it, so this should only happen if + the fsm landed in `AWAIT_DECISION` and the buy modal was dismissed + without a `submitDecision`). `executeVlmAction` now logs: + `[vlm-player] executeVlmAction: btn-roll-dice is disabled — action + dropped.` followed by `{ fsm, turn, winner, turnInFlight }`. Reset + via the Reset button or by clicking the buy modal. +- **Read mode — YOLO has no `dice_number` to publish** (e.g. the + `yolo_dice_detector` node isn't running, the camera can't see the + thrown die, or the user threw it outside the scan area). The script + polls `/api/topics/dice_number` for up to `_READ_POLL_TIMEOUT_S` (8s + by default), and if YOLO never returns a value it emits + `DICE_NUMBER=1` as a fallback and logs a loud error with the last + HTTP status + detail. The chain proceeds (red cube moves, turn ends, + robot turn auto-starts) so the game doesn't dead-end on a silent + 502 — re-roll if the fallback face was wrong. The error log line + starts with `read mode: YOLO never returned a usable dice_number` + and is the right diagnostic for "robot didn't move after I typed". +- **Roll mode — YOLO can't see the dice after the drop** (gripper + occlusion, camera fault). `_wait_for_rolled_dice_number` times out + after `_DICE_POLL_TIMEOUT_S` (5s) past the post-settle window, then + falls back to the latest cached `dice_number` so the chain still + advances. Log line: `No fresh dice_number after drop — falling back + to latest cached value`. +- **Cube pickup failed (`get_piece_info` + fallback search both + miss)** — previously `pick_and_place.py` silently `return`ed with + exit 0, so `apply_robot` advanced the game state while the physical + cube never moved (board overlay teleported, real cube didn't). The + script now `sys.exit(1)`, the router returns `502 PNP_FAILED`, and + the frontend's catch logs `[roll-chain] aborted with error: ...`. + `turnInFlight` resets cleanly in `finally`; re-trigger the turn + after fixing the YOLO occlusion or repositioning the cube. + +### 7.1 Diagnosing "robot didn't move after I typed" + +The roll-dice chain prints to the browser console at every step. Open +DevTools → Console before clicking Ask, then check which line appears +last — that pinpoints where the chain stopped: + +| Last line you see | Meaning | +|---|---| +| `[vlm-player] dispatching roll_and_move via btn-roll-dice click` | Click was issued. If nothing follows, the click handler bailed before any await — usually `turnInFlight` race. | +| `[vlm-player] executeVlmAction: btn-roll-dice is disabled — action dropped.` | Button gated; the attached state object says why. | +| `[roll-chain] dice step: {...}` (no response) | The dice subprocess hung. Check robopoly stdout for `read mode:` / roll-mode timing lines. | +| `[roll-chain] dice response: {...}` then `unexpected fsm: ...` | Server returned 200 but FSM wasn't `MOVING`. The response object shows what came back. | +| `[roll-chain] apply_robot: {...}` (no response) | Physical cube move is running; wait. | +| `[roll-chain] aborted with error: ...` | A fetch threw (502 from server, network). The error contains the HTTP detail — `DICE_NOT_DETECTED`, `PNP_FAILED`, etc. | + +## 8. Code map + +- `static/app.js` + - `VLM_PLAYER_SYSTEM_PROMPT` — default prompt body (§3). + - `vlmInferRaw`, `parseVlmAction`, `buildVlmStateSummary`, + `executeVlmAction`, `vlmPlayerAct` — agent loop primitives. + - `maybeAutoTriggerRobotTurn` — called from `refreshState` and + the WS `hello` handler (§5). + - `setupVlm.askOnce` — branches into `vlmPlayerAct` when + `turn=user, fsm=TURN_START` (§4). + - `ensureVlmPlayerSystemPrompt` — installs the default prompt on + boot (§3). +- Backend endpoints used by the agent loop: + - `POST /api/dice/read_robot` — user turn, read-only (no pickup). + - `POST /api/dice/roll_robot` — robot turn, full pick + drop + read. + - `POST /api/move/apply_robot`, `POST /api/properties/{pid}/decide`, + `POST /api/game/end_turn` — unchanged. +- The two dice endpoints share `_spawn_dice_subprocess(mode, source)` + in `router.py`. The 4th positional arg to `pick_and_place.py` is + `"read"` (calls `_read_dice_only`) or `"roll"` (full chain). diff --git a/movensys_sample/movensys_robopoly/docker/Dockerfile b/movensys_sample/movensys_robopoly/docker/Dockerfile new file mode 100644 index 0000000..08ab758 --- /dev/null +++ b/movensys_sample/movensys_robopoly/docker/Dockerfile @@ -0,0 +1,52 @@ +ARG ROS_DISTRO=jazzy +FROM ros:${ROS_DISTRO}-ros-base +ARG ROS_DISTRO + +USER root + +# Use https mirrors (matches movensys_vlm) +RUN rm -f /etc/apt/sources.list.d/yarn.list || true +RUN if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then \ + sed -i -E 's|http://(archive\|security)\.ubuntu\.com/ubuntu/|https://\1.ubuntu.com/ubuntu/|g' \ + /etc/apt/sources.list.d/ubuntu.sources; \ + fi + +# M0: Python tooling + cyclonedds RMW (the rest of the movensys stack +# uses RMW_IMPLEMENTATION=rmw_cyclonedds_cpp, so we ship it here too +# to avoid rcl aborting the process when the shared lib is missing). +# M4: python3-opencv + sensor_msgs are needed for ros2_node.py to +# subscribe to /image_{top,hand}/* and encode RGB/depth as JPEG for the +# browser WS proxy (§11.5). sensor_msgs is already in ros-base; we add +# opencv (brings numpy as a dep) and std_msgs is already pulled in. +# YOLO overlay: adapters/ros_image.py spins an rclpy subscriber for +# /yolo_{dice,cube}_detector/debug_image so the board pane can swap to a +# camera view during pick_and_place. ros-base ships the C/C++ ROS +# layer; rclpy + sensor-msgs Python bindings are pulled in explicitly. +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-pip \ + python3-opencv \ + ros-${ROS_DISTRO}-rmw-cyclonedds-cpp \ + ros-${ROS_DISTRO}-rclpy \ + ros-${ROS_DISTRO}-sensor-msgs \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt ./ +RUN if [ "${ROS_DISTRO}" = "jazzy" ]; then \ + pip3 install --no-cache-dir --break-system-packages -r requirements.txt; \ + else \ + pip3 install --no-cache-dir -r requirements.txt; \ + fi + +COPY *.py ./ +COPY adapters/ ./adapters/ +COPY game/ ./game/ +COPY static/ ./static/ +COPY doc/ ./doc/ +COPY docker/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 7999 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/movensys_sample/movensys_robopoly/docker/docker-compose.yml b/movensys_sample/movensys_robopoly/docker/docker-compose.yml new file mode 100644 index 0000000..02cb990 --- /dev/null +++ b/movensys_sample/movensys_robopoly/docker/docker-compose.yml @@ -0,0 +1,66 @@ +# Explicit project name — without it compose derives the project from +# the parent directory ("docker"), which collides with the sibling +# movensys_vlm and movensys_manipulator compose files that also live +# under their own docker/ subdir. With shared project names, +# --remove-orphans from one stack wipes the other stacks' containers. +name: movensys-monopoly + +services: + monopoly: + build: + context: .. + dockerfile: docker/Dockerfile + args: + - ROS_DISTRO=${ROS_DISTRO:-jazzy} + image: movensys_monopoly_image:latest + container_name: movensys_monopoly_container + environment: + # ROS 2 + - ROS_DISTRO=${ROS_DISTRO:-jazzy} + - ROS_DOMAIN_ID=${ROS_DOMAIN_ID:-0} + - RMW_IMPLEMENTATION=${RMW_IMPLEMENTATION:-rmw_fastrtps_cpp} + # FastAPI + - MONOPOLY_PORT=${MONOPOLY_PORT:-7999} + - MONOPOLY_LOG_LEVEL=${MONOPOLY_LOG_LEVEL:-INFO} + - MONOPOLY_DEBUG_ROUTES=${MONOPOLY_DEBUG_ROUTES:-true} + # External FastAPI adapters all route through the movensys_vlm + # orchestrator (PRD 8.x). Empty -> stub mode for STT/LLM/Robot. + # Defaults to http://localhost:8000 since network_mode: host is in + # effect and the orchestrator typically runs on the same host; + # override with `MOVENSYS_VLM_URL=` (empty) to force stub mode. + - MOVENSYS_VLM_URL=${MOVENSYS_VLM_URL-http://localhost:8000} + # Isaac Sim card-spawn publisher (PRD 11.5) + - MONOPOLY_ISAAC_TOPIC_CARD_SPAWN=${MONOPOLY_ISAAC_TOPIC_CARD_SPAWN:-/isaac/card_spawn} + # Dry-run mode for pick_and_place.py: when truthy, all manipulator + # HTTP calls are stubbed (no arm motion, no gripper). Used for + # testing the monopoly server without a robot attached. Off by + # default — set MOVENSYS_PNP_DRY_RUN=1 in the shell to enable. + - MOVENSYS_PNP_DRY_RUN=${MOVENSYS_PNP_DRY_RUN:-} + # Optional persistence (PRD 11.3) + - MONOPOLY_PERSISTENCE_PATH=${MONOPOLY_PERSISTENCE_PATH:-} + # Bind-mount the live source tree over the image's baked /app so the + # container always runs the current working-copy code without a rebuild. + # uvicorn (entrypoint.sh) execs in /app, so editing main.py/router.py/ + # adapters/game/static/doc on the host is picked up on container restart + # (or on uvicorn reload, if you switch entrypoint to --reload). Rebuild + # is only needed when requirements.txt or the Dockerfile itself changes. + volumes: + - ../main.py:/app/main.py + - ../router.py:/app/router.py + - ../pick_and_place.py:/app/pick_and_place.py + - ../adapters:/app/adapters + - ../game:/app/game + - ../static:/app/static + - ../doc:/app/doc + - ../saved_status.yaml:/app/saved_status.yaml + restart: unless-stopped + network_mode: host + # Docker-native readiness probe — surfaces in `docker ps` as + # "healthy"/"starting"/"unhealthy". Wrapper scripts (docker/run.sh) + # wait on this before declaring the container up. + healthcheck: + test: ["CMD-SHELL", "curl -sf http://127.0.0.1:${MONOPOLY_PORT:-7999}/api/health || exit 1"] + interval: 5s + timeout: 3s + start_period: 10s + retries: 6 diff --git a/movensys_sample/movensys_robopoly/docker/entrypoint.sh b/movensys_sample/movensys_robopoly/docker/entrypoint.sh new file mode 100755 index 0000000..4912e2a --- /dev/null +++ b/movensys_sample/movensys_robopoly/docker/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# ROS 2 environment is sourced even at M0 so ros2_node.py can be added +# in M4 without changing this file. +source /opt/ros/${ROS_DISTRO}/setup.bash + +cd /app +# Bind loopback only — getUserMedia() requires a secure context, and +# `http://0.0.0.0:*` / `http://:*` are non-secure origins, so +# the browser would block microphone access. `127.0.0.1` (localhost) +# is treated as secure. network_mode: host in compose makes this port +# reachable on the host's loopback exactly the same way. +exec uvicorn main:app --host 127.0.0.1 --port "${MONOPOLY_PORT:-7999}" diff --git a/movensys_sample/movensys_robopoly/docker/run.sh b/movensys_sample/movensys_robopoly/docker/run.sh new file mode 100755 index 0000000..228090f --- /dev/null +++ b/movensys_sample/movensys_robopoly/docker/run.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Launch movensys-monopoly via docker compose with the same safeguards +# as scripts/ci_local.sh: +# - Refuse to start if host port ${MONOPOLY_PORT:-7999} is held by +# something that is NOT our own container (prevents silent shadowing +# by a stray uvicorn on the host). +# - Warn if a stray /movensys_monopoly node is already in the ROS 2 +# graph (duplicate nodes poison /image_*/DDS subscriptions). +# - Bring the stack up with --remove-orphans so old service containers +# from prior compose runs get swept. +# - Wait on the Docker healthcheck and surface a clear error on +# "unhealthy" instead of a blank prompt. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PORT="${MONOPOLY_PORT:-7999}" +CONTAINER="movensys_monopoly_container" + +step() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +fail() { printf '\033[1;31mFAIL:\033[0m %s\n' "$*"; exit 1; } +warn() { printf '\033[1;33mWARN:\033[0m %s\n' "$*"; } + +step "Pre-flight: port :$PORT" +if ss -ltn "sport = :$PORT" 2>/dev/null | grep -q ":$PORT"; then + if docker ps --format '{{.Names}}' | grep -qx "$CONTAINER"; then + echo " port held by our container — compose will recreate" + else + fail "port :$PORT held by a non-container process (uvicorn/other). Stop it, then retry." + fi +fi + +step "Pre-flight: ROS 2 graph" +if command -v ros2 >/dev/null 2>&1; then + if ros2 node list 2>/dev/null | grep -c '^/movensys_monopoly$' | grep -qv '^0$'; then + count=$(ros2 node list 2>/dev/null | grep -c '^/movensys_monopoly$' || true) + if [ "${count:-0}" -gt 0 ]; then + if docker ps --format '{{.Names}}' | grep -qx "$CONTAINER"; then + echo " $count /movensys_monopoly node(s) visible — expected from the running container" + else + warn "$count stray /movensys_monopoly node(s) in the DDS graph with no container running. They'll be replaced on startup but duplicate nodes can delay subscription binding." + fi + fi + fi +else + echo " ros2 CLI not in PATH — skipping graph check" +fi + +step "compose up -d --remove-orphans" +docker compose -f "$HERE/docker-compose.yml" up -d --remove-orphans + +step "Waiting for healthcheck" +deadline=$((SECONDS + 60)) +while :; do + status=$(docker inspect -f '{{.State.Health.Status}}' "$CONTAINER" 2>/dev/null || echo "unknown") + case "$status" in + healthy) echo " healthy"; break ;; + unhealthy) fail "container is unhealthy — last log lines:\n$(docker logs --tail 30 $CONTAINER 2>&1)" ;; + starting|unknown) + [ $SECONDS -ge $deadline ] && fail "healthcheck did not pass within 60s" + sleep 2 ;; + *) fail "unexpected health status: $status" ;; + esac +done + +step "Summary" +# Make it unambiguous that the container keeps running detached after +# this script exits — the prompt returning is not the container dying. +docker ps --filter name="$CONTAINER" --format ' {{.Names}} {{.Status}}' +echo +echo " UI : http://127.0.0.1:$PORT/ cameras: http://127.0.0.1:$PORT/cameras" +echo " logs : docker logs -f $CONTAINER" +echo " stop : $HERE/stop.sh" diff --git a/movensys_sample/movensys_robopoly/docker/stop.sh b/movensys_sample/movensys_robopoly/docker/stop.sh new file mode 100755 index 0000000..88395cd --- /dev/null +++ b/movensys_sample/movensys_robopoly/docker/stop.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tear down movensys-monopoly and verify nothing was left dangling: +# - compose down --remove-orphans removes every service container +# - confirm port :${MONOPOLY_PORT:-7999} is released (catches a rogue +# host-level uvicorn still holding the port) +# - confirm no /movensys_monopoly node remains in the DDS graph (after +# a short grace period for the announcement TTL) + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PORT="${MONOPOLY_PORT:-7999}" + +step() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33mWARN:\033[0m %s\n' "$*"; } + +step "compose down --remove-orphans" +docker compose -f "$HERE/docker-compose.yml" down --remove-orphans + +step "Post-flight: port :$PORT" +if ss -ltn "sport = :$PORT" 2>/dev/null | grep -q ":$PORT"; then + owner=$(ss -ltnp "sport = :$PORT" 2>/dev/null | tail -n +2 | head -1 || true) + warn "port :$PORT is still held after compose down: $owner" +else + echo " released" +fi + +step "Post-flight: ROS 2 graph" +if command -v ros2 >/dev/null 2>&1; then + # DDS node announcements can linger briefly after shutdown; give them a + # grace window before flagging. + sleep 2 + count=$(ros2 node list 2>/dev/null | grep -c '^/movensys_monopoly$' || true) + if [ "${count:-0}" -gt 0 ]; then + warn "$count /movensys_monopoly node(s) still visible in the DDS graph" + else + echo " no stray nodes" + fi +else + echo " ros2 CLI not in PATH — skipped" +fi diff --git a/movensys_sample/movensys_robopoly/game/__init__.py b/movensys_sample/movensys_robopoly/game/__init__.py new file mode 100644 index 0000000..65a56b2 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/__init__.py @@ -0,0 +1,71 @@ +from game.boards import Board, Tile, load_board +from game.decks import Card, Deck, load_chance, load_community_chest +from game.effects import EffectError, apply_effect +from game.properties import ( + compute_rent, + initial_properties, + property_id, + render_card, +) +from game.rules import ( + MoveResult, + RuleError, + TileResolution, + apply_move, + assets_value, + build, + buy_property, + end_turn, + resolve_tile, + sell_tier, + skip_purchase, + start_game, + submit_dice, + total_money, +) +from game.state import ( + FSM, + DiceSource, + GameState, + Player, + PlayerState, + PropertyState, + RuntimeConfig, +) + +__all__ = [ + "Board", + "Card", + "Deck", + "DiceSource", + "EffectError", + "FSM", + "GameState", + "MoveResult", + "Player", + "PlayerState", + "PropertyState", + "RuleError", + "RuntimeConfig", + "Tile", + "TileResolution", + "apply_effect", + "apply_move", + "assets_value", + "build", + "buy_property", + "compute_rent", + "end_turn", + "initial_properties", + "load_board", + "load_chance", + "load_community_chest", + "property_id", + "render_card", + "resolve_tile", + "sell_tier", + "skip_purchase", + "start_game", + "submit_dice", + "total_money", +] diff --git a/movensys_sample/movensys_robopoly/game/boards.py b/movensys_sample/movensys_robopoly/game/boards.py new file mode 100644 index 0000000..2419513 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/boards.py @@ -0,0 +1,68 @@ +"""Board tile loader (PRD §7, §4.4). + +Reads board JSON from static/assets/boards/board{id}.json and validates +the tile list against pydantic schemas. Loaded once and cached. +""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel + +TileKind = Literal[ + "start", + "property", + "railroad", + "utility", + "tax", + "chance", + "community_chest", + "jail_visit", + "go_to_jail", + "free_parking", + "blank", +] + + +class Tile(BaseModel): + index: int + kind: TileKind + name: str = "" + price_buy: int | None = None + price_building: int | None = None + rent_table: list[int] | None = None + amount: int | None = None + + +class BoardLayout(BaseModel): + shape: str | None = None + columns: int | None = None + rows: int | None = None + + +class Board(BaseModel): + board_id: str + tile_count: int + seed_money: int = 0 + start_bonus: int = 0 + tiles: list[Tile] + layout: BoardLayout | None = None + physical_image: str | None = None + blank_svg: str | None = None + + def tile(self, index: int) -> Tile: + return self.tiles[index % self.tile_count] + + +def _assets_dir() -> Path: + return Path(__file__).resolve().parent.parent / "static" / "assets" / "boards" + + +@lru_cache(maxsize=4) +def load_board(board_id: str) -> Board: + path = _assets_dir() / f"board_{board_id}.json" + data = path.read_text() + return Board.model_validate_json(data) diff --git a/movensys_sample/movensys_robopoly/game/chance.json b/movensys_sample/movensys_robopoly/game/chance.json new file mode 100644 index 0000000..e21e368 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/chance.json @@ -0,0 +1,8 @@ +{ + "deck": "chance", + "_note": "Spec §4.4: chance is a random ±$200 coin flip handled inline in rules.py:_resolve_once. This file is kept so the deck loader still finds a valid file, but its cards are no longer drawn during play.", + "cards": [ + { "id": "windfall", "text": "Bank pays you $200.", "effect": { "type": "collect", "amount": 200 } }, + { "id": "unexpected_tax", "text": "Pay $200 to the bank.", "effect": { "type": "pay", "amount": 200, "to": "bank" } } + ] +} diff --git a/movensys_sample/movensys_robopoly/game/community_chest.json b/movensys_sample/movensys_robopoly/game/community_chest.json new file mode 100644 index 0000000..2b207ae --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/community_chest.json @@ -0,0 +1,21 @@ +{ + "deck": "community_chest", + "cards": [ + { "id": "advance_to_go_cc", "text": "Advance to GO. Collect $200.", "effect": { "type": "move_to_tile", "tile_index": 0, "collect_on_pass": true } }, + { "id": "bank_error", "text": "Bank error in your favor. Collect $200.", "effect": { "type": "collect", "amount": 200 } }, + { "id": "doctor_fee", "text": "Doctor's fee. Pay $50.", "effect": { "type": "pay", "amount": 50, "to": "bank" } }, + { "id": "sale_of_stock", "text": "From sale of stock you get $50.", "effect": { "type": "collect", "amount": 50 } }, + { "id": "jail_free_cc", "text": "Get Out of Jail Free. Keep this card until needed.", "effect": { "type": "grant_jail_free_card" } }, + { "id": "go_to_jail_cc", "text": "Go directly to Jail. Do not pass GO.", "effect": { "type": "go_to_jail" } }, + { "id": "holiday_fund", "text": "Holiday fund matures. Collect $100.", "effect": { "type": "collect", "amount": 100 } }, + { "id": "income_tax_refund", "text": "Income tax refund. Collect $20.", "effect": { "type": "collect", "amount": 20 } }, + { "id": "birthday", "text": "It is your birthday. Collect $10 from each player.", "effect": { "type": "collect_from_each_player", "amount": 10 } }, + { "id": "life_insurance", "text": "Life insurance matures. Collect $100.", "effect": { "type": "collect", "amount": 100 } }, + { "id": "hospital", "text": "Hospital fees. Pay $50.", "effect": { "type": "pay", "amount": 50, "to": "bank" } }, + { "id": "school_fees", "text": "School fees. Pay $50.", "effect": { "type": "pay", "amount": 50, "to": "bank" } }, + { "id": "consultancy_fee", "text": "Receive $25 consultancy fee.", "effect": { "type": "collect", "amount": 25 } }, + { "id": "street_repairs", "text": "Street repairs. $40 per house, $115 per hotel.", "effect": { "type": "pay_per_building", "per_house": 40, "per_hotel": 115 } }, + { "id": "beauty_contest", "text": "You have won second prize in a beauty contest. Collect $10.", "effect": { "type": "collect", "amount": 10 } }, + { "id": "inheritance", "text": "You inherit $100.", "effect": { "type": "collect", "amount": 100 } } + ] +} diff --git a/movensys_sample/movensys_robopoly/game/decks.py b/movensys_sample/movensys_robopoly/game/decks.py new file mode 100644 index 0000000..014b3dd --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/decks.py @@ -0,0 +1,71 @@ +"""Chance and Community Chest deck loader (PRD §7.3.6). + +Each deck is a JSON file next to this module. Cards are drawn from the +top; after resolution the card moves to the bottom (Hasbro rule) — except +"Get Out of Jail Free" which is held by the player until consumed. +""" + +from __future__ import annotations + +import json +import random +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_DATA_DIR = Path(__file__).resolve().parent + + +@dataclass +class Card: + id: str + deck: str + text: str + effect: dict[str, Any] + + @classmethod + def from_json(cls, obj: dict, deck: str) -> "Card": + return cls(id=obj["id"], deck=deck, text=obj["text"], effect=obj["effect"]) + + +@dataclass +class Deck: + name: str + cards: list[Card] + _rng: random.Random = field(default_factory=random.Random) + + def shuffle(self, seed: int | None = None) -> None: + if seed is not None: + self._rng = random.Random(seed) + self._rng.shuffle(self.cards) + + def draw(self) -> Card: + """Pop top card. Caller rotates it back with `return_to_bottom` unless + held (e.g. Get Out of Jail Free).""" + if not self.cards: + raise RuntimeError(f"deck {self.name!r} is empty") + return self.cards.pop(0) + + def return_to_bottom(self, card: Card) -> None: + self.cards.append(card) + + def __len__(self) -> int: + return len(self.cards) + + +def _load(name: str, filename: str, seed: int | None = None) -> Deck: + path = _DATA_DIR / filename + data = json.loads(path.read_text()) + cards = [Card.from_json(c, deck=name) for c in data["cards"]] + deck = Deck(name=name, cards=cards) + if seed is not None: + deck.shuffle(seed) + return deck + + +def load_chance(seed: int | None = None) -> Deck: + return _load("chance", "chance.json", seed) + + +def load_community_chest(seed: int | None = None) -> Deck: + return _load("community_chest", "community_chest.json", seed) diff --git a/movensys_sample/movensys_robopoly/game/effects.py b/movensys_sample/movensys_robopoly/game/effects.py new file mode 100644 index 0000000..1335c04 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/effects.py @@ -0,0 +1,214 @@ +"""Chance / Community Chest effect dispatcher (PRD §7.3.6, §5.5). + +Effects are structured JSON (see game/chance.json, game/community_chest.json) +so the rules engine applies them deterministically — no LLM interpretation. + +Each `apply_*` returns a small result dict that callers use as the +payload for WS events. +""" + +from __future__ import annotations + +from typing import Any + +from game.boards import Board +from game.state import GameState, Player + + +class EffectError(ValueError): + def __init__(self, code: str, message: str) -> None: + super().__init__(code, message) + self.code = code + self.message = message + + def __str__(self) -> str: + return self.message + + +# ---- individual effects ---------------------------------------------------- + + +def collect(state: GameState, player: Player, amount: int) -> dict[str, Any]: + state.players[player].balance += amount + return {"kind": "collect", "player": player, "amount": amount, + "balance": state.players[player].balance} + + +def pay(state: GameState, player: Player, amount: int, *, to: str) -> dict[str, Any]: + """`to` is 'bank' or 'opponent'. Negative balance is allowed here — + rules.resolve_bankruptcy owns the bankruptcy trigger.""" + payee = None + if to == "opponent": + payee = state.other(player) + state.players[payee].balance += amount + state.players[player].balance -= amount + return {"kind": "pay", "player": player, "amount": amount, "to": to, + "payee": payee, + "balance": state.players[player].balance} + + +def pay_each_player(state: GameState, player: Player, amount: int) -> dict[str, Any]: + """1v1 game: pay the opponent `amount`.""" + return pay(state, player, amount, to="opponent") + + +def collect_from_each_player(state: GameState, player: Player, amount: int) -> dict[str, Any]: + opp = state.other(player) + state.players[opp].balance -= amount + state.players[player].balance += amount + return {"kind": "collect_from_each_player", "player": player, "amount": amount, + "balance": state.players[player].balance} + + +def pay_per_building( + state: GameState, board: Board, player: Player, *, per_house: int, per_hotel: int +) -> dict[str, Any]: + houses = 0 + hotels = 0 + for p in state.properties.values(): + if p.owner != player: + continue + if p.has_hotel: + hotels += 1 + houses += p.houses + total = houses * per_house + hotels * per_hotel + state.players[player].balance -= total + return {"kind": "pay_per_building", "player": player, "houses": houses, + "hotels": hotels, "amount": total, + "balance": state.players[player].balance} + + +def move_to_tile( + state: GameState, + board: Board, + player: Player, + *, + tile_index: int, + collect_on_pass: bool, + start_bonus: int, +) -> dict[str, Any]: + current = state.positions[player] + # Chance/CC "Advance to X" cards always move forward; we passed START + # only when the destination index is strictly less than where we were. + wrapped = tile_index < current + state.positions[player] = tile_index + collected = 0 + if wrapped and collect_on_pass and start_bonus > 0: + collected = start_bonus + state.players[player].balance += collected + return {"kind": "move_to_tile", "player": player, "from": current, + "to": tile_index, "passed_start": wrapped, "collected": collected} + + +def move_relative( + state: GameState, board: Board, player: Player, *, delta: int +) -> dict[str, Any]: + size = board.tile_count + current = state.positions[player] + nxt = (current + delta) % size + state.positions[player] = nxt + return {"kind": "move_relative", "player": player, "from": current, "to": nxt, + "delta": delta} + + +def move_to_nearest( + state: GameState, board: Board, player: Player, *, kind: str, + collect_on_pass: bool, start_bonus: int, +) -> dict[str, Any]: + size = board.tile_count + current = state.positions[player] + target = None + for step in range(1, size + 1): + cand = (current + step) % size + if board.tiles[cand].kind == kind: + target = cand + break + if target is None: + return {"kind": "move_to_nearest", "player": player, "found": False} + return { + **move_to_tile( + state, board, player, + tile_index=target, collect_on_pass=collect_on_pass, start_bonus=start_bonus, + ), + "kind": "move_to_nearest", + "target_kind": kind, + } + + +def grant_jail_free_card(state: GameState, player: Player) -> dict[str, Any]: + state.players[player].has_jail_free_card = True + return {"kind": "grant_jail_free_card", "player": player} + + +def go_to_jail(state: GameState, board: Board, player: Player) -> dict[str, Any]: + """Teleport to the jail_visit tile and set the in_jail flag (spec §4.5). + + `jail_turns_left = 2` means the player will be stuck for the next two + of their own turns; on the third they auto-release. Rolling a 6 on any + of those turns also escapes immediately. + """ + jail_visit = next((t.index for t in board.tiles if t.kind == "jail_visit"), None) + if jail_visit is None: + return {"kind": "go_to_jail", "player": player, "found": False} + current = state.positions[player] + state.positions[player] = jail_visit + state.players[player].in_jail = True + state.players[player].jail_turns_left = 2 + return {"kind": "go_to_jail", "player": player, "from": current, "to": jail_visit, + "jail_fsm": True} + + +# ---- dispatcher ------------------------------------------------------------ + + +def _require(effect: dict[str, Any], key: str) -> Any: + if key not in effect: + raise EffectError("BAD_REQUEST", f"effect missing required field: {key!r}") + return effect[key] + + +def apply_effect( + state: GameState, board: Board, player: Player, effect: dict[str, Any], +) -> dict[str, Any]: + etype = effect.get("type") + bonus = board.start_bonus + try: + if etype == "collect": + return collect(state, player, int(_require(effect, "amount"))) + if etype == "pay": + return pay(state, player, int(_require(effect, "amount")), + to=effect.get("to", "bank")) + if etype == "pay_each_player": + return pay_each_player(state, player, int(_require(effect, "amount"))) + if etype == "collect_from_each_player": + return collect_from_each_player(state, player, int(_require(effect, "amount"))) + if etype == "pay_per_building": + return pay_per_building( + state, board, player, + per_house=int(_require(effect, "per_house")), + per_hotel=int(_require(effect, "per_hotel")), + ) + if etype == "move_to_tile": + return move_to_tile( + state, board, player, + tile_index=int(_require(effect, "tile_index")), + collect_on_pass=bool(effect.get("collect_on_pass", False)), + start_bonus=bonus, + ) + if etype == "move_relative": + return move_relative(state, board, player, + delta=int(_require(effect, "delta"))) + if etype == "move_to_nearest": + return move_to_nearest( + state, board, player, + kind=_require(effect, "kind"), + collect_on_pass=True, + start_bonus=bonus, + ) + if etype == "grant_jail_free_card": + return grant_jail_free_card(state, player) + if etype == "go_to_jail": + return go_to_jail(state, board, player) + except (TypeError, ValueError) as exc: + raise EffectError("BAD_REQUEST", f"invalid arg in {etype!r}: {exc}") from exc + raise EffectError("BAD_REQUEST", f"unknown effect type: {etype!r}") diff --git a/movensys_sample/movensys_robopoly/game/events.py b/movensys_sample/movensys_robopoly/game/events.py new file mode 100644 index 0000000..3525539 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/events.py @@ -0,0 +1,54 @@ +"""WS event bus (PRD §4.6). + +Tiny in-process pub/sub. Each subscriber gets its own bounded queue so a +slow client backs up only its own stream. Publishers never await the +network — they call `publish_nowait` from inside FSM handlers. +""" + +from __future__ import annotations + +import asyncio +import time +import uuid +from typing import Any + +_QUEUE_MAXSIZE = 256 + + +def make_envelope(event_type: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + return { + "event_id": uuid.uuid4().hex, + "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + "Z", + "type": event_type, + "payload": payload or {}, + } + + +class EventBus: + def __init__(self) -> None: + self._subscribers: list[asyncio.Queue] = [] + + def subscribe(self) -> asyncio.Queue: + q: asyncio.Queue = asyncio.Queue(maxsize=_QUEUE_MAXSIZE) + self._subscribers.append(q) + return q + + def unsubscribe(self, q: asyncio.Queue) -> None: + try: + self._subscribers.remove(q) + except ValueError: + pass + + def publish_nowait(self, event_type: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + envelope = make_envelope(event_type, payload) + for q in list(self._subscribers): + try: + q.put_nowait(envelope) + except asyncio.QueueFull: + # Drop for slow subscribers rather than blocking the engine. + pass + return envelope + + @property + def subscriber_count(self) -> int: + return len(self._subscribers) diff --git a/movensys_sample/movensys_robopoly/game/manager.py b/movensys_sample/movensys_robopoly/game/manager.py new file mode 100644 index 0000000..c373239 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/manager.py @@ -0,0 +1,292 @@ +"""Game manager — serialises FSM transitions (PRD §11.2). + +Wraps GameState with an asyncio.Lock so concurrent REST calls don't race +on state mutations. Emits structured WS events around every transition. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +from typing import Any + +from game import rules +from game.boards import load_board +from game.decks import Deck, load_chance, load_community_chest +from game.effects import apply_effect +from game.events import EventBus +from game.properties import all_cards as _all_cards +from game.state import FSM, GameState, Player, RuntimeConfig + +log = logging.getLogger("monopoly.game") + + +class GameManager: + def __init__(self, bus: EventBus | None = None) -> None: + self.state: GameState = GameState() + self.bus: EventBus = bus or EventBus() + self._lock: asyncio.Lock = asyncio.Lock() + self._chance: Deck | None = None + self._cc: Deck | None = None + + # ---- public API -------------------------------------------------------- + + async def snapshot(self) -> dict[str, Any]: + async with self._lock: + return self.state.model_dump() + + async def start_game(self, board_id: str) -> dict[str, Any]: + async with self._lock: + prev = self.state.fsm + rules.start_game(self.state, board_id) + self._chance = load_chance() + self._cc = load_community_chest() + self._chance.shuffle() + self._cc.shuffle() + self.bus.publish_nowait("game_started", {"board_id": board_id}) + self._emit_transition(prev, self.state.fsm, trigger="start_game") + log.info("game_started", extra={"board_id": board_id}) + return {"fsm": self.state.fsm.value, "turn": self.state.turn} + + async def update_config(self, patch: dict[str, Any]) -> dict[str, Any]: + async with self._lock: + merged = self.state.config.model_dump() + merged.update(patch) + self.state.config = RuntimeConfig.model_validate(merged) + return self.state.config.model_dump() + + async def request_dice(self) -> dict[str, Any]: + """Server-side dice acquisition based on dice_source config (PRD §5.2).""" + async with self._lock: + source = self.state.config.dice_source + if source == "manual": + # /dice/submit must follow — just echo the source. + return {"source": "manual"} + if source == "rng": + value = random.randint(1, 6) + rules.submit_dice(self.state, value) + self.bus.publish_nowait("dice_submitted", {"value": value, "source": "rng"}) + self._emit_transition(FSM.TURN_START, self.state.fsm, trigger="dice_submitted") + return {"source": "rng", "value": value} + # robot source — adapter plumbing lands in M5; fall back to rng for now. + value = random.randint(1, 6) + rules.submit_dice(self.state, value) + self.bus.publish_nowait("dice_submitted", {"value": value, "source": "rng_fallback"}) + self._emit_transition(FSM.TURN_START, self.state.fsm, trigger="dice_submitted") + return {"source": "rng_fallback", "value": value} + + async def submit_dice(self, value: int | tuple[int, int], source: str) -> dict[str, Any]: + async with self._lock: + prev = self.state.fsm + jail = rules.submit_dice(self.state, value) + self.bus.publish_nowait( + "dice_submitted", + {"value": value, "source": source, "sum": self.state.last_dice_sum}, + ) + if jail is not None: + # Spec §4.5: jail_escaped / jail_skipped / jail_released + self.bus.publish_nowait(jail["kind"], jail) + self._emit_transition(prev, self.state.fsm, trigger="dice_submitted") + return { + "fsm": self.state.fsm.value, + "dice": list(self.state.last_dice) if self.state.last_dice else None, + "sum": self.state.last_dice_sum, + } + + async def apply_move(self, player: Player, from_tile: int, to_tile: int) -> dict[str, Any]: + async with self._lock: + prev = self.state.fsm + result = rules.apply_move(self.state, player, from_tile, to_tile) + self.bus.publish_nowait( + "move_applied", + { + "player": result.player, + "from_tile": result.from_tile, + "to_tile": result.to_tile, + "dice_sum": result.dice_sum, + "wrapped": result.wrapped, + }, + ) + if result.wrapped: + self.bus.publish_nowait("lap_completed", {"player": result.player}) + if result.start_bonus_collected > 0: + self.bus.publish_nowait("start_bonus", { + "player": result.player, + "amount": result.start_bonus_collected, + "balance": self.state.players[result.player].balance, + }) + self._emit_transition(prev, self.state.fsm, trigger="move_applied") + + resolved = [] + if result.winner is None and self.state.fsm == FSM.RESOLVE_TILE: + board = load_board(self.state.board_id) + tile_results = rules.resolve_tile( + self.state, board, player, + chance_deck=self._chance, cc_deck=self._cc, + ) + for r in tile_results: + # Narrate auto-liquidation so the UI can render each sale + # before the final rent/tax settlement. + for step in r.payload.get("liquidation", []) or []: + self.bus.publish_nowait(step.get("kind", "liquidation_step"), step) + self.bus.publish_nowait(f"tile_{r.kind}", { + "tile_index": r.tile_index, + "needs_decision": r.needs_decision, + **r.payload, + }) + resolved.append({ + "kind": r.kind, + "tile_index": r.tile_index, + "needs_decision": r.needs_decision, + "payload": r.payload, + }) + if self.state.fsm == FSM.GAME_OVER and self.state.winner: + self.bus.publish_nowait("game_won", {"winner": self.state.winner}) + + if result.winner is not None: + self.bus.publish_nowait("game_won", {"winner": result.winner}) + log.info("game_won", extra={"winner": result.winner}) + + return { + "fsm": self.state.fsm.value, + "resolved": { + "player": result.player, + "to_tile": result.to_tile, + "wrapped": result.wrapped, + "winner": result.winner or self.state.winner, + "tiles": resolved, + }, + } + + # ---- property transactions --------------------------------------------- + + async def buy_property(self, player: Player, pid: str) -> dict[str, Any]: + async with self._lock: + board = load_board(self.state.board_id) + prev = self.state.fsm + res = rules.buy_property(self.state, board, player, pid) + self.bus.publish_nowait("property_bought", res) + self._emit_transition(prev, self.state.fsm, trigger="buy_property") + return res + + async def skip_purchase(self) -> dict[str, Any]: + async with self._lock: + prev = self.state.fsm + rules.skip_purchase(self.state) + self.bus.publish_nowait("purchase_skipped", {}) + self._emit_transition(prev, self.state.fsm, trigger="skip_purchase") + return {"fsm": self.state.fsm.value} + + async def decide_property( + self, player: Player, pid: str, action: str, house_count: int = 0, + ) -> dict[str, Any]: + async with self._lock: + board = load_board(self.state.board_id) + prev = self.state.fsm + if action == "skip": + rules.skip_purchase(self.state) + self.bus.publish_nowait("purchase_skipped", {"property_id": pid}) + self._emit_transition(prev, self.state.fsm, trigger="decide_skip") + return {"action": "skip", "fsm": self.state.fsm.value} + if action == "buy": + res = rules.buy_property(self.state, board, player, pid) + self.bus.publish_nowait("property_bought", res) + self._emit_transition(prev, self.state.fsm, trigger="decide_buy") + return {"action": "buy", **res, "fsm": self.state.fsm.value} + if action in ("build", "build_hotel"): + # On a first arrival the tile is still unowned and we need to + # buy land before building. On a revisit (spec §4.1.2) the + # tile is already owned by `player` and AWAIT_DECISION was + # raised by the upgrade path — skip buy_property in that case. + already_owned = self.state.properties[pid].owner == player + if not already_owned: + bought = rules.buy_property(self.state, board, player, pid) + self.bus.publish_nowait("property_bought", bought) + else: + # Manually drop AWAIT_DECISION → RESOLVE_TILE so build's + # FSM precondition (no AWAIT_DECISION) passes cleanly. + self.state.fsm = self.state.fsm # no-op, build() doesn't check fsm + try: + if action == "build_hotel": + built = rules.build(self.state, board, player, pid, hotel=True) + else: + built = rules.build(self.state, board, player, pid) + self.bus.publish_nowait("property_built", built) + except rules.RuleError as exc: + self.bus.publish_nowait( + "property_build_rejected", + {"property_id": pid, "code": exc.code, "message": str(exc)}, + ) + # Always transition out of AWAIT_DECISION when leaving the modal. + if self.state.fsm == FSM.AWAIT_DECISION: + self.state.fsm = FSM.RESOLVE_TILE + self._emit_transition(prev, self.state.fsm, trigger=f"decide_{action}") + return {"action": action, "fsm": self.state.fsm.value} + raise rules.RuleError("BAD_REQUEST", f"unknown action: {action!r}") + + async def build( + self, player: Player, pid: str, *, houses: int = 1, hotel: bool = False, + ) -> dict[str, Any]: + async with self._lock: + board = load_board(self.state.board_id) + res = rules.build(self.state, board, player, pid, houses=houses, hotel=hotel) + self.bus.publish_nowait("property_built", res) + return res + + # Spec §5.1: no voluntary selling. The mortgage / unmortgage / + # sell_building manager methods are removed; their HTTP routes + # are dropped from router.py. Auto-liquidation calls + # rules.sell_tier directly and emits `tier_sold`. + + async def apply_card_effect(self, player: Player, effect: dict[str, Any]) -> dict[str, Any]: + """Used by /api/effects/* endpoints (PRD §5.5). These mirror what a + Chance/CC card effect would do, callable from the UI for testing.""" + async with self._lock: + board = load_board(self.state.board_id) + res = apply_effect(self.state, board, player, effect) + self.bus.publish_nowait("effect_applied", res) + return res + + def list_properties(self) -> list[dict[str, Any]]: + return list(_all_cards(self.state, load_board(self.state.board_id))) + + def money_snapshot(self) -> dict[str, int]: + return {pid: p.balance for pid, p in self.state.players.items()} + + async def replace_state(self, raw: dict[str, Any]) -> dict[str, Any]: + async with self._lock: + self.state = GameState.model_validate(raw) + if self._chance is None: + self._chance = load_chance() + self._chance.shuffle() + if self._cc is None: + self._cc = load_community_chest() + self._cc.shuffle() + self.bus.publish_nowait("state_loaded", {"board_id": self.state.board_id, + "turn": self.state.turn, + "fsm": self.state.fsm.value}) + return self.state.model_dump() + + async def end_turn(self) -> dict[str, Any]: + async with self._lock: + prev = self.state.fsm + end_payload = rules.end_turn(self.state) + self._emit_transition(prev, self.state.fsm, trigger="end_turn") + if end_payload is not None: + # Spec §6.2 — game ended on the lap cap. + self.bus.publish_nowait("game_won", end_payload) + return {"fsm": self.state.fsm.value, "turn": self.state.turn} + + def winner(self) -> Player | None: + return self.state.winner + + # ---- helpers ----------------------------------------------------------- + + def _emit_transition(self, prev: FSM, nxt: FSM, trigger: str) -> None: + if prev == nxt: + return + self.bus.publish_nowait( + "fsm_transition", + {"from": prev.value, "to": nxt.value, "trigger": trigger}, + ) diff --git a/movensys_sample/movensys_robopoly/game/properties.py b/movensys_sample/movensys_robopoly/game/properties.py new file mode 100644 index 0000000..1b911ba --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/properties.py @@ -0,0 +1,128 @@ +"""Rent computation and property helpers (PRD §7.3). + +Static tile data lives in `game/boards.Board.tiles`. Dynamic ownership +(owner, houses, mortgaged) lives in `GameState.properties`. This module +stitches them together. +""" + +from __future__ import annotations + +import re +from typing import Iterable + +from game.boards import Board, Tile +from game.state import GameState, Player, PropertyState + +_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +def slug(name: str) -> str: + return _SLUG_RE.sub("_", name.lower()).strip("_") + + +def property_id(board_id: str, tile_name: str) -> str: + return f"board{board_id}:{slug(tile_name)}" + + +def initial_properties(board: Board) -> dict[str, PropertyState]: + """Build the dynamic map keyed by property_id for a fresh game.""" + out: dict[str, PropertyState] = {} + for tile in board.tiles: + if tile.kind not in ("property", "railroad", "utility"): + continue + pid = property_id(board.board_id, tile.name) + out[pid] = PropertyState(id=pid, tile_index=tile.index) + return out + + +# ---- ownership queries ----------------------------------------------------- + + +def owned_by(state: GameState, owner: Player) -> list[PropertyState]: + return [p for p in state.properties.values() if p.owner == owner] + + +def railroads_owned(state: GameState, board: Board, owner: Player) -> int: + return sum( + 1 + for tile in board.tiles + if tile.kind == "railroad" + and (p := state.properties.get(property_id(board.board_id, tile.name))) is not None + and p.owner == owner + ) + + +def utilities_owned(state: GameState, board: Board, owner: Player) -> int: + return sum( + 1 + for tile in board.tiles + if tile.kind == "utility" + and (p := state.properties.get(property_id(board.board_id, tile.name))) is not None + and p.owner == owner + ) + + +# ---- rent computation ------------------------------------------------------ + + +# Spec §4.1.3, §4.2.2: rent = the cumulative buy cost for the opponent's +# current tier ($100 land, $200 house, $300 hotel). Utilities cap at land. +_RENT_BY_TIER = {1: 150, 2: 300, 3: 450} + + +def compute_rent( + state: GameState, board: Board, tile_index: int, dice_sum: int | None +) -> int: + """Rent to pay when landing on `tile_index` owned by someone. + + Returns 0 when unowned, self-owned, or when the tile isn't a buyable + kind. `dice_sum` is accepted for back-compat but no longer used + (utility rent is now flat $100, same as land). + """ + tile = board.tiles[tile_index] + if tile.kind not in ("property", "railroad", "utility"): + return 0 + pid = property_id(board.board_id, tile.name) + p = state.properties.get(pid) + if p is None or p.owner is None or p.mortgaged: + return 0 + if p.has_hotel: + tier = 3 + elif p.houses > 0: + tier = 2 + else: + tier = 1 + return _RENT_BY_TIER[tier] + + +# ---- projection to PRD §4.4 PropertyCard shape ---------------------------- + + +def render_card(tile: Tile, p: PropertyState, board_id: str) -> dict: + """Merge static + dynamic view for REST responses (PRD §4.4). + + The per-tile `price_buy` / `price_building` / `rent_table` fields + are no longer relevant under the flat-economy spec; the UI uses + the global $100/$200/$300 ladder. + """ + return { + "id": p.id, + "tile_index": tile.index, + "name": tile.name, + "kind": tile.kind, + "owner": p.owner, + "houses": p.houses, + "has_hotel": p.has_hotel, + "mortgaged": p.mortgaged, + } + + +def all_cards(state: GameState, board: Board) -> Iterable[dict]: + for tile in board.tiles: + if tile.kind not in ("property", "railroad", "utility"): + continue + pid = property_id(board.board_id, tile.name) + p = state.properties.get(pid) + if p is None: + continue + yield render_card(tile, p, board.board_id) diff --git a/movensys_sample/movensys_robopoly/game/rules.py b/movensys_sample/movensys_robopoly/game/rules.py new file mode 100644 index 0000000..ff6b9ac --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/rules.py @@ -0,0 +1,643 @@ +"""Game rule engine (PRD §7). + +Pure functions only — no async, no network, no globals. GameManager +wraps them with a lock and pub/sub. + +- start_game / submit_dice / apply_move / end_turn (M1) +- resolve_tile (M2) routing by tile kind +- buy_property / build / mortgage / unmortgage / sell_building (M2) +- resolve_bankruptcy (M2) liquidate then GAME_OVER +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from game import properties as props_mod +from game.boards import Board, load_board +from game.decks import Deck +from game.effects import apply_effect +from game.properties import ( + initial_properties, + property_id, + render_card, +) +from game.state import FSM, GameState, Player, PlayerState + + +# Game-wide flat economy (spec §2, §4.x). +TIER_PRICE = 100 # cost per tier crossed (and refund on sell) +LAND_PRICE = 1 * TIER_PRICE # tier 1 → $100 cumulative +HOUSE_PRICE = 2 * TIER_PRICE # tier 2 → $200 cumulative +HOTEL_PRICE = 3 * TIER_PRICE # tier 3 → $300 cumulative +TAX_AMOUNT = 100 # flat tax for Non-Free Parking (§4.3) +CHANCE_AMOUNT = 200 # chance card payout magnitude (§4.4) +LAPS_TO_WIN = 2 # end-of-game lap-count cap (§6.2) + + +def tier_of(p) -> int: + """1 = land, 2 = house, 3 = hotel, 0 = unowned.""" + if p.owner is None: + return 0 + if p.has_hotel: + return 3 + if p.houses > 0: + return 2 + return 1 + + +def _set_tier(p, target: int) -> None: + """Mutate p so its tier becomes `target` (1/2/3). Owner must already be set.""" + if target <= 0: + p.owner = None + p.houses = 0 + p.has_hotel = False + p.mortgaged = False + return + p.has_hotel = target == 3 + p.houses = 1 if target == 2 else 0 + p.mortgaged = False + + +def assets_value(state: GameState, player: Player) -> int: + """Sum of tier × $100 over all properties owned by `player` (spec §2.1.2).""" + return sum( + tier_of(p) * TIER_PRICE for p in state.properties.values() if p.owner == player + ) + + +def total_money(state: GameState, player: Player) -> int: + """Liquid + assets (spec §6.2.1).""" + return state.players[player].balance + assets_value(state, player) + + +class RuleError(ValueError): + """Raised when a request violates game rules. Mapped to 409 by the API.""" + + def __init__(self, code: str, message: str, details: dict | None = None) -> None: + super().__init__(code, message, details) + self.code = code + self.message = message + self.details = details or {} + + def __str__(self) -> str: + return self.message + + +# ---- helpers --------------------------------------------------------------- + + +def _init_players(state: GameState, board: Board) -> None: + colors = state.config.player_colors + for pid in ("user", "robot"): + state.players[pid] = PlayerState( + id=pid, balance=board.seed_money, color=colors.get(pid, "#888") + ) + state.positions[pid] = 0 + state.lap_count[pid] = 0 + + +def _tile_pid(board: Board, tile_index: int) -> str: + return property_id(board.board_id, board.tiles[tile_index].name) + + +def _max_tier(tile) -> int: + """Max owned tier for a tile kind (spec §4.1, §4.2).""" + return 3 if tile.kind == "property" else 1 + + +# ---- transitions ----------------------------------------------------------- + + +def start_game(state: GameState, board_id: str) -> Board: + # Singleton-game model: pressing "Start" always begins a fresh game + # even mid-play. The reset below clobbers every in-memory field, so + # there's nothing to protect from a second /game/start call. + if board_id != "final": + raise RuleError("BAD_REQUEST", f"unknown board_id: {board_id!r}") + board = load_board(board_id) + state.board_id = board_id # type: ignore[assignment] + state.fsm = FSM.TURN_START + state.turn = "user" + state.turn_number = 1 + state.last_dice = None + state.last_dice_sum = None + state.pending_dice = None + state.doubles_streak = 0 + state.winner = None + state.players.clear() + state.positions.clear() + state.lap_count.clear() + state.properties.clear() + _init_players(state, board) + state.properties.update(initial_properties(board)) + return board + + +def submit_dice(state: GameState, value: int | tuple[int, int]) -> dict[str, Any] | None: + """Submit a dice roll. Spec §4.5 jail handling: + - if not jailed: normal flow (FSM → MOVING) + - if jailed and the rolled face is 6: escape, normal move + - if jailed and turns_left > 0: decrement, **skip the move** by + going straight to RESOLVE_TILE (the manager runs no resolution + for a jail-skipped turn and end_turn is the next action) + - if jailed and turns_left == 0: auto-release, normal flow + Returns a payload describing the jail outcome, or None if no jail + transition happened. + """ + if state.fsm != FSM.TURN_START: + raise RuleError("INVALID_STATE", f"cannot submit dice in fsm={state.fsm.value}") + if isinstance(value, tuple): + d1, d2 = value + if not (1 <= d1 <= 6 and 1 <= d2 <= 6): + raise RuleError("BAD_REQUEST", f"dice out of range: {value}") + total = d1 + d2 + state.last_dice = (d1, d2) + else: + if not (1 <= value <= 6): + raise RuleError("BAD_REQUEST", f"dice out of range: {value}") + total = value + state.last_dice = (value, 0) + state.last_dice_sum = total + state.pending_dice = total + + player = state.turn + p_state = state.players.get(player) + jail_payload: dict[str, Any] | None = None + if p_state is not None and p_state.in_jail: + # Single-die move from IN_JAIL — escape iff face is 6 (spec §4.5.2.1). + face_for_escape = state.last_dice[0] + if face_for_escape == 6: + p_state.in_jail = False + p_state.jail_turns_left = 0 + jail_payload = {"kind": "jail_escaped", "player": player} + elif p_state.jail_turns_left > 0: + p_state.jail_turns_left -= 1 + state.pending_dice = None + state.fsm = FSM.END_TURN + jail_payload = { + "kind": "jail_skipped", + "player": player, + "turns_left": p_state.jail_turns_left, + } + return jail_payload + else: + p_state.in_jail = False + jail_payload = {"kind": "jail_released", "player": player} + + # House rule: the jail_visit tile (Desert Island) is invisible for + # dice movement — neither landed on nor counted as a step when passed + # through. Only the go_to_jail effect can place a piece there. If the + # path { from+1, …, from+dice } includes a jail_visit tile, bump the + # roll by one so the count of "real" tiles crossed equals the dice. + # Single die ≤ 6 and only one jail_visit per board, so one bump + # suffices (the path covers each tile index at most once). + board = load_board(state.board_id) + size = board.tile_count + from_tile = state.positions.get(player, 0) + dice = state.pending_dice + for jail_idx, tile in enumerate(board.tiles): + if tile.kind != "jail_visit": + continue + offset = (jail_idx - from_tile) % size + if 1 <= offset <= dice: + state.pending_dice += 1 + state.last_dice_sum = state.pending_dice + if jail_payload is None: + jail_payload = { + "kind": "tile_skipped", + "player": player, + "tile_index": jail_idx, + "tile_name": tile.name, + "new_sum": state.pending_dice, + } + break + + state.fsm = FSM.MOVING + return jail_payload + + +@dataclass +class MoveResult: + player: Player + from_tile: int + to_tile: int + dice_sum: int + wrapped: bool # crossed the START tile + lap_completed: bool # this lap is the player's first lap (Board 3 win) + winner: Player | None # set if the move ended the game + start_bonus_collected: int = 0 # +$100 on Board 1, +$200 on Board 2, etc. + + +def apply_move(state: GameState, player: Player, from_tile: int, to_tile: int) -> MoveResult: + if state.fsm != FSM.MOVING: + raise RuleError("INVALID_STATE", f"cannot apply move in fsm={state.fsm.value}") + if player != state.turn: + raise RuleError("INVALID_STATE", f"not {player}'s turn (turn={state.turn})") + if state.pending_dice is None: + raise RuleError("INVALID_STATE", "no pending dice") + + board = load_board(state.board_id) + size = board.tile_count + + expected_from = state.positions.get(player, 0) + if from_tile != expected_from: + raise RuleError( + "TILE_MISMATCH", + f"from_tile {from_tile} does not match server position {expected_from}", + {"from_tile": from_tile, "server": expected_from}, + ) + + dice = state.pending_dice + expected_to = (from_tile + dice) % size + # Landing on the jail_visit tile via dice is "just visiting" — a no_op + # resolution (see _resolve_once). Only the GO_TO_JAIL teleport sets + # in_jail. + if to_tile != expected_to: + raise RuleError( + "TILE_MISMATCH", + f"to_tile {to_tile} does not match computed {expected_to}", + {"dice": dice, "expected_to": expected_to, "given": to_tile}, + ) + + wrapped = to_tile < from_tile or (from_tile + dice) >= size + state.positions[player] = to_tile + state.pending_dice = None + state.fsm = FSM.RESOLVE_TILE + + lap_completed = False + winner: Player | None = None + bonus = 0 + if wrapped: + state.lap_count[player] = state.lap_count.get(player, 0) + 1 + bonus = board.start_bonus + if bonus > 0: + state.players[player].balance += bonus + + return MoveResult( + player=player, + from_tile=from_tile, + to_tile=to_tile, + dice_sum=dice, + wrapped=wrapped, + lap_completed=lap_completed, + winner=winner, + start_bonus_collected=bonus, + ) + + +def end_turn(state: GameState) -> dict[str, Any] | None: + """Hand control to the other player. Also runs the lap-cap check + (spec §6.2): if any player has completed `LAPS_TO_WIN` laps, the + game ends and winner is decided by total_money. Returns the + end-of-game payload when the game ends, else None. + """ + if state.fsm == FSM.GAME_OVER: + return None + if state.fsm not in (FSM.RESOLVE_TILE, FSM.END_TURN): + raise RuleError("INVALID_STATE", f"cannot end turn in fsm={state.fsm.value}") + state.turn = state.other(state.turn) + state.turn_number += 1 + state.last_dice = None + state.last_dice_sum = None + state.pending_dice = None + state.fsm = FSM.TURN_START + + # Spec §6.2 — lap cap. + if any(state.lap_count.get(p, 0) >= LAPS_TO_WIN for p in ("user", "robot")): + totals = {p: total_money(state, p) for p in ("user", "robot")} + if totals["user"] > totals["robot"]: + state.winner = "user" + elif totals["robot"] > totals["user"]: + state.winner = "robot" + else: + state.winner = None # tie — see "draw" flag in payload + state.fsm = FSM.GAME_OVER + return { + "winner": state.winner, + "draw": state.winner is None, + "reason": "lap_cap", + "totals": totals, + "lap_count": dict(state.lap_count), + } + return None + + +# ============================================================================ +# Tile resolution (M2) +# ============================================================================ + + +@dataclass +class TileResolution: + kind: str + tile_index: int + payload: dict[str, Any] = field(default_factory=dict) + needs_decision: bool = False + bankrupt_player: Player | None = None + + +def resolve_tile( + state: GameState, + board: Board, + player: Player, + *, + chance_deck: Deck | None = None, + cc_deck: Deck | None = None, + max_chain: int = 3, +) -> list[TileResolution]: + """Dispatch the player's current tile. May recurse when chance cards + move them to another tile — up to `max_chain` hops. + """ + if state.fsm != FSM.RESOLVE_TILE: + raise RuleError("INVALID_STATE", f"cannot resolve in fsm={state.fsm.value}") + + results: list[TileResolution] = [] + for _ in range(max_chain): + res = _resolve_once(state, board, player, chance_deck=chance_deck, cc_deck=cc_deck) + results.append(res) + if res.needs_decision or res.bankrupt_player is not None: + break + # If the resolution kept us on the same tile (non-movement), stop. + if res.kind not in ("chance_drawn", "community_chest_drawn") or "moved_to" not in res.payload: + break + # Otherwise continue resolving the new tile. + return results + + +def _resolve_once( + state: GameState, + board: Board, + player: Player, + *, + chance_deck: Deck | None, + cc_deck: Deck | None, +) -> TileResolution: + tile_index = state.positions[player] + tile = board.tiles[tile_index] + + if tile.kind == "start": + return TileResolution("start", tile_index) + + if tile.kind in ("property", "railroad", "utility"): + pid = _tile_pid(board, tile_index) + p = state.properties[pid] + if p.owner is None: + if state.players[player].balance >= LAND_PRICE: + state.fsm = FSM.AWAIT_DECISION + return TileResolution( + "property_arrival_buyable", + tile_index, + payload={"property_id": pid, "card": render_card(tile, p, board.board_id), + "current_tier": 0, "max_tier": _max_tier(tile)}, + needs_decision=True, + ) + return TileResolution( + "property_arrival_unaffordable", + tile_index, + payload={"property_id": pid, "card": render_card(tile, p, board.board_id)}, + ) + if p.owner == player: + # Spec §4.1.2: revisiting an already-owned property opens the + # upgrade modal (unless it's already at max tier for this kind: + # property → tier 3, utility/railroad → tier 1). + current = tier_of(p) + max_tier = _max_tier(tile) + if current < max_tier: + state.fsm = FSM.AWAIT_DECISION + return TileResolution( + "property_arrival_buyable", + tile_index, + payload={"property_id": pid, "card": render_card(tile, p, board.board_id), + "current_tier": current, "max_tier": max_tier}, + needs_decision=True, + ) + return TileResolution("property_arrival_self_or_mortgaged", tile_index, + payload={"property_id": pid}) + rent = props_mod.compute_rent(state, board, tile_index, state.last_dice_sum) + bankrupt, liq = _pay_rent_or_bankrupt(state, board, player, p.owner, rent) + return TileResolution( + "rent_paid" if not bankrupt else "rent_bankruptcy", + tile_index, + payload={"property_id": pid, "amount": rent, "payee": p.owner, + "liquidation": liq}, + bankrupt_player=player if bankrupt else None, + ) + + if tile.kind == "tax": + bankrupt, liq = _pay_bank_or_bankrupt(state, board, player, TAX_AMOUNT) + return TileResolution( + "tax_paid" if not bankrupt else "tax_bankruptcy", + tile_index, + payload={"amount": TAX_AMOUNT, "liquidation": liq}, + bankrupt_player=player if bankrupt else None, + ) + + if tile.kind == "chance": + # VLM-driven chance card flow (see router.py:game_chance_card). + # The tile resolution itself is a no-op — the money outcome is + # decided by the orchestrator's VLM reading the physical card. + # `deferred=True` tells the frontend to call /api/game/chance_card + # before /api/game/end_turn so the flow runs while the FSM is + # still on the chance tile. + return TileResolution( + "chance_drawn", + tile_index, + payload={"deferred": True, "player": player, + "balance": state.players[player].balance}, + ) + + if tile.kind == "community_chest" and cc_deck is not None: + card = cc_deck.draw() + apply_res = apply_effect(state, board, player, card.effect) + payload = {"card_id": card.id, "text": card.text, "effect": apply_res} + if card.effect.get("type") != "grant_jail_free_card": + cc_deck.return_to_bottom(card) + if apply_res.get("kind") in ("move_to_tile", "move_to_nearest", "move_relative", "go_to_jail"): + payload["moved_to"] = state.positions[player] + return TileResolution("community_chest_drawn", tile_index, payload=payload) + + if tile.kind == "go_to_jail": + apply_res = apply_effect(state, board, player, {"type": "go_to_jail"}) + return TileResolution("go_to_jail", tile_index, payload=apply_res) + + # free_parking, jail_visit, blank + return TileResolution("no_op", tile_index, payload={"tile_kind": tile.kind}) + + +# ---- property transactions ------------------------------------------------- + + +def buy_property(state: GameState, board: Board, player: Player, pid: str) -> dict[str, Any]: + if state.fsm != FSM.AWAIT_DECISION: + raise RuleError("INVALID_STATE", f"cannot buy in fsm={state.fsm.value}") + if pid not in state.properties: + raise RuleError("NOT_FOUND", f"unknown property {pid!r}") + p = state.properties[pid] + if p.owner is not None: + raise RuleError("PROPERTY_OWNED", f"{pid} already owned by {p.owner}") + price = LAND_PRICE # flat land tier (spec §4.1) + if state.players[player].balance < price: + raise RuleError( + "INSUFFICIENT_FUNDS", f"balance {state.players[player].balance} < {price}", + {"balance": state.players[player].balance, "required": price}, + ) + state.players[player].balance -= price + p.owner = player + p.houses = 0 + p.has_hotel = False + p.mortgaged = False + state.fsm = FSM.RESOLVE_TILE + return {"property_id": pid, "price": price, "owner": player, "tier": 1, + "balance": state.players[player].balance} + + +def skip_purchase(state: GameState) -> None: + if state.fsm != FSM.AWAIT_DECISION: + raise RuleError("INVALID_STATE", f"cannot skip in fsm={state.fsm.value}") + state.fsm = FSM.RESOLVE_TILE + + +def build( + state: GameState, board: Board, player: Player, pid: str, + *, houses: int = 1, hotel: bool = False, +) -> dict[str, Any]: + """Upgrade to a higher tier (spec §4.1.2). Cost = $100 × tiers crossed. + + `hotel=True` targets tier 3, any positive `houses` targets tier 2. + """ + if pid not in state.properties: + raise RuleError("NOT_FOUND", f"unknown property {pid!r}") + p = state.properties[pid] + if p.owner != player: + raise RuleError("NOT_OWNER", f"{pid} is owned by {p.owner}, not {player}") + tile = board.tiles[p.tile_index] + if tile.kind != "property": + raise RuleError("BAD_REQUEST", f"cannot build on {tile.kind} tile") + + current = tier_of(p) + target = 3 if hotel else 2 + if target <= current: + raise RuleError("BAD_REQUEST", f"already at tier {current}") + + cost = (target - current) * TIER_PRICE + if state.players[player].balance < cost: + raise RuleError("INSUFFICIENT_FUNDS", "balance below build cost") + state.players[player].balance -= cost + _set_tier(p, target) + return {"property_id": pid, "tier": target, + "tier_label": "hotel" if target == 3 else "house", + "cost": cost, + "balance": state.players[player].balance} + + +def sell_tier(state: GameState, board: Board, player: Player, pid: str) -> dict[str, Any]: + """Drop one tier (hotel→house, house→land, or land→unowned). + + This is the only sell path under the spec — invoked from auto-liquidation, + never from a voluntary user action. Refund = $100 per tier dropped. + """ + p = state.properties[pid] + if p.owner != player: + raise RuleError("NOT_OWNER", f"{pid} is owned by {p.owner}") + current = tier_of(p) + if current <= 0: + raise RuleError("BAD_REQUEST", "nothing to sell — property is unowned") + new_tier = current - 1 + _set_tier(p, new_tier) + state.players[player].balance += TIER_PRICE + return {"property_id": pid, "refund": TIER_PRICE, + "from_tier": current, "to_tier": new_tier, + "balance": state.players[player].balance} + + +# Back-compat aliases — older code paths (and a few tests) still import these. +# Both collapse to sell_tier under the flat-economy spec. +sell_building = sell_tier +mortgage = sell_tier + + +def unmortgage(*_args, **_kwargs): # pragma: no cover — voluntary mortgage is gone + raise RuleError("BAD_REQUEST", "unmortgage is not part of the spec") + + +# ---- bankruptcy ------------------------------------------------------------ + + +def _pay_rent_or_bankrupt( + state: GameState, board: Board, payer: Player, payee: Player, amount: int, +) -> tuple[bool, list[dict[str, Any]]]: + """Attempt to pay `amount` in rent. Auto-liquidates when short. Returns + (bankrupt, liquidation_steps). On bankruptcy assets transfer to payee + and fsm -> GAME_OVER.""" + if amount <= 0: + return False, [] + steps = _auto_liquidate(state, board, payer, amount) + if state.players[payer].balance >= amount: + state.players[payer].balance -= amount + state.players[payee].balance += amount + return False, steps + # fully bankrupt — transfer everything remaining to payee + state.players[payee].balance += state.players[payer].balance + state.players[payer].balance = 0 + for p in state.properties.values(): + if p.owner == payer: + p.owner = payee + # keep buildings/mortgage flags as-is + state.winner = payee + state.fsm = FSM.GAME_OVER + return True, steps + + +def _pay_bank_or_bankrupt( + state: GameState, board: Board, payer: Player, amount: int, +) -> tuple[bool, list[dict[str, Any]]]: + if amount <= 0: + return False, [] + steps = _auto_liquidate(state, board, payer, amount) + if state.players[payer].balance >= amount: + state.players[payer].balance -= amount + return False, steps + # unpaid tax → bankrupt to bank → properties return to bank (no owner) + state.players[payer].balance = 0 + for p in state.properties.values(): + if p.owner == payer: + p.owner = None + p.houses = 0 + p.has_hotel = False + p.mortgaged = False + state.winner = state.other(payer) + state.fsm = FSM.GAME_OVER + return True, steps + + +def _auto_liquidate( + state: GameState, board: Board, player: Player, target: int, +) -> list[dict[str, Any]]: + """Drop tiers one at a time until balance >= target or nothing left to + sell (spec §5.2). Order: hotels first, then houses, then lands. Each + step refunds $100 and emits a single `tier_sold` event. + """ + steps: list[dict[str, Any]] = [] + + def _drop_at_tier(target_tier: int) -> None: + """Drop one tier on every property currently at exactly `target_tier`.""" + for p in list(state.properties.values()): + if state.players[player].balance >= target: + return + if p.owner != player or tier_of(p) != target_tier: + continue + steps.append({"kind": "tier_sold", **sell_tier(state, board, player, p.id)}) + + # Hotels (tier 3) → houses + _drop_at_tier(3) + if state.players[player].balance >= target: + return steps + # Houses (tier 2) → land + _drop_at_tier(2) + if state.players[player].balance >= target: + return steps + # Land (tier 1) → unowned + _drop_at_tier(1) + return steps diff --git a/movensys_sample/movensys_robopoly/game/state.py b/movensys_sample/movensys_robopoly/game/state.py new file mode 100644 index 0000000..ab00547 --- /dev/null +++ b/movensys_sample/movensys_robopoly/game/state.py @@ -0,0 +1,86 @@ +"""Canonical game state (PRD §4). + +Single source of truth: `GameState`. All REST responses and WS events +project from this. `positions` is the authoritative piece location; the +UI must not display piece positions from WS payloads alone — it should +reconcile against `/api/game/state` on reconnect. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Literal + +from pydantic import BaseModel, Field + +Player = Literal["user", "robot"] +BoardId = Literal["final"] +DiceSource = Literal["manual", "rng", "robot"] + + +class FSM(str, Enum): + IDLE = "IDLE" + TURN_START = "TURN_START" + MOVING = "MOVING" + RESOLVE_TILE = "RESOLVE_TILE" + AWAIT_DECISION = "AWAIT_DECISION" + PAY_RENT = "PAY_RENT" + END_TURN = "END_TURN" + GAME_OVER = "GAME_OVER" + + +class PlayerState(BaseModel): + id: Player + balance: int = 0 + in_jail: bool = False + jail_turns_left: int = 0 + has_jail_free_card: bool = False + color: str = "#888888" + + +class PropertyState(BaseModel): + """Dynamic ownership state for a purchasable tile (PRD §4.4). + + Paired with the static `Tile` from `game/boards.py` to form the full + property card view. + """ + + id: str # "board1:baltic_avenue" + tile_index: int + owner: Player | None = None + houses: int = 0 # 0..4 (property only) + has_hotel: bool = False + mortgaged: bool = False + + +class RuntimeConfig(BaseModel): + dice_source: DiceSource = "rng" + auctions_enabled: bool = False + income_tax_mode: Literal["fixed_200", "choose"] = "fixed_200" + player_colors: dict[Player, str] = Field( + default_factory=lambda: {"user": "#E53935", "robot": "#1E88E5"} + ) + is_YOLO: bool = True + + +class GameState(BaseModel): + board_id: BoardId = "final" + fsm: FSM = FSM.IDLE + turn: Player = "user" + turn_number: int = 0 + positions: dict[Player, int] = Field(default_factory=dict) + players: dict[Player, PlayerState] = Field(default_factory=dict) + properties: dict[str, PropertyState] = Field(default_factory=dict) + last_dice: tuple[int, int] | None = None + last_dice_sum: int | None = None + pending_dice: int | None = None + doubles_streak: int = 0 + lap_count: dict[Player, int] = Field(default_factory=dict) + winner: Player | None = None + config: RuntimeConfig = Field(default_factory=RuntimeConfig) + + def is_active(self) -> bool: + return self.fsm not in (FSM.IDLE, FSM.GAME_OVER) + + def other(self, p: Player) -> Player: + return "robot" if p == "user" else "user" diff --git a/movensys_sample/movensys_robopoly/main.py b/movensys_sample/movensys_robopoly/main.py new file mode 100644 index 0000000..39cc52e --- /dev/null +++ b/movensys_sample/movensys_robopoly/main.py @@ -0,0 +1,78 @@ +"""FastAPI entry point for movensys-monopoly.""" +# app.state, app.mount, app.include_router(api_router) +# For app.mount("/static", StaticFiles(directory=), name="static") 이걸 외워두면 좋다. + + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI, Request +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles + +from adapters import RobotAdapter, RosImageSubscriber, STTAdapter, VLMAdapter +from game.events import EventBus +from game.manager import GameManager +from router import api_router + +log = logging.getLogger("monopoly") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + app.state.stt_adapter = STTAdapter.from_env() + app.state.vlm_adapter = VLMAdapter.from_env() + app.state.robot_adapter = RobotAdapter.from_env() + app.state.event_bus = EventBus() + app.state.game = GameManager(bus=app.state.event_bus) + # Background subscriber for /yolo_{dice,cube}_detector/debug_image — + # powers the board-pane overlay while pick_and_place runs. Safe no-op + # on hosts where rclpy isn't installed (e.g. unit-test environments). + app.state.ros_image = RosImageSubscriber() + app.state.ros_image.start() + log.info( + "startup", + extra={ + "stt_mode": app.state.stt_adapter.mode, + "vlm_mode": app.state.vlm_adapter.mode, + "robot_mode": app.state.robot_adapter.mode, + }, + ) + yield + try: + app.state.ros_image.stop() + except Exception: + log.exception("ros_image stop failed") + log.info("shutdown") + + +app = FastAPI(title="movensys-monopoly", lifespan=lifespan) + + +@app.middleware("http") +async def no_cache_static(request: Request, call_next): + response = await call_next(request) + path = request.url.path + if path == "/" or path.startswith("/static") or path.startswith("/assets"): + response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + return response + + +app.include_router(api_router) + +_static_dir = Path(__file__).parent / "static" +if _static_dir.exists(): + # directory mount하기 + app.mount("/assets", StaticFiles(directory=_static_dir / "assets"), name="assets") + app.mount("/static", StaticFiles(directory=_static_dir), name="static") + + @app.get("/") + async def index() -> FileResponse: + return FileResponse(_static_dir / "index.html") diff --git a/movensys_sample/movensys_robopoly/pick_and_place.py b/movensys_sample/movensys_robopoly/pick_and_place.py new file mode 100644 index 0000000..ccf917e --- /dev/null +++ b/movensys_sample/movensys_robopoly/pick_and_place.py @@ -0,0 +1,892 @@ +import logging +import math +import os +import random +import sys +import time +import requests +from typing import Optional + +logger = logging.getLogger(__name__) +board_positions = { + "GO": { + "red_cube": { + "pos": [-0.38632, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.37876, -0.15907, 0.3] + }, + "green_cube": { + "pos": [-0.32625, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.3323, -0.15907, 0.3] + } + }, + "BOSTON": { + "red_cube": { + "pos": [-0.38640, -0.09308, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.38615, -0.09402, 0.3] + }, + "green_cube": { + "pos": [-0.32625, -0.09308, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.34041, -0.09396, 0.3] + } + }, + "SEOUL": { + "red_cube": { + "pos": [-0.38640, -0.023, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.38615, -0.02562, 0.3] + }, + "green_cube": { + "pos": [-0.32625, -0.023, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.34041, -0.02614, 0.3] + } + }, + "DESERT_ISLAND": { + "red_cube": { + "pos": [-0.38640, 0.04465, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.38615, 0.0405, 0.3] + }, + "green_cube": { + "pos": [-0.32625, 0.04465, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.34041, 0.04039, 0.3] + } + }, + "ELECTRIC_COMPANY": { + "red_cube": { + "pos": [-0.27139, 0.04460, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.27231, 0.0405, 0.3] + }, + "green_cube": { + "pos": [-0.21632, 0.04458, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.23001, 0.04039, 0.3] + } + }, + "TAIPEI": { + "red_cube": { + "pos": [-0.16125, 0.04458, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.15219, 0.0405, 0.3] + }, + "green_cube": { + "pos": [-0.10619, 0.04458, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.10781, 0.04039, 0.3] + } + }, + "SHANGHAI": { + "red_cube": { + "pos": [-0.05614, 0.04458, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.03453, 0.0405, 0.3] + }, + "green_cube": { + "pos": [0.00390, 0.04458, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.00802, 0.04039, 0.3] + } + }, + "NON-FREE_PARKING": { + "red_cube": { + "pos": [0.05398, 0.04458, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.07915, 0.0405, 0.3] + }, + "green_cube": { + "pos": [0.11407, 0.04457, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.12049, 0.04039, 0.3] + } + }, + "TOKYO": { + "red_cube": { + "pos": [0.05393, -0.023, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.07915, -0.02496, 0.3] + }, + "green_cube": { + "pos": [0.10903, -0.02301, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.12049, -0.02767, 0.3] + } + }, + "BUSAN": { + "red_cube": { + "pos": [0.05388, -0.09308, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.07915, -0.09243, 0.3] + }, + "green_cube": { + "pos": [0.11396, -0.09308, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.12049, -0.09362, 0.3] + } + }, + "GO_TO_DESERT_ISLAND": { + "red_cube": { + "pos": [0.0538, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.07991, -0.16428, 0.3] + }, + "green_cube": { + "pos": [0.10895, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.12049, -0.16341, 0.3] + } + }, + "NEW_YORK": { + "red_cube": { + "pos": [-0.05662, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.03824, -0.16428, 0.3] + }, + "green_cube": { + "pos": [0.01015, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [0.00573, -0.16341, 0.3] + } + }, + "CHANCE": { + "red_cube": { + "pos": [-0.16186, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.1527, -0.16428, 0.3] + }, + "green_cube": { + "pos": [-0.10666, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.11251, -0.16341, 0.3] + } + }, + "LONDON": { + "red_cube": { + "pos": [-0.27194, -0.153, 0.30], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.26724, -0.16428, 0.3] + }, + "green_cube": { + "pos": [-0.21182, -0.153, 0.3], + "ori": [3.14, 0.0, -1.57], + "sim_pos": [-0.22484, -0.16341, 0.3] + } + } +} + +URL = "http://localhost:8000" + +# When MOVENSYS_PNP_DRY_RUN is truthy, every HTTP call to the manipulator +# stack is skipped — motion/gripper requests log and return {} instead of +# hitting the arm. get_piece_info synthesizes a fixed pose so the rest of +# pick_and_place() runs end-to-end without hardware. Use for testing the +# monopoly server flow on a workstation with no robot attached. +DRY_RUN = os.environ.get("MOVENSYS_PNP_DRY_RUN", "").strip().lower() in ( + "1", "true", "yes", "y", "on", +) + + +def _post(path: str, payload: dict): + if DRY_RUN: + logger.info("[dry-run] POST %s %s", path, payload) + return {} + start = time.perf_counter() + try: + return requests.post(f"{URL}{path}", json=payload).json() + finally: + logger.info("[timing] POST %s: %.1f ms", path, (time.perf_counter() - start) * 1000.0) + + +def _get(path: str): + if DRY_RUN: + logger.info("[dry-run] GET %s", path) + return {} + start = time.perf_counter() + try: + return requests.get(f"{URL}{path}").json() + finally: + logger.info("[timing] GET %s: %.1f ms", path, (time.perf_counter() - start) * 1000.0) + + +def _sleep(seconds: float) -> None: + # Real-hardware sleeps pace the arm between motion segments. In + # dry-run the HTTP calls are stubbed instantly, so the sleeps become + # pure wall-clock waste (~13s per dice roll) — skip them. + if DRY_RUN: + return + time.sleep(seconds) + + +def _timed_method(label: str): + def deco(fn): + def wrapper(*args, **kwargs): + start = time.perf_counter() + try: + return fn(*args, **kwargs) + finally: + logger.info("[timing] %s: %.1f ms", label, (time.perf_counter() - start) * 1000.0) + return wrapper + return deco + + +# Motion API calls (/api/move/*) are blocking ROS service calls, so no +# pacing sleep is needed between motions. The gripper SetBool service +# returns before the jaws physically settle — keep a small post-gripper +# wait so subsequent motion doesn't drag/drop the cube. +_GRIPPER_SETTLE_S = 0.6 + + +def move_base(): + absolute_joint_pose([0.0, 0.0, 0.5], [3.141, 0.0, -3.141]) + + +# 6 motion movements +def absolute_cartesian_base(pos, ori): + return _post("/api/move/absolute_cartesian_base", {"pos": pos, "ori": ori}) + + +def relative_cartesian_base(pos, ori): + return _post("/api/move/relative_cartesian_base", {"pos": pos, "ori": ori}) + + +def relative_cartesian_tool(pos, ori): + return _post("/api/move/relative_cartesian_tool", {"pos": pos, "ori": ori}) + + +def absolute_joint_pose(pos, ori): + return _post("/api/move/absolute_joint_pose", {"pos": pos, "ori": ori}) + + +def joint_absolute(names, values): + return _post("/api/move/joint_absolute", {"joint_names": names, "joint_values": values}) + + +def joint_relative(names, values): + return _post("/api/move/joint_relative", {"joint_names": names, "joint_values": values}) + + +# 3 assistance functions +def gripper(close: bool): + return _post("/api/services/gripper", {"data": close}) + + +def get_eef_pose(): + return _get("/api/services/get_eef_pose") + + +def set_scales(vel, acc): + return _post("/api/config/scales", {"vel_scale": vel, "acc_scale": acc}) + + +# Isaac-Sim spawn z. Mirrors `z_target_pose_spawn` in +# apriltag_pick_and_place.cpp's yaml (0.07 = table-relative). EEF z would +# be at gripper height — too high for the object's actual rest pose. +_ISAAC_SPAWN_Z = 0.07 + + +def _publish_isaac_target_pose(target_object: str, z: float = _ISAAC_SPAWN_Z) -> None: + """Teleport the Isaac-Sim counterpart of `target_object` to where the + real arm is, so the simulated object stays in lockstep through the + pickup. Matches apriltag_pick_and_place.cpp's target_spawn block — + one POST per pickup, fired just before the gripper closes. No-op in + DRY_RUN; logs and continues on HTTP errors so a misconfigured Isaac + never breaks a real-world pick.""" + if DRY_RUN: + return + try: + resp = requests.post( + f"{URL}/api/isaac/spawn_target", + json={"target": target_object, "z": z}, + timeout=5.0, + ) + if resp.ok: + body = resp.json() + logger.info( + "[isaac] spawn_target %s -> %s @ %s", + target_object, body.get("topic"), + body.get("pose", {}).get("position"), + ) + else: + logger.warning( + "[isaac] spawn_target %s failed: %s %s", + target_object, resp.status_code, resp.text[:200], + ) + except Exception as exc: + logger.warning("[isaac] spawn_target %s error: %s", target_object, exc) + + +class PnP: + _BIN_CENTERS = (0.0, -math.pi / 2, -math.pi, math.pi / 2) + + def __init__(self, target_object: str = "red_cube", is_YOLO: bool = True, delay_exec: float = 2.0): + self.delay_exec = delay_exec + self.is_YOLO = is_YOLO + + # mapping target_object to number + self.target_object = target_object + target_object_mapper: dict = {"red_cube": 0, "green_cube": 1, "dice": 2} + self.target_num = target_object_mapper.get(self.target_object) + if self.target_num is None: + raise ValueError(f"Unknown target_object '{self.target_object}'. Choose one of: {list(target_object_mapper)}") + + # mapping target object to support YOLO & Isaac via ROS2 topic + if is_YOLO: + self.TARGET_STR = ("yolo_cube_red", "yolo_cube_green", "dice") + else: + self.TARGET_STR = ("piece_2", "piece_1", "dice") + + # This offset is dependent for cube size. + self.YOLO_dice_offset_x: float = 0.015 # [m] + self.YOLO_dice_offset_y: float = -0.075 # [m] + self.YOLO_piece_offset_x: float = 0.011 # [m] + self.YOLO_piece_offset_y: float = -0.08 # [m] + + self.pos: Optional[dict] = None + self.ori: Optional[dict] = None + self.yaw: Optional[float] = None + + # Wall-clock time (time.time()) of the instant the gripper opened to + # release the dice in _dest_move. main() uses this to ignore stale + # /api/topics/dice_number cached from before/during the lift — only a + # YOLO publication newer than (drop_time + settle) reflects the rolled + # face. + self._dice_drop_time: Optional[float] = None + + @staticmethod + def _quaternion_to_yaw(qw: float, qx: float, qy: float, qz: float) -> float: + siny_cosp = 2.0 * (qw * qz + qx * qy) + cosy_cosp = 1.0 - 2.0 * (qy * qy + qz * qz) + return math.atan2(siny_cosp, cosy_cosp) + + @staticmethod + def _init_move(target_object: str = "dice"): + if target_object == "dice": + absolute_cartesian_base([0.32040, -0.01058, 0.42], [3.141, 0.0, -3.141]) + else: + # absolute_cartesian_base([-0.12857, 0.0, 0.3500], [3.141, 0.0, -3.141]) + absolute_cartesian_base([-0.18, 0.035, 0.52], [3.141, 0.0, -3.141]) + + @_timed_method("toward_target") + def _toward_target(self, target_object: str = "dice", target_pos: list = None, target_ori: list = None): + if target_pos is None: + target_pos = [0.0, 0.0, 0.0] + if target_ori is None: + target_ori = [0.0, 0.0, 0.0] + if target_object == "dice": + if self.is_YOLO: + relative_cartesian_tool(target_pos, target_ori) + else: + absolute_cartesian_base(target_pos, target_ori) + + # Go down + relative_cartesian_tool([0.0, 0.0, 0.01], [0.0, 0.0, 0.0]) + else: + # Go upside of the piece + if self.is_YOLO: + print(target_pos) + relative_cartesian_tool(target_pos, target_ori) + else: + absolute_cartesian_base(target_pos, target_ori) + + # Go down + relative_cartesian_tool([0.0, 0.0, 0.025], [0.0, 0.0, 0.0]) + + @_timed_method("dest_move") + def _dest_move(self, target_object: str = "dice", board_pos: str = "GO"): + if target_object == "dice": + # Go up + relative_cartesian_tool([0.0, 0.0, -0.1], [0.0, 0.0, 0.0]) + + # place — release the dice and stamp the drop instant so main() + # can wait for a post-roll YOLO detection. + gripper(close=False) + self._dice_drop_time = time.time() + _sleep(_GRIPPER_SETTLE_S) + + # Retreat to the dice init pose. The gripper hovering ~10cm + # above the dropped dice blocks the top camera, so YOLO can + # never see the rolled face. The init pose was clear enough + # for the pre-pickup detection — it's clear enough for the + # post-roll one too. + self._init_move("dice") + else: + # Go up + relative_cartesian_tool([0.0, 0.0, -0.050], [0.0, 0.0, 0.0]) + + # Go upper side of target pos. + if self.is_YOLO: + target_pos = board_positions[board_pos][target_object]["pos"] + else: + target_pos = board_positions[board_pos][target_object]["sim_pos"] + target_pos[2] = target_pos[2] + 0.035 + absolute_cartesian_base(target_pos, board_positions[board_pos][target_object]["ori"]) + + # Go down + relative_cartesian_tool([0.0, 0.0, 0.055], [0.0, 0.0, 0.0]) + + # place + gripper(close=False) + _sleep(_GRIPPER_SETTLE_S) + + # Go up and prepare to go init pos + relative_cartesian_tool([0.0, 0.0, -0.06], [0.0, 0.0, 0.0]) + + @_timed_method("get_piece_info") + def get_piece_info(self, min_received_at: Optional[float] = None) -> bool: + _target_object = self.TARGET_STR[self.target_num] + + if DRY_RUN: + # Synthetic pose — pick_and_place math (yaw checks, target_pos + # construction) needs non-None values. Numbers are arbitrary + # but in the same shape the real topic would return. + self.pos = {"x": 0.0, "y": 0.0, "z": 0.3} + self.ori = {"w": 1.0, "x": 0.0, "y": 0.0, "z": 0.0} + self.yaw = 0.0 + logger.info("[dry-run] synthetic piece info for %s", _target_object) + return True + + if self.is_YOLO: + resp = requests.get(f"{URL}/api/topics/yolo_tf") + + # Error detection 1. YOLO result is published at /tf + if resp.status_code == 503: + logger.info("YOLO /tf unavailable: %s", resp.json().get("detail", "")) + return False + resp.raise_for_status() + + # Read + tf_all = resp.json() + + # Error detection 2. If there is no YOLO detection, there is no result at /tf. + if _target_object not in tf_all: + logger.info("%s not detected (have: %s)", _target_object, list(tf_all)) + return False + + # find the result + tf = tf_all[_target_object] + # Reject cached TF entries older than the caller-supplied cutoff. + # Used by the fallback search to ignore stale detections from + # before the probe motion. + if min_received_at is not None and tf.get("received_at", 0.0) < min_received_at: + logger.info("%s detection is stale (received_at=%.3f < %.3f)", + _target_object, tf.get("received_at", 0.0), min_received_at) + return False + self.pos = {"x": round(tf["translation"]["x"], 5), "y": round(tf["translation"]["y"], 5), "z": round(tf["translation"]["z"], 5)} + self.ori = {"w": tf["rotation"]["w"], "x": tf["rotation"]["x"], "y": tf["rotation"]["y"], "z": tf["rotation"]["z"]} + + # Isaac + else: + URL_topic = f"{URL}/api/topics/{_target_object}" + resp = requests.get(URL_topic) + info = resp.json() + self.pos = {"x": round(info["position"]["x"], 5), "y": round(info["position"]["y"], 5), "z": round(info["position"]["z"], 5)} + self.ori = {"w": info["orientation"]["w"], "x": info["orientation"]["x"], "y": info["orientation"]["y"], "z": info["orientation"]["z"]} + + self.yaw = round(self._quaternion_to_yaw(self.ori["w"], self.ori["x"], self.ori["y"], self.ori["z"]), 5) + return True + + @staticmethod + def _checking_yaw(yaw: float) -> int: + if -math.pi / 4 <= yaw < math.pi / 4: + return 0 + elif -3 * math.pi / 4 <= yaw < -math.pi / 4: + return 1 + elif math.pi / 4 <= yaw < 3 * math.pi / 4: + return 3 + else: + return 2 + + def converting_yaw(self, yaw_status: int, target_yaw_status: int) -> None: + # Shift yaw by the bin-center delta (multiple of pi/2) + delta = self._BIN_CENTERS[target_yaw_status] - self._BIN_CENTERS[yaw_status] + # Then wrap to (-pi, pi]. + self.yaw = (self.yaw + delta + math.pi) % (2 * math.pi) - math.pi + + _SEARCH_OFFSETS = ( + ("front", (0.05, 0.0)), + ("back", (-0.05, 0.0)), + ("right", (0.0, -0.05)), + ("left", (0.0, 0.05)), + ) + _SEARCH_SETTLE_S = 2.5 + + @_timed_method("search_for_target") + def _search_for_target(self) -> bool: + """Fallback: nudge +/-5cm in base XY (front, back, right, left) and + retry detection at each probe. Undoes each probe before the next so + the arm ends at the original pose whether we succeed or fail.""" + for name, (dx, dy) in self._SEARCH_OFFSETS: + logger.info("search: probing %s (dx=%+.2f, dy=%+.2f)", name, dx, dy) + mv = relative_cartesian_base([dx, dy, 0.0], [0.0, 0.0, 0.0]) + if not mv.get("success", False): + logger.warning("search: %s probe motion failed: %s", name, mv.get("message")) + continue + probe_time = time.time() + time.sleep(self._SEARCH_SETTLE_S) + found = self.get_piece_info(min_received_at=probe_time) + if found: + logger.info("search: detected %s after %s probe", self.target_object, name) + return True + + relative_cartesian_base([-dx, -dy, 0.0], [0.0, 0.0, 0.0]) + probe_time = time.time() + time.sleep(self._SEARCH_SETTLE_S) + found = self.get_piece_info(min_received_at=probe_time) + if found: + logger.info("search: detected %s after %s probe", self.target_object, name) + return True + + return False + + @_timed_method("pick_and_place") + def pick_and_place(self, board_pos: str = "GO"): + if board_pos not in board_positions: + raise ValueError(f"Unknown board_pos '{board_pos}'. Choose one of: {list(board_positions)}") + + gripper(close=False) + _sleep(_GRIPPER_SETTLE_S) + # move to initial position + + # For YOLO, we need to set offset + if self.is_YOLO: + if self.target_object == "dice": + self.pos['x'] += self.YOLO_dice_offset_x + self.pos['y'] += self.YOLO_dice_offset_y + else: + self.pos['x'] += self.YOLO_piece_offset_x + self.pos['y'] += self.YOLO_piece_offset_y + + # This is for dice. + if self.target_object == "dice": + yaw_status = self._checking_yaw(self.yaw) + print(self.yaw) + # For using target_yaw_status = 3, we should change `movensys_manipulator's joint6 limitation` + self.converting_yaw(yaw_status=yaw_status, target_yaw_status=1) + # This is for piece pnp. + else: + yaw_status = self._checking_yaw(self.yaw) + # clockwisely rotate 90 degree. (Left-column tiles.) + if board_pos in ("GO", "BOSTON", "SEOUL", "DESERT_ISLAND"): + self.converting_yaw(yaw_status=yaw_status, target_yaw_status=1) + + # Counter clockwisely rotate -90 degree. (Right-column tiles.) + elif board_pos in ("NON-FREE_PARKING", "TOKYO", "BUSAN", "GO_TO_DESERT_ISLAND"): + self.converting_yaw(yaw_status=yaw_status, target_yaw_status=1) + + # Rotate 180 degree. Looking front side. + else: + self.converting_yaw(yaw_status=yaw_status, target_yaw_status=1) + + # move toward target + if self.is_YOLO: + if self.target_object == "dice": + target_pos = [self.pos['x'], self.pos['y'], 0.12] + else: + target_pos = [self.pos['x'], self.pos['y'], 0.22] + target_ori = [0.0, 0.0, self.yaw] + logger.info(f"{self.target_object}: x={self.pos['x']}, y={self.pos['y']}, z={self.pos['z']}, yaw={self.yaw}") + else: + if self.target_object == "dice": + target_pos = [self.pos['y'], -self.pos['x'], 0.3] + else: + target_pos = [self.pos['y'], -self.pos['x'], 0.3] + target_ori = [-3.14, 0.0, self.yaw] + logger.info(f"{self.target_object}: x={self.pos['y']}, y={-self.pos['x']}, z={self.pos['z']}, yaw={self.yaw}") + + self._toward_target(self.target_object, target_pos, target_ori) + + # Sync Isaac Sim: teleport the simulated counterpart of this object + # to the current EEF pose (axis-swapped on the orchestrator side) so + # a simulated arm grabs it in lockstep with the real one. Mirrors + # apriltag_pick_and_place.cpp's `target_spawn` step. + _publish_isaac_target_pose(self.target_object) + + # grasp + gripper(close=True) + _sleep(_GRIPPER_SETTLE_S) + + # move to destination + self._dest_move(self.target_object, board_pos) + + +# After the dice is released we wait this long for it to physically stop +# rolling before trusting a YOLO reading. The polling loop then keeps +# checking up to _DICE_POLL_TIMEOUT_S in case YOLO publishes a little late. +_DICE_SETTLE_S = 1.5 +_DICE_POLL_TIMEOUT_S = 5.0 +_DICE_POLL_INTERVAL_S = 0.1 + + +def _wait_for_rolled_dice_number(drop_time: float) -> Optional[int]: + """Poll /api/topics/dice_number until YOLO publishes a value whose + received_at is past (drop_time + settle) — i.e., detected after the dice + finished rolling. Returns None if no fresh value arrives before timeout. + """ + if DRY_RUN: + # No physical dice was rolled and the orchestrator's cached + # dice_number would just return a stale value forever. Sample. + value = random.randint(1, 6) + logger.info("[dry-run] synthetic rolled dice_number=%s", value) + return value + time.sleep(_DICE_SETTLE_S) + fresh_after = drop_time + _DICE_SETTLE_S + deadline = time.time() + _DICE_POLL_TIMEOUT_S + while time.time() < deadline: + try: + resp = requests.get(f"{URL}/api/topics/dice_number", timeout=2.0) + except Exception as exc: + logger.warning("dice_number poll error: %s", exc) + time.sleep(_DICE_POLL_INTERVAL_S) + continue + if resp.ok: + payload = resp.json() + received_at = payload.get("received_at", 0.0) + value = payload.get("value") + if value is not None and received_at >= fresh_after: + return int(value) + else: + logger.warning("dice_number fetch returned %s: %s", resp.status_code, resp.text) + time.sleep(_DICE_POLL_INTERVAL_S) + return None + + +def _parse_is_yolo(token: str) -> bool: + value = token.strip().lower() + if value in ("1", "true", "yes", "y", "on"): + return True + if value in ("0", "false", "no", "n", "off"): + return False + raise ValueError(f"Unrecognized is_YOLO value '{token}'. Use true/false.") + + +_READ_POLL_TIMEOUT_S = 8.0 +_READ_POLL_INTERVAL_S = 0.2 + + +def _read_dice_only(is_yolo: bool, pnp: "PnP", main_start: float) -> None: + """User-turn dice path: the human has already thrown the die. Move the + arm to the dice scan pose (so the gripper is out of the camera's way) + and read whatever YOLO currently sees. No pickup, no drop, no + freshness check — the dice was rolled BEFORE this script started, so + the cached YOLO publish (which may have a received_at older than the + arm motion) is exactly the value we want. + + Guarantees that DICE_NUMBER= is printed before this function + returns. If YOLO never publishes anything, we fall back to a default + of 1 with a loud error log — emitting *something* lets the calling + chain (router → apply_robot → end_turn) proceed instead of + dead-ending on a 502 with no user-visible message. The operator will + see the warning and can re-roll if the face is wrong. + """ + init_start = time.perf_counter() + logger.info("read mode: moving arm to dice scan pose") + pnp._init_move("dice") + time.sleep(2.0) + logger.info( + "[timing] read_init+settle: %.1f ms", + (time.perf_counter() - init_start) * 1000.0, + ) + + value: Optional[int] = None + last_status: Optional[int] = None + last_detail: Optional[str] = None + + if DRY_RUN: + value = random.randint(1, 6) + logger.info("[dry-run] synthetic read dice_number=%s", value) + # Read mode mirrors the roll-mode sentinel placement: the dice + # face IS the YOLO event, so the close sentinel fires alongside. + print("YOLO_DETECTED", flush=True) + print(f"DICE_NUMBER={value}", flush=True) + logger.info("read mode: emitted DICE_NUMBER=%s", value) + logger.info("[timing] read_total: %.1f ms", (time.perf_counter() - main_start) * 1000.0) + return + + if is_yolo: + deadline = time.time() + _READ_POLL_TIMEOUT_S + attempts = 0 + while time.time() < deadline: + attempts += 1 + try: + resp = requests.get(f"{URL}/api/topics/dice_number", timeout=2.0) + last_status = resp.status_code + if resp.ok: + payload = resp.json() + cached = payload.get("value") + received_at = payload.get("received_at") + if cached is not None: + value = int(cached) + logger.info( + "read mode: got dice_number=%s after %d attempt(s) (received_at=%s)", + value, attempts, received_at, + ) + break + last_detail = "ok but no 'value' field" + else: + # 503 "No dice number received yet" lands here. + try: + last_detail = resp.json().get("detail") + except Exception: + last_detail = resp.text[:200] + except Exception as exc: + last_detail = repr(exc) + logger.warning("dice_number fetch error: %s", exc) + time.sleep(_READ_POLL_INTERVAL_S) + + if value is None: + logger.error( + "read mode: YOLO never returned a usable dice_number after %d attempt(s) " + "(last status=%s, last detail=%s). Falling back to DICE_NUMBER=1 so the " + "turn doesn't dead-end. Re-roll if this face is wrong.", + attempts, last_status, last_detail, + ) + value = 1 + else: + value = random.randint(1, 6) + + print("YOLO_DETECTED", flush=True) + print(f"DICE_NUMBER={value}", flush=True) + logger.info("read mode: emitted DICE_NUMBER=%s", value) + logger.info("[timing] read_total: %.1f ms", (time.perf_counter() - main_start) * 1000.0) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") + + if len(sys.argv) < 4: + raise SystemExit( + "Usage: python3 pick_and_place.py [mode]" + ) + + # is_YOLO drives PnP.TARGET_STR (yolo_cube_* vs piece_*) and per-method + # branches throughout the class, so it must be parsed before PnP() is + # instantiated. + is_yolo = _parse_is_yolo(sys.argv[3]) + + # Optional 4th arg. "roll" (default) is the full pick+drop chain used + # on the robot turn. "read" is the user-turn path — the human has + # already thrown the dice, we only need to look at it. Only valid + # when target_object == "dice". + mode = (sys.argv[4] if len(sys.argv) >= 5 else "roll").strip().lower() + if mode not in ("roll", "read", "chance_init"): + raise SystemExit(f"Unrecognized mode '{mode}'. Use 'roll', 'read', or 'chance_init'.") + if mode == "read" and sys.argv[1] != "dice": + raise SystemExit("mode='read' is only valid for target_object='dice'") + + pnp = PnP(target_object=sys.argv[1], is_YOLO=is_yolo, delay_exec=2.0) + + main_start = time.perf_counter() + + if mode == "read": + _read_dice_only(is_yolo, pnp, main_start) + return + + # Chance-card scan pose: park the arm at the cube-detection init + # position so the top camera has a clear view of the chance card laid + # under the gripper, then settle for 2.0 s before the caller grabs an + # RGB frame for the VLM. Sleep is unconditional (even in dry-run) so + # the caller's timing assumptions don't depend on hardware presence. + if mode == "chance_init": + # _init_move's only branch is dice vs. non-dice — pass the + # caller's target_object (red_cube / green_cube) so the same + # cube scan pose is used. + pnp._init_move(sys.argv[1]) + time.sleep(2.0) + return + + # init + init_start = time.perf_counter() + pnp._init_move(sys.argv[1]) + init_done_at = time.time() + time.sleep(3.0) + logger.info("[timing] init_move+settle: %.1f ms", (time.perf_counter() - init_start) * 1000.0) + + # pick and place + detect_start = time.perf_counter() + if not pnp.get_piece_info(min_received_at=init_done_at if is_yolo else None): + if is_yolo: + logger.info("Initial detection missed, starting 4-direction fallback search") + if not pnp._search_for_target(): + # Exit non-zero so the spawning router sees PNP_FAILED and + # surfaces it to the frontend, instead of silently advancing + # the game state while the physical cube never moved. + logger.error("Failed to detect %s after search, aborting.", sys.argv[1]) + sys.exit(1) + else: + logger.error("Failed to get piece info, aborting.") + sys.exit(1) + logger.info("[timing] detect_phase: %.1f ms", (time.perf_counter() - detect_start) * 1000.0) + + # Cube/piece moves: position detection is the only useful YOLO event, + # so emit the overlay-close sentinel now and let the arm physically + # finish the pick-and-place while the frontend shows the board. + # Dice rolls handle this differently — see below, after the post-drop + # dice_number read. + if sys.argv[1] != "dice": + print("YOLO_DETECTED", flush=True) + + pnp.pick_and_place(board_pos=sys.argv[2]) + + logger.info("[timing] main_total: %.1f ms", (time.perf_counter() - main_start) * 1000.0) + + # Emit the rolled face. For YOLO we must wait until *after* the dice has + # been released and settled — /api/topics/dice_number is just a cached + # latest detection, so reading it without a freshness check would report + # the face from before pickup (or a transient mid-lift detection). + if sys.argv[1] == "dice": + if is_yolo: + drop_time = pnp._dice_drop_time + if drop_time is None: + logger.warning( + "Dice drop_time not recorded; falling back to immediate read (value may be stale)." + ) + drop_time = 0.0 + value = _wait_for_rolled_dice_number(drop_time) + if value is None: + # YOLO never published a post-roll detection. Rather than + # leave the router blocked waiting for DICE_NUMBER (which + # would stall the whole turn), emit the latest cached value + # so the game can advance. We log a warning so the operator + # knows the reading may not reflect the true rolled face. + logger.warning( + "No fresh dice_number after drop — falling back to latest cached value" + ) + try: + resp = requests.get(f"{URL}/api/topics/dice_number", timeout=2.0) + if resp.ok: + cached = resp.json().get("value") + if cached is not None: + value = int(cached) + except Exception as exc: + logger.warning("dice_number fallback fetch failed: %s", exc) + if value is not None: + # Robot dice roll: rolled-face detection is the useful + # YOLO event — emit the overlay-close sentinel here, not + # after the pre-pickup position detection. The 0.5 s + # pause holds the YOLO dice frame on screen a beat longer + # so the operator can register the rolled face before the + # board reappears. + time.sleep(0.5) + print("YOLO_DETECTED", flush=True) + print(f"DICE_NUMBER={value}", flush=True) + logger.info("Detected rolled dice number: %s", value) + else: + logger.error( + "Unable to obtain any dice_number (drop_time=%.3f) — DICE_NUMBER not emitted", + drop_time, + ) + else: + value = random.randint(1, 6) + time.sleep(0.5) + print("YOLO_DETECTED", flush=True) + print(f"DICE_NUMBER={value}", flush=True) + logger.info("Sampled dice number: %s", value) + + +if __name__ == "__main__": + main() diff --git a/movensys_sample/movensys_robopoly/pyproject.toml b/movensys_sample/movensys_robopoly/pyproject.toml new file mode 100644 index 0000000..116b137 --- /dev/null +++ b/movensys_sample/movensys_robopoly/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" diff --git a/movensys_sample/movensys_robopoly/requirements.txt b/movensys_sample/movensys_robopoly/requirements.txt new file mode 100644 index 0000000..07b57fc --- /dev/null +++ b/movensys_sample/movensys_robopoly/requirements.txt @@ -0,0 +1,6 @@ +fastapi>=0.115,<1.0 +uvicorn[standard]>=0.30,<1.0 +httpx>=0.27,<1.0 +pydantic>=2.8,<3.0 +python-multipart>=0.0.9 +requests>=2.32,<3.0 diff --git a/movensys_sample/movensys_robopoly/router.py b/movensys_sample/movensys_robopoly/router.py new file mode 100644 index 0000000..cd482c1 --- /dev/null +++ b/movensys_sample/movensys_robopoly/router.py @@ -0,0 +1,1007 @@ +"""HTTP surface (PRD §5). + +Routes are thin: parse request, call GameManager, convert RuleError into +the PRD §4.7 error envelope. Business logic lives in game/*. +""" + +from __future__ import annotations + +import asyncio +import json as _json +import logging +import os +import re +from pathlib import Path +from typing import Any, Literal + +import httpx +import yaml +from fastapi import APIRouter, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi.responses import PlainTextResponse +from pydantic import BaseModel, Field + +ws_log = logging.getLogger("monopoly.ws") + +from game import RuleError + +api_router = APIRouter(prefix="/api") + + +# ---- health --------------------------------------------------------------- + + +@api_router.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} + + +@api_router.get("/robot/health") +async def robot_health(request: Request) -> dict[str, object]: + return request.app.state.robot_adapter.health() + + +@api_router.get("/stt/health") +async def stt_health(request: Request) -> dict[str, object]: + return request.app.state.stt_adapter.health() + + +@api_router.get("/vlm/health") +async def vlm_health(request: Request) -> dict[str, object]: + return request.app.state.vlm_adapter.health() + + +@api_router.get("/modes") +async def modes(request: Request) -> dict[str, dict[str, object]]: + return { + "stt": request.app.state.stt_adapter.health(), + "vlm": request.app.state.vlm_adapter.health(), + "robot": request.app.state.robot_adapter.health(), + } + + +# ---- request models -------------------------------------------------------- + + +class StartGameRequest(BaseModel): + board: Literal["final"] + + +class DiceSubmitRequest(BaseModel): + value: int | list[int] + source: Literal["manual", "rng", "robot"] = "manual" + + +class DiceRollRobotRequest(BaseModel): + is_YOLO: bool = True + # Caller's snapshot of state.turn_number when the chain was dispatched. + # Server refuses with STALE_TURN if the actual turn has advanced before + # the call lands — guards against multi-client races (browser auto-trigger + # racing the auto-play script, etc.). None disables the check. + expected_turn_number: int | None = None + + +class MoveApplyRequest(BaseModel): + player: Literal["user", "robot"] + from_tile: int = Field(ge=0) + to_tile: int = Field(ge=0) + + +class MoveApplyRobotRequest(MoveApplyRequest): + is_YOLO: bool = True + # Same stale-turn guard as DiceRollRobotRequest. + expected_turn_number: int | None = None + + +class ConfigPatch(BaseModel): + dice_source: Literal["manual", "rng", "robot"] | None = None + auctions_enabled: bool | None = None + income_tax_mode: Literal["fixed_200", "choose"] | None = None + player_colors: dict[str, str] | None = None + is_YOLO: bool | None = None + + +class DecideRequest(BaseModel): + action: Literal["skip", "buy", "build", "build_hotel"] + house_count: int = Field(default=0, ge=0, le=5) + + +class BuildRequest(BaseModel): + houses: int = Field(default=1, ge=0, le=4) + hotel: bool = False + + +class EffectRequest(BaseModel): + player: Literal["user", "robot"] + # Any additional keys go through as effect args + model_config = {"extra": "allow"} + + +# ---- game control --------------------------------------------------------- + + +@api_router.get("/game/state") +async def game_state(request: Request) -> dict[str, Any]: + return await request.app.state.game.snapshot() + + +@api_router.post("/game/start") +async def game_start(request: Request, body: StartGameRequest) -> dict[str, Any]: + try: + return await request.app.state.game.start_game(body.board) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +@api_router.post("/game/end_turn") +async def game_end_turn(request: Request) -> dict[str, Any]: + try: + return await request.app.state.game.end_turn() + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +@api_router.get("/game/winner") +async def game_winner(request: Request) -> dict[str, str | None]: + return {"winner": request.app.state.game.winner()} + + +@api_router.post("/game/config") +async def game_config(request: Request, body: ConfigPatch) -> dict[str, Any]: + patch = body.model_dump(exclude_none=True) + return await request.app.state.game.update_config(patch) + + +_SAVED_STATE_PATH = Path(__file__).resolve().parent / "saved_status.yaml" + + +@api_router.post("/game/save_state") +async def game_save_state(request: Request) -> dict[str, Any]: + # mode="json" so enums (FSM) and tuples become primitives PyYAML can handle. + snapshot = request.app.state.game.state.model_dump(mode="json") + try: + with open(_SAVED_STATE_PATH, "w", encoding="utf-8") as fh: + yaml.safe_dump(snapshot, fh, sort_keys=False, allow_unicode=True) + except OSError as exc: + raise HTTPException( + status_code=500, + detail={"code": "SAVE_FAILED", "message": str(exc)}, + ) + return {"path": str(_SAVED_STATE_PATH), "ok": True} + + +@api_router.post("/game/load_state") +async def game_load_state(request: Request) -> dict[str, Any]: + if not _SAVED_STATE_PATH.exists(): + raise HTTPException( + status_code=404, + detail={"code": "NO_SAVED_STATE", + "message": f"no saved state at {_SAVED_STATE_PATH}"}, + ) + try: + with open(_SAVED_STATE_PATH, "r", encoding="utf-8") as fh: + raw = yaml.safe_load(fh) + except (OSError, yaml.YAMLError) as exc: + raise HTTPException( + status_code=500, + detail={"code": "LOAD_FAILED", "message": str(exc)}, + ) + if not isinstance(raw, dict): + raise HTTPException( + status_code=400, + detail={"code": "BAD_SAVED_STATE", + "message": "saved_status.yaml is not a mapping"}, + ) + try: + new_state = await request.app.state.game.replace_state(raw) + except Exception as exc: + raise HTTPException( + status_code=400, + detail={"code": "INVALID_SAVED_STATE", "message": str(exc)}, + ) + return {"path": str(_SAVED_STATE_PATH), "ok": True, "state": new_state} + + +@api_router.get("/game/next_prompt") +async def game_next_prompt(request: Request) -> dict[str, str]: + # M1 stub; real hint generation lands with Gemma integration at M7. + state = request.app.state.game.state + return { + "hint": f"현재 {state.turn} 턴, FSM={state.fsm.value}. 다음 행동을 지시하세요." + } + + +_RULES_PATH = Path(__file__).resolve().parent / "doc" / "game_logic.md" + + +@api_router.get("/game/rules") +async def game_rules() -> "PlainTextResponse": + """Return the authoritative game spec as raw markdown. Used by the + VLM-player agent loop to seed the system prompt with the rules at + boot (see doc/vlm_as_player.md §3).""" + if not _RULES_PATH.exists(): + raise HTTPException( + status_code=404, + detail={"code": "NO_RULES_DOC", + "message": f"rules doc not found at {_RULES_PATH}"}, + ) + try: + text = _RULES_PATH.read_text(encoding="utf-8") + except OSError as exc: + raise HTTPException( + status_code=500, + detail={"code": "RULES_READ_FAILED", "message": str(exc)}, + ) + return PlainTextResponse(text, media_type="text/markdown") + + +# ---- dice & move ---------------------------------------------------------- + + +@api_router.post("/dice/request") +async def dice_request(request: Request) -> dict[str, Any]: + try: + return await request.app.state.game.request_dice() + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +# Default location of the robot pick-and-place script. It now lives next to +# this router under movensys_sample/movensys_robopoly/. +_DEFAULT_PNP_SCRIPT = Path(__file__).resolve().parent / "pick_and_place.py" +_DICE_LINE_RE = re.compile(rb"DICE_NUMBER=(\d+)") +# pick_and_place.py prints this sentinel on its own line right after +# get_piece_info / _search_for_target succeeds. The spawning route +# forwards it as a `yolo_detection_done` WS event so the frontend +# overlay closes early — operator sees the board during the physical +# pick-and-place instead of staring at a frozen YOLO frame. +_YOLO_DETECTED_SENTINEL = b"YOLO_DETECTED" + +# Board tile index → pick_and_place.py board_positions key. 14-tile board, +# counter-clockwise from GO at bottom-left (Board3_v2). +_TILE_INDEX_TO_BOARD_POS: dict[int, str] = { + 0: "GO", + 1: "BOSTON", + 2: "SEOUL", + 3: "DESERT_ISLAND", + 4: "ELECTRIC_COMPANY", + 5: "TAIPEI", + 6: "SHANGHAI", + 7: "NON-FREE_PARKING", + 8: "TOKYO", + 9: "BUSAN", + 10: "GO_TO_DESERT_ISLAND", + 11: "NEW_YORK", + 12: "CHANCE", + 13: "LONDON", +} +_PLAYER_TO_CUBE: dict[str, str] = {"user": "red_cube", "robot": "green_cube"} + + +async def _spawn_dice_subprocess( + request: Request, + body: DiceRollRobotRequest, + mode: str, + source: str, +) -> dict[str, Any]: + """Shared body for /dice/{roll,read}_robot. + + mode="roll" runs the full pick-and-drop chain (robot turn); + mode="read" only moves the arm to the dice scan pose so YOLO can see + the human-thrown face (user turn). `source` is forwarded to + game.submit_dice — "robot" or "manual". + """ + # Stale-turn guard: refuse before spawning the subprocess if the + # caller dispatched for a turn the server has already moved past + # (browser/script race, slow VLM call straddling end_turn, etc.). + if body.expected_turn_number is not None: + cur = request.app.state.game.state.turn_number + if cur != body.expected_turn_number: + raise HTTPException( + status_code=409, + detail={ + "code": "STALE_TURN", + "message": f"dice dispatched for turn {body.expected_turn_number} " + f"but server is on turn {cur}", + "expected": body.expected_turn_number, + "actual": cur, + }, + ) + script = Path(os.environ.get("MONOPOLY_PNP_SCRIPT", _DEFAULT_PNP_SCRIPT)) + if not script.exists(): + raise HTTPException( + status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", + "message": f"pick_and_place.py not found at {script}"}, + ) + + is_yolo_arg = "true" if body.is_YOLO else "false" + try: + proc = await asyncio.create_subprocess_exec( + "python3", str(script), "dice", "GO", is_yolo_arg, mode, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", "message": str(exc)}) + + dice_value: int | None = None + captured: list[bytes] = [] + yolo_emitted = False + bus = request.app.state.game.bus + assert proc.stdout is not None + while True: + line = await proc.stdout.readline() + if not line: + break + captured.append(line) + if not yolo_emitted and _YOLO_DETECTED_SENTINEL in line: + bus.publish_nowait("yolo_detection_done", {"kind": "dice"}) + yolo_emitted = True + m = _DICE_LINE_RE.search(line) + if m: + dice_value = int(m.group(1)) + break + + if dice_value is None: + await proc.wait() + stderr = b"" + if proc.stderr is not None: + try: + stderr = await proc.stderr.read() + except Exception: + pass + raise HTTPException( + status_code=502, + detail={ + "code": "DICE_NOT_DETECTED", + "message": "robot did not report a dice number", + "stdout": b"".join(captured).decode("utf-8", "replace"), + "stderr": stderr.decode("utf-8", "replace"), + }, + ) + + if not 1 <= dice_value <= 6: + asyncio.create_task(_drain_subprocess(proc)) + raise HTTPException( + status_code=502, + detail={"code": "DICE_INVALID", + "message": f"invalid dice value {dice_value} from robot"}, + ) + + # Detach: roll mode still has the physical pick-and-place finishing; + # read mode is essentially done. Either way, drain in background. + asyncio.create_task(_drain_subprocess(proc)) + + try: + result = await request.app.state.game.submit_dice(dice_value, source) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + result["dice_number"] = dice_value + return result + + +@api_router.post("/dice/read_robot") +async def dice_read_robot(request: Request, body: DiceRollRobotRequest) -> dict[str, Any]: + """User-turn dice path: the human has rolled the die by hand. The arm + moves to the dice scan pose so the camera has a clear view, YOLO is + read, and the value is submitted as source="manual". No pickup, no + drop — see pick_and_place._read_dice_only. + """ + return await _spawn_dice_subprocess(request, body, mode="read", source="manual") + + +@api_router.post("/dice/roll_robot") +async def dice_roll_robot(request: Request, body: DiceRollRobotRequest) -> dict[str, Any]: + """Robot-turn dice path: arm physically picks up, drops, and reads.""" + return await _spawn_dice_subprocess(request, body, mode="roll", source="robot") + + +# ---- chance card (VLM-driven) -------------------------------------------- + +# Four fixed chance-card outcomes the game supports. The VLM reads a +# physical card and the second VLM call maps the reading to one of +# these four — there are no other possible outcomes. +_CHANCE_CHOICES = ("-150", "0_sorry", "100", "200") +_CHANCE_DELTA: dict[str, int] = {"-150": -150, "0_sorry": 0, "100": 100, "200": 200} +_CHANCE_LABEL: dict[str, str] = { + "-150": "-$150", + "0_sorry": "$0 (sorry)", + "100": "+$100", + "200": "+$200", +} +# Token-conscious prompts: the orchestrator pays per token on both +# input AND output and we run this flow every chance card. Both prompts +# stay under ~80 tokens; max_tokens is capped hard in the request body. +_CHANCE_READ_PROMPT = ( + "Read the dollar amount on this chance card.\n" + "Reply with ONLY digits with a sign prefix and a dollar sign. " + 'Examples: "-$150", "$200", "$0", "$100".\n' + "NEVER spell numbers as words. NEVER add prose." +) +_CHANCE_SYSTEM_PROMPT = ( + "Pick ONE of four chance outcomes for the dollar amount in the user message:\n" + ' "-150" if the card is negative (player pays $150)\n' + ' "200" if the card is +$200\n' + ' "100" if the card is +$100\n' + ' "0_sorry" if the card is $0 or says sorry\n' + 'Reply ONLY: {"choice":"-150"|"0_sorry"|"100"|"200"}' +) +_CHANCE_POPUP_HOLD_S = 2.0 +# Regex fallback for the decision step: scan the raw reply for one of the +# four canonical tokens. Order matters — "-150" must beat "150". +_CHANCE_CHOICE_RE = re.compile(r"(-150|0_sorry|200|100|\bsorry\b|\b0\b)", re.IGNORECASE) + + +def _parse_chance_choice(raw: str) -> str: + """Extract one of the four allowed outcomes from a VLM reply. Tries + strict JSON first, then falls back to a regex over the raw text. + Defaults to "0_sorry" (no-op) if nothing matches, so the flow always + advances even when the model goes off-script. + """ + text = (raw or "").strip() + if not text: + return "0_sorry" + # Try strict JSON (handles ```json … ``` fences too) + fenced = re.sub(r"^\s*```(?:json)?\s*|\s*```\s*$", "", text, flags=re.IGNORECASE) + try: + obj = _json.loads(fenced) + choice = str(obj.get("choice", "")).strip() + if choice in _CHANCE_CHOICES: + return choice + except (_json.JSONDecodeError, AttributeError): + pass + # Regex fallback over the whole reply + m = _CHANCE_CHOICE_RE.search(text) + if m: + token = m.group(1).lower() + if token in _CHANCE_CHOICES: + return token + if token in ("sorry", "0"): + return "0_sorry" + return "0_sorry" + + +async def _run_chance_init_subprocess(is_yolo: bool) -> None: + """Spawn pick_and_place.py in chance_init mode: park the arm at the + cube-detection scan pose, sleep 2 s, exit. Raises HTTPException on + subprocess failure so the caller surfaces the same error envelope as + the other PnP routes. + """ + script = Path(os.environ.get("MONOPOLY_PNP_SCRIPT", _DEFAULT_PNP_SCRIPT)) + if not script.exists(): + raise HTTPException( + status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", + "message": f"pick_and_place.py not found at {script}"}, + ) + is_yolo_arg = "true" if is_yolo else "false" + # target_object is required by PnP.__init__ but the chance_init mode + # never touches the YOLO topic — any non-dice cube name works. Pick + # red_cube (the user's piece) so the validator passes. + try: + proc = await asyncio.create_subprocess_exec( + "python3", str(script), "red_cube", "GO", is_yolo_arg, "chance_init", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", "message": str(exc)}) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + raise HTTPException( + status_code=502, + detail={ + "code": "PNP_FAILED", + "message": f"chance_init pick_and_place exited with {proc.returncode}", + "stdout": stdout.decode("utf-8", "replace"), + "stderr": stderr.decode("utf-8", "replace"), + }, + ) + + +async def _vlm_infer( + *, prompt: str, camera: str, system_prompt: str | None = None, + max_tokens: int = 128, temperature: float = 0.0, +) -> str: + """POST {MOVENSYS_VLM_URL}/api/vlm/infer and return the raw response + text. The orchestrator proxies to vLLM (:9000); we keep the call + inline rather than going through VLMAdapter because the adapter's + system prompt is hard-coded to an intent classifier. + """ + base = os.environ.get("MOVENSYS_VLM_URL", "http://localhost:8000").strip() + if not base: + raise HTTPException( + status_code=503, + detail={"code": "VLM_OFFLINE", + "message": "MOVENSYS_VLM_URL is not set — VLM in stub mode"}, + ) + body: dict[str, Any] = { + "camera": camera, + "prompt": prompt, + "client": "robopoly_chance", + "max_tokens": max_tokens, + "temperature": temperature, + } + if system_prompt: + body["system_prompt"] = system_prompt + try: + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.post(f"{base}/api/vlm/infer", json=body) + r.raise_for_status() + data = r.json() + except httpx.HTTPError as exc: + raise HTTPException( + status_code=502, + detail={"code": "VLM_REQUEST_FAILED", "message": str(exc)}, + ) + if data.get("error"): + raise HTTPException( + status_code=502, + detail={"code": "VLM_ERROR", "message": str(data.get("error"))}, + ) + return str(data.get("response") or "").strip() + + +@api_router.post("/game/chance_card") +async def game_chance_card(request: Request) -> dict[str, Any]: + """Run the VLM-driven chance card flow for the current turn player. + + Fired either by (a) the tile-arrival deferred resolution (chance tile + 12) or (b) the frontend V-key hotkey on the user's TURN_START. The + flow is the same for both: + 1. arm → cube scan pose, settle 2 s + 2. VLM call 1 ("read this card") on the top camera frame + 3. publish chance_card_read, hold popup 2 s + 4. VLM call 2 — pick one of -100/+200/0/+100 from the read text + 5. apply the money delta to the current player's balance + 6. publish chance_card_applied, hold popup 2 s + """ + game = request.app.state.game + player = game.state.turn + is_yolo = bool(getattr(game.state.config, "is_YOLO", True)) + + # 1. Park the arm so the top camera has a clean view of the card. + await _run_chance_init_subprocess(is_yolo) + + # 2. VLM read — arm is at the cube-scan pose so the gripper-mounted + # ("hand") camera is the one pointing down at the card. max_tokens + # is tight (24) because the expected reply is at most 6 chars + # ("-$150"); anything longer is prose we'd discard anyway. + try: + read_text = await _vlm_infer( + prompt=_CHANCE_READ_PROMPT, + camera="hand", + max_tokens=24, + temperature=0.0, + ) + except HTTPException: + raise + if not read_text: + read_text = "(VLM returned no text for the card)" + + # 3. Tell the UI to flash the read text in the chat / overlay. + game.bus.publish_nowait("chance_card_read", { + "player": player, + "text": read_text, + "hold_s": _CHANCE_POPUP_HOLD_S, + }) + await asyncio.sleep(_CHANCE_POPUP_HOLD_S) + + # 4. Decision call — small system prompt forces a 1-of-4 outcome. + decision_prompt = ( + f"Read this card and consider it to the game.\n" + f"Card text: {read_text!r}\n" + f'Return JSON: {{"choice": "-100"|"+200"|"0"|"+100"}}.' + ) + decision_raw = await _vlm_infer( + prompt=decision_prompt, + camera="none", + system_prompt=_CHANCE_SYSTEM_PROMPT, + max_tokens=64, + temperature=0.0, + ) + choice = _parse_chance_choice(decision_raw) + delta = _CHANCE_DELTA[choice] + + # 5. Apply the money change to the current turn player's liquid balance. + # No bankruptcy chain here (the deltas are tiny relative to seed + # money / typical balances) — just clamp at 0 on a pay outcome. + async with game._lock: + p_state = game.state.players.get(player) + if p_state is not None: + if delta > 0: + p_state.balance += delta + elif delta < 0: + p_state.balance = max(0, p_state.balance + delta) + new_balance = p_state.balance if p_state is not None else 0 + + # 6. Result popup — frontend listens for chance_card_applied to flash + # "{player}: -$100" etc. and updates the money widget via state refresh. + game.bus.publish_nowait("chance_card_applied", { + "player": player, + "choice": choice, + "label": _CHANCE_LABEL[choice], + "delta": delta, + "balance": new_balance, + "card_text": read_text, + "raw_decision": decision_raw, + "hold_s": _CHANCE_POPUP_HOLD_S, + }) + await asyncio.sleep(_CHANCE_POPUP_HOLD_S) + + return { + "player": player, + "card_text": read_text, + "choice": choice, + "label": _CHANCE_LABEL[choice], + "delta": delta, + "balance": new_balance, + } + + +async def _drain_subprocess(proc: asyncio.subprocess.Process) -> None: + """Consume any remaining stdout/stderr and reap the process.""" + try: + if proc.stdout is not None: + while await proc.stdout.readline(): + pass + if proc.stderr is not None: + await proc.stderr.read() + await proc.wait() + except Exception: + ws_log.exception("drain pick_and_place subprocess failed") + + +@api_router.post("/dice/submit") +async def dice_submit(request: Request, body: DiceSubmitRequest) -> dict[str, Any]: + if isinstance(body.value, list): + if len(body.value) != 2: + raise HTTPException(status_code=400, detail="value must be int or [d1, d2]") + value: int | tuple[int, int] = (body.value[0], body.value[1]) + else: + value = body.value + try: + return await request.app.state.game.submit_dice(value, body.source) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +@api_router.post("/move/apply") +async def move_apply(request: Request, body: MoveApplyRequest) -> dict[str, Any]: + try: + return await request.app.state.game.apply_move(body.player, body.from_tile, body.to_tile) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +@api_router.post("/move/apply_robot") +async def move_apply_robot(request: Request, body: MoveApplyRobotRequest) -> dict[str, Any]: + # Stale-turn guard: refuse before spawning the subprocess if the + # caller dispatched for a turn the server has already moved past. + if body.expected_turn_number is not None: + cur = request.app.state.game.state.turn_number + if cur != body.expected_turn_number: + raise HTTPException( + status_code=409, + detail={ + "code": "STALE_TURN", + "message": f"apply_move dispatched for turn {body.expected_turn_number} " + f"but server is on turn {cur}", + "expected": body.expected_turn_number, + "actual": cur, + }, + ) + # Spawn pick_and_place.py in the background, + # then apply the game move. The HTTP response waits for the physical motion + # so the on-screen piece moves at the same moment as the robot. + board_pos = _TILE_INDEX_TO_BOARD_POS.get(body.to_tile) + cube = _PLAYER_TO_CUBE.get(body.player) + if board_pos is None or cube is None: + raise HTTPException( + status_code=400, + detail={"code": "TILE_UNMAPPED", + "message": f"no board_pos mapping for tile {body.to_tile} / player {body.player}"}, + ) + + script = Path(os.environ.get("MONOPOLY_PNP_SCRIPT", _DEFAULT_PNP_SCRIPT)) + if not script.exists(): + raise HTTPException( + status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", + "message": f"pick_and_place.py not found at {script}"}, + ) + + is_yolo_arg = "true" if body.is_YOLO else "false" + try: + proc = await asyncio.create_subprocess_exec( + "python3", str(script), cube, board_pos, is_yolo_arg, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", "message": str(exc)}) + + # Wait for the physical pick_and_place to finish before updating the game + # state. The frontend piece only moves once the robot is on its new tile. + # We read stdout line-by-line (instead of a single communicate()) so the + # YOLO_DETECTED sentinel can fire a WS event mid-subprocess and the + # frontend overlay closes the moment detection completes. + stdout_lines: list[bytes] = [] + yolo_emitted = False + bus = request.app.state.game.bus + assert proc.stdout is not None + while True: + line = await proc.stdout.readline() + if not line: + break + stdout_lines.append(line) + if not yolo_emitted and _YOLO_DETECTED_SENTINEL in line: + bus.publish_nowait("yolo_detection_done", {"kind": "cube"}) + yolo_emitted = True + await proc.wait() + stdout = b"".join(stdout_lines) + stderr = b"" + if proc.stderr is not None: + try: + stderr = await proc.stderr.read() + except Exception: + pass + if proc.returncode != 0: + raise HTTPException( + status_code=502, + detail={ + "code": "PNP_FAILED", + "message": f"pick_and_place exited with code {proc.returncode}", + "stdout": stdout.decode("utf-8", "replace"), + "stderr": stderr.decode("utf-8", "replace"), + }, + ) + + try: + result = await request.app.state.game.apply_move(body.player, body.from_tile, body.to_tile) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + result["robot"] = {"cube": cube, "board_pos": board_pos} + + # Spec §4.5.1: landing on GO_TO_DESERT_ISLAND teleports the player's + # position to DESERT_ISLAND in-engine; physically move the cube there + # too so the board state matches the game state. + jail_resolved = next( + (r for r in result.get("resolved", {}).get("tiles", []) + if r.get("kind") == "go_to_jail"), + None, + ) + if jail_resolved is not None: + jail_pnp = await _pick_and_place_to_jail(script, cube, body.is_YOLO) + result["robot_jail"] = jail_pnp + return result + + +async def _pick_and_place_to_jail( + script: Path, cube: str, is_yolo: bool, +) -> dict[str, Any]: + """Run pick_and_place.py DESERT_ISLAND for the §4.5.1 + auto-jail move. Raises HTTPException on subprocess failure so the + caller sees the same error envelope as the primary apply_robot path. + """ + is_yolo_arg = "true" if is_yolo else "false" + try: + proc = await asyncio.create_subprocess_exec( + "python3", str(script), cube, "DESERT_ISLAND", is_yolo_arg, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=500, + detail={"code": "SCRIPT_NOT_FOUND", "message": str(exc)}) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + raise HTTPException( + status_code=502, + detail={ + "code": "PNP_JAIL_FAILED", + "message": f"jail pick_and_place exited with code {proc.returncode}", + "stdout": stdout.decode("utf-8", "replace"), + "stderr": stderr.decode("utf-8", "replace"), + }, + ) + return {"cube": cube, "board_pos": "DESERT_ISLAND"} + + +# ---- property ------------------------------------------------------------- + + +@api_router.get("/properties") +async def properties(request: Request) -> list[dict[str, Any]]: + return request.app.state.game.list_properties() + + +@api_router.get("/properties/{pid}") +async def property_detail(request: Request, pid: str) -> dict[str, Any]: + cards = {c["id"]: c for c in request.app.state.game.list_properties()} + if pid not in cards: + raise HTTPException(status_code=404, + detail={"code": "NOT_FOUND", "message": f"unknown property {pid}"}) + return cards[pid] + + +@api_router.post("/properties/{pid}/decide") +async def property_decide(request: Request, pid: str, body: DecideRequest) -> dict[str, Any]: + game = request.app.state.game + player = game.state.turn + try: + return await game.decide_property(player, pid, body.action, body.house_count) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +@api_router.post("/properties/{pid}/buy") +async def property_buy(request: Request, pid: str) -> dict[str, Any]: + game = request.app.state.game + try: + return await game.buy_property(game.state.turn, pid) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +@api_router.post("/properties/{pid}/build") +async def property_build(request: Request, pid: str, body: BuildRequest) -> dict[str, Any]: + game = request.app.state.game + try: + return await game.build(game.state.turn, pid, houses=body.houses, hotel=body.hotel) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + + +# Voluntary mortgage / unmortgage / sell_building routes are intentionally +# removed (spec §5.1: no voluntary selling). The only sell path is +# auto-liquidation, which is internal to rules.py. + + +# ---- money --------------------------------------------------------------- + + +@api_router.get("/money") +async def money_snapshot(request: Request) -> dict[str, int]: + return request.app.state.game.money_snapshot() + + +@api_router.get("/money/{player}") +async def money_player(request: Request, player: Literal["user", "robot"]) -> dict[str, Any]: + snap = request.app.state.game.money_snapshot() + if player not in snap: + raise HTTPException(status_code=404, + detail={"code": "NOT_FOUND", "message": f"unknown player {player}"}) + return {"player": player, "balance": snap[player]} + + +# ---- effects (Chance/CC callable for debugging) -------------------------- + + +@api_router.post("/effects/{effect_type}") +async def effects_apply(request: Request, effect_type: str, body: EffectRequest) -> dict[str, Any]: + from game.effects import EffectError + game = request.app.state.game + payload = body.model_dump(exclude={"player"}) + payload["type"] = effect_type + try: + return await game.apply_card_effect(body.player, payload) + except RuleError as exc: + raise HTTPException(**_http_kwargs(exc)) + except EffectError as exc: + raise HTTPException(status_code=400, + detail={"code": exc.code, "message": str(exc)}) + + +# ---- WebSocket stream ------------------------------------------------------ + + +async def _stream_events(ws: WebSocket, name: str, type_filter: set[str] | None = None) -> None: + """Shared WS subscriber used by /stream/{game,board,money,properties}. + When `type_filter` is set the stream only forwards envelopes whose + `type` matches. /stream/game forwards everything (PRD §4.6).""" + await ws.accept() + bus = ws.app.state.event_bus + queue = bus.subscribe() + from game.events import make_envelope + await ws.send_json(make_envelope("hello", { + "stream": name, + "snapshot": ws.app.state.game.state.model_dump(), + })) + try: + while True: + event = await queue.get() + if type_filter is None or event["type"] in type_filter: + await ws.send_json(event) + except WebSocketDisconnect: + ws_log.info("stream_%s_disconnect", name) + except asyncio.CancelledError: + raise + except Exception: + ws_log.exception("stream_%s_error", name) + finally: + bus.unsubscribe(queue) + + +@api_router.websocket("/stream/game") +async def stream_game(ws: WebSocket) -> None: + await _stream_events(ws, "game") + + +@api_router.websocket("/stream/board") +async def stream_board(ws: WebSocket) -> None: + await _stream_events(ws, "board", {"move_applied", "lap_completed", "fsm_transition", + "game_started", "game_won"}) + + +@api_router.websocket("/stream/money") +async def stream_money(ws: WebSocket) -> None: + await _stream_events(ws, "money", {"effect_applied", "property_bought", + "property_built", "tier_sold", + "tile_rent_paid", "tile_tax_paid", + "tile_rent_bankruptcy", "tile_tax_bankruptcy"}) + + +@api_router.websocket("/stream/properties") +async def stream_properties(ws: WebSocket) -> None: + await _stream_events(ws, "properties", {"property_bought", "property_built", + "tier_sold"}) + + +# ---- YOLO debug image streams --------------------------------------------- +# +# The static UI swaps the board pane for these streams while a +# pick_and_place subprocess is in flight (see static/app.js +# `withYoloStream`). Images are sourced from the rclpy subscriber spun up +# in main.py lifespan; if rclpy is unavailable the latest frame stays +# None and the WS keeps sending {data: null, error: "No data"}. + +async def _stream_image(ws: WebSocket, attr: str, interval: float = 0.1) -> None: + import json as _json + await ws.accept() + try: + while True: + ros_image = getattr(ws.app.state, "ros_image", None) + data = getattr(ros_image, attr, None) if ros_image is not None else None + await ws.send_text(_json.dumps({ + "data": data, + "error": None if data is not None else "No frame yet", + })) + await asyncio.sleep(interval) + except WebSocketDisconnect: + return + except Exception: + ws_log.exception("stream_image_%s_error", attr) + + +@api_router.websocket("/stream/yolo_dice_detector/debug_image") +async def stream_yolo_dice_debug(ws: WebSocket) -> None: + await _stream_image(ws, "latest_dice_debug") + + +@api_router.websocket("/stream/yolo_cube_detector/debug_image") +async def stream_yolo_cube_debug(ws: WebSocket) -> None: + await _stream_image(ws, "latest_cube_debug") + + +@api_router.websocket("/stream/image_hand/rgb") +async def stream_image_hand_rgb(ws: WebSocket) -> None: + """Raw gripper-mounted camera RGB. Used by the chance-card overlay so + the operator can see the card the VLM is reading.""" + await _stream_image(ws, "latest_hand_rgb") + + +# ---- HTTPException helper -------------------------------------------------- + + +def _http_kwargs(exc: RuleError) -> dict[str, Any]: + """FastAPI's HTTPException flow cooperates with our error envelope middleware.""" + status = 409 if exc.code in ("TILE_MISMATCH", "INVALID_STATE", "PROPERTY_OWNED", + "NOT_OWNER", "INSUFFICIENT_FUNDS", + "JAIL_EXIT_UNAVAILABLE") else 400 + if exc.code == "NOT_FOUND": + status = 404 + return {"status_code": status, "detail": {"code": exc.code, "message": str(exc), "details": exc.details}} diff --git a/movensys_sample/movensys_robopoly/saved_status.yaml b/movensys_sample/movensys_robopoly/saved_status.yaml new file mode 100644 index 0000000..8912275 --- /dev/null +++ b/movensys_sample/movensys_robopoly/saved_status.yaml @@ -0,0 +1,102 @@ +board_id: final +fsm: TURN_START +turn: robot +turn_number: 2 +positions: + user: 5 + robot: 0 +players: + user: + id: user + balance: 700 + in_jail: false + jail_turns_left: 0 + has_jail_free_card: false + color: '#E53935' + robot: + id: robot + balance: 1000 + in_jail: false + jail_turns_left: 0 + has_jail_free_card: false + color: '#1E88E5' +properties: + boardfinal:suwon: + id: boardfinal:suwon + tile_index: 1 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:seoul: + id: boardfinal:seoul + tile_index: 2 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:electric_company: + id: boardfinal:electric_company + tile_index: 4 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:jeonju: + id: boardfinal:jeonju + tile_index: 5 + owner: user + houses: 0 + has_hotel: true + mortgaged: false + boardfinal:daejeon: + id: boardfinal:daejeon + tile_index: 6 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:gyeongju: + id: boardfinal:gyeongju + tile_index: 8 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:busan: + id: boardfinal:busan + tile_index: 9 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:daegu: + id: boardfinal:daegu + tile_index: 11 + owner: null + houses: 0 + has_hotel: false + mortgaged: false + boardfinal:bundang: + id: boardfinal:bundang + tile_index: 13 + owner: null + houses: 0 + has_hotel: false + mortgaged: false +last_dice: null +last_dice_sum: null +pending_dice: null +doubles_streak: 0 +lap_count: + user: 0 + robot: 0 +winner: null +config: + dice_source: rng + auctions_enabled: false + income_tax_mode: fixed_200 + player_colors: + user: '#E53935' + robot: '#1E88E5' + is_YOLO: false diff --git a/movensys_sample/movensys_robopoly/scripts/auto_play_dry_run.py b/movensys_sample/movensys_robopoly/scripts/auto_play_dry_run.py new file mode 100755 index 0000000..4d61506 --- /dev/null +++ b/movensys_sample/movensys_robopoly/scripts/auto_play_dry_run.py @@ -0,0 +1,729 @@ +#!/usr/bin/env python3 +"""Auto-play a full robopoly game end-to-end in dry-run mode. + +Drives both user and robot turns by calling the robopoly REST API on +``:7999`` directly, bypassing the VLM agent loop (which would otherwise +need ``movensys_vlm`` + vLLM on ``:8000`` to be running). On the user +side this is the same chain that ``"user just rolled the dice"`` would +trigger through the Ask-VLM textbox: ``/api/dice/read_robot`` (arm to +scan pose, YOLO read) → ``/api/move/apply_robot`` (cube to destination) +→ ``/api/properties/{pid}/decide`` (if a buyable tile pops the modal) +→ ``/api/game/end_turn``. + +Requirements: +- Start the robopoly server with ``MOVENSYS_PNP_DRY_RUN=1`` so + ``pick_and_place.py`` short-circuits to random dice + no arm motion. +- ``movensys_vlm`` does NOT need to be running. + +Decision policy: uniform random over the legal action set +({skip, buy, build, build_hotel}) gated by ``current_tier``, +``max_tier``, and the current player's liquid balance. + +Chance tiles: the VLM-driven ``/api/game/chance_card`` flow is skipped +(it needs ``:8000``). The rules engine left the FSM at RESOLVE_TILE so +``end_turn`` works; the ±$100/$200 delta simply isn't applied. Surfaced +in logs as ``chance tile: skipping VLM card flow``. + +Stops when the engine reaches GAME_OVER (bankruptcy per spec §6.1 or +the 5-lap cap per §6.2), or when ``--max-turns`` is hit. + +Concurrency: the script is tolerant of a browser tab open at +``localhost:7999`` racing it (the page auto-triggers robot turns through +the VLM). Each turn waits for FSM to stabilize at TURN_START; if the +dice value is already in the server (FSM=MOVING when we POSTed), we +adopt it and continue with apply_move. Close the tab for cleaner runs. + +Usage: + # Terminal A — start the server in dry-run. + cd movensys_sample/movensys_robopoly + MOVENSYS_PNP_DRY_RUN=1 python3 -m uvicorn main:app \\ + --host 127.0.0.1 --port 7999 + + # Terminal B — auto-play a full game. + python3 scripts/auto_play_dry_run.py --seed 42 +""" + +from __future__ import annotations + +import argparse +import base64 +import io +import json +import logging +import os +import random +import re +import sys +import time +from pathlib import Path +from typing import Any + +import requests + +try: + from PIL import Image # board image downscale + JPEG re-encode +except ImportError: + Image = None # type: ignore[assignment] + +DEFAULT_BASE = "http://localhost:7999" +DEFAULT_VLM_BASE = "http://localhost:8000" +TILE_COUNT = 14 # board_final.json: 14-tile counter-clockwise loop +LAND_PRICE = 100 # spec §4.1: Land tier (also utility flat price) +TIER_PRICE = 100 # spec §4.1.2: $100 per tier crossed on upgrade + +# Mirror of static/app.js VLM_PLAYER_SYSTEM_PROMPT — kept in sync by hand. +# Sent as system_prompt on every /api/vlm/infer call so the orchestrator +# doesn't need a prior PUT /api/vlm/system_prompt from the browser. +VLM_AGENT_SYSTEM_PROMPT = """You are an action-emitter agent for robopoly, a 2-player Monopoly-style +game. You ARE rolling the dice by emitting JSON — the code reads your +reply and drives the robot arm. Players: "user" (red), "robot" (you, green). + +OUTPUT: exactly one JSON object. No prose, no markdown, no fences. + +Valid actions: + fsm=="TURN_START": {"action":"roll_and_move","player":} + fsm=="AWAIT_DECISION": {"action":"decide","choice":} + +Choice meaning (cumulative cost from unowned = rent opponent pays): + buy tier 1, $100 land + build tier 2, $200 land + house + build_hotel tier 3, $300 land + hotel + skip no purchase +Upgrade delta from owned = $100 × (target_tier − current_tier). +Seed $1000, GO bonus $100, tax $100, chance ±$200, 5 laps to win. + +Constraints: +- decision_pending.kind=="utility" → only buy or skip are legal. +- build needs current_tier<2; build_hotel needs current_tier<3. +""" + +# Static board image used as VLM grounding. Mimics the browser's +# captureBoardImage() but without the live SVG piece overlay — the JSON +# state we attach carries authoritative positions. +_BOARD_PNG_PATH = ( + Path(__file__).resolve().parent.parent + / "static" / "assets" / "boards" / "board.png" +) +# Match BOARD_IMAGE_MAX_WIDTH / BOARD_IMAGE_JPEG_QUALITY in static/app.js. +# Keeps the payload tiny (~256 image tokens in Gemma 4) so VLM latency +# and token cost stay flat. +_BOARD_IMAGE_MAX_WIDTH = 468 +_BOARD_IMAGE_JPEG_QUALITY = 50 +_board_image_b64_cache: str | None = None + +# First balanced {...} block in a VLM reply — tolerant of code fences / +# leading prose. Mirrors parseVlmAction() in app.js. +_JSON_BLOCK_RE = re.compile(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}", re.DOTALL) + +log = logging.getLogger("auto_play") + + +class AutoPlayError(RuntimeError): + pass + + +class Client: + def __init__(self, base: str) -> None: + self.base = base.rstrip("/") + self.session = requests.Session() + + def get(self, path: str) -> dict[str, Any]: + r = self.session.get(f"{self.base}{path}", timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + # Long timeout because /dice/{read,roll}_robot waits on the + # pick_and_place subprocess. In dry-run it returns near-instantly, + # but we leave headroom for real-hardware reuse. + r = self.session.post(f"{self.base}{path}", json=payload or {}, timeout=180) + r.raise_for_status() + return r.json() + + +def _load_board_image_b64() -> str | None: + """Read board.png once, downscale to BOARD_IMAGE_MAX_WIDTH, re-encode + as JPEG at quality BOARD_IMAGE_JPEG_QUALITY, and cache the base64 + payload. Mirrors static/app.js captureBoardImage() — same width cap + and quality so the VLM sees the same token footprint regardless of + which client drove the inference. + """ + global _board_image_b64_cache + if _board_image_b64_cache is not None: + return _board_image_b64_cache + if not _BOARD_PNG_PATH.exists(): + log.warning("vlm: board.png not found at %s — sending camera=none with no image", + _BOARD_PNG_PATH) + return None + if Image is None: + log.warning("vlm: Pillow not installed; sending raw board.png " + "(no downscale, larger payload)") + _board_image_b64_cache = base64.b64encode(_BOARD_PNG_PATH.read_bytes()).decode() + return _board_image_b64_cache + with Image.open(_BOARD_PNG_PATH) as img: + img = img.convert("RGB") + w, h = img.size + if w > _BOARD_IMAGE_MAX_WIDTH: + new_h = round(h * _BOARD_IMAGE_MAX_WIDTH / w) + img = img.resize((_BOARD_IMAGE_MAX_WIDTH, new_h), Image.LANCZOS) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=_BOARD_IMAGE_JPEG_QUALITY) + _board_image_b64_cache = base64.b64encode(buf.getvalue()).decode() + log.info("vlm: board image %d bytes (%dx%d JPEG q=%d)", + len(_board_image_b64_cache), _BOARD_IMAGE_MAX_WIDTH, + round(h * _BOARD_IMAGE_MAX_WIDTH / w) if w > _BOARD_IMAGE_MAX_WIDTH else h, + _BOARD_IMAGE_JPEG_QUALITY) + return _board_image_b64_cache + + +def _build_vlm_state_summary(state: dict[str, Any]) -> dict[str, Any]: + """Mirror of buildVlmStateSummary() in app.js — compact state for + the agent prompt.""" + props_owned: dict[str, list[dict[str, Any]]] = {"user": [], "robot": []} + for _pid, p in (state.get("properties") or {}).items(): + owner = p.get("owner") + if owner not in props_owned: + continue + tier = 3 if p.get("has_hotel") else (2 if p.get("houses", 0) > 0 else 1) + props_owned[owner].append({ + "id": p["id"], "tile_index": p["tile_index"], "tier": tier, + }) + return { + "turn": state["turn"], + "fsm": state["fsm"], + "turn_number": state.get("turn_number"), + "positions": state["positions"], + "balances": {p: state["players"][p]["balance"] for p in ("user", "robot")}, + "lap_count": state.get("lap_count", {}), + "last_dice": state.get("last_dice"), + "last_dice_sum": state.get("last_dice_sum"), + "properties_owned": props_owned, + } + + +def _parse_vlm_action(raw: str) -> dict[str, Any] | None: + """Extract the first balanced {...} JSON object from a VLM reply.""" + if not raw: + return None + # Strip ```json ... ``` fences if present. + text = re.sub(r"^\s*```(?:json)?\s*|\s*```\s*$", "", raw.strip(), flags=re.IGNORECASE) + m = _JSON_BLOCK_RE.search(text) + if not m: + return None + try: + obj = json.loads(m.group(0)) + return obj if isinstance(obj, dict) else None + except json.JSONDecodeError: + return None + + +def vlm_infer(vlm_base: str, state: dict[str, Any], + user_message: str, *, decision: dict[str, Any] | None = None, + timeout_s: float = 30.0) -> dict[str, Any] | None: + """Call the orchestrator's /api/vlm/infer with the agent prompt + a + compact state JSON, parse the returned action. Returns None on any + failure — caller falls back to the deterministic policy. + """ + summary = _build_vlm_state_summary(state) + if decision is not None: + summary["decision_pending"] = { + "property_id": decision.get("property_id"), + "current_tier": decision.get("current_tier", 0), + "max_tier": decision.get("max_tier", 3), + "kind": decision.get("kind", "property"), + } + prompt = ( + f"{user_message}\n\n" + f"State:\n{json.dumps(summary, separators=(',', ':'))}\n\n" + "Your reply (ONE JSON object, nothing else):" + ) + body = { + "client": "robopoly", + "system_prompt": VLM_AGENT_SYSTEM_PROMPT, + "prompt": prompt, + "camera": "none", + "max_tokens": 64, + "temperature": 0.0, + } + image_b64 = _load_board_image_b64() + if image_b64: + body["image_b64"] = image_b64 + try: + r = requests.post(f"{vlm_base.rstrip('/')}/api/vlm/infer", + json=body, timeout=timeout_s) + r.raise_for_status() + resp = r.json() + except (requests.RequestException, ValueError) as exc: + log.warning(" vlm: infer failed: %s", exc) + return None + # Orchestrator returns the model text under various keys depending + # on version: "response", "text", "choices[0].message.content". + raw = (resp.get("response") or resp.get("text") + or _extract_chat_content(resp) or "") + action = _parse_vlm_action(raw) + if action is None: + log.warning(" vlm: unparseable reply: %r", raw[:200]) + return None + return action + + +def _extract_chat_content(resp: dict[str, Any]) -> str | None: + try: + return resp["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return None + + +def _choose_action(current_tier: int, max_tier: int, balance: int, + rng: random.Random) -> str: + """Uniformly pick an action from the legal set for this modal. + + Legal options gated by: + - max_tier (utility caps at 1, property caps at 3) + - current_tier (can't downgrade) + - liquid balance (must cover the delta) + """ + options = ["skip"] + if max_tier >= 1 and current_tier < 1 and balance >= LAND_PRICE: + options.append("buy") + if max_tier >= 2 and current_tier < 2 and balance >= (2 - current_tier) * TIER_PRICE: + options.append("build") + if max_tier >= 3 and current_tier < 3 and balance >= (3 - current_tier) * TIER_PRICE: + options.append("build_hotel") + return rng.choice(options) + + +def _handle_decision(client: Client, move_resp: dict[str, Any], + rng: random.Random, *, vlm_base: str | None = None) -> None: + """If the move resolution opened a buy modal, decide and submit it. + + When `vlm_base` is set AND the current turn is robot, ask the VLM + for the choice; fall back to the random policy on parse failure or + illegal output. + """ + for tile in move_resp.get("resolved", {}).get("tiles", []): + if not tile.get("needs_decision"): + continue + payload = tile.get("payload", {}) + pid = payload.get("property_id") + if pid is None: + log.warning("decision tile without property_id: %s", tile) + continue + current_tier = int(payload.get("current_tier", 0)) + max_tier = int(payload.get("max_tier", 1)) + state = client.get("/api/game/state") + balance = int(state["players"][state["turn"]]["balance"]) + action = None + if vlm_base and state["turn"] == "robot": + decision = { + "property_id": pid, + "current_tier": current_tier, + "max_tier": max_tier, + "kind": (payload.get("card") or {}).get("kind", "property"), + } + vlm_action = vlm_infer( + vlm_base, state, + "It's your turn (robot). Decide on this property tile.", + decision=decision, + ) + if vlm_action and vlm_action.get("action") == "decide": + choice = str(vlm_action.get("choice", "")).strip() + if choice in _legal_choices(current_tier, max_tier, balance): + action = choice + log.info(" vlm-decide: %s -> %s", pid, action) + else: + log.warning(" vlm-decide illegal/unsupported choice %r — falling back", + choice) + if action is None: + action = _choose_action(current_tier, max_tier, balance, rng) + log.info( + " decide: %s tier=%d/%d cash=$%d -> %s", + pid, current_tier, max_tier, balance, action, + ) + try: + client.post(f"/api/properties/{pid}/decide", {"action": action}) + except requests.HTTPError as exc: + # Race: another driver (browser deterministic robot picker, + # second tab) already resolved this decision. Confirm by + # re-reading state and continuing if the modal is gone. + body = exc.response.text if exc.response is not None else "" + if (exc.response is not None and exc.response.status_code == 409 + and "AWAIT_DECISION" not in body): + log.warning(" decide race (%s); already resolved by another client", + body.strip()) + return + raise + + +def _legal_choices(current_tier: int, max_tier: int, balance: int) -> list[str]: + """Compute the set of actions that pass the rules engine for this + tier/cap/cash. `skip` is always legal.""" + options = ["skip"] + if max_tier >= 1 and current_tier < 1 and balance >= LAND_PRICE: + options.append("buy") + if max_tier >= 2 and current_tier < 2 and balance >= (2 - current_tier) * TIER_PRICE: + options.append("build") + if max_tier >= 3 and current_tier < 3 and balance >= (3 - current_tier) * TIER_PRICE: + options.append("build_hotel") + return options + + +def _summarize(state: dict[str, Any]) -> str: + bal = state.get("players", {}) + pos = state.get("positions", {}) + laps = state.get("lap_count", {}) + return ( + f"u@{pos.get('user')} ${bal.get('user', {}).get('balance')} lap={laps.get('user', 0)} | " + f"r@{pos.get('robot')} ${bal.get('robot', {}).get('balance')} lap={laps.get('robot', 0)}" + ) + + +def _resolve_pending_decision(client: Client, state: dict[str, Any], + rng: random.Random) -> None: + """Resolve an AWAIT_DECISION that a racing client opened but won't + close (browser's user-turn buy modal never auto-clicks). Look up the + property at the current player's tile and submit a random decision. + """ + turn = state["turn"] + tile_index = state["positions"][turn] + # /api/properties returns every PropertyState plus its static Tile card. + props = client.get("/api/properties") + target = next((p for p in props if p.get("tile_index") == tile_index), None) + if target is None: + log.warning("no property at tile_index=%d for %s", tile_index, turn) + return + pid = target["id"] + # Tier: 3 if has_hotel, 2 if houses > 0, 1 if owned, else 0. + if target.get("has_hotel"): + current_tier = 3 + elif target.get("houses", 0) > 0: + current_tier = 2 + elif target.get("owner"): + current_tier = 1 + else: + current_tier = 0 + max_tier = 1 if target.get("kind") == "utility" else 3 + balance = int(state["players"][turn]["balance"]) + action = _choose_action(current_tier, max_tier, balance, rng) + log.info( + " race-decide: %s tier=%d/%d cash=$%d -> %s", + pid, current_tier, max_tier, balance, action, + ) + client.post(f"/api/properties/{pid}/decide", {"action": action}) + + +def _wait_for_turn_start(client: Client, rng: random.Random, + timeout_s: float = 90.0, + moving_recovery_after_s: float = 6.0) -> dict[str, Any]: + """Poll until FSM settles at TURN_START or GAME_OVER. Tolerates a + racing browser tab whose previous turn drive is still finishing, + and actively recovers from two stuck states: + + - **AWAIT_DECISION**: opens-and-leaves-open modal (browser auto-decides + only for robot; user-turn modals just sit there). We pick a legal + decision and submit it. + - **MOVING**: dice was submitted but apply_move never landed (racer's + chain crashed mid-step). After `moving_recovery_after_s` we take + over: compute to_tile from state and POST /api/move/apply_robot + ourselves so the game can advance. + """ + deadline = time.time() + timeout_s + last_fsm: str | None = None + moving_since: float | None = None + while time.time() < deadline: + state = client.get("/api/game/state") + fsm = state["fsm"] + if fsm in ("TURN_START", "GAME_OVER"): + return state + if fsm == "AWAIT_DECISION": + log.warning(" race stuck at AWAIT_DECISION; resolving") + _resolve_pending_decision(client, state, rng) + state2 = client.get("/api/game/state") + if state2["fsm"] in ("RESOLVE_TILE", "END_TURN"): + client.post("/api/game/end_turn") + moving_since = None + continue + if fsm == "END_TURN": + client.post("/api/game/end_turn") + moving_since = None + continue + if fsm == "MOVING": + if moving_since is None: + moving_since = time.time() + elif time.time() - moving_since >= moving_recovery_after_s: + _recover_from_stuck_moving(client, state) + moving_since = None + continue + else: + moving_since = None + if fsm != last_fsm: + log.info(" waiting for TURN_START (fsm=%s)", fsm) + last_fsm = fsm + time.sleep(0.5) + raise AutoPlayError(f"timed out waiting for TURN_START (stuck at fsm={last_fsm})") + + +def _recover_from_stuck_moving(client: Client, state: dict[str, Any]) -> None: + """Apply the pending move ourselves when a racer's chain wedged FSM + at MOVING. Reads state.turn / positions / last_dice_sum (already + adjusted for IN_JAIL skip by submit_dice) and POSTs apply_robot. + Best-effort — swallow errors, the outer loop will keep polling. + """ + turn = state["turn"] + from_tile = state["positions"][turn] + dice_sum = state.get("last_dice_sum") + if dice_sum is None: + log.warning(" stuck MOVING with no last_dice_sum; cannot recover") + return + to_tile = (from_tile + dice_sum) % TILE_COUNT + log.warning(" stuck MOVING — taking over apply_move(%s, %d→%d)", + turn, from_tile, to_tile) + try: + client.post("/api/move/apply_robot", { + "player": turn, + "from_tile": from_tile, + "to_tile": to_tile, + "is_YOLO": False, + "expected_turn_number": int(state.get("turn_number", 0)), + }) + except requests.HTTPError as exc: + body = exc.response.text if exc.response is not None else "" + log.warning(" moving-recovery apply_move failed: %s", body.strip()) + + +def play_one_turn(client: Client, rng: random.Random, *, + vlm_base: str | None = None) -> bool: + """Drive exactly one turn. Returns False once the game is over.""" + state = _wait_for_turn_start(client, rng) + if state["fsm"] == "GAME_OVER": + return False + turn = state["turn"] + turn_no = int(state.get("turn_number", 0)) + log.info("turn %s — %s [%s]", turn_no, turn, _summarize(state)) + + # 0. If VLM-mode is on and this is the robot's turn, gate the roll on + # a /api/vlm/infer call so the dice POST mirrors the browser's + # VLM-as-player loop (vlm_as_player.md §5). User turns stay on the + # direct REST path — they're the "human typed/spoke" branch. + if vlm_base and turn == "robot": + vlm_action = vlm_infer( + vlm_base, state, + "It's your turn (robot). Roll the dice and move your cube.", + ) + if vlm_action and vlm_action.get("action") == "roll_and_move": + log.info(" vlm: roll_and_move OK (player=%s)", + vlm_action.get("player")) + elif vlm_action is None: + log.warning(" vlm: no action returned — falling back to direct REST") + else: + log.warning(" vlm: unexpected action %s — falling back", + vlm_action.get("action")) + + # 1. Dice. User turn = read-only (no pickup); robot turn = full roll. + # Both short-circuit to random.randint(1, 6) under MOVENSYS_PNP_DRY_RUN. + # expected_turn_number guards against the browser's roll-and-move + # chain straddling a turn boundary (server returns 409 STALE_TURN). + dice_ep = "/api/dice/read_robot" if turn == "user" else "/api/dice/roll_robot" + try: + dice_resp = client.post(dice_ep, { + "is_YOLO": False, + "expected_turn_number": turn_no, + }) + dice_n = dice_resp.get("dice_number") + log.info(" rolled %s", dice_n) + except requests.HTTPError as exc: + # If a racing client already submitted the dice we land in + # fsm=MOVING (or further). Refresh state and adopt whatever's + # there rather than fighting. + body = exc.response.text if exc.response is not None else "" + if exc.response is not None and exc.response.status_code == 409 and "fsm=" in body: + log.warning(" dice race detected (%s); adopting server state", body.strip()) + else: + raise + + # Re-read state: jail-skip path (spec §4.5.2.2) leaves FSM at + # END_TURN with no move to apply. + state = client.get("/api/game/state") + if state["fsm"] == "END_TURN": + log.info(" jail-skipped — ending turn") + client.post("/api/game/end_turn") + return True + # If a racing client already ran apply_move, we may be past MOVING. + # Bail back to the outer loop — the TURN_START wait will catch up. + if state["fsm"] in ("RESOLVE_TILE", "AWAIT_DECISION", "GAME_OVER"): + log.warning(" race: another client is mid-turn (fsm=%s); skipping", state["fsm"]) + return state["fsm"] != "GAME_OVER" + + # 2. Apply move. The server already adjusted last_dice_sum for the + # IN_JAIL skip rule (spec §4.5.4), so we just trust it. + from_tile = state["positions"][turn] + dice_sum = state["last_dice_sum"] + to_tile = (from_tile + dice_sum) % TILE_COUNT + try: + move_resp = client.post("/api/move/apply_robot", { + "player": turn, + "from_tile": from_tile, + "to_tile": to_tile, + "is_YOLO": False, + "expected_turn_number": turn_no, + }) + except requests.HTTPError as exc: + body = exc.response.text if exc.response is not None else "" + if exc.response is not None and exc.response.status_code == 409: + log.warning(" apply_move race (%s); refreshing state", body.strip()) + move_resp = {"resolved": {"tiles": []}} + else: + raise + resolved = move_resp.get("resolved", {}).get("tiles", []) + if resolved: + log.info(" resolved: %s", + [f"{r.get('kind')}@{r.get('tile_index')}" for r in resolved]) + + # 3. Property decision modal (the only place the chain pauses). + _handle_decision(client, move_resp, rng, vlm_base=vlm_base) + + # 4. Bankruptcy-driven game over surfaces here (spec §6.1). + state = client.get("/api/game/state") + if state["fsm"] == "GAME_OVER": + return False + + # 5. Chance tile: the spec defers the money outcome to the VLM-driven + # /api/game/chance_card flow (router.py). Skip it in dry-run. + if any(t.get("kind") == "chance_drawn" for t in resolved): + log.info(" chance tile: skipping VLM card flow (dry-run, no orchestrator)") + + # 6. End the turn. Lap-cap check (spec §6.2) runs server-side here. + if state["fsm"] in ("RESOLVE_TILE", "END_TURN"): + client.post("/api/game/end_turn") + return True + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Auto-play a full robopoly game in dry-run mode.", + ) + parser.add_argument("--base", default=os.environ.get("ROBOPOLY_BASE", DEFAULT_BASE), + help=f"robopoly base URL (default: {DEFAULT_BASE})") + parser.add_argument("--seed", type=int, default=None, + help="seed the decision RNG (auto-derived per run when --runs > 1)") + parser.add_argument("--max-turns", type=int, default=300, + help="hard cap on turns played per game (safety net)") + parser.add_argument("--no-reset", action="store_true", + help="continue an in-progress game instead of starting fresh") + parser.add_argument("--delay", type=float, default=0.0, + help="seconds to sleep between turns (default: 0)") + parser.add_argument("--runs", type=int, default=1, + help="play N complete games back-to-back, " + "resetting between each (default: 1)") + parser.add_argument("--vlm", action="store_true", + help="drive robot turns through the orchestrator's " + "/api/vlm/infer (mirrors the browser's " + "VLM-as-player loop). User turns stay on the " + "direct REST path.") + parser.add_argument("--vlm-base", default=os.environ.get("VLM_BASE", DEFAULT_VLM_BASE), + help=f"orchestrator base URL for --vlm (default: {DEFAULT_VLM_BASE})") + parser.add_argument("--verbose", "-v", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)s: %(message)s", + datefmt="%H:%M:%S", + ) + + client = Client(args.base) + log.info("connecting to %s", client.base) + try: + client.get("/api/health") + except requests.RequestException as exc: + log.error("server not reachable at %s: %s", client.base, exc) + return 2 + + vlm_base = args.vlm_base.rstrip("/") if args.vlm else None + if vlm_base: + log.info("vlm: robot turns will hit %s/api/vlm/infer", vlm_base) + _load_board_image_b64() # warm cache, surface size/quality log line + + results: list[dict[str, Any]] = [] + overall_start = time.perf_counter() + for run_i in range(1, args.runs + 1): + # Per-run seed: deterministic across runs when --seed is set, but + # each run gets a distinct stream so the games aren't identical. + per_run_seed = (args.seed + run_i - 1) if args.seed is not None else None + rng = random.Random(per_run_seed) + if args.runs > 1: + log.info("========== run %d / %d (seed=%s) ==========", + run_i, args.runs, per_run_seed) + + if run_i > 1 or not args.no_reset: + log.info("starting fresh game (board=final)") + client.post("/api/game/start", {"board": "final"}) + + run_start = time.perf_counter() + outcome: dict[str, Any] = {"run": run_i, "seed": per_run_seed} + try: + for _turn_i in range(args.max_turns): + cont = play_one_turn(client, rng, vlm_base=vlm_base) + if not cont: + state = client.get("/api/game/state") + elapsed = time.perf_counter() - run_start + log.info( + "GAME OVER after %d turns in %.1fs — " + "winner=%s, balances=%s, laps=%s", + state.get("turn_number"), elapsed, state.get("winner"), + {p: state["players"][p]["balance"] + for p in ("user", "robot")}, + state.get("lap_count"), + ) + outcome.update({ + "ok": True, + "winner": state.get("winner"), + "turns": state.get("turn_number"), + "elapsed_s": elapsed, + }) + break + if args.delay: + time.sleep(args.delay) + else: + log.warning("max-turns=%d reached without a winner", + args.max_turns) + outcome.update({"ok": False, "reason": "max_turns"}) + except requests.HTTPError as exc: + body = exc.response.text if exc.response is not None else "" + log.error("HTTP error: %s — %s", exc, body) + outcome.update({"ok": False, "reason": f"http {exc}"}) + except AutoPlayError as exc: + log.error("auto-play aborted: %s", exc) + outcome.update({"ok": False, "reason": str(exc)}) + + results.append(outcome) + # Bail the multi-run loop on the first failure — the user wants + # to fix bugs before continuing. + if not outcome.get("ok"): + log.error("aborting --runs sweep at run %d/%d", run_i, args.runs) + break + + # Summary + if args.runs > 1: + ok_runs = [r for r in results if r.get("ok")] + log.info("==================== summary ====================") + log.info("completed %d / %d runs in %.1fs", + len(ok_runs), args.runs, time.perf_counter() - overall_start) + for r in results: + if r.get("ok"): + log.info( + " run %d: winner=%s in %d turns (%.1fs)", + r["run"], r["winner"], r["turns"], r["elapsed_s"], + ) + else: + log.info(" run %d: FAILED — %s", r["run"], r.get("reason")) + wins = {"user": 0, "robot": 0, None: 0} + for r in ok_runs: + wins[r.get("winner")] = wins.get(r.get("winner"), 0) + 1 + log.info(" wins: user=%d robot=%d draws=%d", + wins["user"], wins["robot"], wins[None]) + return 0 if all(r.get("ok") for r in results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/movensys_sample/movensys_robopoly/scripts/ci_local.sh b/movensys_sample/movensys_robopoly/scripts/ci_local.sh new file mode 100755 index 0000000..d8ea6a1 --- /dev/null +++ b/movensys_sample/movensys_robopoly/scripts/ci_local.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Local reproduction of .github/workflows/movensys-monopoly.yml. +# Keep in 1:1 lockstep with the workflow YAML (see PRD.md §14). +# +# MOVENSYS_VLM_URL is intentionally empty — proves stub-mode invariant. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +cd "$ROOT" + +export MOVENSYS_VLM_URL='' + +step() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +fail() { printf '\033[1;31mFAIL:\033[0m %s\n' "$*"; exit 1; } + +step "Install Python dependencies" +# Match CI: PEP 668 (Ubuntu 24.04) requires --break-system-packages when not in a venv. +PIP_FLAGS=(--no-cache-dir) +if [ -z "${VIRTUAL_ENV:-}" ] && python3 -c "import sys; sys.exit(0 if sys.base_prefix == sys.prefix else 1)"; then + PIP_FLAGS+=(--break-system-packages) +fi +[ -f requirements.txt ] && pip3 install "${PIP_FLAGS[@]}" -r requirements.txt +pip3 install "${PIP_FLAGS[@]}" pytest pytest-asyncio httpx uvicorn fastapi + +step "Python syntax check" +mapfile -t files < <(find . -type f -name '*.py' \ + -not -path './.venv/*' -not -path './build/*' -not -path './__pycache__/*') +for f in "${files[@]}"; do + echo " $f"; python3 -m py_compile "$f" +done + +step "FastAPI health + stub-mode invariant" +if [ -f main.py ]; then + # Pre-flight: refuse to start if port 7999 is already taken — a stale + # uvicorn from a previous run would happily answer /api/health with + # old code and hide regressions. + if ss -ltn "sport = :7999" 2>/dev/null | grep -q ':7999'; then + fail "port 7999 already in use — stop the other uvicorn before rerunning" + fi + + # Own process group so cleanup can sweep any workers uvicorn may spawn, + # not just the direct child. + set -m + python3 -m uvicorn main:app --host 127.0.0.1 --port 7999 & + pid=$! + set +m + + cleanup() { + # TERM the whole process group, then verify nothing survived. Any + # lingering uvicorn on :7999 after this script exits is a bug. + kill -TERM -"$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + if pgrep -f "uvicorn.*main:app.*--port 7999" >/dev/null 2>&1; then + echo " warning: uvicorn survived cleanup — killing -9" >&2 + pkill -9 -f "uvicorn.*main:app.*--port 7999" || true + fi + } + trap cleanup EXIT INT TERM + + for _ in $(seq 1 20); do + curl -sf http://127.0.0.1:7999/api/health >/dev/null 2>&1 && break + sleep 1 + done + body=$(curl -sf http://127.0.0.1:7999/api/health) || fail "/api/health did not respond" + echo " health: $body" + echo "$body" | grep -q '"status"' || fail "health missing status" + + robot=$(curl -sf http://127.0.0.1:7999/api/robot/health) || fail "/api/robot/health did not respond" + echo " robot: $robot" + echo "$robot" | grep -Eq '"mode"[[:space:]]*:[[:space:]]*"stub"' \ + || fail "adapter not in stub mode with empty MOVENSYS_VLM_URL" + + cleanup + trap - EXIT INT TERM +else + echo " main.py not present — skipped" +fi + +step "Pytest" +if [ -d tests ]; then + rc=0 + python3 -m pytest -v tests/ || rc=$? + # Exit code 5 = "no tests collected" — acceptable at early milestones. + if [ "$rc" != "0" ] && [ "$rc" != "5" ]; then exit "$rc"; fi +else + echo " tests/ not present — nothing to run" +fi + +printf '\n\033[1;32m==> ci_local passed\033[0m\n' diff --git a/movensys_sample/movensys_robopoly/scripts/render_cards.py b/movensys_sample/movensys_robopoly/scripts/render_cards.py new file mode 100755 index 0000000..08c6652 --- /dev/null +++ b/movensys_sample/movensys_robopoly/scripts/render_cards.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Render property/railroad/utility SVG cards from a board JSON definition. + +Usage: + python3 scripts/render_cards.py static/assets/boards/board2.json \ + --out static/assets/property_cards/board2/ + +Reads board JSON (§4.4 of PRD.md), stamps templates, writes one SVG per tile. +Templates live next to this script's output dir: + static/assets/property_cards/{template,_railroad_template,_utility_template}.svg +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +ASSETS = Path(__file__).resolve().parent.parent / "static" / "assets" / "property_cards" +SLUG_RE = re.compile(r"[^a-z0-9]+") +COMMENT_RE = re.compile(r"", re.DOTALL) + + +def strip_comments(svg: str) -> str: + """Drop template doc comments so placeholder text isn't accidentally rendered.""" + return COMMENT_RE.sub("", svg) + + +def slug(name: str) -> str: + return SLUG_RE.sub("_", name.lower()).strip("_") + + +def fmt(n: int) -> str: + return f"${n}" + + +def render_property(tile: dict, color_hex: str, tmpl: str) -> str: + r = tile["rent_table"] + replacements = { + "NAME": tile["name"], + "COLOR_HEX": color_hex, + "PRICE_BUY": fmt(tile["price_buy"]), + "RENT_BASE": fmt(r[0]), + "RENT_H1": fmt(r[1]), + "RENT_H2": fmt(r[2]), + "RENT_H3": fmt(r[3]), + "RENT_H4": fmt(r[4]), + "RENT_HOTEL": fmt(r[5]), + "PRICE_HOUSE": fmt(tile["price_building"]), + "PRICE_HOTEL": f'{fmt(tile["price_building"])} + 4 houses', + } + out = tmpl + for k, v in replacements.items(): + out = out.replace("{{" + k + "}}", v) + return out + + +def render_railroad(tile: dict, tmpl: str) -> str: + return tmpl.replace("{{NAME}}", tile["name"]) + + +def render_utility(tile: dict, tmpl: str) -> str: + return tmpl.replace("{{NAME}}", tile["name"]) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("board_json") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + board = json.loads(Path(args.board_json).read_text()) + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + + prop_tmpl = strip_comments((ASSETS / "template.svg").read_text()) + rail_tmpl = strip_comments((ASSETS / "_railroad_template.svg").read_text()) + util_tmpl = strip_comments((ASSETS / "_utility_template.svg").read_text()) + colors = board.get("color_group_hex", {}) + + count = 0 + for tile in board["tiles"]: + kind = tile["kind"] + name = tile["name"] + if kind == "property": + color = colors.get(tile.get("color_group"), "#888") + svg = render_property(tile, color, prop_tmpl) + elif kind == "railroad": + svg = render_railroad(tile, rail_tmpl) + elif kind == "utility": + svg = render_utility(tile, util_tmpl) + else: + continue + (out_dir / f"{slug(name)}.svg").write_text(svg) + count += 1 + + print(f"wrote {count} card SVGs to {out_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/movensys_sample/movensys_robopoly/scripts/render_whiteboard_png.py b/movensys_sample/movensys_robopoly/scripts/render_whiteboard_png.py new file mode 100644 index 0000000..12e88e2 --- /dev/null +++ b/movensys_sample/movensys_robopoly/scripts/render_whiteboard_png.py @@ -0,0 +1,126 @@ +"""Regenerate the printable whiteboard PNGs under boards/. + +Three themes (Korea, USA, numbers) rendered as 1800x3000 portrait PNGs. +The board is drawn landscape (5x3 perimeter, matching board3_blank.svg) +and then rotated 90° counter-clockwise. +""" +from __future__ import annotations + +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFont + +ROOT = Path(__file__).resolve().parent.parent +OUT_DIR = ROOT / "boards" + +CELL = 600 +COLS, ROWS = 5, 3 +W, H = COLS * CELL, ROWS * CELL # 3000 x 1800 landscape + +FONT_PATH = "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf" + +# Perimeter walked clockwise from START in the landscape coord frame. +PERIMETER = [ + (0, 0), (1, 0), (2, 0), (3, 0), (4, 0), + (4, 1), + (4, 2), (3, 2), (2, 2), (1, 2), (0, 2), + (0, 1), +] + +KOREA = ["START", "Incheon", "Suwon", "Daejeon", "Jeonju", + "Gwangju", + "Busan", "Ulsan", "Daegu", "Gyeongju", "Gangneung", + "Chuncheon"] + +USA = ["START", "New York", "Boston", "Philadelphia", "Washington", + "Atlanta", + "Miami", "Houston", "Dallas", "Denver", "Los Angeles", + "Chicago"] + +NUMBERS = ["START"] + [str(i) for i in range(1, 12)] + +LINE_COLOR = (17, 17, 17) +TEXT_COLOR = (17, 17, 17) +BG = (255, 255, 255) +INNER_BG = (250, 250, 250) +LINE_W = 14 +OUTER_LINE_W = LINE_W + 6 + + +def fit_font(text: str, max_width: int, base_size: int) -> ImageFont.FreeTypeFont: + size = base_size + while size > 30: + font = ImageFont.truetype(FONT_PATH, size) + bbox = font.getbbox(text) + if (bbox[2] - bbox[0]) <= max_width: + return font + size -= 6 + return ImageFont.truetype(FONT_PATH, size) + + +VPAD = 40 # padding from cell edge for "up"/"down" alignment + + +def render( + labels: list[str], + base_font_size: int, + out_path: Path, + align: str = "middle", + rotate: bool = True, +) -> None: + assert len(labels) == 12, "expected 12 perimeter labels" + assert align in ("up", "middle", "down") + img = Image.new("RGB", (W, H), BG) + draw = ImageDraw.Draw(img) + + inner = (CELL, CELL, 4 * CELL, 2 * CELL) + draw.rectangle(inner, fill=INNER_BG) + + for (c, r) in PERIMETER: + x0, y0 = c * CELL, r * CELL + draw.rectangle((x0, y0, x0 + CELL, y0 + CELL), + outline=LINE_COLOR, width=LINE_W) + + draw.rectangle(inner, outline=LINE_COLOR, width=LINE_W) + draw.rectangle((0, 0, W - 1, H - 1), + outline=LINE_COLOR, width=OUTER_LINE_W) + + for (c, r), label in zip(PERIMETER, labels): + x0, y0 = c * CELL, r * CELL + cx = x0 + CELL // 2 + font = fit_font(label, CELL - 80, base_font_size) + bb = draw.textbbox((0, 0), label, font=font) + bw, bh = bb[2] - bb[0], bb[3] - bb[1] + if align == "up": + ty = y0 + VPAD - bb[1] + elif align == "down": + ty = y0 + CELL - VPAD - bh - bb[1] + else: + ty = y0 + CELL // 2 - bh // 2 - bb[1] + draw.text((cx - bw // 2 - bb[0], ty), + label, font=font, fill=TEXT_COLOR) + + out = img.rotate(90, expand=True) if rotate else img + out.save(out_path, format="PNG", optimize=True) + print(f"wrote {out_path} ({out.size[0]}x{out.size[1]})") + + +def main() -> None: + themes = [ + ("korea", KOREA, 150), + ("usa", USA, 130), + ("w_number", NUMBERS, 280), + ] + for name, labels, size in themes: + for align in ("up", "middle", "down"): + render( + labels, + size, + OUT_DIR / f"whiteboard_{name}_{align}.png", + align=align, + rotate=False, + ) + + +if __name__ == "__main__": + main() diff --git a/movensys_sample/movensys_robopoly/static/app.css b/movensys_sample/movensys_robopoly/static/app.css new file mode 100644 index 0000000..5345318 --- /dev/null +++ b/movensys_sample/movensys_robopoly/static/app.css @@ -0,0 +1,606 @@ +:root { + --bg: #1a1a1a; + --panel: #242424; + --panel-light: #303030; + --fg: #e8e8e8; + --muted: #888; + --accent: #4caf50; + --off: #555; + --warn: #ff9800; + --user: #EF5350; + --robot: #2E7D32; + --border: #333; +} +* { box-sizing: border-box; } +html, body { margin: 0; padding: 0; background: var(--bg); color: var(--fg); + font: 14px/1.5 "Inter", system-ui, -apple-system, sans-serif; } + +header { + padding: 10px 16px; border-bottom: 1px solid var(--border); + display: flex; align-items: center; gap: 14px; flex-wrap: wrap; +} +header h1 { margin: 0; font-size: 16px; font-weight: 600; } +header .spacer { flex: 1; } +.badges { display: flex; gap: 6px; } +.badge { + display: inline-flex; align-items: center; gap: 6px; + padding: 3px 9px; border-radius: 12px; + background: var(--panel); color: var(--muted); + font-size: 11px; font-family: "JetBrains Mono", ui-monospace, monospace; +} +.badge .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--off); } +.badge.live .dot, .badge.on .dot { background: var(--accent); } +.badge.stub .dot, .badge.off .dot { background: var(--off); } + +.mode-toggle-badge { + border: 1px solid var(--border); cursor: pointer; + font-weight: 600; color: var(--fg); background: var(--panel-light); +} +.mode-toggle-badge:hover { background: var(--panel); } + +/* ---- Game mode layout --------------------------------------------------- + Hide debug-only panels and let the board fill the freed columns. The + mic device is configured in debug mode; in game mode it's reused + silently (Z/X hotkeys still work) so the mic-bay UI is hidden. */ +body.game-mode .grid { grid-template-columns: 1fr; } +body.game-mode .sidebar-right { display: none; } +body.game-mode .sidebar-left { display: none; } +body.game-mode .vlm-header > :not(h2) { display: none; } +body.game-mode .vlm-controls { display: none; } +body.game-mode .vlm-mic-bay { display: none; } +body.game-mode .vlm-col-prompt { display: none; } +body.game-mode .vlm-col-header { display: none; } +body.game-mode #board-coords { display: none; } +body.game-mode .ownership-group.unowned { display: none; } +body.game-mode #notification { display: none; } +body.game-mode .board-pane-label { display: none; } +body.game-mode .money-bar { display: none; } +body.game-mode .board-wrap { width: 85%; margin: 0 auto; } + +/* Z/X 핫키로 켜는 풀스크린 ASK VLM 오버레이. status 오버레이가 뜨면 + 자동으로 닫힌다. ESC로도 닫을 수 있다. */ +body.game-mode.chat-overlay-active .sidebar-left { + display: flex; + position: fixed; inset: 0; z-index: 1400; + background: rgba(10, 10, 10, 0.94); + padding: 4vh 6vw; + overflow: hidden; +} +body.game-mode.chat-overlay-active .vlm-section { + flex: 1; display: flex; flex-direction: column; + max-width: 1200px; margin: 0 auto; + background: transparent; +} +body.game-mode.chat-overlay-active .vlm-body { flex: 1; min-height: 0; } +body.game-mode.chat-overlay-active .vlm-col-query { + flex: 1; display: flex; flex-direction: column; min-height: 0; +} +body.game-mode.chat-overlay-active #vlm-chat { + flex: 1; max-height: none; font-size: 15px; +} +body.game-mode.chat-overlay-active .vlm-header h2 { + font-size: 14px; +} +.vlm-hotkey-hint-game { display: none; } +body.game-mode .vlm-hotkey-hint-game { display: inline; } +body.game-mode .vlm-hotkey-hint-debug { display: none; } + +/* Centered status flash for game mode. Hidden by default; .visible + fades it in with a dimmed backdrop for ~1s on each announce. */ +.game-overlay { + position: fixed; inset: 0; z-index: 1000; + display: flex; align-items: center; justify-content: center; + background: rgba(0, 0, 0, 0.62); + opacity: 0; pointer-events: none; + transition: opacity 180ms ease-in-out; +} +.game-overlay.visible { opacity: 1; } +.game-overlay-text { + color: #fff; + font-family: "Inter", system-ui, -apple-system, sans-serif; + font-weight: 700; + font-size: clamp(34px, 5.4vw, 77px); + text-align: center; + padding: 0 32px; + max-width: 86%; + text-shadow: 0 2px 10px rgba(0, 0, 0, 0.45); + line-height: 1.2; + white-space: pre-line; +} + +/* money bar — sits below the game board */ +.money-bar { + display: flex; gap: 10px; justify-content: center; + margin-top: 8px; +} + +/* Human-readable announcement below the board (turn changes, buys, etc.) */ +.notification { + margin-top: 12px; + padding: 42px 54px; + background: var(--panel); + border-radius: 10px; + border-left: 5px solid var(--muted); + font-size: 48px; font-weight: 600; + font-family: "JetBrains Mono", ui-monospace, monospace; + min-height: 144px; + display: flex; align-items: center; + transition: border-color 200ms, background 200ms; +} +.notification.empty { color: var(--muted); font-weight: 400; font-style: italic; } +.notification.kind-turn { border-left-color: var(--accent); } +.notification.kind-dice { border-left-color: #c084fc; } +.notification.kind-move { border-left-color: #60a5fa; } +.notification.kind-buy { border-left-color: #fbbf24; } +.notification.kind-build { border-left-color: #34d399; } +.notification.kind-money { border-left-color: #f97316; } +.notification.kind-win { border-left-color: #f87171; color: #fcd34d; font-size: 54px; } +.money-player { + display: inline-flex; align-items: center; gap: 12px; + background: var(--panel); padding: 12px 20px; border-radius: 12px; + font-family: "JetBrains Mono", ui-monospace, monospace; +} +.money-color { width: 18px; height: 18px; border-radius: 50%; } +.money-label { color: var(--muted); font-size: 14px; min-width: 48px; } +.money-stack { display: flex; flex-direction: column; gap: 2px; } +.money-row { display: inline-flex; align-items: baseline; gap: 8px; } +.money-row-label { + font-size: 10px; color: var(--muted); + text-transform: uppercase; letter-spacing: 0.05em; + min-width: 48px; +} +.money-assets { + font-size: 14px; font-weight: 600; color: var(--muted); + font-family: "JetBrains Mono", ui-monospace, monospace; +} +.money-value { + font-size: 24px; font-weight: 700; + transition: color 300ms; +} +.money-value.flash-up { color: var(--accent); } +.money-value.flash-down { color: var(--user); } + +.grid { + display: grid; + grid-template-columns: 360px 1fr 320px; + gap: 12px; padding: 12px; + max-width: 1800px; margin: 0 auto; +} +.sidebar { display: flex; flex-direction: column; gap: 12px; min-width: 0; } +.sidebar-left { order: 0; } +.sidebar-right { order: 2; } + +.card { + background: var(--panel); + border-radius: 10px; + padding: 12px 14px; +} +.card h2 { + margin: 0 0 10px 0; + font-size: 11px; text-transform: uppercase; letter-spacing: 0.05em; + color: var(--muted); +} + +dl { display: grid; grid-template-columns: auto 1fr; gap: 3px 12px; margin: 0; + font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 13px; } +dt { color: var(--muted); } +dd { margin: 0; } +dd.yolo-row { display: flex; gap: 8px; align-items: center; } +.btn-inline { padding: 2px 8px; font-size: 12px; line-height: 1.2; } + +/* Game state: data on the left, SAVE/LOAD stacked on the right */ +.state-body { + display: flex; align-items: flex-start; gap: 12px; +} +.state-body > dl { flex: 1; min-width: 0; } +.state-actions { + display: flex; flex-direction: column; gap: 6px; + flex-shrink: 0; +} +.state-actions > button { min-width: 64px; } + +/* Dice Status: left buttons | dice face | right buttons */ +.dice-row { + display: flex; align-items: center; gap: 10px; +} +.dice-row > .dice-face { margin: 0; flex-shrink: 0; } +.dice-actions-col { + flex: 1; min-width: 0; + display: flex; flex-direction: column; gap: 6px; +} +.dice-actions-col > button { width: 100%; } + +.row { display: flex; gap: 6px; align-items: end; margin-bottom: 8px; } +.row:last-child { margin-bottom: 0; } +.row label { display: flex; flex-direction: column; gap: 2px; font-size: 11px; + color: var(--muted); flex: 1; } +select, input[type=number] { + background: var(--panel-light); color: var(--fg); + border: 1px solid var(--border); border-radius: 4px; + padding: 5px 8px; font: inherit; width: 100%; +} +button { + background: var(--panel-light); color: var(--fg); + border: 1px solid var(--border); border-radius: 4px; + padding: 6px 12px; font: inherit; cursor: pointer; + transition: background 120ms; +} +button:hover:not(:disabled) { background: #3a3a3a; } +button:disabled { opacity: 0.4; cursor: not-allowed; } +.btn-secondary { background: transparent; } + +/* dice face */ +.dice-face { + width: 84px; height: 84px; + margin: 4px auto 12px; + background: #D2B48C; + border: 1px solid #000; + border-radius: 14px; + display: grid; + grid-template-columns: repeat(3, 1fr); + grid-template-rows: repeat(3, 1fr); + padding: 12px; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.4); +} +.dice-face.empty { + background: var(--panel-light); + box-shadow: none; + border-color: var(--border); +} +.dice-pip { + background: #1a1a1a; + border-radius: 50%; + width: 12px; height: 12px; + align-self: center; + justify-self: center; +} + +/* board */ +.board-pane { + background: var(--panel); + border-radius: 10px; + padding: 12px; + display: flex; flex-direction: column; align-items: stretch; justify-content: flex-start; + gap: 8px; + align-self: start; +} +.board-pane-label { + font-size: 11px; text-transform: uppercase; letter-spacing: 0.05em; + color: var(--muted); + font-family: "JetBrains Mono", ui-monospace, monospace; +} +.board-wrap { + position: relative; + width: 100%; + aspect-ratio: 1559/794; +} +#board-host { + width: 100%; height: 100%; + background: var(--panel); border-radius: 6px; overflow: hidden; + display: flex; align-items: center; justify-content: center; +} +#board-host svg, #board-host img { width: 100%; height: 100%; object-fit: contain; display: block; } +#pieces { + position: absolute; inset: 0; + width: 100%; height: 100%; + pointer-events: none; +} +#pieces rect { transition: x 500ms ease-out, y 500ms ease-out; pointer-events: auto; cursor: grab; } +#pieces rect.dragging { cursor: grabbing; transition: none; } + +.yolo-overlay { + position: absolute; inset: 0; z-index: 6; + background: rgba(0, 0, 0, 0.9); + border-radius: 6px; + display: flex; flex-direction: column; align-items: center; justify-content: center; + gap: 8px; + padding: 12px; +} +.yolo-overlay.hidden { display: none; } +.yolo-overlay-label { + font-family: "JetBrains Mono", ui-monospace, monospace; + font-size: 12px; color: var(--accent); + letter-spacing: 0.04em; text-transform: uppercase; +} +.yolo-overlay-status { + font-family: "JetBrains Mono", ui-monospace, monospace; + font-size: 11px; color: var(--muted); +} +.yolo-overlay img { + max-width: 100%; + max-height: calc(100% - 56px); + object-fit: contain; + border-radius: 4px; + background: #000; + transform: rotate(180deg); +} +.board-coords { + font-family: "JetBrains Mono", ui-monospace, monospace; + font-size: 11px; + color: var(--muted); + text-align: center; + user-select: text; + white-space: pre-line; + line-height: 1.45; +} + + +/* event log */ +#event-log { + list-style: none; padding: 0; margin: 0; + max-height: 240px; overflow-y: auto; + font-family: "JetBrains Mono", ui-monospace, monospace; font-size: 11px; +} +#event-log li { + padding: 3px 6px; border-bottom: 1px solid var(--border); + color: var(--fg); +} +#event-log li.type-fsm_transition { color: #bdbdbd; } +#event-log li.type-game_won { color: var(--accent); font-weight: 600; } +#event-log li.type-lap_completed { color: var(--warn); } +#event-log li.type-tile_rent_paid { color: #ffab91; } +#event-log li.type-tile_rent_bankruptcy { color: var(--user); font-weight: 600; } +#event-log li .ts { color: var(--muted); margin-right: 6px; } + +/* decision modal */ +.modal-backdrop { + position: fixed; inset: 0; + background: rgba(0,0,0,0.7); + display: flex; align-items: center; justify-content: center; + z-index: 100; +} +.modal-backdrop.hidden { display: none; } +.modal { + background: var(--panel); border-radius: 10px; + padding: 20px 24px; min-width: 360px; max-width: 500px; + border: 1px solid var(--border); +} +.modal h3 { margin: 0 0 12px 0; font-size: 15px; } +.card-preview { + padding: 16px; border-radius: 6px; margin-bottom: 16px; + color: #222; +} +.card-preview .name { font-weight: 700; font-size: 16px; margin-bottom: 4px; } +.card-preview .price { font-size: 14px; } +.card-preview .kind { font-size: 10px; text-transform: uppercase; letter-spacing: 0.05em; opacity: 0.7; } +.modal-buttons { display: flex; justify-content: flex-end; gap: 8px; } + +.stream-dot { + width: 8px; height: 8px; border-radius: 50%; + background: var(--off); + flex-shrink: 0; +} +.stream-dot.live { background: var(--accent); } + +/* VLM section — matches .card panel style */ +.vlm-section { + background: var(--panel); + border-radius: 10px; + overflow: hidden; +} +.vlm-header { + padding: 12px 14px 0; + display: flex; align-items: center; gap: 10px; +} +.vlm-header h2 { + margin: 0; + font-size: 11px; text-transform: uppercase; letter-spacing: 0.05em; + color: var(--muted); +} +.vlm-loop-status { + font-size: 11px; color: var(--muted); + font-family: "JetBrains Mono", ui-monospace, monospace; +} +.hotkey-pill { + display: inline-flex; align-items: center; justify-content: center; + width: 22px; height: 18px; + margin-left: 4px; + border-radius: 4px; + background: var(--panel-light); + color: var(--muted); + border: 1px solid var(--border); + font-family: "JetBrains Mono", ui-monospace, monospace; + font-size: 10px; font-weight: 700; + transition: background 80ms ease, color 80ms ease, transform 80ms ease; + user-select: none; +} +.hotkey-pill.live { + background: var(--user); + color: #fff; + border-color: var(--user); + transform: scale(1.12); + box-shadow: 0 0 0 2px rgba(239, 83, 80, 0.3); +} +.vlm-body { + padding: 12px 14px 14px; + display: flex; flex-direction: column; gap: 12px; + min-width: 0; +} + +.vlm-col { + display: flex; flex-direction: column; gap: 10px; + min-width: 0; + padding: 10px 12px; + border-radius: 8px; + background: var(--panel-light); +} +.vlm-col-header { + display: flex; align-items: baseline; gap: 10px; + padding-bottom: 6px; + border-bottom: 1px solid var(--border); +} +.vlm-col-title { + font-size: 10px; text-transform: uppercase; letter-spacing: 0.05em; + font-weight: 600; + color: var(--muted); +} + +.vlm-controls { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.vlm-controls input[type="text"], +.vlm-controls select { + background: var(--panel); color: var(--fg); + border: 1px solid var(--border); border-radius: 4px; + padding: 5px 8px; font: inherit; +} +.vlm-controls input[type="text"] { flex: 1; min-width: 140px; } +.vlm-controls input[type="text"]:focus, +.vlm-controls select:focus { outline: none; border-color: var(--muted); } + +.vlm-check { + display: flex; align-items: center; gap: 6px; + font-size: 11px; color: var(--muted); cursor: pointer; user-select: none; +} +.vlm-check input[type="checkbox"] { + width: 13px; height: 13px; accent-color: var(--accent); cursor: pointer; +} + +.vlm-btn { + background: var(--panel-light); color: var(--fg); + border: 1px solid var(--border); border-radius: 4px; + padding: 5px 12px; font: inherit; cursor: pointer; + transition: background 120ms; +} +.vlm-btn:hover:not(:disabled) { background: #3a3a3a; } +.vlm-btn:disabled { opacity: 0.4; cursor: not-allowed; } +.vlm-col .vlm-btn { background: var(--panel); } +.vlm-col .vlm-btn:hover:not(:disabled) { background: #3a3a3a; } +.vlm-btn-secondary { background: transparent; } + +/* Mic bay — circular Rec button, ported from /vlm */ +.vlm-mic-bay { + display: flex; flex-direction: column; align-items: center; + gap: 8px; + padding: 8px 0 4px; +} +.vlm-mic-select { width: 100%; } +.vlm-mic-circle { + width: 64px; height: 64px; + border-radius: 50%; + background: #dc2626; + border: 3px solid #7f1d1d; + color: #fff; + font-size: 13px; font-weight: 700; letter-spacing: 0.04em; + cursor: pointer; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4); + display: flex; align-items: center; justify-content: center; + transition: transform 0.1s ease, background 0.1s ease; +} +.vlm-mic-circle:hover:not(:disabled) { background: #b91c1c; transform: scale(1.05); } +.vlm-mic-circle:disabled { opacity: 0.6; cursor: not-allowed; } +.vlm-mic-circle.recording { animation: vlm-mic-pulse-circle 1.2s ease-in-out infinite; } +@keyframes vlm-mic-pulse-circle { + 0%, 100% { box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4), 0 0 0 0 rgba(220, 38, 38, 0.7); } + 50% { box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4), 0 0 0 14px rgba(220, 38, 38, 0); } +} +.vlm-response { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 4px; + padding: 8px 10px; + flex: 1; min-height: 90px; overflow-y: auto; + font-size: 12px; line-height: 1.5; + color: var(--fg); + white-space: pre-wrap; word-wrap: break-word; +} +.vlm-response.empty { color: var(--muted); font-style: italic; } +.vlm-response.error { color: var(--user); border-color: var(--user); } + +/* Chat-style transcript (replaces single-block response in Query & response). */ +.vlm-hotkey-hint { + display: flex; align-items: center; gap: 6px; + font-size: 10px; color: var(--muted); + font-family: "JetBrains Mono", ui-monospace, monospace; + padding-bottom: 4px; +} +.vlm-hotkey-hint kbd { + display: inline-block; + background: var(--panel); color: var(--fg); + border: 1px solid var(--border); border-bottom-width: 2px; + border-radius: 3px; + padding: 1px 5px; + font-family: inherit; font-size: 10px; + min-width: 14px; text-align: center; +} +.vlm-hotkey-hint kbd.live { + background: var(--user); color: #fff; border-color: var(--user); +} +.vlm-hotkey-state { margin-left: auto; font-style: italic; } +.vlm-hotkey-state.error { color: var(--user); } + +.vlm-chat { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 6px; + padding: 10px; + flex: 1; min-height: 220px; max-height: 420px; + overflow-y: auto; + display: flex; flex-direction: column; gap: 8px; + scroll-behavior: smooth; +} +.vlm-chat:empty::before { + content: "Press Z to act · Press X to ask anything"; + color: var(--muted); font-style: italic; font-size: 11px; + align-self: center; margin: auto 0; +} +.vlm-msg { + display: flex; flex-direction: column; + max-width: 86%; padding: 7px 11px; + border-radius: 14px; + font-size: 12px; line-height: 1.45; + word-wrap: break-word; white-space: pre-wrap; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.25); +} +.vlm-msg .vlm-msg-meta { + font-family: "JetBrains Mono", ui-monospace, monospace; + font-size: 9px; + color: rgba(255, 255, 255, 0.55); + margin-top: 3px; + align-self: flex-end; +} +.vlm-msg.bot .vlm-msg-meta { color: var(--muted); align-self: flex-start; } +.vlm-msg.me { + align-self: flex-end; + background: var(--user); color: #fff; + border-bottom-right-radius: 4px; +} +.vlm-msg.bot { + align-self: flex-start; + background: var(--panel-light); color: var(--fg); + border-bottom-left-radius: 4px; +} +.vlm-msg.sys { + align-self: center; + background: transparent; color: var(--muted); + font-style: italic; font-size: 11px; + padding: 2px 8px; box-shadow: none; + max-width: 100%; text-align: center; +} +.vlm-msg.bot.error { + background: rgba(239, 83, 80, 0.18); + color: var(--fg); + border: 1px solid var(--user); +} +.vlm-msg.pending { opacity: 0.7; font-style: italic; } +.vlm-msg-role { + font-size: 9px; text-transform: uppercase; letter-spacing: 0.06em; + opacity: 0.65; margin-bottom: 2px; +} +.vlm-meta { + font-size: 10px; color: var(--muted); + font-family: "JetBrains Mono", ui-monospace, monospace; +} + +.vlm-textarea { + background: var(--panel); color: var(--fg); + border: 1px solid var(--border); border-radius: 4px; + padding: 8px 10px; + font-size: 12px; line-height: 1.5; + font-family: "JetBrains Mono", ui-monospace, monospace; + resize: vertical; flex: 1; min-height: 120px; width: 100%; +} +.vlm-textarea:focus { outline: none; border-color: var(--muted); } + +@media (max-width: 900px) { + .grid { grid-template-columns: 1fr; } + .sidebar { order: 1 !important; } +} diff --git a/movensys_sample/movensys_robopoly/static/app.js b/movensys_sample/movensys_robopoly/static/app.js new file mode 100644 index 0000000..3f995fe --- /dev/null +++ b/movensys_sample/movensys_robopoly/static/app.js @@ -0,0 +1,2757 @@ +/** + * movensys-monopoly UI. + * + * 14-tile JSON model rendered on a 14-cell rectangular perimeter (5 wide × 4 + * tall grid). Indexing is counter-clockwise from GO at the bottom-left + * corner. viewBox matches board.png (PDF page 1559 × 794 → ≈ 1.96:1). + */ + +// Cell geometry, corners 1.5× side cells: +// width units: 1.5 + 1 + 1 + 1 + 1.5 = 6 → unit ≈ 259.83 +// height units: 1.5 + 1 + 1 + 1.5 = 5 → unit ≈ 158.80 +// corner ≈ 390×238, top/bot side ≈ 260×238, left/right side ≈ 390×159 +// +// Piece centers below sit at the geometric centre of each cell. The pieces +// in #pieces are pointer-draggable (see setupPieceDragging) so the +// operator can fine-tune on the printed board and read the new coords off +// the board-coords readout. +const BOARD_FINAL_LAYOUT = { + viewBox: { w: 1559, h: 794 }, + centers: { + 0: { user: [81.06, 710.60], robot: [165.93, 710.60] }, // GO (BL) + 1: { user: [63.20, 545.48], robot: [148.08, 545.48] }, // BOSTON (left, lower mid) + 2: { user: [63.20, 344.65], robot: [146.59, 344.65] }, // SEOUL (left, upper mid) + 3: { user: [63.20, 146.79], robot: [148.08, 146.79] }, // IN THE DESERT ISLAND (TL) + 4: { user: [372.63, 143.82], robot: [453.04, 143.82] }, // ELECTRIC COMPANY (top) + 5: { user: [689.50, 143.82], robot: [774.37, 143.82] }, // TAIPEI (top) + 6: { user: [998.92, 146.79], robot: [1086.88, 146.90] }, // SHANGHAI (top) + 7: { user: [1311.69, 146.79], robot: [1391.94, 146.90] }, // NON-FREE PARKING (TR) + 8: { user: [1299.36, 342.46], robot: [1379.62, 341.03] }, // TOKYO (right, upper mid) + 9: { user: [1297.82, 542.76], robot: [1385.78, 542.86] }, // BUSAN (right, lower mid) + 10: { user: [1300.91, 739.97], robot: [1385.78, 738.53] }, // GO TO DESERT ISLAND (BR) + 11: { user: [992.76, 739.97], robot: [1076.10, 740.07] }, // NEW YORK (bottom) + 12: { user: [678.45, 741.51], robot: [757.17, 740.07] }, // CHANCE (bottom) + 13: { user: [368.77, 739.97], robot: [447.48, 740.07] }, // LONDON (bottom) + }, +}; + +const BOARD_LAYOUTS = { + "final": BOARD_FINAL_LAYOUT, +}; + +// ---- HTTP helpers ---------------------------------------------------------- + +async function fetchJson(path, init) { + const r = await fetch(path, init); + if (!r.ok) throw Object.assign(new Error(`${path} -> ${r.status}`), + { status: r.status, body: await r.text() }); + return r.json(); +} +async function postJson(path, body) { + return fetchJson(path, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body || {}), + }); +} + +// ---- adapter badges ------------------------------------------------------- + +function modeBadge(label, health) { + const mode = health.mode || (health.enabled ? "on" : "off"); + const cls = ["live", "on"].includes(mode) ? mode : "stub"; + return `${label}:${mode}`; +} +async function loadBadges() { + // Adapter modes are set at server startup (lifespan() from env) and + // don't change during a session — fetch once at boot, no interval. + try { + const m = await fetchJson("/api/modes"); + document.getElementById("modes").innerHTML = + modeBadge("STT", m.stt) + modeBadge("VLM", m.vlm) + + modeBadge("Robot", m.robot) + modeBadge("ROS2", m.ros2); + } catch (err) { console.warn("modes fetch failed:", err); } +} + +// ---- board rendering ------------------------------------------------------ + +let boardTiles = null; + +async function loadBoardVisual(boardId) { + const host = document.getElementById("board-host"); + const pieces = document.getElementById("pieces"); + + const layout = BOARD_LAYOUTS[boardId]; + pieces.setAttribute("viewBox", `0 0 ${layout.viewBox.w} ${layout.viewBox.h}`); + + const boardJson = await fetchJson(`/assets/boards/board_${boardId}.json`); + boardTiles = boardJson.tiles; + host.innerHTML = `Board ${boardId}`; +} + +// Cell horizontal extent per tile, used to size the ownership rectangle +// (~65% of the cell width). Corners and left/right edges are narrower; the +// inner top/bottom edges are wider. +const TILE_CELL_WIDTH = { + 0: 240, 1: 240, 2: 240, 3: 240, + 4: 320, 5: 320, 6: 320, + 7: 240, 8: 240, 9: 240, 10: 240, + 11: 320, 12: 320, 13: 320, +}; +const PIECE_HEIGHT = 60; +const OWNERSHIP_LABELS = [ + "GO", "BOSTON", "SEOUL", "DESERT", + "ELECTRIC", "TAIPEI", "SHANGHAI", + "PARKING", "TOKYO", "BUSAN", + "GO_DESERT", "NEWYORK", "CHANCE", "LONDON", +]; +// Tiles that do not display an ownership rectangle. +const OWNERSHIP_HIDDEN = new Set([0, 3, 7, 10, 12]); +// Per-tile rectangle size override [width, height]. Tiles without an entry +// use the default size (cell_width * 0.65 wide × piece_height/2 tall). +// ELECTRIC uses a smaller utility-style box: width 110% of the piece width, +// height 65% of the piece height. +const OWNERSHIP_RECT_SIZE = { + 4: [PIECE_HEIGHT * 1.1, PIECE_HEIGHT * 0.65], +}; +// Calibrated rectangle centers per tile (in viewBox coords). Tiles without +// an entry default to a position just below the pieces and rely on the +// operator dragging to calibrate. +const OWNERSHIP_CENTERS = { + 1: [99.48, 478.39], + 2: [98.73, 274.47], + 4: [553.04, 48.87], + 5: [744.26, 82.89], + 6: [1055.23, 82.83], + 8: [1342.57, 276.19], + 9: [1340.26, 477.26], + 11: [1056.00, 671.38], + 13: [431.24, 674.47], +}; + +// Pre-create one per tile, each containing a rect +// + text label. The group is positioned by a transform="translate(dx,dy)" +// that the operator can drag-tune; the rect/text keep stable base coords +// so renderOwnership only needs to update fill+label, leaving any dragged +// translate intact. +function setupOwnershipRects() { + const svg = document.getElementById("pieces"); + if (!svg) return; + let overlay = document.getElementById("ownership-overlay"); + if (overlay) overlay.remove(); + overlay = document.createElementNS("http://www.w3.org/2000/svg", "g"); + overlay.setAttribute("id", "ownership-overlay"); + svg.insertBefore(overlay, svg.firstChild); + + const layout = BOARD_LAYOUTS["final"]; + if (!layout) return; + for (let idx = 0; idx < 14; idx++) { + if (OWNERSHIP_HIDDEN.has(idx)) continue; + const tile = layout.centers[idx]; + if (!tile) continue; + const sizeOverride = OWNERSHIP_RECT_SIZE[idx]; + const rectW = sizeOverride ? sizeOverride[0] : (TILE_CELL_WIDTH[idx] ?? 240) * 0.65; + const rectH = sizeOverride ? sizeOverride[1] : PIECE_HEIGHT / 2; + const center = OWNERSHIP_CENTERS[idx]; + const rectCx = center ? center[0] : (tile.user[0] + tile.robot[0]) / 2; + const rectCy = center + ? center[1] + : (tile.user[1] + tile.robot[1]) / 2 + PIECE_HEIGHT / 2 + 5 + rectH / 2; + const rectX = rectCx - rectW / 2; + const rectY = rectCy - rectH / 2; + + const g = document.createElementNS("http://www.w3.org/2000/svg", "g"); + g.setAttribute("id", `ownership-${idx}`); + g.setAttribute("class", "ownership-group unowned"); + g.setAttribute("transform", "translate(0,0)"); + g.dataset.tileIndex = String(idx); + + const rect = document.createElementNS("http://www.w3.org/2000/svg", "rect"); + rect.setAttribute("x", String(rectX)); + rect.setAttribute("y", String(rectY)); + rect.setAttribute("width", String(rectW)); + rect.setAttribute("height", String(rectH)); + rect.setAttribute("fill", "rgba(180,180,180,0.30)"); + rect.setAttribute("stroke", "#555"); + rect.setAttribute("stroke-width", "2"); + rect.setAttribute("stroke-dasharray", "5 3"); + g.appendChild(rect); + + const text = document.createElementNS("http://www.w3.org/2000/svg", "text"); + text.setAttribute("x", String(rectCx)); + text.setAttribute("y", String(rectCy)); + text.setAttribute("text-anchor", "middle"); + text.setAttribute("dominant-baseline", "central"); + text.setAttribute("fill", "#fff"); + text.setAttribute("font-family", "JetBrains Mono, ui-monospace, monospace"); + text.setAttribute("font-size", "20"); + text.setAttribute("font-weight", "700"); + text.setAttribute("paint-order", "stroke"); + text.setAttribute("stroke", "#000"); + text.setAttribute("stroke-width", "3"); + text.setAttribute("stroke-linejoin", "round"); + text.setAttribute("pointer-events", "none"); + text.textContent = ""; + g.appendChild(text); + + overlay.appendChild(g); + } +} + +// Ownership overlay refresh: just toggle fill/stroke/label per tile based +// on the current state. The 14 groups are created once by +// setupOwnershipRects() and stay drag-tunable across state updates. +function renderOwnership(state) { + const props = state?.properties || {}; + const byTile = {}; + for (const p of Object.values(props)) { + if (p && p.tile_index !== undefined) byTile[p.tile_index] = p; + } + for (let idx = 0; idx < 14; idx++) { + const g = document.getElementById(`ownership-${idx}`); + if (!g) continue; + const rect = g.querySelector("rect"); + const text = g.querySelector("text"); + if (!rect || !text) continue; + const p = byTile[idx]; + if (p && p.owner) { + const fill = p.owner === "user" ? "var(--user)" : "var(--robot)"; + const label = p.has_hotel ? "HOTEL" : (p.houses > 0 ? "HOUSE" : "LAND"); + rect.setAttribute("fill", fill); + rect.setAttribute("stroke", "#000"); + rect.setAttribute("stroke-width", "2.5"); + rect.removeAttribute("stroke-dasharray"); + text.textContent = label; + g.classList.remove("unowned"); + } else { + rect.setAttribute("fill", "rgba(180,180,180,0.30)"); + rect.setAttribute("stroke", "#555"); + rect.setAttribute("stroke-width", "2"); + rect.setAttribute("stroke-dasharray", "5 3"); + text.textContent = ""; + g.classList.add("unowned"); + } + } +} + +function movePiece(player, tileIndex, boardId) { + const layout = BOARD_LAYOUTS[boardId]; + const tile = layout?.centers?.[tileIndex]; + const coords = tile?.[player]; + if (!coords) return; + const [cx, cy] = coords; + const el = document.getElementById(`piece-${player}`); + const half = parseFloat(el.getAttribute("width")) / 2; + el.setAttribute("x", cx - half); + el.setAttribute("y", cy - half); + el.removeAttribute("transform"); + updatePieceReadout(); +} + +// ---- piece drag calibration ---------------------------------------------- + +let updatePieceReadout = () => {}; + +function setupPieceDragging() { + const pieces = document.getElementById("pieces"); + const userRect = document.getElementById("piece-user"); + const robotRect = document.getElementById("piece-robot"); + const readout = document.getElementById("board-coords"); + if (!pieces || !userRect || !robotRect || !readout) return; + + const parseTranslate = (el) => { + const tr = el.getAttribute("transform") || ""; + const m = tr.match(/translate\(\s*(-?[\d.]+)[\s,]+(-?[\d.]+)/); + return m ? [parseFloat(m[1]), parseFloat(m[2])] : [0, 0]; + }; + + const rectBaseCenter = (rect) => { + const x = parseFloat(rect.getAttribute("x")); + const y = parseFloat(rect.getAttribute("y")); + const w = parseFloat(rect.getAttribute("width")); + const h = parseFloat(rect.getAttribute("height")); + return [x + w / 2, y + h / 2]; + }; + + const pieceCenter = (rect) => { + const [bcx, bcy] = rectBaseCenter(rect); + const [tx, ty] = parseTranslate(rect); + return [bcx + tx, bcy + ty]; + }; + + const ownershipCenter = (g) => { + const rect = g.querySelector("rect"); + if (!rect) return [0, 0]; + const [bcx, bcy] = rectBaseCenter(rect); + const [tx, ty] = parseTranslate(g); + return [bcx + tx, bcy + ty]; + }; + + const fmt = (n) => n.toFixed(2); + + updatePieceReadout = () => { + const [ux, uy] = pieceCenter(userRect); + const [rx, ry] = pieceCenter(robotRect); + const lines = [ + `user: (${fmt(ux)}, ${fmt(uy)}) | robot: (${fmt(rx)}, ${fmt(ry)})`, + ]; + const parts = []; + for (let idx = 0; idx < 14; idx++) { + const g = document.getElementById(`ownership-${idx}`); + if (!g) continue; + const [cx, cy] = ownershipCenter(g); + parts.push(`${OWNERSHIP_LABELS[idx]}: (${fmt(cx)}, ${fmt(cy)})`); + } + for (let i = 0; i < parts.length; i += 4) { + lines.push(parts.slice(i, i + 4).join(" | ")); + } + readout.textContent = lines.join("\n"); + }; + + const svgPoint = (evt) => { + const pt = pieces.createSVGPoint(); + pt.x = evt.clientX; + pt.y = evt.clientY; + return pt.matrixTransform(pieces.getScreenCTM().inverse()); + }; + + let dragging = null; + let dragOffset = { x: 0, y: 0 }; + + const wirePiece = (rect) => { + rect.addEventListener("pointerdown", (evt) => { + dragging = { kind: "piece", handle: rect }; + rect.classList.add("dragging"); + const pt = svgPoint(evt); + const [cx, cy] = pieceCenter(rect); + dragOffset.x = pt.x - cx; + dragOffset.y = pt.y - cy; + rect.setPointerCapture(evt.pointerId); + evt.preventDefault(); + }); + rect.addEventListener("pointermove", (evt) => { + if (!dragging || dragging.handle !== rect) return; + const pt = svgPoint(evt); + const newCx = pt.x - dragOffset.x; + const newCy = pt.y - dragOffset.y; + const w = parseFloat(rect.getAttribute("width")); + const h = parseFloat(rect.getAttribute("height")); + const [tx, ty] = parseTranslate(rect); + rect.setAttribute("x", newCx - w / 2 - tx); + rect.setAttribute("y", newCy - h / 2 - ty); + updatePieceReadout(); + }); + const stop = (evt) => { + if (!dragging || dragging.handle !== rect) return; + dragging = null; + rect.classList.remove("dragging"); + try { rect.releasePointerCapture(evt.pointerId); } catch (_) {} + }; + rect.addEventListener("pointerup", stop); + rect.addEventListener("pointercancel", stop); + }; + + const wireOwnership = (g) => { + const rect = g.querySelector("rect"); + if (!rect) return; + rect.addEventListener("pointerdown", (evt) => { + dragging = { kind: "ownership", handle: rect, group: g }; + g.classList.add("dragging"); + rect.classList.add("dragging"); + const pt = svgPoint(evt); + const [cx, cy] = ownershipCenter(g); + dragOffset.x = pt.x - cx; + dragOffset.y = pt.y - cy; + rect.setPointerCapture(evt.pointerId); + evt.preventDefault(); + }); + rect.addEventListener("pointermove", (evt) => { + if (!dragging || dragging.handle !== rect) return; + const pt = svgPoint(evt); + const newCx = pt.x - dragOffset.x; + const newCy = pt.y - dragOffset.y; + const [bcx, bcy] = rectBaseCenter(rect); + g.setAttribute("transform", `translate(${newCx - bcx},${newCy - bcy})`); + updatePieceReadout(); + }); + const stop = (evt) => { + if (!dragging || dragging.handle !== rect) return; + dragging = null; + g.classList.remove("dragging"); + rect.classList.remove("dragging"); + try { rect.releasePointerCapture(evt.pointerId); } catch (_) {} + }; + rect.addEventListener("pointerup", stop); + rect.addEventListener("pointercancel", stop); + }; + + wirePiece(userRect); + wirePiece(robotRect); + for (let idx = 0; idx < 14; idx++) { + const g = document.getElementById(`ownership-${idx}`); + if (g) wireOwnership(g); + } + + updatePieceReadout(); +} + +// ---- money widget --------------------------------------------------------- + +const lastBalance = { user: null, robot: null }; +function renderMoney(liquidSnap, assetsSnap) { + for (const p of ["user", "robot"]) { + const el = document.getElementById(`money-${p}-value`); + const prev = lastBalance[p]; + const curr = liquidSnap[p] ?? 0; + el.textContent = `$${curr}`; + if (prev !== null && curr !== prev) { + el.classList.remove("flash-up", "flash-down"); + void el.offsetWidth; // reflow so animation replays + el.classList.add(curr > prev ? "flash-up" : "flash-down"); + setTimeout(() => el.classList.remove("flash-up", "flash-down"), 600); + } + lastBalance[p] = curr; + + const aEl = document.getElementById(`money-${p}-assets`); + if (aEl) aEl.textContent = `$${assetsSnap?.[p] ?? 0}`; + } +} + +// ---- dice face ------------------------------------------------------------ + +// Pip positions on a 3×3 grid (row, col) for each die value. +const DICE_PIPS = { + 1: [[1, 1]], + 2: [[0, 0], [2, 2]], + 3: [[0, 0], [1, 1], [2, 2]], + 4: [[0, 0], [0, 2], [2, 0], [2, 2]], + 5: [[0, 0], [0, 2], [1, 1], [2, 0], [2, 2]], + 6: [[0, 0], [0, 2], [1, 0], [1, 2], [2, 0], [2, 2]], +}; + +function renderDiceFace(value) { + const face = document.getElementById("dice-face"); + const pips = DICE_PIPS[value]; + if (!pips) { + face.classList.add("empty"); + face.innerHTML = ""; + return; + } + face.classList.remove("empty"); + face.innerHTML = pips.map(([r, c]) => + `` + ).join(""); +} + +// ---- decision modal ------------------------------------------------------- + +let pendingDecision = null; // { property_id, card } +// Voice intent cached from the user's earlier utterance ("buy a house" +// spoken during TURN_START while the dice was rolling). When AWAIT_DECISION +// fires, dispatchVoiceAction's fast path missed it because pendingDecision +// wasn't set yet — we replay the cached choice here. Cleared on use or +// when the turn changes. +let pendingVoiceIntent = null; // { choice: "skip"|"buy"|"build"|"build_hotel", text } + +function showDecision(decision) { + pendingDecision = decision; + // If the user already voiced a choice this turn, apply it instead of + // forcing them to repeat themselves. + if (pendingVoiceIntent) { + const intent = pendingVoiceIntent; + pendingVoiceIntent = null; + const choice = legalizeDecisionChoice(intent.choice, decision); + appendChat({ role: "sys", + text: `Voice (cached "${intent.text}") → ${choice}` }); + submitDecision(choice, choice === "build" ? 1 : 0) + .catch((err) => console.warn("[voice-cache] submit:", err)); + return; + } + const { card } = decision; + const currentTier = decision.current_tier ?? 0; + const maxTier = decision.max_tier ?? (card.kind === "property" ? 3 : 1); + const tierLabel = ["unowned", "land", "house", "hotel"]; + const host = document.getElementById("decision-card"); + const subtitle = currentTier > 0 + ? `You already own this — current tier: ${tierLabel[currentTier]}` + : "Unowned — pick a tier to buy directly"; + host.innerHTML = ` +
+
${card.kind}
+
${card.name}
+
${subtitle}
+
+ `; + const title = currentTier > 0 + ? `Upgrade ${card.name}` + : `Land on ${card.name}`; + document.getElementById("decision-title").textContent = title; + + const btnBuy = document.getElementById("btn-buy"); // → land (tier 1) + const btnHouse = document.getElementById("btn-buy-build"); // → house (tier 2) + const btnHotel = document.getElementById("btn-buy-hotel"); // → hotel (tier 3) + + // Show only the upgrade paths that actually advance the tier. + btnBuy.style.display = currentTier < 1 ? "inline-block" : "none"; + btnHouse.style.display = (currentTier < 2 && maxTier >= 2) ? "inline-block" : "none"; + btnHotel.style.display = (currentTier < 3 && maxTier >= 3) ? "inline-block" : "none"; + + // Re-label with the actual delta cost from where the player is now. + btnBuy.textContent = "Buy land ($100)"; + btnHouse.textContent = `Buy + house ($${(2 - currentTier) * 100})`; + btnHotel.textContent = `Buy + hotel ($${(3 - currentTier) * 100})`; + + document.getElementById("decision-modal").classList.remove("hidden"); +} + +function hideDecision() { + pendingDecision = null; + document.getElementById("decision-modal").classList.add("hidden"); +} + +async function submitDecision(action, houseCount = 0) { + if (!pendingDecision) return; + const pid = pendingDecision.property_id; + let decideOk = true; + try { + await postJson(`/api/properties/${encodeURIComponent(pid)}/decide`, + { action, house_count: houseCount }); + } catch (err) { + decideOk = false; + console.warn("decide:", err); + // Clear the auto-trigger dedup key so the robot's AWAIT_DECISION + // state can re-fire. Without this, an illegal choice (e.g. "buy" + // on a tile the robot already owns) leaves fsm=AWAIT_DECISION but + // the (turn, fsm, turn_number, pid) key still matches — the loop + // never retries and the game deadlocks. + vlmPlayerLastTurnKey = null; + } + hideDecision(); + // Game mode: hold the result on screen before swapping turns — + // STATUS_FLASH_MS for the buy/build flash to play out, then another 3s + // of clean board view so the operator can take in the new building + // before "It's robot's turn" overlays the screen. + if (document.body.classList.contains("game-mode") && action !== "skip" && decideOk) { + await new Promise((res) => setTimeout(res, STATUS_FLASH_MS + 3000)); + } + // Spec §3.4: end-turn is automatic. After the buy/skip/build choice + // the FSM is back at RESOLVE_TILE, so end_turn is safe to call. + try { + if (decideOk) await postJson("/api/game/end_turn"); + } catch (err) { + console.warn("post-decide end_turn:", err); + vlmPlayerLastTurnKey = null; + } finally { + turnInFlight = false; + } +} + +// ---- announcement (human-readable banner below the board) ----------------- + +let lastAnnouncedTurn = null; + +function tileName(idx) { + if (idx === null || idx === undefined) return "—"; + return boardTiles?.[idx]?.name ?? `tile ${idx}`; +} + +function propertyName(pid) { + if (!pid) return "a property"; + const prop = currentState?.properties?.[pid]; + if (prop && typeof prop.tile_index === "number") return tileName(prop.tile_index); + // property_id is usually `${board_id}:${tile_name_slug}` — fall back to the suffix. + const idx = String(pid).lastIndexOf(":"); + return idx >= 0 ? String(pid).slice(idx + 1).replace(/_/g, " ") : pid; +} + +// Unified display duration for every status/state flash overlay. +const STATUS_FLASH_MS = 2000; +let gameOverlayTimer = null; +function flashGameOverlay(text, opts = {}) { + const overlay = document.getElementById("game-overlay"); + const slot = document.getElementById("game-overlay-text"); + if (!overlay || !slot) return; + // Status flash takes over the screen — close the chat overlay if open. + document.body.classList.remove("chat-overlay-active"); + if (opts.html != null) slot.innerHTML = opts.html; + else slot.textContent = text; + overlay.classList.add("visible"); + if (gameOverlayTimer) clearTimeout(gameOverlayTimer); + gameOverlayTimer = null; + // opts.sticky keeps the overlay up until the next flash (e.g. a new + // game's "Game started" / "It's X's turn" overwrites it). Used for + // game_won so the operator doesn't blink and miss the 2s flash. + if (opts.sticky) return; + gameOverlayTimer = setTimeout(() => { + overlay.classList.remove("visible"); + gameOverlayTimer = null; + }, opts.durationMs ?? STATUS_FLASH_MS); +} + +function isStatusOverlayActive() { + const overlay = document.getElementById("game-overlay"); + return !!(overlay && overlay.classList.contains("visible")); +} + +// Keywords that mark a transcribed Z-key utterance as a "show board state" +// request. We match a single character/word from this set so a question like +// "지금 몇 턴이지" or "show me the money" triggers the state flash. +const STATE_INQUIRY_RE = /(보드|상태|돈|머니|턴|board|state|status|money|turn|balance|cash)/i; +function isStateInquiry(text) { + return !!text && STATE_INQUIRY_RE.test(text); +} +function flashCurrentStateOverlay() { + if (!currentState) { + flashGameOverlay("No game state yet"); + return; + } + const turn = currentState.turn_number ?? 0; + const userBal = currentState.players?.user?.balance ?? 0; + const robotBal = currentState.players?.robot?.balance ?? 0; + const assets = { user: 0, robot: 0 }; + for (const p of Object.values(currentState.properties || {})) { + if (!p.owner) continue; + const tier = p.has_hotel ? 3 : (p.houses > 0 ? 2 : 1); + assets[p.owner] = (assets[p.owner] ?? 0) + tier * 100; + } + const html = + `Turn ${turn}\n` + + `User: $${userBal} (assets $${assets.user})\n` + + `Robot: $${robotBal} (assets $${assets.robot})`; + flashGameOverlay(null, { html, durationMs: 5000 }); +} +function maybeOpenChatOverlay() { + if (!document.body.classList.contains("game-mode")) return; + if (isStatusOverlayActive()) return; + document.body.classList.add("chat-overlay-active"); +} +function closeChatOverlay() { + document.body.classList.remove("chat-overlay-active"); +} +function toggleChatOverlay() { + if (document.body.classList.contains("chat-overlay-active")) closeChatOverlay(); + else maybeOpenChatOverlay(); +} + +function announce(text, kind = "info", opts = {}) { + const el = document.getElementById("notification"); + if (el) { + el.classList.remove("empty"); + el.className = `notification kind-${kind}`; + el.textContent = text; + } + if (document.body.classList.contains("game-mode")) { + // opts.durationMs overrides STATUS_FLASH_MS — used by the dice + // popup to flash for just 1 s instead of the default 2 s. + flashGameOverlay(text, opts); + } + // Mirror the same string into the chat transcript as a system bubble so + // the operator sees turn changes / buys / etc. inline with the dialogue. + // We only mirror the "high-signal" notifications that map to a single + // chat-worthy event — the noisy ones (rent, tax, move spam) stay on the + // banner only. + const CHAT_KINDS = new Set(["turn", "win", "buy", "build"]); + if (CHAT_KINDS.has(kind)) appendChat({ role: "sys", text }); +} + +// ---- chat transcript (Query & response panel) ---------------------------- + +const CHAT_BOTTOM_THRESHOLD = 40; +let chatStickToBottom = true; + +function chatEl() { return document.getElementById("vlm-chat"); } + +function appendChat({ role, text, meta = "", error = false, pending = false }) { + const host = chatEl(); + if (!host) return null; + // Lock scroll behaviour to whether the user is currently pinned to the + // bottom — if they scrolled up to read history we don't drag them back. + const distFromBottom = host.scrollHeight - host.scrollTop - host.clientHeight; + const wasAtBottom = distFromBottom <= CHAT_BOTTOM_THRESHOLD; + + const msg = document.createElement("div"); + const classes = ["vlm-msg", role]; + if (error) classes.push("error"); + if (pending) classes.push("pending"); + msg.className = classes.join(" "); + + if (role !== "sys") { + const role_el = document.createElement("div"); + role_el.className = "vlm-msg-role"; + role_el.textContent = role === "me" ? "you" : "vlm"; + msg.appendChild(role_el); + } + + const body = document.createElement("div"); + body.textContent = text; + msg.appendChild(body); + + if (meta) { + const m = document.createElement("div"); + m.className = "vlm-msg-meta"; + m.textContent = meta; + msg.appendChild(m); + } + host.appendChild(msg); + if (wasAtBottom) host.scrollTop = host.scrollHeight; + return msg; +} + +function updateChatMsg(node, { text, meta, error = false, pending = false }) { + if (!node) return; + const bodyNode = node.querySelector("div:not(.vlm-msg-role):not(.vlm-msg-meta)"); + if (bodyNode && text !== undefined) bodyNode.textContent = text; + const metaNode = node.querySelector(".vlm-msg-meta"); + if (meta !== undefined) { + if (metaNode) metaNode.textContent = meta; + else if (meta) { + const m = document.createElement("div"); + m.className = "vlm-msg-meta"; + m.textContent = meta; + node.appendChild(m); + } + } + node.classList.toggle("error", !!error); + node.classList.toggle("pending", !!pending); + const host = chatEl(); + if (host) host.scrollTop = host.scrollHeight; +} + +function setHotkeyState(text, isError = false) { + const el = document.getElementById("vlm-hotkey-state"); + if (!el) return; + el.textContent = text || ""; + el.classList.toggle("error", isError); +} + +function announceFromEvent(env) { + const { type, payload = {} } = env; + switch (type) { + case "game_started": + announce("Game started", "turn"); + lastAnnouncedTurn = null; + break; + case "dice_submitted": { + const player = currentState?.turn ?? "player"; + // `value` is the real face read from the die. `sum` (last_dice_sum) + // is +1 when rules.submit_dice bumped past jail_visit (Desert + // Island skip). The popup always shows the REAL face; the bump is + // mentioned as a side note so the player knows their physical roll + // wasn't ignored. + const real = Array.isArray(payload.value) + ? payload.value.reduce((a, b) => a + b, 0) + : payload.value; + const reflected = payload.sum ?? real; + const text = reflected !== real + ? `${player} rolled ${real}\n(+1 for crossing Desert Island)` + : `${player} rolled ${real}`; + // Short flash — the dice face is the only signal the operator needs + // from this popup, and the next overlay (move/buy/etc.) lands fast. + announce(text, "dice", { durationMs: 1000 }); + break; + } + case "tile_skipped": + // Mirror the skip into the chat transcript / event log so it's + // visible after the dice flash fades. The popup is handled by + // dice_submitted above. + announce( + `Crossing ${payload.tile_name || "Desert Island"} — dice reflected to ${payload.new_sum}`, + "move", + ); + break; + case "move_applied": + announce(`${payload.player} moved to ${tileName(payload.to_tile)}`, "move"); + break; + case "lap_completed": + announce(`${payload.player} passed GO`, "move"); + break; + case "start_bonus": + announce(`${payload.player} collected $${payload.amount} for passing GO`, "money"); + break; + case "property_bought": + announce(`${payload.owner} bought ${propertyName(payload.property_id)} for $${payload.price}`, "buy"); + break; + case "purchase_skipped": + announce(`Purchase skipped`, "buy"); + break; + case "property_built": { + const label = payload.tier_label || (payload.tier === 3 ? "hotel" : "house"); + announce(`Built ${label} on ${propertyName(payload.property_id)}`, "build"); + break; + } + case "tier_sold": { + const labels = ["unowned", "land", "house", "hotel"]; + const what = labels[payload.from_tier] || "tier"; + announce(`Sold ${what} on ${propertyName(payload.property_id)} (+$${payload.refund})`, "money"); + break; + } + case "tile_rent_paid": + announce(`Rent paid${payload.amount ? ` ($${payload.amount})` : ""}`, "money"); + break; + case "tile_tax_paid": + announce(`Tax paid${payload.amount ? ` ($${payload.amount})` : ""}`, "money"); + break; + case "tile_chance_drawn": { + // VLM-driven chance: the per-card text/effect comes through + // chance_card_read + chance_card_applied below. Skip the banner + // here so we don't print "Chance: pay $undefined". + if (payload.deferred) break; + const dir = payload.direction; + const amt = payload.amount; + if (dir === "collect") announce(`Chance: collect $${amt}`, "money"); + else if (dir === "pay") announce(`Chance: pay $${Math.abs(amt)}`, "money"); + break; + } + case "chance_card_read": { + // Step 1 of the VLM chance flow — flash whatever the model read off + // the card. flashGameOverlay only fires in game-mode; mirror to chat + // unconditionally so the operator still has a record in debug mode. + const txt = `Chance card: ${payload.text || "(no text)"}`; + const hold = Math.max(1000, Math.round((payload.hold_s ?? 2) * 1000)); + flashGameOverlay(txt, { durationMs: hold }); + appendChat({ role: "sys", text: txt }); + break; + } + case "chance_card_applied": { + // Step 2 — the model picked one of the 4 outcomes. Flash the result + // for hold_s so the user can read it before the turn moves on. + const player = payload.player || "player"; + const label = payload.label || payload.choice || "$0"; + const txt = `Chance result: ${player} ${label}`; + const hold = Math.max(1000, Math.round((payload.hold_s ?? 2) * 1000)); + flashGameOverlay(txt, { durationMs: hold }); + appendChat({ role: "sys", text: txt }); + break; + } + case "jail_escaped": + announce(`${payload.player} rolled 6 and escaped jail!`, "turn"); + break; + case "jail_skipped": + announce(`${payload.player} is in jail (${payload.turns_left} turn(s) left)`, "money"); + break; + case "jail_released": + announce(`${payload.player} served their time and is free`, "turn"); + break; + case "game_won": { + // Sticky overlay — leave the win banner up until Reset / new game. + // The 2s default flash is too brief for a game-end event. + let text; + if (payload.draw) { + const t = payload.totals || {}; + text = `🤝 Draw — both players at $${t.user ?? "?"}`; + } else { + const reason = payload.reason === "lap_cap" ? " (2 laps)" : ""; + text = `🏆 ${payload.winner} wins the game!${reason}`; + } + announce(text, "win", { sticky: true }); + // Force the overlay even in debug-mode so the operator still sees + // a full-screen banner instead of only the notification strip. + if (!document.body.classList.contains("game-mode")) { + flashGameOverlay(text, { sticky: true }); + } + break; + } + case "state_loaded": + announce("Game state loaded", "turn"); + lastAnnouncedTurn = null; + break; + } +} + +// ---- event log ------------------------------------------------------------ + +function logEvent(env) { + const ol = document.getElementById("event-log"); + const li = document.createElement("li"); + li.className = `type-${env.type}`; + const ts = env.ts ? env.ts.slice(11, 19) : ""; + li.innerHTML = `${ts}${env.type} ${JSON.stringify(env.payload)}`; + ol.prepend(li); + while (ol.children.length > 80) ol.removeChild(ol.lastChild); +} + +// ---- is_YOLO toggle ------------------------------------------------------- + +// is_YOLO lives in server RuntimeConfig so SAVE/LOAD STATE persists it. +// Mirror it locally so the Toggle button can flip without a round trip. +let isYOLO = true; + +function renderYoloStatus() { + const el = document.getElementById("st-is-yolo"); + if (el) el.textContent = isYOLO ? "ON" : "OFF"; +} + +// ---- state reconciliation ------------------------------------------------- + +let currentState = null; +// Spec §3: one button drives the whole turn. This flag gates Roll-dice +// while the roll → move → resolve → end-turn chain is in flight (including +// while the Buy modal is open waiting for a choice). +let turnInFlight = false; + +function renderState(state) { + if (state.config && typeof state.config.is_YOLO === "boolean") { + isYOLO = state.config.is_YOLO; + renderYoloStatus(); + } + document.getElementById("st-turn").textContent = state.turn; + // Single-die manual input stores as (value, 0); only render the "+ d2" + // part when we actually rolled two dice (doubles detection). + const ld = state.last_dice; + document.getElementById("st-dice").textContent = !ld + ? "—" + : ld[1] > 0 + ? `${ld[0]} + ${ld[1]} = ${state.last_dice_sum}` + : String(ld[0]); + renderDiceFace(ld ? ld[0] : null); + + for (const p of ["user", "robot"]) { + const pos = state.positions[p]; + const tileName = boardTiles?.[pos]?.name; + document.getElementById(`st-pos-${p}`).textContent = + pos === undefined ? "—" : (tileName ?? pos); + if (pos !== undefined) movePiece(p, pos, state.board_id); + } + renderOwnership(state); + const money = {}; + const assets = { user: 0, robot: 0 }; + for (const [pid, ps] of Object.entries(state.players || {})) money[pid] = ps.balance; + for (const p of Object.values(state.properties || {})) { + if (!p.owner) continue; + const tier = p.has_hotel ? 3 : (p.houses > 0 ? 2 : 1); + assets[p.owner] = (assets[p.owner] ?? 0) + tier * 100; + } + renderMoney(money, assets); + + const winner = state.winner; + // Spec §3: Roll dice is the only turn button; it chains move + end-turn + // automatically. Disabled while a turn is in flight or a buy modal is open. + document.getElementById("btn-roll-dice").disabled = + state.fsm !== "TURN_START" || winner || turnInFlight; + + // Whose turn is it? — announce when it flips. Skip if a winner has been + // declared so the "X wins" banner isn't overwritten by a stale turn label. + if (!winner && state.turn && state.turn !== lastAnnouncedTurn) { + lastAnnouncedTurn = state.turn; + announce(`It's ${state.turn}'s turn`, "turn"); + // Voice intents are turn-scoped — don't leak last turn's "buy a house" + // into the next one. + pendingVoiceIntent = null; + } +} + +async function refreshState() { + currentState = await fetchJson("/api/game/state"); + renderState(currentState); + if (typeof maybeAutoTriggerRobotTurn === "function") maybeAutoTriggerRobotTurn(); +} + +// ---- WebSocket stream ---------------------------------------------------- + +function openStream() { + const proto = location.protocol === "https:" ? "wss:" : "ws:"; + const ws = new WebSocket(`${proto}//${location.host}/api/stream/game`); + ws.onmessage = async (ev) => { + const env = JSON.parse(ev.data); + logEvent(env); + if (env.type === "hello" && env.payload.snapshot) { + currentState = env.payload.snapshot; + renderState(currentState); + if (typeof maybeAutoTriggerRobotTurn === "function") maybeAutoTriggerRobotTurn(); + return; + } + if (env.type === "tile_property_arrival_buyable" && env.payload.needs_decision) { + const decision = { + property_id: env.payload.property_id, + card: env.payload.card, + current_tier: env.payload.current_tier ?? 0, + max_tier: env.payload.max_tier, + }; + // Robot decides via the VLM agent loop; keep the JS state but skip + // the modal so the operator only ever sees buy choices for the user. + if (currentState?.turn === "robot") pendingDecision = decision; + else showDecision(decision); + } + // Close the modal when *any* client (script, second tab, agent loop) + // resolves the decision via REST. Without this the modal stays + // painted because hideDecision() was only wired to the local button + // click in submitDecision(). + if (env.type === "purchase_skipped" || env.type === "property_bought" || + env.type === "property_built") { + if (pendingDecision) hideDecision(); + } else if (env.type === "fsm_transition" && + env.payload?.from === "AWAIT_DECISION" && + env.payload?.to !== "AWAIT_DECISION") { + // Safety net: any path that leaves AWAIT_DECISION should clear it. + if (pendingDecision) hideDecision(); + } + // Early-close the YOLO overlay the instant pick_and_place reports a + // successful detection. yoloStreamClose is depth-aware and idempotent, + // so the withYoloStream wrapper's own close at fetch-end is a no-op. + if (env.type === "yolo_detection_done") { + yoloStreamClose(); + } + announceFromEvent(env); + await refreshState(); + }; + ws.onclose = () => setTimeout(openStream, 1500); + ws.onerror = () => ws.close(); +} + +// ---- YOLO debug-image overlay -------------------------------------------- +// +// Swap the board pane for /yolo_{dice,cube}_detector/debug_image while +// pick_and_place is in flight. The frames are sourced from the +// movensys_vlm orchestrator on :8000 (same ROS host as joint_states / +// eef_pose) — robopoly's UI cross-origins to it like it already does +// for the other ROS-fed streams. +// +// Lifecycle: yoloStreamOpen(kind) opens a WS and replaces the board with +// the latest JPEG frame; yoloStreamClose() tears it down and the board +// becomes visible again. Wrap a pnp-issuing fetch in +// `withYoloStream(kind, fn)` to bind the overlay's visibility to the +// fetch's promise. +const YOLO_STREAM_HOST = `${location.hostname}:8000`; +const YOLO_STREAM_TOPICS = { + dice: { + path: "/api/stream/yolo_dice_detector/debug_image", + label: "YOLO — dice detector", + }, + cube: { + path: "/api/stream/yolo_cube_detector/debug_image", + label: "YOLO — cube detector", + }, + // Raw gripper-mounted camera — driven during the chance card flow so + // the operator can see the card the VLM is reading. Not a YOLO debug + // topic, but it reuses the same overlay machinery. + hand: { + path: "/api/stream/image_hand/rgb", + label: "Hand camera — chance card", + }, +}; +let yoloStreamWs = null; +let yoloStreamDepth = 0; // re-entrancy: nested pnp calls keep overlay open + +function yoloStreamOpen(kind) { + const topic = YOLO_STREAM_TOPICS[kind]; + if (!topic) return; + yoloStreamDepth += 1; + const overlay = document.getElementById("yolo-overlay"); + const label = document.getElementById("yolo-overlay-label"); + const status = document.getElementById("yolo-overlay-status"); + if (!overlay) return; + if (label) label.textContent = topic.label; + if (status) status.textContent = "Waiting for frames…"; + overlay.classList.remove("hidden"); + overlay.setAttribute("aria-hidden", "false"); + + if (yoloStreamWs) { + try { yoloStreamWs.close(); } catch (_) {} + yoloStreamWs = null; + } + const proto = location.protocol === "https:" ? "wss:" : "ws:"; + const url = `${proto}//${YOLO_STREAM_HOST}${topic.path}`; + let ws; + try { + ws = new WebSocket(url); + } catch (err) { + if (status) status.textContent = `stream error: ${err}`; + return; + } + yoloStreamWs = ws; + ws.onmessage = (ev) => { + let env; + try { env = JSON.parse(ev.data); } catch { return; } + const data = env && env.data; + if (!data || !data.data) { + if (status) status.textContent = env.error || "No frame"; + return; + } + const img = document.getElementById("yolo-overlay-img"); + if (img) img.src = `data:image/jpeg;base64,${data.data}`; + if (status) status.textContent = `${data.width || "?"}×${data.height || "?"}`; + }; + ws.onerror = () => { + if (status) status.textContent = "stream error (is movensys_vlm up?)"; + }; + ws.onclose = () => { + if (yoloStreamWs === ws) yoloStreamWs = null; + }; +} + +function yoloStreamClose() { + if (yoloStreamDepth > 0) yoloStreamDepth -= 1; + if (yoloStreamDepth > 0) return; // still inside another pnp — keep open + const overlay = document.getElementById("yolo-overlay"); + if (overlay) { + overlay.classList.add("hidden"); + overlay.setAttribute("aria-hidden", "true"); + } + if (yoloStreamWs) { + try { yoloStreamWs.close(); } catch (_) {} + yoloStreamWs = null; + } + const img = document.getElementById("yolo-overlay-img"); + if (img) img.removeAttribute("src"); +} + +async function withYoloStream(kind, fn) { + yoloStreamOpen(kind); + try { + return await fn(); + } finally { + yoloStreamClose(); + } +} + +// ---- manual controls ------------------------------------------------------ + +// Spec §3: Roll dice chains roll → physical move → tile resolution → end turn. +// The only pause for human input is the Buy modal (§4.1.1 / §4.1.2); rent, +// tax, chance, and auto-liquidation all resolve server-side. +document.getElementById("btn-roll-dice").addEventListener("click", async () => { + if (turnInFlight) return; + const btn = document.getElementById("btn-roll-dice"); + const prevText = btn.textContent; + turnInFlight = true; + btn.disabled = true; + btn.textContent = "Rolling…"; + + try { + // Snapshot the dispatch turn so the server can refuse with STALE_TURN + // if this chain straddled a turn boundary (e.g. VLM latency made our + // POST land after another client already advanced the game). + const chainTurnNumber = currentState ? currentState.turn_number : null; + // 1. Get the dice value. Two paths depending on whose turn it is: + // - User turn → /api/dice/read_robot. The human already threw the + // die by hand; the arm only moves to the scan pose so the camera + // has a clear view. No pickup, no drop. + // - Robot turn → /api/dice/roll_robot. The arm physically picks + // up, lifts, drops the die, then reads the rolled face. + const diceEndpoint = (currentState && currentState.turn === "user") + ? "/api/dice/read_robot" + : "/api/dice/roll_robot"; + console.log("[roll-chain] dice step:", + { endpoint: diceEndpoint, turn: currentState && currentState.turn, + expected_turn_number: chainTurnNumber }); + // While pick_and_place runs the dice routine, overlay + // /yolo_dice_detector/debug_image on the board pane. The overlay is + // bound to this fetch's promise: it closes the moment the server + // returns DICE_NUMBER, restoring the default board view. + let rollRes; + try { + rollRes = await withYoloStream("dice", () => + postJson(diceEndpoint, { + is_YOLO: isYOLO, + expected_turn_number: chainTurnNumber, + }) + ); + } catch (err) { + if (err && err.status === 409 && typeof err.body === "string" + && err.body.includes("STALE_TURN")) { + console.warn("[roll-chain] dice dropped — STALE_TURN:", err.body); + return; + } + throw err; + } + console.log("[roll-chain] dice response:", rollRes); + if (rollRes && typeof rollRes.dice_number === "number") { + renderDiceFace(rollRes.dice_number); + } + + // 1b. Jail-skip path: rules.submit_dice transitions straight to END_TURN + // when the jailed player rolls a non-6 with turns_left > 0. + if (rollRes && rollRes.fsm === "END_TURN") { + await postJson("/api/game/end_turn"); + return; + } + if (!rollRes || rollRes.fsm !== "MOVING") { + console.warn("[roll-chain] unexpected fsm:", rollRes && rollRes.fsm, + "— skipping apply_robot. Full response:", rollRes); + return; + } + + // 2. Apply move (physical arm). Compute destination from the current + // player's tile + dice sum, modulo the board size. + const player = currentState && currentState.turn; + if (!player || !currentState) { + console.warn("apply_robot: missing currentState"); + return; + } + const from = currentState.positions[player]; + const size = BOARD_LAYOUTS[currentState.board_id] + ? Object.keys(BOARD_LAYOUTS[currentState.board_id].centers).length + : 40; + const to = (from + rollRes.sum) % size; + btn.textContent = "Moving…"; + console.log("[roll-chain] apply_robot:", + { player, from_tile: from, to_tile: to, is_YOLO: isYOLO, + expected_turn_number: chainTurnNumber }); + // Move phase: swap the board for /yolo_cube_detector/debug_image so + // the operator sees the cube-detection pipeline that's driving the + // arm. Overlay closes when the move HTTP call returns. + let moveRes; + try { + moveRes = await withYoloStream("cube", () => + postJson("/api/move/apply_robot", { + player, from_tile: from, to_tile: to, is_YOLO: isYOLO, + expected_turn_number: chainTurnNumber, + }) + ); + } catch (err) { + if (err && err.status === 409 && typeof err.body === "string" + && err.body.includes("STALE_TURN")) { + console.warn("[roll-chain] apply_robot dropped — STALE_TURN:", err.body); + return; + } + throw err; + } + console.log("[roll-chain] apply_robot response:", moveRes); + + // 3. If the tile arrival needs a human decision (Buy modal), stop here. + // The WS event already popped the modal; submitDecision will call + // end_turn after the user picks an option. + if (moveRes && moveRes.fsm === "AWAIT_DECISION") { + return; + } + + // 3b. Deferred chance card: rules.py left the money outcome to the + // VLM. Run the full chance flow (arm move → 2x VLM call → apply + // money → 2 popups) before ending the turn, so the new balance + // reflects on the board before "It's 's turn" overlays. + const tiles = moveRes && moveRes.resolved && moveRes.resolved.tiles; + const deferredChance = Array.isArray(tiles) + && tiles.some((t) => t && t.kind === "chance_drawn" && t.payload && t.payload.deferred); + if (deferredChance) { + // Replace the Ask-VLM chat overlay with the gripper camera feed + // for the duration of the chance flow — operator sees the card + // the VLM is reading instead of the empty chat panel. + closeChatOverlay(); + try { + await withYoloStream("hand", () => postJson("/api/game/chance_card")); + } catch (err) { + console.warn("[roll-chain] chance_card failed:", err, "body:", err && err.body); + appendChat({ + role: "sys", + text: `Chance card failed: ${err.message || err}\n${err && err.body ? err.body : ""}`, + error: true, + }); + } + } + + // 4. Auto end-turn — rent / tax / chance / bankruptcy already resolved + // inside apply_move on the server side. + await postJson("/api/game/end_turn"); + } catch (err) { + console.warn("[roll-chain] aborted with error:", err); + } finally { + btn.textContent = prevText; + // Re-enable when the chain stops here (errors, jail-skip, or end_turn). + // If we're still mid-modal (AWAIT_DECISION), keep the flag set — + // submitDecision will clear it after the post-modal end_turn lands. + if (!currentState || currentState.fsm !== "AWAIT_DECISION") { + turnInFlight = false; + } + } +}); +document.getElementById("btn-reset").addEventListener("click", () => { + resetGame().catch((err) => console.warn("reset failed", err)); +}); +document.getElementById("btn-toggle-yolo").addEventListener("click", async () => { + const next = !isYOLO; + try { + const cfg = await postJson("/api/game/config", { is_YOLO: next }); + isYOLO = !!cfg.is_YOLO; + } catch (err) { + console.warn("toggle is_YOLO:", err); + isYOLO = next; // local fallback so the UI still reflects the click + } + renderYoloStatus(); +}); +document.getElementById("btn-save-state").addEventListener("click", async () => { + const btn = document.getElementById("btn-save-state"); + const prevText = btn.textContent; + btn.disabled = true; + btn.textContent = "Saving…"; + try { + await postJson("/api/game/save_state"); + } catch (err) { + console.warn("save_state:", err); + alert(`Save failed: ${err.message || err}`); + } finally { + btn.textContent = prevText; + btn.disabled = false; + } +}); +document.getElementById("btn-load-state").addEventListener("click", async () => { + const btn = document.getElementById("btn-load-state"); + const prevText = btn.textContent; + btn.disabled = true; + btn.textContent = "Loading…"; + try { + await postJson("/api/game/load_state"); + await refreshState(); + } catch (err) { + console.warn("load_state:", err); + alert(`Load failed: ${err.message || err}`); + } finally { + btn.textContent = prevText; + btn.disabled = false; + } +}); +document.getElementById("btn-skip").addEventListener("click", () => submitDecision("skip")); +document.getElementById("btn-buy").addEventListener("click", () => submitDecision("buy")); +document.getElementById("btn-buy-build").addEventListener("click", () => submitDecision("build", 1)); +document.getElementById("btn-buy-hotel").addEventListener("click", () => submitDecision("build_hotel")); + +// ---- VLM ask -------------------------------------------------------------- + +const VLM_HOST = `${location.hostname}:8000`; +const VLM_BASE = `${location.protocol}//${VLM_HOST}`; + +function setupVlm() { + const prompt = document.getElementById("vlm-prompt"); + const askBtn = document.getElementById("vlm-ask"); + const repeat = document.getElementById("vlm-repeat"); + const interval = document.getElementById("vlm-interval"); + const loopDot = document.getElementById("vlm-loop-dot"); + const loopLbl = document.getElementById("vlm-loop-status"); + + let loopTimer = null; + let inFlight = false; + + async function askOnce() { + if (inFlight) return; + const userText = (prompt.value || "").trim(); + // Spec doc/vlm_as_player.md §4.2: when the user types during their + // TURN_START, the textbox is the user-turn trigger — route the message + // through the VLM-player action loop instead of the free-form Q&A path. + const isUserTurnStart = currentState + && currentState.turn === "user" + && currentState.fsm === "TURN_START" + && !currentState.winner; + // Phrases that clearly mean "I'm trying to take my turn" — if the user + // types one of these but the game isn't actually in user-TURN_START, + // they're hitting the free-form Q&A path by accident and wondering + // why nothing moves. Surface the real state instead of silently + // forwarding the message to the VLM as a generic question. + const looksLikeTurnIntent = + /\b(roll|just rolled|i rolled|user roll|user just|my turn|moved?)\b/i.test(userText); + if (!isUserTurnStart && looksLikeTurnIntent && currentState) { + appendChat({ role: "me", text: userText }); + const reason = !currentState + ? "no game state yet — start a new game" + : currentState.winner + ? `game is over (winner: ${currentState.winner}) — reset to play again` + : currentState.turn !== "user" + ? `it's ${currentState.turn}'s turn, not yours` + : currentState.fsm !== "TURN_START" + ? `fsm is "${currentState.fsm}", not "TURN_START" — a prior turn didn't finish. Try the Roll-dice button, Reset, or wait for the move to complete.` + : "unknown gate failure"; + appendChat({ + role: "bot", + text: `Can't dispatch your turn: ${reason}`, + error: true, + meta: new Date().toLocaleTimeString(), + }); + prompt.value = ""; + return; + } + if (isUserTurnStart) { + const msg = userText || "I rolled the dice."; + inFlight = true; + askBtn.disabled = true; + askBtn.textContent = "Playing…"; + appendChat({ role: "me", text: msg }); + const pending = appendChat({ role: "bot", text: "Acting on your turn…", pending: true }); + try { + await vlmPlayerAct(msg); + updateChatMsg(pending, { text: "(turn dispatched)", meta: new Date().toLocaleTimeString() }); + } catch (err) { + updateChatMsg(pending, { text: String(err), error: true }); + } finally { + inFlight = false; + askBtn.disabled = false; + askBtn.textContent = "Ask"; + prompt.value = ""; + } + return; + } + inFlight = true; + askBtn.disabled = true; + askBtn.textContent = "Thinking…"; + if (userText) appendChat({ role: "me", text: userText }); + const pending = appendChat({ role: "bot", text: "Waiting for VLM response…", pending: true }); + const started = performance.now(); + try { + // Each Ask is a fresh standalone query: wipe the orchestrator's + // vector-DB memory first so no prior turns leak into the LLM's + // recall step. The chat UI still keeps the visible transcript. + try { + await fetch(`${VLM_BASE}/api/vlm/memory`, { method: "DELETE" }); + } catch (_) { /* memory clear is best-effort */ } + const r = await fetch(`${VLM_BASE}/api/vlm/infer`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ camera: "none", prompt: userText || null, client: "robopoly" }), + }); + const body = await r.json(); + const elapsedMs = Math.round(performance.now() - started); + if (!r.ok) { + updateChatMsg(pending, { + text: body.detail || `HTTP ${r.status}`, + meta: `${elapsedMs} ms`, + error: true, + }); + return; + } + const ts = new Date().toLocaleTimeString(); + updateChatMsg(pending, { + text: body.response || "(empty response)", + meta: `${elapsedMs} ms · ${ts}`, + }); + } catch (err) { + updateChatMsg(pending, { text: String(err), error: true }); + } finally { + inFlight = false; + askBtn.disabled = false; + askBtn.textContent = "Ask"; + prompt.value = ""; + } + } + + function scheduleNext() { + if (!repeat.checked) return; + const delayMs = Number(interval.value) * 1000; + loopTimer = setTimeout(async () => { + if (!repeat.checked) return; + await askOnce(); + scheduleNext(); + }, delayMs); + } + function stopLoop() { + if (loopTimer !== null) { clearTimeout(loopTimer); loopTimer = null; } + loopDot.classList.remove("live"); + loopLbl.textContent = "idle"; + } + function startLoop() { + stopLoop(); + loopDot.classList.add("live"); + loopLbl.textContent = `every ${interval.value}s`; + askOnce().then(scheduleNext); + } + + askBtn.addEventListener("click", () => { + if (repeat.checked) startLoop(); else askOnce(); + }); + prompt.addEventListener("keydown", (e) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + if (repeat.checked) startLoop(); else askOnce(); + } + }); + repeat.addEventListener("change", () => { + if (repeat.checked) startLoop(); else stopLoop(); + }); + interval.addEventListener("change", () => { + if (repeat.checked) { + loopLbl.textContent = `every ${interval.value}s`; + if (loopTimer !== null) { clearTimeout(loopTimer); loopTimer = null; } + if (!inFlight) scheduleNext(); + } + }); + + // System prompt + const sp = document.getElementById("vlm-system-prompt"); + const spSave = document.getElementById("vlm-sp-save"); + const spReset = document.getElementById("vlm-sp-reset"); + const spStatus = document.getElementById("vlm-sp-status"); + let spServer = ""; + + function setSpStatus(text, isError = false) { + spStatus.textContent = text; + spStatus.style.color = isError ? "#fca5a5" : "#64748b"; + } + function updateDirty() { + const dirty = sp.value !== spServer; + spSave.disabled = !dirty; + if (dirty) setSpStatus("unsaved changes"); + } + async function loadSp() { + try { + const r = await fetch(`${VLM_BASE}/api/vlm/system_prompt?client=robopoly`); + const body = await r.json(); + spServer = body.system_prompt || ""; + sp.value = spServer; + setSpStatus("loaded"); + spSave.disabled = true; + } catch (err) { setSpStatus(`load failed: ${err}`, true); } + } + async function saveSp() { + spSave.disabled = true; + setSpStatus("saving…"); + try { + const r = await fetch(`${VLM_BASE}/api/vlm/system_prompt?client=robopoly`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ system_prompt: sp.value }), + }); + const body = await r.json(); + if (!r.ok) { setSpStatus(body.detail || `HTTP ${r.status}`, true); return; } + spServer = body.system_prompt; + sp.value = spServer; + setSpStatus("saved"); + } catch (err) { setSpStatus(`save failed: ${err}`, true); } + finally { updateDirty(); } + } + async function resetSp() { + if (!confirm("Reset system prompt to default?")) return; + setSpStatus("resetting…"); + try { + const r = await fetch(`${VLM_BASE}/api/vlm/system_prompt?client=robopoly`, { method: "DELETE" }); + const body = await r.json(); + if (!r.ok) { setSpStatus(body.detail || `HTTP ${r.status}`, true); return; } + spServer = body.system_prompt; + sp.value = spServer; + setSpStatus("reset to default"); + spSave.disabled = true; + } catch (err) { setSpStatus(`reset failed: ${err}`, true); } + } + sp.addEventListener("input", updateDirty); + spSave.addEventListener("click", saveSp); + spReset.addEventListener("click", resetSp); + loadSp(); + + // Speech-to-text — routed through the orchestrator's /api/whisper/transcribe. + const mic = document.getElementById("vlm-mic"); + const micDevice = document.getElementById("vlm-mic-device"); + const sttStatus = document.getElementById("vlm-stt-status"); + + let sttRecorder = null; + let sttChunks = []; + let sttStream = null; + + function setSttStatus(text, isError = false) { + sttStatus.textContent = text; + sttStatus.style.color = isError ? "#fca5a5" : "#64748b"; + } + function stopSttTracks() { + if (sttStream) { + sttStream.getTracks().forEach(t => t.stop()); + sttStream = null; + } + } + async function populateMicDevices() { + if (!navigator.mediaDevices?.enumerateDevices) return; + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const audioInputs = devices.filter((d) => d.kind === "audioinput"); + const previous = micDevice.value; + micDevice.innerHTML = ''; + audioInputs.forEach((d, i) => { + const opt = document.createElement("option"); + opt.value = d.deviceId; + opt.textContent = d.label || `Microphone ${i + 1}`; + micDevice.appendChild(opt); + }); + if (previous && [...micDevice.options].some((o) => o.value === previous)) { + micDevice.value = previous; + } + } catch (err) { + console.warn("enumerateDevices failed:", err); + } + } + populateMicDevices(); + navigator.mediaDevices?.addEventListener?.("devicechange", populateMicDevices); + + async function transcribeBlob(blob) { + const form = new FormData(); + const ext = (blob.type.includes("webm") ? "webm" + : blob.type.includes("ogg") ? "ogg" + : blob.type.includes("mp4") ? "mp4" + : "wav"); + form.append("file", blob, `mic.${ext}`); + form.append("language", "en"); + + setSttStatus("transcribing…"); + const started = performance.now(); + try { + const r = await fetch(`${VLM_BASE}/api/whisper/transcribe`, { method: "POST", body: form }); + const elapsedMs = Math.round(performance.now() - started); + if (!r.ok) { + let detail = `HTTP ${r.status}`; + try { const j = await r.json(); if (j.detail) detail = j.detail; } catch {} + setSttStatus(`${detail} (${elapsedMs} ms)`, true); + return; + } + const body = await r.json(); + if (body.error) { setSttStatus(`failed: ${body.error}`, true); return; } + const rawText = (body.text || "").trim(); + const text = isWhisperHallucination(rawText) ? "" : rawText; + if (text) { + prompt.value = prompt.value + ? `${prompt.value.trimEnd()} ${text}` + : text; + prompt.focus(); + } + setSttStatus(text ? `transcribed (${elapsedMs} ms)` : `empty result (${elapsedMs} ms)`); + } catch (err) { + setSttStatus(`failed: ${err}`, true); + } + } + async function startRecording() { + if (!navigator.mediaDevices?.getUserMedia) { + setSttStatus("mic not available in this browser", true); + return; + } + try { + const deviceId = micDevice.value; + const constraints = { audio: deviceId ? { deviceId: { exact: deviceId } } : true }; + sttStream = await navigator.mediaDevices.getUserMedia(constraints); + // Labels are only revealed after permission is granted — repopulate. + populateMicDevices(); + } catch (err) { + setSttStatus(`mic denied: ${err.name || err}`, true); + return; + } + sttChunks = []; + const mime = MediaRecorder.isTypeSupported("audio/webm;codecs=opus") + ? "audio/webm;codecs=opus" + : (MediaRecorder.isTypeSupported("audio/webm") ? "audio/webm" : ""); + sttRecorder = mime ? new MediaRecorder(sttStream, { mimeType: mime }) + : new MediaRecorder(sttStream); + sttRecorder.ondataavailable = (e) => { if (e.data && e.data.size > 0) sttChunks.push(e.data); }; + sttRecorder.onstop = async () => { + stopSttTracks(); + mic.classList.remove("recording"); + mic.textContent = "Rec"; + mic.title = "Record voice and transcribe via Whisper"; + if (sttChunks.length === 0) { setSttStatus("no audio captured", true); return; } + const blob = new Blob(sttChunks, { type: sttRecorder.mimeType || "audio/webm" }); + sttChunks = []; + await transcribeBlob(blob); + }; + sttRecorder.start(); + mic.classList.add("recording"); + mic.textContent = "Stop"; + mic.title = "Stop recording"; + setSttStatus("recording…"); + } + function stopRecording() { + if (sttRecorder && sttRecorder.state !== "inactive") { + sttRecorder.stop(); + } else { + stopSttTracks(); + } + } + mic.addEventListener("click", () => { + if (sttRecorder && sttRecorder.state === "recording") stopRecording(); + else startRecording(); + }); + + // VLM memory (vector DB) — talks to the orchestrator at VLM_BASE. + const memStatus = document.getElementById("vlm-mem-status"); + const memClear = document.getElementById("vlm-mem-clear"); + + async function refreshMemoryStatus() { + try { + const r = await fetch(`${VLM_BASE}/api/vlm/memory`); + const body = await r.json(); + if (!r.ok) { memStatus.textContent = "memory: error"; return; } + const tag = body.enabled ? "" : " (disabled)"; + memStatus.textContent = body.count === null + ? `memory: unreachable${tag}` + : `memory: ${body.count} stored${tag}`; + } catch { + memStatus.textContent = "memory: unreachable"; + } + } + async function clearMemory() { + if (!confirm("Delete all stored memories from the vector DB? This cannot be undone.")) return; + memClear.disabled = true; + const prev = memClear.textContent; + memClear.textContent = "Clearing…"; + try { + const r = await fetch(`${VLM_BASE}/api/vlm/memory`, { method: "DELETE" }); + const body = await r.json(); + if (!r.ok || !body.ok) alert(`Clear failed: ${body.error || r.status}`); + } catch (err) { + alert(`Clear failed: ${err}`); + } finally { + memClear.disabled = false; + memClear.textContent = prev; + refreshMemoryStatus(); + } + } + memClear.addEventListener("click", clearMemory); + refreshMemoryStatus(); + setInterval(refreshMemoryStatus, 5000); + + // Refresh memory counter right after every inference. + const _askOnceOrig = askOnce; + askOnce = async function() { + await _askOnceOrig(); + refreshMemoryStatus(); + }; +} + +// ==== VLM as robot player (doc/vlm_as_player.md) =========================== +// Single agent loop: the orchestrator's VLM plays the "robot" side. On the +// user's turn the user types in the Ask VLM textbox to nudge the same agent; +// on the robot's turn the agent fires automatically. The VLM never touches +// the arm directly — it emits a JSON action and the frontend dispatches it +// through the existing /api endpoints (so all rules / pick-and-place logic +// stay server-side). + +const VLM_PLAYER_SYSTEM_PROMPT = `You are an action-emitter agent for robopoly, a 2-player Monopoly-style +game. You ARE rolling the dice by emitting JSON — the code reads your +reply and drives the robot arm. Players: "user" (red), "robot" (you, green). + +OUTPUT: exactly one JSON object. No prose, no markdown, no fences. + +Valid actions: + fsm=="TURN_START": {"action":"roll_and_move","player":} + fsm=="AWAIT_DECISION": {"action":"decide","choice":} + +Choice meaning (cumulative cost from unowned = rent opponent pays): + buy tier 1, $100 land + build tier 2, $200 land + house + build_hotel tier 3, $300 land + hotel + skip no purchase +Upgrade delta from owned = $100 × (target_tier − current_tier). +Seed $500, GO bonus $100, tax $100, chance ±$200, 2 laps to win. Rent $150/$300/$450 per tier. + +Constraints: +- decision_pending.kind=="utility" → only buy or skip are legal. +- build needs current_tier<2; build_hotel needs current_tier<3. + +User voice intent (Context: line) — highest priority, head-noun wins: + "hotel"→build_hotel; "house"→build; "land" or bare buy→buy; + "skip"/"pass"/"no"→skip. ("Buy a hotel" → build_hotel, not buy.) +If illegal for the tile, fall back to the closest legal choice +(utility→buy; over-tier→next legal upgrade; else skip). + +Robot AWAIT_DECISION is handled deterministically in the frontend — you +will not be asked to pick on the robot's behalf, only to honor the +user's voice intent on user turns.`; + +let vlmPlayerInFlight = false; +let vlmPlayerLastTurnKey = null; + +// The VLM sees the on-screen rendered game board (background PNG + +// pieces + ownership rectangles, composited into a single JPEG) on every +// inference call. If the canvas capture fails (e.g. tainted by a +// cross-origin asset), we fall back to the physical top-down camera so +// the agent still has *some* visual grounding. +const VLM_PLAYER_FALLBACK_CAMERA = "top"; + +// Downscale the composited board hard before sending to the VLM. Because +// the board is a CLEAN RENDERED DRAWING (flat colors, vector pieces, +// solid ownership rectangles) — not a noisy camera frame — it stays +// legible at very low resolutions. Capping the long side at ~30% of the +// native viewBox (1559 → 468 px) keeps the frame well inside a single +// Pan-and-Scan crop in Gemma 4's vision encoder (~256 image tokens +// instead of the 500–700 the full-size frame would generate), and JPEG +// q=0.5 compresses flat regions to a fraction of the original payload. +// The on-screen board is unaffected — only the off-screen capture canvas +// uses these values. +const BOARD_IMAGE_MAX_WIDTH = 468; +const BOARD_IMAGE_JPEG_QUALITY = 0.5; + +async function captureBoardImage() { + try { + const svg = document.getElementById("pieces"); + if (!svg) return null; + const vb = (svg.getAttribute("viewBox") || "0 0 1559 794").split(/\s+/).map(Number); + const vbW = vb[2] || 1559; + const vbH = vb[3] || 794; + const scale = Math.min(1, BOARD_IMAGE_MAX_WIDTH / vbW); + const w = Math.round(vbW * scale); + const h = Math.round(vbH * scale); + + const canvas = document.createElement("canvas"); + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext("2d"); + ctx.fillStyle = "#ffffff"; + ctx.fillRect(0, 0, w, h); + + // Background board image (same-origin under /assets). + const bgImg = document.querySelector("#board-host img"); + if (bgImg) { + if (!(bgImg.complete && bgImg.naturalWidth > 0)) { + await new Promise((res, rej) => { + bgImg.addEventListener("load", res, { once: true }); + bgImg.addEventListener("error", rej, { once: true }); + }); + } + ctx.drawImage(bgImg, 0, 0, w, h); + } + + // Clone the SVG and inject a + + + +
+

Movensys Manipulator Control

+ FastAPI Bridge +
+ Cameras → + VLM → +
+ +
+ + + + + +
+ +
+
Quick Actions
+
+ + +
+
+ +
+
Gripper
+
+ + + +
+
+ +
+
Movement Commands
+
+ + + + + + + +
+
+ + +
+
Absolute Cartesian Move — Base Frame
+
+
+ +
+ + + +
+
+
+ +
+ + + +
+
+ +
+
+ + +
+
Relative Cartesian Move — Base Frame
+
+
+ +
+ + + +
+
+
+ +
+ + + +
+
+ +
+
+ + +
+
Relative Cartesian Move — Tool Frame
+
+
+ +
+ + + +
+
+
+ +
+ + + +
+
+ +
+
+ + +
+
Absolute Joint-Space Move — Pose Target (base frame)
+
+
+ +
+ + + +
+
+
+ +
+ + + +
+
+ +
+
+ + +
+
Joint Movement — Absolute
+
+
+ +
+ + +
+
+ + +
+
Joint Movement — Relative (increment)
+
+
+ +
+ + +
+
+ + +
+
Camera Feed
+
+ + +
+
/image_top
+ +
+
+
+
RGB
+ Top RGB feed +
Waiting for stream…
+
+
+
Depth (turbo, 0.1–5 m)
+ Top Depth feed +
Waiting for stream…
+
+
+
+
+
Camera Info (/image_top/camera_info)
+ +
+
+
resolution—
+
distortion—
+
fx—
+
fy—
+
cx—
+
cy—
+
+
+
K matrix (3×3)
+
+
—
—
—
+
—
—
—
+
—
—
—
+
+
+
+ + +
+
+
/image_hand
+ +
+
+
+
RGB
+ Hand RGB feed +
Waiting for stream…
+
+
+
Depth (turbo, 0.1–5 m)
+ Hand Depth feed +
Waiting for stream…
+
+
+
+
+
Camera Info (/image_hand/camera_info)
+ +
+
+
resolution—
+
distortion—
+
fx—
+
fy—
+
cx—
+
cy—
+
+
+
K matrix (3×3)
+
+
—
—
—
+
—
—
—
+
—
—
—
+
+
+
+
+ + +
+
+
TF Static — world_manipulator → camera_top
+ +
+
camera_top_color_optical_frame
+
+
tx—
+
qx—
+
ty—
+
qy—
+
tz—
+
qz—
+
+
qw—
+
+
+ +
+
+ +
+ + +
+
RESPONSE LOG
+
+ +
+ + + + diff --git a/movensys_vlm/static/vlm.html b/movensys_vlm/static/vlm.html new file mode 100644 index 0000000..f594662 --- /dev/null +++ b/movensys_vlm/static/vlm.html @@ -0,0 +1,481 @@ + + + + + +Movensys — VLM + + + + +
+

VLM Inference

+
+ Cameras → + ← Control Panel +
+ +
+ +
+
+

✨ Ask VLM

+ + idle +
+
+ +
+
+ Query & response +
+
+ + + +
+
+ + + memory: — + +
+
No response yet.
+
+
+ + +
+
+ + +
+
+ Image sent to VLM + + +
+
+ +
No image yet — click Ask to capture.
+
+
+ + +
+
+ System prompt + +
+ +
+ + +
+
+
+
+ +
+ + + + diff --git a/movensys_vlm/vlm_client.py b/movensys_vlm/vlm_client.py new file mode 100644 index 0000000..7f84185 --- /dev/null +++ b/movensys_vlm/vlm_client.py @@ -0,0 +1,90 @@ +import asyncio +import os +from typing import Optional + +from openai import AsyncOpenAI + +import memory_client + +DEFAULT_SYSTEM_PROMPT = """You are a vision assistant for a board game played on a printed grid.""" + +DEFAULT_CLIENT = "default" +_system_prompts: dict[str, str] = {DEFAULT_CLIENT: DEFAULT_SYSTEM_PROMPT} +_client: AsyncOpenAI | None = None + + +def _client_key(client: Optional[str]) -> str: + return (client or DEFAULT_CLIENT).strip() or DEFAULT_CLIENT + + +def get_system_prompt(client: Optional[str] = None) -> str: + return _system_prompts.get(_client_key(client), DEFAULT_SYSTEM_PROMPT) + + +def set_system_prompt(prompt: str, client: Optional[str] = None) -> str: + key = _client_key(client) + _system_prompts[key] = prompt + return _system_prompts[key] + + +def reset_system_prompt(client: Optional[str] = None) -> str: + key = _client_key(client) + _system_prompts[key] = DEFAULT_SYSTEM_PROMPT + return _system_prompts[key] + + +def get_client() -> AsyncOpenAI: + global _client + if _client is None: + base_url = os.environ.get("VLM_BASE_URL", "http://localhost:9000/v1") + api_key = os.environ.get("HF_TOKEN") or "EMPTY" + timeout = float(os.environ.get("VLM_TIMEOUT") or 60) + _client = AsyncOpenAI(base_url=base_url, api_key=api_key, timeout=timeout) + return _client + + +async def infer( + image_b64: Optional[str] = None, + user_prompt: str = "Report the tokens on the board and the die value.", + system_prompt: Optional[str] = None, + max_tokens: int = 128, + temperature: float = 0.2, + client_id: Optional[str] = None, +) -> str: + client = get_client() + model = os.environ.get("VLM_MODEL_NAME") + + base_system = system_prompt if system_prompt is not None else get_system_prompt(client_id) + # Memory belongs to conversation, not perception: skip recall/store on + # camera-grounded frames so per-frame polling can't pollute the store + # with stale token reports. + use_memory = image_b64 is None + memory_block = ( + memory_client.format_recall(await memory_client.recall(user_prompt)) + if use_memory else "" + ) + effective_system = f"{base_system}\n\n{memory_block}" if memory_block else base_system + + user_content: list = [] + if image_b64: + user_content.append({ + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}, + }) + user_content.append({"type": "text", "text": user_prompt}) + response = await client.chat.completions.create( + model=model, + max_tokens=max_tokens, + temperature=temperature, + messages=[ + {"role": "system", "content": effective_system}, + {"role": "user", "content": user_content}, + ], + ) + answer = response.choices[0].message.content or "" + if use_memory: + asyncio.create_task(memory_client.store( + f"Q: {user_prompt}\nA: {answer}", + metadata={"model": model}, + )) + return answer diff --git a/movensys_vlm/whisper_client.py b/movensys_vlm/whisper_client.py new file mode 100644 index 0000000..665a647 --- /dev/null +++ b/movensys_vlm/whisper_client.py @@ -0,0 +1,36 @@ +import os +from typing import Optional + +from openai import AsyncOpenAI + +_client: AsyncOpenAI | None = None + + +def get_client() -> AsyncOpenAI: + global _client + if _client is None: + base_url = os.environ.get("WHISPER_BASE_URL", "http://localhost:9010/v1") + api_key = os.environ.get("HF_TOKEN") or "EMPTY" + timeout = float(os.environ.get("WHISPER_TIMEOUT") or 60) + _client = AsyncOpenAI(base_url=base_url, api_key=api_key, timeout=timeout) + return _client + + +async def transcribe( + audio_bytes: bytes, + filename: str = "audio.wav", + content_type: str = "audio/wav", + language: Optional[str] = None, + response_format: str = "json", +) -> str: + client = get_client() + model = os.environ.get("HF_WHISPER_REPO") or "whisper" + kwargs: dict = { + "file": (filename, audio_bytes, content_type), + "model": model, + "response_format": response_format, + } + if language: + kwargs["language"] = language + result = await client.audio.transcriptions.create(**kwargs) + return result.text if hasattr(result, "text") else str(result) diff --git a/movensys_vlm/whisper_server.py b/movensys_vlm/whisper_server.py new file mode 100644 index 0000000..66c6ccf --- /dev/null +++ b/movensys_vlm/whisper_server.py @@ -0,0 +1,294 @@ +"""OpenAI-compatible Whisper STT server, backend selectable at startup. + +Two backends share this server: + +- `openvino` (default): OpenVINO GenAI WhisperPipeline, targets the Intel NPU + on Panther Lake (Core Ultra). Requires the `openvino_genai` runtime in the + image; see Dockerfile.whisper-npu. +- `transformers`: HuggingFace transformers + PyTorch on CUDA, targets the + Jetson AGX Thor iGPU (Blackwell, sm_110). We use transformers rather than + faster-whisper here because the upstream ctranslate2 wheels for arm64 are + not built with CUDA support, so faster-whisper would silently fall back to + CPU on Thor. transformers + torch ride on the sm_110-tuned PyTorch shipped + in the NVIDIA Jetson base image — no source build, no Jetson-wheel-version + matching. See Dockerfile.whisper-thor. + +The HTTP surface is identical across backends: POST /v1/audio/transcriptions +with a multipart `file=` field, response shape `{"text": "..."}` (or plain +text when `response_format=text`). Clients pointed at base_url=http://host:9010/v1 +work against either deployment with no code changes — only the base_url moves. + +Why one server file: the two backends differ only in (a) how the model is +loaded and (b) the single transcribe call. Splitting the FastAPI plumbing, +audio decoding, and request validation across two files would invite drift. +A small if/else at module scope keeps both paths visible and lets the same +deployment knobs (`HF_WHISPER_REPO`, `WHISPER_MODEL_DIR`, +`WHISPER_DEFAULT_LANGUAGE`) apply uniformly. +""" + +from __future__ import annotations + +import logging +import os +import tempfile +from pathlib import Path +from typing import Optional, Protocol + +import librosa +import numpy as np +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, PlainTextResponse +from huggingface_hub import snapshot_download + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger("whisper") + +BACKEND = os.environ.get("WHISPER_BACKEND", "openvino").lower() + + +def _autodetect_ov_device() -> str: + # NPU > GPU > CPU. We query OpenVINO directly because that's the same list + # the WhisperPipeline will see at compile time — no point preferring a + # device the runtime can't actually target. + try: + import openvino as ov + + available = {d.split(".")[0].upper() for d in ov.Core().available_devices} + except Exception as e: + log.warning("openvino device probe failed (%s); defaulting to CPU", e) + return "CPU" + for preferred in ("NPU", "GPU", "CPU"): + if preferred in available: + return preferred + return "CPU" + + +# Per-device OV-IR model defaults. `movensys/whisper-large-v3-turbo-fp16-ov-npu` +# is a modern stateful export (single-file decoder + `beam_idx` input) that +# OV 2026 will load on NPU, GPU, *and* CPU. We use it for every device rather +# than swapping in legacy FluidInference NPU IRs, which OV 2026's NPU plugin +# rejects via the StatefulToStateless transform. See +# docker/whisper/whisper-npu-README.md for the export-format rationale. +_MOVENSYS_OV_REPO = ("movensys/whisper-large-v3-turbo-fp16-ov-npu", + "/models/whisper-large-v3-turbo-fp16-ov-npu") +_OV_MODEL_BY_DEVICE = { + "NPU": _MOVENSYS_OV_REPO, + "GPU": _MOVENSYS_OV_REPO, + "CPU": _MOVENSYS_OV_REPO, +} + +if BACKEND == "openvino": + _requested = os.environ.get("WHISPER_DEVICE", "AUTO").strip().upper() or "AUTO" + _DEFAULT_DEVICE = _autodetect_ov_device() if _requested == "AUTO" else _requested + _DEFAULT_REPO, _DEFAULT_DIR = _OV_MODEL_BY_DEVICE.get( + _DEFAULT_DEVICE, _OV_MODEL_BY_DEVICE["CPU"] + ) + log.info( + "openvino backend: device=%s (requested=%s) default_model=%s", + _DEFAULT_DEVICE, _requested, _DEFAULT_REPO, + ) +elif BACKEND == "transformers": + _DEFAULT_REPO = "openai/whisper-large-v3" + _DEFAULT_DIR = "/models/whisper-large-v3" + _DEFAULT_DEVICE = os.environ.get("WHISPER_DEVICE", "cuda") +else: + raise RuntimeError(f"unknown WHISPER_BACKEND={BACKEND!r}; expected 'openvino' or 'transformers'") + +MODEL_REPO = os.environ.get("HF_WHISPER_REPO", _DEFAULT_REPO) +MODEL_DIR = os.environ.get("WHISPER_MODEL_DIR", _DEFAULT_DIR) +DEVICE = _DEFAULT_DEVICE +DEFAULT_LANGUAGE = os.environ.get("WHISPER_DEFAULT_LANGUAGE", "") +# WHISPER_TASK — "transcribe" (default) keeps the source language in the +# output. "translate" forces English output regardless of source language, +# which is the right setting when the operator may mix languages (e.g. +# Korean accent slipping in) but the downstream prompt path is strictly +# English. The `language` parameter still works in translate mode — it +# acts as a source-language hint, not an output-language selector. +TASK = os.environ.get("WHISPER_TASK", "transcribe").strip().lower() or "transcribe" +if TASK not in ("transcribe", "translate"): + raise RuntimeError(f"unknown WHISPER_TASK={TASK!r}; expected 'transcribe' or 'translate'") +MAX_NEW_TOKENS = int(os.environ.get("WHISPER_MAX_NEW_TOKENS", "448")) +TORCH_DTYPE = os.environ.get("WHISPER_TORCH_DTYPE", "float16") # transformers only +CHUNK_LENGTH_S = int(os.environ.get("WHISPER_CHUNK_LENGTH_S", "30")) # transformers only +TARGET_SR = 16000 + + +class _Backend(Protocol): + def transcribe(self, audio: np.ndarray, language: Optional[str], temperature: float) -> str: ... + + +def _resolve_model_path() -> str: + if Path(MODEL_DIR).exists() and any(Path(MODEL_DIR).iterdir()): + log.info("using local model at %s", MODEL_DIR) + return MODEL_DIR + log.info("downloading %s -> %s", MODEL_REPO, MODEL_DIR) + return snapshot_download(repo_id=MODEL_REPO, local_dir=MODEL_DIR) + + +class _OpenVINOBackend: + def __init__(self) -> None: + import openvino_genai + + model_path = _resolve_model_path() + log.info( + "loading WhisperPipeline on %s from %s (first-run compile may take ~30s)", + DEVICE, model_path, + ) + if DEVICE.upper() == "NPU": + self._pipe = openvino_genai.WhisperPipeline(model_path, DEVICE, STATIC_PIPELINE=True) + else: + self._pipe = openvino_genai.WhisperPipeline(model_path, DEVICE) + + def transcribe(self, audio: np.ndarray, language: Optional[str], temperature: float) -> str: + del temperature # OpenVINO GenAI Whisper doesn't expose a sampling temperature knob + config = self._pipe.get_generation_config() + config.task = TASK + config.max_new_tokens = MAX_NEW_TOKENS + lang_token = _normalize_lang_ov(language) or _normalize_lang_ov(DEFAULT_LANGUAGE) + if lang_token: + config.language = lang_token + return str(self._pipe.generate(audio, config)).strip() + + +class _TransformersBackend: + def __init__(self) -> None: + import torch + from transformers import pipeline + + # transformers resolves HF repos itself, but we still prefer a pre-populated + # local dir when available so we don't pay a download on container start. + model_id = MODEL_DIR if (Path(MODEL_DIR).exists() and any(Path(MODEL_DIR).iterdir())) else MODEL_REPO + dtype = { + "float16": torch.float16, + "fp16": torch.float16, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + "float32": torch.float32, + "fp32": torch.float32, + }.get(TORCH_DTYPE.lower()) + if dtype is None: + raise RuntimeError(f"unknown WHISPER_TORCH_DTYPE={TORCH_DTYPE!r}") + + log.info( + "loading transformers ASR pipeline(%s) on device=%s dtype=%s", + model_id, DEVICE, TORCH_DTYPE, + ) + # NB: do NOT set chunk_length_s on the pipeline at construction time. + # Doing so silently turns on the timestamp-decoding path for *every* + # call, which on short clips (<30s) drops the model into a dash/dot + # repetition loop because `return_timestamps=True` is then required + # but the timestamp logits processor doesn't always get attached on + # transformers 4.45+. Instead we pass chunk_length_s per call below, + # only when the audio is actually long enough to need chunking. + self._pipe = pipeline( + task="automatic-speech-recognition", + model=model_id, + torch_dtype=dtype, + device=DEVICE, + ) + + def transcribe(self, audio: np.ndarray, language: Optional[str], temperature: float) -> str: + lang = (language or "").strip() or (DEFAULT_LANGUAGE or "").strip() or None + # transformers passes language/task through generate_kwargs to Whisper's + # forced decoder ids. temperature=0 is the deterministic-greedy default + # that the OpenAI client sends; we only pass it through when nonzero so + # we don't accidentally enable sampling on a 0.0 request. TASK is + # process-wide ("transcribe" or "translate"); see the WHISPER_TASK env + # var description at the top of this module. + generate_kwargs: dict = {"task": TASK} + if lang: + generate_kwargs["language"] = lang + if temperature and temperature > 0: + generate_kwargs["do_sample"] = True + generate_kwargs["temperature"] = float(temperature) + + call_kwargs: dict = {"generate_kwargs": generate_kwargs} + # Whisper's native window is 30s. For longer audio we hand the pipeline + # a chunk length and ask for timestamps, which is the only way the HF + # pipeline knows how to stitch chunks back together. Short clips skip + # both — that path goes through plain greedy decoding and is robust. + duration_s = len(audio) / TARGET_SR + if duration_s > 30.0: + call_kwargs["chunk_length_s"] = CHUNK_LENGTH_S + call_kwargs["return_timestamps"] = True + + result = self._pipe({"array": audio, "sampling_rate": TARGET_SR}, **call_kwargs) + return str(result["text"]).strip() + + +def _normalize_lang_ov(lang: Optional[str]) -> Optional[str]: + """OpenVINO GenAI wants the Whisper language token form, e.g. '<|en|>'.""" + if not lang: + return None + lang = lang.strip() + if not lang: + return None + return lang if lang.startswith("<|") else f"<|{lang}|>" + + +def _load_backend() -> _Backend: + if BACKEND == "openvino": + return _OpenVINOBackend() + return _TransformersBackend() + + +app = FastAPI(title=f"whisper-{BACKEND}", version="0.1") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) +backend: _Backend = _load_backend() + + +def _decode_audio(raw: bytes, filename: str) -> np.ndarray: + suffix = Path(filename).suffix or ".wav" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as tf: + tf.write(raw) + tf.flush() + audio, _ = librosa.load(tf.name, sr=TARGET_SR, mono=True) + return audio.astype(np.float32, copy=False) + + +@app.get("/health") +def health() -> dict: + return {"status": "ok", "backend": BACKEND, "device": DEVICE, "model": MODEL_REPO} + + +@app.get("/v1/models") +def list_models() -> dict: + return { + "object": "list", + "data": [{"id": MODEL_REPO, "object": "model", "owned_by": "local"}], + } + + +@app.post("/v1/audio/transcriptions") +async def transcriptions( + file: UploadFile = File(...), + model: str = Form(default=MODEL_REPO), + language: Optional[str] = Form(default=None), + response_format: str = Form(default="json"), + temperature: float = Form(default=0.0), +): + del model # accepted for OpenAI client compatibility, ignored + raw = await file.read() + if not raw: + raise HTTPException(status_code=400, detail="empty audio upload") + + try: + audio = _decode_audio(raw, file.filename or "audio.wav") + except Exception as e: + raise HTTPException(status_code=400, detail=f"could not decode audio: {e}") from e + + try: + text = backend.transcribe(audio, language, temperature) + except Exception as e: + log.exception("inference failed") + raise HTTPException(status_code=500, detail=f"inference failed: {e}") from e + + if response_format in ("text", "txt"): + return PlainTextResponse(text) + return JSONResponse({"text": text})