Devel

.flake8

@@ -0,0 +1,20 @@
+[flake8]
+max-line-length = 100
+extend-ignore =
+    E203,
+    E501,
+    W503,
+    E402,
+    # B008: false positive on FastAPI's Depends()/File()/Query()/Form() defaults
+    B008
+exclude =
+    .git,
+    __pycache__,
+    .venv,
+    build,
+    dist,
+    movensys_vlm/models,
+    movensys_sample/movensys_vlm,
+    movensys_sample/.venv
+per-file-ignores =
+    movensys_sample/movensys_robopoly/tests/game/test_rules_m2.py:F821,F841

.github/workflows/lint.yml

@@ -0,0 +1,45 @@
+name: Lint
+
+on:
+  pull_request:
+    branches: [main, devel]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  lint:
+    name: flake8 (${{ matrix.python-version }} / ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            python-version: '3.10'
+          - os: ubuntu-24.04
+            python-version: '3.12'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install flake8
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install flake8 flake8-bugbear
+
+      - name: flake8 movensys_sample/movensys_robopoly
+        run: |
+          flake8 movensys_sample/movensys_robopoly \
+            --exclude=.venv,__pycache__,build,dist
+
+      - name: flake8 movensys_vlm
+        run: |
+          flake8 movensys_vlm \
+            --exclude=.venv,__pycache__,models,static

.github/workflows/movensys-intelligence.yml

@@ -0,0 +1,65 @@
+name: CI - movensys_vlm
+
+on:
+  pull_request:
+    branches: [main, devel]
+    paths:
+      - 'movensys_vlm/**'
+
+jobs:
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ros_distro: jazzy
+            os: ubuntu-24.04
+          - ros_distro: humble
+            os: ubuntu-22.04
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Pre-install ros2-apt-source pinned to a known release so setup-ros
+      # skips its unauthenticated api.github.com call (rate-limited on
+      # shared runner IPs → empty version → 404 → curl exit 22).
+      - name: Pre-install ros2-apt-source
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends curl ca-certificates lsb-release
+          codename=$(. /etc/os-release && echo "${UBUNTU_CODENAME:-${VERSION_CODENAME}}")
+          version=1.2.0
+          deb="ros2-apt-source_${version}.${codename}_all.deb"
+          curl --fail --location --retry 5 --retry-delay 3 \
+            -o "/tmp/${deb}" \
+            "https://github.com/ros-infrastructure/ros-apt-source/releases/download/${version}/${deb}"
+          sudo dpkg -i "/tmp/${deb}"
+          rm -f "/tmp/${deb}"
+          sudo apt-get update
+
+      - name: Setup ROS ${{ matrix.ros_distro }}
+        uses: ros-tooling/setup-ros@v0.7
+        with:
+          required-ros-distributions: ${{ matrix.ros_distro }}
+
+      - name: Install Python dependencies
+        run: |
+          if [ "${{ matrix.ros_distro }}" = "jazzy" ]; then
+            pip3 install --no-cache-dir --break-system-packages \
+              fastapi "uvicorn[standard]" pydantic "openai>=1.30.0" Pillow
+          else
+            pip3 install --no-cache-dir \
+              fastapi "uvicorn[standard]" pydantic "openai>=1.30.0" Pillow
+          fi
+
+      - name: Test Python syntax
+        run: |
+          for f in main.py router.py ros2_node.py; do
+            echo "=== syntax check: $f ==="
+            python3 -m py_compile ${{ github.workspace }}/movensys_vlm/$f
+            echo "  OK"
+          done

.github/workflows/unit-test.yml

@@ -0,0 +1,101 @@
+name: Unit Tests
+
+on:
+  pull_request:
+    branches: [main, devel]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  robopoly:
+    name: pytest movensys_robopoly (${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            python-version: '3.10'
+          - os: ubuntu-24.04
+            python-version: '3.12'
+
+    # Stub-mode invariant: external service URLs unset so adapters use stubs.
+    env:
+      STT_SERVICE_URL: ''
+      LLM_SERVICE_URL: ''
+      ROBOT_SERVICE_URL: ''
+
+    defaults:
+      run:
+        working-directory: movensys_sample/movensys_robopoly
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then
+            python -m pip install -r requirements.txt
+          fi
+          python -m pip install pytest pytest-asyncio httpx
+
+      - name: Python syntax check
+        run: |
+          find . -type f -name '*.py' \
+            -not -path './.venv/*' \
+            -not -path './build/*' \
+            -not -path './__pycache__/*' \
+            -print0 | xargs -0 -n1 python -m py_compile
+
+      - name: pytest
+        run: |
+          # Exit code 5 = "no tests collected" is acceptable (all tests may be
+          # marked stale and skipped at module level).
+          rc=0
+          python -m pytest -v tests/ --maxfail=10 || rc=$?
+          if [ "$rc" != "0" ] && [ "$rc" != "5" ]; then
+            exit "$rc"
+          fi
+
+  vlm-syntax:
+    name: Python syntax check movensys_vlm (${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-22.04
+            python-version: '3.10'
+          - os: ubuntu-24.04
+            python-version: '3.12'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install runtime dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install fastapi 'uvicorn[standard]' pydantic 'openai>=1.30.0' Pillow
+
+      - name: py_compile movensys_vlm Python files
+        run: |
+          for f in main.py router.py ros2_node.py memory_client.py vlm_client.py whisper_client.py whisper_server.py; do
+            if [ -f "movensys_vlm/$f" ]; then
+              echo "=== $f ==="
+              python -m py_compile "movensys_vlm/$f"
+              echo "  OK"
+            fi
+          done

.gitignore

@@ -0,0 +1,3 @@
+movensys_vlm/models
+movensys_vlm/vllm
+.venv/

README.md

@@ -1 +1,154 @@
-# movensys-intellegence
+# Movensys Intelligence
+
+Vision-language and speech intelligence layer for the
+[`movensys-manipulator`](https://github.com/movensys/movensys-manipulator)
+stack. Adds a FastAPI VLM service, a Whisper speech endpoint, a Qdrant
+vector memory, optional Phoenix tracing, and sample applications that drive
+the manipulator from natural language.
+
+## Overview
+
+This repository sits on top of the WMX ROS 2 manipulator stack and gives it
+a higher-level reasoning layer:
+
+- **VLM service** — FastAPI server wrapping a vLLM-hosted Gemma 4 model
+  with image input, exposed as REST + WebSocket. It bridges to ROS 2 so it
+  can call manipulator services (`MovePose`, `MoveJoints`, `GetEefPose`,
+  etc.) directly.
+- **Whisper service** — streaming speech-to-text used to issue commands by
+  voice.
+- **Vector memory** — Qdrant-backed long-term memory for the VLM agent.
+- **Sample apps** — `movensys_robopoly`, a board-game demo where the robot
+  picks and places pieces under VLM control, with a YOLO + AprilTag
+  perception pipeline and a dry-run mode that exercises the full stack
+  without moving the arm.
+
+The entire stack runs as a set of Docker compose services and supports
+NVIDIA desktop GPUs, Jetson Thor, and Intel B60 / Panther Lake XPU.
+
+## Repository Layout
+
+```
+.
+├── movensys_vlm/
+│   ├── main.py / router.py / ros2_node.py   # FastAPI app + ROS 2 bridge
+│   ├── vlm_client.py / whisper_client.py    # vLLM + Whisper clients
+│   ├── memory_client.py                     # Qdrant vector memory client
+│   ├── models/                              # Local model assets (Gemma, Whisper, embeddings)
+│   ├── docker/                              # Compose files: vllm, whisper, vectordb, vlm
+│   └── doc/running.md                       # Step-by-step bring-up
+└── movensys_sample/
+    └── movensys_robopoly/                   # Board-game demo (FastAPI + adapters)
+        ├── main.py / router.py
+        ├── pick_and_place.py
+        ├── adapters/                        # robot, ros_image, stt, vlm
+        ├── game/                            # rules, manager, decks, boards
+        ├── scripts/                         # auto_play_dry_run, render helpers
+        └── docker/                          # Compose stack
+```
+
+## Services and Ports
+
+| Service                | Default port | Purpose                                  |
+|------------------------|--------------|------------------------------------------|
+| `movensys_vlm` (FastAPI) | 8000       | VLM REST/WebSocket API + ROS 2 bridge    |
+| `vllm`                  | 9000        | vLLM OpenAI-compatible inference server |
+| `whisper`               | 9010        | Speech-to-text server                    |
+| `vectordb` (Qdrant)     | 6333        | Long-term vector memory                  |
+| `movensys_robopoly`     | 7999        | Robopoly demo UI/API                     |
+| `phoenix` (optional)    | 6006        | OpenTelemetry/LLM traces UI              |
+
+## Requirements
+
+- Ubuntu 22.04 or 24.04
+- Docker with `docker compose`
+- Hardware: NVIDIA GPU (desktop or Jetson Thor) or Intel XPU (B60 / Panther Lake)
+- Local model weights placed under `movensys_vlm/models/` (Gemma 4 E2B/E4B, Whisper large-v3, embedding model)
+- The [`movensys-manipulator`](https://github.com/movensys/movensys-manipulator) stack running (the VLM publishes/calls its ROS 2 services)
+
+## Quick Start
+
+### 1. Configure the host environment
+
+Add the following to your `~/.bashrc`:
+
+```
+export XPU_CORE=nvidia-gpu        # {nvidia-gpu, intel-xpu}
+export CPU_ARCH=amd64             # {amd64, arm64}
+```
+
+```
+source ~/.bashrc
+```
+
+### 2. Clone the repository
+
+```
+mkdir -p ~/workspaces
+cd ~/workspaces
+git clone https://github.com/movensys/movensys-intelligence.git
+```
+
+### 3. Start the VLM stack
+
+For Nvidia desktop, Jetson Thor, or Intel B60:
+
+```
+cd ~/workspaces/movensys-intelligence/movensys_vlm/docker
+COMPOSE_PROFILES=$XPU_CORE docker compose -f vllm.yaml up -d --build
+COMPOSE_PROFILES=$CPU_ARCH docker compose -f vectordb.yaml up -d --build
+COMPOSE_PROFILES=$XPU_CORE docker compose -f whisper.yaml up -d --build
+COMPOSE_PROFILES=$XPU_CORE docker compose -f movensys_vlm.yaml up -d --build
+```
+
+For Intel Panther Lake (vLLM uses a separate build path):
+
+```
+cd ~/workspaces/movensys-intelligence/movensys_vlm/docker
+./vllm-intel-build.sh
+./vllm-intel-run.sh
+```
+
+Wait for `application startup complete` in the vLLM logs before continuing.
+
+> On Jetson Thor or Intel Panther Lake, drop kernel caches between restarts
+> if memory pressure builds up:
+> `sync && sudo sysctl vm.drop_caches=3`
+
+Full bring-up, teardown, and Phoenix-tracing options are documented in
+[`movensys_vlm/doc/running.md`](movensys_vlm/doc/running.md).
+
+### 4. Run a sample application
+
+The Robopoly board-game demo drives the manipulator via the VLM stack. With
+the `movensys-manipulator` YOLO simulation example running (see
+[`movensys-manipulator/doc/6a_yolo_simulation.md`](https://github.com/movensys/movensys-manipulator/blob/main/doc/6a_yolo_simulation.md)):
+
+```
+export MOVENSYS_PNP_DRY_RUN=0     # set to 1 to skip arm motion
+cd ~/workspaces/movensys-intelligence/movensys_sample/movensys_robopoly/docker
+docker compose up -d --build
+```
+
+Open the UI on `http://localhost:7999/`, toggle `is_YOLO` on, and start a
+game. Dry-run mode and the auto-play test script are described in
+[`movensys_sample/doc/1a_robopoly_simulation.md`](movensys_sample/doc/1a_robopoly_simulation.md).
+
+### Pick-and-place from the command line
+
+```
+cd ~/workspaces/movensys-intelligence
+python3 movensys_sample/movensys_robopoly/pick_and_place.py red_cube GO true 2>&1 | tee baseline.log
+grep '\[timing\]' baseline.log
+```
+
+## Related Repositories
+
+- [movensys-manipulator](https://github.com/movensys/movensys-manipulator) — ROS 2 manipulator stack driven by this layer
+- [movensys-simulation](https://github.com/movensys/movensys-simulation) — Isaac Sim scenes used by the demos
+- [wmx-ros2](https://github.com/movensys/wmx-ros2) — Core WMX motion control packages
+- [wmx-ros2-doc](https://github.com/movensys/wmx-ros2-doc) — WMX ROS 2 documentation site
+
+## License
+
+Released under the MIT License.