From e4393e51ca8305c703845660fab4581b1104958c Mon Sep 17 00:00:00 2001 From: yoyowz Date: Tue, 7 Jul 2026 12:21:57 +0800 Subject: [PATCH 1/5] Fix OpenVINO optimization notebook deployment flow --- .../tutorials/003_OpenVINO_Optimization.ipynb | 336 ++++++++++++++---- 1 file changed, 274 insertions(+), 62 deletions(-) diff --git a/examples/tutorials/003_OpenVINO_Optimization.ipynb b/examples/tutorials/003_OpenVINO_Optimization.ipynb index 443fcd94..878aaa96 100644 --- a/examples/tutorials/003_OpenVINO_Optimization.ipynb +++ b/examples/tutorials/003_OpenVINO_Optimization.ipynb @@ -49,7 +49,9 @@ "id": "3c28a826-a284-4623-8a7b-74765529bdd6", "metadata": {}, "source": [ - "Clone the OpenVINO Physical AI repo and install." + "Clone the OpenVINO Physical AI repo and install a pinned revision.\n", + "\n", + "> **Why pin the revision?** The OpenVINO Physical AI APIs are evolving quickly. This notebook depends on a tested `physicalai` runtime revision, so the install cell below checks out a known-good commit before installing the package in editable mode. Update `PHYSICALAI_COMMIT` only after re-validating the rest of this notebook." ] }, { @@ -60,16 +62,37 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "import subprocess\n", + "import sys\n", "\n", "requirements_file = Path(\"requirements.txt\")\n", "if not requirements_file.exists():\n", " requirements_file = Path(\"notebooks/requirements.txt\")\n", "\n", - "if not Path(\"physicalai\").exists():\n", - " !git clone https://github.com/openvinotoolkit/physicalai.git\n", + "# Tested with this notebook. Pinning avoids breakage from fast-moving API changes.\n", + "# The install flow checks out the commit directly so it does not depend on the\n", + "# docs/tutorials branch name after these notebooks are merged into the default branch.\n", + "PHYSICALAI_REPO = \"https://github.com/openvinotoolkit/physicalai.git\"\n", + "PHYSICALAI_REF = \"docs/tutorials\"\n", + "PHYSICALAI_COMMIT = \"6f764b8b4534ad32bf2fdacea066e19cf0921cb0\"\n", + "physicalai_dir = Path(\"physicalai\")\n", + "\n", + "if not physicalai_dir.exists():\n", + " subprocess.check_call([\"git\", \"clone\", PHYSICALAI_REPO, str(physicalai_dir)])\n", + "\n", + "has_pinned_commit = subprocess.run(\n", + " [\"git\", \"-C\", str(physicalai_dir), \"cat-file\", \"-e\", f\"{PHYSICALAI_COMMIT}^{{commit}}\"],\n", + " check=False,\n", + ").returncode == 0\n", + "if not has_pinned_commit:\n", + " subprocess.check_call([\"git\", \"-C\", str(physicalai_dir), \"fetch\", \"origin\", PHYSICALAI_REF])\n", + "\n", + "subprocess.check_call([\"git\", \"-C\", str(physicalai_dir), \"checkout\", PHYSICALAI_COMMIT])\n", + "print(\"Using physicalai commit:\")\n", + "subprocess.check_call([\"git\", \"-C\", str(physicalai_dir), \"rev-parse\", \"--short\", \"HEAD\"])\n", "\n", "%pip install -q --extra-index-url https://download.pytorch.org/whl/cpu -r {requirements_file}\n", - "%pip install -q -e physicalai" + "%pip install -q -e physicalai\n" ] }, { @@ -138,7 +161,56 @@ "id": "2694f43e-7753-4ce5-9124-7faf9bc32490", "metadata": {}, "source": [ - "## 3) Discover and connect to Robot and Cameras" + "## 3) Discover and connect to Robot and Cameras\n", + "\n", + "> **Before continuing on Linux:** make sure the user running Jupyter has permission to access USB cameras and the SO101 serial device. If the robot appears as `/dev/ttyACM0`, the user usually needs to be in the `dialout` group. Camera access may require membership in the `video` group." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "linux-permission-check", + "metadata": {}, + "outputs": [], + "source": [ + "# Optional Linux permission check for cameras and the SO101 serial device.\n", + "# If the current user is missing the required groups, run the printed commands\n", + "# in a terminal, then log out/in or restart the machine before continuing.\n", + "import getpass\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "try:\n", + " import grp\n", + "except ImportError:\n", + " grp = None\n", + "\n", + "user = getpass.getuser()\n", + "print(\"Current user:\", user)\n", + "\n", + "if grp is None:\n", + " print(\"Group membership checks are only available on Unix-like systems.\")\n", + "else:\n", + " groups = {g.gr_name for g in grp.getgrall() if user in g.gr_mem}\n", + " try:\n", + " groups.add(grp.getgrgid(os.getgid()).gr_name)\n", + " except KeyError:\n", + " pass\n", + " print(\"Groups:\", \" \".join(sorted(groups)))\n", + "\n", + " for group in [\"video\", \"dialout\"]:\n", + " if group not in groups:\n", + " print(f\"[ACTION REQUIRED] Add {user!r} to the {group!r} group:\")\n", + " print(f\" sudo usermod -aG {group} {user}\")\n", + "\n", + "serial_port = Path(\"/dev/ttyACM0\")\n", + "if serial_port.exists():\n", + " print(\"/dev/ttyACM0 mode/owner:\", oct(serial_port.stat().st_mode), serial_port.stat().st_uid, serial_port.stat().st_gid)\n", + " if not os.access(serial_port, os.R_OK | os.W_OK):\n", + " print(\"[ACTION REQUIRED] Current process cannot read/write /dev/ttyACM0.\")\n", + " print(\"After adding the user to dialout, restart the login session/Jupyter server.\")\n", + "else:\n", + " print(\"/dev/ttyACM0 was not found. Update SO101_PORT later if your robot uses a different serial device.\")\n" ] }, { @@ -146,7 +218,9 @@ "id": "7c427976-07a9-49ac-86da-8b68a2a6dbdd", "metadata": {}, "source": [ - "Discover cameras." + "
\n", + "Discover cameras.\n", + "
\n" ] }, { @@ -173,10 +247,22 @@ "id": "85241b2b-6119-4bba-8986-aa2fa0f57123", "metadata": {}, "source": [ - "To set up cameras
\n", - "* Use the output of previous cell in the next cell
\n", - "* Use the same names of camera as set in the dataset
\n", - "\"Description\"\n" + "
\n", + "ACTION REQUIRED: Configure your cameras before running the next code cell.\n", + "
\n", + "\n", + "
\n", + "Use the device IDs printed by the previous cell. Keep the camera names exactly the same as the names used during dataset collection and model export.\n", + "
\n", + "\n", + "For the released Pi0.5 pick-and-place model used in this tutorial, the model expects **two** cameras. Do not add a third camera such as `table-cam` unless your exported model was trained with that camera.\n", + "\n", + "Expected names for the example model:\n", + "\n", + "- `top-cam`\n", + "- `gripper-cam`\n", + "\n", + "\"Camera\n" ] }, { @@ -186,13 +272,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Cameras — use the same names and setup as used for the dataset collection\n", + "# Cameras - use the same names and setup as used for dataset collection and model export.\n", "# ============================================================\n", - "# 🔴 USER PARAMETERS — Edit these before running 🔧\n", + "# ACTION REQUIRED - Edit these before running.\n", "# ============================================================\n", + "# Use the LEFT column from the camera discovery output as device_id.\n", + "# For UVC cameras, prefer stable /dev/v4l/by-id/... paths instead of numeric IDs.\n", + "#\n", + "# The example Pi0.5 model expects exactly two cameras:\n", + "# - \"top-cam\"\n", + "# - \"gripper-cam\"\n", + "# Remove or rename cameras only if your exported model was trained with different names.\n", "CAMERAS = [\n", - " (, \"uvc\", ),\n", - " (, \"uvc\", ),\n", + " (\"top-cam\", \"uvc\", \"/dev/v4l/by-id/\"),\n", + " (\"gripper-cam\", \"uvc\", \"/dev/v4l/by-id/\"),\n", "]\n", "\n", "CAMERA_WIDTH = 640\n", @@ -201,7 +294,7 @@ "\n", "# Runtime\n", "FPS = 30\n", - "DURATION_S = 60.0" + "DURATION_S = 60.0\n" ] }, { @@ -210,33 +303,59 @@ "id": "19ae4018-7f3d-4c98-917c-843c58074b8a", "metadata": {}, "source": [ - "Define robot and get the robot ID from Physical AI Studio

\n", - "\"Description\"\n", - "\n" + "Define robot and get the robot ID from Physical AI Studio.\n", + "\n", + "\"Robot\n", + "\n", + "
\n", + "ACTION REQUIRED: Configure the SO101 serial port and calibration file before connecting the robot.\n", + "
\n", + "\n", + "The calibration file path depends on how you collected data:\n", + "\n", + "- Physical AI Studio: `~/.cache/physicalai/robots//calibrations/.json`\n", + "- LeRobot: `~/.cache/calibration/so101_follower.json`\n", + "\n", + "Make sure the Jupyter user can read the calibration JSON. If you see `PermissionError`, fix file ownership/permissions or copy the calibration file to a readable path.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "35aa390b-c7e4-4bc2-ab08-13aecff652d5", + "id": "so101-port-config", "metadata": {}, "outputs": [], "source": [ "# ============================================================\n", - "# 🔴 USER PARAMETERS — Edit these before running 🔧\n", + "# ACTION REQUIRED - Edit these before running.\n", "# ============================================================\n", - "# Robot\n", - "SO101_PORT = \"/dev/ttyACM0\"\n", - "\n", + "# Robot serial port. Common Linux values are /dev/ttyACM0 or /dev/ttyUSB0.\n", + "SO101_PORT = \"/dev/ttyACM0\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "so101-calibration-config", + "metadata": {}, + "outputs": [], + "source": [ "# ============================================================\n", - "# 🔴 USER PARAMETERS — Edit these before running 🔧\n", + "# ACTION REQUIRED - Edit this calibration path before running the connect cell.\n", "# ============================================================\n", - "# Calibration file location depends on how you collected data:\n", - "# Physical AI Studio: ~/.cache/physicalai/robots//calibrations/.json\n", - "# LeRobot: ~/.cache/calibration/so101_follower.json\n", + "# Physical AI Studio example:\n", + "# SO101_CALIBRATION = \"~/.cache/physicalai/robots//calibrations/.json\"\n", + "#\n", + "# LeRobot example:\n", + "# SO101_CALIBRATION = \"~/.cache/calibration/so101_follower.json\"\n", + "SO101_CALIBRATION = \"~/.cache/physicalai/robots//calibrations/.json\"\n", "\n", - "#SO101_CALIBRATION = \"~/.cache/physicalai/robots//calibrations/.json\"\n", - "SO101_CALIBRATION = \"/home/intel/.cache/physicalai/robots/7549dd0c-a292-41c4-a8c0-712aae14c55f/calibrations/b28a2ed2-66db-4f61-848e-b65c88827d48.json\"" + "SO101_CALIBRATION = str(Path(SO101_CALIBRATION).expanduser())\n", + "if \"<\" in SO101_CALIBRATION or \">\" in SO101_CALIBRATION:\n", + " raise ValueError(\"Update SO101_CALIBRATION to your actual calibration JSON path before continuing.\")\n", + "if not Path(SO101_CALIBRATION).is_file():\n", + " raise FileNotFoundError(f\"Calibration file not found: {SO101_CALIBRATION}\")\n", + "print(\"Using SO101 calibration:\", SO101_CALIBRATION)\n" ] }, { @@ -244,8 +363,18 @@ "id": "6c86a75f-9cd6-46ba-99fb-880521a689c8", "metadata": {}, "source": [ - "Connect cameras and robot together. This is an important step as it sets up an environment for robot and camera together.\n", - "* Make sure Physical AI Studio application is closed and no other application is using the cameras or robot" + "Connect cameras and robot together. This sets up the synchronized robot/camera runtime environment.\n", + "\n", + "
\n", + "ACTION REQUIRED: Confirm cameras, robot connection, and permissions before running the next code cell.\n", + "
\n", + "\n", + "Before running this cell:\n", + "\n", + "- Make sure Physical AI Studio and other camera applications are closed.\n", + "- Confirm the current user can access the SO101 serial port, for example `/dev/ttyACM0`.\n", + "- Confirm `CAMERAS` contains exactly the camera names expected by your exported model. The example model expects `top-cam` and `gripper-cam` only.\n", + "- If a previous connection attempt failed, disconnect cameras or restart the kernel before trying again.\n" ] }, { @@ -260,17 +389,25 @@ "\n", "# Cameras\n", "cameras = {}\n", - "for name, driver, device_id in CAMERAS:\n", - " kwargs = {\"serial_number\": device_id} if driver == \"realsense\" else {\"device\": device_id}\n", - " cam = SharedCamera(driver, **kwargs, width=CAMERA_WIDTH, height=CAMERA_HEIGHT, fps=CAMERA_FPS)\n", - " cam.connect()\n", - " cameras[name] = cam\n", - " print(f\"Camera '{name}' connected: {cam.actual_width}x{cam.actual_height} @ {cam.actual_fps}fps\")\n", - "\n", - "# Robot\n", - "robot = SO101(port=SO101_PORT, calibration=SO101_CALIBRATION, role=\"follower\")\n", - "robot.connect()\n", - "print(f\"Robot connected on {SO101_PORT}\")" + "try:\n", + " for name, driver, device_id in CAMERAS:\n", + " kwargs = {\"serial_number\": device_id} if driver == \"realsense\" else {\"device\": device_id}\n", + " cam = SharedCamera(driver, **kwargs, width=CAMERA_WIDTH, height=CAMERA_HEIGHT, fps=CAMERA_FPS)\n", + " cam.connect()\n", + " cameras[name] = cam\n", + " print(f\"Camera '{name}' connected: {cam.actual_width}x{cam.actual_height} @ {cam.actual_fps}fps\")\n", + "\n", + " # Robot\n", + " robot = SO101(port=SO101_PORT, calibration=SO101_CALIBRATION, role=\"follower\")\n", + " robot.connect()\n", + " print(f\"Robot connected on {SO101_PORT}\")\n", + "except Exception:\n", + " for cam in cameras.values():\n", + " try:\n", + " cam.disconnect()\n", + " except Exception:\n", + " pass\n", + " raise\n" ] }, { @@ -278,7 +415,14 @@ "id": "66334671-4691-44d7-a37b-59bc92693798", "metadata": {}, "source": [ - "## 4) Load the model" + "## 4) Load the model\n", + "\n", + "This notebook supports two model sources:\n", + "\n", + "1. **Your own exported model** from Physical AI Studio or LeRobot. Set `MODEL_SOURCE = \"local\"` and point `EXPORT_DIR` to the exported OpenVINO policy package.\n", + "2. **The public example model** from Hugging Face. Set `MODEL_SOURCE = \"hub\"`; the notebook downloads the OpenVINO policy package and uses it for deployment.\n", + "\n", + "The exported model must match your runtime setup, especially the number and names of cameras.\n" ] }, { @@ -289,11 +433,24 @@ "outputs": [], "source": [ "# ============================================================\n", - "# 🔴 USER PARAMETERS — Edit these before running 🔧\n", + "# ACTION REQUIRED - Choose model source and deployment device.\n", "# ============================================================\n", + "# \"local\": use your own exported model from Physical AI Studio or LeRobot.\n", + "# \"hub\": download the public example OpenVINO Pi0.5 policy package.\n", + "MODEL_SOURCE = \"local\" # \"local\" or \"hub\"\n", + "\n", + "# Used when MODEL_SOURCE == \"local\".\n", "EXPORT_DIR = \"exports/pi05-pick-place-purple-cube\"\n", - "DEVICE = \"GPU\" # OpenVINO can run on \"GPU\", \"CPU\", \"NPU\"\n", - "TASK = \"pick up the box\"" + "\n", + "# Used when MODEL_SOURCE == \"hub\".\n", + "MODEL_REPO_ID = \"eugene123tw/pi05-pick-place-purple-cube\"\n", + "DATASET_REPO_ID = \"gtamir/pick-place-purple-cube\"\n", + "DATASET_NAME = \"pick-place-purple-cube\"\n", + "DOWNLOAD_EXAMPLE_DATASET_METADATA = False\n", + "ASSETS_DIR = Path(\"physicalai_assets\").resolve()\n", + "\n", + "DEVICE = \"GPU\" # OpenVINO can run on \"GPU\", \"CPU\", \"NPU\".\n", + "TASK = \"pick up the box\"\n" ] }, { @@ -301,10 +458,11 @@ "id": "d4ed8ff1-4c78-4060-b9d2-6e8fb9e9c891", "metadata": {}, "source": [ - "
Make sure to copy all the model files from the studio to the $EXPORT_DIR\n", - "
Then, load the model to the target device\n", - "

\n", - "\"Description\"" + "If `MODEL_SOURCE = \"local\"`, make sure `EXPORT_DIR` contains the full exported policy package, including `manifest.json`, model IR files, tokenizer files, and metadata.\n", + "\n", + "If `MODEL_SOURCE = \"hub\"`, the next cell downloads the public example model package from Hugging Face. This is useful for validating the notebook before using your own model.\n", + "\n", + "\"Model\n" ] }, { @@ -314,11 +472,49 @@ "metadata": {}, "outputs": [], "source": [ - "import openvino_tokenizers \n", + "import openvino_tokenizers # noqa: F401 - registers OpenVINO tokenizer custom ops\n", + "from huggingface_hub import snapshot_download\n", "from physicalai.inference import InferenceModel\n", "\n", - "policy = InferenceModel.load(EXPORT_DIR, device=DEVICE)\n", - "print(f\"Model loaded on {DEVICE}\")" + "if MODEL_SOURCE == \"hub\":\n", + " EXPORT_DIR = Path(\n", + " snapshot_download(\n", + " repo_id=MODEL_REPO_ID,\n", + " local_dir=ASSETS_DIR / \"models\" / MODEL_REPO_ID.replace(\"/\", \"__\"),\n", + " allow_patterns=[\n", + " \"manifest.json\",\n", + " \"pi05.xml\",\n", + " \"pi05.bin\",\n", + " \"tokenizer.xml\",\n", + " \"tokenizer.bin\",\n", + " \"metadata.yaml\",\n", + " \"README.md\",\n", + " ],\n", + " local_dir_use_symlinks=False,\n", + " )\n", + " ).resolve()\n", + "\n", + " if DOWNLOAD_EXAMPLE_DATASET_METADATA:\n", + " dataset_dir = snapshot_download(\n", + " repo_id=DATASET_REPO_ID,\n", + " repo_type=\"dataset\",\n", + " local_dir=ASSETS_DIR / \"datasets\" / DATASET_REPO_ID.replace(\"/\", \"__\"),\n", + " allow_patterns=[f\"{DATASET_NAME}/meta/**\"],\n", + " local_dir_use_symlinks=False,\n", + " )\n", + " print(\"Downloaded example dataset metadata to:\", dataset_dir)\n", + "elif MODEL_SOURCE == \"local\":\n", + " EXPORT_DIR = Path(EXPORT_DIR).expanduser().resolve()\n", + "else:\n", + " raise ValueError(\"MODEL_SOURCE must be 'local' or 'hub'.\")\n", + "\n", + "required_model_files = [\"manifest.json\", \"pi05.xml\", \"pi05.bin\", \"tokenizer.xml\", \"tokenizer.bin\"]\n", + "missing = [name for name in required_model_files if not (EXPORT_DIR / name).exists()]\n", + "if missing:\n", + " raise FileNotFoundError(f\"Missing required model package files in {EXPORT_DIR}: {missing}\")\n", + "\n", + "policy = InferenceModel.load(EXPORT_DIR, backend=\"openvino\", device=DEVICE)\n", + "print(f\"Model loaded from {EXPORT_DIR} on {DEVICE}\")\n" ] }, { @@ -344,23 +540,31 @@ "metadata": {}, "outputs": [], "source": [ - "from physicalai.runtime import ActionQueue, PolicyRuntime, SyncExecution\n", + "from physicalai.runtime import PolicyRuntime, SyncExecution\n", "\n", "runtime = PolicyRuntime(\n", " robot=robot,\n", " model=policy,\n", - " execution=SyncExecution(fps=FPS, request_threshold=0.5),\n", + " execution=SyncExecution(request_threshold=0.5),\n", " fps=FPS,\n", " cameras=cameras,\n", - " action_queue=ActionQueue(),\n", " task=TASK,\n", ")\n", "\n", + "# Optional sanity check: the example Pi0.5 model expects two camera inputs.\n", + "sample_obs = runtime._build_model_input()\n", + "print(\"Runtime model inputs:\")\n", + "for key, value in sample_obs.items():\n", + " print(\" \", key, getattr(value, \"shape\", None))\n", + "image_keys = [key for key in sample_obs if key.startswith(\"images.\")]\n", + "if len(image_keys) != len(CAMERAS):\n", + " raise RuntimeError(f\"Camera input mismatch: CAMERAS has {len(CAMERAS)} entries but runtime built {len(image_keys)} image inputs.\")\n", + "\n", "with runtime:\n", - " print(f\"Running policy at {FPS} fps for {DURATION_S}s — task: {TASK!r}\")\n", + " print(f\"Running policy at {FPS} fps for {DURATION_S}s - task: {TASK!r}\")\n", " stats = runtime.run(duration_s=DURATION_S)\n", "\n", - "print(f\"\\nDone — {stats.steps} steps, {stats.inference_count} inferences, {stats.total_holds} holds\")" + "print(f\"\\nDone - {stats.steps} steps, {stats.inference_count} inferences, {stats.total_holds} holds\")\n" ] }, { @@ -379,11 +583,19 @@ "outputs": [], "source": [ "for name, cam in cameras.items():\n", - " cam.disconnect()\n", - " print(f\"Camera '{name}' disconnected\")\n", + " try:\n", + " cam.disconnect()\n", + " print(f\"Camera '{name}' disconnected\")\n", + " except Exception as exc:\n", + " print(f\"Camera '{name}' disconnect warning: {exc}\")\n", "\n", - "robot.disconnect()\n", - "print(\"Robot disconnected\")" + "try:\n", + " robot.disconnect()\n", + " print(\"Robot disconnected\")\n", + "except NameError:\n", + " print(\"Robot was not created\")\n", + "except Exception as exc:\n", + " print(f\"Robot disconnect warning: {exc}\")\n" ] } ], From 7999915f5cda6a10cea64576a31838cf23ec4036 Mon Sep 17 00:00:00 2001 From: yoyowz Date: Mon, 13 Jul 2026 16:46:56 +0800 Subject: [PATCH 2/5] fix: install released physicalai version in tutorial --- .../tutorials/003_OpenVINO_Optimization.ipynb | 30 +++---------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/examples/tutorials/003_OpenVINO_Optimization.ipynb b/examples/tutorials/003_OpenVINO_Optimization.ipynb index 878aaa96..079502cb 100644 --- a/examples/tutorials/003_OpenVINO_Optimization.ipynb +++ b/examples/tutorials/003_OpenVINO_Optimization.ipynb @@ -49,9 +49,9 @@ "id": "3c28a826-a284-4623-8a7b-74765529bdd6", "metadata": {}, "source": [ - "Clone the OpenVINO Physical AI repo and install a pinned revision.\n", + "Install OpenVINO Physical AI from PyPI with a pinned released version.\n", "\n", - "> **Why pin the revision?** The OpenVINO Physical AI APIs are evolving quickly. This notebook depends on a tested `physicalai` runtime revision, so the install cell below checks out a known-good commit before installing the package in editable mode. Update `PHYSICALAI_COMMIT` only after re-validating the rest of this notebook." + "> **Why pin the version?** The OpenVINO Physical AI APIs are evolving quickly. This notebook depends on a tested `physicalai` release, so the install cell below pins the package version. Update `PHYSICALAI_VERSION` only after re-validating the rest of this notebook." ] }, { @@ -62,37 +62,15 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "import subprocess\n", - "import sys\n", - "\n", "requirements_file = Path(\"requirements.txt\")\n", "if not requirements_file.exists():\n", " requirements_file = Path(\"notebooks/requirements.txt\")\n", "\n", "# Tested with this notebook. Pinning avoids breakage from fast-moving API changes.\n", - "# The install flow checks out the commit directly so it does not depend on the\n", - "# docs/tutorials branch name after these notebooks are merged into the default branch.\n", - "PHYSICALAI_REPO = \"https://github.com/openvinotoolkit/physicalai.git\"\n", - "PHYSICALAI_REF = \"docs/tutorials\"\n", - "PHYSICALAI_COMMIT = \"6f764b8b4534ad32bf2fdacea066e19cf0921cb0\"\n", - "physicalai_dir = Path(\"physicalai\")\n", - "\n", - "if not physicalai_dir.exists():\n", - " subprocess.check_call([\"git\", \"clone\", PHYSICALAI_REPO, str(physicalai_dir)])\n", - "\n", - "has_pinned_commit = subprocess.run(\n", - " [\"git\", \"-C\", str(physicalai_dir), \"cat-file\", \"-e\", f\"{PHYSICALAI_COMMIT}^{{commit}}\"],\n", - " check=False,\n", - ").returncode == 0\n", - "if not has_pinned_commit:\n", - " subprocess.check_call([\"git\", \"-C\", str(physicalai_dir), \"fetch\", \"origin\", PHYSICALAI_REF])\n", - "\n", - "subprocess.check_call([\"git\", \"-C\", str(physicalai_dir), \"checkout\", PHYSICALAI_COMMIT])\n", - "print(\"Using physicalai commit:\")\n", - "subprocess.check_call([\"git\", \"-C\", str(physicalai_dir), \"rev-parse\", \"--short\", \"HEAD\"])\n", + "PHYSICALAI_VERSION = \"0.1.1\"\n", "\n", "%pip install -q --extra-index-url https://download.pytorch.org/whl/cpu -r {requirements_file}\n", - "%pip install -q -e physicalai\n" + "%pip install -q physicalai=={PHYSICALAI_VERSION}\n" ] }, { From 4ec6feb662faf18c34da55feceb713bcb3b48e58 Mon Sep 17 00:00:00 2001 From: yoyowz Date: Wed, 15 Jul 2026 18:08:16 +0800 Subject: [PATCH 3/5] fix: align tutorial dependencies with released packages --- .../tutorials/003_OpenVINO_Optimization.ipynb | 75 ++++++++----------- 1 file changed, 32 insertions(+), 43 deletions(-) diff --git a/examples/tutorials/003_OpenVINO_Optimization.ipynb b/examples/tutorials/003_OpenVINO_Optimization.ipynb index 079502cb..e239b9bb 100644 --- a/examples/tutorials/003_OpenVINO_Optimization.ipynb +++ b/examples/tutorials/003_OpenVINO_Optimization.ipynb @@ -30,10 +30,10 @@ "id": "39464140-04c5-4f8a-b386-6a7bb735836b", "metadata": {}, "source": [ - "## 1) Train a model using the Physical AI Studio or LeRobot\n", - "* Use Pi0.5 policy supported in Physical AI Studio or LeRobot to train a model.
\n", - "* The output of this stage should be a trained Pi0.5 model
\n", - "[Open Notebook](002_Using_Physical_AI_Studio.ipynb)" + "## 1) Train a model using Physical AI Studio or LeRobot\n", + "* Use the Pi0.5 policy supported in Physical AI Studio or LeRobot to train a model.
\n", + "* The output of this stage should be a trained Pi0.5 model.
\n", + "* For the Physical AI Studio workflow, follow the official [Physical AI Studio documentation](https://github.com/open-edge-platform/physical-ai-studio/blob/main/application/README.md)." ] }, { @@ -89,19 +89,9 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "import json\n", - "import os\n", - "import time\n", - "\n", - "import numpy as np\n", - "import openvino as ov\n", - "from huggingface_hub import hf_hub_download, snapshot_download\n", - "from physicalai.inference import InferenceModel\n", "\n", "WORKSPACE = Path.cwd().resolve()\n", - "RUNTIME_ROOT = WORKSPACE / \"physicalai\"\n", - "ROOT = RUNTIME_ROOT\n", - "os.chdir(ROOT)" + "print(\"Workspace:\", WORKSPACE)" ] }, { @@ -119,19 +109,19 @@ "metadata": {}, "outputs": [], "source": [ - "INSTALL_HARDWARE_DEPS = \"True\"\n", - "USE_SHARED_CAMERA = \"True\"\n", + "INSTALL_HARDWARE_DEPS = True\n", + "USE_SHARED_CAMERA = True\n", "\n", "if INSTALL_HARDWARE_DEPS:\n", " import subprocess\n", " import sys\n", "\n", " hardware_extras = \"transport,so101\" if USE_SHARED_CAMERA else \"so101\"\n", - " runtime_with_hardware_extras = str(WORKSPACE / \"physicalai\") + f\"[{hardware_extras}]\"\n", - " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", runtime_with_hardware_extras])\n", - " print(f\"[DONE] Installed PhysicalAI hardware extras: {hardware_extras}\")\n", + " package_spec = f\"physicalai[{hardware_extras}]=={PHYSICALAI_VERSION}\"\n", + " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", package_spec])\n", + " print(f\"[DONE] Installed PhysicalAI hardware extras: {package_spec}\")\n", "else:\n", - " print(\"[SKIP] Hardware extras are not installed. Set INSTALL_HARDWARE_DEPS = True if this environment needs them.\")\n" + " print(\"[SKIP] Hardware extras are not installed. Set INSTALL_HARDWARE_DEPS = True if this environment needs them.\")" ] }, { @@ -397,10 +387,10 @@ "\n", "This notebook supports two model sources:\n", "\n", - "1. **Your own exported model** from Physical AI Studio or LeRobot. Set `MODEL_SOURCE = \"local\"` and point `EXPORT_DIR` to the exported OpenVINO policy package.\n", - "2. **The public example model** from Hugging Face. Set `MODEL_SOURCE = \"hub\"`; the notebook downloads the OpenVINO policy package and uses it for deployment.\n", + "1. **Your own exported model** from Physical AI Studio or LeRobot. Set `MODEL_SOURCE = \"local\"` and point `EXPORT_DIR` to the exported OpenVINO policy package. Use this path for real SO101 robot deployment.\n", + "2. **The official OpenVINO example model** from Hugging Face. Set `MODEL_SOURCE = \"hub\"`; the notebook downloads a released OpenVINO policy package from the official [OpenVINO Physical AI collection](https://huggingface.co/collections/OpenVINO/physical-ai). This is useful for validating model package download and loading before using your own exported model.\n", "\n", - "The exported model must match your runtime setup, especially the number and names of cameras.\n" + "The exported model must match your runtime setup, especially the robot action space and the number and names of cameras." ] }, { @@ -414,21 +404,22 @@ "# ACTION REQUIRED - Choose model source and deployment device.\n", "# ============================================================\n", "# \"local\": use your own exported model from Physical AI Studio or LeRobot.\n", - "# \"hub\": download the public example OpenVINO Pi0.5 policy package.\n", + "# \"hub\": download an official OpenVINO example Pi0.5 policy package.\n", "MODEL_SOURCE = \"local\" # \"local\" or \"hub\"\n", "\n", "# Used when MODEL_SOURCE == \"local\".\n", "EXPORT_DIR = \"exports/pi05-pick-place-purple-cube\"\n", "\n", "# Used when MODEL_SOURCE == \"hub\".\n", - "MODEL_REPO_ID = \"eugene123tw/pi05-pick-place-purple-cube\"\n", - "DATASET_REPO_ID = \"gtamir/pick-place-purple-cube\"\n", - "DATASET_NAME = \"pick-place-purple-cube\"\n", - "DOWNLOAD_EXAMPLE_DATASET_METADATA = False\n", + "# This official example is from https://huggingface.co/collections/OpenVINO/physical-ai.\n", + "# It is a LIBERO benchmark model, so do not deploy it on a real SO101 robot unless\n", + "# you have verified that the robot, cameras, and action schema match your setup.\n", + "MODEL_REPO_ID = \"OpenVINO/pi05-libero-fp16-ov\"\n", + "HUB_MODEL_IS_ROBOT_COMPATIBLE = False\n", "ASSETS_DIR = Path(\"physicalai_assets\").resolve()\n", "\n", "DEVICE = \"GPU\" # OpenVINO can run on \"GPU\", \"CPU\", \"NPU\".\n", - "TASK = \"pick up the box\"\n" + "TASK = \"pick up the box\"" ] }, { @@ -438,9 +429,9 @@ "source": [ "If `MODEL_SOURCE = \"local\"`, make sure `EXPORT_DIR` contains the full exported policy package, including `manifest.json`, model IR files, tokenizer files, and metadata.\n", "\n", - "If `MODEL_SOURCE = \"hub\"`, the next cell downloads the public example model package from Hugging Face. This is useful for validating the notebook before using your own model.\n", + "If `MODEL_SOURCE = \"hub\"`, the next cell downloads the official example model package from the OpenVINO Hugging Face collection. Use this path to validate model loading; for real robot deployment, use a model exported from a dataset collected with the same robot, camera names, and action schema.\n", "\n", - "\"Model\n" + "\"Model" ] }, { @@ -471,16 +462,6 @@ " local_dir_use_symlinks=False,\n", " )\n", " ).resolve()\n", - "\n", - " if DOWNLOAD_EXAMPLE_DATASET_METADATA:\n", - " dataset_dir = snapshot_download(\n", - " repo_id=DATASET_REPO_ID,\n", - " repo_type=\"dataset\",\n", - " local_dir=ASSETS_DIR / \"datasets\" / DATASET_REPO_ID.replace(\"/\", \"__\"),\n", - " allow_patterns=[f\"{DATASET_NAME}/meta/**\"],\n", - " local_dir_use_symlinks=False,\n", - " )\n", - " print(\"Downloaded example dataset metadata to:\", dataset_dir)\n", "elif MODEL_SOURCE == \"local\":\n", " EXPORT_DIR = Path(EXPORT_DIR).expanduser().resolve()\n", "else:\n", @@ -492,7 +473,7 @@ " raise FileNotFoundError(f\"Missing required model package files in {EXPORT_DIR}: {missing}\")\n", "\n", "policy = InferenceModel.load(EXPORT_DIR, backend=\"openvino\", device=DEVICE)\n", - "print(f\"Model loaded from {EXPORT_DIR} on {DEVICE}\")\n" + "print(f\"Model loaded from {EXPORT_DIR} on {DEVICE}\")" ] }, { @@ -518,6 +499,14 @@ "metadata": {}, "outputs": [], "source": [ + "if MODEL_SOURCE == \"hub\" and not HUB_MODEL_IS_ROBOT_COMPATIBLE:\n", + " raise RuntimeError(\n", + " \"The official Hugging Face example model is intended for model-loading validation. \"\n", + " \"Before running it on a real SO101 robot, verify that its robot action space, \"\n", + " \"camera count, camera names, and preprocessing metadata match your setup, then set \"\n", + " \"HUB_MODEL_IS_ROBOT_COMPATIBLE = True.\"\n", + " )\n", + "\n", "from physicalai.runtime import PolicyRuntime, SyncExecution\n", "\n", "runtime = PolicyRuntime(\n", From 1b1126ae7e84a47d56a93f06e3ddd39996720e1c Mon Sep 17 00:00:00 2001 From: yoyowz Date: Thu, 16 Jul 2026 17:50:56 +0800 Subject: [PATCH 4/5] fix: bootstrap tutorial dependencies in notebook 003 --- .../tutorials/003_OpenVINO_Optimization.ipynb | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/examples/tutorials/003_OpenVINO_Optimization.ipynb b/examples/tutorials/003_OpenVINO_Optimization.ipynb index e239b9bb..7749f3f6 100644 --- a/examples/tutorials/003_OpenVINO_Optimization.ipynb +++ b/examples/tutorials/003_OpenVINO_Optimization.ipynb @@ -49,9 +49,9 @@ "id": "3c28a826-a284-4623-8a7b-74765529bdd6", "metadata": {}, "source": [ - "Install OpenVINO Physical AI from PyPI with a pinned released version.\n", + "Install OpenVINO Physical AI from PyPI with a pinned released version. If this notebook is run outside a full repository checkout, the install cell also clones the tutorial sources into `_physicalai_tutorial_repo` so it can find `requirements.txt`.\n", "\n", - "> **Why pin the version?** The OpenVINO Physical AI APIs are evolving quickly. This notebook depends on a tested `physicalai` release, so the install cell below pins the package version. Update `PHYSICALAI_VERSION` only after re-validating the rest of this notebook." + "> **Why pin the version?** The OpenVINO Physical AI APIs are evolving quickly. This notebook depends on a tested `physicalai` release, so the install cell below pins the package version. Update `PHYSICALAI_VERSION` only after re-validating the rest of this notebook.\n" ] }, { @@ -62,9 +62,34 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "requirements_file = Path(\"requirements.txt\")\n", - "if not requirements_file.exists():\n", - " requirements_file = Path(\"notebooks/requirements.txt\")\n", + "import subprocess\n", + "import sys\n", + "\n", + "WORKSPACE = Path.cwd().resolve()\n", + "TUTORIALS_REF = \"docs/tutorials\"\n", + "TUTORIALS_REPO = \"https://github.com/openvinotoolkit/physicalai.git\"\n", + "TUTORIAL_REPO_DIR = WORKSPACE / \"_physicalai_tutorial_repo\"\n", + "\n", + "local_requirements = WORKSPACE / \"requirements.txt\"\n", + "fallback_requirements = WORKSPACE / \"notebooks\" / \"requirements.txt\"\n", + "\n", + "if local_requirements.exists():\n", + " requirements_file = local_requirements\n", + "elif fallback_requirements.exists():\n", + " requirements_file = fallback_requirements\n", + "else:\n", + " if not TUTORIAL_REPO_DIR.exists():\n", + " subprocess.check_call([\n", + " \"git\",\n", + " \"clone\",\n", + " \"--depth\",\n", + " \"1\",\n", + " \"--branch\",\n", + " TUTORIALS_REF,\n", + " TUTORIALS_REPO,\n", + " str(TUTORIAL_REPO_DIR),\n", + " ])\n", + " requirements_file = TUTORIAL_REPO_DIR / \"examples\" / \"tutorials\" / \"requirements.txt\"\n", "\n", "# Tested with this notebook. Pinning avoids breakage from fast-moving API changes.\n", "PHYSICALAI_VERSION = \"0.1.1\"\n", From b889615681d533d419172eba9020464fcd68c3d7 Mon Sep 17 00:00:00 2001 From: yoyowz Date: Fri, 31 Jul 2026 11:36:30 +0800 Subject: [PATCH 5/5] fix: align 003 deployment workflow with current APIs --- .../tutorials/003_OpenVINO_Optimization.ipynb | 701 +++++++----------- 1 file changed, 263 insertions(+), 438 deletions(-) diff --git a/examples/tutorials/003_OpenVINO_Optimization.ipynb b/examples/tutorials/003_OpenVINO_Optimization.ipynb index 7749f3f6..bae987ff 100644 --- a/examples/tutorials/003_OpenVINO_Optimization.ipynb +++ b/examples/tutorials/003_OpenVINO_Optimization.ipynb @@ -2,592 +2,417 @@ "cells": [ { "cell_type": "markdown", - "id": "4d74971e-0379-4237-96f2-d2755405ec96", + "id": "title003", "metadata": {}, "source": [ - "# OpenVINO Physical AI APIs enabling deployment of optimized model" - ] - }, - { - "cell_type": "markdown", - "id": "08eaf7c2-a51a-497b-8f98-41967e972426", - "metadata": {}, - "source": [ - "### Minimal code to run a Pi0.5 model on an SO101 robot" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "40e9e0bb-3b87-42cd-a587-ae6acc72409e", - "metadata": {}, - "source": [ - "" + "# Deploy an OpenVINO Policy on an SO101 Robot\n", + "\n", + "This notebook loads an exported robot policy with the OpenVINO Physical AI runtime, checks that its inputs and outputs match an SO101 deployment, and optionally runs it on real hardware.\n", + "\n", + "Real robot execution is disabled by default. Review the model, calibration, camera names, workspace, and emergency-stop procedure before enabling it." ] }, { "cell_type": "markdown", - "id": "39464140-04c5-4f8a-b386-6a7bb735836b", + "id": "install1", "metadata": {}, "source": [ - "## 1) Train a model using Physical AI Studio or LeRobot\n", - "* Use the Pi0.5 policy supported in Physical AI Studio or LeRobot to train a model.
\n", - "* The output of this stage should be a trained Pi0.5 model.
\n", - "* For the Physical AI Studio workflow, follow the official [Physical AI Studio documentation](https://github.com/open-edge-platform/physical-ai-studio/blob/main/application/README.md)." + "## 1) Install OpenVINO Physical AI\n", + "\n", + "The runtime APIs used below are newer than the latest PyPI release. Install the current package directly from the Physical AI repository with the SO101 and shared-camera transport extras.\n", + "\n", + "The general notebook environment should already be installed as described in [the tutorials README](README.md)." ] }, { - "cell_type": "markdown", - "id": "a34dcf91-070a-4772-ad8f-dde8c6483c57", + "cell_type": "code", + "execution_count": null, + "id": "install2", "metadata": {}, + "outputs": [], "source": [ - "## 2) Install OpenVINO Physical AI on your target platform" + "%pip install -q \"physicalai[so101,transport] @ git+https://github.com/openvinotoolkit/physicalai.git@main\"" ] }, { "cell_type": "markdown", - "id": "3c28a826-a284-4623-8a7b-74765529bdd6", + "id": "model001", "metadata": {}, "source": [ - "Install OpenVINO Physical AI from PyPI with a pinned released version. If this notebook is run outside a full repository checkout, the install cell also clones the tutorial sources into `_physicalai_tutorial_repo` so it can find `requirements.txt`.\n", + "## 2) Load an OpenVINO Policy\n", "\n", - "> **Why pin the version?** The OpenVINO Physical AI APIs are evolving quickly. This notebook depends on a tested `physicalai` release, so the install cell below pins the package version. Update `PHYSICALAI_VERSION` only after re-validating the rest of this notebook.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "907c2724-8da0-4745-bea7-2d44337e1645", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import subprocess\n", - "import sys\n", + "The default path downloads the official `OpenVINO/pi05-libero-fp16-ov` package from Hugging Face with `InferenceModel.from_pretrained`. It lets every developer validate model download and OpenVINO loading without preparing a local export.\n", "\n", - "WORKSPACE = Path.cwd().resolve()\n", - "TUTORIALS_REF = \"docs/tutorials\"\n", - "TUTORIALS_REPO = \"https://github.com/openvinotoolkit/physicalai.git\"\n", - "TUTORIAL_REPO_DIR = WORKSPACE / \"_physicalai_tutorial_repo\"\n", + "Use one of two loading paths:\n", "\n", - "local_requirements = WORKSPACE / \"requirements.txt\"\n", - "fallback_requirements = WORKSPACE / \"notebooks\" / \"requirements.txt\"\n", + "1. **Hugging Face (default):** keep the official model ID for a loading test, or replace it with your own published OpenVINO policy package.\n", + "2. **Local OpenVINO package:** point to an extracted policy package on disk. For real robot deployment, obtain this package using one of these workflows:\n", + " - **Physical AI Studio export (recommended):** train or import the policy in a current Studio version, export the OpenVINO backend, download the export, and extract it locally. This workflow preserves the manifest and runtime metadata expected by the current Physical AI APIs.\n", + " - **External training and export:** train on another platform, such as a CUDA system, then export a complete OpenVINO policy package yourself. Keep the model IR, manifest, preprocessing/postprocessing assets, normalization statistics, and feature schema together. A standalone `.xml`/`.bin` pair may not contain enough deployment metadata.\n", "\n", - "if local_requirements.exists():\n", - " requirements_file = local_requirements\n", - "elif fallback_requirements.exists():\n", - " requirements_file = fallback_requirements\n", - "else:\n", - " if not TUTORIAL_REPO_DIR.exists():\n", - " subprocess.check_call([\n", - " \"git\",\n", - " \"clone\",\n", - " \"--depth\",\n", - " \"1\",\n", - " \"--branch\",\n", - " TUTORIALS_REF,\n", - " TUTORIALS_REPO,\n", - " str(TUTORIAL_REPO_DIR),\n", - " ])\n", - " requirements_file = TUTORIAL_REPO_DIR / \"examples\" / \"tutorials\" / \"requirements.txt\"\n", - "\n", - "# Tested with this notebook. Pinning avoids breakage from fast-moving API changes.\n", - "PHYSICALAI_VERSION = \"0.1.1\"\n", - "\n", - "%pip install -q --extra-index-url https://download.pytorch.org/whl/cpu -r {requirements_file}\n", - "%pip install -q physicalai=={PHYSICALAI_VERSION}\n" + "The official Pi0.5 checkpoint is trained for the LIBERO benchmark. Its 8-dimensional state and 7-dimensional action interface are not compatible with a six-motor SO101, so keep `RUN_ON_ROBOT = False` when using this default model.\n", + "\n", + "See the [Physical AI Studio documentation](https://github.com/open-edge-platform/physical-ai-studio/blob/main/application/README.md) for its model and hardware workflows." ] }, { "cell_type": "markdown", - "id": "55c25328-afc8-48e1-9448-db67a492caaf", + "id": "model002", "metadata": {}, "source": [ - "Import libraries, and set up a working directory" + "### Action required: Select and configure a model source\n", + "\n", + "Keep `MODEL_SOURCE = \"huggingface\"` for the default loading validation, or set it to `\"local\"` for an extracted Studio export or an externally exported package.\n", + "\n", + "- For `\"huggingface\"`, keep the official model ID or set `HUGGINGFACE_MODEL_ID` to your published policy package.\n", + "- For `\"local\"`, set `LOCAL_MODEL_PATH` to the root of the extracted OpenVINO policy package.\n", + "- Set `TASK` to the instruction used by a language-conditioned policy, or `None` for a policy that does not use language.\n", + "- Current Studio exports should declare feature metadata. For an older or external package that does not, set `FALLBACK_CAMERA_NAMES` to the exact camera feature names used during data collection and training.\n", + "\n", + "Keep `RUN_ON_ROBOT = False` until the compatibility section confirms that the policy is intended for this SO101 and its six-dimensional state and action interfaces." ] }, { "cell_type": "code", "execution_count": null, - "id": "9cd0bf28-873b-4cce-bb84-21c4ef513ea2", + "id": "model003", "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", "\n", - "WORKSPACE = Path.cwd().resolve()\n", - "print(\"Workspace:\", WORKSPACE)" - ] - }, - { - "cell_type": "markdown", - "id": "65ff5c43-c32a-44ca-ac2a-9e4bee141acc", - "metadata": {}, - "source": [ - "Install extras, mainly for hardware acceleration (e.g. for SO101 or specific camera types)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28eab95d-26c2-4e88-a67d-c19b3cc7e9dc", - "metadata": {}, - "outputs": [], - "source": [ - "INSTALL_HARDWARE_DEPS = True\n", - "USE_SHARED_CAMERA = True\n", + "MODEL_SOURCE = \"huggingface\"\n", "\n", - "if INSTALL_HARDWARE_DEPS:\n", - " import subprocess\n", - " import sys\n", + "LOCAL_MODEL_PATH = Path(\"physicalai_assets/models/my-so101-openvino-export\")\n", "\n", - " hardware_extras = \"transport,so101\" if USE_SHARED_CAMERA else \"so101\"\n", - " package_spec = f\"physicalai[{hardware_extras}]=={PHYSICALAI_VERSION}\"\n", - " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", package_spec])\n", - " print(f\"[DONE] Installed PhysicalAI hardware extras: {package_spec}\")\n", - "else:\n", - " print(\"[SKIP] Hardware extras are not installed. Set INSTALL_HARDWARE_DEPS = True if this environment needs them.\")" - ] - }, - { - "cell_type": "markdown", - "id": "2694f43e-7753-4ce5-9124-7faf9bc32490", - "metadata": {}, - "source": [ - "## 3) Discover and connect to Robot and Cameras\n", + "HUGGINGFACE_MODEL_ID = \"OpenVINO/pi05-libero-fp16-ov\"\n", + "HUGGINGFACE_REVISION = None\n", + "HUGGINGFACE_CACHE_DIR = Path(\"physicalai_assets/models\")\n", "\n", - "> **Before continuing on Linux:** make sure the user running Jupyter has permission to access USB cameras and the SO101 serial device. If the robot appears as `/dev/ttyACM0`, the user usually needs to be in the `dialout` group. Camera access may require membership in the `video` group." + "DEVICE = \"AUTO\"\n", + "TASK = \"pick up the box\"\n", + "FALLBACK_CAMERA_NAMES = [\"image\", \"image2\"]\n", + "FPS = 30.0\n", + "DURATION_S = 60.0\n", + "RUN_ON_ROBOT = False" ] }, { "cell_type": "code", "execution_count": null, - "id": "linux-permission-check", + "id": "model004", "metadata": {}, "outputs": [], "source": [ - "# Optional Linux permission check for cameras and the SO101 serial device.\n", - "# If the current user is missing the required groups, run the printed commands\n", - "# in a terminal, then log out/in or restart the machine before continuing.\n", - "import getpass\n", - "import os\n", - "from pathlib import Path\n", - "\n", - "try:\n", - " import grp\n", - "except ImportError:\n", - " grp = None\n", + "from physicalai.inference import InferenceModel\n", "\n", - "user = getpass.getuser()\n", - "print(\"Current user:\", user)\n", + "if MODEL_SOURCE == \"local\":\n", + " model_path = LOCAL_MODEL_PATH.expanduser().resolve()\n", + " if not model_path.is_dir():\n", + " raise FileNotFoundError(\n", + " f\"Local policy directory not found: {model_path}. \"\n", + " \"Update LOCAL_MODEL_PATH or select MODEL_SOURCE='huggingface'.\"\n", + " )\n", + " policy = InferenceModel(\n", + " model_path,\n", + " backend=\"openvino\",\n", + " device=DEVICE,\n", + " )\n", + " print(f\"Loaded local policy: {model_path}\")\n", + "elif MODEL_SOURCE == \"huggingface\":\n", + " policy = InferenceModel.from_pretrained(\n", + " HUGGINGFACE_MODEL_ID,\n", + " revision=HUGGINGFACE_REVISION,\n", + " cache_dir=HUGGINGFACE_CACHE_DIR,\n", + " backend=\"openvino\",\n", + " device=DEVICE,\n", + " )\n", + " print(f\"Loaded Hugging Face policy: {HUGGINGFACE_MODEL_ID}\")\n", + "else:\n", + " raise ValueError(\"MODEL_SOURCE must be 'local' or 'huggingface'.\")\n", "\n", - "if grp is None:\n", - " print(\"Group membership checks are only available on Unix-like systems.\")\n", + "print(policy)\n", + "if policy.input_features:\n", + " print(\"Input features:\")\n", + " for feature in policy.input_features:\n", + " print(f\" {feature.name}: {feature.ftype}, shape={feature.shape}, dtype={feature.dtype}\")\n", "else:\n", - " groups = {g.gr_name for g in grp.getgrall() if user in g.gr_mem}\n", - " try:\n", - " groups.add(grp.getgrgid(os.getgid()).gr_name)\n", - " except KeyError:\n", - " pass\n", - " print(\"Groups:\", \" \".join(sorted(groups)))\n", - "\n", - " for group in [\"video\", \"dialout\"]:\n", - " if group not in groups:\n", - " print(f\"[ACTION REQUIRED] Add {user!r} to the {group!r} group:\")\n", - " print(f\" sudo usermod -aG {group} {user}\")\n", - "\n", - "serial_port = Path(\"/dev/ttyACM0\")\n", - "if serial_port.exists():\n", - " print(\"/dev/ttyACM0 mode/owner:\", oct(serial_port.stat().st_mode), serial_port.stat().st_uid, serial_port.stat().st_gid)\n", - " if not os.access(serial_port, os.R_OK | os.W_OK):\n", - " print(\"[ACTION REQUIRED] Current process cannot read/write /dev/ttyACM0.\")\n", - " print(\"After adding the user to dialout, restart the login session/Jupyter server.\")\n", + " print(\"The package does not declare input feature metadata; configure the documented fallbacks before deployment.\")\n", + "\n", + "if policy.output_features:\n", + " print(\"Output features:\")\n", + " for feature in policy.output_features:\n", + " print(f\" {feature.name}: {feature.ftype}, shape={feature.shape}, dtype={feature.dtype}\")\n", "else:\n", - " print(\"/dev/ttyACM0 was not found. Update SO101_PORT later if your robot uses a different serial device.\")\n" + " print(\"The package does not declare output feature metadata; confirm its SO101 action schema before deployment.\")" ] }, { "cell_type": "markdown", - "id": "7c427976-07a9-49ac-86da-8b68a2a6dbdd", + "id": "hardware", "metadata": {}, "source": [ - "
\n", - "Discover cameras.\n", - "
\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fad5138f-f397-4143-8e94-9d44ff7ad1a6", - "metadata": {}, - "outputs": [], - "source": [ - "# Discover available cameras — use the device IDs from this output in the config below\n", - "from physicalai.capture import discover_all\n", - "\n", - "for driver, devices in discover_all().items():\n", - " if not devices:\n", - " continue\n", - " print(f\"\\n[{driver}]\")\n", - " for dev in devices:\n", - " print(f\" {dev.device_id} — {dev.name}\")\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "85241b2b-6119-4bba-8986-aa2fa0f57123", - "metadata": {}, - "source": [ - "
\n", - "ACTION REQUIRED: Configure your cameras before running the next code cell.\n", - "
\n", + "## 3) Configure SO101 Hardware\n", "\n", - "
\n", - "Use the device IDs printed by the previous cell. Keep the camera names exactly the same as the names used during dataset collection and model export.\n", - "
\n", + "Skip this section when `RUN_ON_ROBOT` is `False`.\n", "\n", - "For the released Pi0.5 pick-and-place model used in this tutorial, the model expects **two** cameras. Do not add a third camera such as `table-cam` unless your exported model was trained with that camera.\n", + "For a real deployment, keep the model, calibration, and camera configuration from the same data-collection workflow:\n", "\n", - "Expected names for the example model:\n", + "1. **Studio-assisted workflow (recommended):** download the OpenVINO model export and SO101 calibration JSON from a current Physical AI Studio setup. During camera selection, use the exact camera names, resolution, and FPS from the Studio environment used to collect the training dataset.\n", + "2. **External/manual workflow:** provide a compatible OpenVINO policy package and SO101 calibration JSON produced outside current Studio, then configure the physical cameras explicitly. The feature names and ordering must still match the training dataset.\n", "\n", - "- `top-cam`\n", - "- `gripper-cam`\n", + "### Linux device permissions\n", "\n", - "\"Camera\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55d09462-1ef0-4d92-a03e-5b2b21684bad", - "metadata": {}, - "outputs": [], - "source": [ - "# Cameras - use the same names and setup as used for dataset collection and model export.\n", - "# ============================================================\n", - "# ACTION REQUIRED - Edit these before running.\n", - "# ============================================================\n", - "# Use the LEFT column from the camera discovery output as device_id.\n", - "# For UVC cameras, prefer stable /dev/v4l/by-id/... paths instead of numeric IDs.\n", - "#\n", - "# The example Pi0.5 model expects exactly two cameras:\n", - "# - \"top-cam\"\n", - "# - \"gripper-cam\"\n", - "# Remove or rename cameras only if your exported model was trained with different names.\n", - "CAMERAS = [\n", - " (\"top-cam\", \"uvc\", \"/dev/v4l/by-id/\"),\n", - " (\"gripper-cam\", \"uvc\", \"/dev/v4l/by-id/\"),\n", - "]\n", + "The notebook user needs access to the robot serial port, camera devices, and, when applicable, the OpenVINO GPU device. Check `groups` and device ownership before starting Jupyter. A typical one-time setup is:\n", "\n", - "CAMERA_WIDTH = 640\n", - "CAMERA_HEIGHT = 480\n", - "CAMERA_FPS = 30\n", + "```bash\n", + "sudo usermod -aG dialout,video,render \"$USER\"\n", + "```\n", "\n", - "# Runtime\n", - "FPS = 30\n", - "DURATION_S = 60.0\n" + "Log out and back in after changing group membership. Do not run Jupyter as root." ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "19ae4018-7f3d-4c98-917c-843c58074b8a", + "id": "robot001", "metadata": {}, "source": [ - "Define robot and get the robot ID from Physical AI Studio.\n", - "\n", - "\"Robot\n", - "\n", - "
\n", - "ACTION REQUIRED: Configure the SO101 serial port and calibration file before connecting the robot.\n", - "
\n", + "### Action required: Select the calibration source\n", "\n", - "The calibration file path depends on how you collected data:\n", + "Choose one of two calibration paths:\n", "\n", - "- Physical AI Studio: `~/.cache/physicalai/robots//calibrations/.json`\n", - "- LeRobot: `~/.cache/calibration/so101_follower.json`\n", + "1. **Studio download (recommended):** download the calibration JSON for this SO101 from a current Physical AI Studio robot configuration, place it in a local directory, and set `STUDIO_CALIBRATION_PATH`.\n", + "2. **Custom path:** for an older Studio setup, LeRobot, or another calibration workflow, set `CUSTOM_CALIBRATION_PATH` to its compatible SO101 calibration JSON.\n", "\n", - "Make sure the Jupyter user can read the calibration JSON. If you see `PermissionError`, fix file ownership/permissions or copy the calibration file to a readable path.\n" + "Set `CALIBRATION_SOURCE` to `\"studio_download\"` or `\"custom_path\"`. The notebook does not assume a Studio cache location. The selected file must belong to this physical robot and contain calibration data for all six motors." ] }, { "cell_type": "code", "execution_count": null, - "id": "so101-port-config", + "id": "robot002", "metadata": {}, "outputs": [], "source": [ - "# ============================================================\n", - "# ACTION REQUIRED - Edit these before running.\n", - "# ============================================================\n", - "# Robot serial port. Common Linux values are /dev/ttyACM0 or /dev/ttyUSB0.\n", - "SO101_PORT = \"/dev/ttyACM0\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "so101-calibration-config", - "metadata": {}, - "outputs": [], - "source": [ - "# ============================================================\n", - "# ACTION REQUIRED - Edit this calibration path before running the connect cell.\n", - "# ============================================================\n", - "# Physical AI Studio example:\n", - "# SO101_CALIBRATION = \"~/.cache/physicalai/robots//calibrations/.json\"\n", - "#\n", - "# LeRobot example:\n", - "# SO101_CALIBRATION = \"~/.cache/calibration/so101_follower.json\"\n", - "SO101_CALIBRATION = \"~/.cache/physicalai/robots//calibrations/.json\"\n", - "\n", - "SO101_CALIBRATION = str(Path(SO101_CALIBRATION).expanduser())\n", - "if \"<\" in SO101_CALIBRATION or \">\" in SO101_CALIBRATION:\n", - " raise ValueError(\"Update SO101_CALIBRATION to your actual calibration JSON path before continuing.\")\n", - "if not Path(SO101_CALIBRATION).is_file():\n", - " raise FileNotFoundError(f\"Calibration file not found: {SO101_CALIBRATION}\")\n", - "print(\"Using SO101 calibration:\", SO101_CALIBRATION)\n" - ] - }, - { - "cell_type": "markdown", - "id": "6c86a75f-9cd6-46ba-99fb-880521a689c8", - "metadata": {}, - "source": [ - "Connect cameras and robot together. This sets up the synchronized robot/camera runtime environment.\n", + "SO101_PORT = \"/dev/ttyACM0\"\n", "\n", - "
\n", - "ACTION REQUIRED: Confirm cameras, robot connection, and permissions before running the next code cell.\n", - "
\n", + "CALIBRATION_SOURCE = \"studio_download\"\n", + "STUDIO_CALIBRATION_PATH = Path(\"physicalai_assets/calibrations/so101_calibration.json\")\n", + "CUSTOM_CALIBRATION_PATH = Path(\"/path/to/existing-so101-calibration.json\")\n", "\n", - "Before running this cell:\n", + "if CALIBRATION_SOURCE == \"studio_download\":\n", + " SO101_CALIBRATION = STUDIO_CALIBRATION_PATH.expanduser()\n", + "elif CALIBRATION_SOURCE == \"custom_path\":\n", + " SO101_CALIBRATION = CUSTOM_CALIBRATION_PATH.expanduser()\n", + "else:\n", + " raise ValueError(\"CALIBRATION_SOURCE must be 'studio_download' or 'custom_path'.\")\n", "\n", - "- Make sure Physical AI Studio and other camera applications are closed.\n", - "- Confirm the current user can access the SO101 serial port, for example `/dev/ttyACM0`.\n", - "- Confirm `CAMERAS` contains exactly the camera names expected by your exported model. The example model expects `top-cam` and `gripper-cam` only.\n", - "- If a previous connection attempt failed, disconnect cameras or restart the kernel before trying again.\n" + "CAMERA_WIDTH = 640\n", + "CAMERA_HEIGHT = 480\n", + "CAMERA_FPS = 30\n", + "\n", + "CAMERA_SOURCE = \"interactive\"\n", + "MANUAL_CAMERA_CONFIGS = [\n", + " {\n", + " \"name\": \"image\",\n", + " \"camera_type\": \"uvc\",\n", + " \"init_args\": {\"device\": \"/dev/v4l/by-id/usb-example-overhead-video-index0\"},\n", + " },\n", + " {\n", + " \"name\": \"image2\",\n", + " \"camera_type\": \"uvc\",\n", + " \"init_args\": {\"device\": \"/dev/v4l/by-id/usb-example-arm-video-index0\"},\n", + " },\n", + "]" ] }, { "cell_type": "code", "execution_count": null, - "id": "b56dabbf-20f5-4896-894f-7bafc7283625", + "id": "robot003", "metadata": {}, "outputs": [], "source": [ - "from physicalai.capture import SharedCamera\n", "from physicalai.robot import SO101\n", "\n", - "# Cameras\n", - "cameras = {}\n", - "try:\n", - " for name, driver, device_id in CAMERAS:\n", - " kwargs = {\"serial_number\": device_id} if driver == \"realsense\" else {\"device\": device_id}\n", - " cam = SharedCamera(driver, **kwargs, width=CAMERA_WIDTH, height=CAMERA_HEIGHT, fps=CAMERA_FPS)\n", - " cam.connect()\n", - " cameras[name] = cam\n", - " print(f\"Camera '{name}' connected: {cam.actual_width}x{cam.actual_height} @ {cam.actual_fps}fps\")\n", - "\n", - " # Robot\n", - " robot = SO101(port=SO101_PORT, calibration=SO101_CALIBRATION, role=\"follower\")\n", - " robot.connect()\n", - " print(f\"Robot connected on {SO101_PORT}\")\n", - "except Exception:\n", - " for cam in cameras.values():\n", - " try:\n", - " cam.disconnect()\n", - " except Exception:\n", - " pass\n", - " raise\n" + "robot = None\n", + "if RUN_ON_ROBOT:\n", + " if not SO101_CALIBRATION.is_file():\n", + " raise FileNotFoundError(f\"Calibration file not found: {SO101_CALIBRATION}\")\n", + " robot = SO101(\n", + " port=SO101_PORT,\n", + " calibration=SO101_CALIBRATION,\n", + " role=\"follower\",\n", + " )\n", + "else:\n", + " print(\"Robot setup skipped. Set RUN_ON_ROBOT = True after validating a compatible policy.\")" ] }, { "cell_type": "markdown", - "id": "66334671-4691-44d7-a37b-59bc92693798", + "id": "compat01", "metadata": {}, "source": [ - "## 4) Load the model\n", + "## 4) Validate Deployment Compatibility\n", "\n", - "This notebook supports two model sources:\n", + "The runtime maps robot observations and named camera frames to the public feature schema in `policy.input_features`. The model state and action dimensions must match the robot joint count. Multi-camera names must match the suffixes of the model's visual feature names.\n", "\n", - "1. **Your own exported model** from Physical AI Studio or LeRobot. Set `MODEL_SOURCE = \"local\"` and point `EXPORT_DIR` to the exported OpenVINO policy package. Use this path for real SO101 robot deployment.\n", - "2. **The official OpenVINO example model** from Hugging Face. Set `MODEL_SOURCE = \"hub\"`; the notebook downloads a released OpenVINO policy package from the official [OpenVINO Physical AI collection](https://huggingface.co/collections/OpenVINO/physical-ai). This is useful for validating model package download and loading before using your own exported model.\n", - "\n", - "The exported model must match your runtime setup, especially the robot action space and the number and names of cameras." + "This validation uses only public APIs; it does not construct a private runtime input." ] }, { "cell_type": "code", "execution_count": null, - "id": "4f9581ef-405e-4719-a201-8908a2940414", + "id": "compat02", "metadata": {}, "outputs": [], "source": [ - "# ============================================================\n", - "# ACTION REQUIRED - Choose model source and deployment device.\n", - "# ============================================================\n", - "# \"local\": use your own exported model from Physical AI Studio or LeRobot.\n", - "# \"hub\": download an official OpenVINO example Pi0.5 policy package.\n", - "MODEL_SOURCE = \"local\" # \"local\" or \"hub\"\n", - "\n", - "# Used when MODEL_SOURCE == \"local\".\n", - "EXPORT_DIR = \"exports/pi05-pick-place-purple-cube\"\n", - "\n", - "# Used when MODEL_SOURCE == \"hub\".\n", - "# This official example is from https://huggingface.co/collections/OpenVINO/physical-ai.\n", - "# It is a LIBERO benchmark model, so do not deploy it on a real SO101 robot unless\n", - "# you have verified that the robot, cameras, and action schema match your setup.\n", - "MODEL_REPO_ID = \"OpenVINO/pi05-libero-fp16-ov\"\n", - "HUB_MODEL_IS_ROBOT_COMPATIBLE = False\n", - "ASSETS_DIR = Path(\"physicalai_assets\").resolve()\n", - "\n", - "DEVICE = \"GPU\" # OpenVINO can run on \"GPU\", \"CPU\", \"NPU\".\n", - "TASK = \"pick up the box\"" - ] - }, - { - "cell_type": "markdown", - "id": "d4ed8ff1-4c78-4060-b9d2-6e8fb9e9c891", - "metadata": {}, - "source": [ - "If `MODEL_SOURCE = \"local\"`, make sure `EXPORT_DIR` contains the full exported policy package, including `manifest.json`, model IR files, tokenizer files, and metadata.\n", + "def features_of_type(features, feature_type):\n", + " return [feature for feature in features if feature.ftype == feature_type]\n", "\n", - "If `MODEL_SOURCE = \"hub\"`, the next cell downloads the official example model package from the OpenVINO Hugging Face collection. Use this path to validate model loading; for real robot deployment, use a model exported from a dataset collected with the same robot, camera names, and action schema.\n", "\n", - "\"Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "667d3981-8a63-433e-9e43-f62952331c01", - "metadata": {}, - "outputs": [], - "source": [ - "import openvino_tokenizers # noqa: F401 - registers OpenVINO tokenizer custom ops\n", - "from huggingface_hub import snapshot_download\n", - "from physicalai.inference import InferenceModel\n", + "visual_features = features_of_type(policy.input_features, \"VISUAL\")\n", + "state_features = features_of_type(policy.input_features, \"STATE\")\n", + "action_features = features_of_type(policy.output_features, \"ACTION\")\n", "\n", - "if MODEL_SOURCE == \"hub\":\n", - " EXPORT_DIR = Path(\n", - " snapshot_download(\n", - " repo_id=MODEL_REPO_ID,\n", - " local_dir=ASSETS_DIR / \"models\" / MODEL_REPO_ID.replace(\"/\", \"__\"),\n", - " allow_patterns=[\n", - " \"manifest.json\",\n", - " \"pi05.xml\",\n", - " \"pi05.bin\",\n", - " \"tokenizer.xml\",\n", - " \"tokenizer.bin\",\n", - " \"metadata.yaml\",\n", - " \"README.md\",\n", - " ],\n", - " local_dir_use_symlinks=False,\n", - " )\n", - " ).resolve()\n", - "elif MODEL_SOURCE == \"local\":\n", - " EXPORT_DIR = Path(EXPORT_DIR).expanduser().resolve()\n", + "declared_camera_names = [\n", + " feature.name.removeprefix(\"images.\")\n", + " for feature in visual_features\n", + "]\n", + "if declared_camera_names:\n", + " expected_camera_names = declared_camera_names\n", + " camera_name_source = \"model feature metadata\"\n", "else:\n", - " raise ValueError(\"MODEL_SOURCE must be 'local' or 'hub'.\")\n", + " expected_camera_names = list(FALLBACK_CAMERA_NAMES)\n", + " camera_name_source = \"FALLBACK_CAMERA_NAMES\"\n", + " if not expected_camera_names:\n", + " raise ValueError(\n", + " \"The model has no visual feature metadata. \"\n", + " \"Set FALLBACK_CAMERA_NAMES to the camera names used during training.\"\n", + " )\n", "\n", - "required_model_files = [\"manifest.json\", \"pi05.xml\", \"pi05.bin\", \"tokenizer.xml\", \"tokenizer.bin\"]\n", - "missing = [name for name in required_model_files if not (EXPORT_DIR / name).exists()]\n", - "if missing:\n", - " raise FileNotFoundError(f\"Missing required model package files in {EXPORT_DIR}: {missing}\")\n", + "if len(set(expected_camera_names)) != len(expected_camera_names):\n", + " raise ValueError(f\"Camera names must be unique: {expected_camera_names}\")\n", "\n", - "policy = InferenceModel.load(EXPORT_DIR, backend=\"openvino\", device=DEVICE)\n", - "print(f\"Model loaded from {EXPORT_DIR} on {DEVICE}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3e99fb03-e422-4c37-beb7-6a0d70783606", - "metadata": {}, - "source": [ - "## 5) Run the model (policy) on the robot" + "state_dims = sorted({feature.shape[-1] for feature in state_features if feature.shape})\n", + "action_dims = sorted({feature.shape[-1] for feature in action_features if feature.shape})\n", + "\n", + "print(f\"Expected cameras ({camera_name_source}): {expected_camera_names}\")\n", + "print(f\"Expected state dimensions: {state_dims or 'none declared'}\")\n", + "print(f\"Expected action dimensions: {action_dims or 'none declared'}\")\n", + "\n", + "if RUN_ON_ROBOT:\n", + " robot_dim = len(robot.joint_names)\n", + " if state_dims and state_dims != [robot_dim]:\n", + " raise ValueError(f\"Model state dimensions {state_dims} do not match SO101 joint count {robot_dim}.\")\n", + " if action_dims and action_dims != [robot_dim]:\n", + " raise ValueError(f\"Model action dimensions {action_dims} do not match SO101 joint count {robot_dim}.\")\n", + " print(f\"SO101 state/action dimensions match its {robot_dim} joints.\")" ] }, { "cell_type": "markdown", - "id": "13d14a2f-fe48-4c08-9846-775278cdb7e8", + "id": "camera01", "metadata": {}, "source": [ - "Use OpenVINO Physical AI to perform Inference." + "### Action required: Configure cameras\n", + "\n", + "Choose one of two camera paths:\n", + "\n", + "1. **Interactive selection (recommended):** keep `CAMERA_SOURCE = \"interactive\"`. The public `select_cameras_interactive` API discovers connected cameras. Select one index at a time and assign the exact feature names printed by the compatibility cell. For a Studio-trained policy, use the names from the Studio environment used during data collection.\n", + "2. **Manual configuration:** set `CAMERA_SOURCE = \"manual\"` and update `MANUAL_CAMERA_CONFIGS`. This is useful for an external setup or unattended reruns. Prefer stable Linux device IDs such as `/dev/v4l/by-id/...-video-index0` instead of `/dev/videoN`.\n", + "\n", + "Both paths create shared cameras through public Physical AI APIs. They are not connected yet; the `RobotRuntime` context manager connects and disconnects them together with the robot." ] }, { "cell_type": "code", "execution_count": null, - "id": "0d363fee-8116-4662-86fa-a074ac6ecb63", + "id": "camera02", "metadata": {}, "outputs": [], "source": [ - "if MODEL_SOURCE == \"hub\" and not HUB_MODEL_IS_ROBOT_COMPATIBLE:\n", - " raise RuntimeError(\n", - " \"The official Hugging Face example model is intended for model-loading validation. \"\n", - " \"Before running it on a real SO101 robot, verify that its robot action space, \"\n", - " \"camera count, camera names, and preprocessing metadata match your setup, then set \"\n", - " \"HUB_MODEL_IS_ROBOT_COMPATIBLE = True.\"\n", - " )\n", + "from physicalai.capture import create_camera, select_cameras_interactive\n", "\n", - "from physicalai.runtime import PolicyRuntime, SyncExecution\n", - "\n", - "runtime = PolicyRuntime(\n", - " robot=robot,\n", - " model=policy,\n", - " execution=SyncExecution(request_threshold=0.5),\n", - " fps=FPS,\n", - " cameras=cameras,\n", - " task=TASK,\n", - ")\n", - "\n", - "# Optional sanity check: the example Pi0.5 model expects two camera inputs.\n", - "sample_obs = runtime._build_model_input()\n", - "print(\"Runtime model inputs:\")\n", - "for key, value in sample_obs.items():\n", - " print(\" \", key, getattr(value, \"shape\", None))\n", - "image_keys = [key for key in sample_obs if key.startswith(\"images.\")]\n", - "if len(image_keys) != len(CAMERAS):\n", - " raise RuntimeError(f\"Camera input mismatch: CAMERAS has {len(CAMERAS)} entries but runtime built {len(image_keys)} image inputs.\")\n", - "\n", - "with runtime:\n", - " print(f\"Running policy at {FPS} fps for {DURATION_S}s - task: {TASK!r}\")\n", - " stats = runtime.run(duration_s=DURATION_S)\n", - "\n", - "print(f\"\\nDone - {stats.steps} steps, {stats.inference_count} inferences, {stats.total_holds} holds\")\n" + "cameras = {}\n", + "if RUN_ON_ROBOT:\n", + " if CAMERA_SOURCE == \"interactive\":\n", + " cameras = select_cameras_interactive(\n", + " width=CAMERA_WIDTH,\n", + " height=CAMERA_HEIGHT,\n", + " fps=CAMERA_FPS,\n", + " )\n", + " elif CAMERA_SOURCE == \"manual\":\n", + " for config in MANUAL_CAMERA_CONFIGS:\n", + " name = config[\"name\"]\n", + " if name in cameras:\n", + " raise ValueError(f\"Duplicate manual camera name: {name}\")\n", + " cameras[name] = create_camera(\n", + " config[\"camera_type\"],\n", + " shared=True,\n", + " width=CAMERA_WIDTH,\n", + " height=CAMERA_HEIGHT,\n", + " fps=CAMERA_FPS,\n", + " **config[\"init_args\"],\n", + " )\n", + " else:\n", + " raise ValueError(\"CAMERA_SOURCE must be 'interactive' or 'manual'.\")\n", + "\n", + " if len(cameras) != len(expected_camera_names):\n", + " raise ValueError(\n", + " f\"Selected {len(cameras)} camera(s), but {camera_name_source} \"\n", + " f\"expects {len(expected_camera_names)}: {expected_camera_names}.\"\n", + " )\n", + " if set(cameras) != set(expected_camera_names):\n", + " raise ValueError(\n", + " f\"Camera names {sorted(cameras)} do not match the expected names \"\n", + " f\"{sorted(expected_camera_names)} from {camera_name_source}.\"\n", + " )\n", + "else:\n", + " print(\"Camera selection skipped.\")" ] }, { "cell_type": "markdown", - "id": "c2cb63f2-e963-484d-aa8c-57fbf499aa1c", + "id": "runtime1", "metadata": {}, "source": [ - "## 6) Disconnect" + "## 5) Run the Policy on the Robot\n", + "\n", + "Clear the robot workspace and make the emergency stop accessible before running this cell. The `RobotRuntime` context manager owns hardware connection and cleanup, including cleanup after an exception or keyboard interrupt." ] }, { "cell_type": "code", "execution_count": null, - "id": "bae9cb87-132b-4a12-a412-80bf3d4e5760", + "id": "runtime2", "metadata": {}, "outputs": [], "source": [ - "for name, cam in cameras.items():\n", - " try:\n", - " cam.disconnect()\n", - " print(f\"Camera '{name}' disconnected\")\n", - " except Exception as exc:\n", - " print(f\"Camera '{name}' disconnect warning: {exc}\")\n", - "\n", - "try:\n", - " robot.disconnect()\n", - " print(\"Robot disconnected\")\n", - "except NameError:\n", - " print(\"Robot was not created\")\n", - "except Exception as exc:\n", - " print(f\"Robot disconnect warning: {exc}\")\n" + "from physicalai.runtime import PolicySource, RobotRuntime, SyncExecution\n", + "\n", + "if RUN_ON_ROBOT:\n", + " execution = SyncExecution(request_threshold=0.5)\n", + " policy_source = PolicySource(\n", + " model=policy,\n", + " execution=execution,\n", + " task=TASK,\n", + " )\n", + " runtime = RobotRuntime(\n", + " robot=robot,\n", + " action_source=policy_source,\n", + " fps=FPS,\n", + " cameras=cameras,\n", + " )\n", + "\n", + " with runtime:\n", + " steps = runtime.run(duration_s=DURATION_S)\n", + "\n", + " print(f\"Completed steps: {steps}\")\n", + " print(f\"Inference requests after warmup: {execution.inference_count}\")\n", + " print(f\"Action queue holds: {policy_source.action_queue.total_holds}\")\n", + "else:\n", + " print(\"Deployment skipped. Set RUN_ON_ROBOT = True only with a compatible SO101 policy.\")" ] } ],