Add bidirectional clipboard sync for Remote Desktop

A new CLIPBOARD message type carries a JSON envelope so viewers and
the host can swap clipboards explicitly:

  {"kind": "text", "text": "..."}
  {"kind": "image", "format": "png", "data_b64": "..."}

Existing utils/clipboard/clipboard.py is extended with
get_clipboard_image / set_clipboard_image. Windows uses CF_DIB via
ctypes (Pillow rasterises PNG -> BMP -> DIB); Linux shells out to
'xclip -t image/png'; macOS get works via Pillow ImageGrab and set
raises a clear NotImplementedError pending a PyObjC backend.

Host: broadcast_clipboard_text / broadcast_clipboard_image push to
every authenticated viewer; incoming CLIPBOARD messages from a viewer
are decoded and applied to the host's local clipboard via the helpers
above.

Viewer: send_clipboard_text / send_clipboard_image push to the host;
incoming CLIPBOARD messages fire an on_clipboard(kind, data) callback
so the GUI / library user controls when (and whether) to set the local
clipboard. Sync is explicit per-call — no auto-polling that could
create paste loops between the two sides.

Tests cover the JSON serialisation contract (text + image, malformed
input, unknown kinds, missing fields) and end-to-end host<->viewer
flow with a recording host that captures apply calls instead of
touching the OS clipboard.

Author: JE-Chen

je_auto_control/utils/clipboard/clipboard.py

@@ -1,15 +1,19 @@
-"""Headless cross-platform text clipboard.
+"""Headless cross-platform text + image clipboard.
 
-Windows uses Win32 clipboard API via ctypes.
-macOS shells out to pbcopy / pbpaste.
-Linux shells out to xclip or xsel (whichever is available).
+Windows uses Win32 clipboard API via ctypes (CF_UNICODETEXT for text,
+CF_DIB for image).
+macOS shells out to pbcopy / pbpaste for text; image support requires
+PyObjC and is best effort.
+Linux shells out to xclip / xsel for text and ``xclip -t image/png`` for
+images.
 
 All functions raise ``RuntimeError`` if the platform backend is missing so
 callers can degrade gracefully.
 """
 import shutil
 import subprocess  # nosec B404  # reason: required for pbcopy/pbpaste/xclip/xsel
 import sys
+from io import BytesIO
 from typing import Optional
 
 
@@ -35,6 +39,30 @@ def set_clipboard(text: str) -> None:
     _linux_set(text)
 
 
+def get_clipboard_image() -> Optional[bytes]:
+    """Return the clipboard's image as PNG bytes, or ``None`` if no image."""
+    if sys.platform.startswith("win"):
+        return _win_get_image()
+    if sys.platform == "darwin":
+        return _mac_get_image()
+    return _linux_get_image()
+
+
+def set_clipboard_image(png_bytes: bytes) -> None:
+    """Place a PNG image (as bytes) onto the clipboard."""
+    if not isinstance(png_bytes, (bytes, bytearray)):
+        raise TypeError("set_clipboard_image expects bytes")
+    if not png_bytes:
+        raise ValueError("png_bytes is empty")
+    if sys.platform.startswith("win"):
+        _win_set_image(bytes(png_bytes))
+        return
+    if sys.platform == "darwin":
+        _mac_set_image(bytes(png_bytes))
+        return
+    _linux_set_image(bytes(png_bytes))
+
+
 # === Windows backend =========================================================
 
 def _win_get() -> str:
@@ -160,3 +188,125 @@ def _linux_set(text: str) -> None:
         write_cmd, input=text.encode("utf-8"),
         check=True, timeout=5,
     )
+
+
+# === Image clipboard backends ===============================================
+
+
+def _win_get_image() -> Optional[bytes]:
+    """Return the Windows clipboard image as PNG bytes, or None."""
+    try:
+        from PIL import ImageGrab  # noqa: PLC0415  lazy import
+    except ImportError as error:
+        raise RuntimeError(
+            "Pillow is required for clipboard image support"
+        ) from error
+    image = ImageGrab.grabclipboard()
+    if image is None or isinstance(image, list):
+        return None
+    buffer = BytesIO()
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+    image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def _win_set_image(png_bytes: bytes) -> None:
+    """Set the Windows clipboard image from PNG bytes (CF_DIB)."""
+    try:
+        from PIL import Image  # noqa: PLC0415  lazy import
+    except ImportError as error:
+        raise RuntimeError(
+            "Pillow is required for clipboard image support"
+        ) from error
+    image = Image.open(BytesIO(png_bytes))
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+    bmp_buf = BytesIO()
+    image.save(bmp_buf, format="BMP")
+    # CF_DIB excludes the 14-byte BITMAPFILEHEADER prefix that BMP files use.
+    dib = bmp_buf.getvalue()[14:]
+
+    import ctypes  # noqa: PLC0415
+    from ctypes import wintypes  # noqa: PLC0415
+
+    user32 = ctypes.WinDLL("user32", use_last_error=True)
+    kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    cf_dib = 8
+    gmem_moveable = 0x0002
+
+    user32.OpenClipboard.argtypes = [wintypes.HWND]
+    user32.OpenClipboard.restype = wintypes.BOOL
+    user32.EmptyClipboard.restype = wintypes.BOOL
+    user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE]
+    user32.SetClipboardData.restype = wintypes.HANDLE
+    user32.CloseClipboard.restype = wintypes.BOOL
+    kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t]
+    kernel32.GlobalAlloc.restype = wintypes.HGLOBAL
+    kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL]
+    kernel32.GlobalLock.restype = ctypes.c_void_p
+    kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL]
+
+    handle = kernel32.GlobalAlloc(gmem_moveable, len(dib))
+    if not handle:
+        raise RuntimeError("GlobalAlloc failed")
+    pointer = kernel32.GlobalLock(handle)
+    if not pointer:
+        raise RuntimeError("GlobalLock failed")
+    ctypes.memmove(pointer, dib, len(dib))
+    kernel32.GlobalUnlock(handle)
+    if not user32.OpenClipboard(None):
+        raise RuntimeError("OpenClipboard failed")
+    try:
+        user32.EmptyClipboard()
+        if not user32.SetClipboardData(cf_dib, handle):
+            raise RuntimeError("SetClipboardData(CF_DIB) failed")
+    finally:
+        user32.CloseClipboard()
+
+
+def _mac_get_image() -> Optional[bytes]:
+    """Read clipboard image via Pillow's ImageGrab; raises if PIL missing."""
+    try:
+        from PIL import ImageGrab  # noqa: PLC0415
+    except ImportError as error:
+        raise RuntimeError(
+            "Pillow is required for clipboard image support on macOS"
+        ) from error
+    image = ImageGrab.grabclipboard()
+    if image is None or isinstance(image, list):
+        return None
+    buffer = BytesIO()
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+    image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def _mac_set_image(_png_bytes: bytes) -> None:
+    raise RuntimeError(
+        "Setting clipboard images on macOS requires PyObjC; not yet supported"
+    )
+
+
+def _linux_get_image() -> Optional[bytes]:
+    if not shutil.which("xclip"):
+        raise RuntimeError("Install xclip for Linux clipboard image support")
+    # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit
+    result = subprocess.run(  # nosec B603 B607  # reason: hard-coded argv to xclip
+        ["xclip", "-selection", "clipboard", "-t", "image/png", "-o"],
+        capture_output=True, check=False, timeout=5,
+    )
+    if result.returncode != 0 or not result.stdout:
+        return None
+    return result.stdout
+
+
+def _linux_set_image(png_bytes: bytes) -> None:
+    if not shutil.which("xclip"):
+        raise RuntimeError("Install xclip for Linux clipboard image support")
+    # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit
+    subprocess.run(  # nosec B603 B607  # reason: hard-coded argv to xclip
+        ["xclip", "-selection", "clipboard", "-t", "image/png", "-i"],
+        input=png_bytes, check=True, timeout=5,
+    )

je_auto_control/utils/remote_desktop/__init__.py

@@ -13,6 +13,9 @@
     AudioBackendError, AudioCapture, AudioPlayer,
     is_audio_backend_available,
 )
+from je_auto_control.utils.remote_desktop.clipboard_sync import (
+    ClipboardSyncError,
+)
 from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost
 from je_auto_control.utils.remote_desktop.host_id import (
     HostIdError, format_host_id, generate_host_id, load_or_create_host_id,
@@ -42,4 +45,5 @@
     "load_or_create_host_id", "parse_host_id", "validate_host_id",
     "AudioBackendError", "AudioCapture", "AudioPlayer",
     "is_audio_backend_available",
+    "ClipboardSyncError",
 ]

je_auto_control/utils/remote_desktop/clipboard_sync.py

@@ -0,0 +1,72 @@
+"""Serialization helpers for CLIPBOARD messages.
+
+The wire format is a JSON envelope so adding new payload kinds (rich
+text, file lists, ...) doesn't require touching the framing layer:
+
+* ``{"kind": "text", "text": "..."}``
+* ``{"kind": "image", "format": "png", "data_b64": "..."}``
+"""
+import base64
+import json
+from typing import Any, Dict, Tuple
+
+
+class ClipboardSyncError(ValueError):
+    """Raised when a CLIPBOARD payload is malformed or unsupported."""
+
+
+def encode_text(text: str) -> bytes:
+    """Encode a text-clipboard payload."""
+    if not isinstance(text, str):
+        raise TypeError("text must be a string")
+    return json.dumps(
+        {"kind": "text", "text": text}, ensure_ascii=False,
+    ).encode("utf-8")
+
+
+def encode_image(png_bytes: bytes) -> bytes:
+    """Encode a PNG image as a clipboard payload."""
+    if not isinstance(png_bytes, (bytes, bytearray)):
+        raise TypeError("png_bytes must be bytes")
+    if not png_bytes:
+        raise ValueError("png_bytes is empty")
+    return json.dumps({
+        "kind": "image",
+        "format": "png",
+        "data_b64": base64.b64encode(bytes(png_bytes)).decode("ascii"),
+    }, ensure_ascii=False).encode("utf-8")
+
+
+def decode(payload: bytes) -> Tuple[str, Any]:
+    """Parse a CLIPBOARD payload; return ``(kind, data)``.
+
+    For ``"text"`` ``data`` is a ``str``; for ``"image"`` it is the raw
+    PNG bytes (already base64-decoded).
+    """
+    try:
+        envelope: Dict[str, Any] = json.loads(payload.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as error:
+        raise ClipboardSyncError(f"invalid CLIPBOARD JSON: {error}") from error
+    if not isinstance(envelope, dict):
+        raise ClipboardSyncError("CLIPBOARD payload must be a JSON object")
+    kind = envelope.get("kind")
+    if kind == "text":
+        text = envelope.get("text")
+        if not isinstance(text, str):
+            raise ClipboardSyncError("text payload missing 'text' string")
+        return ("text", text)
+    if kind == "image":
+        if envelope.get("format") != "png":
+            raise ClipboardSyncError(
+                f"image format {envelope.get('format')!r} not supported"
+            )
+        encoded = envelope.get("data_b64", "")
+        if not isinstance(encoded, str):
+            raise ClipboardSyncError("image payload missing 'data_b64'")
+        try:
+            return ("image", base64.b64decode(encoded))
+        except (ValueError, TypeError) as error:
+            raise ClipboardSyncError(
+                f"invalid base64 image payload: {error}"
+            ) from error
+    raise ClipboardSyncError(f"unknown clipboard kind: {kind!r}")

je_auto_control/utils/remote_desktop/host.py

@@ -17,6 +17,9 @@
 from je_auto_control.utils.remote_desktop.auth import (
     NONCE_BYTES, make_nonce, verify_response,
 )
+from je_auto_control.utils.remote_desktop.clipboard_sync import (
+    ClipboardSyncError, decode as decode_clipboard, encode_image, encode_text,
+)
 from je_auto_control.utils.remote_desktop.host_id import (
     load_or_create_host_id, validate_host_id,
 )
@@ -204,11 +207,31 @@ def _recv_loop(self) -> None:
             if msg_type is MessageType.INPUT:
                 self._handle_input_payload(payload)
                 continue
+            if msg_type is MessageType.CLIPBOARD:
+                self._handle_clipboard_payload(payload)
+                continue
             autocontrol_logger.info(
                 "remote_desktop unexpected msg %s from %s",
                 msg_type.name, self._address,
             )
 
+    def _handle_clipboard_payload(self, payload: bytes) -> None:
+        try:
+            kind, data = decode_clipboard(payload)
+        except ClipboardSyncError as error:
+            autocontrol_logger.info(
+                "remote_desktop bad CLIPBOARD from %s: %r",
+                self._address, error,
+            )
+            return
+        try:
+            self._host._apply_clipboard(kind, data)
+        except (OSError, RuntimeError, TypeError, ValueError) as error:
+            autocontrol_logger.warning(
+                "remote_desktop clipboard apply failed for %s: %r",
+                self._address, error,
+            )
+
     def _handle_input_payload(self, payload: bytes) -> None:
         try:
             message = json.loads(payload.decode("utf-8"))
@@ -428,6 +451,47 @@ def _broadcast_audio(self, chunk: bytes) -> None:
         for client in clients:
             client.push_audio(chunk)
 
+    def broadcast_clipboard_text(self, text: str) -> int:
+        """Send a text-clipboard message to every authenticated viewer."""
+        return self._broadcast_clipboard_payload(encode_text(text))
+
+    def broadcast_clipboard_image(self, png_bytes: bytes) -> int:
+        """Send a PNG image to every authenticated viewer's clipboard."""
+        return self._broadcast_clipboard_payload(encode_image(png_bytes))
+
+    def _broadcast_clipboard_payload(self, payload: bytes) -> int:
+        with self._clients_lock:
+            clients = [c for c in self._clients
+                       if c.authenticated and not c._shutdown.is_set()]
+        sent = 0
+        for client in clients:
+            try:
+                client._channel.send_typed(MessageType.CLIPBOARD, payload)
+                sent += 1
+            except (OSError, ConnectionError) as error:
+                autocontrol_logger.info(
+                    "remote_desktop clipboard send to %s failed: %r",
+                    client.address, error,
+                )
+                client.stop()
+        return sent
+
+    def _apply_clipboard(self, kind: str, data: Any) -> None:
+        """Set this host's local clipboard from a decoded CLIPBOARD payload.
+
+        Subclasses or tests may override; the default routes to the
+        utils.clipboard helpers and accepts ``"text"`` / ``"image"`` kinds.
+        """
+        from je_auto_control.utils.clipboard.clipboard import (
+            set_clipboard, set_clipboard_image,
+        )
+        if kind == "text":
+            set_clipboard(data)
+        elif kind == "image":
+            set_clipboard_image(data)
+        else:
+            raise ValueError(f"unsupported clipboard kind: {kind!r}")
+
     # internals -----------------------------------------------------------
 
     def _accept_loop(self) -> None:

je_auto_control/utils/remote_desktop/protocol.py

@@ -33,6 +33,7 @@ class MessageType(enum.IntEnum):
     FRAME = 0x10  # host -> viewer: JPEG frame
     AUDIO = 0x11  # host -> viewer: PCM audio chunk
     INPUT = 0x20  # viewer -> host: JSON input message
+    CLIPBOARD = 0x21  # either way: clipboard payload (text or image)
     PING = 0x30  # either way: liveness