From 63b0ffd310828363bcaf19211991aa614c397487 Mon Sep 17 00:00:00 2001 From: Alexander Poon Date: Sun, 2 Aug 2026 21:21:31 -0400 Subject: [PATCH 1/3] Add BodyMaps AI assistant: streaming chat, organ measurement, and vision --- .../src/components/AIAssistant/AISidebar.css | 617 +++++++++- .../src/components/AIAssistant/AISidebar.tsx | 1080 ++++++++++++----- .../src/components/AIAssistant/types.ts | 35 + PanTS-Demo/src/components/MeshViewer.tsx | 5 +- PanTS-Demo/src/routes/VisualizationPage.css | 27 +- PanTS-Demo/src/routes/VisualizationPage.tsx | 125 +- flask-server/.env.example | 109 +- flask-server/api/api_blueprint.py | 954 +++++++++++++-- flask-server/api/utils.py | 25 +- flask-server/scripts/precompute_meshes.py | 144 +++ flask-server/services/ollama_client.py | 156 ++- 11 files changed, 2757 insertions(+), 520 deletions(-) create mode 100644 flask-server/scripts/precompute_meshes.py diff --git a/PanTS-Demo/src/components/AIAssistant/AISidebar.css b/PanTS-Demo/src/components/AIAssistant/AISidebar.css index 2b1ea4a..5d006d4 100644 --- a/PanTS-Demo/src/components/AIAssistant/AISidebar.css +++ b/PanTS-Demo/src/components/AIAssistant/AISidebar.css @@ -8,7 +8,8 @@ height: 100dvh; overflow: hidden; color: var(--vp-text, rgba(255, 255, 255, 0.92)); - background: rgba(14, 15, 18, 0.86); + /* Match the top toolbar's "clear black" (.vp-topbar background) exactly. */ + background: rgba(14, 15, 18, 0.92); border-left: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); box-shadow: 0 24px 60px -20px rgba(0, 0, 0, 0.8); backdrop-filter: blur(18px) saturate(140%); @@ -199,59 +200,19 @@ } } -.ai-sidebar__suggestions { - display: flex; - gap: 7px; - overflow-x: auto; - padding: 0 16px 10px; - scrollbar-width: none; -} - -.ai-sidebar__suggestions::-webkit-scrollbar { - display: none; -} - -.ai-suggestion { - flex: 0 0 auto; - max-width: 245px; - overflow: hidden; - padding: 6px 9px; - border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); - border-radius: 999px; - background: transparent; - color: var(--vp-text-faint, rgba(255, 255, 255, 0.36)); - cursor: pointer; - font: inherit; - font-size: 11px; - line-height: 1.25; - text-overflow: ellipsis; - white-space: nowrap; - transition: color 140ms ease, background 140ms ease, border-color 140ms ease; -} - -.ai-suggestion:hover { - color: var(--vp-text-dim, rgba(255, 255, 255, 0.5)); - background: var(--vp-panel, rgba(255, 255, 255, 0.045)); - border-color: var(--vp-border-strong, rgba(255, 255, 255, 0.16)); -} - .ai-sidebar__composer-wrap { flex: 0 0 auto; padding: 0 14px max(14px, env(safe-area-inset-bottom)); } +/* A distinct, slightly darker box than the panel so the typing area is easy + to see, with a visible (but not flashy) outline. No bright focus highlight. */ .ai-composer { overflow: visible; - border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); + border: 1px solid rgba(255, 255, 255, 0.16); border-radius: 16px; - background: rgba(255, 255, 255, 0.048); - box-shadow: 0 12px 28px rgba(0, 0, 0, 0.2); - transition: border-color 140ms ease, background 140ms ease; -} - -.ai-composer:focus-within { - border-color: var(--vp-border-strong, rgba(255, 255, 255, 0.16)); - background: rgba(255, 255, 255, 0.058); + background: rgba(0, 0, 0, 0.32); + box-shadow: none; } .ai-composer__textarea { @@ -292,7 +253,7 @@ .ai-model-picker { position: relative; min-width: 0; - max-width: calc(100% - 45px); + max-width: 100%; } .ai-model-picker__button { @@ -379,6 +340,13 @@ -webkit-backdrop-filter: blur(18px) saturate(130%); } +/* Anchored to the picker's right edge so it opens leftward and stays inside + the sidebar when the model picker sits next to Send. */ +.ai-model-menu--right { + left: auto; + right: 0; +} + .ai-model-menu__heading { padding: 6px 8px 7px; color: rgba(255, 255, 255, 0.43); @@ -395,7 +363,7 @@ justify-content: space-between; gap: 14px; width: 100%; - min-height: 48px; + min-height: 34px; padding: 8px 9px; border: 0; border-radius: 8px; @@ -494,12 +462,34 @@ transform: translate(0.5px, -0.5px); } -.ai-sidebar button:focus-visible, -.ai-sidebar textarea:focus-visible { +/* Stop button (shown while the assistant is generating). */ +.ai-composer__send--stop { + background: #fff; + opacity: 1; +} + +.ai-composer__send--stop:hover { + background: #fff; +} + +.ai-stop-square { + width: 10px; + height: 10px; + border-radius: 2.5px; + background: #08090b; +} + +.ai-sidebar button:focus-visible { outline: 1px solid rgba(255, 255, 255, 0.48); outline-offset: 2px; } +/* The typing area itself never shows a focus box. */ +.ai-composer__textarea:focus, +.ai-composer__textarea:focus-visible { + outline: none; +} + @media (max-width: 899px) { .ai-sidebar { width: 100vw; @@ -510,11 +500,536 @@ } } +/* ---- Assistant "thinking" (streamed reasoning) ---- */ +.ai-thinking { + margin-bottom: 8px; + border-left: 2px solid var(--vp-border, rgba(255, 255, 255, 0.09)); + padding-left: 10px; +} + +.ai-thinking__summary { + list-style: none; + cursor: pointer; + color: var(--vp-text-faint, rgba(255, 255, 255, 0.36)); + font-size: 11px; + font-weight: 560; + letter-spacing: 0.02em; + user-select: none; +} + +.ai-thinking__summary::-webkit-details-marker { + display: none; +} + +.ai-thinking__summary::before { + content: "›"; + display: inline-block; + margin-right: 6px; + transition: transform 140ms ease; +} + +.ai-thinking[open] .ai-thinking__summary::before { + transform: rotate(90deg); +} + +.ai-thinking__body { + margin-top: 6px; + color: var(--vp-text-faint, rgba(255, 255, 255, 0.4)); + font-size: 12px; + line-height: 1.55; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +/* ---- Per-message actions (copy / read aloud) ---- */ +.ai-msg__actions { + display: flex; + gap: 2px; + margin-top: 6px; + opacity: 0.55; + transition: opacity 140ms ease; +} + +.ai-msg--assistant:hover .ai-msg__actions { + opacity: 1; +} + +.ai-msg-action { + display: inline-grid; + place-items: center; + width: 26px; + height: 26px; + padding: 0; + border: 0; + border-radius: 7px; + background: transparent; + color: var(--vp-text-faint, rgba(255, 255, 255, 0.4)); + cursor: pointer; + line-height: 0; + transition: color 140ms ease, background 140ms ease; +} + +.ai-msg-action:hover { + color: var(--vp-text, rgba(255, 255, 255, 0.92)); + background: var(--vp-hover, rgba(255, 255, 255, 0.08)); +} + +.ai-msg-action[data-active="true"] { + color: var(--vp-text, rgba(255, 255, 255, 0.92)); +} + +.ai-msg-action svg { + width: 15px; + height: 15px; +} + +/* ---- Streaming status line (shown before the first token) ---- */ +.ai-status { + display: inline-flex; + align-items: center; + gap: 8px; + color: var(--vp-text-faint, rgba(255, 255, 255, 0.4)); + font-size: 12.5px; +} + +.ai-status__shimmer { + background: linear-gradient( + 90deg, + rgba(255, 255, 255, 0.32) 0%, + rgba(255, 255, 255, 0.85) 50%, + rgba(255, 255, 255, 0.32) 100% + ); + background-size: 200% 100%; + -webkit-background-clip: text; + background-clip: text; + -webkit-text-fill-color: transparent; + animation: ai-shimmer 1.6s linear infinite; +} + +.ai-status__dots { + display: inline-flex; + align-items: center; + gap: 4px; +} + +@keyframes ai-shimmer { + 0% { + background-position: 200% 0; + } + 100% { + background-position: -200% 0; + } +} + +/* Blinking caret at the end of streaming assistant text. */ +.ai-caret { + display: inline-block; + width: 7px; + height: 14px; + margin-left: 2px; + transform: translateY(2px); + background: var(--vp-text-dim, rgba(255, 255, 255, 0.5)); + border-radius: 1px; + animation: ai-caret-blink 1s steps(2, start) infinite; +} + +@keyframes ai-caret-blink { + 0%, + 100% { + opacity: 0; + } + 50% { + opacity: 1; + } +} + +/* ---- Attachments shown inside a sent user message ---- */ +.ai-msg__attachments { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 7px; + justify-content: flex-end; +} + +.ai-attach-thumb { + width: 66px; + height: 66px; + object-fit: cover; + border-radius: 9px; + border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); + cursor: zoom-in; +} + +.ai-thumb-chip__img { + cursor: zoom-in; +} + +/* ---- Lightbox (click a thumbnail to enlarge) ---- */ +.ai-lightbox { + position: fixed; + inset: 0; + z-index: 99999; + display: grid; + place-items: center; + padding: 4vh 4vw; + background: rgba(0, 0, 0, 0.82); + backdrop-filter: blur(6px); + -webkit-backdrop-filter: blur(6px); + cursor: zoom-out; +} + +.ai-lightbox__img { + max-width: 94vw; + max-height: 92vh; + object-fit: contain; + border-radius: 10px; + box-shadow: 0 24px 70px rgba(0, 0, 0, 0.6); +} + +.ai-lightbox__close { + position: fixed; + top: 18px; + right: 18px; + z-index: 100000; + display: inline-grid; + place-items: center; + width: 38px; + height: 38px; + padding: 0; + border: 0; + border-radius: 10px; + background: rgba(255, 255, 255, 0.12); + color: #fff; + cursor: pointer; + line-height: 0; +} + +.ai-lightbox__close:hover { + background: rgba(255, 255, 255, 0.22); +} + +.ai-lightbox__close svg { + width: 20px; + height: 20px; +} + +.ai-attach-file { + max-width: 180px; + padding: 5px 9px; + border-radius: 8px; + border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); + background: var(--vp-panel, rgba(255, 255, 255, 0.045)); + color: var(--vp-text-dim, rgba(255, 255, 255, 0.5)); + font-size: 11px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* ---- Composer attachment chips (pending, before send) ---- */ +.ai-composer__chips { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 8px; +} + +.ai-chip { + display: inline-flex; + align-items: center; + gap: 6px; + max-width: 190px; + padding: 4px 6px 4px 4px; + border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); + border-radius: 9px; + background: var(--vp-panel, rgba(255, 255, 255, 0.045)); +} + +.ai-chip__thumb { + width: 26px; + height: 26px; + object-fit: cover; + border-radius: 6px; +} + +.ai-chip__icon { + display: inline-grid; + place-items: center; + width: 26px; + height: 26px; + border-radius: 6px; + background: rgba(255, 255, 255, 0.06); + color: var(--vp-text-dim, rgba(255, 255, 255, 0.5)); + font-size: 13px; +} + +.ai-chip__label { + overflow: hidden; + color: var(--vp-text-dim, rgba(255, 255, 255, 0.6)); + font-size: 11px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.ai-chip__remove { + display: inline-grid; + place-items: center; + width: 18px; + height: 18px; + flex: 0 0 18px; + padding: 0; + border: 0; + border-radius: 5px; + background: transparent; + color: var(--vp-text-faint, rgba(255, 255, 255, 0.36)); + cursor: pointer; + line-height: 0; +} + +.ai-chip__remove:hover { + color: var(--vp-text, rgba(255, 255, 255, 0.92)); + background: var(--vp-hover, rgba(255, 255, 255, 0.1)); +} + +.ai-chip__remove svg { + width: 12px; + height: 12px; +} + +/* ---- Composer tools (plus / camera / model picker) ---- */ +.ai-composer__tools { + display: inline-flex; + align-items: center; + gap: 6px; + min-width: 0; +} + +/* Right side of the composer footer: model picker directly beside Send. */ +.ai-composer__right { + display: inline-flex; + align-items: center; + gap: 8px; + min-width: 0; +} + +.ai-composer__right .ai-model-picker { + max-width: 190px; +} + +.ai-hidden-file { + display: none; +} + +.ai-tool-btn { + display: inline-grid; + place-items: center; + width: 31px; + height: 31px; + flex: 0 0 31px; + padding: 0; + border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); + border-radius: 9px; + background: transparent; + color: var(--vp-text-dim, rgba(255, 255, 255, 0.55)); + cursor: pointer; + line-height: 0; + transition: color 140ms ease, background 140ms ease, border-color 140ms ease; +} + +.ai-tool-btn:hover:not(:disabled) { + color: var(--vp-text, rgba(255, 255, 255, 0.92)); + background: var(--vp-hover, rgba(255, 255, 255, 0.08)); + border-color: var(--vp-border-strong, rgba(255, 255, 255, 0.16)); +} + +.ai-tool-btn:disabled { + cursor: default; + opacity: 0.4; +} + +.ai-tool-btn[data-active="true"] { + color: var(--vp-text, rgba(255, 255, 255, 0.92)); + background: var(--vp-hover, rgba(255, 255, 255, 0.12)); + border-color: var(--vp-border-strong, rgba(255, 255, 255, 0.22)); +} + +.ai-tool-btn svg { + width: 16px; + height: 16px; +} + +/* ---- Drag-to-resize handle (left edge) ---- */ +.ai-resize-handle { + position: absolute; + top: 0; + left: 0; + width: 8px; + height: 100%; + z-index: 5; + cursor: ew-resize; + touch-action: none; +} + +.ai-resize-handle::after { + content: ""; + position: absolute; + top: 0; + left: 0; + width: 2px; + height: 100%; + background: transparent; + transition: background 140ms ease; +} + +.ai-resize-handle:hover::after { + background: var(--vp-border-strong, rgba(255, 255, 255, 0.28)); +} + +/* ---- Welcome / how-to summary (empty state) ---- */ +.ai-welcome { + margin: auto; + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + gap: 12px; + max-width: 300px; + padding: 24px 8px; +} + +.ai-welcome__mark { + display: inline-grid; + place-items: center; + width: 44px; + height: 44px; + border: 1px solid var(--vp-border-strong, rgba(255, 255, 255, 0.16)); + border-radius: 12px; + background: var(--vp-panel, rgba(255, 255, 255, 0.045)); + color: rgba(255, 255, 255, 0.75); +} + +.ai-welcome__mark svg { + width: 24px; + height: 24px; +} + +.ai-welcome__title { + margin: 0; + font-size: 17px; + font-weight: 650; + letter-spacing: -0.01em; + color: var(--vp-text, rgba(255, 255, 255, 0.92)); +} + +.ai-welcome__text { + margin: 0; + font-size: 12.5px; + line-height: 1.6; + color: var(--vp-text-dim, rgba(255, 255, 255, 0.5)); +} + +.ai-welcome__text strong { + color: var(--vp-text, rgba(255, 255, 255, 0.85)); + font-weight: 650; +} + +/* ---- Compact image thumbnail chips (fit 4 captured views on one row) ---- */ +.ai-thumb-chip { + position: relative; + flex: 0 0 auto; + width: 62px; + height: 62px; + border-radius: 9px; + overflow: hidden; + border: 1px solid var(--vp-border, rgba(255, 255, 255, 0.09)); +} + +.ai-thumb-chip__img { + width: 100%; + height: 100%; + object-fit: cover; + display: block; +} + +.ai-thumb-chip__remove { + position: absolute; + top: 2px; + right: 2px; + display: inline-grid; + place-items: center; + width: 17px; + height: 17px; + padding: 0; + border: 0; + border-radius: 5px; + background: rgba(0, 0, 0, 0.6); + color: #fff; + cursor: pointer; + line-height: 0; +} + +.ai-thumb-chip__remove svg { + width: 11px; + height: 11px; +} + +/* ---- File-type icon badge (composer chip + sent message) ---- */ +.ai-chip__type, +.ai-attach-file__icon { + display: inline-grid; + place-items: center; + width: 26px; + height: 26px; + flex: 0 0 26px; + border-radius: 6px; + background: rgba(255, 255, 255, 0.06); + color: var(--vp-text-dim, rgba(255, 255, 255, 0.6)); +} + +.ai-chip__type svg, +.ai-attach-file__icon svg { + width: 15px; + height: 15px; +} + +.ai-chip__type[data-type="pdf"], +.ai-attach-file__icon[data-type="pdf"] { + background: rgba(232, 89, 89, 0.16); + color: #f0a0a0; +} + +.ai-chip__type[data-type="scan"], +.ai-attach-file__icon[data-type="scan"] { + background: rgba(110, 168, 254, 0.16); + color: #a8c8ff; +} + +.ai-chip__type[data-type="image"], +.ai-attach-file__icon[data-type="image"] { + background: rgba(120, 200, 140, 0.16); + color: #a6d8b4; +} + +.ai-attach-file { + display: inline-flex; + align-items: center; + gap: 7px; +} + +.ai-attach-file__name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + @media (prefers-reduced-motion: reduce) { .ai-sidebar, .ai-typing__dot, .ai-model-picker__dot, - .ai-model-picker__button > svg { + .ai-model-picker__button > svg, + .ai-status__shimmer, + .ai-caret { animation: none; transition: none; } diff --git a/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx b/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx index f880f73..1f885a9 100644 --- a/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx +++ b/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx @@ -1,15 +1,22 @@ -// Trigger typecheck using the latest AI assistant files. import React, { useCallback, useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; import { API_BASE } from "../../helpers/constants"; import type { AIAction, AIModelInfo, AISidebarProps, + ChatAttachment, ChatMessage, } from "./types"; import "./AISidebar.css"; -const MODEL_STORAGE_KEY = "bodymaps-ai-model"; +// Bumped to v2 so a previously-stored reasoning model (e.g. qwen3) is reset — +// the default now prefers a non-reasoning model that never leaks "thinking". +const MODEL_STORAGE_KEY = "bodymaps-ai-model-v2"; + +// Reasoning models emit a chain-of-thought that can leak into the answer on +// older Ollama; we avoid picking them as the initial default. +const REASONING_MODEL = /qwen3|deepseek-r1|-r1\b|:think|marco-o1|qwq/i; const SendIcon = () => ( ( ); +const PlusIcon = () => ( + +); + +const CameraIcon = () => ( + +); + const BotIcon = () => ( ( ); +const CopyIcon = () => ( + +); + +const SpeakerIcon = () => ( + +); + +const StopIcon = () => ( + +); + const ChevronIcon = () => ( ( ); +// Attachment type → a distinct little icon (PDF / image / scan / generic file). +type FileType = "pdf" | "image" | "scan" | "file"; + +function fileTypeOf(name: string): FileType { + const n = name.toLowerCase(); + if (n.endsWith(".pdf")) return "pdf"; + if (/\.(png|jpe?g|gif|webp|bmp|tiff?)$/.test(n)) return "image"; + if (/\.(nii|nii\.gz|dcm|dicom|nrrd|mha|mhd)$/.test(n)) return "scan"; + return "file"; +} + +const DocIcon = () => ( + +); + +const ImageFileIcon = () => ( + +); + +const ScanFileIcon = () => ( + +); + +function AttachmentTypeIcon({ name }: { name: string }) { + const type = fileTypeOf(name); + if (type === "image") return ; + if (type === "scan") return ; + return ; +} + const CheckIcon = () => ( ( ); -let messageCounter = 0; +let idCounter = 0; +function makeId(prefix = "id") { + idCounter += 1; + return `${prefix}-${Date.now()}-${idCounter}`; +} -function makeId() { - messageCounter += 1; - return `msg-${Date.now()}-${messageCounter}`; +function readFileAsDataURL(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result)); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(file); + }); } +// Minimal markdown: **bold** and line breaks. Kept intentionally small so the +// assistant text stays clean and minimalist rather than heavily styled. function renderMessageText(content: string) { return content.split("\n").map((line, lineIndex, lines) => { const parts = line.split(/(\*\*[^*]+\*\*)/g); - return ( {parts.map((part, partIndex) => { if (part.startsWith("**") && part.endsWith("**")) { - return ( - - {part.slice(2, -2)} - - ); + return {part.slice(2, -2)}; } - return ( - - {part} - + {part} ); })} - {lineIndex < lines.length - 1 ?
: null} ); }); } -const SUGGESTION_CHIPS = [ - "Segment the liver and tell me its volume", - "What does the liver do?", - "What organs are visible in this scan?", -]; - -type AICommandResponse = { - reply?: string; - actions?: AIAction[]; - source?: string; - model?: string | null; -}; - type ModelState = "loading" | "ollama" | "fallback"; +type StreamEvent = + | { type: "status"; text?: string } + | { type: "thinking"; delta?: string } + | { type: "reply"; delta?: string } + | { type: "actions"; actions?: AIAction[] } + | { type: "final"; reply?: string; actions?: AIAction[]; source?: string; model?: string | null } + | { type: "done" } + | { type: "error"; message?: string }; + export default function AISidebar({ open, onClose, @@ -145,32 +262,42 @@ export default function AISidebar({ organReferences = [], demographics = null, actions, + captureViewport, + getMaskLegend, + onResize, + onResizeEnd, }: AISidebarProps) { const [messages, setMessages] = useState([]); const [input, setInput] = useState(""); + const [attachments, setAttachments] = useState([]); const [loading, setLoading] = useState(false); + const [capturing, setCapturing] = useState(false); const [models, setModels] = useState([]); const [selectedModel, setSelectedModel] = useState(""); const [modelState, setModelState] = useState("loading"); const [modelMenuOpen, setModelMenuOpen] = useState(false); + const [copiedId, setCopiedId] = useState(null); + const [speakingId, setSpeakingId] = useState(null); + const [lightboxUrl, setLightboxUrl] = useState(null); const chatEndRef = useRef(null); + const chatScrollRef = useRef(null); const textareaRef = useRef(null); const modelPickerRef = useRef(null); + const fileInputRef = useRef(null); + const abortRef = useRef(null); + // Whether the chat is scrolled to (near) the bottom. Only auto-scroll when it + // is, so scrolling up to read during generation isn't yanked back down. + const pinnedToBottomRef = useRef(true); const loadModels = useCallback(async () => { setModelState("loading"); - try { const response = await fetch(`${API_BASE}/api/ai-models`); const data = await response.json(); - if (!response.ok) { - throw new Error(data.error || `HTTP ${response.status}`); - } + if (!response.ok) throw new Error(data.error || `HTTP ${response.status}`); - const nextModels: AIModelInfo[] = Array.isArray(data.models) - ? data.models - : []; + const nextModels: AIModelInfo[] = Array.isArray(data.models) ? data.models : []; setModels(nextModels); if (!data.available || nextModels.length === 0) { @@ -179,18 +306,17 @@ export default function AISidebar({ return; } - const storedModel = - window.localStorage.getItem(MODEL_STORAGE_KEY) ?? ""; - const defaultModel = String( - data.default_model || nextModels[0].name - ); - const nextSelection = nextModels.some( - (model) => model.name === storedModel - ) + const storedModel = window.localStorage.getItem(MODEL_STORAGE_KEY) ?? ""; + const backendDefault = String(data.default_model || ""); + // Prefer a non-reasoning model as the initial default (clean output), + // unless the user has already picked one this session. + const cleanModel = nextModels.find((model) => !REASONING_MODEL.test(model.name)); + const nextSelection = nextModels.some((model) => model.name === storedModel) ? storedModel - : nextModels.some((model) => model.name === defaultModel) - ? defaultModel - : nextModels[0].name; + : !REASONING_MODEL.test(backendDefault) && + nextModels.some((model) => model.name === backendDefault) + ? backendDefault + : cleanModel?.name ?? (backendDefault || nextModels[0].name); setSelectedModel(nextSelection); setModelState("ollama"); @@ -204,29 +330,35 @@ export default function AISidebar({ useEffect(() => { if (!open) return; - setModelMenuOpen(false); void loadModels(); - const focusTimer = window.setTimeout(() => { - textareaRef.current?.focus(); - }, 180); - + const focusTimer = window.setTimeout(() => textareaRef.current?.focus(), 180); return () => window.clearTimeout(focusTimer); }, [open, loadModels]); useEffect(() => { + abortRef.current?.abort(); setMessages([]); setInput(""); + setAttachments([]); setModelMenuOpen(false); }, [caseId, sessionId]); + // Auto-scroll to the newest content ONLY when the user is already near the + // bottom. If they've scrolled up to read, leave them where they are. useEffect(() => { - chatEndRef.current?.scrollIntoView({ - behavior: "smooth", - block: "end", - }); + if (pinnedToBottomRef.current) { + chatEndRef.current?.scrollIntoView({ behavior: "smooth", block: "end" }); + } }, [messages, loading]); + const handleChatScroll = useCallback(() => { + const el = chatScrollRef.current; + if (!el) return; + const distanceFromBottom = el.scrollHeight - el.scrollTop - el.clientHeight; + pinnedToBottomRef.current = distanceFromBottom < 80; + }, []); + const closeSidebar = useCallback(() => { setModelMenuOpen(false); onClose(); @@ -234,7 +366,6 @@ export default function AISidebar({ useEffect(() => { if (!open) return; - const handlePointerDown = (event: PointerEvent) => { if ( modelMenuOpen && @@ -244,19 +375,13 @@ export default function AISidebar({ setModelMenuOpen(false); } }; - const handleEscape = (event: KeyboardEvent) => { if (event.key !== "Escape") return; - if (modelMenuOpen) { - setModelMenuOpen(false); - } else { - closeSidebar(); - } + if (modelMenuOpen) setModelMenuOpen(false); + else closeSidebar(); }; - window.addEventListener("pointerdown", handlePointerDown); window.addEventListener("keydown", handleEscape); - return () => { window.removeEventListener("pointerdown", handlePointerDown); window.removeEventListener("keydown", handleEscape); @@ -266,7 +391,6 @@ export default function AISidebar({ const selectModel = (value: string) => { setSelectedModel(value); setModelMenuOpen(false); - if (value) { window.localStorage.setItem(MODEL_STORAGE_KEY, value); setModelState("ollama"); @@ -276,19 +400,22 @@ export default function AISidebar({ } }; - const handleInput = ( - event: React.ChangeEvent - ) => { + const handleInput = (event: React.ChangeEvent) => { setInput(event.target.value); - const element = event.target; element.style.height = "auto"; - element.style.height = `${Math.min( - element.scrollHeight, - 124 - )}px`; + element.style.height = `${Math.min(element.scrollHeight, 124)}px`; }; + const updateMessage = useCallback( + (id: string, updater: (message: ChatMessage) => ChatMessage) => { + setMessages((previous) => + previous.map((message) => (message.id === id ? updater(message) : message)) + ); + }, + [] + ); + const executeAction = useCallback( async (action: AIAction): Promise => { switch (action.type) { @@ -350,109 +477,373 @@ export default function AISidebar({ try { await executeAction(action); } catch (error) { - console.error( - "[BodyMaps AI action error]", - error, - action - ); + console.error("[BodyMaps AI action error]", error, action); } } }, [executeAction] ); + // ---- Attachments --------------------------------------------------------- + + const addFiles = useCallback(async (files: FileList | File[]) => { + const list = Array.from(files); + const next: ChatAttachment[] = []; + for (const file of list) { + if (file.type.startsWith("image/")) { + try { + const dataUrl = await readFileAsDataURL(file); + next.push({ id: makeId("att"), name: file.name, kind: "image", dataUrl, source: "upload" }); + } catch { + next.push({ id: makeId("att"), name: file.name, kind: "file", source: "upload" }); + } + } else { + next.push({ id: makeId("att"), name: file.name, kind: "file", source: "upload" }); + } + } + if (next.length) setAttachments((previous) => [...previous, ...next]); + }, []); + + const handleFilePick = (event: React.ChangeEvent) => { + if (event.target.files && event.target.files.length) void addFiles(event.target.files); + event.target.value = ""; + }; + + const handleCapture = useCallback(async () => { + if (!captureViewport || capturing) return; + // Toggle: if screenshots are already attached, one more click clears them + // all instead of stacking another set. + if (attachments.some((att) => att.source === "screenshot")) { + setAttachments((previous) => previous.filter((att) => att.source !== "screenshot")); + return; + } + setCapturing(true); + try { + const shots = await captureViewport(); + if (!shots.length) return; + const next: ChatAttachment[] = shots.map((shot) => ({ + id: makeId("shot"), + name: `${shot.name} view`, + kind: "image", + dataUrl: shot.dataUrl, + label: shot.name, + source: "screenshot", + })); + setAttachments((previous) => [...previous, ...next]); + } catch (error) { + console.error("[BodyMaps AI capture error]", error); + } finally { + setCapturing(false); + } + }, [captureViewport, capturing, attachments]); + + const removeAttachment = (id: string) => { + setAttachments((previous) => previous.filter((item) => item.id !== id)); + }; + + // ---- Copy / read aloud (per assistant message) --------------------------- + + const handleCopy = useCallback(async (id: string, text: string) => { + try { + await navigator.clipboard.writeText(text); + setCopiedId(id); + window.setTimeout(() => setCopiedId((current) => (current === id ? null : current)), 1400); + } catch (error) { + console.error("[BodyMaps AI copy error]", error); + } + }, []); + + const handleSpeak = useCallback( + (id: string, text: string) => { + const synth = window.speechSynthesis; + if (!synth) return; + // Toggle: clicking the speaker on the message that's talking stops it. + if (speakingId === id) { + synth.cancel(); + setSpeakingId(null); + return; + } + synth.cancel(); + const utterance = new SpeechSynthesisUtterance(text); + utterance.rate = 1.02; + utterance.onend = () => setSpeakingId((current) => (current === id ? null : current)); + utterance.onerror = () => setSpeakingId((current) => (current === id ? null : current)); + setSpeakingId(id); + synth.speak(utterance); + }, + [speakingId] + ); + + // ---- Drag-to-resize (left edge) ------------------------------------------ + + const startResize = useCallback( + (event: React.PointerEvent) => { + if (!onResize) return; + event.preventDefault(); + const move = (e: PointerEvent) => onResize(e.clientX); + const up = () => { + window.removeEventListener("pointermove", move); + window.removeEventListener("pointerup", up); + document.body.style.cursor = ""; + document.body.style.userSelect = ""; + onResizeEnd?.(); + }; + document.body.style.cursor = "ew-resize"; + document.body.style.userSelect = "none"; + window.addEventListener("pointermove", move); + window.addEventListener("pointerup", up); + }, + [onResize, onResizeEnd] + ); + + // Stop any narration when the panel closes or the case changes. + useEffect(() => { + if (!open) { + window.speechSynthesis?.cancel(); + setSpeakingId(null); + } + }, [open]); + + useEffect(() => { + return () => { + window.speechSynthesis?.cancel(); + }; + }, []); + + // ---- Streaming send ------------------------------------------------------ + + const streamResponse = useCallback( + async ( + assistantId: string, + payload: Record, + signal?: AbortSignal + ): Promise => { + const response = await fetch(`${API_BASE}/api/ai-command-stream`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal, + }); + if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`); + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + let actionsApplied = false; + + const handleEvent = (event: StreamEvent) => { + switch (event.type) { + case "status": + updateMessage(assistantId, (m) => ({ ...m, status: event.text ?? "" })); + break; + case "thinking": + updateMessage(assistantId, (m) => ({ + ...m, + thinking: (m.thinking ?? "") + (event.delta ?? ""), + })); + break; + case "reply": + updateMessage(assistantId, (m) => ({ + ...m, + content: m.content + (event.delta ?? ""), + status: undefined, + })); + break; + case "actions": + if (Array.isArray(event.actions) && event.actions.length) { + actionsApplied = true; + void applyReturnedActions(event.actions); + } + break; + case "final": + if (typeof event.reply === "string") { + updateMessage(assistantId, (m) => ({ ...m, content: event.reply as string, status: undefined })); + } + if (!actionsApplied && Array.isArray(event.actions) && event.actions.length) { + actionsApplied = true; + void applyReturnedActions(event.actions); + } + break; + case "error": + updateMessage(assistantId, (m) => ({ + ...m, + content: + m.content || + event.message || + "The assistant ran into an error while answering.", + status: undefined, + })); + break; + case "done": + break; + } + }; + + // Read the NDJSON stream line by line. + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let newlineIndex = buffer.indexOf("\n"); + while (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex).trim(); + buffer = buffer.slice(newlineIndex + 1); + if (line) { + try { + handleEvent(JSON.parse(line) as StreamEvent); + } catch (error) { + console.warn("[BodyMaps AI stream parse]", error, line); + } + } + newlineIndex = buffer.indexOf("\n"); + } + } + + const tail = buffer.trim(); + if (tail) { + try { + handleEvent(JSON.parse(tail) as StreamEvent); + } catch { + /* ignore trailing partial */ + } + } + return true; + }, + [applyReturnedActions, updateMessage] + ); + + // Non-streaming fallback for environments where the stream endpoint is + // unavailable (older backend, proxy that buffers the response, etc.). + const sendNonStreaming = useCallback( + async (assistantId: string, payload: Record, signal?: AbortSignal) => { + const response = await fetch(`${API_BASE}/api/ai-command`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal, + }); + const data = await response.json(); + if (!response.ok) throw new Error(data.reply || `HTTP ${response.status}`); + const returnedActions: AIAction[] = Array.isArray(data.actions) ? data.actions : []; + if (returnedActions.length) void applyReturnedActions(returnedActions); + updateMessage(assistantId, (m) => ({ + ...m, + content: data.reply ?? "Done.", + status: undefined, + })); + }, + [applyReturnedActions, updateMessage] + ); + const handleSend = useCallback( async (overrideText?: string) => { const text = (overrideText ?? input).trim(); - if (!text || loading) return; + const outgoingAttachments = attachments; + if ((!text && outgoingAttachments.length === 0) || loading) return; const conversation = messages - .filter( - (message) => - message.role === "user" || - message.role === "assistant" - ) + .filter((message) => message.role === "user" || message.role === "assistant") .slice(-12) - .map((message) => ({ - role: message.role, - content: message.content, - })); + .map((message) => ({ role: message.role, content: message.content })); setInput(""); - if (textareaRef.current) { - textareaRef.current.style.height = "auto"; - } + setAttachments([]); + if (textareaRef.current) textareaRef.current.style.height = "auto"; + + // A new turn: pin to the bottom so the reply is visible as it starts. + pinnedToBottomRef.current = true; + + const userId = makeId("user"); + const assistantId = makeId("assistant"); setMessages((previous) => [ ...previous, { - id: makeId(), + id: userId, role: "user", content: text, timestamp: Date.now(), + attachments: outgoingAttachments.length ? outgoingAttachments : undefined, + }, + { + id: assistantId, + role: "assistant", + content: "", + timestamp: Date.now(), + streaming: true, + status: "Thinking", }, ]); setLoading(true); - try { - const response = await fetch(`${API_BASE}/api/ai-command`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - message: text, - conversation, - case_id: caseId, - session_id: sessionId ?? null, - available_organs: availableOrgans, - viewer_state: viewerState, - organ_metrics: organMetrics, - organ_references: organReferences, - demographics, - model: selectedModel || null, - }), - }); - - const data = (await response.json()) as AICommandResponse; - if (!response.ok) { - throw new Error(data.reply || `HTTP ${response.status}`); - } + const images = outgoingAttachments + .filter((item) => item.kind === "image" && item.dataUrl) + .map((item) => item.dataUrl as string); - const returnedActions = Array.isArray(data.actions) - ? data.actions - : []; - if (returnedActions.length > 0) { - void applyReturnedActions(returnedActions); - } + const fileNames = outgoingAttachments + .filter((item) => item.kind === "file") + .map((item) => item.name); + + const composedMessage = fileNames.length + ? `${text}\n\n[Attached files: ${fileNames.join(", ")}]`.trim() + : text; + + // When screenshots are attached, include the color→organ legend so the + // vision model can identify each colored region. + const maskLegend = + images.length && getMaskLegend ? getMaskLegend() : []; - setMessages((previous) => [ - ...previous, - { - id: makeId(), - role: "assistant", - content: data.reply ?? "Done.", - timestamp: Date.now(), - }, - ]); - } catch (error) { - console.error("[BodyMaps AI send error]", error); - setMessages((previous) => [ - ...previous, - { - id: makeId(), - role: "assistant", - content: - "The assistant service is unavailable right now. Viewer controls are still available from the left panel.", - timestamp: Date.now(), - }, - ]); + const payload: Record = { + message: composedMessage, + conversation, + case_id: caseId, + session_id: sessionId ?? null, + available_organs: availableOrgans, + viewer_state: viewerState, + organ_metrics: organMetrics, + organ_references: organReferences, + demographics, + model: selectedModel || null, + images, + mask_legend: maskLegend, + }; + + const controller = new AbortController(); + abortRef.current = controller; + const isAbort = (e: unknown) => + e instanceof DOMException ? e.name === "AbortError" : (e as { name?: string })?.name === "AbortError"; + + try { + await streamResponse(assistantId, payload, controller.signal); + } catch (streamError) { + if (isAbort(streamError)) { + // User pressed Stop — keep whatever was streamed, no error. + } else { + console.warn("[BodyMaps AI stream] falling back:", streamError); + try { + await sendNonStreaming(assistantId, payload, controller.signal); + } catch (error) { + if (!isAbort(error)) { + console.error("[BodyMaps AI send error]", error); + updateMessage(assistantId, (m) => ({ + ...m, + content: + m.content || + "The assistant service is unavailable right now. Viewer controls are still available from the left panel.", + status: undefined, + })); + } + } + } } finally { + updateMessage(assistantId, (m) => ({ ...m, streaming: false, status: undefined })); setLoading(false); + abortRef.current = null; } }, [ input, + attachments, loading, messages, caseId, @@ -463,13 +854,18 @@ export default function AISidebar({ organReferences, demographics, selectedModel, - applyReturnedActions, + getMaskLegend, + streamResponse, + sendNonStreaming, + updateMessage, ] ); - const handleKeyDown = ( - event: React.KeyboardEvent - ) => { + const handleStop = useCallback(() => { + abortRef.current?.abort(); + }, []); + + const handleKeyDown = (event: React.KeyboardEvent) => { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); void handleSend(); @@ -481,7 +877,9 @@ export default function AISidebar({ ? "Loading models" : modelState === "fallback" ? "Local fallback" - : selectedModel || "Automatic"; + : selectedModel || models[0]?.name || "Model"; + + const canSend = !loading && (input.trim().length > 0 || attachments.length > 0); return (