From 8e879874e5ca2bf8be9495120e2a925de3c3bab0 Mon Sep 17 00:00:00 2001 From: "nedas.vi" Date: Tue, 21 Jul 2026 09:47:43 +0300 Subject: [PATCH 1/9] Add human handoff, CSAT capture, and analytics dashboard Closes the P0 competitive gaps vs. Intercom Fin/Zendesk AI Agents: sessions can now escalate to a human agent (server-side tool + max-tool-rounds fallback), operators can reply/resolve escalated sessions from the dashboard, users can submit a 1-5 CSAT rating after a resolved turn, and the dashboard exposes a resolution-rate/escalation-rate/CSAT analytics page. Co-Authored-By: Claude Sonnet 5 --- .env.example | 5 + agent/config.py | 4 + agent/middleware/internal_auth.py | 24 ++ agent/models/__init__.py | 2 + agent/models/session.py | 9 +- agent/models/session_feedback.py | 25 ++ agent/routers/chat_events.py | 54 +++- agent/routers/sessions.py | 34 +++ agent/schemas/session.py | 17 ++ agent/services/orchestrator.py | 66 ++++- agent/services/session_service.py | 2 +- .../024_add_session_escalation_fields.py | 28 ++ alembic/versions/025_add_session_feedback.py | 39 +++ dashboard/prisma/schema.prisma | 16 ++ .../app/v1/apps/[appId]/analytics/route.ts | 98 +++++++ .../sessions/[sessionId]/reply/route.ts | 34 +++ .../sessions/[sessionId]/resolve/route.ts | 31 +++ dashboard/src/components/AppSidebar.tsx | 4 + dashboard/src/components/ui/AppNav.tsx | 1 + dashboard/src/dashboard-app.tsx | 2 + dashboard/src/dashboard_pages/Analytics.tsx | 243 ++++++++++++++++++ dashboard/src/lib/server/agent-service.ts | 73 ++++++ docs/generated/openapi/agent.openapi.json | 201 +++++++++++++++ docs/generated/openapi/dashboard.openapi.json | 63 +++++ tests/test_analytics_aggregation_contract.py | 24 ++ tests/test_orchestrator_chat_unavailable.py | 3 + .../test_orchestrator_kb_tool_result_flow.py | 9 +- ...st_orchestrator_router_enriched_context.py | 3 + tests/test_session_escalation_contract.py | 25 ++ tests/test_session_feedback_contract.py | 23 ++ 30 files changed, 1154 insertions(+), 8 deletions(-) create mode 100644 agent/middleware/internal_auth.py create mode 100644 agent/models/session_feedback.py create mode 100644 alembic/versions/024_add_session_escalation_fields.py create mode 100644 alembic/versions/025_add_session_feedback.py create mode 100644 dashboard/src/app/v1/apps/[appId]/analytics/route.ts create mode 100644 dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/reply/route.ts create mode 100644 dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/resolve/route.ts create mode 100644 dashboard/src/dashboard_pages/Analytics.tsx create mode 100644 dashboard/src/lib/server/agent-service.ts create mode 100644 tests/test_analytics_aggregation_contract.py create mode 100644 tests/test_session_escalation_contract.py create mode 100644 tests/test_session_feedback_contract.py diff --git a/.env.example b/.env.example index ff0bc6e..31e81fb 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,11 @@ RK_KNOWLEDGE_BASES_AUDIENCE=kb-service RK_KNOWLEDGE_BASES_JWT_ALGORITHM=HS256 RK_KNOWLEDGE_BASES_TIMEOUT_SECONDS=20 RK_KNOWLEDGE_BASES_CONNECT_TIMEOUT_SECONDS=5 +# Service-to-service auth for dashboard `api` -> agent internal endpoints (/internal/*). +# Must match RK_INTERNAL_SERVICE_SIGNING_KEY on the dashboard side. +RK_INTERNAL_SERVICE_SIGNING_KEY=change-me-internal-service-signing-key +RK_INTERNAL_SERVICE_AUDIENCE=agent-service +RK_INTERNAL_SERVICE_JWT_ALGORITHM=HS256 # Frontend NEXT_PUBLIC_API_BASE_URL=http://localhost:3002 diff --git a/agent/config.py b/agent/config.py index ab96abd..f6dd05e 100644 --- a/agent/config.py +++ b/agent/config.py @@ -43,5 +43,9 @@ class Settings(BaseSettings): knowledge_bases_timeout_seconds: float = 20.0 knowledge_bases_connect_timeout_seconds: float = 5.0 + internal_service_signing_key: str = "change-me-internal-service-signing-key" + internal_service_audience: str = "agent-service" + internal_service_jwt_algorithm: str = "HS256" + settings = Settings() diff --git a/agent/middleware/internal_auth.py b/agent/middleware/internal_auth.py new file mode 100644 index 0000000..047bb43 --- /dev/null +++ b/agent/middleware/internal_auth.py @@ -0,0 +1,24 @@ +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +import jwt +from jwt import InvalidTokenError + +from agent.config import settings + +bearer_scheme = HTTPBearer() + + +async def require_internal_service( + credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme), +) -> None: + """Validates a service-to-service JWT issued by the dashboard `api`, mirroring + the knowledge_bases service's own internal-auth pattern.""" + try: + jwt.decode( + credentials.credentials, + settings.internal_service_signing_key, + algorithms=[settings.internal_service_jwt_algorithm], + audience=settings.internal_service_audience, + ) + except InvalidTokenError as exc: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid service token") from exc diff --git a/agent/models/__init__.py b/agent/models/__init__.py index e3e896b..dee50ad 100644 --- a/agent/models/__init__.py +++ b/agent/models/__init__.py @@ -12,6 +12,7 @@ from agent.models.knowledge_base_ref import KnowledgeBaseRef from agent.models.session import ChatSession from agent.models.message import Message +from agent.models.session_feedback import SessionFeedback from agent.models.llm_usage_event import LLMUsageEvent from agent.models.playbook import Playbook, PlaybookFunction @@ -30,6 +31,7 @@ "KnowledgeBaseRef", "ChatSession", "Message", + "SessionFeedback", "LLMUsageEvent", "Playbook", "PlaybookFunction", diff --git a/agent/models/session.py b/agent/models/session.py index c5a26a1..326ad14 100644 --- a/agent/models/session.py +++ b/agent/models/session.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any -from sqlalchemy import DateTime, ForeignKey, String +from sqlalchemy import DateTime, ForeignKey, String, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -11,6 +11,7 @@ if TYPE_CHECKING: from agent.models.app import App from agent.models.message import Message + from agent.models.session_feedback import SessionFeedback class ChatSession(Base, UUIDMixin, TimestampMixin): @@ -18,7 +19,7 @@ class ChatSession(Base, UUIDMixin, TimestampMixin): app_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("apps.id", ondelete="CASCADE")) device_id: Mapped[str | None] = mapped_column(String(255)) - status: Mapped[str] = mapped_column(String(20), default="active") # active, expired, closed + status: Mapped[str] = mapped_column(String(20), default="active") # active, expired, closed, escalated last_activity_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now(timezone.utc) ) @@ -26,6 +27,10 @@ class ChatSession(Base, UUIDMixin, TimestampMixin): llm_context: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict) available_function_names: Mapped[list[str]] = mapped_column(JSONB, default=list) locale: Mapped[str] = mapped_column(String(16), default="en") + escalated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + escalation_reason: Mapped[str | None] = mapped_column(Text, nullable=True) + resolved_by: Mapped[str | None] = mapped_column(String(20), nullable=True) # ai, human app: Mapped["App"] = relationship(back_populates="sessions") messages: Mapped[list["Message"]] = relationship(back_populates="session", cascade="all, delete-orphan", order_by="Message.sequence_number") + feedback: Mapped["SessionFeedback | None"] = relationship(back_populates="session", cascade="all, delete-orphan", uselist=False) diff --git a/agent/models/session_feedback.py b/agent/models/session_feedback.py new file mode 100644 index 0000000..f7bd06d --- /dev/null +++ b/agent/models/session_feedback.py @@ -0,0 +1,25 @@ +import uuid +from typing import TYPE_CHECKING + +from sqlalchemy import CheckConstraint, ForeignKey, Integer, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from agent.models.base import Base, TimestampMixin, UUIDMixin + +if TYPE_CHECKING: + from agent.models.session import ChatSession + + +class SessionFeedback(Base, UUIDMixin, TimestampMixin): + __tablename__ = "session_feedback" + __table_args__ = ( + CheckConstraint("rating >= 1 AND rating <= 5", name="ck_session_feedback_rating_range"), + ) + + session_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("chat_sessions.id", ondelete="CASCADE"), unique=True + ) + rating: Mapped[int] = mapped_column(Integer) + comment: Mapped[str | None] = mapped_column(Text, nullable=True) + + session: Mapped["ChatSession"] = relationship(back_populates="feedback") diff --git a/agent/routers/chat_events.py b/agent/routers/chat_events.py index 133834d..0c03b91 100644 --- a/agent/routers/chat_events.py +++ b/agent/routers/chat_events.py @@ -15,8 +15,10 @@ from agent.database import async_session_factory, get_db from agent.middleware.auth import get_app_from_sdk_auth +from agent.middleware.internal_auth import require_internal_service from agent.models.agent_config import AgentConfig from agent.models.app import App +from agent.models.message import Message from agent.models.organization_llm_provider_profile import OrganizationLLMProviderProfile from agent.models.session import ChatSession from agent.services.chat_access_service import ( @@ -33,7 +35,7 @@ register_pending_tool_result, resolve_pending_tool_result, ) -from agent.services.session_service import is_session_expired +from agent.services.session_service import get_next_sequence, is_session_expired from agent.services.turn_state_service import turn_state_store router = APIRouter(tags=["chat-events"]) @@ -63,6 +65,10 @@ class ChatMessageAccepted(BaseModel): status: str = "accepted" +class HumanMessageBody(BaseModel): + text: str = Field(min_length=1, max_length=_MAX_MESSAGE_TEXT_BYTES) + + class ToolResultBody(BaseModel): turn_id: str idempotency_key: str = Field(min_length=1, max_length=255) @@ -131,9 +137,15 @@ async def send_tool_call_request( async def send_turn_complete(self, full_text: str, usage: dict | None) -> None: await self._push("turn_complete", {"full_text": full_text, "usage": usage}) + async def send_feedback_requested(self) -> None: + await self._push("feedback_requested", {}) + async def send_error(self, code: str, message: str, recoverable: bool = True) -> None: await self._push("error", {"code": code, "message": message, "recoverable": recoverable}) + async def send_session_escalated(self, reason: str) -> None: + await self._push("session_escalated", {"reason": reason}) + async def wait_for_tool_result(self, call_id: str, timeout: int) -> dict[str, Any]: fut = self._pending.get(call_id) if not fut: @@ -340,3 +352,43 @@ async def submit_tool_result( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No pending tool call with this ID") return {"status": "ok", "deduplicated": False} + + +@router.post("/internal/sessions/{session_id}/human-message", dependencies=[Depends(require_internal_service)]) +async def post_human_message( + session_id: uuid.UUID, + body: HumanMessageBody, + db: AsyncSession = Depends(get_db), +): + session = await db.get(ChatSession, session_id) + if session is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found") + + seq = await get_next_sequence(db, session.id) + message = Message( + session_id=session.id, + sequence_number=seq, + role="human_agent", + content=body.text, + ) + db.add(message) + await db.commit() + await db.refresh(message) + + await event_stream_store.append( + session_id=session.id, + app_id=session.app_id, + turn_id=str(uuid.uuid4()), + request_id=str(uuid.uuid4()), + event_type="human_message", + payload={"message_id": str(message.id), "text": body.text}, + ) + + return { + "id": str(message.id), + "created_at": message.created_at.isoformat(), + "session_id": str(message.session_id), + "sequence_number": message.sequence_number, + "role": message.role, + "content": message.content, + } diff --git a/agent/routers/sessions.py b/agent/routers/sessions.py index 953c4ed..4705256 100644 --- a/agent/routers/sessions.py +++ b/agent/routers/sessions.py @@ -10,11 +10,14 @@ from agent.models.app import App from agent.models.message import Message from agent.models.session import ChatSession +from agent.models.session_feedback import SessionFeedback from agent.schemas.session import ( MessageOut, SessionContextOut, SessionContextPatch, SessionCreate, + SessionFeedbackCreate, + SessionFeedbackOut, SessionOut, ) from agent.services.chat_localization_service import effective_texts, resolve_locale @@ -199,3 +202,34 @@ async def get_session_messages_sdk( ).order_by(Message.sequence_number) ) return result.scalars().all() + + +@sdk_router.post("/{session_id}/feedback", response_model=SessionFeedbackOut, status_code=status.HTTP_201_CREATED) +async def submit_session_feedback( + session_id: uuid.UUID, + body: SessionFeedbackCreate, + request: Request, + app: App = Depends(get_app_from_sdk_auth), + db: AsyncSession = Depends(get_db), +): + validate_chat_capability_token( + token=resolve_chat_capability_token(request.headers), + session_id=session_id, + app=app, + ) + + session = await db.get(ChatSession, session_id) + if not session or session.app_id != app.id: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found") + + existing = await db.execute( + select(SessionFeedback).where(SessionFeedback.session_id == session_id) + ) + if existing.scalar_one_or_none() is not None: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Feedback already submitted for this session") + + feedback = SessionFeedback(session_id=session_id, rating=body.rating, comment=body.comment) + db.add(feedback) + await db.commit() + await db.refresh(feedback) + return feedback diff --git a/agent/schemas/session.py b/agent/schemas/session.py index 2ad8303..529c252 100644 --- a/agent/schemas/session.py +++ b/agent/schemas/session.py @@ -118,6 +118,23 @@ class MessageOut(BaseModel): model_config = {"from_attributes": True} +class SessionFeedbackCreate(BaseModel): + rating: int = Field(ge=1, le=5) + comment: str | None = Field(default=None, max_length=2000) + + model_config = {"extra": "forbid"} + + +class SessionFeedbackOut(BaseModel): + id: uuid.UUID + session_id: uuid.UUID + rating: int + comment: str | None + created_at: datetime + + model_config = {"from_attributes": True} + + class SessionContextOut(BaseModel): id: uuid.UUID app_id: uuid.UUID diff --git a/agent/services/orchestrator.py b/agent/services/orchestrator.py index 6a611b2..5ddde90 100644 --- a/agent/services/orchestrator.py +++ b/agent/services/orchestrator.py @@ -4,6 +4,7 @@ import re import uuid from dataclasses import dataclass +from datetime import datetime, timezone from typing import Any from sqlalchemy import select @@ -38,6 +39,7 @@ logger = logging.getLogger(__name__) KB_SEARCH_TOOL_NAME = "kb_search" +ESCALATE_TOOL_NAME = "escalate_to_human" KB_PREFETCH_MAX_ITEMS = 5 KB_PREFETCH_MAX_CHARS = 1200 KB_INDEX_MAX_ITEMS = 8 @@ -859,6 +861,39 @@ def _build_kb_search_tool() -> dict[str, Any]: } +def _build_escalate_tool() -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": ESCALATE_TOOL_NAME, + "description": ( + "Escalate this conversation to a human support agent. Use this when the user " + "explicitly asks for a human, or when you cannot resolve the issue yourself." + ), + "parameters": { + "type": "object", + "properties": { + "reason": {"type": "string", "description": "Why this conversation needs a human."}, + }, + "required": ["reason"], + }, + }, + } + + +async def escalate_session( + db: AsyncSession, + session: ChatSession, + sender: "MessageSender", + reason: str, +) -> None: + session.status = "escalated" + session.escalated_at = datetime.now(timezone.utc) + session.escalation_reason = reason + await db.commit() + await sender.send_session_escalated(reason) + + async def _load_kb_assignment_context( db: AsyncSession, app_id: uuid.UUID, @@ -951,12 +986,18 @@ async def send_tool_call_request( async def send_turn_complete(self, full_text: str, usage: dict | None) -> None: raise NotImplementedError + async def send_feedback_requested(self) -> None: + raise NotImplementedError + async def send_error(self, code: str, message: str, recoverable: bool = True) -> None: raise NotImplementedError async def wait_for_tool_result(self, call_id: str, timeout: int) -> dict[str, Any]: raise NotImplementedError + async def send_session_escalated(self, reason: str) -> None: + raise NotImplementedError + async def build_playbook_prompt( db: AsyncSession, @@ -1139,6 +1180,7 @@ async def run_agent_loop( tools = list(sdk_tools) if kb_service_enabled: tools.append(_build_kb_search_tool()) + tools.append(_build_escalate_tool()) tools_payload = tools or None tool_round = 0 @@ -1240,6 +1282,7 @@ async def run_agent_loop( db.add(assistant_msg) await db.commit() await sender.send_turn_complete(accumulated_text, usage_data) + await sender.send_feedback_requested() return # 5. If tool calls → send to iOS and wait for results @@ -1271,6 +1314,7 @@ async def run_agent_loop( "arguments": arguments, "description": (fn.description_override or fn.description) if fn else "", "is_kb_internal": fn_name == KB_SEARCH_TOOL_NAME, + "is_escalation": fn_name == ESCALATE_TOOL_NAME, }) # 5a. Execute internal KB tools server-side. @@ -1314,8 +1358,24 @@ async def run_agent_loop( db.add(result_msg) await db.commit() + # 5a2. Handle escalation to human, server-side. + escalation_info = next((info for info in tc_infos if info["is_escalation"]), None) + if escalation_info is not None: + reason = str(escalation_info["arguments"].get("reason", "")).strip() or "Escalated by assistant" + seq = await get_next_sequence(db, session.id) + result_msg = Message( + session_id=session.id, + sequence_number=seq, + role="tool_result", + tool_call_id=escalation_info["id"], + content=json.dumps({"status": "escalated"}), + ) + db.add(result_msg) + await escalate_session(db, session, sender, reason) + return + # 5b. Send SDK function tools to iOS - sdk_tc_infos = [info for info in tc_infos if not info["is_kb_internal"]] + sdk_tc_infos = [info for info in tc_infos if not info["is_kb_internal"] and not info["is_escalation"]] descriptions = ( await generate_tool_descriptions(config, sdk_tc_infos, session_id=session.id) if sdk_tc_infos @@ -1413,6 +1473,6 @@ async def run_agent_loop( # No text and no tool calls — shouldn't happen but break to be safe break - # Max tool rounds exceeded — force text response + # Max tool rounds exceeded — escalate to a human rather than dead-ending the session if tool_round >= config.max_tool_rounds: - await sender.send_error("max_tool_rounds", "Maximum tool calling rounds exceeded", recoverable=False) + await escalate_session(db, session, sender, "Maximum tool calling rounds exceeded") diff --git a/agent/services/session_service.py b/agent/services/session_service.py index 608504c..a8a652a 100644 --- a/agent/services/session_service.py +++ b/agent/services/session_service.py @@ -90,7 +90,7 @@ async def expire_stale_sessions(db: AsyncSession) -> int: ChatSession.status == "active", ChatSession.last_activity_at < func.now() - AgentConfig.session_ttl_minutes * text("interval '1 minute'"), ) - .values(status="expired") + .values(status="expired", resolved_by="ai") ) await db.commit() return result.rowcount diff --git a/alembic/versions/024_add_session_escalation_fields.py b/alembic/versions/024_add_session_escalation_fields.py new file mode 100644 index 0000000..ef7aca5 --- /dev/null +++ b/alembic/versions/024_add_session_escalation_fields.py @@ -0,0 +1,28 @@ +"""Add escalation fields to chat_sessions + +Revision ID: 024 +Revises: 023 +Create Date: 2026-07-21 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "024" +down_revision: Union[str, None] = "023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("chat_sessions", sa.Column("escalated_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("chat_sessions", sa.Column("escalation_reason", sa.Text(), nullable=True)) + op.add_column("chat_sessions", sa.Column("resolved_by", sa.String(length=20), nullable=True)) + + +def downgrade() -> None: + op.drop_column("chat_sessions", "resolved_by") + op.drop_column("chat_sessions", "escalation_reason") + op.drop_column("chat_sessions", "escalated_at") diff --git a/alembic/versions/025_add_session_feedback.py b/alembic/versions/025_add_session_feedback.py new file mode 100644 index 0000000..3de8265 --- /dev/null +++ b/alembic/versions/025_add_session_feedback.py @@ -0,0 +1,39 @@ +"""Add session_feedback table for CSAT capture + +Revision ID: 025 +Revises: 024 +Create Date: 2026-07-21 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "025" +down_revision: Union[str, None] = "024" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "session_feedback", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + "session_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("chat_sessions.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ), + sa.Column("rating", sa.Integer(), nullable=False), + sa.Column("comment", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("rating >= 1 AND rating <= 5", name="ck_session_feedback_rating_range"), + ) + + +def downgrade() -> None: + op.drop_table("session_feedback") diff --git a/dashboard/prisma/schema.prisma b/dashboard/prisma/schema.prisma index ab7a68e..72534f3 100644 --- a/dashboard/prisma/schema.prisma +++ b/dashboard/prisma/schema.prisma @@ -196,15 +196,31 @@ model ChatSession { llmContext Json @default("{}") @map("llm_context") availableFunctionNames Json @default("[]") @map("available_function_names") locale String @default("en") @db.VarChar(16) + escalatedAt DateTime? @map("escalated_at") @db.Timestamptz(6) + escalationReason String? @map("escalation_reason") + resolvedBy String? @map("resolved_by") @db.VarChar(20) app App @relation(fields: [appId], references: [id], onDelete: Cascade) messages Message[] llmUsageEvents LlmUsageEvent[] + feedback SessionFeedback? @@index([appId], map: "ix_chat_sessions_app_id") @@map("chat_sessions") } +model SessionFeedback { + id String @id @db.Uuid + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) + sessionId String @unique @map("session_id") @db.Uuid + rating Int + comment String? + + session ChatSession @relation(fields: [sessionId], references: [id], onDelete: Cascade) + + @@map("session_feedback") +} + model LlmUsageEvent { id String @id @db.Uuid createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) diff --git a/dashboard/src/app/v1/apps/[appId]/analytics/route.ts b/dashboard/src/app/v1/apps/[appId]/analytics/route.ts new file mode 100644 index 0000000..57ab9a2 --- /dev/null +++ b/dashboard/src/app/v1/apps/[appId]/analytics/route.ts @@ -0,0 +1,98 @@ +import { Prisma } from "@prisma/client"; +import { NextRequest, NextResponse } from "next/server"; + +import { getDeveloperFromRequest } from "@/lib/server/auth"; +import { getOwnedAppOrNull } from "@/lib/server/apps"; +import { detail } from "@/lib/server/http"; +import { coerceDbNumber } from "@/lib/server/numbers"; +import { prisma } from "@/lib/server/prisma"; + +export const dynamic = "force-dynamic"; + +type DailyRow = { + day: Date; + total: number | string | bigint; + resolved: number | string | bigint; + escalated: number | string | bigint; +}; + +export async function GET(request: NextRequest, context: { params: Promise<{ appId: string }> }) { + const developer = await getDeveloperFromRequest(request); + if (!developer) return detail(401, "Invalid token"); + + const { appId } = await context.params; + const app = await getOwnedAppOrNull(appId, developer); + if (!app) return detail(404, "App not found"); + + const fromParam = request.nextUrl.searchParams.get("from"); + const toParam = request.nextUrl.searchParams.get("to"); + const to = toParam ? new Date(toParam) : new Date(); + const from = fromParam ? new Date(fromParam) : new Date(to.getTime() - 30 * 24 * 60 * 60 * 1000); + if (Number.isNaN(from.valueOf()) || Number.isNaN(to.valueOf())) { + return detail(400, "Invalid from/to date"); + } + + const [totalSessions, resolvedSessions, escalatedByStatus, escalatedByResolver, feedbackAgg, dailyRows] = + await Promise.all([ + prisma.chatSession.count({ + where: { appId: app.id, createdAt: { gte: from, lte: to } }, + }), + prisma.chatSession.count({ + where: { appId: app.id, createdAt: { gte: from, lte: to }, resolvedBy: "ai" }, + }), + prisma.chatSession.count({ + where: { appId: app.id, createdAt: { gte: from, lte: to }, status: "escalated" }, + }), + prisma.chatSession.count({ + where: { appId: app.id, createdAt: { gte: from, lte: to }, resolvedBy: "human" }, + }), + prisma.sessionFeedback.aggregate({ + where: { session: { appId: app.id, createdAt: { gte: from, lte: to } } }, + _avg: { rating: true }, + _count: { rating: true }, + }), + prisma.$queryRaw(Prisma.sql` + SELECT + date_trunc('day', created_at) AS day, + COUNT(*)::bigint AS total, + COUNT(*) FILTER (WHERE resolved_by = 'ai')::bigint AS resolved, + COUNT(*) FILTER (WHERE status = 'escalated' OR resolved_by = 'human')::bigint AS escalated + FROM chat_sessions + WHERE app_id = ${app.id}::uuid + AND created_at >= ${from} + AND created_at <= ${to} + GROUP BY 1 + ORDER BY 1 + `), + ]); + + const ratingDistribution = await prisma.sessionFeedback.groupBy({ + by: ["rating"], + where: { session: { appId: app.id, createdAt: { gte: from, lte: to } } }, + _count: { rating: true }, + }); + + const escalatedSessions = Math.max(escalatedByStatus, escalatedByResolver); + + return NextResponse.json({ + from: from.toISOString(), + to: to.toISOString(), + total_sessions: totalSessions, + resolved_sessions: resolvedSessions, + escalated_sessions: escalatedSessions, + abandoned_sessions: Math.max(0, totalSessions - resolvedSessions - escalatedSessions), + resolution_rate: totalSessions > 0 ? resolvedSessions / totalSessions : 0, + escalation_rate: totalSessions > 0 ? escalatedSessions / totalSessions : 0, + avg_csat: feedbackAgg._avg.rating ?? null, + csat_response_count: feedbackAgg._count.rating, + csat_distribution: ratingDistribution + .map((row) => ({ rating: row.rating, count: row._count.rating })) + .sort((a, b) => a.rating - b.rating), + daily: dailyRows.map((row) => ({ + date: row.day.toISOString().slice(0, 10), + total: coerceDbNumber(row.total), + resolved: coerceDbNumber(row.resolved), + escalated: coerceDbNumber(row.escalated), + })), + }); +} diff --git a/dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/reply/route.ts b/dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/reply/route.ts new file mode 100644 index 0000000..27b6c5e --- /dev/null +++ b/dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/reply/route.ts @@ -0,0 +1,34 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { getDeveloperFromRequest } from "@/lib/server/auth"; +import { getOwnedAppOrNull } from "@/lib/server/apps"; +import { detail } from "@/lib/server/http"; +import { prisma } from "@/lib/server/prisma"; +import { postHumanMessage } from "@/lib/server/agent-service"; + +export const dynamic = "force-dynamic"; + +export async function POST( + request: NextRequest, + context: { params: Promise<{ appId: string; sessionId: string }> }, +) { + const developer = await getDeveloperFromRequest(request); + if (!developer) return detail(401, "Invalid token"); + + const { appId, sessionId } = await context.params; + const app = await getOwnedAppOrNull(appId, developer); + if (!app) return detail(404, "App not found"); + + const session = await prisma.chatSession.findUnique({ where: { id: sessionId } }); + if (!session || session.appId !== app.id) return detail(404, "Session not found"); + + const body = await request.json().catch(() => null); + const text = typeof body?.text === "string" ? body.text.trim() : ""; + if (!text) return detail(400, "text is required"); + + // The agent service owns the Message write and pushes it onto the session's + // live SSE stream — do not also write it here, or the message would be duplicated. + const message = await postHumanMessage(sessionId, text); + + return NextResponse.json(message); +} diff --git a/dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/resolve/route.ts b/dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/resolve/route.ts new file mode 100644 index 0000000..82ecab0 --- /dev/null +++ b/dashboard/src/app/v1/apps/[appId]/sessions/[sessionId]/resolve/route.ts @@ -0,0 +1,31 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { getDeveloperFromRequest } from "@/lib/server/auth"; +import { getOwnedAppOrNull } from "@/lib/server/apps"; +import { detail } from "@/lib/server/http"; +import { prisma } from "@/lib/server/prisma"; +import { sessionOut } from "@/lib/server/serializers"; + +export const dynamic = "force-dynamic"; + +export async function POST( + request: NextRequest, + context: { params: Promise<{ appId: string; sessionId: string }> }, +) { + const developer = await getDeveloperFromRequest(request); + if (!developer) return detail(401, "Invalid token"); + + const { appId, sessionId } = await context.params; + const app = await getOwnedAppOrNull(appId, developer); + if (!app) return detail(404, "App not found"); + + const session = await prisma.chatSession.findUnique({ where: { id: sessionId } }); + if (!session || session.appId !== app.id) return detail(404, "Session not found"); + + const updated = await prisma.chatSession.update({ + where: { id: sessionId }, + data: { status: "closed", resolvedBy: "human" }, + }); + + return NextResponse.json(sessionOut(updated)); +} diff --git a/dashboard/src/components/AppSidebar.tsx b/dashboard/src/components/AppSidebar.tsx index fcda323..58594eb 100644 --- a/dashboard/src/components/AppSidebar.tsx +++ b/dashboard/src/components/AppSidebar.tsx @@ -26,12 +26,14 @@ const CUSTOMIZATION_ITEMS: NavItem[] = [ ]; const CUSTOMIZATION_PARENT_ITEM: NavItem = { label: "Customization", slug: "chat-theme" }; const SESSIONS_ITEM: NavItem = { label: "Sessions", slug: "sessions" }; +const ANALYTICS_ITEM: NavItem = { label: "Analytics", slug: "analytics" }; const AUDIT_ITEM: NavItem = { label: "Audit Log", slug: "audit" }; const ROUTE_LABEL_ITEMS: NavItem[] = [ ...AGENT_CHILD_ITEMS, API_KEYS_ITEM, ...CUSTOMIZATION_ITEMS, SESSIONS_ITEM, + ANALYTICS_ITEM, AUDIT_ITEM, ]; const AGENT_SECTION_SLUGS = AGENT_CHILD_ITEMS.map((item) => item.slug); @@ -218,6 +220,7 @@ export default function AppSidebar({ variant = "desktop" }: AppSidebarProps) { )} {renderNavButton(SESSIONS_ITEM)} + {renderNavButton(ANALYTICS_ITEM)} {renderNavButton(AUDIT_ITEM)} @@ -280,6 +283,7 @@ export default function AppSidebar({ variant = "desktop" }: AppSidebarProps) { )} {renderNavButton(SESSIONS_ITEM)} + {renderNavButton(ANALYTICS_ITEM)} {renderNavButton(AUDIT_ITEM)} diff --git a/dashboard/src/components/ui/AppNav.tsx b/dashboard/src/components/ui/AppNav.tsx index 33d54ab..266d762 100644 --- a/dashboard/src/components/ui/AppNav.tsx +++ b/dashboard/src/components/ui/AppNav.tsx @@ -18,6 +18,7 @@ const TABS = [ { label: "Chat Theme", slug: "chat-theme" }, { label: "Localization", slug: "languages" }, { label: "Sessions", slug: "sessions" }, + { label: "Analytics", slug: "analytics" }, { label: "Audit Log", slug: "audit" }, ]; diff --git a/dashboard/src/dashboard-app.tsx b/dashboard/src/dashboard-app.tsx index 111f53f..63cb7f0 100644 --- a/dashboard/src/dashboard-app.tsx +++ b/dashboard/src/dashboard-app.tsx @@ -14,6 +14,7 @@ import AppKnowledgeBases from "./dashboard_pages/AppKnowledgeBases"; import OrganizationAdmin from "./dashboard_pages/OrganizationAdmin"; import Playbooks from "./dashboard_pages/Playbooks"; import Sessions from "./dashboard_pages/Sessions"; +import Analytics from "./dashboard_pages/Analytics"; import ChatTheme from "./dashboard_pages/ChatTheme"; export function RootRedirect() { @@ -37,6 +38,7 @@ export function DashboardApp() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/dashboard/src/dashboard_pages/Analytics.tsx b/dashboard/src/dashboard_pages/Analytics.tsx new file mode 100644 index 0000000..1e1d43b --- /dev/null +++ b/dashboard/src/dashboard_pages/Analytics.tsx @@ -0,0 +1,243 @@ +import { useEffect, useMemo, useState } from "react"; +import { useParams } from "react-router-dom"; + +import { api } from "../api/client"; +import { PageSpinner } from "../components/ui"; +import { PageHeader } from "../components/layout/PageHeader"; + +interface DailyPoint { + date: string; + total: number; + resolved: number; + escalated: number; +} + +interface AnalyticsSummary { + total_sessions: number; + resolved_sessions: number; + escalated_sessions: number; + abandoned_sessions: number; + resolution_rate: number; + escalation_rate: number; + avg_csat: number | null; + csat_response_count: number; + csat_distribution: Array<{ rating: number; count: number }>; + daily: DailyPoint[]; +} + +function formatPercent(value: number): string { + return `${Math.round(value * 100)}%`; +} + +function StatTile({ label, value, sublabel }: { label: string; value: string; sublabel?: string }) { + return ( +
+

{label}

+

{value}

+ {sublabel &&

{sublabel}

} +
+ ); +} + +function DailyChart({ daily }: { daily: DailyPoint[] }) { + const [hoverIndex, setHoverIndex] = useState(null); + + const width = 640; + const height = 220; + const padding = { top: 12, right: 12, bottom: 24, left: 12 }; + const plotWidth = width - padding.left - padding.right; + const plotHeight = height - padding.top - padding.bottom; + + const maxTotal = Math.max(1, ...daily.map((d) => d.total)); + const stepX = daily.length > 1 ? plotWidth / (daily.length - 1) : 0; + + const points = (key: "resolved" | "escalated") => + daily + .map((d, i) => { + const x = padding.left + i * stepX; + const y = padding.top + plotHeight - (d[key] / maxTotal) * plotHeight; + return `${x},${y}`; + }) + .join(" "); + + if (daily.length === 0) { + return

No session activity in this range.

; + } + + const hovered = hoverIndex !== null ? daily[hoverIndex] : null; + + return ( +
+ setHoverIndex(null)} + onMouseMove={(e) => { + const rect = e.currentTarget.getBoundingClientRect(); + const relX = ((e.clientX - rect.left) / rect.width) * width - padding.left; + const idx = stepX > 0 ? Math.round(relX / stepX) : 0; + setHoverIndex(Math.min(daily.length - 1, Math.max(0, idx))); + }} + > + + + + {hoverIndex !== null && ( + + )} + + {hovered && ( +
+

{hovered.date}

+

resolved: {hovered.resolved}

+

escalated: {hovered.escalated}

+
+ )} +
+ + Resolved (AI) + + + Escalated + +
+
+ ); +} + +export default function Analytics() { + const { appId } = useParams<{ appId: string }>(); + const [data, setData] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [rangeDays, setRangeDays] = useState(30); + + useEffect(() => { + setIsLoading(true); + setError(null); + const to = new Date(); + const from = new Date(to.getTime() - rangeDays * 24 * 60 * 60 * 1000); + api( + `/v1/apps/${appId}/analytics?from=${from.toISOString()}&to=${to.toISOString()}` + ) + .then(setData) + .catch(() => setError("Failed to load analytics")) + .finally(() => setIsLoading(false)); + }, [appId, rangeDays]); + + const maxDistCount = useMemo( + () => Math.max(1, ...(data?.csat_distribution.map((row) => row.count) ?? [])), + [data] + ); + + if (isLoading) return ; + + return ( +
+ + +
+ +
+ + {error &&

{error}

} + + {data && ( + <> +
+ + + + +
+ +
+
+

Daily Sessions

+ +
+ +
+

CSAT Distribution

+ {data.csat_distribution.length === 0 ? ( +

No feedback submitted yet.

+ ) : ( +
+ {[1, 2, 3, 4, 5].map((rating) => { + const row = data.csat_distribution.find((r) => r.rating === rating); + const count = row?.count ?? 0; + return ( +
+ {rating}★ +
+
+
+ {count} +
+ ); + })} +
+ )} +
+
+ + )} +
+ ); +} diff --git a/dashboard/src/lib/server/agent-service.ts b/dashboard/src/lib/server/agent-service.ts new file mode 100644 index 0000000..e69faa4 --- /dev/null +++ b/dashboard/src/lib/server/agent-service.ts @@ -0,0 +1,73 @@ +import { SignJWT } from "jose"; + +const AGENT_BASE_URL = (process.env.RESOLVEKIT_SERVER_AGENT_BASE_URL ?? "http://localhost:8000").replace(/\/$/, ""); +const INTERNAL_AUDIENCE = process.env.RK_INTERNAL_SERVICE_AUDIENCE ?? "agent-service"; +const INTERNAL_JWT_ALGORITHM = process.env.RK_INTERNAL_SERVICE_JWT_ALGORITHM ?? "HS256"; + +const INSECURE_KEY_VALUES = new Set(["", "change-me-internal-service-signing-key"]); + +function resolveSigningKey(): string { + const value = (process.env.RK_INTERNAL_SERVICE_SIGNING_KEY ?? "").trim(); + if (INSECURE_KEY_VALUES.has(value)) { + if (process.env.NODE_ENV === "test") { + return "test-only-internal-service-signing-key"; + } + // Skip during `next build` — runtime secrets are not available at build time. + if (process.env.NEXT_PHASE !== "phase-production-build") { + throw new Error("RK_INTERNAL_SERVICE_SIGNING_KEY must be set to a secure non-default value"); + } + return "build-phase-placeholder-internal-signing-key"; + } + return value; +} + +let _signingKey: string | null = null; +function getSigningKey(): string { + if (_signingKey === null) _signingKey = resolveSigningKey(); + return _signingKey; +} + +function jwtSecretBytes(): Uint8Array { + return new TextEncoder().encode(getSigningKey()); +} + +async function buildServiceToken(): Promise { + const now = Math.floor(Date.now() / 1000); + return new SignJWT({}) + .setIssuer("core-api") + .setAudience(INTERNAL_AUDIENCE) + .setSubject("core-api") + .setIssuedAt(now) + .setExpirationTime(now + 120) + .setProtectedHeader({ alg: INTERNAL_JWT_ALGORITHM }) + .sign(jwtSecretBytes()); +} + +export type AgentHumanMessage = { + id: string; + created_at: string; + session_id: string; + sequence_number: number; + role: string; + content: string | null; +}; + +export async function postHumanMessage(sessionId: string, text: string): Promise { + const token = await buildServiceToken(); + const response = await fetch(`${AGENT_BASE_URL}/internal/sessions/${sessionId}/human-message`, { + method: "POST", + cache: "no-store", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ text }), + }); + + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error(`Agent internal human-message call failed: ${response.status} ${body}`); + } + + return (await response.json()) as AgentHumanMessage; +} diff --git a/docs/generated/openapi/agent.openapi.json b/docs/generated/openapi/agent.openapi.json index 8a62933..ff64cee 100644 --- a/docs/generated/openapi/agent.openapi.json +++ b/docs/generated/openapi/agent.openapi.json @@ -348,6 +348,21 @@ "title": "HTTPValidationError", "type": "object" }, + "HumanMessageBody": { + "properties": { + "text": { + "maxLength": 32768, + "minLength": 1, + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "HumanMessageBody", + "type": "object" + }, "MessageOut": { "properties": { "content": { @@ -846,6 +861,77 @@ "title": "SessionCreate", "type": "object" }, + "SessionFeedbackCreate": { + "additionalProperties": false, + "properties": { + "comment": { + "anyOf": [ + { + "maxLength": 2000, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Comment" + }, + "rating": { + "maximum": 5.0, + "minimum": 1.0, + "title": "Rating", + "type": "integer" + } + }, + "required": [ + "rating" + ], + "title": "SessionFeedbackCreate", + "type": "object" + }, + "SessionFeedbackOut": { + "properties": { + "comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Comment" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "rating": { + "title": "Rating", + "type": "integer" + }, + "session_id": { + "format": "uuid", + "title": "Session Id", + "type": "string" + } + }, + "required": [ + "id", + "session_id", + "rating", + "comment", + "created_at" + ], + "title": "SessionFeedbackOut", + "type": "object" + }, "SessionOut": { "properties": { "app_id": { @@ -1068,6 +1154,12 @@ "title": "ValidationError", "type": "object" } + }, + "securitySchemes": { + "HTTPBearer": { + "scheme": "bearer", + "type": "http" + } } }, "info": { @@ -1093,6 +1185,62 @@ "summary": "Health" } }, + "/internal/sessions/{session_id}/human-message": { + "post": { + "operationId": "post_human_message_internal_sessions__session_id__human_message_post", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Session Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HumanMessageBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Post Human Message", + "tags": [ + "chat-events" + ] + } + }, "/v1/functions": { "get": { "operationId": "list_functions_sdk_v1_functions_get", @@ -1479,6 +1627,59 @@ ] } }, + "/v1/sessions/{session_id}/feedback": { + "post": { + "operationId": "submit_session_feedback_v1_sessions__session_id__feedback_post", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Session Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionFeedbackCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionFeedbackOut" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Submit Session Feedback", + "tags": [ + "sessions-sdk" + ] + } + }, "/v1/sessions/{session_id}/localization": { "get": { "operationId": "get_session_localization_v1_sessions__session_id__localization_get", diff --git a/docs/generated/openapi/dashboard.openapi.json b/docs/generated/openapi/dashboard.openapi.json index dfbc77f..2cd7877 100644 --- a/docs/generated/openapi/dashboard.openapi.json +++ b/docs/generated/openapi/dashboard.openapi.json @@ -119,6 +119,27 @@ ] } }, + "/v1/apps/appId/analytics": { + "get": { + "operationId": "get_v1_apps_appId_analytics", + "responses": { + "200": { + "description": "Success" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "tags": [ + "apps" + ] + } + }, "/v1/apps/appId/api-keys": { "get": { "operationId": "get_v1_apps_appId_api-keys", @@ -687,6 +708,48 @@ ] } }, + "/v1/apps/appId/sessions/sessionId/reply": { + "post": { + "operationId": "post_v1_apps_appId_sessions_sessionId_reply", + "responses": { + "200": { + "description": "Success" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "tags": [ + "apps" + ] + } + }, + "/v1/apps/appId/sessions/sessionId/resolve": { + "post": { + "operationId": "post_v1_apps_appId_sessions_sessionId_resolve", + "responses": { + "200": { + "description": "Success" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "bearerAuth": [] + } + ], + "tags": [ + "apps" + ] + } + }, "/v1/auth/login": { "post": { "operationId": "post_v1_auth_login", diff --git a/tests/test_analytics_aggregation_contract.py b/tests/test_analytics_aggregation_contract.py new file mode 100644 index 0000000..b45455e --- /dev/null +++ b/tests/test_analytics_aggregation_contract.py @@ -0,0 +1,24 @@ +from pathlib import Path + + +def test_analytics_route_computes_resolution_and_escalation_rates() -> None: + text = Path("dashboard/src/app/v1/apps/[appId]/analytics/route.ts").read_text(encoding="utf-8") + + assert 'resolvedBy: "ai"' in text + assert 'status: "escalated"' in text + assert 'resolvedBy: "human"' in text + assert "resolution_rate: totalSessions > 0 ? resolvedSessions / totalSessions : 0" in text + assert "escalation_rate: totalSessions > 0 ? escalatedSessions / totalSessions : 0" in text + + +def test_analytics_route_aggregates_csat_from_session_feedback() -> None: + text = Path("dashboard/src/app/v1/apps/[appId]/analytics/route.ts").read_text(encoding="utf-8") + + assert "prisma.sessionFeedback.aggregate(" in text + assert "prisma.sessionFeedback.groupBy(" in text + + +def test_expire_stale_sessions_marks_unescalated_sessions_ai_resolved() -> None: + text = Path("agent/services/session_service.py").read_text(encoding="utf-8") + + assert '.values(status="expired", resolved_by="ai")' in text diff --git a/tests/test_orchestrator_chat_unavailable.py b/tests/test_orchestrator_chat_unavailable.py index 187504c..936f508 100644 --- a/tests/test_orchestrator_chat_unavailable.py +++ b/tests/test_orchestrator_chat_unavailable.py @@ -32,6 +32,9 @@ async def send_tool_call_request( async def send_turn_complete(self, full_text: str, usage: dict | None) -> None: raise AssertionError("turn_complete should not be called when llm call fails") + async def send_feedback_requested(self) -> None: + return None + async def send_error(self, code: str, message: str, recoverable: bool = True) -> None: self.code = code self.message = message diff --git a/tests/test_orchestrator_kb_tool_result_flow.py b/tests/test_orchestrator_kb_tool_result_flow.py index d22616b..19768f5 100644 --- a/tests/test_orchestrator_kb_tool_result_flow.py +++ b/tests/test_orchestrator_kb_tool_result_flow.py @@ -37,6 +37,9 @@ async def send_turn_complete(self, full_text: str, usage: dict | None) -> None: self.turn_complete_text = full_text self.turn_complete_usage = usage + async def send_feedback_requested(self) -> None: + return None + async def send_error(self, code: str, message: str, recoverable: bool = True) -> None: raise AssertionError(f"Unexpected error sent to client: {code} {message}") @@ -217,5 +220,9 @@ async def fake_next_sequence(_db, _session_id): # noqa: ANN001 ) llm_mock.assert_awaited_once() - assert llm_mock.await_args.args[2] is None + tools_sent = llm_mock.await_args.args[2] + assert tools_sent is not None + tool_names = {tool["function"]["name"] for tool in tools_sent} + assert "kb_search" not in tool_names + assert "escalate_to_human" in tool_names assert sender.turn_complete_text == "Use Settings > Account > Reset Password." diff --git a/tests/test_orchestrator_router_enriched_context.py b/tests/test_orchestrator_router_enriched_context.py index bc45ccf..558760f 100644 --- a/tests/test_orchestrator_router_enriched_context.py +++ b/tests/test_orchestrator_router_enriched_context.py @@ -37,6 +37,9 @@ async def send_turn_complete(self, full_text: str, usage: dict | None) -> None: self.turn_complete_text = full_text self.turn_complete_usage = usage + async def send_feedback_requested(self) -> None: + return None + async def send_error(self, code: str, message: str, recoverable: bool = True) -> None: self.errors.append((code, message, recoverable)) diff --git a/tests/test_session_escalation_contract.py b/tests/test_session_escalation_contract.py new file mode 100644 index 0000000..6c2e310 --- /dev/null +++ b/tests/test_session_escalation_contract.py @@ -0,0 +1,25 @@ +from pathlib import Path + + +def test_orchestrator_registers_and_handles_escalation_tool() -> None: + text = Path("agent/services/orchestrator.py").read_text(encoding="utf-8") + + assert 'ESCALATE_TOOL_NAME = "escalate_to_human"' in text + assert "tools.append(_build_escalate_tool())" in text + assert 'session.status = "escalated"' in text + assert "session.escalated_at = datetime.now(timezone.utc)" in text + assert "session.escalation_reason = reason" in text + + +def test_orchestrator_escalates_on_max_tool_rounds_exceeded() -> None: + text = Path("agent/services/orchestrator.py").read_text(encoding="utf-8") + + assert "if tool_round >= config.max_tool_rounds:" in text + assert 'await escalate_session(db, session, sender, "Maximum tool calling rounds exceeded")' in text + + +def test_event_stream_sender_emits_escalation_and_feedback_events() -> None: + text = Path("agent/routers/chat_events.py").read_text(encoding="utf-8") + + assert 'await self._push("session_escalated", {"reason": reason})' in text + assert 'await self._push("feedback_requested", {})' in text diff --git a/tests/test_session_feedback_contract.py b/tests/test_session_feedback_contract.py new file mode 100644 index 0000000..7a7d5a6 --- /dev/null +++ b/tests/test_session_feedback_contract.py @@ -0,0 +1,23 @@ +from pathlib import Path + + +def test_feedback_endpoint_rejects_duplicate_ratings() -> None: + text = Path("agent/routers/sessions.py").read_text(encoding="utf-8") + + assert '@sdk_router.post("/{session_id}/feedback"' in text + assert "status_code=status.HTTP_409_CONFLICT" in text + assert "Feedback already submitted for this session" in text + + +def test_feedback_schema_bounds_rating_one_to_five() -> None: + text = Path("agent/schemas/session.py").read_text(encoding="utf-8") + + assert "class SessionFeedbackCreate(BaseModel):" in text + assert "rating: int = Field(ge=1, le=5)" in text + + +def test_session_feedback_model_enforces_one_rating_per_session() -> None: + text = Path("agent/models/session_feedback.py").read_text(encoding="utf-8") + + assert 'ForeignKey("chat_sessions.id", ondelete="CASCADE"), unique=True' in text + assert 'CheckConstraint("rating >= 1 AND rating <= 5"' in text From 9ff26d7b04de5f13f488ffd409f420a156fd58df Mon Sep 17 00:00:00 2001 From: "nedas.vi" Date: Tue, 21 Jul 2026 10:04:53 +0300 Subject: [PATCH 2/9] Include human_agent messages in SDK session history The reused-session-history endpoint filtered messages to user/assistant only, so a human_agent reply sent while a client was offline would never appear on reconnect (only via the live human_message SSE event). Co-Authored-By: Claude Sonnet 5 --- agent/routers/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/routers/sessions.py b/agent/routers/sessions.py index 4705256..b91fd24 100644 --- a/agent/routers/sessions.py +++ b/agent/routers/sessions.py @@ -198,7 +198,7 @@ async def get_session_messages_sdk( result = await db.execute( select(Message).where( Message.session_id == session_id, - Message.role.in_(["user", "assistant"]), + Message.role.in_(["user", "assistant", "human_agent"]), ).order_by(Message.sequence_number) ) return result.scalars().all() From fc78a4bd6dfa7ab1b38718739596019a21800189 Mon Sep 17 00:00:00 2001 From: "nedas.vi" Date: Tue, 21 Jul 2026 10:29:55 +0300 Subject: [PATCH 3/9] Add reply/resolve UI for escalated sessions in the dashboard Sessions previously only had reply/resolve as raw API endpoints with no way to use them from the dashboard. Adds an escalation banner with escalation reason, a "Mark resolved" button, and a reply composer on escalated sessions in the Sessions page. Also surfaces escalated_at/ escalation_reason/resolved_by from sessionOut, which the serializer was previously dropping. Co-Authored-By: Claude Sonnet 5 --- dashboard/src/dashboard_pages/Sessions.tsx | 89 +++++++++++++++++++++- dashboard/src/lib/server/serializers.ts | 6 ++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/dashboard/src/dashboard_pages/Sessions.tsx b/dashboard/src/dashboard_pages/Sessions.tsx index 9f05b01..9f63782 100644 --- a/dashboard/src/dashboard_pages/Sessions.tsx +++ b/dashboard/src/dashboard_pages/Sessions.tsx @@ -14,6 +14,7 @@ interface Session { client_context?: Record; llm_context?: Record; cost_summary?: SessionCostSummary | null; + escalation_reason?: string | null; } interface Message { @@ -71,10 +72,11 @@ type KbSearchResult = { error: string | null; }; -function statusVariant(status: string): "active" | "expired" | "closed" | "default" { +function statusVariant(status: string): "active" | "expired" | "closed" | "danger" | "default" { if (status === "active") return "active"; if (status === "expired") return "expired"; if (status === "closed") return "closed"; + if (status === "escalated") return "danger"; return "default"; } @@ -285,6 +287,10 @@ export default function Sessions() { const [selectedCostSummary, setSelectedCostSummary] = useState(null); const [costsLoading, setCostsLoading] = useState(false); const [costsError, setCostsError] = useState(null); + const [replyText, setReplyText] = useState(""); + const [isSendingReply, setIsSendingReply] = useState(false); + const [replyError, setReplyError] = useState(null); + const [isResolving, setIsResolving] = useState(false); const abortRef = useRef(null); const pollRef = useRef | null>(null); @@ -331,6 +337,8 @@ export default function Sessions() { setSelectedCostSummary(sessions.find((entry) => entry.id === sessionId)?.cost_summary ?? null); setCostsError(null); setCostsLoading(true); + setReplyText(""); + setReplyError(null); if (pollRef.current) { clearInterval(pollRef.current); @@ -353,7 +361,7 @@ export default function Sessions() { setCostsLoading(false); const session = sessions.find((entry) => entry.id === sessionId); - if (session?.status === "active") { + if (session?.status === "active" || session?.status === "escalated") { pollRef.current = setInterval(async () => { try { const updated = await api(`/v1/apps/${appId}/sessions/${sessionId}/messages`); @@ -382,6 +390,42 @@ export default function Sessions() { }; }, []); + async function sendReply() { + const sessionId = selectedId; + const text = replyText.trim(); + if (!sessionId || !text) return; + setIsSendingReply(true); + setReplyError(null); + try { + const message = await api(`/v1/apps/${appId}/sessions/${sessionId}/reply`, { + method: "POST", + body: JSON.stringify({ text }), + }); + setMessages((prev) => [...prev, message]); + setReplyText(""); + } catch (err: unknown) { + setReplyError(err instanceof ApiError ? err.detail : "Failed to send reply"); + } finally { + setIsSendingReply(false); + } + } + + async function resolveSession() { + const sessionId = selectedId; + if (!sessionId) return; + setIsResolving(true); + try { + const updated = await api(`/v1/apps/${appId}/sessions/${sessionId}/resolve`, { + method: "POST", + }); + setSessions((prev) => prev.map((entry) => (entry.id === sessionId ? updated : entry))); + } catch (err: unknown) { + setReplyError(err instanceof ApiError ? err.detail : "Failed to resolve session"); + } finally { + setIsResolving(false); + } + } + const selectedSession = sessions.find((session) => session.id === selectedId); const toolResultByCallId = useMemo(() => { const map = new Map(); @@ -569,6 +613,24 @@ export default function Sessions() {
)} {selectedSession.llm_context && } + {selectedSession.status === "escalated" && ( +
+
+

Escalated to human

+ {selectedSession.escalation_reason && ( +

{selectedSession.escalation_reason}

+ )} +
+ +
+ )}
@@ -628,6 +690,29 @@ export default function Sessions() { }) )}
+ + {selectedSession.status === "escalated" && ( +
+ {replyError &&

{replyError}

} +
+