From 42fc2d45bccaf102a792d3ccc9f8c6e36e2e0977 Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:29:02 -0700 Subject: [PATCH 1/4] feat(search): weighted FTS ranking + searchable message bodies Results ranked by relevance instead of date-only: subject > sender > body (setweight A-D + ts_rank_cd), prefix matching as you type, quoted phrases, served by a GIN index over a trigger-maintained search_fts column. Backfill runs as a resumable background drainer (fast-DDL migrations 0035-0037, no boot-blocking rewrite); not-yet-backfilled rows fall back to the previous query path. Body text is materialized by a provider-gated background IMAP drainer with circuit breakers, poison-message forward progress, and progress reporting. Filter-only queries stay date-ordered. No infrastructure changes - runs on stock postgres:16-alpine. Split 1/3 of #283. Co-Authored-By: Claude Fable 5 --- backend/migrations/0035_search_fts.sql | 55 +++ backend/migrations/0036_background_jobs.sql | 19 + backend/migrations/0037_search_fts_index.sql | 24 ++ backend/src/index.js | 6 + backend/src/routes/accounts.js | 16 +- backend/src/routes/indexing.js | 13 + backend/src/routes/search.js | 230 +---------- backend/src/routes/search.test.js | 192 +++------ backend/src/services/backgroundJobs.js | 26 ++ backend/src/services/backgroundJobs.test.js | 25 ++ backend/src/services/bodyBackfill.js | 181 ++++++++ backend/src/services/bodyBackfill.test.js | 290 +++++++++++++ backend/src/services/imapManager.js | 107 +++-- backend/src/services/imapManager.test.js | 133 +++++- backend/src/services/search/ftsBackfill.js | 108 +++++ .../src/services/search/ftsBackfill.test.js | 45 ++ backend/src/services/search/lexicalRepo.js | 267 ++++++++++++ .../src/services/search/lexicalRepo.test.js | 385 ++++++++++++++++++ .../src/services/search/migrations.test.js | 72 ++++ backend/src/services/search/queryParser.js | 185 +++++++++ .../src/services/search/queryParser.test.js | 225 ++++++++++ backend/src/services/search/searchService.js | 43 ++ .../src/services/search/searchService.test.js | 67 +++ backend/src/testSupport/testAccount.js | 19 + frontend/src/components/MessageList.jsx | 8 +- frontend/src/utils/api.search.test.js | 18 + 26 files changed, 2367 insertions(+), 392 deletions(-) create mode 100644 backend/migrations/0035_search_fts.sql create mode 100644 backend/migrations/0036_background_jobs.sql create mode 100644 backend/migrations/0037_search_fts_index.sql create mode 100644 backend/src/routes/indexing.js create mode 100644 backend/src/services/backgroundJobs.js create mode 100644 backend/src/services/backgroundJobs.test.js create mode 100644 backend/src/services/bodyBackfill.js create mode 100644 backend/src/services/bodyBackfill.test.js create mode 100644 backend/src/services/search/ftsBackfill.js create mode 100644 backend/src/services/search/ftsBackfill.test.js create mode 100644 backend/src/services/search/lexicalRepo.js create mode 100644 backend/src/services/search/lexicalRepo.test.js create mode 100644 backend/src/services/search/migrations.test.js create mode 100644 backend/src/services/search/queryParser.js create mode 100644 backend/src/services/search/queryParser.test.js create mode 100644 backend/src/services/search/searchService.js create mode 100644 backend/src/services/search/searchService.test.js create mode 100644 backend/src/testSupport/testAccount.js create mode 100644 frontend/src/utils/api.search.test.js diff --git a/backend/migrations/0035_search_fts.sql b/backend/migrations/0035_search_fts.sql new file mode 100644 index 00000000..519c213c --- /dev/null +++ b/backend/migrations/0035_search_fts.sql @@ -0,0 +1,55 @@ +-- Weighted full-text search column (search_fts) + version stamp, kept fresh by +-- a BEFORE trigger. Fast, metadata-only DDL only (README D1): the nullable +-- columns force no table rewrite on PG16, and the function/trigger are instant. +-- Pre-existing rows are populated by the resumable drainer (ftsBackfill.js); +-- the GIN index is built CONCURRENTLY in the separate no-transaction migration +-- 0037 (a $$-quoted plpgsql body cannot survive the no-transaction ; splitter, +-- so the trigger and the CONCURRENTLY index MUST live in different files). +-- +-- The setweight(...) expression MUST stay identical to +-- lexicalRepo.searchFtsExpr('NEW'); fts_version = 1 matches lexicalRepo.FTS_VERSION. +-- Idempotent (IF NOT EXISTS / CREATE OR REPLACE / DROP IF EXISTS) because a +-- crash before the schema_migrations INSERT retries this migration. + +ALTER TABLE messages ADD COLUMN IF NOT EXISTS search_fts tsvector; +ALTER TABLE messages ADD COLUMN IF NOT EXISTS fts_version int; + +CREATE OR REPLACE FUNCTION messages_search_fts_refresh() RETURNS trigger AS $$ +BEGIN + -- Skip recompute on an UPDATE that changes none of the indexed source + -- columns (read/star flag flips, snippet-only writes), so the trigger does + -- not tax hot sync UPSERTs; this also lets the backfill's explicit SET win. + IF TG_OP = 'UPDATE' + AND NEW.subject IS NOT DISTINCT FROM OLD.subject + AND NEW.from_name IS NOT DISTINCT FROM OLD.from_name + AND NEW.from_email IS NOT DISTINCT FROM OLD.from_email + AND NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses + AND NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses + AND NEW.body_text IS NOT DISTINCT FROM OLD.body_text + THEN + RETURN NEW; + END IF; + + BEGIN + NEW.search_fts := setweight(to_tsvector('english', coalesce(NEW.subject,'')), 'A') || + setweight(to_tsvector('english', coalesce(NEW.from_name,'') || ' ' || coalesce(NEW.from_email,'')), 'B') || + setweight(to_tsvector('english', coalesce(NEW.to_addresses::text,'') || ' ' || coalesce(NEW.cc_addresses::text,'')), 'C') || + setweight(to_tsvector('english', LEFT(coalesce(NEW.body_text,''), 600000)), 'D'); + NEW.fts_version := 1; + EXCEPTION WHEN program_limit_exceeded THEN + -- Even with the 600k LEFT cap, a pathologically dense/multibyte body can + -- exceed Postgres's ~1MB tsvector limit (SQLSTATE 54000). Never fail the + -- row write: leave search_fts NULL so the message still persists and stays + -- findable via the ILIKE fallback; the backfill's row-by-row skip stamps it. + NEW.search_fts := NULL; + NEW.fts_version := NULL; + END; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_messages_search_fts ON messages; +CREATE TRIGGER trg_messages_search_fts + BEFORE INSERT OR UPDATE ON messages + FOR EACH ROW EXECUTE FUNCTION messages_search_fts_refresh(); diff --git a/backend/migrations/0036_background_jobs.sql b/backend/migrations/0036_background_jobs.sql new file mode 100644 index 00000000..d189933c --- /dev/null +++ b/backend/migrations/0036_background_jobs.sql @@ -0,0 +1,19 @@ +-- Generic progress substrate for observable background drainers (first consumer: +-- the FTS backfill). Plain table, fast DDL. One row per (kind, account); global +-- jobs use a NULL account_id, COALESCE'd to '' in the unique index so the upsert +-- has a single conflict target for both cases. + +CREATE TABLE IF NOT EXISTS background_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + kind VARCHAR(64) NOT NULL, + account_id UUID REFERENCES email_accounts(id) ON DELETE CASCADE, + state VARCHAR(20) NOT NULL DEFAULT 'idle', + processed BIGINT NOT NULL DEFAULT 0, + total BIGINT NOT NULL DEFAULT 0, + last_error TEXT, + started_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_background_jobs_kind_account + ON background_jobs (kind, COALESCE(account_id::text, '')); diff --git a/backend/migrations/0037_search_fts_index.sql b/backend/migrations/0037_search_fts_index.sql new file mode 100644 index 00000000..f82b7dbb --- /dev/null +++ b/backend/migrations/0037_search_fts_index.sql @@ -0,0 +1,24 @@ +-- no-transaction +-- +-- GIN index that serves `search_fts @@ tsquery`, plus a partial btree that lets +-- the backfill drainer (and any "needs backfill" probe) find not-yet-stamped +-- rows without a full seq scan; it self-prunes as fts_version = 1 fills in. +-- CONCURRENTLY must run outside a transaction, so these live apart from 0035's +-- trigger. Idempotent via IF NOT EXISTS (retried if a crash precedes the +-- schema_migrations INSERT). +-- +-- DROP ... IF EXISTS before each CREATE: a cancelled or crashed CREATE INDEX +-- CONCURRENTLY leaves an INVALID index under the target name. On retry, plain +-- IF NOT EXISTS sees that name and silently skips the create, recording the +-- migration as done while the scan stays unindexed forever. This file only re-runs +-- after such a failure — in which case the index is either absent (drop is a no-op) +-- or invalid (drop removes the dead stub) — so dropping first is safe and cheap. + +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_search_fts; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts + ON messages USING GIN (search_fts); + +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_fts_stale_v1; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_fts_stale_v1 + ON messages (date DESC) + WHERE fts_version IS DISTINCT FROM 1; diff --git a/backend/src/index.js b/backend/src/index.js index 483dcfa0..70b0a85f 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -18,6 +18,7 @@ import accountRoutes from './routes/accounts.js'; import mailRoutes from './routes/mail.js'; import searchRoutes from './routes/search.js'; import adminRoutes from './routes/admin.js'; +import indexingRoutes from './routes/indexing.js'; import totpRoutes from './routes/totp.js'; import oidcApiRouter, { oidcBrowserRouter } from './routes/oidc.js'; import rulesRoutes from './routes/rules.js'; @@ -30,6 +31,7 @@ import gtdRoutes from './routes/gtd.js'; import carddavRouter from './routes/carddav.js'; import carddavAccountRouter from './routes/carddavAccount.js'; import { startCardavScheduler } from './services/carddavSync.js'; +import { scheduleFtsBackfill } from './services/search/ftsBackfill.js'; import { encryptExistingCredentials, query } from './services/db.js'; import { runMigrations } from './services/migrations.js'; import { parseVCard } from './utils/vcard.js'; @@ -167,6 +169,7 @@ app.use('/api/mail', sendRoutes); app.use('/api/mail', draftRoutes); app.use('/api/search', searchRoutes); app.use('/api/admin', adminRoutes); +app.use('/api/admin/indexing', indexingRoutes); app.use('/api/totp', totpRoutes); app.use('/api/rules', rulesRoutes); app.use('/api/block-list', blockListRoutes); @@ -244,6 +247,9 @@ imapManager.startSnoozeWatcher(); // Schedule periodic CardDAV contact sync for any connected accounts. startCardavScheduler(); +// Postgres-only drainer: populate search_fts for pre-existing rows (no IMAP). +scheduleFtsBackfill(); + // Re-connect all enabled IMAP accounts on startup with bounded concurrency so a // large user base doesn't hammer IMAP servers and the DB connection pool at once. try { diff --git a/backend/src/routes/accounts.js b/backend/src/routes/accounts.js index 3ef08d76..d947ead3 100644 --- a/backend/src/routes/accounts.js +++ b/backend/src/routes/accounts.js @@ -9,6 +9,9 @@ import { getConnectionPolicy } from '../services/connectionPolicy.js'; import { invalidateGtdConfigCache, sanitizeGtdFoldersDetailed, findGtdFolderCollisions, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js'; import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; import { createKeyedSerializer } from '../utils/keyedSerializer.js'; +import { startAccountBodyBackfill } from '../services/bodyBackfill.js'; +import { providerProfile } from '../services/imapManager.js'; +import { upsertJob } from '../services/backgroundJobs.js'; // Serialize an account's reconnect triggers so a rapid settings change (e.g. a // gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains — @@ -421,7 +424,18 @@ router.post('/:id/reindex', async (req, res) => { console.error(`Manual reindex error for ${account.email_address}:`, err.message) ); } - res.json({ ok: true, alreadyRunning }); + + // Also kick a body-materialization pass for allowlisted providers. Fire-and-forget: the + // drainer self-guards against concurrent runs and caps each session, and Gmail/PurelyMail/ + // Microsoft are gated off inside startBodyBackfill. Progress lands in background_jobs. + const bodyBackfillEnabled = providerProfile(account).bodyBackfill; + if (bodyBackfillEnabled) { + startAccountBodyBackfill(account, imapManager, upsertJob).catch(err => + console.error(`Body backfill error for ${account.email_address}:`, err.message) + ); + } + + res.json({ ok: true, alreadyRunning, bodyBackfillEnabled }); } catch (err) { console.error('POST /accounts/:id/reindex error:', err.message); res.status(500).json({ error: 'Failed to start reindex' }); diff --git a/backend/src/routes/indexing.js b/backend/src/routes/indexing.js new file mode 100644 index 00000000..d8966d2d --- /dev/null +++ b/backend/src/routes/indexing.js @@ -0,0 +1,13 @@ +import { Router } from 'express'; +import { requireAdmin } from '../middleware/auth.js'; +import { listJobs } from '../services/backgroundJobs.js'; + +const router = Router(); + +// GET /api/admin/indexing/status — admin-gated (same guard as routes/ai.js). +// Surfaces every background drainer's progress, e.g. "FTS backfill: N/M". +router.get('/status', requireAdmin, async (_req, res) => { + res.json({ jobs: await listJobs() }); +}); + +export default router; diff --git a/backend/src/routes/search.js b/backend/src/routes/search.js index dbd4379a..cb3c8a6c 100644 --- a/backend/src/routes/search.js +++ b/backend/src/routes/search.js @@ -1,6 +1,8 @@ import { Router } from 'express'; import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; const router = Router(); router.use(requireAuth); @@ -30,230 +32,24 @@ function searchLimiter(req, res, next) { next(); } -// Parses a raw search string into structured operator filters and free-text -// terms. Supports from: to: subject: has: is: after: before: in:, quoted values -// (from:"John Smith"), and a leading '-' that negates either an operator -// (-from:smith) or a bare word (-invoice). Filters are a list (not a map) so -// repeated/negated operators like `from:a -from:b` are all preserved. -export function parseSearchQuery(raw) { - const filters = []; - const terms = []; - - // The leading (-?) captures optional negation. \b sits between an optional '-' - // and the operator name, so both `from:` and `-from:` match. - const opPattern = /(-?)\b(from|to|subject|has|is|after|before|in):("([^"]*)"|([\S]+))/gi; - const remaining = raw.replace(opPattern, (_, neg, key, _v, quoted, unquoted) => { - const k = key.toLowerCase(); - const v = (quoted !== undefined ? quoted : (unquoted || '')).toLowerCase().trim(); - if (v) filters.push({ key: k, value: v, negate: neg === '-' }); - return ' '; - }).trim(); - - for (const word of remaining.split(/\s+/)) { - let w = word.trim(); - if (!w || w === '-') continue; // skip blanks and a lone '-' (nothing to negate) - let negate = false; - if (w[0] === '-' && w.length > 1) { negate = true; w = w.slice(1); } - terms.push({ value: w, negate }); - } - - return { filters, terms }; -} - -// Wraps a positive condition so that when negated it also matches rows where the -// underlying columns are NULL (COALESCE(..., false) treats NULL as "not a match", -// which NOT then flips to a match — the intuitive meaning of exclusion). -function negateCond(sql) { - return `NOT COALESCE((${sql}), false)`; -} - -export function resolveSearchFolderScope(filters, folderParam = '') { - let folderScope; - let folderFuzzy = false; // in: matches loosely; the folder param is exact - - for (const f of filters) { - if (f.key !== 'in') continue; - if (f.value === 'all') { folderScope = null; } - else { folderScope = f.value; folderFuzzy = true; } - } - - if (folderScope === undefined) { - folderScope = (folderParam || '').trim() || null; - folderFuzzy = false; - } - - return { folderScope, folderFuzzy }; -} - -export function shouldExcludeTrashFromSearch(folderScope) { - return folderScope === null; -} - -export function trashFolderExclusionCondition() { - return `NOT EXISTS ( - SELECT 1 - FROM folders f - WHERE f.account_id = m.account_id - AND f.path = m.folder - AND (f.special_use = '\\Trash' - OR lower(f.name) LIKE '%trash%' - OR lower(f.name) LIKE '%deleted%') - )`; -} - -// Postgres refuses to build a tsvector larger than ~1MB of packed lexemes -// (SQLSTATE 54000), so an oversized body would 500 the whole search. Cap the -// text fed to to_tsvector at 600k chars — matching msgvault's maxFTSBodyChars -// (internal/store/dialect_pg.go) — so one huge email can't crash the query. -// Exported because slice 02's search_fts trigger caps the same way. -export const FTS_BODY_CHAR_CAP = 600000; - -// Builds the per-term free-text OR-condition: a term matches if it appears in -// the sender, the subject, the stored search_vector, or the length-capped body. -// Extracted so the body cap is a single, testable source of truth. -export function freeTextTermCondition(likeIdx, ftsIdx) { - return `( - m.from_name ILIKE $${likeIdx} - OR m.from_email ILIKE $${likeIdx} - OR m.subject ILIKE $${likeIdx} - OR m.search_vector @@ plainto_tsquery('english', $${ftsIdx}) - OR to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})) @@ plainto_tsquery('english', $${ftsIdx}) - )`; -} - router.get('/', searchLimiter, async (req, res) => { const { q, accountId, limit = 50, offset = 0 } = req.query; const trimmed = (q || '').trim(); if (!trimmed) return res.json({ messages: [] }); if (trimmed.length > 500) return res.status(400).json({ error: 'Search query too long' }); - const accountsResult = await query( - 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', - [req.session.userId] - ); - const userAccountIds = accountsResult.rows.map(r => r.id); - if (!userAccountIds.length) return res.json({ messages: [] }); - - const targetIds = accountId && userAccountIds.includes(accountId) - ? [accountId] : userAccountIds; - - const cap = Math.max(1, Math.min(parseInt(limit) || 50, 200)); - const { filters, terms } = parseSearchQuery(trimmed); - - const conditions = []; - const params = [targetIds]; - let p = 2; - - // Folder scope. `in:` in the query wins; otherwise the client-supplied `folder` - // param (the folder the user is currently viewing) applies. `undefined` means - // no in: operator was given, so we fall back to the param below. - // folderScope === null → search all folders - // folderScope === string → restrict to that folder - - // ── Operator filters ────────────────────────────────────────────────────── - - for (const f of filters) { - // in: controls scope rather than adding a row condition; negation is - // meaningless here so it's ignored. - if (f.key === 'in') { - continue; - } - - let cond = null; - - if (f.key === 'from') { - params.push(`%${f.value}%`); - cond = `(m.from_email ILIKE $${p} OR m.from_name ILIKE $${p})`; - p++; - } else if (f.key === 'subject') { - params.push(`%${f.value}%`); - cond = `m.subject ILIKE $${p++}`; - } else if (f.key === 'to') { - // to: searches the to/cc address JSON — cast to text covers name and email - params.push(`%${f.value}%`); - cond = `(m.to_addresses::text ILIKE $${p} OR m.cc_addresses::text ILIKE $${p})`; - p++; - } else if (f.key === 'has') { - if (f.value === 'attachment' || f.value === 'attachments') cond = `m.has_attachments = true`; - } else if (f.key === 'is') { - if (f.value === 'unread') cond = `m.is_read = false`; - else if (f.value === 'read') cond = `m.is_read = true`; - else if (f.value === 'starred') cond = `m.is_starred = true`; - } else if (f.key === 'after') { - const d = new Date(f.value); - if (!isNaN(d)) { params.push(d.toISOString()); cond = `m.date >= $${p++}`; } - } else if (f.key === 'before') { - const d = new Date(f.value); - if (!isNaN(d)) { params.push(d.toISOString()); cond = `m.date < $${p++}`; } - } - - if (cond) conditions.push(f.negate ? negateCond(cond) : cond); - } - - // ── Free-text terms ─────────────────────────────────────────────────────── - // Each term must match at least one of: from, subject (ILIKE — good for names - // and partial words), or body content (FTS — good for large text with stemming). - // AND between all terms: every word must appear somewhere in the email. - // A negated term (-word) must appear nowhere. - - for (const term of terms.slice(0, 10)) { - if (term.value.length < 2) continue; // single-char terms are too broad and expensive - params.push(`%${term.value}%`); // ILIKE pattern - const likeIdx = p++; - - params.push(term.value); // raw term for plainto_tsquery - const ftsIdx = p++; - - const cond = freeTextTermCondition(likeIdx, ftsIdx); - conditions.push(term.negate ? negateCond(cond) : cond); - } - - // Require at least one real search condition before applying folder scope, so a - // bare `in:inbox` (or a lone folder param) never dumps an entire folder. - if (!conditions.length) return res.json({ messages: [], query: q }); - - // Resolve folder scope: in: operator wins; otherwise use the param. - const { folderScope, folderFuzzy } = resolveSearchFolderScope(filters, req.query.folder || ''); - if (folderScope) { - if (folderFuzzy) { - // in: — case-insensitive match on a folder named exactly that, or a - // nested folder whose path ends in it (in:sent → "Sent" or "Personal/Sent"). - // A multi-word leaf like "[Gmail]/Sent Mail" needs the quoted form in:"sent mail". - params.push(folderScope); - params.push(`%/${folderScope}`); - conditions.push(`(m.folder ILIKE $${p} OR m.folder ILIKE $${p + 1})`); - p += 2; - } else { - params.push(folderScope); - conditions.push(`m.folder = $${p++}`); - } - } else if (shouldExcludeTrashFromSearch(folderScope)) { - // Deleting moves mail into Trash, where it remains searchable by explicit - // folder queries like in:trash. Keep ordinary all-folder searches from - // resurfacing freshly-deleted messages after the optimistic UI guard expires. - conditions.push(trashFolderExclusionCondition()); - } - - const off = Math.max(0, parseInt(offset) || 0); - params.push(cap); - params.push(off); - + const parsed = parseQuery(trimmed); try { - const result = await query(` - SELECT - m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, - m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, - a.name as account_name, a.email_address as account_email, a.color as account_color - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.account_id = ANY($1) - AND m.is_deleted = false - AND ${conditions.join('\n AND ')} - ORDER BY m.date DESC - LIMIT $${p} OFFSET $${p + 1} - `, params); - - res.json({ messages: result.rows, query: q }); + const result = await search({ + userId: req.session.userId, + accountId, + parsed, + folderParam: req.query.folder || '', + limit, + offset, + }); + // Response is a strict superset of the historical { messages, query }. + res.json({ ...result, query: q, unsupported: parsed.unsupported, errors: parsed.errors }); } catch (err) { console.error('Search error:', err); res.status(500).json({ error: 'Search failed' }); diff --git a/backend/src/routes/search.test.js b/backend/src/routes/search.test.js index 3ef54c76..9b305e10 100644 --- a/backend/src/routes/search.test.js +++ b/backend/src/routes/search.test.js @@ -1,157 +1,61 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; -// search.js opens a DB handle and registers auth middleware at import time; -// neither is exercised by the pure parser under test, so stub them out. +// The route must not hold SQL: it parses, calls the service, and JSON-frames. +// Mock inline + import the mocked binding (avoids the vi.mock hoisting TDZ trap). +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ requireAuth: (req, _res, next) => { req.session = { userId: 'u1' }; next(); } })); vi.mock('../services/db.js', () => ({ query: vi.fn() })); -vi.mock('../middleware/auth.js', () => ({ requireAuth: vi.fn() })); -import { - parseSearchQuery, - resolveSearchFolderScope, - shouldExcludeTrashFromSearch, - trashFolderExclusionCondition, - freeTextTermCondition, - FTS_BODY_CHAR_CAP, -} from './search.js'; - -describe('parseSearchQuery', () => { - it('treats bare words as free-text terms', () => { - const { filters, terms } = parseSearchQuery('hello world'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'hello', negate: false }, - { value: 'world', negate: false }, - ]); - }); - - it('extracts positive operators and lowercases their values', () => { - const { filters, terms } = parseSearchQuery('from:Amazon subject:Invoice hello'); - expect(filters).toEqual([ - { key: 'from', value: 'amazon', negate: false }, - { key: 'subject', value: 'invoice', negate: false }, - ]); - expect(terms).toEqual([{ value: 'hello', negate: false }]); - }); - - it('supports quoted operator values with spaces', () => { - const { filters, terms } = parseSearchQuery('from:"John Smith" report'); - expect(filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); - expect(terms).toEqual([{ value: 'report', negate: false }]); - }); - - it('negates an operator when prefixed with -', () => { - const { filters, terms } = parseSearchQuery('-from:Smith report'); - expect(filters).toEqual([{ key: 'from', value: 'smith', negate: true }]); - expect(terms).toEqual([{ value: 'report', negate: false }]); - }); - - it('negates a free-text term when prefixed with -', () => { - const { filters, terms } = parseSearchQuery('report -invoice'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'report', negate: false }, - { value: 'invoice', negate: true }, - ]); - }); - - it('parses the in: scope operator (in:all and named folders)', () => { - expect(parseSearchQuery('in:all invoice').filters).toEqual([ - { key: 'in', value: 'all', negate: false }, - ]); - expect(parseSearchQuery('in:Sent proposal').filters).toEqual([ - { key: 'in', value: 'sent', negate: false }, - ]); - }); - - it('preserves repeated and mixed positive/negative operators', () => { - const { filters } = parseSearchQuery('from:alice -from:bob is:unread'); - expect(filters).toEqual([ - { key: 'from', value: 'alice', negate: false }, - { key: 'from', value: 'bob', negate: true }, - { key: 'is', value: 'unread', negate: false }, - ]); - }); - - it('ignores a lone - so it is not treated as a negated empty term', () => { - const { filters, terms } = parseSearchQuery('report - draft'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'report', negate: false }, - { value: 'draft', negate: false }, - ]); - }); - - it('returns empty structures for blank input', () => { - expect(parseSearchQuery('')).toEqual({ filters: [], terms: [] }); - expect(parseSearchQuery(' ')).toEqual({ filters: [], terms: [] }); - }); - - it('does not treat unknown prefixes as operators', () => { - const { filters, terms } = parseSearchQuery('label:work'); - expect(filters).toEqual([]); - expect(terms).toEqual([{ value: 'label:work', negate: false }]); - }); - - it('handles all supported operator keys', () => { - const raw = 'from:a to:b subject:c has:attachment is:starred after:2024-01-01 before:2024-12-31 in:archive'; - const keys = parseSearchQuery(raw).filters.map(f => f.key); - expect(keys).toEqual(['from', 'to', 'subject', 'has', 'is', 'after', 'before', 'in']); - }); - - it('scopes search to the client folder param when no in: operator is present', () => { - const { filters } = parseSearchQuery('subject:newsletter'); - expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ - folderScope: 'INBOX', - folderFuzzy: false, +import { search } from '../services/search/searchService.js'; +import searchRouter from './search.js'; + +function makeApp() { + const app = express(); + app.use((req, _res, next) => { req.session = { userId: 'u1' }; next(); }); + app.use('/api/search', searchRouter); + return app; +} + +async function get(app, path) { + const { default: request } = await import('node:http'); + return new Promise((resolve) => { + const server = app.listen(0, () => { + const port = server.address().port; + request.get(`http://127.0.0.1:${port}${path}`, (res) => { + let body = ''; + res.on('data', c => (body += c)); + res.on('end', () => { server.close(); resolve({ status: res.statusCode, json: JSON.parse(body || '{}') }); }); + }); }); }); +} - it('lets in: override the client folder param', () => { - const { filters } = parseSearchQuery('in:trash subject:newsletter'); - expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ - folderScope: 'trash', - folderFuzzy: true, - }); - }); - - it('excludes trash-like folders from ordinary all-folder searches', () => { - const { filters } = parseSearchQuery('subject:newsletter'); - const { folderScope } = resolveSearchFolderScope(filters); - expect(folderScope).toBeNull(); - expect(shouldExcludeTrashFromSearch(folderScope)).toBe(true); - expect(trashFolderExclusionCondition()).toContain('NOT EXISTS'); - expect(trashFolderExclusionCondition()).toContain('%trash%'); - expect(trashFolderExclusionCondition()).toContain('%deleted%'); - }); - - it('keeps explicit folder searches eligible to find trash messages', () => { - const { filters } = parseSearchQuery('in:trash subject:newsletter'); - const { folderScope } = resolveSearchFolderScope(filters); - expect(shouldExcludeTrashFromSearch(folderScope)).toBe(false); - }); -}); +beforeEach(() => search.mockReset()); -describe('freeTextTermCondition (oversized-body crash hotfix)', () => { - it('caps the body tsvector so a multi-megabyte email cannot exceed the 1MB tsvector limit', () => { - const cond = freeTextTermCondition(3, 4); - // The body text is fed to to_tsvector through LEFT(..., FTS_BODY_CHAR_CAP). - // Without this cap a single oversized body raises SQLSTATE 54000 and 500s search. - expect(cond).toContain( - `to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))` - ); - // Guard against a regression back to the uncapped expression. - expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); +describe('GET /api/search', () => { + it('short-circuits a blank query with { messages: [] } and never calls the service', async () => { + const res = await get(makeApp(), '/api/search?q='); + expect(res.status).toBe(200); + expect(res.json).toEqual({ messages: [] }); + expect(search).not.toHaveBeenCalled(); }); - it('pins the cap at 600000 chars (msgvault maxFTSBodyChars parity)', () => { - expect(FTS_BODY_CHAR_CAP).toBe(600000); + it('rejects an over-500-char query with 400', async () => { + const res = await get(makeApp(), `/api/search?q=${'a'.repeat(501)}`); + expect(res.status).toBe(400); }); - it('still matches sender, subject, and the stored search_vector at the given param positions', () => { - const cond = freeTextTermCondition(3, 4); - expect(cond).toContain('m.from_name ILIKE $3'); - expect(cond).toContain('m.from_email ILIKE $3'); - expect(cond).toContain('m.subject ILIKE $3'); - expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + it('returns a superset response: messages + query + mode + page + unsupported + errors', async () => { + search.mockResolvedValue({ messages: [{ id: 'm1' }], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + const res = await get(makeApp(), '/api/search?q=larger:5M%20invoice'); + expect(res.status).toBe(200); + expect(res.json.messages).toEqual([{ id: 'm1' }]); + expect(res.json.query).toBe('larger:5M invoice'); + expect(res.json.mode).toBe('lexical'); + expect(res.json.page).toEqual({ offset: 0, limit: 50, hasMore: false }); + // larger: is recognized but unserviceable — surfaced, not silently dropped. + expect(res.json.unsupported).toEqual([{ key: 'larger', token: 'larger:5m' }]); + expect(res.json.errors).toEqual([]); }); }); diff --git a/backend/src/services/backgroundJobs.js b/backend/src/services/backgroundJobs.js new file mode 100644 index 00000000..799b05e0 --- /dev/null +++ b/backend/src/services/backgroundJobs.js @@ -0,0 +1,26 @@ +import { query } from './db.js'; + +// Upsert a drainer's progress row. One row per (kind, account); global jobs +// pass accountId = null. started_at is set once and preserved across updates. +export async function upsertJob({ kind, accountId = null, state, processed = 0, total = 0, lastError = null }) { + await query(` + INSERT INTO background_jobs (kind, account_id, state, processed, total, last_error, started_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW()) + ON CONFLICT (kind, COALESCE(account_id::text, '')) DO UPDATE SET + state = EXCLUDED.state, + processed = EXCLUDED.processed, + total = EXCLUDED.total, + last_error = EXCLUDED.last_error, + started_at = COALESCE(background_jobs.started_at, EXCLUDED.started_at), + updated_at = NOW() + `, [kind, accountId, state, processed, total, lastError]); +} + +export async function listJobs() { + const { rows } = await query( + `SELECT kind, account_id, state, processed, total, last_error, started_at, updated_at + FROM background_jobs + ORDER BY updated_at DESC` + ); + return rows; +} diff --git a/backend/src/services/backgroundJobs.test.js b/backend/src/services/backgroundJobs.test.js new file mode 100644 index 00000000..f3ecfd24 --- /dev/null +++ b/backend/src/services/backgroundJobs.test.js @@ -0,0 +1,25 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('./db.js', () => ({ query: vi.fn() })); +import { query } from './db.js'; +import { upsertJob, listJobs } from './backgroundJobs.js'; + +beforeEach(() => query.mockReset()); + +describe('backgroundJobs', () => { + it('upserts one row per (kind, account) via the COALESCE unique index', async () => { + query.mockResolvedValue({ rows: [] }); + await upsertJob({ kind: 'fts_backfill', state: 'running', processed: 10, total: 100 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('INSERT INTO background_jobs'); + expect(sql).toContain("ON CONFLICT (kind, COALESCE(account_id::text, ''))"); + expect(sql).toContain('started_at = COALESCE(background_jobs.started_at, EXCLUDED.started_at)'); + expect(params).toEqual(['fts_backfill', null, 'running', 10, 100, null]); + }); + + it('lists jobs newest-first', async () => { + query.mockResolvedValue({ rows: [{ kind: 'fts_backfill' }] }); + const jobs = await listJobs(); + expect(jobs).toEqual([{ kind: 'fts_backfill' }]); + expect(query.mock.calls[0][0]).toContain('ORDER BY updated_at DESC'); + }); +}); diff --git a/backend/src/services/bodyBackfill.js b/backend/src/services/bodyBackfill.js new file mode 100644 index 00000000..ccb855b7 --- /dev/null +++ b/backend/src/services/bodyBackfill.js @@ -0,0 +1,181 @@ +import { query } from './db.js'; +import { providerProfile } from './imapManager.js'; + +// Body-materialization drainer. Fills messages.body_text/body_html over IMAP so weight-D +// lexical search has material to work with, WITHOUT growing the imapManager +// singleton (README invariant) and WITHOUT touching throttle-hostile providers. ALL policy — +// the scan predicate, batching, pacing, provider gating, session cap, quiet-window +// backpressure, and the per-account circuit breaker — lives here. The singleton exposes only +// the narrow fetchBodiesForMessages() entry point this module calls. +// +// Shape is copied from imapManager.startSnippetIndexer, but this module is standalone: its +// per-account run guard and circuit-breaker state are module-level (not instance fields), and +// its IMAP/quiet-window/progress/clock dependencies are injected so it unit-tests against a +// fake fetchBodies with no real IMAP. + +const BATCH_SIZE = 50; // messages fetched per batch +const MIN_BATCH_DELAY_MS = 2000; // floor between batches (provider batchDelay may raise it) +const MAX_BATCHES_PER_RUN = 200; // session cap: 10,000 messages per run, then resume next kick +const QUIET_WINDOW_MS = 8000; // pause extra when the user opened a message this recently +const MAX_CONSECUTIVE_ERRORS = 3; // abort the run after this many failing batches in a row +const BACKOFF_BASE_MS = 10 * 60 * 1000; // first circuit-breaker back-off +const BACKOFF_MAX_MS = 2 * 60 * 60 * 1000; // cap + +// Per-account run guard and circuit breaker. Module-level so a single process runs at most one +// body drainer per account (the snippet indexer uses instance fields for the same purpose). +const running = new Set(); // accountId +const backoff = new Map(); // accountId -> { failures, until } + +// Test seam: reset module state between unit tests. +export function resetBodyBackfillState() { + running.clear(); + backoff.clear(); +} + +const realSleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Core drainer. Dependencies are injected so tests supply a fake fetchBodies: +// fetchBodies(accountId, ids) -> Promise<{ fetched: number }> (writes bodies; throws on a +// connection-level failure so this loop can back off) +// getLastActivityMs(accountId) -> number (ms timestamp of the user's last live body open) +// isSnippetIndexerRunning(accountId) -> boolean (the snippet indexer does its own sustained +// BODY[]-ish fetches over an independent connection; running +// both loops for the same account at once double-books IMAP +// connections against the same provider limit) +// upsertJobProgress({ accountId, kind, processed, total, state }) -> Promise (this is +// backgroundJobs.upsertJob's own shape — callers inject it +// directly; the drainer speaks its vocabulary, no adapter) +// sleep(ms) -> Promise (injectable for fast, deterministic tests) +// now() -> number (injectable clock) +export async function startBodyBackfill(account, deps) { + const { + fetchBodies, + getLastActivityMs = () => 0, + isSnippetIndexerRunning = () => false, + upsertJobProgress = async () => {}, + sleep = realSleep, + now = Date.now, + } = deps; + + const cfg = providerProfile(account); + if (!cfg.bodyBackfill) return; // provider gate (Gmail/PurelyMail/etc.) + if (running.has(account.id)) return; // one drainer per account + const bo = backoff.get(account.id); + if (bo && now() < bo.until) return; // circuit breaker open + running.add(account.id); + + const batchDelay = Math.max(cfg.batchDelay || 0, MIN_BATCH_DELAY_MS); + const kind = 'body_backfill'; + let batchCount = 0; + let failed = false; + + try { + // Denominator + starting coverage for progress. total is the whole eligible mailbox so the + // progress bar reflects real coverage, not just this run. + const countRes = await query( + `SELECT count(*)::int AS total, + count(*) FILTER (WHERE body_text IS NOT NULL OR body_html IS NOT NULL)::int AS have_body + FROM messages WHERE account_id = $1 AND is_deleted = false`, + [account.id] + ); + const total = countRes.rows[0].total; + let haveBody = countRes.rows[0].have_body; + if (total - haveBody <= 0) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'done' }); + return; + } + + // Keyset by id ascending: id is a non-null, unique UUID, so the cursor advances past every + // row exactly once per run — including rows whose body cannot be fetched (they stay NULL but + // are not re-selected this run, so the run always terminates). Recency ordering is + // intentionally not used: body coverage is a background quality lever. + let cursorId = null; + let consecutiveErrors = 0; + + while (true) { + // Stop if the account was deleted mid-run. + const alive = await query('SELECT id FROM email_accounts WHERE id = $1', [account.id]); + if (!alive.rows.length) return; + + // Defer to the snippet indexer rather than double-book BODY[] IMAP connections for the + // same account — checked every iteration (not just at kick time) so a session already in + // progress backs off cleanly the moment the indexer starts, instead of racing it. + if (isSnippetIndexerRunning(account.id)) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'deferred' }); + return; + } + + if (batchCount >= MAX_BATCHES_PER_RUN) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'paused' }); + return; + } + + const batchRes = await query( + `SELECT id FROM messages + WHERE account_id = $1 AND body_text IS NULL AND body_html IS NULL AND is_deleted = false + AND ($2::uuid IS NULL OR id > $2) + ORDER BY id ASC + LIMIT $3`, + [account.id, cursorId, BATCH_SIZE] + ); + if (!batchRes.rows.length) break; // drained + const ids = batchRes.rows.map((r) => r.id); + + try { + const { fetched } = await fetchBodies(account.id, ids); + haveBody += fetched; + cursorId = ids[ids.length - 1]; // advance only on success + batchCount++; + consecutiveErrors = 0; + } catch { + consecutiveErrors++; + await sleep(cfg.errorDelay || batchDelay); + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + failed = true; + // Write a terminal 'error' state (ftsBackfill.js convention) so background_jobs + // doesn't keep showing a stale 'running' row for an account that's actually stalled. + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'error' }); + return; // finally trips the circuit breaker + } + continue; // retry the same batch (cursor not advanced); fetchBodies reopens IMAP + } + + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'running' }); + + // Quiet-window backpressure: pause longer when the user is actively opening messages so + // background BODY[] traffic doesn't compete with click-time fetches. + const quietFor = now() - getLastActivityMs(account.id); + const extraDelay = quietFor < QUIET_WINDOW_MS ? QUIET_WINDOW_MS - quietFor : 0; + await sleep(batchDelay + extraDelay); + } + + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'done' }); + } catch { + failed = true; + } finally { + running.delete(account.id); + // Circuit breaker: a run that failed without draining a single batch (e.g. the provider + // refuses the extra connection) backs off exponentially so we stop reopening competing IMAP + // connections. Any progress — or a clean finish — clears the backoff. + if (failed && batchCount === 0) { + const failures = (backoff.get(account.id)?.failures || 0) + 1; + const delay = Math.min(BACKOFF_BASE_MS * 2 ** (failures - 1), BACKOFF_MAX_MS); + backoff.set(account.id, { failures, until: now() + delay }); + } else { + backoff.delete(account.id); + } + } +} + +// Convenience wrapper for the reindex route: builds the injected deps from the imapManager +// singleton and the injected progress sink (backgroundJobs.upsertJob — the drainer already +// emits its shape), then runs the core drainer. Kept here (not in the route) so the wiring is +// unit-testable and bodyBackfill.js never imports backgroundJobs.js directly. +export function startAccountBodyBackfill(account, imapManager, upsertJobProgress) { + return startBodyBackfill(account, { + fetchBodies: (accountId, ids) => imapManager.fetchBodiesForMessages(accountId, ids), + getLastActivityMs: (accountId) => imapManager.lastUserActivity.get(accountId) || 0, + isSnippetIndexerRunning: (accountId) => imapManager.snippetIndexerRunning.has(accountId), + upsertJobProgress, + }); +} diff --git a/backend/src/services/bodyBackfill.test.js b/backend/src/services/bodyBackfill.test.js new file mode 100644 index 00000000..3b3da49f --- /dev/null +++ b/backend/src/services/bodyBackfill.test.js @@ -0,0 +1,290 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('./db.js', () => ({ query: vi.fn() })); +vi.mock('./imapManager.js', () => ({ providerProfile: vi.fn() })); + +import { query } from './db.js'; +import { providerProfile } from './imapManager.js'; +import { startBodyBackfill, startAccountBodyBackfill, resetBodyBackfillState } from './bodyBackfill.js'; + +const account = { id: 'acct-1', imap_host: 'imap.icloud.com' }; +const enabledProfile = { bodyBackfill: true, batchDelay: 2000, errorDelay: 100 }; + +// Instant, assertable deps so the drainer runs with no wall-clock waits. +function baseDeps(overrides = {}) { + return { + fetchBodies: vi.fn().mockResolvedValue({ fetched: 0 }), + getLastActivityMs: () => 0, + isSnippetIndexerRunning: () => false, + upsertJobProgress: vi.fn().mockResolvedValue(), + sleep: vi.fn().mockResolvedValue(), + now: () => 1_000_000, + ...overrides, + }; +} + +// Route the mocked query by SQL shape. `batches` is an array of id-arrays returned in order; +// once exhausted it returns an empty batch (drained). +function mockQuery({ total = 0, haveBody = 0, batches = [], alive = true } = {}) { + let i = 0; + query.mockImplementation((sql) => { + if (/AS total/.test(sql)) return Promise.resolve({ rows: [{ total, have_body: haveBody }] }); + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: alive ? [{ id: 'acct-1' }] : [] }); + if (/body_text IS NULL/.test(sql)) { + const rows = (batches[i++] || []).map((id) => ({ id })); + return Promise.resolve({ rows }); + } + return Promise.resolve({ rows: [] }); + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + resetBodyBackfillState(); +}); + +describe('startBodyBackfill — provider gate', () => { + it('no-ops for a provider with bodyBackfill:false and never queries', async () => { + providerProfile.mockReturnValue({ bodyBackfill: false }); + const deps = baseDeps(); + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('startBodyBackfill — nothing to do', () => { + it('reports complete and fetches nothing when coverage is already full', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 100 }); + const deps = baseDeps(); + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(deps.upsertJobProgress).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'done', processed: 100, total: 100 }) + ); + }); +}); + +describe('startBodyBackfill — batching + keyset + progress', () => { + it('drains 120 messages in batches of 50/50/20 and advances the cursor', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = Array.from({ length: 50 }, (_, k) => `a${k}`); + const b2 = Array.from({ length: 50 }, (_, k) => `b${k}`); + const b3 = Array.from({ length: 20 }, (_, k) => `c${k}`); + mockQuery({ total: 120, haveBody: 0, batches: [b1, b2, b3] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockImplementation((_id, ids) => Promise.resolve({ fetched: ids.length })) }); + + await startBodyBackfill(account, deps); + + expect(deps.fetchBodies).toHaveBeenCalledTimes(3); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(1, 'acct-1', b1); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(2, 'acct-1', b2); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(3, 'acct-1', b3); + // Cursor advances to the last id of the previous batch (keyset param $2). + const scanCalls = query.mock.calls.filter(([sql]) => /body_text IS NULL/.test(sql)); + expect(scanCalls[0][1][1]).toBeNull(); // first scan: cursor null + expect(scanCalls[1][1][1]).toBe('a49'); // second scan: last id of batch 1 + expect(scanCalls[2][1][1]).toBe('b49'); // third scan: last id of batch 2 + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 120, total: 120 }) + ); + }); +}); + +describe('startBodyBackfill — session cap + resume', () => { + it('pauses after MAX_BATCHES_PER_RUN batches, then resumes on the next call', async () => { + providerProfile.mockReturnValue(enabledProfile); + // Never drains: every scan returns 50 fresh ids so the run hits the batch cap. + let n = 0; + query.mockImplementation((sql) => { + if (/AS total/.test(sql)) return Promise.resolve({ rows: [{ total: 100000, have_body: 0 }] }); + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/body_text IS NULL/.test(sql)) return Promise.resolve({ rows: Array.from({ length: 50 }, () => ({ id: `id-${n++}` })) }); + return Promise.resolve({ rows: [] }); + }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 50 }) }); + + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).toHaveBeenCalledTimes(200); // MAX_BATCHES_PER_RUN + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'paused' }) + ); + + // Resume: the run guard was released in finally, so a second call proceeds (not gated). + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).toHaveBeenCalledTimes(400); + }); +}); + +describe('startBodyBackfill — single drainer per account', () => { + it('a second concurrent call for the same account is a no-op', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 50, haveBody: 0, batches: [['x1']] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }) }); + + // Both are invoked synchronously in the same tick: the first adds the account to the + // run guard before its first await, so the second sees it and returns immediately. + const p1 = startBodyBackfill(account, deps); + const p2 = startBodyBackfill(account, deps); + await Promise.all([p1, p2]); + + expect(deps.fetchBodies).toHaveBeenCalledTimes(1); // only p1 did work + }); +}); + +describe('startBodyBackfill — circuit breaker trip + recovery', () => { + it('trips after 3 consecutive batch errors, blocks retries in the window, recovers after it', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 0, batches: [['x1'], ['x1'], ['x1']] }); + let clock = 1_000_000; + const failing = baseDeps({ + fetchBodies: vi.fn().mockRejectedValue(new Error('Command failed')), + now: () => clock, + }); + + // Trip: 3 consecutive rejections abort the run and open the breaker. + await startBodyBackfill(account, failing); + expect(failing.fetchBodies).toHaveBeenCalledTimes(3); + // A terminal 'error' progress row is written so background_jobs doesn't keep showing a + // stale 'running' state for an account that's actually stalled. + expect(failing.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'error' }) + ); + + // Blocked: a call inside the back-off window no-ops (breaker open). + const blocked = baseDeps({ fetchBodies: vi.fn(), now: () => clock }); + await startBodyBackfill(account, blocked); + expect(blocked.fetchBodies).not.toHaveBeenCalled(); + + // Recover: advance past the back-off window; a fresh call runs again. + clock += 3 * 60 * 60 * 1000; // beyond BACKOFF_MAX_MS (2h) + mockQuery({ total: 100, haveBody: 0, batches: [['y1']] }); + const recovered = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }), now: () => clock }); + await startBodyBackfill(account, recovered); + expect(recovered.fetchBodies).toHaveBeenCalledTimes(1); + }); +}); + +describe('startBodyBackfill — quiet-window backpressure', () => { + it('adds the remaining quiet window to the post-batch delay when the user is active', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const NOW = 5_000_000; + const deps = baseDeps({ + fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }), + now: () => NOW, + getLastActivityMs: () => NOW - 2000, // user opened a message 2s ago + }); + + await startBodyBackfill(account, deps); + + // batchDelay = max(2000, 2000) = 2000; quiet window 8000 - 2000 elapsed = 6000 extra. + expect(deps.sleep).toHaveBeenCalledWith(8000); + }); +}); + +describe('startBodyBackfill — defers to the snippet indexer', () => { + it('gates the kick: never fetches when the indexer is already running', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 0, batches: [['x1']] }); + const deps = baseDeps({ isSnippetIndexerRunning: () => true }); + + await startBodyBackfill(account, deps); + + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(deps.upsertJobProgress).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'deferred' }) + ); + }); + + it('re-checks every iteration: defers mid-session once the indexer starts, without tripping the breaker', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = ['a1']; + const b2 = ['b1']; + mockQuery({ total: 100, haveBody: 0, batches: [b1, b2] }); + let indexerRunning = false; + const deps = baseDeps({ + fetchBodies: vi.fn().mockImplementation(() => { + indexerRunning = true; // simulate the snippet indexer kicking in right after batch 1 + return Promise.resolve({ fetched: 1 }); + }), + isSnippetIndexerRunning: () => indexerRunning, + }); + + await startBodyBackfill(account, deps); + + // Batch 1 runs (indexer not yet running); batch 2's scan never happens because the + // re-check at the top of the next iteration sees the indexer is now running and bails. + expect(deps.fetchBodies).toHaveBeenCalledTimes(1); + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'deferred' }) + ); + + // Deferring is not a failure: the circuit breaker must not be armed by it. + const resumed = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }) }); + mockQuery({ total: 100, haveBody: 1, batches: [['c1']] }); + await startBodyBackfill(account, resumed); + expect(resumed.fetchBodies).toHaveBeenCalledTimes(1); + }); +}); + +describe('startBodyBackfill — forward progress', () => { + it('drains to completion even when every batch reports {fetched:0} (resolved, not rejected)', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = Array.from({ length: 50 }, (_, k) => `a${k}`); + const b2 = Array.from({ length: 10 }, (_, k) => `b${k}`); + mockQuery({ total: 60, haveBody: 0, batches: [b1, b2] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 0 }) }); + + await startBodyBackfill(account, deps); + + // The cursor still advances on a resolved (non-throwing) call even when it wrote nothing — + // e.g. every message in the batch legitimately has no fetchable body — so the run still + // reaches completion instead of looping forever or being mistaken for a failure. + expect(deps.fetchBodies).toHaveBeenCalledTimes(2); + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 0, total: 60 }) + ); + }); +}); + +describe('startAccountBodyBackfill — adapter wiring', () => { + it('wires imapManager.fetchBodiesForMessages and lastUserActivity into the core drainer', async () => { + vi.useFakeTimers(); + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const fakeImap = { + fetchBodiesForMessages: vi.fn().mockResolvedValue({ fetched: 1 }), + lastUserActivity: new Map([['acct-1', Date.now()]]), + snippetIndexerRunning: new Set(), + }; + const upsert = vi.fn().mockResolvedValue(); + + const p = startAccountBodyBackfill(account, fakeImap, upsert); + await vi.runAllTimersAsync(); // flush the real setTimeout used by the default sleep + await p; + + expect(fakeImap.fetchBodiesForMessages).toHaveBeenCalledWith('acct-1', ['m1']); + expect(upsert).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', accountId: 'acct-1' }) + ); + vi.useRealTimers(); + }); + + it('wires imapManager.snippetIndexerRunning so an already-running indexer defers the kick', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const fakeImap = { + fetchBodiesForMessages: vi.fn().mockResolvedValue({ fetched: 1 }), + lastUserActivity: new Map(), + snippetIndexerRunning: new Set(['acct-1']), + }; + const upsert = vi.fn().mockResolvedValue(); + + await startAccountBodyBackfill(account, fakeImap, upsert); + + expect(fakeImap.fetchBodiesForMessages).not.toHaveBeenCalled(); + expect(upsert).toHaveBeenCalledWith(expect.objectContaining({ state: 'deferred' })); + }); +}); diff --git a/backend/src/services/imapManager.js b/backend/src/services/imapManager.js index ed83496b..f6219422 100644 --- a/backend/src/services/imapManager.js +++ b/backend/src/services/imapManager.js @@ -418,6 +418,10 @@ function safeDate(d) { // flagPollEveryTicks: for non-push flag providers, poll flags every N successful sync ticks. // snippetIndex: run the background snippet indexer after backfill. // Disabled for providers that throttle body fetches too aggressively. +// bodyBackfill: run the background body-materialization drainer +// (services/bodyBackfill.js) for this provider. Enabled only for +// providers that tolerate sustained background BODY[] fetches; +// Gmail/PurelyMail/Microsoft stay off in v1 (unproven / throttle-hostile). // skipFolderPatterns: folder path substrings to skip during backfill (label-view dedup). // skipFolderNames: exact folder paths to skip (non-selectable namespace containers). // batchSize/Delay/errorDelay/batchesPerConn: backfill rate-limit tuning. @@ -432,6 +436,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: false, snippetIndex: false, + bodyBackfill: false, speculativeFetch: false, skipFolderPatterns: ['all mail', '[gmail]/starred', '[gmail]/important'], // [Gmail] is a namespace container — not a selectable mailbox. It must be @@ -443,6 +448,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: false, skipFolderPatterns: [], skipFolderNames: [], @@ -453,6 +459,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -462,6 +469,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: false, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -489,6 +497,7 @@ const PROVIDERS = { usesIdle: false, pushesFlags: false, snippetIndex: false, + bodyBackfill: false, speculativeFetch: false, preferFreshBodyFetch: true, freshInboxSync: true, @@ -507,6 +516,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -3723,6 +3733,30 @@ export class ImapManager { // Called in the background (via setImmediate) so it doesn't block the sync path. // By the time the user clicks the email (typically 2–10s later), the body is already // in the DB and the click returns instantly without a live IMAP round-trip. + // Shared fetch+sanitize+snippet+UPDATE step behind every body-materialization path + // (opportunistic new-mail prefetch, on-view folder prefetch, and the body-backfill + // drainer). Kept as one helper so the UPDATE shape that fires the search_fts + // triggers never drifts between callers. Returns true if a body was actually written. + // Private (underscore-prefixed, matching this file's convention for internal helpers like + // _enqueueFlagPush) — not new public surface, so it doesn't count against the + // one-narrow-method invariant that governs fetchBodiesForMessages. + async _fetchAndStoreBody(account, msg) { + const { html, text, attachments } = await this.fetchMessageBody( + account, msg.uid, msg.folder || 'INBOX' + ); + const safeHtml = html ? sanitizeEmail(html) : null; + if (!safeHtml && !text) return false; + const snip = snippetFromBody(text, safeHtml || html); + await query( + `UPDATE messages + SET body_html = $1, body_text = $2, attachments = $3, + snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END + WHERE id = $4`, + [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] + ); + return true; + } + async prefetchNewMessageBodies(account, messages) { for (const msg of messages) { try { @@ -3733,26 +3767,58 @@ export class ImapManager { ); if (existing.rows.length) continue; - const { html, text, attachments } = await this.fetchMessageBody( - account, msg.uid, msg.folder || 'INBOX' - ); - const safeHtml = html ? sanitizeEmail(html) : null; - if (safeHtml || text) { - const snip = snippetFromBody(text, safeHtml || html); - await query( - `UPDATE messages - SET body_html = $1, body_text = $2, attachments = $3, - snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END - WHERE id = $4`, - [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] - ); - } + await this._fetchAndStoreBody(account, msg); } catch (err) { console.warn(`Body prefetch failed for uid ${msg.uid}:`, err.message); } } } + // Narrow entry point for the body-materialization drainer (services/bodyBackfill.js). + // Given a set of message IDs, fetch each body over IMAP and persist it with the same + // UPDATE the prefetch path uses. This is the ONLY method the drainer calls into the + // singleton — all batching, pacing, provider gating, and progress live in bodyBackfill.js + // (imapManager invariant, specs/search-overhaul/README.md). Returns the number of bodies + // actually written. + // + // A single poison message (deleted/renamed folder, a per-message server-side rejection) + // must not wedge the whole batch — bodyBackfill.js's keyset cursor only advances past ids + // this method actually processed, and its circuit breaker only trips on a batch that made + // zero progress. So each message's fetch+store is individually caught and skipped here; + // only when the WHOLE batch fails (zero successes and at least one error — almost always a + // dead connection, not a bad message) do we rethrow, so the drainer's breaker can still back + // a genuinely broken account off. + async fetchBodiesForMessages(accountId, messageIds) { + if (!messageIds.length) return { fetched: 0 }; + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + if (!accountResult.rows.length) return { fetched: 0 }; + const account = accountResult.rows[0]; + + // Only touch rows that still lack a body — a concurrent open/prefetch may have filled + // some since the drainer selected them. + const rowsResult = await query( + `SELECT id, uid, folder FROM messages + WHERE id = ANY($1::uuid[]) AND body_html IS NULL AND body_text IS NULL`, + [messageIds] + ); + + let fetched = 0; + let errors = 0; + let lastError = null; + for (const msg of rowsResult.rows) { + try { + if (await this._fetchAndStoreBody(account, msg)) fetched++; + } catch (err) { + errors++; + lastError = err; + console.warn(`Body backfill: skipped message ${msg.id} (uid ${msg.uid}):`, err.message); + } + } + if (fetched === 0 && errors > 0) throw lastError; + return { fetched }; + } + // Background body prefetch for messages currently visible in a folder. // Called after GET /messages responds so the user gets a fast first impression // without waiting for this work. Respects the quiet window — pauses between @@ -3786,18 +3852,7 @@ export class ImapManager { ); if (existing.rows.length) continue; - const { html, text, attachments } = await this.fetchMessageBody(account, msg.uid, msg.folder); - const safeHtml = html ? sanitizeEmail(html) : null; - if (safeHtml || text) { - const snip = snippetFromBody(text, safeHtml || html); - await query( - `UPDATE messages - SET body_html = $1, body_text = $2, attachments = $3, - snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END - WHERE id = $4`, - [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] - ); - } + await this._fetchAndStoreBody(account, msg); } catch (err) { console.warn(`Folder body prefetch failed for uid ${msg.uid}:`, err.message); } diff --git a/backend/src/services/imapManager.test.js b/backend/src/services/imapManager.test.js index 15402984..51203646 100644 --- a/backend/src/services/imapManager.test.js +++ b/backend/src/services/imapManager.test.js @@ -15,7 +15,8 @@ import { ImapManager, providerProfile, makeClientCfg, gtdRelocateGuard, insertCo import { query } from './db.js'; import { invalidateGtdConfigCache } from './gtdConfig.js'; import { runGtdTransitions, threadKeysInFolders } from './gtdTransitions.js'; -import { parseMessage } from './messageParser.js'; +import { sanitizeEmail } from './emailSanitizer.js'; +import { parseMessage, snippetFromBody } from './messageParser.js'; const account = (imap_host, oauth_provider = null) => ({ imap_host, oauth_provider }); @@ -104,6 +105,22 @@ describe('providerProfile — host detection', () => { }); }); +// ── providerProfile — bodyBackfill allowlist ───────────────────────────────── + +describe('providerProfile — bodyBackfill allowlist', () => { + it('enables body backfill for well-behaved providers (apple, yahoo, generic)', () => { + expect(providerProfile(account('imap.icloud.com')).bodyBackfill).toBe(true); + expect(providerProfile(account('imap.mail.yahoo.com')).bodyBackfill).toBe(true); + expect(providerProfile(account('imap.fastmail.com')).bodyBackfill).toBe(true); + }); + + it('excludes throttle-hostile / unproven providers (google, purelymail, microsoft)', () => { + expect(providerProfile(account('imap.gmail.com')).bodyBackfill).toBe(false); + expect(providerProfile(account('imap.purelymail.com')).bodyBackfill).toBe(false); + expect(providerProfile(account('outlook.office365.com')).bodyBackfill).toBe(false); + }); +}); + // ── providerProfile — oauth_provider detection ──────────────────────────────── describe('providerProfile — oauth_provider fallback', () => { @@ -1154,3 +1171,117 @@ describe('syncMessages — empty local cache vs nonempty server (wiring)', () => expect(query.mock.calls.some(([sql]) => sql.includes('UPDATE folders SET highest_modseq'))).toBe(false); }); }); + +describe('ImapManager.fetchBodiesForMessages', () => { + // The constructor starts four setInterval timers; clear them so tests leave no open handles. + function makeManager() { + const mgr = new ImapManager({}); + clearInterval(mgr._healthCheckTimer); + clearInterval(mgr._snippetSchedulerTimer); + clearInterval(mgr._stalenessCheckTimer); + clearInterval(mgr._flagPushReconcilerTimer); + return mgr; + } + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns {fetched:0} without touching IMAP when given no ids', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn(); + const result = await mgr.fetchBodiesForMessages('acct-1', []); + expect(result).toEqual({ fetched: 0 }); + expect(mgr.fetchMessageBody).not.toHaveBeenCalled(); + }); + + it('fetches each still-empty message and writes its body via UPDATE', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockResolvedValue({ html: '

hi

', text: 'hi', attachments: [] }); + sanitizeEmail.mockReturnValue('

hi

'); + snippetFromBody.mockReturnValue('hi'); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1', email_address: 'x@y' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }, { id: 'm2', uid: 6, folder: 'Sent' }] }); + } + return Promise.resolve({ rows: [] }); // UPDATE + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2']); + + expect(result).toEqual({ fetched: 2 }); + expect(mgr.fetchMessageBody).toHaveBeenCalledWith(expect.objectContaining({ id: 'acct-1' }), 5, 'INBOX'); + expect(mgr.fetchMessageBody).toHaveBeenCalledWith(expect.objectContaining({ id: 'acct-1' }), 6, 'Sent'); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(2); + }); + + it('skips a message whose body comes back empty (no html and no text)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockResolvedValue({ html: null, text: null, attachments: [] }); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }] }); + return Promise.resolve({ rows: [] }); + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1']); + + expect(result).toEqual({ fetched: 0 }); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(0); + }); + + it('propagates a fetchMessageBody failure so the drainer can back off', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockRejectedValue(new Error('Command failed')); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }] }); + return Promise.resolve({ rows: [] }); + }); + + await expect(mgr.fetchBodiesForMessages('acct-1', ['m1'])).rejects.toThrow('Command failed'); + }); + + it('skips a poison message that throws mid-batch and still fetches the rest (no wedge)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn() + .mockRejectedValueOnce(new Error("Mailbox doesn't exist")) + .mockResolvedValueOnce({ html: '

hi

', text: 'hi', attachments: [] }); + sanitizeEmail.mockReturnValue('

hi

'); + snippetFromBody.mockReturnValue('hi'); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'Stale' }, { id: 'm2', uid: 6, folder: 'INBOX' }] }); + } + return Promise.resolve({ rows: [] }); + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2']); + + // The poison message (m1) is skipped, not thrown — the loop keeps going so a batch + // containing it still makes forward progress instead of wedging the account forever. + expect(result).toEqual({ fetched: 1 }); + expect(mgr.fetchMessageBody).toHaveBeenCalledTimes(2); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(1); + }); + + it('rethrows when every message in the batch fails (genuine outage, not one bad message)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockRejectedValue(new Error('Connection closed')); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }, { id: 'm2', uid: 6, folder: 'INBOX' }] }); + } + return Promise.resolve({ rows: [] }); + }); + + await expect(mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2'])).rejects.toThrow('Connection closed'); + expect(mgr.fetchMessageBody).toHaveBeenCalledTimes(2); // both attempted, neither skipped silently + }); +}); diff --git a/backend/src/services/search/ftsBackfill.js b/backend/src/services/search/ftsBackfill.js new file mode 100644 index 00000000..bacef25b --- /dev/null +++ b/backend/src/services/search/ftsBackfill.js @@ -0,0 +1,108 @@ +import { query } from '../db.js'; +import { searchFtsExpr, FTS_VERSION } from './lexicalRepo.js'; +import { upsertJob } from '../backgroundJobs.js'; + +const KIND = 'fts_backfill'; +const BATCH = 3000; // 2–5k rows/batch +const IDLE_MS = 5 * 60 * 1000; +const defaultSleep = () => new Promise((r) => setTimeout(r, 250)); + +let running = false; + +// FTS_VERSION is inlined as a literal (not a bind param) so the predicate +// matches the partial index idx_messages_fts_stale_v1 exactly and the batch +// scan is index-served rather than a growing seq scan as the tail drains. +const STALE_PRED = `fts_version IS DISTINCT FROM ${FTS_VERSION}`; + +const batchSql = ` + WITH batch AS ( + SELECT id FROM messages + WHERE ${STALE_PRED} + ORDER BY date DESC NULLS LAST + LIMIT $1 + ) + UPDATE messages m + SET search_fts = ${searchFtsExpr('m')}, + fts_version = ${FTS_VERSION} + FROM batch + WHERE m.id = batch.id +`; + +async function processBatch() { + try { + const res = await query(batchSql, [BATCH]); + return res.rowCount; + } catch (err) { + if (err.code !== '54000') throw err; // program_limit_exceeded + return processBatchRowByRow(); + } +} + +// A single pathological body tripped SQLSTATE 54000 for the whole batch. +// Re-process row by row so one bad row can't wedge the drainer. +async function processBatchRowByRow() { + const { rows } = await query( + `SELECT id FROM messages WHERE ${STALE_PRED} ORDER BY date DESC NULLS LAST LIMIT $1`, + [BATCH] + ); + let done = 0; + for (const { id } of rows) { + try { + await query( + `UPDATE messages m SET search_fts = ${searchFtsExpr('m')}, fts_version = ${FTS_VERSION} WHERE m.id = $1`, + [id] + ); + } catch (err) { + if (err.code !== '54000') throw err; + // Stamp the version so it is never retried; leave search_fts NULL (still + // served by the ILIKE fallback in lexicalRepo). + await query(`UPDATE messages SET fts_version = ${FTS_VERSION} WHERE id = $1 AND search_fts IS NULL`, [id]); + console.warn(`FTS backfill: skipped oversized message ${id} (tsvector too large)`); + } + done++; + } + return done; +} + +export async function runFtsBackfill({ sleep = defaultSleep } = {}) { + if (running) return; // single-flight + running = true; + try { + const { rows: [{ remaining }] } = await query( + `SELECT count(*)::int AS remaining FROM messages WHERE ${STALE_PRED}` + ); + if (remaining === 0) { + await upsertJob({ kind: KIND, state: 'done', processed: 0, total: 0 }); + return; + } + const total = remaining; + let processed = 0; + await upsertJob({ kind: KIND, state: 'running', processed, total }); + + while (true) { + let n; + try { + n = await processBatch(); + } catch (err) { + await upsertJob({ kind: KIND, state: 'error', processed, total, lastError: err.message }); + throw err; + } + if (n === 0) break; + processed += n; + await upsertJob({ kind: KIND, state: 'running', processed: Math.min(processed, total), total }); + await sleep(); + } + await upsertJob({ kind: KIND, state: 'done', processed: total, total }); + } finally { + running = false; + } +} + +export function scheduleFtsBackfill() { + runFtsBackfill().catch((err) => console.error('FTS backfill error:', err.message)); + const timer = setInterval(() => { + runFtsBackfill().catch((err) => console.error('FTS backfill error:', err.message)); + }, IDLE_MS); + timer.unref?.(); + return timer; +} diff --git a/backend/src/services/search/ftsBackfill.test.js b/backend/src/services/search/ftsBackfill.test.js new file mode 100644 index 00000000..3539739a --- /dev/null +++ b/backend/src/services/search/ftsBackfill.test.js @@ -0,0 +1,45 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +// Mock inline + import the mocked bindings (avoids the vi.mock hoisting TDZ trap). +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../backgroundJobs.js', () => ({ upsertJob: vi.fn() })); +import { query } from '../db.js'; +import { upsertJob } from '../backgroundJobs.js'; +import { runFtsBackfill } from './ftsBackfill.js'; + +const noSleep = () => Promise.resolve(); +beforeEach(() => { query.mockReset(); upsertJob.mockReset(); }); + +describe('runFtsBackfill', () => { + it('marks the job done and does no work when nothing is stale', async () => { + query.mockResolvedValueOnce({ rows: [{ remaining: 0 }] }); // count + await runFtsBackfill({ sleep: noSleep }); + expect(query).toHaveBeenCalledTimes(1); // count only, no batch UPDATE + expect(upsertJob).toHaveBeenCalledWith(expect.objectContaining({ kind: 'fts_backfill', state: 'done' })); + }); + + it('drains in batches until an UPDATE affects zero rows, reporting progress', async () => { + query + .mockResolvedValueOnce({ rows: [{ remaining: 5000 }] }) // count + .mockResolvedValueOnce({ rowCount: 3000 }) // batch 1 + .mockResolvedValueOnce({ rowCount: 2000 }) // batch 2 + .mockResolvedValueOnce({ rowCount: 0 }); // batch 3 → stop + await runFtsBackfill({ sleep: noSleep }); + const batchSql = query.mock.calls[1][0]; + expect(batchSql).toContain('fts_version IS DISTINCT FROM 1'); + expect(batchSql).toContain("setweight(to_tsvector('english', coalesce(m.subject,'')), 'A')"); + expect(upsertJob).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 5000, total: 5000 }) + ); + }); + + it('is single-flight: a second concurrent call returns immediately', async () => { + let release; + const gate = new Promise((r) => (release = r)); + query.mockImplementationOnce(async () => { await gate; return { rows: [{ remaining: 0 }] }; }); + const first = runFtsBackfill({ sleep: noSleep }); + await runFtsBackfill({ sleep: noSleep }); // returns immediately (guarded) + release(); + await first; + expect(query).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/services/search/lexicalRepo.js b/backend/src/services/search/lexicalRepo.js new file mode 100644 index 00000000..4d346fce --- /dev/null +++ b/backend/src/services/search/lexicalRepo.js @@ -0,0 +1,267 @@ +import { shouldExcludeTrashFromSearch } from './queryParser.js'; + +// Postgres refuses to build a tsvector larger than ~1MB of packed lexemes +// (SQLSTATE 54000). Cap the text fed to to_tsvector at 600k chars — matching +// msgvault's maxFTSBodyChars (internal/store/dialect_pg.go). Reused by the +// slice-02 search_fts trigger and backfill via searchFtsExpr(). +export const FTS_BODY_CHAR_CAP = 600000; + +// Single source of truth for the stored-tsvector layout version. The trigger +// and backfill both stamp fts_version = FTS_VERSION; a layout/dictionary +// change bumps this and re-runs the backfill. +export const FTS_VERSION = 1; + +// Wraps a positive condition so that when negated it also matches rows where +// the underlying columns are NULL. +export function negateCond(sql) { + return `NOT COALESCE((${sql}), false)`; +} + +export function trashFolderExclusionCondition() { + return `NOT EXISTS ( + SELECT 1 + FROM folders f + WHERE f.account_id = m.account_id + AND f.path = m.folder + AND (f.special_use = '\\Trash' + OR lower(f.name) LIKE '%trash%' + OR lower(f.name) LIKE '%deleted%') + )`; +} + +// One free-text term matches if it appears in the sender, subject, the stored +// search_vector, or the length-capped body. Extracted so the body cap is a +// single, testable source of truth (the ranked, search_fts-first variant takes +// over once rows are backfilled). +export function freeTextTermCondition(likeIdx, ftsIdx) { + return `( + m.from_name ILIKE $${likeIdx} + OR m.from_email ILIKE $${likeIdx} + OR m.subject ILIKE $${likeIdx} + OR m.search_vector @@ plainto_tsquery('english', $${ftsIdx}) + OR to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})) @@ plainto_tsquery('english', $${ftsIdx}) + )`; +} + +// BM25-style lexical rank (ts_rank_cd; class weights D,C,B,A → 10:4:1 +// subject:from:rest; length normalization 32). +export const LEXICAL_RANK_SQL = (vectorExpr, queryExpr) => + `ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], ${vectorExpr}, ${queryExpr}, 32)`; + +// A term counts as searchable text only if it has at least one letter or +// digit; msgvault's hasFTSToken drops punctuation-only tokens ("!!!", "***") +// the same way, since they'd normalize to zero lexemes and can't usefully +// match or rank anything. +export function hasSearchableToken(term) { + return /[\p{L}\p{N}]/u.test(term); +} + +// A bare free-text word is a single search token; a term containing whitespace +// only arises from a quoted multi-word phrase (queryParser doesn't emit these +// today, but the builders below stay correct if it ever does). Phrases keep +// ordinary non-prefix matching — prefix-expanding a multi-word phrase isn't a +// single lexeme operation and would change its meaning. +function isPhraseTerm(term) { + return /\s/.test(term); +} + +// msgvault's BuildFTSArg prefix-matches every bare word (typing "amaz" still +// finds "amazon") — plainto_tsquery only matches whole stemmed words, which +// regressed recall vs the old ILIKE substring behavior once a row is +// backfilled onto search_fts. quote_literal($N) is a plain SQL bind param +// reference (no JS-side string building), so the raw term travels through as +// an ordinary parameter; quote_literal() at query time safely wraps it as a +// single tsquery lexeme literal — neutralizing any &, |, !, (, ), ', or : +// the term might contain — before ':*' marks it for prefix matching. +export function ftsTermQueryArg(ftsIdx, term) { + return isPhraseTerm(term) + ? `plainto_tsquery('english', $${ftsIdx})` + : `to_tsquery('english', quote_literal($${ftsIdx}) || ':*')`; +} + +function ftsMatchExpr(vectorExpr, ftsIdx, term) { + return `${vectorExpr} @@ ${ftsTermQueryArg(ftsIdx, term)}`; +} + +// Ranked variant used once rows carry search_fts: backfilled rows match via the +// GIN-indexed tsvector; rows not yet backfilled (search_fts IS NULL) fall back +// to the legacy ILIKE/search_vector/body branch so recall never regresses +// mid-backfill. The fallback matches nothing once search_fts is populated, +// letting the planner use the GIN index exclusively. +export function freeTextTermConditionRanked(likeIdx, ftsIdx, term) { + return `( + ${ftsMatchExpr('m.search_fts', ftsIdx, term)} + OR (m.search_fts IS NULL AND ${freeTextTermCondition(likeIdx, ftsIdx)}) + )`; +} + +// English stopwords ("the", "for", "you", …) normalize to an EMPTY tsquery +// (numnode = 0), and `tsvector @@ ` is FALSE for every row — so one +// stopword in an AND'd term chain silently nuked ALL results once rows were +// backfilled onto search_fts ("waiting for invoice" → 0 hits, because of +// "for"; the ILIKE fallback arm is dead once search_fts is populated). +// A term whose tsquery normalizes empty must contribute NOTHING instead: +// this wraps the term's already-polarity-applied condition so it is +// vacuously TRUE — for positive AND negated terms alike — whenever the +// term's tsquery is empty. The numnode() probe runs on the SAME bind (and +// the SAME prefix-or-phrase construction, via ftsTermQueryArg) the match +// uses, so guard and match can never disagree; a query of ONLY stopwords +// degrades to a filter-only, date-ordered search. The relevance rank needs +// no guard: `&&` drops an empty tsquery operand and ts_rank_cd over a fully +// empty tsquery is 0. +export function stopwordSafeCondition(ftsIdx, term, condition) { + return `(numnode(${ftsTermQueryArg(ftsIdx, term)}) = 0 OR ${condition})`; +} + +// The one owner of a free-text term's full predicate: ranked FTS match, +// un-backfilled ILIKE fallback, polarity, and stopword vacuity. `bind(value) → +// '$n'` pushes onto the caller's params; the raw ordinals expected by +// freeTextTermConditionRanked are recovered from the placeholders. +export function freeTextTermClause(term, negate, bind) { + const likeIdx = Number(bind(`%${term}%`).slice(1)); + const ftsIdx = Number(bind(term).slice(1)); + const cond = freeTextTermConditionRanked(likeIdx, ftsIdx, term); + return stopwordSafeCondition(ftsIdx, term, negate ? negateCond(cond) : cond); +} + +// The weighted tsvector written into messages.search_fts. `ref` is the row +// reference: 'm' for the backfill UPDATE, 'NEW' for the BEFORE trigger. The +// class weights map to ts_rank_cd's D,C,B,A array (10:4:1 subject:from:rest). +// MUST stay identical to migration 0035's trigger body. +export function searchFtsExpr(ref) { + return `setweight(to_tsvector('english', coalesce(${ref}.subject,'')), 'A') || + setweight(to_tsvector('english', coalesce(${ref}.from_name,'') || ' ' || coalesce(${ref}.from_email,'')), 'B') || + setweight(to_tsvector('english', coalesce(${ref}.to_addresses::text,'') || ' ' || coalesce(${ref}.cc_addresses::text,'')), 'C') || + setweight(to_tsvector('english', LEFT(coalesce(${ref}.body_text,''), ${FTS_BODY_CHAR_CAP})), 'D')`; +} + +// Structured operator predicates (from/to/cc/subject/has/is/after/before), +// negation-aware. `bind(value) → '$n'` pushes value onto the caller's param +// list and returns its placeholder — the caller owns `params`/the running +// index, so searchLexical remains the single owner of these predicates. +// Excludes free-text terms and folder scope (`in:`), which are not +// structured row predicates. Extracted from searchLexical byte-identically — +// same branches, same ILIKE-arm reuse, same skip rules. +export function buildOperatorClauses(filters, bind) { + const conditions = []; + for (const f of filters) { + if (f.key === 'in') continue; // controls scope, not a row condition + let cond = null; + + if (f.key === 'from') { + const idx = bind(`%${f.value}%`); + cond = `(m.from_email ILIKE ${idx} OR m.from_name ILIKE ${idx})`; + } else if (f.key === 'subject') { + cond = `m.subject ILIKE ${bind(`%${f.value}%`)}`; + } else if (f.key === 'to') { + const idx = bind(`%${f.value}%`); + cond = `(m.to_addresses::text ILIKE ${idx} OR m.cc_addresses::text ILIKE ${idx})`; + } else if (f.key === 'cc') { + cond = `m.cc_addresses::text ILIKE ${bind(`%${f.value}%`)}`; + } else if (f.key === 'has') { + if (f.value === 'attachment' || f.value === 'attachments') cond = `m.has_attachments = true`; + } else if (f.key === 'is') { + if (f.value === 'unread') cond = `m.is_read = false`; + else if (f.value === 'read') cond = `m.is_read = true`; + else if (f.value === 'starred') cond = `m.is_starred = true`; + } else if (f.key === 'after') { + const d = new Date(f.value); + if (!isNaN(d)) cond = `m.date >= ${bind(d.toISOString())}`; + } else if (f.key === 'before') { + const d = new Date(f.value); + if (!isNaN(d)) cond = `m.date < ${bind(d.toISOString())}`; + } + + if (cond) conditions.push(f.negate ? negateCond(cond) : cond); + } + return conditions; +} + +// Folder-scope predicate(s) for the messages table. `folderScope`/`folderFuzzy` come from +// resolveSearchFolderScope: a truthy fuzzy scope (in:) matches the bare +// name OR any .../ path; an exact scope (REST ?folder=) matches the full +// path; a null scope carries no explicit folder and excludes trash-like folders +// (the default search scope). `bind(value) → '$n'` — the caller owns params. +export function buildFolderScopeClauses(folderScope, folderFuzzy, bind) { + const conditions = []; + if (folderScope) { + if (folderFuzzy) { + const name = bind(folderScope); + const path = bind(`%/${folderScope}`); + conditions.push(`(m.folder ILIKE ${name} OR m.folder ILIKE ${path})`); + } else { + conditions.push(`m.folder = ${bind(folderScope)}`); + } + } else if (shouldExcludeTrashFromSearch(folderScope)) { + conditions.push(trashFolderExclusionCondition()); + } + return conditions; +} + +export async function searchLexical(client, { parsed, accountIds, folderScope, folderFuzzy, ordering, limit, offset }) { + const { filters, terms } = parsed; + const params = [accountIds]; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const conditions = buildOperatorClauses(filters, bind); + + for (const term of terms.slice(0, 10)) { + // Single-char terms are too broad/expensive; a punctuation-only term + // (e.g. "!!!") has no letters/digits to search on — msgvault's + // hasFTSToken drops it the same way rather than handing Postgres a + // token that would normalize to zero lexemes. + if (term.value.length < 2 || !hasSearchableToken(term.value)) continue; + conditions.push(freeTextTermClause(term.value, term.negate, bind)); + } + + // A bare in:inbox (or lone folder param) must never dump a whole folder. + if (!conditions.length) return { rows: [], hasCondition: false }; + + for (const cond of buildFolderScopeClauses(folderScope, folderFuzzy, bind)) conditions.push(cond); + + // D5: rank a free-text search by ts_rank_cd over the combined positive terms + // (date/id tiebreak); a filter-only search stays date-ordered. The rank arg + // must be bound before LIMIT/OFFSET. + let orderBy = 'ORDER BY m.date DESC'; + const positiveTerms = terms.slice(0, 10) + .filter(t => !t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => t.value); + if (ordering === 'relevance' && positiveTerms.length) { + // Rank with the SAME prefix-aware tsquery the MATCH predicate uses + // (freeTextTermConditionRanked → ftsTermQueryArg): one bind per term, + // combined with && (ts_rank_cd takes a single tsquery). Reusing the one + // term→tsquery-arg builder is what makes + // predicate and rank impossible to diverge — a prefix-only hit ("invo" + // matching "invoice") then ranks by ts_rank_cd instead of getting rank 0 + // (which COALESCE(...,0) would collapse to date order). Phrases keep + // non-prefix matching via ftsTermQueryArg, exactly as the predicate does. + const rankArgs = positiveTerms.map((term) => { params.push(term); return p++; }); + const rankQuery = positiveTerms.map((term, i) => ftsTermQueryArg(rankArgs[i], term)).join(' && '); + const rankExpr = LEXICAL_RANK_SQL('m.search_fts', rankQuery); + // ts_rank_cd(..., NULL::tsvector, ...) returns NULL for un-backfilled rows + // (search_fts IS NULL), and Postgres sorts NULLs FIRST in DESC order — so + // without COALESCE every un-backfilled row would outrank every properly + // ranked hit during the backfill window. COALESCE(...,0) keeps them + // sorting by the date/id tiebreak instead, below any real rank score. + orderBy = `ORDER BY COALESCE(${rankExpr}, 0) DESC, m.date DESC, m.id DESC`; + } + + params.push(limit); + params.push(offset); + + const result = await client(` + SELECT + m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, + m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, + a.name as account_name, a.email_address as account_email, a.color as account_color + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND ${conditions.join('\n AND ')} + ${orderBy} + LIMIT $${p} OFFSET $${p + 1} + `, params); + + return { rows: result.rows, hasCondition: true }; +} diff --git a/backend/src/services/search/lexicalRepo.test.js b/backend/src/services/search/lexicalRepo.test.js new file mode 100644 index 00000000..0144e31e --- /dev/null +++ b/backend/src/services/search/lexicalRepo.test.js @@ -0,0 +1,385 @@ +import { describe, it, expect, vi } from 'vitest'; +import { + FTS_BODY_CHAR_CAP, + FTS_VERSION, + negateCond, + trashFolderExclusionCondition, + freeTextTermCondition, + searchFtsExpr, + searchLexical, + LEXICAL_RANK_SQL, + freeTextTermConditionRanked, + freeTextTermClause, + stopwordSafeCondition, + buildOperatorClauses, + buildFolderScopeClauses, +} from './lexicalRepo.js'; + +describe('SQL fragment builders', () => { + it('pins the body cap at 600000 and FTS version at 1 (msgvault parity)', () => { + expect(FTS_BODY_CHAR_CAP).toBe(600000); + expect(FTS_VERSION).toBe(1); + }); + + it('caps the body tsvector inside the free-text condition', () => { + const cond = freeTextTermCondition(3, 4); + expect(cond).toContain( + `to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))` + ); + expect(cond).toContain('m.from_name ILIKE $3'); + expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); + }); + + it('negateCond wraps a positive condition to also match NULL columns', () => { + expect(negateCond('m.is_read = true')).toBe('NOT COALESCE((m.is_read = true), false)'); + }); + + it('trashFolderExclusionCondition targets trash-like folders', () => { + const sql = trashFolderExclusionCondition(); + expect(sql).toContain('NOT EXISTS'); + expect(sql).toContain('%trash%'); + expect(sql).toContain('%deleted%'); + }); + + it('searchFtsExpr builds a weighted A/B/C/D tsvector for the given row alias', () => { + const expr = searchFtsExpr('m'); + expect(expr).toContain("setweight(to_tsvector('english', coalesce(m.subject,'')), 'A')"); + expect(expr).toContain("coalesce(m.from_name,'') || ' ' || coalesce(m.from_email,'')"); + expect(expr).toContain("LEFT(coalesce(m.body_text,''), 600000)), 'D')"); + // NEW.-form for the trigger must be produced by the same builder. + expect(searchFtsExpr('NEW')).toContain("coalesce(NEW.subject,'')"); + }); +}); + +describe('searchLexical', () => { + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [{ id: 'x' }] }; }); + return { fn, calls }; + } + + it('returns hasCondition:false and never queries when there is no real condition', async () => { + const { fn } = mockClient(); + const res = await searchLexical(fn, { + parsed: { filters: [{ key: 'in', value: 'inbox', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'inbox', folderFuzzy: true, ordering: 'date', limit: 50, offset: 0, + }); + expect(res).toEqual({ rows: [], hasCondition: false }); + expect(fn).not.toHaveBeenCalled(); + }); + + it('emits the pre-existing SQL shape and bind params for a from:+term all-folder query', async () => { + const { fn, calls } = mockClient(); + const res = await searchLexical(fn, { + parsed: { + filters: [{ key: 'from', value: 'amazon', negate: false }], + terms: [{ value: 'invoice', negate: false }], + }, + accountIds: ['a1', 'a2'], folderScope: null, folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(res.hasCondition).toBe(true); + expect(res.rows).toEqual([{ id: 'x' }]); + const { text, params } = calls[0]; + // Params: [accountIds, from-like, term-like, term-fts, limit, offset] + expect(params).toEqual([['a1', 'a2'], '%amazon%', '%invoice%', 'invoice', 50, 0]); + // Load-bearing SQL shape (byte-identical gate: the human-runnable old-vs-new diff). + expect(text).toContain('WHERE m.account_id = ANY($1)'); + expect(text).toContain('AND m.is_deleted = false'); + expect(text).toContain('(m.from_email ILIKE $2 OR m.from_name ILIKE $2)'); + expect(text).toContain('NOT EXISTS'); // trash exclusion on all-folder search + expect(text).toContain('ORDER BY m.date DESC'); + expect(text).toContain('LIMIT $5 OFFSET $6'); + }); + + it('adds a cc_addresses predicate for the new cc: operator', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [{ key: 'cc', value: 'boss', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain('m.cc_addresses::text ILIKE $2'); + expect(calls[0].params).toEqual([['a1'], '%boss%', 'INBOX', 50, 0]); + }); + +}); + +describe('ranked lexical query (slice 02)', () => { + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [] }; }); + return { fn, calls }; + } + + it('LEXICAL_RANK_SQL emits ts_rank_cd with the D,C,B,A weight array and normalization 32', () => { + const expr = LEXICAL_RANK_SQL('m.search_fts', "plainto_tsquery('english', $7)"); + expect(expr).toBe("ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, plainto_tsquery('english', $7), 32)"); + }); + + it('freeTextTermConditionRanked prefix-matches a single-word term against search_fts, falling back only when it IS NULL', () => { + const cond = freeTextTermConditionRanked(3, 4, 'invoice'); + expect(cond).toContain("m.search_fts @@ to_tsquery('english', quote_literal($4) || ':*')"); + expect(cond).not.toContain("m.search_fts @@ plainto_tsquery"); + expect(cond).toContain('m.search_fts IS NULL AND'); + // The IS-NULL fallback branch is untouched legacy SQL (freeTextTermCondition): + // still plainto_tsquery + ILIKE, since it only serves un-backfilled rows. + expect(cond).toContain(`LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})`); + expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + }); + + it('freeTextTermConditionRanked keeps non-prefix phrase matching for a quoted multi-word term', () => { + const cond = freeTextTermConditionRanked(3, 4, 'weekly report'); + expect(cond).toContain("m.search_fts @@ plainto_tsquery('english', $4)"); + expect(cond).not.toContain("m.search_fts @@ to_tsquery"); + }); + + it('prefix-matching passes the raw term through as an ordinary bind param — quoting/escaping happens in SQL via quote_literal, never in JS', async () => { + const { fn, calls } = mockClient(); + const weird = `d'Angelo:*(evil)`; + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: weird, negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // The term is bound VERBATIM (no JS-side escaping) — quote_literal() at + // query time is what neutralizes '/:/* /()/etc. so it can't act as tsquery + // syntax or break out of the quoted lexeme. + expect(params).toContain(weird); + expect(text).toContain("to_tsquery('english', quote_literal($3) || ':*')"); + }); + + it('drops a punctuation-only term (msgvault hasFTSToken parity) instead of emitting an invalid tsquery', async () => { + const { fn } = mockClient(); + const res = await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: '!!!', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + // A punctuation-only term contributes no condition at all — same + // treatment as an under-length term — so a bare "!!!" search never + // dumps the whole folder and never reaches Postgres with a term that + // would normalize to zero lexemes. + expect(res).toEqual({ rows: [], hasCondition: false }); + expect(fn).not.toHaveBeenCalled(); + }); + + it('drops a punctuation-only term while keeping a real term alongside it', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: '***', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + // Only the real term's params were pushed: [accountIds, like, fts, folder, limit, offset]. + expect(calls[0].params).toEqual([['a1'], '%invoice%', 'invoice', 'INBOX', 50, 0]); + }); + + it('orders by ts_rank_cd then date/id for a relevance query, binding the rank args before LIMIT/OFFSET', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'urgent', negate: false }] }, + accountIds: ['a1'], folderScope: null, folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // NULL-safe: ts_rank_cd(..., NULL::tsvector, ...) returns NULL, and Postgres + // sorts NULLs FIRST in DESC order — without COALESCE, every un-backfilled + // (search_fts IS NULL) row would outrank every properly ranked hit during + // the backfill window. COALESCE(..., 0) keeps un-backfilled rows sorting + // by date/id like today, below any real (non-negative) rank score. + expect(text).toContain('ORDER BY COALESCE(ts_rank_cd'); + expect(text).toContain('32), 0) DESC, m.date DESC, m.id DESC'); + // Fix 5: rank reuses the SAME prefix-aware per-term tsquery as the MATCH + // predicate (ftsTermQueryArg), one bind per term combined with && — NOT a + // single plainto_tsquery over the joined string — so a prefix-only hit ranks + // by ts_rank_cd instead of collapsing to COALESCE(0) date order. + expect(text).toContain( + "ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, to_tsquery('english', quote_literal($6) || ':*') && to_tsquery('english', quote_literal($7) || ':*'), 32)" + ); + // Params: [accountIds, t1-like, t1-fts, t2-like, t2-fts, t1-rank, t2-rank, limit, offset] + expect(params).toEqual([['a1'], '%invoice%', 'invoice', '%urgent%', 'urgent', 'invoice', 'urgent', 50, 0]); + }); + + it('ranks a prefix-only term with the SAME builder output as the MATCH predicate (Fix 5 — never rank 0)', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invo', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // The MATCH predicate prefix-matches via ftsTermQueryArg; the rank arg is the + // SAME per-term construction (different bind ordinal only), so predicate and + // rank can never diverge on prefix-vs-plainto. + expect(text).toContain("m.search_fts @@ to_tsquery('english', quote_literal($3) || ':*')"); // predicate + expect(text).toContain( + "ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, to_tsquery('english', quote_literal($5) || ':*'), 32)" // rank + ); + // Params: [accountIds, term-like, term-fts(predicate), folder, term-fts(rank), limit, offset] + expect(params).toEqual([['a1'], '%invo%', 'invo', 'INBOX', 'invo', 50, 0]); + }); + + it('keeps non-prefix phrase matching in the rank for a quoted multi-word positive term (mirrors the predicate)', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'weekly report', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text } = calls[0]; + // A phrase term ranks (and matches) via plainto_tsquery — no prefix ':*'. + expect(text).toContain("ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, plainto_tsquery('english', $5), 32)"); + expect(text).not.toContain("quote_literal($5) || ':*'"); + }); + + it('keeps date ordering (no ts_rank_cd) for a filter-only query', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain('ORDER BY m.date DESC'); + expect(calls[0].text).not.toContain('ts_rank_cd'); + }); +}); + +describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { + // Root cause: + // to_tsquery('english', quote_literal('for') || ':*') normalizes to an + // EMPTY tsquery (numnode = 0), and `tsvector @@ ` is FALSE for + // every row — so once rows were backfilled onto search_fts, one english + // stopword in an AND'd term chain ("waiting for invoice") nuked ALL + // results. The guard makes such a term vacuously TRUE instead. + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [] }; }); + return { fn, calls }; + } + + it('stopwordSafeCondition wraps a clause with a numnode()=0 escape on the SAME bind as the match', () => { + expect(stopwordSafeCondition(4, 'invoice', 'X')).toBe( + "(numnode(to_tsquery('english', quote_literal($4) || ':*')) = 0 OR X)" + ); + // Phrase terms probe emptiness through the SAME plainto construction the + // match uses ("the on" normalizes empty exactly like a stopword word). + expect(stopwordSafeCondition(4, 'weekly report', 'X')).toBe( + "(numnode(plainto_tsquery('english', $4)) = 0 OR X)" + ); + }); + + it('wraps every positive free-text term condition, binding NO extra params', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // One guard per term, reusing that term's fts bind ordinal ($3, $5, $7). + for (const ftsIdx of [3, 5, 7]) { + expect(text).toContain(`(numnode(to_tsquery('english', quote_literal($${ftsIdx}) || ':*')) = 0 OR (`); + } + // Guard adds no binds: [accountIds, like1, fts1, like2, fts2, like3, fts3, folder, limit, offset] + expect(params).toEqual([['a1'], '%waiting%', 'waiting', '%for%', 'for', '%invoice%', 'invoice', 'INBOX', 50, 0]); + }); + + it('applies the guard OUTSIDE the negation so a negated stopword also contributes nothing', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'the', negate: true }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text } = calls[0]; + // (empty OR NOT COALESCE(match)) — vacuously TRUE for a stopword in BOTH + // polarities; a plain NOT-wrap of the guarded condition would instead be + // FALSE and exclude everything. + expect(text).toContain("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE("); + }); + + it('freeTextTermClause owns the complete ranked lexical predicate', () => { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const clause = freeTextTermClause('invoice', false, bind); + expect(params).toEqual(['%invoice%', 'invoice']); + expect(clause.startsWith("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR (")).toBe(true); + expect(clause).toContain("m.search_fts @@ to_tsquery('english', quote_literal($3) || ':*')"); + expect(clause).toContain('m.search_fts IS NULL AND'); + }); + +}); + +describe('buildOperatorClauses (Phase 4 Task 2a — extracted from searchLexical)', () => { + function bindHarness(start = 2) { + const params = []; + let p = start; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { bind, params }; + } + + it('builds a from: predicate reusing one bind for both ILIKE arms', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'from', value: 'amazon', negate: false }], bind); + expect(conds).toEqual(['(m.from_email ILIKE $2 OR m.from_name ILIKE $2)']); + expect(params).toEqual(['%amazon%']); + }); + + it('builds a cc: predicate', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'cc', value: 'boss', negate: false }], bind); + expect(conds).toEqual(['m.cc_addresses::text ILIKE $2']); + expect(params).toEqual(['%boss%']); + }); + + it('negates a structured operator via negateCond', () => { + const { bind } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'subject', value: 'invoice', negate: true }], bind); + expect(conds).toEqual(['NOT COALESCE((m.subject ILIKE $2), false)']); + }); + + it('skips in: (scope control, not a row condition) and malformed after/before', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([ + { key: 'in', value: 'inbox', negate: false }, + { key: 'after', value: 'not-a-date', negate: false }, + ], bind); + expect(conds).toEqual([]); + expect(params).toEqual([]); + }); + + it('does not bind free-text terms or folder scope (structured operators only)', () => { + const { bind } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'is', value: 'unread', negate: false }], bind); + expect(conds).toEqual(['m.is_read = false']); + }); +}); + +describe('buildFolderScopeClauses', () => { + function bindHarness(start = 2) { + const params = []; + let p = start; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { bind, params }; + } + + it('fuzzy in: matches the bare name or any .../ path', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses('sent', true, bind); + expect(conds).toEqual(['(m.folder ILIKE $2 OR m.folder ILIKE $3)']); + expect(params).toEqual(['sent', '%/sent']); + }); + + it('an exact folderScope (REST ?folder=) matches the full path', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses('INBOX', false, bind); + expect(conds).toEqual(['m.folder = $2']); + expect(params).toEqual(['INBOX']); + }); + + it('a null folderScope excludes trash-like folders (default search scope), binding nothing', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses(null, false, bind); + expect(conds).toHaveLength(1); + expect(conds[0]).toContain('NOT EXISTS'); + expect(conds[0]).toContain('%trash%'); + expect(params).toEqual([]); + }); +}); diff --git a/backend/src/services/search/migrations.test.js b/backend/src/services/search/migrations.test.js new file mode 100644 index 00000000..53f2a8c8 --- /dev/null +++ b/backend/src/services/search/migrations.test.js @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { searchFtsExpr, FTS_VERSION } from './lexicalRepo.js'; + +const MIGRATIONS = join(dirname(fileURLToPath(import.meta.url)), '../../../migrations'); +const read = (f) => readFileSync(join(MIGRATIONS, f), 'utf8'); +const norm = (s) => s.replace(/\s+/g, ' ').trim(); + +describe('0035_search_fts.sql', () => { + const sql = read('0035_search_fts.sql'); + + it('is transactional (no no-transaction header — the $$ body cannot survive the ; splitter)', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(false); + }); + + it('adds nullable search_fts + fts_version with IF NOT EXISTS (fast metadata DDL, D1)', () => { + expect(norm(sql)).toContain('ADD COLUMN IF NOT EXISTS search_fts tsvector'); + expect(norm(sql)).toContain('ADD COLUMN IF NOT EXISTS fts_version int'); + expect(sql).not.toContain('GENERATED ALWAYS AS'); // D1: not a generated column + }); + + it('installs a BEFORE INSERT OR UPDATE trigger whose body equals searchFtsExpr(NEW)', () => { + expect(norm(sql)).toContain('BEFORE INSERT OR UPDATE ON messages'); + expect(norm(sql)).toContain(norm(searchFtsExpr('NEW'))); + expect(norm(sql)).toContain(`NEW.fts_version := ${FTS_VERSION}`); + }); + + it('handles the oversized-tsvector case gracefully (never fails the row write)', () => { + expect(sql).toContain('EXCEPTION WHEN program_limit_exceeded'); + }); + + it('skips recompute only on columns that actually feed searchFtsExpr — snippet is not one of them', () => { + // snippet isn't part of the weighted A/B/C/D expression (subject, from, + // to/cc, body_text), so guarding on it made a snippet-only UPDATE (e.g. + // read/star-adjacent metadata writes that also touch snippet) recompute + // an identical search_fts for nothing. + expect(sql).toContain('NEW.subject IS NOT DISTINCT FROM OLD.subject'); + expect(sql).toContain('NEW.from_name IS NOT DISTINCT FROM OLD.from_name'); + expect(sql).toContain('NEW.from_email IS NOT DISTINCT FROM OLD.from_email'); + expect(sql).toContain('NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses'); + expect(sql).toContain('NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses'); + expect(sql).toContain('NEW.body_text IS NOT DISTINCT FROM OLD.body_text'); + expect(sql).not.toContain('OLD.snippet'); + expect(sql).not.toContain('NEW.snippet'); + }); +}); + +describe('0037_search_fts_index.sql', () => { + const sql = read('0037_search_fts_index.sql'); + + it('runs outside a transaction and builds both indexes CONCURRENTLY', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(true); + expect(sql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts'); + expect(sql).toContain('USING GIN (search_fts)'); + expect(sql).toContain('idx_messages_fts_stale_v1'); + expect(sql).toContain(`WHERE fts_version IS DISTINCT FROM ${FTS_VERSION}`); + expect(sql).not.toContain('$$'); // no function bodies — safe for the ; splitter + }); + + // A cancelled/crashed CREATE INDEX CONCURRENTLY leaves an INVALID index under the + // target name; retrying with IF NOT EXISTS silently skips it, so the scan stays + // unindexed forever. Drop-if-exists before each create makes the retry crash-idempotent. + it('drops each index CONCURRENTLY before creating it (invalid-index retry hazard)', () => { + for (const name of ['idx_messages_search_fts', 'idx_messages_fts_stale_v1']) { + expect(sql).toContain(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`); + expect(sql.indexOf(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)) + .toBeLessThan(sql.indexOf(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${name}`)); + } + }); +}); diff --git a/backend/src/services/search/queryParser.js b/backend/src/services/search/queryParser.js new file mode 100644 index 00000000..bdba1e36 --- /dev/null +++ b/backend/src/services/search/queryParser.js @@ -0,0 +1,185 @@ +// Pure query grammar for lexical search. Ports msgvault's operator set +// (internal/search/parser.go); operators Mailflow's schema cannot serve +// (larger:/smaller: — no size column, bcc: — not stored, label:/l: — no +// labels) are RECORDED as unsupported, never silently widened into a match. + +// Multi-char keys precede single-char `l` so the alternation resolves +// `label:` before `l:`. The leading (-?) captures optional negation; \b sits +// between an optional '-' and the key so both `from:` and `-from:` match. +const OP_KEYS = 'from|to|cc|bcc|subject|has|is|after|before|in|older_than|newer_than|larger|smaller|label|l'; +const OP_PATTERN = new RegExp(`(-?)\\b(${OP_KEYS}):("([^"]*)"|([\\S]+))`, 'gi'); + +// Bare quoted phrases (msgvault tokenize parity, parser.go:395-464): a double- +// OR single-quoted span that STARTS a token (preceded by start/whitespace, +// optionally negated with '-') becomes ONE phrase term. A quote glued to the +// tail of a token — from:"John Smith", d'Angelo — never starts a phrase: +// operator values belong to OP_PATTERN and mid-word apostrophes are text. +// Backslash escapes keep a quote char from terminating the span. +const PHRASE_PATTERN = /(^|\s)(-?)(["'])((?:\\.|(?!\3)[^\\])*)\3/g; + +// Port of msgvault's unescapeQuotedValue: `\\` and an escaped quote collapse +// to the bare char; any other `\x` keeps its backslash literally. +function unescapePhrase(s) { + let out = ''; + let escaped = false; + for (const ch of s) { + if (escaped) { + out += (ch === '\\' || ch === '"' || ch === "'") ? ch : '\\' + ch; + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else { + out += ch; + } + } + if (escaped) out += '\\'; + return out; +} + +// Sizes: 5M / 100K / 1G → bytes. Longer suffixes checked first. Returns null +// on anything unparseable (port of parser.go parseSize). +function parseSize(value) { + const v = value.trim().toUpperCase(); + const mult = { KB: 1024, MB: 1048576, GB: 1073741824, K: 1024, M: 1048576, G: 1073741824 }; + for (const suffix of ['KB', 'MB', 'GB', 'K', 'M', 'G']) { + if (v.endsWith(suffix)) { + const num = parseFloat(v.slice(0, -suffix.length)); + if (Number.isNaN(num)) return null; + return Math.floor(num * mult[suffix]); + } + } + return /^\d+$/.test(v) ? parseInt(v, 10) : null; +} + +// Relative ages: 7d / 2w / 1m / 1y → an absolute ISO timestamp relative to +// `now` (port of parser.go parseRelativeDate). Returns null if unparseable. +function relativeAgeToISO(value, now) { + const m = /^(\d+)([dwmy])$/.exec(value.trim().toLowerCase()); + if (!m) return null; + const n = parseInt(m[1], 10); + const d = new Date(now.getTime()); + switch (m[2]) { + case 'd': d.setUTCDate(d.getUTCDate() - n); break; + case 'w': d.setUTCDate(d.getUTCDate() - n * 7); break; + case 'm': d.setUTCMonth(d.getUTCMonth() - n); break; + case 'y': d.setUTCFullYear(d.getUTCFullYear() - n); break; + default: return null; + } + return d.toISOString(); +} + +function applyOperator(key, value, negate, ctx) { + const { filters, unsupported, errors, now } = ctx; + switch (key) { + case 'from': case 'to': case 'cc': + case 'subject': case 'has': case 'is': + case 'in': + if (value) filters.push({ key, value, negate }); + return; + case 'after': + case 'before': { + if (!value) return; + // A bad date must be RECORDED, not silently dropped downstream + // (buildOperatorClauses skips unparseable dates via isNaN, which would + // silently WIDEN the results). Same uniform message parser.go's + // operatorValueError emits for a bad before:/after: value. + if (isNaN(new Date(value))) { + errors.push(`invalid value "${value}" for ${key}: — expected a date like YYYY-MM-DD`); + return; + } + filters.push({ key, value, negate }); + return; + } + case 'newer_than': + case 'older_than': { + const iso = relativeAgeToISO(value, now); + if (!iso) { + errors.push(`invalid value "${value}" for ${key}: — expected a relative age like 7d, 2w, 1m, or 1y`); + return; + } + filters.push({ key: key === 'newer_than' ? 'after' : 'before', value: iso, negate }); + return; + } + case 'larger': + case 'smaller': { + if (parseSize(value) === null) { + errors.push(`invalid value "${value}" for ${key}: — expected a size like 5M, 100K, or 1G`); + return; + } + // Recognized and well-formed, but messages has no size column. + unsupported.push({ key, token: `${key}:${value}` }); + return; + } + case 'bcc': + case 'label': + case 'l': + // No bcc column / no labels concept in Mailflow. + if (value) unsupported.push({ key: key === 'l' ? 'label' : key, token: `${key}:${value}` }); + return; + default: + return; + } +} + +export function parseQuery(raw, { now = new Date() } = {}) { + const filters = []; + const terms = []; + const unsupported = []; + const errors = []; + const ctx = { filters, unsupported, errors, now }; + + // Phase 1: lift bare quoted phrases out BEFORE the operator grammar runs, so + // a colon inside a phrase ("subject:not an operator") can't be parsed as an + // operator. Each phrase leaves a U+E000-delimited placeholder in the string + // (a Private Use Area char) — restored in term order below — so phrases and + // bare words keep their relative positions. U+E000 is stripped from the raw + // input first, so user text can never forge a placeholder. + const phrases = []; + const withPhrases = (raw || '').replace(/\uE000/g, '').replace(PHRASE_PATTERN, (_, pre, neg, _q, body) => { + const value = unescapePhrase(body); + if (!value.trim()) return pre; // empty phrase ("") contributes nothing + phrases.push({ value, negate: neg === '-' }); + return `${pre}\uE000${phrases.length - 1}\uE000`; + }); + + const remaining = withPhrases.replace(OP_PATTERN, (_, neg, key, _v, quoted, unquoted) => { + const k = key.toLowerCase(); + const value = (quoted !== undefined ? quoted : (unquoted || '')).toLowerCase().trim(); + applyOperator(k, value, neg === '-', ctx); + return ' '; + }).trim(); + + for (const word of remaining.split(/\s+/)) { + let w = word.trim(); + if (!w || w === '-') continue; // skip blanks and a lone '-' (nothing to negate) + const ph = /^\uE000(\d+)\uE000$/.exec(w); + if (ph) { terms.push(phrases[Number(ph[1])]); continue; } + let negate = false; + if (w[0] === '-' && w.length > 1) { negate = true; w = w.slice(1); } + terms.push({ value: w, negate }); + } + + return { filters, terms, unsupported, errors }; +} + +export function resolveSearchFolderScope(filters, folderParam = '') { + let folderScope; + let folderFuzzy = false; // in: matches loosely; the folder param is exact + + for (const f of filters) { + if (f.key !== 'in') continue; + if (f.value === 'all') { folderScope = null; } + else { folderScope = f.value; folderFuzzy = true; } + } + + if (folderScope === undefined) { + folderScope = (folderParam || '').trim() || null; + folderFuzzy = false; + } + + return { folderScope, folderFuzzy }; +} + +export function shouldExcludeTrashFromSearch(folderScope) { + return folderScope === null; +} diff --git a/backend/src/services/search/queryParser.test.js b/backend/src/services/search/queryParser.test.js new file mode 100644 index 00000000..58485aa4 --- /dev/null +++ b/backend/src/services/search/queryParser.test.js @@ -0,0 +1,225 @@ +import { describe, it, expect } from 'vitest'; +import { + parseQuery, + resolveSearchFolderScope, + shouldExcludeTrashFromSearch, +} from './queryParser.js'; + +const NOW = new Date('2026-07-16T00:00:00.000Z'); + +describe('parseQuery — preserved grammar', () => { + it('treats bare words as free-text terms', () => { + const { filters, terms } = parseQuery('hello world'); + expect(filters).toEqual([]); + expect(terms).toEqual([ + { value: 'hello', negate: false }, + { value: 'world', negate: false }, + ]); + }); + + it('extracts positive operators and lowercases their values', () => { + const { filters, terms } = parseQuery('from:Amazon subject:Invoice hello'); + expect(filters).toEqual([ + { key: 'from', value: 'amazon', negate: false }, + { key: 'subject', value: 'invoice', negate: false }, + ]); + expect(terms).toEqual([{ value: 'hello', negate: false }]); + }); + + it('supports quoted operator values with spaces', () => { + const { filters } = parseQuery('from:"John Smith" report'); + expect(filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); + }); + + it('negates an operator when prefixed with -', () => { + const { filters } = parseQuery('-from:Smith report'); + expect(filters).toEqual([{ key: 'from', value: 'smith', negate: true }]); + }); + + it('negates a free-text term when prefixed with -', () => { + const { terms } = parseQuery('report -invoice'); + expect(terms).toEqual([ + { value: 'report', negate: false }, + { value: 'invoice', negate: true }, + ]); + }); + + it('preserves repeated and mixed positive/negative operators', () => { + const { filters } = parseQuery('from:alice -from:bob is:unread'); + expect(filters).toEqual([ + { key: 'from', value: 'alice', negate: false }, + { key: 'from', value: 'bob', negate: true }, + { key: 'is', value: 'unread', negate: false }, + ]); + }); + + it('ignores a lone - so it is not treated as a negated empty term', () => { + const { terms } = parseQuery('report - draft'); + expect(terms).toEqual([ + { value: 'report', negate: false }, + { value: 'draft', negate: false }, + ]); + }); + + it('returns empty structures for blank input', () => { + expect(parseQuery('')).toEqual({ filters: [], terms: [], unsupported: [], errors: [] }); + expect(parseQuery(' ')).toEqual({ filters: [], terms: [], unsupported: [], errors: [] }); + }); + + it('parses in:all and named folders', () => { + expect(parseQuery('in:all invoice').filters).toEqual([ + { key: 'in', value: 'all', negate: false }, + ]); + expect(parseQuery('in:Sent proposal').filters).toEqual([ + { key: 'in', value: 'sent', negate: false }, + ]); + }); +}); + +describe('parseQuery — new msgvault operators', () => { + it('adds cc: as a real filter', () => { + const { filters } = parseQuery('cc:boss@corp.com report'); + expect(filters).toEqual([{ key: 'cc', value: 'boss@corp.com', negate: false }]); + }); + + it('maps newer_than: to an after: date and older_than: to a before: date', () => { + const newer = parseQuery('newer_than:2w', { now: NOW }).filters; + expect(newer[0].key).toBe('after'); + expect(newer[0].value.startsWith('2026-07-02')).toBe(true); + + const older = parseQuery('older_than:7d', { now: NOW }).filters; + expect(older[0].key).toBe('before'); + expect(older[0].value.startsWith('2026-07-09')).toBe(true); + }); + + it('records larger:/smaller:/bcc:/label:/l: as recognized-but-unsupported (never widened)', () => { + const { filters, unsupported } = parseQuery('larger:5M bcc:x@y.com label:work l:home smaller:100K'); + expect(filters).toEqual([]); // none of these silently become predicates + expect(unsupported).toEqual([ + { key: 'larger', token: 'larger:5m' }, + { key: 'bcc', token: 'bcc:x@y.com' }, + { key: 'label', token: 'label:work' }, + { key: 'label', token: 'l:home' }, + { key: 'smaller', token: 'smaller:100k' }, + ]); + }); + + it('records a malformed typed value as an error, not a filter', () => { + const { errors, unsupported } = parseQuery('larger:5X older_than:3q'); + expect(unsupported).toEqual([]); + expect(errors).toHaveLength(2); + expect(errors[0]).toContain('larger'); + expect(errors[1]).toContain('older_than'); + }); +}); + +describe('parseQuery — bare quoted phrases (Wave D Fix 4, msgvault tokenize parity)', () => { + it('treats a double-quoted phrase as ONE term without the quote chars (port "quoted phrase")', () => { + expect(parseQuery('"hello world"').terms).toEqual([{ value: 'hello world', negate: false }]); + }); + + it('mixes phrases with operators and bare words in order (port "mixed operators and text")', () => { + const { filters, terms } = parseQuery('from:alice@example.com "meeting notes" urgent'); + expect(filters).toEqual([{ key: 'from', value: 'alice@example.com', negate: false }]); + expect(terms).toEqual([ + { value: 'meeting notes', negate: false }, + { value: 'urgent', negate: false }, + ]); + }); + + it('keeps colons inside a phrase from becoming operators (port QuotedPhrasesWithColons)', () => { + expect(parseQuery('"foo:bar"').terms).toEqual([{ value: 'foo:bar', negate: false }]); + expect(parseQuery('"meeting at 10:30"').terms).toEqual([{ value: 'meeting at 10:30', negate: false }]); + expect(parseQuery('"check http://example.com"').terms).toEqual([{ value: 'check http://example.com', negate: false }]); + expect(parseQuery('"a:b:c:d"').terms).toEqual([{ value: 'a:b:c:d', negate: false }]); + }); + + it('parses a colon phrase alongside a real operator (port "quoted colon phrase mixed with real operator")', () => { + const { filters, terms } = parseQuery('from:alice@example.com "subject:not an operator"'); + expect(filters).toEqual([{ key: 'from', value: 'alice@example.com', negate: false }]); + expect(terms).toEqual([{ value: 'subject:not an operator', negate: false }]); + }); + + it('parses a leading phrase before an operator (port "operator followed by quoted colon phrase")', () => { + const { filters, terms } = parseQuery('"re: meeting notes" from:bob@example.com'); + expect(filters).toEqual([{ key: 'from', value: 'bob@example.com', negate: false }]); + expect(terms).toEqual([{ value: 're: meeting notes', negate: false }]); + }); + + it('accepts single-quoted phrases (msgvault tokenize takes both quote chars)', () => { + expect(parseQuery("'hello world'").terms).toEqual([{ value: 'hello world', negate: false }]); + }); + + it('never starts a phrase at an apostrophe INSIDE a word', () => { + expect(parseQuery("d'Angelo report").terms).toEqual([ + { value: "d'Angelo", negate: false }, + { value: 'report', negate: false }, + ]); + }); + + it('unescapes backslash-escaped quotes inside a phrase (msgvault unescapeQuotedValue)', () => { + expect(parseQuery('"say \\"hi\\" now"').terms).toEqual([{ value: 'say "hi" now', negate: false }]); + expect(parseQuery('"a\\\\b"').terms).toEqual([{ value: 'a\\b', negate: false }]); + }); + + it('negates a phrase with a leading -', () => { + expect(parseQuery('report -"weekly digest"').terms).toEqual([ + { value: 'report', negate: false }, + { value: 'weekly digest', negate: true }, + ]); + }); + + it('drops an empty phrase and leaves op:"value" quoting to the operator grammar', () => { + expect(parseQuery('""').terms).toEqual([]); + expect(parseQuery('from:"John Smith"').filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); + expect(parseQuery('from:"John Smith"').terms).toEqual([]); + }); +}); + +describe('parseQuery — before:/after: validation (Wave D Fix 5)', () => { + it('records an invalid before:/after: value as an error, never a filter (no silent widening)', () => { + const { filters, errors } = parseQuery('before:notadate after:2025-99-99 invoice'); + expect(filters).toEqual([]); + expect(errors).toEqual([ + 'invalid value "notadate" for before: — expected a date like YYYY-MM-DD', + 'invalid value "2025-99-99" for after: — expected a date like YYYY-MM-DD', + ]); + }); + + it('keeps valid dates as filters (msgvault parseDate accepts several forms)', () => { + const { filters, errors } = parseQuery('after:2025-01-02 before:2025/03/04'); + expect(errors).toEqual([]); + expect(filters.map((f) => f.key)).toEqual(['after', 'before']); + }); +}); + +describe('resolveSearchFolderScope / shouldExcludeTrashFromSearch', () => { + it('scopes to the client folder param when no in: operator is present', () => { + const { filters } = parseQuery('subject:newsletter'); + expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ + folderScope: 'INBOX', + folderFuzzy: false, + }); + }); + + it('lets in: override the client folder param', () => { + const { filters } = parseQuery('in:trash subject:newsletter'); + expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ + folderScope: 'trash', + folderFuzzy: true, + }); + }); + + it('flags all-folder searches for trash exclusion', () => { + const { filters } = parseQuery('subject:newsletter'); + const { folderScope } = resolveSearchFolderScope(filters); + expect(folderScope).toBeNull(); + expect(shouldExcludeTrashFromSearch(folderScope)).toBe(true); + }); + + it('keeps explicit folder searches eligible to find trash', () => { + const { filters } = parseQuery('in:trash subject:newsletter'); + const { folderScope } = resolveSearchFolderScope(filters); + expect(shouldExcludeTrashFromSearch(folderScope)).toBe(false); + }); +}); diff --git a/backend/src/services/search/searchService.js b/backend/src/services/search/searchService.js new file mode 100644 index 00000000..49e6114d --- /dev/null +++ b/backend/src/services/search/searchService.js @@ -0,0 +1,43 @@ +import { query } from '../db.js'; +import { searchLexical } from './lexicalRepo.js'; +import { resolveSearchFolderScope } from './queryParser.js'; + +function clampLimit(limit) { + return Math.max(1, Math.min(parseInt(limit) || 50, 200)); +} + +export async function search(request) { + const { userId, accountId, parsed, folderParam = '', limit = 50, offset = 0 } = request; + + const cap = clampLimit(limit); + const off = Math.max(0, parseInt(offset) || 0); + const emptyPage = { offset: off, limit: cap, hasMore: false }; + + const accountsResult = await query( + 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId] + ); + let accountIds = accountsResult.rows.map(r => r.id); + if (!accountIds.length) return { messages: [], mode: 'lexical', page: emptyPage }; + + // Optional single-account narrowing, only within the authenticated user's scope. + if (accountId && accountIds.includes(accountId)) accountIds = [accountId]; + + const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, folderParam); + + // D5: a free-text search (≥1 positive, non-trivial term) ranks by relevance; + // a filter-only search stays date-ordered. + const hasPositiveText = parsed.terms.some(t => !t.negate && t.value.length >= 2); + const ordering = hasPositiveText ? 'relevance' : 'date'; + + const { rows, hasCondition } = await searchLexical(query, { + parsed, accountIds, folderScope, folderFuzzy, ordering, limit: cap, offset: off, + }); + if (!hasCondition) return { messages: [], mode: 'lexical', page: emptyPage }; + + return { + messages: rows, + mode: 'lexical', + page: { offset: off, limit: cap, hasMore: rows.length === cap }, + }; +} diff --git a/backend/src/services/search/searchService.test.js b/backend/src/services/search/searchService.test.js new file mode 100644 index 00000000..4735be64 --- /dev/null +++ b/backend/src/services/search/searchService.test.js @@ -0,0 +1,67 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('./lexicalRepo.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, searchLexical: vi.fn() }; +}); + +import { query } from '../db.js'; +import { searchLexical } from './lexicalRepo.js'; +import { search } from './searchService.js'; + +beforeEach(() => { + query.mockReset(); + searchLexical.mockReset(); +}); + +function withAccounts(ids) { + query.mockResolvedValueOnce({ rows: ids.map(id => ({ id })) }); +} + +describe('searchService.search', () => { + it('returns an empty shaped result when the user has no enabled accounts', async () => { + withAccounts([]); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(res).toEqual({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + expect(searchLexical).not.toHaveBeenCalled(); + }); + + it('scopes to a single account only when it belongs to the user', async () => { + withAccounts(['a1', 'a2']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', accountId: 'a2', parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a2']); + }); + + it('chooses relevance ordering for free text and date ordering for filter-only queries', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].ordering).toBe('relevance'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] } }); + expect(searchLexical.mock.calls[0][1].ordering).toBe('date'); + }); + + it('shapes rows into messages + page, clamping the limit to 200 and flagging hasMore on a full page', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: new Array(200).fill({ id: 'm' }), hasCondition: true }); + const res = await search({ userId: 'u1', limit: 9999, offset: 40, parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(res.messages).toHaveLength(200); + expect(res.mode).toBe('lexical'); + expect(res.page).toEqual({ offset: 40, limit: 200, hasMore: true }); + expect(searchLexical.mock.calls[0][1].limit).toBe(200); + }); + + it('returns empty when there is no real search condition', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: false }); + const res = await search({ userId: 'u1', parsed: { filters: [{ key: 'in', value: 'inbox', negate: false }], terms: [] } }); + expect(res.messages).toEqual([]); + expect(res.page.hasMore).toBe(false); + }); +}); diff --git a/backend/src/testSupport/testAccount.js b/backend/src/testSupport/testAccount.js new file mode 100644 index 00000000..88b0afc3 --- /dev/null +++ b/backend/src/testSupport/testAccount.js @@ -0,0 +1,19 @@ +// Shared IT/dev-script fixture: seed and tear down a throwaway user + email account +// against a real DB. `db` is anything with .query(text, params) — a pg.Client, a +// pooled client, or the app pool. Deleting the user cascades to email_accounts → +// messages (ON DELETE CASCADE), so cleanupAccount removes everything in one step. +import { randomUUID } from 'crypto'; + +export async function seedAccount(db, label = 'it') { + const u = await db.query('INSERT INTO users (username) VALUES ($1) RETURNING id', [`${label}-${randomUUID()}`]); + const userId = u.rows[0].id; + const a = await db.query( + 'INSERT INTO email_accounts (user_id, name, email_address) VALUES ($1, $2, $3) RETURNING id', + [userId, label, `${label}-${randomUUID()}@example.com`], + ); + return { userId, accountId: a.rows[0].id }; +} + +export async function cleanupAccount(db, userId) { + if (userId) await db.query('DELETE FROM users WHERE id = $1', [userId]); +} diff --git a/frontend/src/components/MessageList.jsx b/frontend/src/components/MessageList.jsx index efb16df3..f4665182 100644 --- a/frontend/src/components/MessageList.jsx +++ b/frontend/src/components/MessageList.jsx @@ -2171,7 +2171,9 @@ export default function MessageList() { const showInboxIcon = !isUnified && selectedFolder === 'INBOX' && !searchQuery.trim(); const label = searchQuery.trim() - ? `Search: "${searchQuery}"` + // No quotes around the query: when the header truncates with a CSS ellipsis + // the closing quote would be lost, stranding an unbalanced opening one. + ? `Search: ${searchQuery}` : isUnified ? t('sidebar.allInboxes') : selectedFolder; // Non-INBOX folders omitted: byAccount is account-total, not folder-specific, so it would mislead. @@ -2246,7 +2248,7 @@ export default function MessageList() { - ) : label} + ) : {label}} {headerUnread > 0 && !searchQuery.trim() && ( - ) : label} + ) : {label}}
{messagesTotal > 0 && !searchQuery && ( diff --git a/frontend/src/utils/api.search.test.js b/frontend/src/utils/api.search.test.js new file mode 100644 index 00000000..a237a775 --- /dev/null +++ b/frontend/src/utils/api.search.test.js @@ -0,0 +1,18 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { api } from './api.js'; + +test('search sends lexical pagination and folder parameters', async () => { + let seen; + const orig = globalThis.fetch; + globalThis.fetch = async (url) => { seen = url; return { ok: true, json: async () => ({ messages: [] }) }; }; + try { + await api.search('quarterly report', 'a1', { offset: 50, limit: 25, folder: 'INBOX' }); + } finally { globalThis.fetch = orig; } + const url = new URL(seen, 'http://localhost'); + assert.equal(url.searchParams.get('q'), 'quarterly report'); + assert.equal(url.searchParams.get('accountId'), 'a1'); + assert.equal(url.searchParams.get('offset'), '50'); + assert.equal(url.searchParams.get('limit'), '25'); + assert.equal(url.searchParams.get('folder'), 'INBOX'); +}); From 741c142e85be31701be2604065670e4c7013f856 Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:41:05 -0700 Subject: [PATCH 2/4] feat(search): optional semantic embeddings with hybrid ranking (pgvector) Explicit opt-in embeddings pipeline against any OpenAI-compatible /v1/embeddings endpoint (key encrypted at rest, host-validated, masked on read). Fingerprinted generations (config change => clean rebuild), a crash-safe fill worker, per-dimension HNSW partial indexes, re-embedding of late-arriving bodies. Query side adds hybrid BM25+ANN reciprocal-rank fusion behind an in-input Semantic toggle; lexical stays the default, with silent fallback while the index builds. Settings UI with explicit privacy copy and live Test/Build progress, in all 7 locales. Migrations 0038-0039. Includes the search-eval harness that produced the published quality numbers, and its explain/total diagnostics seams. Infra: compose moves postgres:16-alpine -> pgvector/pgvector:pg16 (same PG16 major). Run REINDEX DATABASE once after the first boot: the musl -> glibc collation change can misorder existing text btree indexes; the app logs a loud warning when it detects this. Split 2/3 of #283, stacked on the weighted-FTS PR. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + README.md | 8 + backend/migrations/0038_embed_watermark.sql | 44 ++ .../migrations/0039_embed_pending_index.sql | 17 + .../scripts/bench-last-modified-trigger.js | 45 ++ backend/scripts/search-eval.mjs | 453 +++++++++++ backend/scripts/vector-probe.js | 38 + backend/src/index.js | 18 +- .../routes/ai.buildEmbeddingsConfig.test.js | 32 + backend/src/routes/ai.js | 52 +- backend/src/routes/aiEmbeddings.js | 148 ++++ backend/src/routes/aiEmbeddings.test.js | 149 ++++ backend/src/routes/search.js | 5 + backend/src/routes/search.test.js | 35 + backend/src/services/bodyBackfill.js | 5 +- backend/src/services/embeddings/__testdb__.js | 60 ++ .../services/embeddings/activeGeneration.js | 44 ++ .../embeddings/activeGeneration.test.js | 29 + backend/src/services/embeddings/chunk.js | 82 ++ backend/src/services/embeddings/chunk.test.js | 134 ++++ backend/src/services/embeddings/client.js | 99 +++ .../src/services/embeddings/client.test.js | 119 +++ backend/src/services/embeddings/config.js | 58 ++ .../src/services/embeddings/config.test.js | 53 ++ .../src/services/embeddings/embedRunLock.js | 18 + .../services/embeddings/embedRunLock.test.js | 20 + .../embeddings/embedScan.integration.test.js | 63 ++ .../embeddings/generations.chunkCount.test.js | 29 + .../generations.integration.test.js | 60 ++ .../src/services/embeddings/generations.js | 145 ++++ .../generations.summaryFields.test.js | 38 + backend/src/services/embeddings/hybrid.js | 135 ++++ .../src/services/embeddings/hybrid.test.js | 177 +++++ .../migration0038.integration.test.js | 81 ++ .../postStampReembed.integration.test.js | 62 ++ backend/src/services/embeddings/preprocess.js | 105 +++ .../services/embeddings/preprocess.test.js | 87 +++ .../embeddings/rankingQuality.test.js | 83 ++ backend/src/services/embeddings/scheduler.js | 91 +++ .../src/services/embeddings/scheduler.test.js | 118 +++ .../stampSkipped.integration.test.js | 76 ++ .../embeddings/testSupport.js} | 0 .../src/services/embeddings/vectorErrors.js | 5 + .../vectorSchema.integration.test.js | 45 ++ .../embeddings/vectorSchema.unit.test.js | 32 + .../embeddings/vectorStore.fused.test.js | 296 +++++++ .../embeddings/vectorStore.fusedSql.test.js | 112 +++ .../vectorStore.integration.test.js | 85 ++ .../src/services/embeddings/vectorStore.js | 731 ++++++++++++++++++ .../embeddings/vectorStore.unit.test.js | 92 +++ .../embeddings/worker.integration.test.js | 58 ++ backend/src/services/embeddings/worker.js | 352 +++++++++ .../src/services/embeddings/worker.test.js | 250 ++++++ backend/src/services/imapManager.js | 2 +- .../src/services/migrations.collation.test.js | 58 ++ backend/src/services/migrations.js | 67 ++ backend/src/services/search/lexicalRepo.js | 71 +- .../src/services/search/lexicalRepo.test.js | 14 +- .../services/search/lexicalRepo.total.test.js | 22 + .../src/services/search/migrations.test.js | 17 + backend/src/services/search/searchService.js | 136 +++- .../src/services/search/searchService.test.js | 265 ++++++- .../search/searchService.total.test.js | 15 + docker-compose.ghcr.yml | 2 +- docker-compose.yml | 2 +- frontend/src/components/AdminPanel.jsx | 190 ++++- frontend/src/components/MessageList.jsx | 155 +++- frontend/src/locales/de.json | 35 +- frontend/src/locales/en.json | 35 +- frontend/src/locales/es.json | 35 +- frontend/src/locales/fr.json | 35 +- frontend/src/locales/i18n.test.js | 5 + frontend/src/locales/it.json | 35 +- frontend/src/locales/ru.json | 35 +- frontend/src/locales/zhCN.json | 35 +- frontend/src/store/index.js | 5 + frontend/src/utils/api.js | 8 +- frontend/src/utils/api.search.test.js | 19 +- frontend/src/utils/embeddingsSettings.js | 101 +++ frontend/src/utils/embeddingsSettings.test.js | 99 +++ frontend/src/utils/searchMode.js | 80 ++ frontend/src/utils/searchMode.test.js | 87 +++ 82 files changed, 6731 insertions(+), 109 deletions(-) create mode 100644 backend/migrations/0038_embed_watermark.sql create mode 100644 backend/migrations/0039_embed_pending_index.sql create mode 100644 backend/scripts/bench-last-modified-trigger.js create mode 100644 backend/scripts/search-eval.mjs create mode 100644 backend/scripts/vector-probe.js create mode 100644 backend/src/routes/ai.buildEmbeddingsConfig.test.js create mode 100644 backend/src/routes/aiEmbeddings.js create mode 100644 backend/src/routes/aiEmbeddings.test.js create mode 100644 backend/src/services/embeddings/__testdb__.js create mode 100644 backend/src/services/embeddings/activeGeneration.js create mode 100644 backend/src/services/embeddings/activeGeneration.test.js create mode 100644 backend/src/services/embeddings/chunk.js create mode 100644 backend/src/services/embeddings/chunk.test.js create mode 100644 backend/src/services/embeddings/client.js create mode 100644 backend/src/services/embeddings/client.test.js create mode 100644 backend/src/services/embeddings/config.js create mode 100644 backend/src/services/embeddings/config.test.js create mode 100644 backend/src/services/embeddings/embedRunLock.js create mode 100644 backend/src/services/embeddings/embedRunLock.test.js create mode 100644 backend/src/services/embeddings/embedScan.integration.test.js create mode 100644 backend/src/services/embeddings/generations.chunkCount.test.js create mode 100644 backend/src/services/embeddings/generations.integration.test.js create mode 100644 backend/src/services/embeddings/generations.js create mode 100644 backend/src/services/embeddings/generations.summaryFields.test.js create mode 100644 backend/src/services/embeddings/hybrid.js create mode 100644 backend/src/services/embeddings/hybrid.test.js create mode 100644 backend/src/services/embeddings/migration0038.integration.test.js create mode 100644 backend/src/services/embeddings/postStampReembed.integration.test.js create mode 100644 backend/src/services/embeddings/preprocess.js create mode 100644 backend/src/services/embeddings/preprocess.test.js create mode 100644 backend/src/services/embeddings/rankingQuality.test.js create mode 100644 backend/src/services/embeddings/scheduler.js create mode 100644 backend/src/services/embeddings/scheduler.test.js create mode 100644 backend/src/services/embeddings/stampSkipped.integration.test.js rename backend/src/{testSupport/testAccount.js => services/embeddings/testSupport.js} (100%) create mode 100644 backend/src/services/embeddings/vectorErrors.js create mode 100644 backend/src/services/embeddings/vectorSchema.integration.test.js create mode 100644 backend/src/services/embeddings/vectorSchema.unit.test.js create mode 100644 backend/src/services/embeddings/vectorStore.fused.test.js create mode 100644 backend/src/services/embeddings/vectorStore.fusedSql.test.js create mode 100644 backend/src/services/embeddings/vectorStore.integration.test.js create mode 100644 backend/src/services/embeddings/vectorStore.js create mode 100644 backend/src/services/embeddings/vectorStore.unit.test.js create mode 100644 backend/src/services/embeddings/worker.integration.test.js create mode 100644 backend/src/services/embeddings/worker.js create mode 100644 backend/src/services/embeddings/worker.test.js create mode 100644 backend/src/services/migrations.collation.test.js create mode 100644 backend/src/services/search/lexicalRepo.total.test.js create mode 100644 backend/src/services/search/searchService.total.test.js create mode 100644 frontend/src/utils/embeddingsSettings.js create mode 100644 frontend/src/utils/embeddingsSettings.test.js create mode 100644 frontend/src/utils/searchMode.js create mode 100644 frontend/src/utils/searchMode.test.js diff --git a/.gitignore b/.gitignore index 97a8fe08..de9fce71 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ Thumbs.db .superpowers/ output/ + +# Search-eval query cache — paraphrases of real mail; the harness regenerates it diff --git a/README.md b/README.md index 0f5c1c04..4188a1bd 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,14 @@ docker compose up -d To pin to a specific version instead of `latest`, add `MAILFLOW_VERSION=1.9.0` to your `.env`. +> **Upgrading an existing install across the Postgres image change** (`postgres:16-alpine` → `pgvector/pgvector:pg16`): the new image uses a different C library (musl → glibc), which changes text collation order. Reindex once after the switch or text indexes can silently return wrong results — the backend also prints this warning at startup when it detects the mismatch: +> +> ```bash +> docker compose exec postgres psql -U mailflow -d mailflow \ +> -c 'REINDEX DATABASE "mailflow";' \ +> -c 'ALTER DATABASE "mailflow" REFRESH COLLATION VERSION;' +> ``` + --- ## Option B — Build from source diff --git a/backend/migrations/0038_embed_watermark.sql b/backend/migrations/0038_embed_watermark.sql new file mode 100644 index 00000000..ecf8214a --- /dev/null +++ b/backend/migrations/0038_embed_watermark.sql @@ -0,0 +1,44 @@ +-- Vector-substrate CAS columns + last_modified trigger (slice 04). +-- Extension-independent, fast DDL only. NO vector-typed DDL here — the +-- embeddings/index_generations/embed_watermark/embed_runs tables and the HNSW +-- index are created by ensureVectorSchema() at startup (README invariant). +-- Transactional migration (NOT -- no-transaction): the no-transaction runner +-- splits on ';' and would break the dollar-quoted function below. + +-- last_modified: content-change CAS token the embed worker compares to detect a +-- late-arriving body invalidating a stale subject-only embedding. NOT NULL DEFAULT +-- now() is metadata-only on PG16 (now() is STABLE → one stored missing-value, no +-- table rewrite). +ALTER TABLE messages ADD COLUMN IF NOT EXISTS last_modified TIMESTAMPTZ NOT NULL DEFAULT now(); + +-- embed_gen: the index generation this row is embedded under. NULL = needs embedding. +-- Plain BIGINT soft-stamp (no FK): a generation can be retired/deleted while stamps linger. +ALTER TABLE messages ADD COLUMN IF NOT EXISTS embed_gen BIGINT; + +-- Bump last_modified AND clear embed_gen when an embedding-input column changes. The +-- WHEN clause filters at the C level so unchanged-content UPDATEs (the hot re-sync path) +-- and stamp-only UPDATEs (the worker setting embed_gen) skip the function entirely. +-- Clearing embed_gen is what re-surfaces a late-arriving body: a row embedded +-- subject-only and stamped, whose body later lands (phase-2 drainer / on-open fetch), +-- has its stamp cleared here so the NULL-only embed scan re-finds it and the idempotent +-- upsert replaces the stale chunks. The CAS only guards the read→stamp window; this +-- trigger covers the post-stamp case (Mailflow's late-arriving bodies — msgvault's rows +-- are immutable after ingest, so it never needed this). Together with createGeneration's +-- stamp-reset (which handles generation rebuilds: unchanged content, new fingerprint), +-- the invariant is exact: embed_gen IS NULL ⟺ the row needs embedding. +CREATE OR REPLACE FUNCTION messages_bump_last_modified() RETURNS trigger AS $$ +BEGIN + NEW.last_modified := now(); + NEW.embed_gen := NULL; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_messages_last_modified ON messages; +CREATE TRIGGER trg_messages_last_modified + BEFORE UPDATE ON messages + FOR EACH ROW + WHEN (NEW.subject IS DISTINCT FROM OLD.subject + OR NEW.body_text IS DISTINCT FROM OLD.body_text + OR NEW.body_html IS DISTINCT FROM OLD.body_html) + EXECUTE FUNCTION messages_bump_last_modified(); diff --git a/backend/migrations/0039_embed_pending_index.sql b/backend/migrations/0039_embed_pending_index.sql new file mode 100644 index 00000000..abf9a874 --- /dev/null +++ b/backend/migrations/0039_embed_pending_index.sql @@ -0,0 +1,17 @@ +-- no-transaction +-- Partial index over the embed-scan's steady-state hot predicate. Once a generation +-- reaches full coverage, the only live rows still needing work are newly-arrived +-- messages with embed_gen IS NULL, so the 60s scheduler scan (scanForEmbedding) should +-- touch O(pending), not O(mailbox). Built CONCURRENTLY (hence -- no-transaction) so it +-- never blocks boot on a large messages table; extension-independent and cheap to +-- maintain (only the sparse NULL set is indexed). Idempotent (IF NOT EXISTS) because a +-- crash before the schema_migrations INSERT retries the migration. +-- +-- DROP ... IF EXISTS before the CREATE: a cancelled or crashed CREATE INDEX +-- CONCURRENTLY leaves an INVALID index under this name, and plain IF NOT EXISTS would +-- then silently skip the create on retry — recording the migration as done while the +-- embed scan stays unindexed forever. This file only re-runs after such a failure, so +-- the index is either absent (drop is a no-op) or invalid (drop clears the dead stub). +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending + ON messages (id) WHERE embed_gen IS NULL; diff --git a/backend/scripts/bench-last-modified-trigger.js b/backend/scripts/bench-last-modified-trigger.js new file mode 100644 index 00000000..4593994b --- /dev/null +++ b/backend/scripts/bench-last-modified-trigger.js @@ -0,0 +1,45 @@ +// Benchmark the last_modified trigger overhead on the hot re-sync UPSERT. +// Usage: node scripts/bench-last-modified-trigger.js [rows=5000] [iters=20000] +// Requires DB_* env pointing at a migrated Postgres (trigger present). +import { pool } from '../src/services/db.js'; +import { seedAccount, cleanupAccount } from '../src/services/embeddings/testSupport.js'; + +const ROWS = Number(process.argv[2]) || 5000; +const ITERS = Number(process.argv[3]) || 20000; + +const { accountId: acctId, userId } = await seedAccount(pool, 'bench'); +const ids = []; +for (let i = 0; i < ROWS; i++) { + const r = await pool.query(`INSERT INTO messages (account_id, uid, folder, subject, is_read) + VALUES ($1, $2, 'INBOX', 'bench', false) RETURNING id`, [acctId, 700000 + i]); + ids.push(r.rows[0].id); +} + +// Flag-only churn: toggles is_read (NOT an embedding-input column) so the trigger's +// WHEN clause should skip the function entirely. +async function churn() { + const t0 = process.hrtime.bigint(); + for (let i = 0; i < ITERS; i++) { + const id = ids[i % ids.length]; + await pool.query('UPDATE messages SET is_read = NOT is_read WHERE id = $1', [id]); + } + return Number(process.hrtime.bigint() - t0) / 1e6; // ms +} + +const withTrig = await churn(); +await pool.query('DROP TRIGGER IF EXISTS trg_messages_last_modified ON messages'); +const withoutTrig = await churn(); +// Restore the trigger. +await pool.query(`CREATE TRIGGER trg_messages_last_modified + BEFORE UPDATE ON messages FOR EACH ROW + WHEN (NEW.subject IS DISTINCT FROM OLD.subject OR NEW.body_text IS DISTINCT FROM OLD.body_text OR NEW.body_html IS DISTINCT FROM OLD.body_html) + EXECUTE FUNCTION messages_bump_last_modified()`); + +await cleanupAccount(pool, userId); +await pool.end(); + +const regression = ((withTrig - withoutTrig) / withoutTrig) * 100; +console.log(`with trigger: ${withTrig.toFixed(0)} ms`); +console.log(`without trigger: ${withoutTrig.toFixed(0)} ms`); +console.log(`regression: ${regression.toFixed(1)}% (gate: < 10%)`); +process.exit(regression < 10 ? 0 : 2); diff --git a/backend/scripts/search-eval.mjs b/backend/scripts/search-eval.mjs new file mode 100644 index 00000000..a4ce1cb9 --- /dev/null +++ b/backend/scripts/search-eval.mjs @@ -0,0 +1,453 @@ +#!/usr/bin/env node +// search-eval.mjs — IR relevance eval harness for Mailflow's lexical/vector/hybrid search. +// +// WHY THIS EXISTS +// A user reported they "can't tell the difference" between search modes on their real +// 18k-message mailbox and suspected hybrid might be *worse* than pure vector. Phase 4's +// rankingQuality.test.js proved hybrid never LOSES a lexical hit on a synthetic 16-doc +// fixture; this harness is the complementary real-corpus measurement: it builds labeled +// query sets FROM the live mailbox, runs them through the deployed REST API in all three +// modes, and reports the standard IR metrics (Recall@1/5/20, MRR@20) plus cross-mode +// result overlap and hybrid explain-score composition — so the "is hybrid working?" +// question is answered with numbers instead of vibes. +// +// DESIGN +// * No new deps. Node >=18 global fetch; psql sampling via `docker exec` (the same +// read-only path the eval brief documents); explain scores via one `docker exec ... +// node` pass against the in-container searchService seam (the deployed REST route does +// not plumb `explain` through, but the seam it calls does). +// * Deterministic. Message sampling is ordered by md5(id || seed); paraphrase queries +// are cached to JSON keyed by {messageId, promptVersion}, so reruns are free & stable. +// * Two query sets, both with a single ground-truth message id: +// KEYWORD — 2 distinctive subject tokens (lexical should win/tie). +// PARAPHRASE — an LLM rewrites the email's topic WITHOUT its distinctive keywords +// (semantic recall test; degrades to hand-written queries if the LLM +// is unavailable). +// +// USAGE +// EVAL_USER='admin@example.com' \ # login username (no default) +// EVAL_PASS='' \ +// OPENAI_API_KEY="$(cat /path/to/key)" \ # or EVAL_KEY_FILE=/path/to/key +// node backend/scripts/search-eval.mjs +// +// Reruns after the first are offline for query generation (cache hit) but still hit the +// live REST API for the actual searches. Set EVAL_SKIP_EXPLAIN=1 to skip the in-container +// diagnostic. All knobs are env vars (see CFG below). No secrets are written to disk. + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const EVAL_DIR = resolve(__dirname, '..', '..', 'specs', 'search-overhaul', 'evals'); + +const PROMPT_VERSION = 'v1'; // bump to invalidate the paraphrase cache + +function readKeyFile() { + const f = process.env.EVAL_KEY_FILE; + if (f && existsSync(f)) return readFileSync(f, 'utf8'); + return ''; +} + +const CFG = { + baseUrl: process.env.EVAL_BASE_URL || 'http://127.0.0.1:8087', + loginUser: process.env.EVAL_USER || '', // required for a fresh login; never defaulted + loginPass: process.env.EVAL_PASS || '', // required for a fresh login; never defaulted + pgContainer: process.env.EVAL_PG_CONTAINER || 'mailflow-postgres', + pgUser: process.env.EVAL_PG_USER || 'mailflow', + pgDb: process.env.EVAL_PG_DB || 'mailflow', + backendContainer: process.env.EVAL_BACKEND_CONTAINER || 'mailflow-backend', + openaiBase: process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1', + openaiModel: process.env.OPENAI_MODEL || 'gpt-4o-mini', + openaiKey: (process.env.OPENAI_API_KEY || readKeyFile()).trim(), + seed: process.env.EVAL_SEED || 'mailflow-eval-2026-07-16', + nKeyword: Number(process.env.EVAL_N_KEYWORD || 15), + nParaphrase: Number(process.env.EVAL_N_PARAPHRASE || 25), + limit: 20, + spacingMs: Number(process.env.EVAL_REQUEST_SPACING_MS || 3300), // stay just under 20/min + llmSpacingMs: Number(process.env.EVAL_LLM_SPACING_MS || 3000), // pace flaky low-tier keys + generateOnly: process.env.EVAL_GENERATE_ONLY === '1', // populate the cache, then exit + skipExplain: process.env.EVAL_SKIP_EXPLAIN === '1', + cacheFile: resolve(EVAL_DIR, 'query-cache.json'), + outFile: process.env.EVAL_OUT || resolve(EVAL_DIR, `results-${new Date().toISOString().slice(0, 10)}.json`), +}; + +const MODES = ['lexical', 'vector', 'hybrid']; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// ── DB sampling (docker exec psql, read-only) ────────────────────────────────────────── +const FSEP = '\x1f'; +function psql(sql) { + const out = execFileSync( + 'docker', + ['exec', CFG.pgContainer, 'psql', '-U', CFG.pgUser, '-d', CFG.pgDb, '-tAF', FSEP, '-c', sql], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + return out.split('\n').filter((l) => l.length > 0).map((l) => l.split(FSEP)); +} + +function resolveScope() { + const rows = psql( + `SELECT u.id, (SELECT string_agg(id::text, ',') FROM email_accounts WHERE user_id=u.id AND enabled=true) + FROM users u WHERE u.username='${CFG.loginUser.replace(/'/g, "''")}'`, + ); + if (!rows.length) throw new Error(`no user ${CFG.loginUser}`); + const [userId, accts] = rows[0]; + return { userId, accountIds: (accts || '').split(',').filter(Boolean) }; +} + +// Sanitize subject/snippet in SQL so newlines/separators never break TSV row parsing. +const CLEAN = (col) => `regexp_replace(coalesce(${col},''), '\\s+', ' ', 'g')`; + +function sampleMessages({ salt, where, n }) { + const acctList = SCOPE.accountIds.map((a) => `'${a}'`).join(','); + return psql( + `SELECT id, ${CLEAN('subject')}, ${CLEAN('snippet')}, coalesce(from_name,'') + FROM messages m + WHERE m.is_deleted=false AND m.account_id IN (${acctList}) AND ${where} + ORDER BY md5(m.id::text || '${salt.replace(/'/g, "''")}') + LIMIT ${n}`, + ).map(([id, subject, snippet, fromName]) => ({ id, subject, snippet, fromName })); +} + +// ── KEYWORD query derivation ─────────────────────────────────────────────────────────── +const STOP = new Set([ + 'the', 'and', 'for', 'you', 'your', 'with', 'from', 'this', 'that', 'have', 'has', 'was', + 'are', 'will', 'not', 'new', 'can', 'all', 'our', 'out', 'get', 'now', 'about', 'been', + 'account', 'email', 'please', 'update', 'updated', 'notification', 're', 'fwd', 'fw', + 'order', 'invoice', 'payment', 'confirm', 'confirmation', 'reminder', 'receipt', 'alert', + 'security', 'verify', 'verification', 'here', 'more', 'just', 'been', 'they', 'them', + 'default', 'routing', 'transaction', 'needs', 'sent', 'may', 'copy', 'left', 'comment', +]); +function keywordTokens(subject) { + const toks = (subject.toLowerCase().match(/[a-z][a-z0-9'-]{3,}/g) || []) + .map((t) => t.replace(/^[-']+|[-']+$/g, '')) + .filter((t) => t.length >= 4 && !STOP.has(t)); + const uniq = [...new Set(toks)].sort((a, b) => b.length - a.length); + return uniq.slice(0, 2); +} + +function buildKeywordSet() { + const cands = sampleMessages({ + salt: `${CFG.seed}:kw`, + where: `length(coalesce(m.subject,'')) BETWEEN 10 AND 160`, + n: CFG.nKeyword * 4, + }); + const set = []; + for (const m of cands) { + if (set.length >= CFG.nKeyword) break; + const toks = keywordTokens(m.subject); + if (toks.length < 2) continue; + set.push({ id: m.id, set: 'keyword', query: toks.join(' '), subject: m.subject }); + } + return set; +} + +// ── PARAPHRASE query generation (LLM, cached) ────────────────────────────────────────── +async function openaiParaphrase(subject, snippet) { + const body = { + model: CFG.openaiModel, + temperature: 0.3, + max_tokens: 40, + messages: [ + { role: 'system', content: 'You write realistic email-search queries the way a busy person would type them months later from memory.' }, + { role: 'user', content: + `Email subject: ${subject}\nEmail preview: ${snippet}\n\n` + + 'Write ONE short natural-language search query (4-9 words) that a person might type to re-find THIS email later. ' + + 'Describe its topic or purpose in everyday words. Do NOT reuse the distinctive names, brands, product names, codes, or rare keywords from the subject — paraphrase them with common synonyms. ' + + 'Output only the query text, no quotes, no punctuation at the end.' }, + ], + }; + // Low-tier keys rate-limit bursts (429), sometimes even reporting it as a quota error; + // back off (3s,6s,12s,24s) and retry before giving up to the hand-written fallback. + const backoff = [3000, 6000, 12000, 24000]; + for (let attempt = 0; attempt <= backoff.length; attempt++) { + const res = await fetch(`${CFG.openaiBase}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${CFG.openaiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (res.status === 429 && attempt < backoff.length) { await sleep(backoff[attempt]); continue; } + if (!res.ok) throw new Error(`openai ${res.status}: ${(await res.text()).slice(0, 120)}`); + const j = await res.json(); + return (j.choices?.[0]?.message?.content || '').trim().replace(/^["']|["']$/g, ''); + } + throw new Error('openai 429: retries exhausted'); +} + +// Hand-written fallbacks, only used if the LLM is unavailable AND nothing is cached. +function fallbackParaphrase(subject) { + const t = keywordTokens(subject); + return t.length ? `email about ${t.join(' and ')}` : 'that email i got recently'; +} + +async function buildParaphraseSet(cache) { + const cands = sampleMessages({ + salt: `${CFG.seed}:para`, + where: `length(coalesce(m.subject,'')) BETWEEN 12 AND 160 AND m.snippet IS NOT NULL AND length(m.snippet) >= 40`, + n: CFG.nParaphrase, + }); + const set = []; + let generated = 0; let usedFallback = 0; + for (const m of cands) { + const key = `${m.id}:${PROMPT_VERSION}`; + let query = cache[key]?.query; + if (!query) { + if (CFG.openaiKey) { + try { + query = await openaiParaphrase(m.subject, m.snippet); + generated++; + await sleep(CFG.llmSpacingMs); // pace LLM calls to avoid burst rate-limits on low-tier keys + } catch (err) { + console.warn(` paraphrase LLM failed for ${m.id.slice(0, 8)}: ${err.message}`); + } + } + if (!query) { query = fallbackParaphrase(m.subject); usedFallback++; } + cache[key] = { query, subject: m.subject, source: generated && query !== fallbackParaphrase(m.subject) ? 'llm' : 'fallback' }; + } + set.push({ id: m.id, set: 'paraphrase', query, subject: m.subject }); + } + // Tally the provenance of every query actually used (cache may hold llm/handwritten/fallback). + const sourceCounts = {}; + for (const m of cands) { + const s = cache[`${m.id}:${PROMPT_VERSION}`]?.source || 'unknown'; + sourceCounts[s] = (sourceCounts[s] || 0) + 1; + } + if (generated || usedFallback) console.log(` paraphrases: ${generated} generated, ${usedFallback} fallback, ${set.length - generated - usedFallback} cached`); + console.log(` paraphrase sources: ${JSON.stringify(sourceCounts)}`); + return { set, usedFallback, sourceCounts }; +} + +// ── Live REST search (session + 429-aware throttling) ────────────────────────────────── +let COOKIE = ''; +async function login() { + if (!CFG.loginUser) throw new Error('EVAL_USER is required for a fresh login (no default; local test creds)'); + if (!CFG.loginPass) throw new Error('EVAL_PASS is required for a fresh login (no default; local test creds)'); + const res = await fetch(`${CFG.baseUrl}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify({ username: CFG.loginUser, password: CFG.loginPass }), + }); + if (!res.ok) throw new Error(`login ${res.status}`); + const set = res.headers.getSetCookie?.() || []; + COOKIE = set.map((c) => c.split(';')[0]).join('; '); + if (!COOKIE) throw new Error('login returned no session cookie'); +} + +async function restSearch(query, mode) { + const url = `${CFG.baseUrl}/api/search/?q=${encodeURIComponent(query)}&mode=${mode}&limit=${CFG.limit}`; + for (let attempt = 0; attempt < 6; attempt++) { + const res = await fetch(url, { headers: { Cookie: COOKIE, 'X-Requested-With': 'XMLHttpRequest' } }); + if (res.status === 429) { + const retry = Number(res.headers.get('retry-after') || 5); + console.log(` 429 rate-limited, sleeping ${retry + 1}s`); + await sleep((retry + 1) * 1000); + continue; + } + if (!res.ok) throw new Error(`search ${res.status} for "${query}" (${mode})`); + return res.json(); + } + throw new Error(`search gave up after retries: "${query}" (${mode})`); +} + +// ── Metrics ──────────────────────────────────────────────────────────────────────────── +function rankOf(ids, targetId) { + const i = ids.indexOf(targetId); + return i < 0 ? null : i + 1; // 1-indexed +} +function jaccard(a, b) { + const sa = new Set(a); const sb = new Set(b); + if (!sa.size && !sb.size) return 1; + let inter = 0; for (const x of sa) if (sb.has(x)) inter++; + return inter / (sa.size + sb.size - inter); +} +function summarize(ranks) { + const n = ranks.length; + const rec = (k) => ranks.filter((r) => r != null && r <= k).length / n; + const mrr = ranks.reduce((s, r) => s + (r != null && r <= 20 ? 1 / r : 0), 0) / n; + const found = ranks.filter((r) => r != null).length; + return { n, recall_at_1: rec(1), recall_at_5: rec(5), recall_at_20: rec(20), mrr_at_20: mrr, found }; +} + +// ── In-container explain diagnostic (single node pass) ───────────────────────────────── +function explainDiagnostic(queries) { + const payload = JSON.stringify(queries.map((q) => ({ id: q.id, q: q.query, set: q.set }))); + const script = ` +import { search } from './src/services/search/searchService.js'; +import { parseQuery } from './src/services/search/queryParser.js'; +const userId = process.env.EVAL_USER_ID; +const queries = JSON.parse(process.env.EVAL_Q); +const out = []; +for (const item of queries) { + const parsed = parseQuery(item.q); + const row = { id: item.id, q: item.q, set: item.set }; + for (const mode of ['hybrid', 'vector']) { + try { + const r = await search({ userId, parsed, mode, limit: 20, explain: true }); + const hits = r.messages || []; + const ids = hits.map((h) => h.id); + const gi = ids.indexOf(item.id); + row[mode] = { + fellBack: !!r.fellBack, mode: r.mode, pool_saturated: !!r.pool_saturated, n: hits.length, + n_bm25: hits.filter((h) => h.score && h.score.bm25 != null).length, + n_vector: hits.filter((h) => h.score && h.score.vector != null).length, + n_both: hits.filter((h) => h.score && h.score.bm25 != null && h.score.vector != null).length, + n_subject_boosted: hits.filter((h) => h.score && h.score.subject_boosted).length, + gt_rank: gi < 0 ? null : gi + 1, + gt_score: gi < 0 ? null : hits[gi].score, + }; + } catch (e) { row[mode] = { error: String(e.message || e) }; } + } + out.push(row); +} +process.stdout.write(JSON.stringify(out)); +`; + const raw = execFileSync( + 'docker', + ['exec', '-e', `EVAL_Q=${payload}`, '-e', `EVAL_USER_ID=${SCOPE.userId}`, + CFG.backendContainer, 'node', '--input-type=module', '-e', script], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + return JSON.parse(raw); +} + +// ── Main ─────────────────────────────────────────────────────────────────────────────── +let SCOPE; +async function main() { + if (!existsSync(EVAL_DIR)) mkdirSync(EVAL_DIR, { recursive: true }); + SCOPE = resolveScope(); + console.log(`user=${SCOPE.userId.slice(0, 8)} accounts=${SCOPE.accountIds.length}`); + + const cache = existsSync(CFG.cacheFile) ? JSON.parse(readFileSync(CFG.cacheFile, 'utf8')) : {}; + + console.log('building keyword set...'); + const keywordSet = buildKeywordSet(); + console.log(`building paraphrase set (LLM=${CFG.openaiKey ? 'on' : 'off'})...`); + const { set: paraphraseSet, usedFallback, sourceCounts } = await buildParaphraseSet(cache); + // cacheFile holds paraphrases of real mail — gitignored, regenerated on demand; never committed. + writeFileSync(CFG.cacheFile, JSON.stringify(cache, null, 2)); + if (CFG.generateOnly) { + console.log(`generate-only: cache written (${paraphraseSet.length} paraphrases, ${usedFallback} still fallback). Exiting.`); + return; + } + + const queries = [...keywordSet, ...paraphraseSet]; + console.log(`total queries: ${queries.length} (${keywordSet.length} keyword, ${paraphraseSet.length} paraphrase)`); + + console.log('logging in...'); + await login(); + + // Run every query in every mode against the live REST API. + const perQuery = []; + let done = 0; + for (const q of queries) { + const rec = { id: q.id, set: q.set, query: q.query, subject: q.subject, byMode: {} }; + for (const mode of MODES) { + const r = await restSearch(q.query, mode); + const ids = (r.messages || []).map((m) => m.id); + rec.byMode[mode] = { ids, rank: rankOf(ids, q.id), fellBack: !!r.fellBack, total: r.total }; + await sleep(CFG.spacingMs); + } + perQuery.push(rec); + done++; + if (done % 5 === 0) console.log(` ${done}/${queries.length} queries searched`); + } + + // Explain diagnostics (in-container seam). + let explain = []; + if (!CFG.skipExplain) { + console.log('collecting hybrid/vector explain scores (in-container)...'); + try { explain = explainDiagnostic(queries); } + catch (e) { console.warn(` explain diagnostic failed: ${e.message}`); } + } + const explainById = Object.fromEntries(explain.map((e) => [`${e.set}:${e.id}`, e])); + + // Aggregate metrics. + const sets = ['keyword', 'paraphrase', 'overall']; + const metrics = {}; + for (const s of sets) { + metrics[s] = {}; + const rows = perQuery.filter((r) => s === 'overall' || r.set === s); + for (const mode of MODES) metrics[s][mode] = summarize(rows.map((r) => r.byMode[mode].rank)); + } + + // Cross-mode overlap (how identical are the result sets?). + const overlap = {}; + for (const s of ['keyword', 'paraphrase', 'overall']) { + const rows = perQuery.filter((r) => s === 'overall' || r.set === s); + const pairMean = (a, b) => rows.reduce((acc, r) => acc + jaccard(r.byMode[a].ids, r.byMode[b].ids), 0) / rows.length; + const identicalTop20 = (a, b) => rows.filter((r) => jaccard(r.byMode[a].ids, r.byMode[b].ids) === 1).length / rows.length; + overlap[s] = { + jaccard_hybrid_vector: pairMean('hybrid', 'vector'), + jaccard_hybrid_lexical: pairMean('hybrid', 'lexical'), + jaccard_vector_lexical: pairMean('vector', 'lexical'), + identical_top20_hybrid_vector: identicalTop20('hybrid', 'vector'), + identical_top20_hybrid_lexical: identicalTop20('hybrid', 'lexical'), + }; + } + + // Explain composition rollup (per set): how often did the BM25 leg actually contribute? + const explainRollup = {}; + for (const s of ['keyword', 'paraphrase']) { + const rows = explain.filter((e) => e.set === s && e.hybrid && !e.hybrid.error); + if (!rows.length) continue; + const avg = (f) => rows.reduce((a, e) => a + f(e), 0) / rows.length; + explainRollup[s] = { + n: rows.length, + queries_with_any_bm25_hit: rows.filter((e) => e.hybrid.n_bm25 > 0).length, + avg_hybrid_bm25_hits: avg((e) => e.hybrid.n_bm25), + avg_hybrid_vector_hits: avg((e) => e.hybrid.n_vector), + avg_hybrid_subject_boosted: avg((e) => e.hybrid.n_subject_boosted), + queries_pool_saturated: rows.filter((e) => e.hybrid.pool_saturated).length, + }; + } + + // ── Print human-readable tables ── + const pct = (x) => (x * 100).toFixed(1).padStart(5); + const num = (x) => x.toFixed(3); + console.log('\n================ RESULTS ================'); + for (const s of sets) { + console.log(`\n[${s.toUpperCase()}] (n=${metrics[s].lexical.n})`); + console.log(' mode R@1 R@5 R@20 MRR@20 found'); + for (const mode of MODES) { + const m = metrics[s][mode]; + console.log(` ${mode.padEnd(7)} ${pct(m.recall_at_1)}% ${pct(m.recall_at_5)}% ${pct(m.recall_at_20)}% ${num(m.mrr_at_20)} ${m.found}/${m.n}`); + } + } + console.log('\n[OVERLAP] mean Jaccard@20 of result sets'); + for (const s of ['keyword', 'paraphrase']) { + const o = overlap[s]; + console.log(` ${s.padEnd(11)} hybrid~vector=${num(o.jaccard_hybrid_vector)} hybrid~lexical=${num(o.jaccard_hybrid_lexical)} vector~lexical=${num(o.jaccard_vector_lexical)} (identical hybrid==vector top20: ${pct(o.identical_top20_hybrid_vector)}%)`); + } + console.log('\n[EXPLAIN] hybrid BM25-leg contribution'); + for (const s of ['keyword', 'paraphrase']) { + const e = explainRollup[s]; if (!e) continue; + console.log(` ${s.padEnd(11)} queries with >=1 BM25 hit: ${e.queries_with_any_bm25_hit}/${e.n} avg BM25 hits=${num(e.avg_hybrid_bm25_hits)} avg vector hits=${num(e.avg_hybrid_vector_hits)} avg subject-boosted=${num(e.avg_hybrid_subject_boosted)}`); + } + + // ── Write results JSON (NO secrets) ── + const results = { + generated_at: new Date().toISOString(), + config: { + baseUrl: CFG.baseUrl, seed: CFG.seed, limit: CFG.limit, + openaiModel: CFG.openaiKey ? CFG.openaiModel : null, + n_keyword: keywordSet.length, n_paraphrase: paraphraseSet.length, + paraphrase_fallback_used: usedFallback, + paraphrase_sources: sourceCounts, + }, + corpus: { messages: Number(psql('SELECT count(*) FROM messages')[0][0]), note: 'bodies mostly NULL; embeddings ~subject-only' }, + metrics, overlap, explainRollup, + perQuery: perQuery.map((r) => ({ + id: r.id, set: r.set, query: r.query, subject: r.subject, + rank: { lexical: r.byMode.lexical.rank, vector: r.byMode.vector.rank, hybrid: r.byMode.hybrid.rank }, + fellBack: { vector: r.byMode.vector.fellBack, hybrid: r.byMode.hybrid.fellBack }, + explain: explainById[`${r.set}:${r.id}`] || null, + })), + }; + writeFileSync(CFG.outFile, JSON.stringify(results, null, 2)); + console.log(`\nwrote ${CFG.outFile}`); + console.log(`wrote ${CFG.cacheFile}`); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/backend/scripts/vector-probe.js b/backend/scripts/vector-probe.js new file mode 100644 index 00000000..763bfd4d --- /dev/null +++ b/backend/scripts/vector-probe.js @@ -0,0 +1,38 @@ +// Dev probe: bring up the vector schema, insert N random D-dim vectors under a throwaway +// generation, then run an ANN query and print the nearest neighbors with scores. +// Usage: node scripts/vector-probe.js [N=100] [D=8] +// Requires DB_* env pointing at a pgvector-enabled Postgres. +import { randomUUID } from 'crypto'; +import { pool } from '../src/services/db.js'; +import { ensureVectorSchema, ensureVectorIndex, upsert, annSearch } from '../src/services/embeddings/vectorStore.js'; +import { createGeneration } from '../src/services/embeddings/generations.js'; +import { seedAccount, cleanupAccount } from '../src/services/embeddings/testSupport.js'; + +const N = Number(process.argv[2]) || 100; +const D = Number(process.argv[3]) || 8; + +function randVec(d) { return Array.from({ length: d }, () => Math.random() * 2 - 1); } + +const { vectorAvailable } = await ensureVectorSchema(); +if (!vectorAvailable) { console.error('vector unavailable — is this a pgvector image?'); process.exit(1); } +await ensureVectorIndex(D); + +const gen = await createGeneration('probe-model', D, `probe:${randomUUID()}`); +// Seed N messages (probe rows) so the ANN liveness EXISTS matches. Uses a throwaway account. +const { accountId: acctId, userId } = await seedAccount(pool, 'probe'); +const chunks = []; +for (let i = 0; i < N; i++) { + const m = await pool.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1,$2,'INBOX',$3) RETURNING id`, + [acctId, 900000 + i, `probe ${i}`]); + chunks.push({ messageId: m.rows[0].id, chunkIndex: 0, vector: randVec(D), sourceCharLen: 8, chunkCharStart: 0, chunkCharEnd: 8, truncated: false }); +} +await upsert(gen, chunks); + +const q = randVec(D); +console.log(`query = [${q.map((x) => x.toFixed(3)).join(', ')}]`); +const hits = await annSearch(gen, q, 5); +for (const h of hits) console.log(` rank ${h.rank} msg ${h.messageId} score ${h.score.toFixed(4)}`); + +await cleanupAccount(pool, userId); +await pool.end(); +process.exit(0); diff --git a/backend/src/index.js b/backend/src/index.js index 70b0a85f..fbf579de 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -26,6 +26,7 @@ import blockListRoutes from './routes/blockList.js'; import contactsRoutes from './routes/contacts.js'; import todoistRoutes from './routes/todoist.js'; import aiRoutes from './routes/ai.js'; +import aiEmbeddingsRoutes from './routes/aiEmbeddings.js'; import categoriesRoutes from './routes/categories.js'; import gtdRoutes from './routes/gtd.js'; import carddavRouter from './routes/carddav.js'; @@ -33,7 +34,9 @@ import carddavAccountRouter from './routes/carddavAccount.js'; import { startCardavScheduler } from './services/carddavSync.js'; import { scheduleFtsBackfill } from './services/search/ftsBackfill.js'; import { encryptExistingCredentials, query } from './services/db.js'; -import { runMigrations } from './services/migrations.js'; +import { runMigrations, warnOnCollationMismatch } from './services/migrations.js'; +import { ensureVectorSchema } from './services/embeddings/vectorStore.js'; +import { startEmbeddingScheduler } from './services/embeddings/scheduler.js'; import { parseVCard } from './utils/vcard.js'; import { reloadAuthSettings } from './services/authLimiter.js'; import { setupWebSocket } from './services/websocket.js'; @@ -177,6 +180,7 @@ app.use('/api/contacts', contactsRoutes); app.use('/api/todoist', todoistRoutes); app.use('/api/carddav', carddavAccountRouter); app.use('/api', aiRoutes); +app.use('/api', aiEmbeddingsRoutes); app.use('/api', categoriesRoutes); // Mounted at the /api/gtd subtree (not bare /api) so gtd.js's router-level // requireAuth cannot intercept the unauthenticated /api/health and /api/version @@ -213,6 +217,11 @@ setupWebSocket(wss, sessionMiddleware, imapManager); // Run pending schema migrations then start await runMigrations(); +// Loud, best-effort drift check: a Postgres image swap (e.g. postgres:16-alpine → +// pgvector/pgvector:pg16) changes the libc collation and silently corrupts text-index +// ordering until a REINDEX. Logs the remedy; never blocks the boot. +await warnOnCollationMismatch(); + // One-time backfill: populate photo_data from existing vcard column for contacts // that were synced before CardDAV PUT started persisting photo_data. async function backfillContactPhotos() { @@ -241,6 +250,13 @@ await encryptExistingCredentials(); // Load OAuth integration configs from DB into process.env await loadIntegrationConfigs(); +// Best-effort vector schema bring-up. On stock postgres:16-alpine this logs +// "Vector disabled" and semantic search stays off; lexical is unaffected. +await ensureVectorSchema(); + +// Periodic embedding nudge — drives any building/active generation toward coverage. +startEmbeddingScheduler(); + // Start background snooze watcher — polls every 60 seconds to restore snoozed messages imapManager.startSnoozeWatcher(); diff --git a/backend/src/routes/ai.buildEmbeddingsConfig.test.js b/backend/src/routes/ai.buildEmbeddingsConfig.test.js new file mode 100644 index 00000000..8505fc8b --- /dev/null +++ b/backend/src/routes/ai.buildEmbeddingsConfig.test.js @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../services/encryption.js', () => ({ + encrypt: (v) => `enc:${v}`, + decrypt: (v) => (v ? String(v).replace(/^enc:/, '') : v), +})); +import { buildEmbeddingsConfig } from './ai.js'; + +describe('buildEmbeddingsConfig', () => { + it('encrypts a freshly supplied apiKey', () => { + const out = buildEmbeddingsConfig( + { enabled: true, endpoint: 'http://h/v1', apiKey: 'sk-1', model: 'm', dimension: 768 }, + null, + ); + expect(out.apiKey).toBe('enc:sk-1'); + expect(out.dimension).toBe(768); + expect(out.preprocess.stripHTML).toBe(true); + }); + it('keeps the existing key when the masked sentinel is sent back', () => { + const out = buildEmbeddingsConfig( + { apiKey: '••••••••', model: 'm', dimension: 4 }, + { apiKey: 'enc:old' }, + ); + expect(out.apiKey).toBe('enc:old'); + }); + it('preserves an explicit-false preprocess flag', () => { + const out = buildEmbeddingsConfig( + { model: 'm', dimension: 4, preprocess: { stripQuotes: false } }, + null, + ); + expect(out.preprocess.stripQuotes).toBe(false); + }); +}); diff --git a/backend/src/routes/ai.js b/backend/src/routes/ai.js index 088fc385..8c32ade7 100644 --- a/backend/src/routes/ai.js +++ b/backend/src/routes/ai.js @@ -4,9 +4,24 @@ import { requireAuth, requireAdmin } from '../middleware/auth.js'; import { encrypt, decrypt } from '../services/encryption.js'; import { validateHost } from '../services/hostValidation.js'; import { getConnectionPolicy } from '../services/connectionPolicy.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import { applyEmbedDefaults } from '../services/embeddings/config.js'; const router = Router(); +// Merge an embeddings sub-config from a PATCH body with the existing stored block. +// config.js owns every field default (including endpoint trim + trailing-slash strip) +// via applyEmbedDefaults, so the read and write paths normalize identically; here we +// layer only the write-path apiKey concern: a fresh key is encrypted, the masked +// sentinel keeps the stored key. +export function buildEmbeddingsConfig(body = {}, existing = null) { + const resolved = applyEmbedDefaults(body); + resolved.apiKey = body.apiKey && body.apiKey !== '••••••••' + ? encrypt(body.apiKey) + : (existing?.apiKey || null); + return resolved; +} + // ── Admin: AI provider configuration ────────────────────────────────────────── router.get('/admin/ai', requireAdmin, async (req, res) => { @@ -14,7 +29,11 @@ router.get('/admin/ai', requireAdmin, async (req, res) => { if (!result.rows.length) return res.json({ config: null }); try { const cfg = JSON.parse(result.rows[0].value); - res.json({ config: { ...cfg, apiKey: cfg.apiKey ? '••••••••' : '' } }); + const masked = { ...cfg, apiKey: cfg.apiKey ? '••••••••' : '' }; + if (cfg.embeddings) { + masked.embeddings = { ...cfg.embeddings, apiKey: cfg.embeddings.apiKey ? '••••••••' : '' }; + } + res.json({ config: masked }); } catch { res.json({ config: null }); } @@ -49,6 +68,30 @@ router.patch('/admin/ai', requireAdmin, async (req, res) => { } } + let existingEmbeddings = null; + if (existing.rows.length) { + try { existingEmbeddings = JSON.parse(existing.rows[0].value).embeddings || null; } catch { /* keep null */ } + } + const embeddings = req.body.embeddings + ? buildEmbeddingsConfig(req.body.embeddings, existingEmbeddings) + : existingEmbeddings; + + // Host-validate the embeddings endpoint the same way baseUrl is validated. + if (embeddings?.endpoint) { + let embHost; + try { embHost = new URL(embeddings.endpoint).hostname; } catch { + return res.status(400).json({ error: 'Invalid embeddings endpoint URL' }); + } + const policy = await getConnectionPolicy(); + const embErr = await validateHost(embHost, { allowPrivate: policy.allowPrivateHosts }); + if (embErr) { + const hint = embErr.includes('private or reserved') + ? ' To use a local network address, enable "Allow private hosts" in Settings → Security.' + : ''; + return res.status(400).json({ error: `Embeddings endpoint: ${embErr}.${hint}` }); + } + } + const cfg = { enabled: enabled !== false, baseUrl: trimmedBaseUrl, @@ -58,6 +101,7 @@ router.patch('/admin/ai', requireAdmin, async (req, res) => { compose: features?.compose !== false, summarize: features?.summarize !== false, }, + ...(embeddings ? { embeddings } : {}), }; await query( @@ -121,15 +165,17 @@ router.post('/admin/ai/test', requireAdmin, async (req, res) => { router.get('/ai/status', requireAuth, async (req, res) => { const result = await query("SELECT value FROM system_settings WHERE key = 'ai_config'"); - if (!result.rows.length) return res.json({ enabled: false, features: {} }); + if (!result.rows.length) return res.json({ enabled: false, features: {}, vectorAvailable: isVectorAvailable() }); try { const cfg = JSON.parse(result.rows[0].value); res.json({ enabled: cfg.enabled === true && !!cfg.baseUrl && !!cfg.model, features: cfg.features || {}, + vectorAvailable: isVectorAvailable(), + embeddingsEnabled: cfg.embeddings?.enabled === true && !!cfg.embeddings?.endpoint && !!cfg.embeddings?.model, }); } catch { - res.json({ enabled: false, features: {} }); + res.json({ enabled: false, features: {}, vectorAvailable: isVectorAvailable() }); } }); diff --git a/backend/src/routes/aiEmbeddings.js b/backend/src/routes/aiEmbeddings.js new file mode 100644 index 00000000..e41021e6 --- /dev/null +++ b/backend/src/routes/aiEmbeddings.js @@ -0,0 +1,148 @@ +import { Router } from 'express'; +import { requireAdmin } from '../middleware/auth.js'; +import { resolveEmbedConfig, generationFingerprint } from '../services/embeddings/config.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import * as store from '../services/embeddings/vectorStore.js'; +import * as generations from '../services/embeddings/generations.js'; +const { createGeneration, buildingGeneration, retireGeneration, BuildingInProgressError } = generations; +import { EmbeddingClient } from '../services/embeddings/client.js'; +import { EmbeddingWorker } from '../services/embeddings/worker.js'; +import { tryAcquireEmbedRun, releaseEmbedRun } from '../services/embeddings/embedRunLock.js'; +import { upsertJob } from '../services/backgroundJobs.js'; +import { query } from '../services/db.js'; + +const router = Router(); + +// Returns an error string when the config cannot start a build, else null. Pure helper. +export function validateBuildConfig(cfg) { + if (!isVectorAvailable()) return 'Vector extension unavailable — semantic search is disabled on this database'; + if (!cfg) return 'Embeddings not configured'; + if (!cfg.enabled) return 'Embeddings are disabled'; + if (!cfg.endpoint) return 'Embeddings endpoint is required'; + if (!cfg.model) return 'Embeddings model is required'; + if (!(cfg.dimension > 0)) return 'Embeddings dimension must be a positive integer'; + return null; +} + +// Probe the embedding endpoint with one input and echo the returned dimension. Pure helper. +export async function probeEmbeddings(client) { + const vecs = await client.embed(['mailflow embeddings connectivity probe']); + return { ok: true, dimension: vecs[0].length }; +} + +// The probe client intentionally omits the dimension expectation (dimension: null +// skips the client's per-vector assertion): Test's whole job is to DISCOVER the +// endpoint's real dimension so the UI can reconcile a wrong saved value +// (reconcileDimension auto-fill). With the assertion in place, a mismatch threw +// before the probed dimension ever reached the response, making that UI path +// unreachable. Worker/query paths keep the strict assertion — they construct +// their clients with cfg.dimension. +export function buildProbeClient(cfg) { + return new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: null }); +} + +router.post('/admin/ai/embeddings/test-embeddings', requireAdmin, async (req, res) => { + const cfg = await resolveEmbedConfig(); + if (!cfg || !cfg.endpoint || !cfg.model || !(cfg.dimension > 0)) { + return res.status(400).json({ error: 'Embeddings endpoint, model, and dimension are required' }); + } + try { + res.json(await probeEmbeddings(buildProbeClient(cfg))); + } catch (err) { + res.status(400).json({ error: err.message }); + } +}); + +// Count live messages still needing embedding under generation `gen` (the build's +// initial "total"). Default collaborator for startEmbeddingBuild. +async function countPending(gen) { + const r = await query( + 'SELECT COUNT(*)::int n FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $1) AND is_deleted = false', [gen], + ); + return r.rows[0].n; +} + +// Fire-and-forget worker run toward coverage. Returns worker.runOnce's promise so the +// caller can chain job-state updates and the single-flight release onto its settlement. +function runWorker(gen, total, cfg) { + const client = new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: cfg.dimension }); + const worker = new EmbeddingWorker({ + // `generations` lets the worker activate this building generation once its + // scan drains to full coverage (the shared activation seam in worker.js). + store, client, generations, preprocessCfg: cfg.preprocess, maxInputChars: cfg.maxInputChars, batchSize: cfg.batchSize, + onProgress: (p) => { upsertJob({ kind: 'embeddings', state: 'running', processed: p.done, total }).catch(() => {}); }, + }); + return worker.runOnce(gen); +} + +// Collaborators for startEmbeddingBuild, bound to the real modules by default and +// overridable in tests (the injection pattern used across the embeddings services). +export const BUILD_DEPS = { + tryAcquireEmbedRun, releaseEmbedRun, + createGeneration, buildingGeneration, retireGeneration, generationFingerprint, + countPending, upsertJob, runWorker, log: console.log, +}; + +// Orchestrates an embeddings (re)build. Returns { status, body } for the route to send. +// +// Ordering matters: createGeneration atomically NULLs every live embed_gen stamp. If an +// embed run were mid-flight when that reset lands, it could re-stamp rows with the OLD +// generation id afterward — and the new generation's scan (embed_gen IS NULL only) would +// never see them, so activation's coverage gate blocks forever. So we take the single- +// flight lock BEFORE createGeneration, making the stamp-reset mutually exclusive with any +// embed run. If the lock is busy we return an honest, retryable 409 and never touch the +// stamps. On every non-success exit the lock is released exactly once; on success the +// fire-and-forget worker chain owns the single release when the run settles. +export async function startEmbeddingBuild(cfg, username, deps = {}) { + const d = { ...BUILD_DEPS, ...deps }; + const fingerprint = d.generationFingerprint(cfg); + + if (!d.tryAcquireEmbedRun()) { + return { status: 409, body: { error: 'An embedding run is in progress — retry in a moment' } }; + } + + let gen, total; + try { + try { + gen = await d.createGeneration(cfg.model, cfg.dimension, fingerprint); + } catch (err) { + // A building generation with a DIFFERENT fingerprint blocks this build. A + // new-fingerprint build supersedes an incomplete old-fingerprint one, so retire + // the stale gen (deletes its rows — generations never mix) and retry once. + if (!(err instanceof BuildingInProgressError)) throw err; + const stale = await d.buildingGeneration(); + if (!stale || stale.fingerprint === fingerprint) throw err; + await d.retireGeneration(stale.id); + d.log(`[admin] ${username} retired stale building gen ${stale.id} (fingerprint ${stale.fingerprint}); superseded by ${fingerprint}`); + gen = await d.createGeneration(cfg.model, cfg.dimension, fingerprint); + } + total = await d.countPending(gen); + await d.upsertJob({ kind: 'embeddings', state: 'running', processed: 0, total }); + } catch (err) { + // Any failure before the worker chain is attached below leaves the lock ours to + // free — release it so a failed start never wedges every future build. + d.releaseEmbedRun(); + return { status: 409, body: { error: err.message } }; + } + + // We already hold the single-flight lock; fire the worker and release exactly once + // when it settles, on every outcome. The request returns immediately. + d.runWorker(gen, total, cfg) + .then((r) => d.upsertJob({ kind: 'embeddings', state: 'done', processed: r.succeeded, total })) + .catch((err) => d.upsertJob({ kind: 'embeddings', state: 'error', processed: 0, total, lastError: err.message })) + .finally(() => d.releaseEmbedRun()) + .catch(() => {}); + d.log(`[admin] ${username} started embeddings build gen ${gen} (${total} pending)`); + return { status: 200, body: { ok: true, generationId: gen, total } }; +} + +router.post('/admin/ai/embeddings/build', requireAdmin, async (req, res) => { + const cfg = await resolveEmbedConfig(); + const invalid = validateBuildConfig(cfg); + if (invalid) return res.status(400).json({ error: invalid }); + + const { status, body } = await startEmbeddingBuild(cfg, req.session.username); + res.status(status).json(body); +}); + +export default router; diff --git a/backend/src/routes/aiEmbeddings.test.js b/backend/src/routes/aiEmbeddings.test.js new file mode 100644 index 00000000..7ce05b84 --- /dev/null +++ b/backend/src/routes/aiEmbeddings.test.js @@ -0,0 +1,149 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +vi.mock('../services/embeddings/vectorStore.js', () => ({ isVectorAvailable: vi.fn(() => true) })); +import { validateBuildConfig, probeEmbeddings, buildProbeClient, startEmbeddingBuild } from './aiEmbeddings.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import { BuildingInProgressError } from '../services/embeddings/generations.js'; + +const full = { enabled: true, endpoint: 'http://h/v1', model: 'm', dimension: 768, maxInputChars: 32768, batchSize: 32, preprocess: {} }; + +describe('validateBuildConfig', () => { + it('accepts a complete config when vector is available', () => { + expect(validateBuildConfig(full)).toBeNull(); + }); + it('rejects when vector is unavailable', () => { + isVectorAvailable.mockReturnValueOnce(false); + expect(validateBuildConfig(full)).toMatch(/vector/i); + }); + it('rejects an incomplete config', () => { + expect(validateBuildConfig({ ...full, dimension: 0 })).toMatch(/dimension/i); + expect(validateBuildConfig(null)).toMatch(/not configured/i); + }); +}); + +describe('probeEmbeddings', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('echoes the returned dimension', async () => { + const fakeClient = { embed: vi.fn().mockResolvedValue([[0.1, 0.2, 0.3]]) }; + const out = await probeEmbeddings(fakeClient); + expect(out).toEqual({ ok: true, dimension: 3 }); + }); + it('surfaces an embed error', async () => { + const fakeClient = { embed: vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')) }; + await expect(probeEmbeddings(fakeClient)).rejects.toThrow(/ECONNREFUSED/); + }); + it('returns the endpoint\'s ACTUAL dimension even when the saved config disagrees', async () => { + // Regression pin: with the probe client asserting cfg.dimension, a mismatch threw + // before Test could return the probed value, so the UI's reconcileDimension + // auto-fill was unreachable. The probe client must skip the assertion. + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + status: 200, + headers: { get: () => null }, + json: async () => ({ data: [{ index: 0, embedding: Array(768).fill(0.1) }] }), + text: async () => '', + })); + const out = await probeEmbeddings(buildProbeClient({ ...full, dimension: 1536 })); + expect(out).toEqual({ ok: true, dimension: 768 }); + }); +}); + +// Let the fire-and-forget worker chain (.then/.catch/.finally) settle. +const flush = () => new Promise((r) => setTimeout(r, 0)); + +function makeDeps(overrides = {}) { + return { + tryAcquireEmbedRun: vi.fn(() => true), + releaseEmbedRun: vi.fn(), + createGeneration: vi.fn().mockResolvedValue('gen-1'), + buildingGeneration: vi.fn().mockResolvedValue(null), + retireGeneration: vi.fn().mockResolvedValue(undefined), + generationFingerprint: vi.fn(() => 'fp'), + countPending: vi.fn().mockResolvedValue(5), + upsertJob: vi.fn().mockResolvedValue(undefined), + runWorker: vi.fn().mockResolvedValue({ succeeded: 5 }), + log: vi.fn(), + ...overrides, + }; +} + +describe('startEmbeddingBuild', () => { + it('returns a retryable 409 and never resets stamps when an embed run is active (Fix 1)', async () => { + const deps = makeDeps({ tryAcquireEmbedRun: vi.fn(() => false) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toMatch(/in progress/i); + expect(deps.createGeneration).not.toHaveBeenCalled(); // stamp-reset never runs + expect(deps.releaseEmbedRun).not.toHaveBeenCalled(); // never acquired ⇒ nothing to release + }); + + it('acquires the single-flight lock before createGeneration (Fix 1)', async () => { + const deps = makeDeps(); + await startEmbeddingBuild(full, 'admin', deps); + expect(deps.tryAcquireEmbedRun).toHaveBeenCalled(); + expect(deps.createGeneration).toHaveBeenCalledWith('m', 768, 'fp'); + expect(deps.tryAcquireEmbedRun.mock.invocationCallOrder[0]) + .toBeLessThan(deps.createGeneration.mock.invocationCallOrder[0]); + await flush(); + }); + + it('on success fires the worker and releases the lock exactly once (Fix 1)', async () => { + const deps = makeDeps(); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true, generationId: 'gen-1', total: 5 }); + expect(deps.runWorker).toHaveBeenCalledWith('gen-1', 5, full); + await flush(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('releases the lock and returns 409 when createGeneration fails outright (Fix 1)', async () => { + const deps = makeDeps({ createGeneration: vi.fn().mockRejectedValue(new Error('boom')) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toBe('boom'); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + expect(deps.retireGeneration).not.toHaveBeenCalled(); + }); + + it('releases the lock if the build fails after createGeneration but before the worker starts (Fix 1)', async () => { + // A failing countPending would otherwise leave the lock held forever (it is taken + // before createGeneration now), wedging every future build. + const deps = makeDeps({ countPending: vi.fn().mockRejectedValue(new Error('db down')) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toBe('db down'); + expect(deps.createGeneration).toHaveBeenCalledTimes(1); + expect(deps.runWorker).not.toHaveBeenCalled(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('retires a stale different-fingerprint building gen and retries createGeneration once (Fix 2b)', async () => { + const createGeneration = vi.fn() + .mockRejectedValueOnce(new BuildingInProgressError('building fingerprint=old-fp, requested=fp')) + .mockResolvedValueOnce('gen-2'); + const deps = makeDeps({ + createGeneration, + buildingGeneration: vi.fn().mockResolvedValue({ id: 'stale-1', fingerprint: 'old-fp' }), + }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(deps.retireGeneration).toHaveBeenCalledWith('stale-1'); + expect(createGeneration).toHaveBeenCalledTimes(2); + expect(res.status).toBe(200); + expect(res.body.generationId).toBe('gen-2'); + await flush(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('surfaces a same-fingerprint BuildingInProgressError as 409 without retiring (Fix 2b)', async () => { + const createGeneration = vi.fn().mockRejectedValue(new BuildingInProgressError('already building')); + const deps = makeDeps({ + createGeneration, + buildingGeneration: vi.fn().mockResolvedValue({ id: 'x', fingerprint: 'fp' }), + }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(deps.retireGeneration).not.toHaveBeenCalled(); + expect(createGeneration).toHaveBeenCalledTimes(1); // no infinite retry + expect(res.status).toBe(409); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/routes/search.js b/backend/src/routes/search.js index cb3c8a6c..63980648 100644 --- a/backend/src/routes/search.js +++ b/backend/src/routes/search.js @@ -7,6 +7,8 @@ import { search } from '../services/search/searchService.js'; const router = Router(); router.use(requireAuth); +const ALLOWED_MODES = new Set(['lexical', 'vector', 'hybrid']); + // Simple in-memory rate limiter: 20 searches per minute per user. const searchBuckets = new Map(); setInterval(() => { @@ -39,6 +41,8 @@ router.get('/', searchLimiter, async (req, res) => { if (trimmed.length > 500) return res.status(400).json({ error: 'Search query too long' }); const parsed = parseQuery(trimmed); + const mode = ALLOWED_MODES.has(req.query.mode) ? req.query.mode : 'lexical'; + try { const result = await search({ userId: req.session.userId, @@ -47,6 +51,7 @@ router.get('/', searchLimiter, async (req, res) => { folderParam: req.query.folder || '', limit, offset, + mode, }); // Response is a strict superset of the historical { messages, query }. res.json({ ...result, query: q, unsupported: parsed.unsupported, errors: parsed.errors }); diff --git a/backend/src/routes/search.test.js b/backend/src/routes/search.test.js index 9b305e10..26bcbe62 100644 --- a/backend/src/routes/search.test.js +++ b/backend/src/routes/search.test.js @@ -59,3 +59,38 @@ describe('GET /api/search', () => { expect(res.json.errors).toEqual([]); }); }); + +describe('GET /api/search mode passthrough (Phase 4 Task 7)', () => { + beforeEach(() => search.mockReset()); + + it('defaults mode to lexical when absent', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello'); + expect(search.mock.calls[0][0].mode).toBe('lexical'); + }); + + it('passes mode=hybrid straight through', async () => { + search.mockResolvedValue({ messages: [], mode: 'hybrid', pool_saturated: false, generation: null, page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=hybrid'); + expect(search.mock.calls[0][0].mode).toBe('hybrid'); + }); + + it('passes mode=vector straight through', async () => { + search.mockResolvedValue({ messages: [], mode: 'vector', pool_saturated: false, generation: null, page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=vector'); + expect(search.mock.calls[0][0].mode).toBe('vector'); + }); + + it('coerces an unknown mode to lexical', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=bogus'); + expect(search.mock.calls[0][0].mode).toBe('lexical'); + }); + + it('returns the service response including fellBack (superset)', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', fellBack: true, page: { offset: 0, limit: 50, hasMore: false } }); + const res = await get(makeApp(), '/api/search?q=hello&mode=hybrid'); + expect(res.json.mode).toBe('lexical'); + expect(res.json.fellBack).toBe(true); + }); +}); diff --git a/backend/src/services/bodyBackfill.js b/backend/src/services/bodyBackfill.js index ccb855b7..11e6cbea 100644 --- a/backend/src/services/bodyBackfill.js +++ b/backend/src/services/bodyBackfill.js @@ -2,7 +2,7 @@ import { query } from './db.js'; import { providerProfile } from './imapManager.js'; // Body-materialization drainer. Fills messages.body_text/body_html over IMAP so weight-D -// lexical search has material to work with, WITHOUT growing the imapManager +// lexical search and embeddings have material to work with, WITHOUT growing the imapManager // singleton (README invariant) and WITHOUT touching throttle-hostile providers. ALL policy — // the scan predicate, batching, pacing, provider gating, session cap, quiet-window // backpressure, and the per-account circuit breaker — lives here. The singleton exposes only @@ -88,7 +88,8 @@ export async function startBodyBackfill(account, deps) { // Keyset by id ascending: id is a non-null, unique UUID, so the cursor advances past every // row exactly once per run — including rows whose body cannot be fetched (they stay NULL but // are not re-selected this run, so the run always terminates). Recency ordering is - // intentionally not used: body coverage is a background quality lever. + // intentionally not used: body coverage is a background quality lever, and the embedding + // backstop picks up late arrivals regardless (README D4). let cursorId = null; let consecutiveErrors = 0; diff --git a/backend/src/services/embeddings/__testdb__.js b/backend/src/services/embeddings/__testdb__.js new file mode 100644 index 00000000..15ae3282 --- /dev/null +++ b/backend/src/services/embeddings/__testdb__.js @@ -0,0 +1,60 @@ +// pgvector test-DB harness for fused search + ranking-quality gates. +// +// Reuses Phase 3's established IT convention (env var VECTOR_IT_DB, the SAME +// var every *.integration.test.js in this directory already gates on) rather +// than introducing a second, parallel gating mechanism. Migrations are applied +// ONCE out-of-band via the real runMigrations() against the scratch DB (see +// implementation.md) — this harness does not run them itself, matching every +// existing IT file here. `ensureVectorSchema()` takes no arguments (reads its +// own module-level pool from db.js), so targeting the scratch DB requires the +// same trick those files use: mutate DB_HOST/DB_PORT/DB_NAME/DB_USER/DB_PASSWORD +// from the DSN, then dynamically `import()` the module AFTER that mutation so +// db.js's pool (constructed once, at import time) picks up the right env. +import pg from 'pg'; + +const DSN = process.env.VECTOR_IT_DB || ''; + +export function hasTestDb() { + return DSN.length > 0; +} + +let _pool = null; +let _ready = null; + +async function ready() { + if (_ready) return _ready; + _ready = (async () => { + const u = new URL(DSN); + Object.assign(process.env, { + DB_HOST: u.hostname, + DB_PORT: u.port, + DB_NAME: u.pathname.slice(1), + DB_USER: u.username, + DB_PASSWORD: u.password, + }); + const { ensureVectorSchema } = await import('./vectorStore.js'); + await ensureVectorSchema(); + _pool = new pg.Pool({ connectionString: DSN, max: 4 }); + return _pool; + })(); + return _ready; +} + +// Runs fn with a clean slate: truncate every table the fused-search / ranking +// tests touch, in one statement (Postgres resolves FK order within a single +// TRUNCATE ... CASCADE regardless of listed order). embed_runs has no ON +// DELETE CASCADE from index_generations by design (Phase 3 finding — the +// production code never hard-deletes a generation), so it must be listed +// explicitly rather than relying on cascade from index_generations alone. +export async function withTestDb(fn) { + const pool = await ready(); + await pool.query( + `TRUNCATE embeddings, embed_runs, embed_watermark, index_generations, + messages, folders, email_accounts, users RESTART IDENTITY CASCADE` + ); + return fn(pool); +} + +export async function closeTestDb() { + if (_pool) { await _pool.end(); _pool = null; _ready = null; } +} diff --git a/backend/src/services/embeddings/activeGeneration.js b/backend/src/services/embeddings/activeGeneration.js new file mode 100644 index 00000000..9eef0e4c --- /dev/null +++ b/backend/src/services/embeddings/activeGeneration.js @@ -0,0 +1,44 @@ +import { VectorUnavailableError } from './vectorErrors.js'; +import * as generations from './generations.js'; +import { resolveEmbedConfig, generationFingerprint } from './config.js'; + +const DEFAULTS = { + activeGeneration: generations.activeGeneration, + buildingGeneration: generations.buildingGeneration, +}; + +// Port of internal/vector/generations.go ResolveActiveForFingerprint. +// Throws VectorUnavailableError('index_stale') on a fingerprint mismatch +// (a rebuild superseded the caller's config), ('index_building') when only +// a build is in progress, ('no_active_generation') when neither exists. +export async function resolveActiveGeneration(fingerprint, overrides = {}) { + const d = { ...DEFAULTS, ...overrides }; + const active = await d.activeGeneration(); + if (active) { + if (fingerprint && active.fingerprint !== fingerprint) throw new VectorUnavailableError('index_stale'); + return { id: active.id, model: active.model, dimension: active.dimension, fingerprint: active.fingerprint, state: active.state }; + } + if (await d.buildingGeneration()) throw new VectorUnavailableError('index_building'); + throw new VectorUnavailableError('no_active_generation'); +} + +const PREAMBLE_DEFAULTS = { + resolveEmbedConfig, + generationFingerprint, + resolveActiveGeneration, +}; + +// The full vector-availability gate shared by every caller that needs an active +// generation from the stored embed config: resolve the config, reject an absent or +// disabled one as vector_not_enabled, then resolve the active generation by +// fingerprint (which throws index_stale/index_building/no_active_generation on a +// degraded index). Returns the resolved cfg alongside the generation — callers need +// both (the embed client + preprocess config, and the generation id/dimension). The +// collaborators are injectable so hybridSearch can thread its own fakes through. +export async function resolveActiveGenerationFromConfig(overrides = {}) { + const d = { ...PREAMBLE_DEFAULTS, ...overrides }; + const cfg = await d.resolveEmbedConfig(); + if (!cfg || cfg.enabled === false) throw new VectorUnavailableError('vector_not_enabled'); + const generation = await d.resolveActiveGeneration(d.generationFingerprint(cfg)); + return { cfg, generation }; +} diff --git a/backend/src/services/embeddings/activeGeneration.test.js b/backend/src/services/embeddings/activeGeneration.test.js new file mode 100644 index 00000000..40a293be --- /dev/null +++ b/backend/src/services/embeddings/activeGeneration.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { resolveActiveGeneration } from './activeGeneration.js'; +import { VectorUnavailableError } from './vectorErrors.js'; + +const gen = { id: 3, model: 'm', dimension: 4, fingerprint: 'm:4:fp', state: 'active', messageCount: 10 }; + +describe('resolveActiveGeneration', () => { + it('returns the rich active generation when the fingerprint matches', async () => { + const g = await resolveActiveGeneration('m:4:fp', + { activeGeneration: async () => gen, buildingGeneration: async () => null }); + expect(g).toEqual({ id: 3, model: 'm', dimension: 4, fingerprint: 'm:4:fp', state: 'active' }); + }); + it('throws index_stale on a fingerprint mismatch', async () => { + const err = await resolveActiveGeneration('OTHER', + { activeGeneration: async () => gen, buildingGeneration: async () => null }).catch(e => e); + expect(err).toBeInstanceOf(VectorUnavailableError); + expect(err.reason).toBe('index_stale'); + }); + it('throws index_building when a build is in progress and none active', async () => { + const err = await resolveActiveGeneration('x', + { activeGeneration: async () => null, buildingGeneration: async () => ({ id: 9 }) }).catch(e => e); + expect(err.reason).toBe('index_building'); + }); + it('throws no_active_generation when nothing exists', async () => { + const err = await resolveActiveGeneration('x', + { activeGeneration: async () => null, buildingGeneration: async () => null }).catch(e => e); + expect(err.reason).toBe('no_active_generation'); + }); +}); diff --git a/backend/src/services/embeddings/chunk.js b/backend/src/services/embeddings/chunk.js new file mode 100644 index 00000000..54d7a594 --- /dev/null +++ b/backend/src/services/embeddings/chunk.js @@ -0,0 +1,82 @@ +// Port of internal/vector/embed/chunk.go, in code-point (rune) space. IMPORTANT: any +// change to the window/overlap/soft-break logic MUST bump EMBED_POLICY_VERSION in config.js. + +const SENTENCE_TERMS = [['.', ' '], ['?', ' '], ['!', ' '], ['.', '\n'], ['?', '\n'], ['!', '\n']]; + +export const MAX_SPANS = 64; // worker.go maxSpansPerMessage + +export function chunkOverlapFor(maxRunes) { + if (maxRunes <= 0) return 0; + if (maxRunes < 200) return 0; + return Math.floor(maxRunes / 30); +} + +// Largest i in [floor, ceil-2] with cps[i]===a && cps[i+1]===b, else -1. +function lastPair(cps, a, b, floor, ceil) { + for (let i = ceil - 2; i >= floor; i--) if (cps[i] === a && cps[i + 1] === b) return i; + return -1; +} +// Largest i in [floor, ceil-1] with cps[i]===ch, else -1. +function lastChar(cps, ch, floor, ceil) { + for (let i = ceil - 1; i >= floor; i--) if (cps[i] === ch) return i; + return -1; +} + +// Return the code-point index of the preferred soft break in [floor, ceil): paragraph +// beats sentence beats word; ceil when none found. +function findSoftBreak(cps, floor, ceil) { + const para = lastPair(cps, '\n', '\n', floor, ceil); + if (para >= 0) return para + 2; + + let sentenceEnd = -1; + let threshold = floor - 1; + for (const [a, b] of SENTENCE_TERMS) { + const start = lastPair(cps, a, b, floor, ceil); + if (start >= 0 && start > threshold) { sentenceEnd = start + 2; threshold = start + 2; } + } + if (sentenceEnd >= 0) return sentenceEnd; + + const sp = lastChar(cps, ' ', floor, ceil); + if (sp > floor) return sp + 1; + + return ceil; +} + +export function chunkText(text, maxRunes, overlapRunes, maxSpans) { + if (text === '') return { spans: [], tailDropped: false }; + if (maxRunes <= 0) return { spans: [{ text, charStart: 0, charEnd: Array.from(text).length }], tailDropped: false }; + + let tailDropped = false; + // Early cap WITHOUT materializing the whole input (guards the 10M-rune fixture). + if (maxSpans > 0) { + const keep = maxSpans * maxRunes; + let count = 0, u16 = 0; + for (const ch of text) { + if (count >= keep) { text = text.slice(0, u16); tailDropped = true; break; } + count++; u16 += ch.length; + } + } + + const cps = Array.from(text); + const total = cps.length; + if (total <= maxRunes) return { spans: [{ text: cps.join(''), charStart: 0, charEnd: total }], tailDropped }; + + let overlap = overlapRunes < 0 ? 0 : overlapRunes; + if (overlap >= maxRunes) overlap = Math.floor(maxRunes / 2); + + const spans = []; + let cursor = 0; + while (cursor < total) { + if (maxSpans > 0 && spans.length >= maxSpans) { tailDropped = true; break; } + const windowEnd = Math.min(cursor + maxRunes, total); + let cut = windowEnd; + if (windowEnd < total) { + const searchFloor = Math.max(cursor + Math.floor((maxRunes * 3) / 4), cursor + 1); + cut = findSoftBreak(cps, searchFloor, windowEnd); + } + spans.push({ text: cps.slice(cursor, cut).join(''), charStart: cursor, charEnd: cut }); + if (cut >= total) break; + cursor += Math.max((cut - cursor) - overlap, 1); + } + return { spans, tailDropped }; +} diff --git a/backend/src/services/embeddings/chunk.test.js b/backend/src/services/embeddings/chunk.test.js new file mode 100644 index 00000000..1693e98f --- /dev/null +++ b/backend/src/services/embeddings/chunk.test.js @@ -0,0 +1,134 @@ +import { describe, it, expect } from 'vitest'; +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; + +const cp = (s) => Array.from(s).length; + +describe('shared chunking policy (one owner)', () => { + it('exports MAX_SPANS = 64 (worker.go maxSpansPerMessage)', () => { + expect(MAX_SPANS).toBe(64); + }); + it('chunkOverlapFor: 0 below 200 runes, floor(maxRunes/30) at or above', () => { + expect(chunkOverlapFor(0)).toBe(0); + expect(chunkOverlapFor(-5)).toBe(0); + expect(chunkOverlapFor(199)).toBe(0); + expect(chunkOverlapFor(200)).toBe(6); // floor(200/30) + expect(chunkOverlapFor(3000)).toBe(100); + }); +}); + +describe('chunkText (msgvault fixture parity)', () => { + it('EmptyInputReturnsNil', () => { expect(chunkText('', 100, 10, 0).spans).toEqual([]); }); + + it('ShortInputReturnsSingleSpan', () => { + const { spans } = chunkText('hello world', 100, 10, 0); + expect(spans).toHaveLength(1); + expect(spans[0].text).toBe('hello world'); + expect(spans[0].charStart).toBe(0); + expect(spans[0].charEnd).toBe(11); + }); + + it('MaxRunesZeroDisablesChunking', () => { + const text = 'x'.repeat(1000); + const { spans } = chunkText(text, 0, 10, 0); + expect(spans).toHaveLength(1); + expect(spans[0].text).toBe(text); + }); + + it('CutsAtParagraphBreakInBackQuarter', () => { + const text = 'a'.repeat(80) + '\n\n' + 'b'.repeat(50); + const { spans } = chunkText(text, 100, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charEnd).toBe(82); + expect(spans[0].text.endsWith('\n\n')).toBe(true); + }); + + it('CutsAtSentenceBoundaryWhenNoParagraph', () => { + const text = 'a'.repeat(80) + '. ' + 'b'.repeat(50); + const { spans } = chunkText(text, 100, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charEnd).toBe(82); + }); + + it('CutsAtWordBoundaryWhenNoSentence', () => { + const text = 'a'.repeat(90) + ' ' + 'b'.repeat(50); + const { spans } = chunkText(text, 100, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charEnd).toBe(91); + }); + + it('HardCutsWhenNoSoftBreakInBackQuarter', () => { + const { spans } = chunkText('a'.repeat(1000), 100, 0, 0); + expect(spans).toHaveLength(10); + for (const s of spans) expect(s.charEnd - s.charStart).toBe(100); + }); + + it('OverlapBetweenConsecutiveChunks', () => { + const { spans } = chunkText('a'.repeat(300), 100, 20, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[1].charStart).toBe(80); + }); + + it('OverlapClampedToHalfWindow', () => { + const { spans } = chunkText('a'.repeat(300), 100, 500, 0); + expect(spans.length).toBeGreaterThan(0); + if (spans.length >= 2) expect(spans[1].charStart).toBe(50); + }); + + it('AllSpansHaveValidUTF8AndCorrectText', () => { + let text = ''; + for (let i = 0; i < 50; i++) text += 'Hello world. ' + 'こんにちは世界。'; + const { spans } = chunkText(text, 80, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + const runes = Array.from(text); + for (const s of spans) { + expect(s.charStart).toBeGreaterThanOrEqual(0); + expect(s.charEnd).toBeLessThanOrEqual(runes.length); + expect(s.charStart).toBeLessThan(s.charEnd); + expect(s.text).toBe(runes.slice(s.charStart, s.charEnd).join('')); + } + }); + + it('MaxSpansCapsInputBytesProcessed', () => { + const text = 'a'.repeat(10_000_000); + const t0 = Date.now(); + const { spans } = chunkText(text, 100, 0, 3); + expect(spans).toHaveLength(3); + for (const s of spans) expect(s.charEnd).toBeLessThanOrEqual(300); + expect(Date.now() - t0).toBeLessThan(500); // O(cap), not O(10M) + }); + + it('MaxSpansCapsOutputAndDropsTail', () => { + const { spans } = chunkText('a'.repeat(1000), 100, 0, 3); + expect(spans).toHaveLength(3); + expect(spans[0].charStart).toBe(0); + expect(spans[spans.length - 1].charEnd).toBeLessThan(1000); + }); + + it('TailDroppedFlagsCapWhenLastChunkLandsOnSoftBreak', () => { + let text = ''; + for (let i = 0; i < 5; i++) text += 'a'.repeat(85) + '. '; + const { spans, tailDropped } = chunkText(text, 90, 0, 2); + expect(spans).toHaveLength(2); + expect(tailDropped).toBe(true); + }); + + it('TailDroppedFalseWhenAllContentEmitted', () => { + const { spans, tailDropped } = chunkText('a'.repeat(150), 100, 0, 10); + expect(spans.length).toBeGreaterThan(0); + expect(tailDropped).toBe(false); + }); + + it('MaxSpansZeroIsUnlimited', () => { expect(chunkText('a'.repeat(1000), 100, 0, 0).spans).toHaveLength(10); }); + it('MaxSpansLargerThanNaturalChunkCountIsNoop', () => { expect(chunkText('a'.repeat(300), 100, 0, 100).spans).toHaveLength(3); }); + + it('ConcatenationCoversInputModuloOverlap', () => { + const text = 'Lorem ipsum dolor sit amet. '.repeat(200); + const { spans } = chunkText(text, 200, 30, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charStart).toBe(0); + expect(spans[spans.length - 1].charEnd).toBe(cp(text)); + for (let i = 1; i < spans.length; i++) { + expect(spans[i].charStart).toBeLessThanOrEqual(spans[i - 1].charEnd); + } + }); +}); diff --git a/backend/src/services/embeddings/client.js b/backend/src/services/embeddings/client.js new file mode 100644 index 00000000..44d84088 --- /dev/null +++ b/backend/src/services/embeddings/client.js @@ -0,0 +1,99 @@ +// Port of internal/vector/embed/client.go. + +export class Permanent4xxError extends Error { + constructor(message) { super(message); this.name = 'Permanent4xxError'; this.permanent4xx = true; } +} +export function isPermanent4xx(err) { return err instanceof Permanent4xxError || err?.permanent4xx === true; } + +function parseRetryAfter(v) { + if (!v) return null; + const s = String(v).trim(); + const MAX = 3600 * 1000; + if (/^\d+$/.test(s)) return Math.min(Number(s) * 1000, MAX); + const t = Date.parse(s); + if (!Number.isNaN(t)) { const d = t - Date.now(); return d <= 0 ? 0 : Math.min(d, MAX); } + return null; +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Best-effort drain of a response body we are about to abandon. undici keeps the +// socket checked out until the body is consumed or cancelled; the retry paths +// (429/5xx) throw without decoding a body, so drain here to release the connection +// back to the pool before we back off and retry — otherwise a burst of rate +// limiting across scheduler ticks leaks connections. Errors are swallowed: a failed +// drain must not mask the retryable status we are reporting. +async function drainBody(resp) { + try { await resp.text(); } catch { /* already released/aborted — nothing to free */ } +} + +export class EmbeddingClient { + // `dimension` is the expected vector length; every returned embedding is asserted + // against it. Pass null/undefined to skip the assertion — the admin test-embeddings + // probe does this because its job is to DISCOVER the endpoint's real dimension. + // Worker/query paths always construct with the configured dimension. + constructor({ endpoint, apiKey, model, dimension, timeout = 30000, maxRetries = 3 }) { + this.endpoint = endpoint; + this.apiKey = apiKey || null; + this.model = model; + this.dimension = dimension ?? null; + this.timeout = timeout; + this.maxRetries = maxRetries; + } + + async embed(inputs) { + if (!inputs.length) return []; + const body = JSON.stringify({ input: inputs, model: this.model }); + let lastErr; + for (let attempt = 1; attempt <= this.maxRetries; attempt++) { + try { + return await this._doOnce(body, inputs.length); + } catch (err) { + lastErr = err; + if (!err._retry) throw err; // permanent 4xx or dimension mismatch — no retry + if (attempt === this.maxRetries) break; + let backoff = Math.min(2 ** Math.min(attempt, 8), 256) * 100; + if (err._retryAfterSet) backoff = err._retryAfter; + if (backoff > 0) await sleep(backoff); + } + } + throw new Error(`embed: giving up after ${this.maxRetries} attempts: ${lastErr.message}`); + } + + async _doOnce(body, want) { + const headers = { 'Content-Type': 'application/json' }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + let resp; + try { + resp = await fetch(`${this.endpoint}/embeddings`, { method: 'POST', headers, body, signal: AbortSignal.timeout(this.timeout) }); + } catch (e) { const err = new Error(`http do: ${e.message}`); err._retry = true; throw err; } + + if (resp.status === 429) { + const ra = parseRetryAfter(resp.headers.get('Retry-After')); + await drainBody(resp); // free the socket before backing off — we discard this body + const err = new Error('embed: HTTP 429 (rate limited)'); err._retry = true; + if (ra !== null) { err._retryAfterSet = true; err._retryAfter = ra; } + throw err; + } + if (resp.status >= 500) { + await drainBody(resp); // free the socket before retrying — we discard this body + const err = new Error(`embed: HTTP ${resp.status}`); err._retry = true; throw err; + } + if (resp.status >= 400) { + const txt = (await resp.text().catch(() => '')).slice(0, 4096).trim(); + throw new Permanent4xxError(`embed: HTTP ${resp.status}${txt ? ': ' + txt : ''}`); + } + let data; + try { data = await resp.json(); } catch (e) { const err = new Error(`decode response: ${e.message}`); err._retry = true; throw err; } + + const vecs = new Array(want).fill(null); + for (const d of data.data || []) { + if (d.index < 0 || d.index >= want) throw new Error(`embed: invalid index ${d.index} (len=${want})`); + if (this.dimension !== null && d.embedding.length !== this.dimension) { + throw new Error(`embed: dimension mismatch: got ${d.embedding.length}, configured ${this.dimension}`); + } + vecs[d.index] = d.embedding; + } + for (let i = 0; i < want; i++) if (vecs[i] === null) throw new Error(`embed: missing embedding at index ${i}`); + return vecs; + } +} diff --git a/backend/src/services/embeddings/client.test.js b/backend/src/services/embeddings/client.test.js new file mode 100644 index 00000000..e2fd7ad2 --- /dev/null +++ b/backend/src/services/embeddings/client.test.js @@ -0,0 +1,119 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { EmbeddingClient, isPermanent4xx } from './client.js'; + +function mockFetchOnce(status, body, headers = {}) { + return { + status, + headers: { get: (k) => headers[k.toLowerCase()] ?? null }, + json: async () => body, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + }; +} +const okBody = (n, dim = 3) => ({ data: Array.from({ length: n }, (_, i) => ({ index: i, embedding: Array(dim).fill(0.1) })), model: 'm' }); + +afterEach(() => vi.unstubAllGlobals()); + +describe('EmbeddingClient.embed', () => { + it('returns one vector per input in order', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(mockFetchOnce(200, okBody(2)))); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3 }); + const out = await c.embed(['a', 'b']); + expect(out).toHaveLength(2); + expect(out[0]).toEqual([0.1, 0.1, 0.1]); + }); + + it('empty input is a no-op (no HTTP call)', async () => { + const f = vi.fn(); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3 }); + expect(await c.embed([])).toEqual([]); + expect(f).not.toHaveBeenCalled(); + }); + + it('retries a 429 (Retry-After: 0) then succeeds', async () => { + const f = vi.fn() + .mockResolvedValueOnce(mockFetchOnce(429, 'slow down', { 'retry-after': '0' })) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + const out = await c.embed(['a']); + expect(out).toHaveLength(1); + expect(f).toHaveBeenCalledTimes(2); + }); + + it('rejects a dimension mismatch without retrying', async () => { + const f = vi.fn().mockResolvedValueOnce(mockFetchOnce(200, okBody(1, 5))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3 }); + await expect(c.embed(['a'])).rejects.toThrow(/dimension mismatch/); + expect(f).toHaveBeenCalledTimes(1); + }); + + it('accepts any vector length when constructed without a dimension expectation (probe mode)', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(mockFetchOnce(200, okBody(1, 5)))); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: null }); + const out = await c.embed(['a']); + expect(out[0]).toHaveLength(5); // discovered, not asserted — the Test probe path + }); + + it('marks a 400 as a permanent 4xx (no retry)', async () => { + const f = vi.fn().mockResolvedValueOnce(mockFetchOnce(400, 'bad request')); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + const err = await c.embed(['a']).catch((e) => e); + expect(isPermanent4xx(err)).toBe(true); + expect(f).toHaveBeenCalledTimes(1); + }); + + it('gives up after maxRetries on a persistent network error', async () => { + const f = vi.fn().mockRejectedValue(new Error('ECONNREFUSED')); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 1 }); + await expect(c.embed(['a'])).rejects.toThrow(/giving up after 1 attempts/); + expect(f).toHaveBeenCalledTimes(1); + }); + + // Undici keeps the socket checked out until the body is read/cancelled; the + // retry paths throw without decoding a body, so they must drain it first or a + // burst of rate limiting across scheduler ticks leaks connections. + it('consumes the response body on a retryable 429 before throwing', async () => { + const textSpy = vi.fn(async () => 'slow down'); + const resp429 = { status: 429, headers: { get: () => null }, json: async () => ({}), text: textSpy }; + const f = vi.fn() + .mockResolvedValueOnce(resp429) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + await c.embed(['a']); + expect(textSpy).toHaveBeenCalledTimes(1); + }); + + it('consumes the response body on a retryable 5xx before throwing', async () => { + const textSpy = vi.fn(async () => 'upstream boom'); + const resp503 = { status: 503, headers: { get: () => null }, json: async () => ({}), text: textSpy }; + const f = vi.fn() + .mockResolvedValueOnce(resp503) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + await c.embed(['a']); + expect(textSpy).toHaveBeenCalledTimes(1); + }); + + it('still retries a 429 even if draining the body throws (best-effort)', async () => { + const resp429 = { + status: 429, + headers: { get: () => null }, + json: async () => ({}), + text: vi.fn().mockRejectedValue(new Error('body already released')), + }; + const f = vi.fn() + .mockResolvedValueOnce(resp429) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + const out = await c.embed(['a']); + expect(out).toHaveLength(1); + expect(f).toHaveBeenCalledTimes(2); + }); +}); diff --git a/backend/src/services/embeddings/config.js b/backend/src/services/embeddings/config.js new file mode 100644 index 00000000..44679927 --- /dev/null +++ b/backend/src/services/embeddings/config.js @@ -0,0 +1,58 @@ +import { query } from '../db.js'; +import { decrypt } from '../encryption.js'; + +// Bump when preprocess.js changes its output for the same flags (tighten a regex, +// add a tracking param, add a default-on transform). Folds into the generation +// fingerprint so a policy change forces a new generation. Port of config.go preprocessVersion. +export const PREPROCESS_VERSION = 1; +// Bump when worker.js/chunk.js change the vector layout for the same preprocess +// output (chunk window/overlap/cap). Port of config.go embedPolicyVersion. +export const EMBED_POLICY_VERSION = 1; + +const PREPROCESS_FLAG_ORDER = [ + 'stripQuotes', 'stripSignatures', 'stripHTML', 'stripBase64', 'stripURLTracking', 'collapseWhitespace', +]; + +export function preprocessFingerprint(pp) { + const bits = PREPROCESS_FLAG_ORDER.map((k) => (pp[k] ? '1' : '0')).join(''); + return `p${PREPROCESS_VERSION}-${bits}`; +} + +export function generationFingerprint(cfg) { + return `${cfg.model}:${cfg.dimension}:${preprocessFingerprint(cfg.preprocess)}:c${cfg.maxInputChars}:e${EMBED_POLICY_VERSION}`; +} + +function resolveFlag(v) { return v === undefined || v === null ? true : v === true; } + +export function applyEmbedDefaults(raw) { + const pp = raw.preprocess || {}; + return { + enabled: raw.enabled === true, + endpoint: (raw.endpoint || '').trim().replace(/\/+$/, ''), + apiKey: raw.apiKey || null, + model: (raw.model || '').trim(), + dimension: Number(raw.dimension) || 0, + maxInputChars: Number(raw.maxInputChars) > 0 ? Number(raw.maxInputChars) : 32768, + batchSize: Number(raw.batchSize) > 0 ? Number(raw.batchSize) : 32, + skipExtensionCreate: raw.skipExtensionCreate === true, + preprocess: { + stripQuotes: resolveFlag(pp.stripQuotes), + stripSignatures: resolveFlag(pp.stripSignatures), + stripHTML: resolveFlag(pp.stripHTML), + stripBase64: resolveFlag(pp.stripBase64), + stripURLTracking: resolveFlag(pp.stripURLTracking), + collapseWhitespace: resolveFlag(pp.collapseWhitespace), + }, + }; +} + +export async function resolveEmbedConfig() { + const result = await query("SELECT value FROM system_settings WHERE key = 'ai_config'"); + if (!result.rows.length) return null; + let cfg; + try { cfg = JSON.parse(result.rows[0].value); } catch { return null; } + if (!cfg.embeddings) return null; + const resolved = applyEmbedDefaults(cfg.embeddings); + resolved.apiKey = resolved.apiKey ? decrypt(resolved.apiKey) : null; + return resolved; +} diff --git a/backend/src/services/embeddings/config.test.js b/backend/src/services/embeddings/config.test.js new file mode 100644 index 00000000..cb196dec --- /dev/null +++ b/backend/src/services/embeddings/config.test.js @@ -0,0 +1,53 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../encryption.js', () => ({ decrypt: (v) => (v ? v.replace(/^enc:/, '') : v) })); +const { query } = await import('../db.js'); +import { preprocessFingerprint, generationFingerprint, applyEmbedDefaults, resolveEmbedConfig } from './config.js'; + +const allOn = { stripQuotes: true, stripSignatures: true, stripHTML: true, stripBase64: true, stripURLTracking: true, collapseWhitespace: true }; + +describe('preprocessFingerprint', () => { + it('is p1-111111 when all flags on', () => { + expect(preprocessFingerprint(allOn)).toBe('p1-111111'); + }); + it('flips the 3rd bit (strip_html) off in field-declaration order', () => { + expect(preprocessFingerprint({ ...allOn, stripHTML: false })).toBe('p1-110111'); + }); +}); + +describe('generationFingerprint', () => { + it('joins model:dim:preprocess:c:e', () => { + const cfg = { model: 'nomic-embed-text', dimension: 768, maxInputChars: 32768, preprocess: allOn }; + expect(generationFingerprint(cfg)).toBe('nomic-embed-text:768:p1-111111:c32768:e1'); + }); +}); + +describe('applyEmbedDefaults', () => { + it('fills batchSize 32, maxInputChars 32768, all preprocess flags on', () => { + const out = applyEmbedDefaults({ model: 'm', dimension: 4 }); + expect(out.batchSize).toBe(32); + expect(out.maxInputChars).toBe(32768); + expect(out.preprocess).toEqual(allOn); + }); + it('preserves an explicit false preprocess flag', () => { + const out = applyEmbedDefaults({ model: 'm', dimension: 4, preprocess: { stripQuotes: false } }); + expect(out.preprocess.stripQuotes).toBe(false); + expect(out.preprocess.stripHTML).toBe(true); + }); +}); + +describe('resolveEmbedConfig', () => { + it('returns null when ai_config has no embeddings block', async () => { + query.mockResolvedValueOnce({ rows: [{ value: JSON.stringify({ baseUrl: 'x' }) }] }); + expect(await resolveEmbedConfig()).toBeNull(); + }); + it('decrypts the apiKey and applies defaults', async () => { + query.mockResolvedValueOnce({ rows: [{ value: JSON.stringify({ + embeddings: { enabled: true, endpoint: 'http://h:8080/v1', apiKey: 'enc:secret', model: 'm', dimension: 4 }, + }) }] }); + const cfg = await resolveEmbedConfig(); + expect(cfg.apiKey).toBe('secret'); + expect(cfg.batchSize).toBe(32); + expect(cfg.model).toBe('m'); + }); +}); diff --git a/backend/src/services/embeddings/embedRunLock.js b/backend/src/services/embeddings/embedRunLock.js new file mode 100644 index 00000000..f51df0e3 --- /dev/null +++ b/backend/src/services/embeddings/embedRunLock.js @@ -0,0 +1,18 @@ +// Process-wide single-flight guard for the embed worker. Both the periodic scheduler +// tick and the manual admin build acquire this before running a worker, so at most one +// runs at a time in-process — a scheduler tick skips while a manual build is active, and +// a manual build defers while a scheduler tick is mid-run (C1). It is a plain in-memory +// flag: it does not coordinate across processes (a multi-instance deploy relies on the +// idempotent upsert + CAS + coverage gate for cross-process safety, not this lock). +let _active = false; + +// Returns true and takes the lock when free; returns false when already held. +export function tryAcquireEmbedRun() { + if (_active) return false; + _active = true; + return true; +} + +export function releaseEmbedRun() { _active = false; } + +export function isEmbedRunActive() { return _active; } diff --git a/backend/src/services/embeddings/embedRunLock.test.js b/backend/src/services/embeddings/embedRunLock.test.js new file mode 100644 index 00000000..e65eb843 --- /dev/null +++ b/backend/src/services/embeddings/embedRunLock.test.js @@ -0,0 +1,20 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { tryAcquireEmbedRun, releaseEmbedRun, isEmbedRunActive } from './embedRunLock.js'; + +beforeEach(() => releaseEmbedRun()); + +describe('embedRunLock (single-flight)', () => { + it('grants the first acquire and refuses a second while held', () => { + expect(isEmbedRunActive()).toBe(false); + expect(tryAcquireEmbedRun()).toBe(true); + expect(isEmbedRunActive()).toBe(true); + expect(tryAcquireEmbedRun()).toBe(false); // second caller (scheduler vs manual build) is refused + }); + + it('re-acquires after release', () => { + expect(tryAcquireEmbedRun()).toBe(true); + releaseEmbedRun(); + expect(isEmbedRunActive()).toBe(false); + expect(tryAcquireEmbedRun()).toBe(true); + }); +}); diff --git a/backend/src/services/embeddings/embedScan.integration.test.js b/backend/src/services/embeddings/embedScan.integration.test.js new file mode 100644 index 00000000..f37353f8 --- /dev/null +++ b/backend/src/services/embeddings/embedScan.integration.test.js @@ -0,0 +1,63 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// S1: the steady-state embed scan must be O(pending), not O(mailbox). The partial +// index idx_messages_embed_pending (migration 0039) only helps if the scan predicate +// is `embed_gen IS NULL` (an OR with `embed_gen <> target` forces a seq scan), which +// in turn is only correct if createGeneration resets stale stamps to NULL so a +// generation rebuild still finds prior-generation rows via the NULL scan. +d('embed scan O(pending) + reset-on-create', () => { + let store, gens, client, acctId, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + gens = await import('./generations.js'); + await store.ensureVectorSchema(); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'scan')); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('scanForEmbedding returns only embed_gen-IS-NULL rows (steady state)', async () => { + // 2000 stamped under an active-like gen id, 3 new (NULL). + await client.query("INSERT INTO messages (account_id, uid, folder, subject, embed_gen) SELECT $1, 500000+g, 'INBOX', 'x', 424242 FROM generate_series(0,1999) g", [acctId]); + const nulls = await client.query("INSERT INTO messages (account_id, uid, folder, subject, embed_gen) SELECT $1, 520000+g, 'INBOX', 'x', NULL FROM generate_series(0,2) g RETURNING id", [acctId]); + const nullIds = nulls.rows.map((r) => r.id).sort(); + const got = (await store.scanForEmbedding('424242', store.ZERO_UUID, 100)).sort(); + expect(got).toEqual(nullIds); + }); + + it('the scan plan uses the partial pending index (no Seq Scan)', async () => { + await client.query('ANALYZE messages'); + const r = await client.query( + `EXPLAIN SELECT id FROM messages WHERE embed_gen IS NULL AND is_deleted = false AND id > $1 ORDER BY id LIMIT $2`, + [store.ZERO_UUID, 32], + ); + const plan = r.rows.map((x) => x['QUERY PLAN']).join('\n'); + expect(plan).toMatch(/idx_messages_embed_pending/); + expect(plan).not.toMatch(/Seq Scan/); + }); + + it('createGeneration resets stale non-null stamps to NULL (so a rebuild re-finds them)', async () => { + // All current rows are stamped 424242 (a prior gen). A new-fingerprint generation + // must reset them to NULL so scanForEmbedding finds them for re-embedding. + const before = await client.query('SELECT COUNT(*)::int n FROM messages WHERE embed_gen IS NULL AND account_id = $1', [acctId]); + expect(before.rows[0].n).toBe(3); // only the 3 new rows are NULL pre-create + const g = await gens.createGeneration('m', 4, 'fp-reset'); + const after = await client.query('SELECT COUNT(*)::int nulls, COUNT(*) FILTER (WHERE embed_gen IS NOT NULL)::int stamped FROM messages WHERE account_id = $1', [acctId]); + expect(after.rows[0].stamped).toBe(0); // every live stamp reset + expect(after.rows[0].nulls).toBe(2003); // all rows now pending for the new gen + // and the scan now surfaces them (bounded here by limit) + const pending = await store.scanForEmbedding(g, store.ZERO_UUID, 100); + expect(pending.length).toBe(100); + // cleanup the building gen so later IT files start clean + await gens.retireGeneration(g, true); + }); +}); diff --git a/backend/src/services/embeddings/generations.chunkCount.test.js b/backend/src/services/embeddings/generations.chunkCount.test.js new file mode 100644 index 00000000..e3181083 --- /dev/null +++ b/backend/src/services/embeddings/generations.chunkCount.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Unit-pin generations.chunkCount, the seam get_stats' collectStats calls for +// active_generation.message_count and the building generation's progress.done. +// It was listed in the frozen cross-phase contract but never implemented in the +// real module (only ever a vi.fn() in test mocks) — so live collectStats threw +// "generations.chunkCount is not a function" and the vector_search block vanished. +vi.mock('../db.js', () => ({ pool: { query: vi.fn() }, withTransaction: vi.fn() })); +import { pool } from '../db.js'; +import * as generations from './generations.js'; + +beforeEach(() => { pool.query.mockReset(); }); + +describe('generations.chunkCount (msgvault Stats.EmbeddingCount semantics)', () => { + it('counts DISTINCT message_id (not chunk rows) scoped to the generation, returned as a number', async () => { + // msgvault: "EmbeddingCount is distinct messages, not chunk rows". Live parity: + // 18,193 chunk rows / 18,192 distinct messages → message_count 18,192. + pool.query.mockResolvedValueOnce({ rows: [{ n: '18192' }] }); + const n = await generations.chunkCount(7); + expect(n).toBe(18192); + expect(typeof n).toBe('number'); + const [sql, params] = pool.query.mock.calls[0]; + expect(sql).toMatch(/COUNT\(\s*DISTINCT\s+message_id\s*\)/i); + expect(sql).not.toMatch(/COUNT\(\s*\*\s*\)/); // raw chunk rows would inflate message_count + expect(sql).toMatch(/FROM\s+embeddings/i); + expect(sql).toMatch(/generation_id\s*=\s*\$1/); + expect(params).toEqual([7]); + }); +}); diff --git a/backend/src/services/embeddings/generations.integration.test.js b/backend/src/services/embeddings/generations.integration.test.js new file mode 100644 index 00000000..b1d4a5ed --- /dev/null +++ b/backend/src/services/embeddings/generations.integration.test.js @@ -0,0 +1,60 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('generations lifecycle', () => { + let gens, store, client, acctId, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + gens = await import('./generations.js'); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // Clean slate for generations (integration DB is shared across tasks). embed_runs + // FKs index_generations without ON DELETE CASCADE, so clear it before the parent. + await client.query('DELETE FROM embeddings'); + await client.query('DELETE FROM embed_runs'); + await client.query('DELETE FROM index_generations'); + ({ accountId: acctId, userId } = await seedAccount(client, 'gen')); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('creates a building generation, then a same-fingerprint create resumes it', async () => { + const g1 = await gens.createGeneration('m', 4, 'fp-A'); + const g2 = await gens.createGeneration('m', 4, 'fp-A'); + expect(g2).toBe(g1); + const building = await gens.buildingGeneration(); + expect(building.id).toBe(g1); + }); + + it('rejects a mismatched-fingerprint create while building', async () => { + await expect(gens.createGeneration('m', 4, 'fp-B')).rejects.toBeInstanceOf(gens.BuildingInProgressError); + }); + + it('blocks activation while a live message still needs embedding', async () => { + const g = (await gens.buildingGeneration()).id; + await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 82001, 'INBOX', 'needs work')`, [acctId]); + await expect(gens.activateGeneration(g)).rejects.toThrow(/needing embedding/); + }); + + it('activates once coverage is complete and deletes retired rows on the next activate', async () => { + const g = (await gens.buildingGeneration()).id; + await client.query('UPDATE messages SET embed_gen = $1 WHERE account_id = $2', [g, acctId]); + await gens.activateGeneration(g); + expect((await gens.activeGeneration()).id).toBe(g); + + // New generation covering the same corpus; activating it must delete g's rows. + await client.query(`INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, dimension, embedding) + SELECT $1, id, 0, 0, 1, 4, '[1,0,0,0]' FROM messages WHERE account_id = $2`, [g, acctId]); + const g3 = await gens.createGeneration('m', 4, 'fp-C'); + await client.query('UPDATE messages SET embed_gen = $1 WHERE account_id = $2', [g3, acctId]); + await gens.activateGeneration(g3); + const left = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1', [g]); + expect(left.rows[0].n).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/generations.js b/backend/src/services/embeddings/generations.js new file mode 100644 index 00000000..79d03e9c --- /dev/null +++ b/backend/src/services/embeddings/generations.js @@ -0,0 +1,145 @@ +import { pool, withTransaction } from '../db.js'; +import { ensureVectorIndex } from './vectorStore.js'; + +export class BuildingInProgressError extends Error {} +export class NoActiveGenerationError extends Error {} + +const LIVE = 'is_deleted = false'; + +// Coverage gate predicate: a generation is fully covered when NO live message still +// needs embedding for it. Port of backend.go missingForGenExistsClause. +export async function coverageMissing(gen) { + const r = await pool.query( + `SELECT EXISTS (SELECT 1 FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $1) AND ${LIVE}) missing`, + [gen], + ); + return r.rows[0].missing; +} + +// activated_at/started_at are epoch SECONDS (bigint); get_stats' generation +// summaries surface them (active → activated_at, building → started_at), so both +// must be selected here or vectorStats only ever sees undefined and the wire +// fields collapse to "". node-pg returns bigint as a string; the wire formatter +// in vectorStats coerces + renders RFC3339. +async function generationByState(state) { + const r = await pool.query( + `SELECT id, model, dimension, fingerprint, state, message_count AS "messageCount", + activated_at AS "activatedAt", started_at AS "startedAt" + FROM index_generations WHERE state = $1`, + [state], + ); + return r.rows[0] || null; +} + +export async function activeGeneration() { return generationByState('active'); } +export async function buildingGeneration() { return generationByState('building'); } + +// Distinct embedded-message count for a generation — a direct port of msgvault's +// Backend.Stats EmbeddingCount (internal/vector/pgvector/backend.go): "distinct +// messages, not chunk rows — a long message occupies multiple rows but counts as +// one embedded message". get_stats' collectStats consumes it for both +// active_generation.message_count and the building generation's progress.done, +// so COUNT(DISTINCT message_id) is required — COUNT(*) would inflate the count by +// every extra chunk a long message carries (live: 18,193 chunk rows vs 18,192 +// distinct messages → message_count 18,192, matching the index_generations +// .message_count column upsert maintains). Scoped to one generation_id (msgvault's +// gen != 0 path); the aggregate gen == 0 path is unused here. Kept named +// `chunkCount` per the frozen cross-phase contract (README vectorStore bullet) +// even though the value is a message count, not a chunk count. +export async function chunkCount(gen) { + const r = await pool.query( + 'SELECT COUNT(DISTINCT message_id)::bigint AS n FROM embeddings WHERE generation_id = $1', + [gen], + ); + return Number(r.rows[0].n); +} + +// Claim-or-insert a building generation. A same-fingerprint building row is resumed +// (returns its id); a mismatched building fingerprint → BuildingInProgressError. Ensures +// the per-dimension HNSW index exists first (port of backend.go CreateGeneration). +export async function createGeneration(model, dim, fingerprint) { + await ensureVectorIndex(dim); + const fp = fingerprint || `${model}:${dim}`; + const now = Math.floor(Date.now() / 1000); + const existing = await buildingGeneration(); + if (existing) { + if (existing.fingerprint !== fp) { + throw new BuildingInProgressError(`building fingerprint=${existing.fingerprint}, requested=${fp} — activate or retire it first`); + } + return existing.id; + } + try { + return await withTransaction(async (client) => { + await client.query('SET LOCAL statement_timeout = 0'); // corpus-size reset may exceed 30s + const r = await client.query( + `INSERT INTO index_generations (model, dimension, fingerprint, started_at, seeded_at, state) + VALUES ($1,$2,$3,$4,$4,'building') RETURNING id`, + [model, dim, fp, now], + ); + // Reset every live row's stale stamp so scanForEmbedding (embed_gen IS NULL) + // finds each row that must be (re)embedded under the new generation — the + // invariant that lets the scan use the partial pending index. Atomic with the + // insert so a crash never strands prior-generation rows the NULL-only scan can't + // see. Skips already-NULL rows; only fires the last_modified trigger's WHEN + // clause on subject/body columns, so a pure embed_gen reset never bumps it. + await client.query('UPDATE messages SET embed_gen = NULL WHERE embed_gen IS NOT NULL AND is_deleted = false'); + return r.rows[0].id; + }); + } catch (err) { + if (err.code === '23505') { // unique_violation on idx_generations_building — a race + const raced = await buildingGeneration(); + if (raced && raced.fingerprint === fp) return raced.id; + throw new BuildingInProgressError(`building generation exists with a different fingerprint`); + } + throw err; + } +} + +// Retire the current active (if any) and promote gen. The auto-retire DELETEs the +// demoted generation's rows in the same tx (shared per-dimension HNSW graph, README +// invariant). Coverage gate folded into the promote UPDATE unless force. +export async function activateGeneration(gen, force = false) { + const now = Math.floor(Date.now() / 1000); + await withTransaction(async (client) => { + await client.query('SET LOCAL statement_timeout = 0'); // corpus-size DELETE may exceed 30s + const demoted = await client.query( + `UPDATE index_generations SET state = 'retired', completed_at = COALESCE(completed_at, $1) + WHERE state = 'active' RETURNING id`, + [now], + ); + if (demoted.rows.length) { + await client.query('DELETE FROM embeddings WHERE generation_id = $1', [demoted.rows[0].id]); + } + const res = await client.query( + `UPDATE index_generations + SET state = 'active', activated_at = $1, completed_at = COALESCE(completed_at, $2) + WHERE id = $3 AND state = 'building' + AND ($4 OR NOT EXISTS (SELECT 1 FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $3) AND ${LIVE}))`, + [now, now, gen, force], + ); + if (res.rowCount === 0) { + const st = await client.query('SELECT state FROM index_generations WHERE id = $1', [gen]); + if (!st.rows.length) throw new Error(`unknown generation ${gen}`); + if (st.rows[0].state !== 'building') throw new Error(`generation ${gen} not in 'building' state`); + throw new Error(`generation ${gen} still has messages needing embedding; pass force to override`); + } + }); +} + +// Mark gen retired and DELETE its rows. Refuses to retire an active generation unless force. +export async function retireGeneration(gen, force = false) { + await withTransaction(async (client) => { + await client.query('SET LOCAL statement_timeout = 0'); + const res = await client.query( + `UPDATE index_generations SET state = 'retired' WHERE id = $1 AND ($2 OR state != 'active')`, + [gen, force], + ); + if (res.rowCount === 0) { + const st = await client.query('SELECT state FROM index_generations WHERE id = $1', [gen]); + if (!st.rows.length) throw new Error(`unknown generation ${gen}`); + if (st.rows[0].state === 'active' && !force) throw new Error(`refusing to retire active generation ${gen} without force`); + throw new Error(`retire generation ${gen}: no rows affected (state=${st.rows[0].state})`); + } + await client.query('DELETE FROM embeddings WHERE generation_id = $1', [gen]); + }); +} diff --git a/backend/src/services/embeddings/generations.summaryFields.test.js b/backend/src/services/embeddings/generations.summaryFields.test.js new file mode 100644 index 00000000..1e5b9f6f --- /dev/null +++ b/backend/src/services/embeddings/generations.summaryFields.test.js @@ -0,0 +1,38 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// generationByState feeds get_stats' active/building generation summaries. It must +// SELECT the epoch-seconds timestamp columns those summaries emit — activated_at +// (active) and started_at (building) — or vectorStats only ever sees undefined and +// the wire fields collapse to "". (Same family as the chunkCount gap.) +vi.mock('../db.js', () => ({ pool: { query: vi.fn() }, withTransaction: vi.fn() })); +import { pool } from '../db.js'; +import { activeGeneration, buildingGeneration } from './generations.js'; + +beforeEach(() => { pool.query.mockReset(); }); + +describe('generationByState surfaces the timestamp columns get_stats emits', () => { + it('activeGeneration SELECTs activated_at + started_at (camelCase) and returns them', async () => { + pool.query.mockResolvedValueOnce({ rows: [{ + id: 1, model: 'm', dimension: 4, fingerprint: 'f', state: 'active', + messageCount: 5, activatedAt: 1784235357, startedAt: 1784200000, + }] }); + const g = await activeGeneration(); + const [sql, params] = pool.query.mock.calls[0]; + expect(sql).toMatch(/activated_at AS "activatedAt"/); + expect(sql).toMatch(/started_at AS "startedAt"/); + expect(params).toEqual(['active']); + expect(g.activatedAt).toBe(1784235357); + expect(g.startedAt).toBe(1784200000); + }); + + it('buildingGeneration uses the same SELECT so started_at is surfaced', async () => { + pool.query.mockResolvedValueOnce({ rows: [{ + id: 2, model: 'm', dimension: 4, fingerprint: 'f', state: 'building', + messageCount: 0, startedAt: 1784200000, + }] }); + const g = await buildingGeneration(); + expect(pool.query.mock.calls[0][0]).toMatch(/started_at AS "startedAt"/); + expect(pool.query.mock.calls[0][1]).toEqual(['building']); + expect(g.startedAt).toBe(1784200000); + }); +}); diff --git a/backend/src/services/embeddings/hybrid.js b/backend/src/services/embeddings/hybrid.js new file mode 100644 index 00000000..8d1cfc64 --- /dev/null +++ b/backend/src/services/embeddings/hybrid.js @@ -0,0 +1,135 @@ +import { resolveEmbedConfig, generationFingerprint } from './config.js'; +import { resolveActiveGeneration, resolveActiveGenerationFromConfig } from './activeGeneration.js'; +import { VectorUnavailableError } from './vectorErrors.js'; +import { EmbeddingClient } from './client.js'; +import { fusedSearch } from './vectorStore.js'; + +const RRF_K = 60; +const K_PER_SIGNAL = 100; +const SUBJECT_BOOST = 2.0; + +export class MissingFreeTextError extends Error { + constructor() { super('missing_free_text'); this.name = 'MissingFreeTextError'; this.code = 'MISSING_FREE_TEXT'; } +} + +const ORCHESTRATOR_DEFAULTS = { + resolveEmbedConfig, + generationFingerprint, // sync (cfg) => string + resolveActiveGeneration, // throws VectorUnavailableError + resolveActiveGenerationFromConfig, + makeClient: (cfg) => new EmbeddingClient(cfg), + fusedSearch, +}; + +// Port of msgvault's internal/vector/hybrid/{engine,rrf}.go orchestration: +// embed the free text once, resolve the active generation by fingerprint, +// dispatch to fusedSearch (mode:'vector' skips the BM25 leg), then rerank in +// JS. Every degradation (disabled/absent embed config, stale/building/no +// generation, a transient embed failure) collapses to VectorUnavailableError +// so searchService has one predicate (isLexicalFallback) to decide whether +// to fall back silently (REST) or raise (MCP strictVector). +export async function hybridSearch(req, overrides = {}) { + const d = { ...ORCHESTRATOR_DEFAULTS, ...overrides }; + const { mode, accountIds, buildFilters, limit } = req; + const freeText = (req.freeText || '').trim(); + if (!freeText) throw new MissingFreeTextError(); + + // One owner (activeGeneration.js) resolves the embed config, rejects a disabled/absent + // one as VectorUnavailableError('vector_not_enabled'), and resolves the active generation + // by fingerprint (throws index_stale | index_building | no_active_generation). Fakes thread + // through `d`. + const { cfg, generation } = await d.resolveActiveGenerationFromConfig(d); + + let queryVec; + try { + const vecs = await d.makeClient(cfg).embed([freeText]); + if (!Array.isArray(vecs) || vecs.length !== 1) { + throw new Error(`embedder returned ${vecs && vecs.length} vectors, want 1`); + } + queryVec = vecs[0]; + } catch (err) { + if (err instanceof VectorUnavailableError) throw err; + // Embed-step failure (network/timeout/bad upstream response). The config + // already resolved as ENABLED above, so this is transient unavailability, + // not "vector search is not configured" — msgvault's engine.go wraps it + // as ErrEmbeddingTimeout the same way. The reason string is a frozen + // contract with the MCP error mapper (mcp/vectorErrors.js). REST behavior + // is unchanged: isLexicalFallback still matches, so the route silently + // falls back to lexical with fellBack:true. + throw new VectorUnavailableError('embedding_timeout'); + } + + const subjectTerms = freeText.toLowerCase().split(/\s+/).filter(w => w.length >= 2); + // The subject boost is a lexical-signal nudge — msgvault's vector-only path + // never applies it, and it "has no meaning without a BM25 + // leg to nudge against". `wantBoost` only decides whether to fetch a WIDENED + // pool (so the boost can reorder without the SQL LIMIT pre-cutting + // candidates); whether the boost actually runs is gated below on the BM25 + // leg having contributed a hit. + const wantBoost = mode === 'hybrid' && SUBJECT_BOOST > 1.0 && subjectTerms.length > 0; + const sqlLimit = wantBoost ? Math.max(2 * K_PER_SIGNAL, limit) : limit; + + const { hits, poolSaturated, generation: gen } = await d.fusedSearch({ + ftsQuery: mode === 'vector' ? null : freeText, + queryVec, + generation, + accountIds, + buildFilters, + rrfK: RRF_K, + kPerSignal: K_PER_SIGNAL, + limit: sqlLimit, + }); + + // Gate the boost on the BM25 leg actually contributing (any hit with a + // non-null bm25_score). When the FTS AND-leg is empty — no message contains + // ALL query terms, the common case for paraphrase / natural-language queries + // — the fused ranking IS the pure-ANN ranking, so a subject-substring rerank + // over the widened pool would only EVICT keyword-free-subject hits that + // vector correctly surfaced. Real-corpus eval (2026-07-16) measured hybrid + // losing to pure vector on paraphrase R@5/R@20/MRR for exactly this reason. + // Gating makes hybrid ranking == vector ranking when there is no lexical + // signal, while keyword queries (BM25 leg live) keep the nudge that made + // hybrid win there. Documented divergence from msgvault (boosts uncondition- + // ally); see specs/search-overhaul/README.md "Semantic excerpt seam". + const bm25Contributed = hits.some(h => h.bm25_score != null); + let ranked = wantBoost && bm25Contributed ? applySubjectBoost(hits, subjectTerms, SUBJECT_BOOST) : hits; + if (ranked.length > limit) ranked = ranked.slice(0, limit); + return { hits: ranked, poolSaturated, generation: gen }; +} + +export function isLexicalFallback(err) { + return err instanceof VectorUnavailableError || err instanceof MissingFreeTextError; +} + +// hybrid.js is the module's public face: re-export the vector-unavailability types so +// searchService and the MCP handlers pull them from one place. +// Definition stays in the leaf modules — no vectorStore → hybrid import cycle. +export { VectorUnavailableError } from './vectorErrors.js'; +export { resolveActiveGeneration, resolveActiveGenerationFromConfig } from './activeGeneration.js'; + +// Case-insensitive UUID-ascending tie-break, matching the SQL ORDER BY. +function byScoreThenId(a, b) { + if (b.rrf_score !== a.rrf_score) return b.rrf_score - a.rrf_score; + return a.message_id < b.message_id ? -1 : a.message_id > b.message_id ? 1 : 0; +} + +// JS subject-boost rerank (port of fused.go:429-469 + rrf.go boost logic). +// Multiplies rrf_score for any hit whose subject contains one of the +// lowercased free-text terms as a case-insensitive substring, then re-sorts. +// No-op when boost <= 1 or subjectTerms is empty. +export function applySubjectBoost(hits, subjectTerms, boost) { + // Only "boost disabled" skips the whole pass (including the tie-break + // sort) — a boost-enabled call with no terms still re-sorts deterministically, + // it just boosts nothing. + if (!(boost > 1.0)) return hits; + const terms = (subjectTerms || []).map(t => (t || '').toLowerCase()).filter(Boolean); + for (const hit of hits) { + const subj = (hit.subject || '').toLowerCase(); + if (subj && terms.length && terms.some(t => subj.includes(t))) { + hit.rrf_score *= boost; + hit.subject_boosted = true; + } + } + hits.sort(byScoreThenId); + return hits; +} diff --git a/backend/src/services/embeddings/hybrid.test.js b/backend/src/services/embeddings/hybrid.test.js new file mode 100644 index 00000000..22cb2aeb --- /dev/null +++ b/backend/src/services/embeddings/hybrid.test.js @@ -0,0 +1,177 @@ +import { describe, it, expect } from 'vitest'; +import { applySubjectBoost, hybridSearch, isLexicalFallback, MissingFreeTextError } from './hybrid.js'; +import { VectorUnavailableError } from './vectorErrors.js'; + +const h = (id, rrf, subject) => ({ message_id: id, rrf_score: rrf, subject }); + +describe('applySubjectBoost', () => { + it('lifts a subject-term match above an equal-scoring non-match (port TestFuse_SubjectBoost)', () => { + const hits = [h('a', 1 / 61, 'ordinary email'), h('b', 1 / 61, 'Quarterly Review meeting')]; + const out = applySubjectBoost(hits, ['meeting'], 2.0); + expect(out[0].message_id).toBe('b'); + expect(out[0].subject_boosted).toBe(true); + expect(out.find(x => x.message_id === 'a').subject_boosted).toBeUndefined(); + }); + + it('matches case-insensitively (port TestFuse_SubjectBoost_CaseInsensitive)', () => { + const out = applySubjectBoost([h('a', 1 / 61, 'MEETING Minutes')], ['meeting'], 2.0); + expect(out[0].subject_boosted).toBe(true); + }); + + it('does nothing when boost <= 1 (port TestFuse_NoBoostWhenFlagUnset)', () => { + const out = applySubjectBoost([h('a', 1 / 61, 'meeting subject')], ['meeting'], 1.0); + expect(out[0].subject_boosted).toBeUndefined(); + expect(out[0].rrf_score).toBeCloseTo(1 / 61, 12); + }); + + it('breaks ties by message_id ascending, deterministically (port TestFuse_TiedRRFScoresStableByMessageID)', () => { + for (let i = 0; i < 20; i++) { + const out = applySubjectBoost([h('m7', 0.05, 'x'), h('m3', 0.05, 'y')], [], 2.0); + expect(out.map(x => x.message_id)).toEqual(['m3', 'm7']); + } + }); +}); + +function fakes(over = {}) { + const calls = { embed: [], fused: [] }; + const base = { + resolveEmbedConfig: async () => ({ enabled: true, model: 'm', dimension: 4 }), + generationFingerprint: (cfg) => `${cfg.model}:${cfg.dimension}:test`, + resolveActiveGeneration: async () => ({ id: 1, model: 'm', dimension: 4, fingerprint: 'm:4:test', state: 'active' }), + makeClient: () => ({ embed: async (texts) => { calls.embed.push(texts); return [[1, 0, 0, 0]]; } }), + // Hit 'a' also surfaced via the FTS leg (bm25_score set) so the subject + // boost is legitimately live; 'b' is ANN-only. (The boost is gated on the + // BM25 leg contributing — see the empty-leg regression test below.) + fusedSearch: async (r) => { calls.fused.push(r); return { + hits: [{ message_id: 'a', rrf_score: 0.02, subject: 'quarterly meeting', bm25_score: 0.5, vector_score: 0.7 }, + { message_id: 'b', rrf_score: 0.03, subject: 'ordinary', bm25_score: null, vector_score: 0.8 }], + poolSaturated: false, generation: r.generation }; }, // echoes the rich generation it was given + }; + return { deps: { ...base, ...over }, calls }; +} + +describe('hybridSearch', () => { + const req = (m) => ({ mode: m, freeText: 'quarterly meeting', accountIds: ['acc'], buildFilters: () => [], limit: 10 }); + + it('vector mode calls fusedSearch with ftsQuery=null and embeds once', async () => { + const { deps, calls } = fakes(); + await hybridSearch(req('vector'), deps); + expect(calls.embed).toHaveLength(1); + expect(calls.fused[0].ftsQuery).toBeNull(); + expect(calls.fused[0].queryVec).toEqual([1, 0, 0, 0]); + }); + + it('hybrid mode passes the free text as the BM25 leg', async () => { + const { deps, calls } = fakes(); + await hybridSearch(req('hybrid'), deps); + expect(calls.fused[0].ftsQuery).toBe('quarterly meeting'); + }); + + it('applies the subject boost so a subject-term hit outranks a higher raw RRF (hybrid mode)', async () => { + const { deps } = fakes(); + const { hits } = await hybridSearch(req('hybrid'), deps); + expect(hits[0].message_id).toBe('a'); // 0.02*2 = 0.04 > 0.03 + expect(hits[0].subject_boosted).toBe(true); + }); + + it('does NOT apply the subject boost in vector-only mode — msgvault\'s vector path is pure ANN (review MINOR 1)', async () => { + const { deps } = fakes(); + const { hits } = await hybridSearch(req('vector'), deps); + const a = hits.find(h => h.message_id === 'a'); + expect(a.rrf_score).toBeCloseTo(0.02, 12); // unboosted — NOT 0.02*2 + expect(hits.every(h => h.subject_boosted === undefined)).toBe(true); + }); + + // Regression: real-corpus eval (2026-07-16) showed hybrid LOSING to pure vector on + // paraphrase queries because the BM25 AND-leg is empty (no doc has all terms) yet the + // subject boost still reranked the widened ANN pool by loose substring matches and + // EVICTED keyword-free-subject targets. Gate: an empty BM25 leg ⇒ no boost ⇒ hybrid + // ranking == vector ranking. (Documented divergence from msgvault, which boosts + // unconditionally.) + it('does NOT boost when the BM25 leg is empty — hybrid ranking == vector ranking (eval fix)', async () => { + // Pure-ANN pool: every hit's bm25_score is null (FTS matched nothing). The + // lower-ranked hit's subject contains both query terms; it must NOT be lifted. + const annOnly = async (r) => ({ + hits: [ + { message_id: 'top', rrf_score: 1 / 61, subject: 'no relevant words in here', bm25_score: null, vector_score: 0.90 }, + { message_id: 'low', rrf_score: 1 / 62, subject: 'quarterly meeting notes', bm25_score: null, vector_score: 0.80 }, + ], + poolSaturated: false, generation: r.generation, + }); + const { deps } = fakes({ fusedSearch: annOnly }); + const { hits } = await hybridSearch(req('hybrid'), deps); + expect(hits.map(h => h.message_id)).toEqual(['top', 'low']); // ANN order preserved, no eviction + expect(hits.every(h => !h.subject_boosted)).toBe(true); // nothing boosted + }); + + it('still boosts when the BM25 leg contributed at least one hit (keyword behavior preserved)', async () => { + // Mixed pool: 'low' also surfaced via the FTS leg (bm25_score set), so the lexical + // signal is live and the subject nudge legitimately applies. + const mixed = async (r) => ({ + hits: [ + { message_id: 'top', rrf_score: 1 / 61, subject: 'no relevant words in here', bm25_score: null, vector_score: 0.90 }, + { message_id: 'low', rrf_score: 1 / 62, subject: 'quarterly meeting notes', bm25_score: 0.5, vector_score: 0.80 }, + ], + poolSaturated: false, generation: r.generation, + }); + const { deps } = fakes({ fusedSearch: mixed }); + const { hits } = await hybridSearch(req('hybrid'), deps); + expect(hits[0].message_id).toBe('low'); // (1/62)*2 ≈ 0.0323 > 1/61 ≈ 0.0164 + expect(hits[0].subject_boosted).toBe(true); + }); + + it('rejects a filter-only (no free text) query with MissingFreeTextError', async () => { + const { deps } = fakes(); + const err = await hybridSearch({ ...req('hybrid'), freeText: ' ' }, deps).catch(e => e); + expect(err).toBeInstanceOf(MissingFreeTextError); + expect(isLexicalFallback(err)).toBe(true); + }); + + it('treats a disabled embed config as VectorUnavailableError(vector_not_enabled)', async () => { + const { deps } = fakes({ resolveEmbedConfig: async () => ({ enabled: false }) }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err).toBeInstanceOf(VectorUnavailableError); + expect(err.reason).toBe('vector_not_enabled'); + expect(isLexicalFallback(err)).toBe(true); + }); + + it('treats a missing (null) embed config as VectorUnavailableError(vector_not_enabled)', async () => { + const { deps } = fakes({ resolveEmbedConfig: async () => null }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err.reason).toBe('vector_not_enabled'); + }); + + it('wraps a transient embed failure as VectorUnavailableError(embedding_timeout) — NOT vector_not_enabled (Wave D Fix 6)', async () => { + // The config was already resolved as enabled by this point, so a failed + // embed call is an upstream/transient event (msgvault engine.go wraps it + // as ErrEmbeddingTimeout), not a "vector search is not configured" one. + const { deps } = fakes({ makeClient: () => ({ embed: async () => { throw new Error('econnrefused'); } }) }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err).toBeInstanceOf(VectorUnavailableError); + expect(err.reason).toBe('embedding_timeout'); + expect(isLexicalFallback(err)).toBe(true); // REST still falls back to lexical silently + }); + + it('wraps a malformed embedder response as embedding_timeout too (embed-step failure)', async () => { + const { deps } = fakes({ makeClient: () => ({ embed: async () => [] }) }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err.reason).toBe('embedding_timeout'); + }); + + it('propagates a stale-index VectorUnavailableError from the resolver', async () => { + const stale = new VectorUnavailableError('index_stale'); + const { deps } = fakes({ resolveActiveGeneration: async () => { throw stale; } }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err).toBe(stale); + expect(isLexicalFallback(err)).toBe(true); + }); +}); + +describe('hybrid.js public re-exports', () => { + it('exposes VectorUnavailableError + resolveActiveGeneration from ./hybrid.js (phase-5 imports them here)', async () => { + const mod = await import('./hybrid.js'); + expect(typeof mod.resolveActiveGeneration).toBe('function'); + expect(typeof mod.VectorUnavailableError).toBe('function'); + expect(new mod.VectorUnavailableError('index_stale').reason).toBe('index_stale'); + }); +}); diff --git a/backend/src/services/embeddings/migration0038.integration.test.js b/backend/src/services/embeddings/migration0038.integration.test.js new file mode 100644 index 00000000..a83de0c4 --- /dev/null +++ b/backend/src/services/embeddings/migration0038.integration.test.js @@ -0,0 +1,81 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('0038 last_modified trigger', () => { + let client; + let acctId, userId; + beforeAll(async () => { + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + ({ accountId: acctId, userId } = await seedAccount(client, 'mig')); + }); + afterAll(async () => { + await cleanupAccount(client, userId); + await client.end(); + }); + + async function insertMsg(uid) { + const r = await client.query( + `INSERT INTO messages (account_id, uid, folder, subject, body_text) + VALUES ($1, $2, 'INBOX', 'orig subject', NULL) RETURNING id, last_modified`, + [acctId, uid] + ); + return r.rows[0]; + } + + it('does NOT bump last_modified on a flag-only UPDATE', async () => { + const m = await insertMsg(90001); + await client.query('UPDATE messages SET is_read = true WHERE id = $1', [m.id]); + const after = await client.query('SELECT last_modified FROM messages WHERE id = $1', [m.id]); + expect(after.rows[0].last_modified.getTime()).toBe(m.last_modified.getTime()); + }); + + it('bumps last_modified when body_text changes', async () => { + const m = await insertMsg(90002); + await new Promise(r => setTimeout(r, 5)); + await client.query("UPDATE messages SET body_text = 'a late body arrived' WHERE id = $1", [m.id]); + const after = await client.query('SELECT last_modified FROM messages WHERE id = $1', [m.id]); + expect(after.rows[0].last_modified.getTime()).toBeGreaterThan(m.last_modified.getTime()); + }); + + it('embed_gen defaults to NULL', async () => { + const m = await insertMsg(90003); + const r = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [m.id]); + expect(r.rows[0].embed_gen).toBeNull(); + }); + + it('clears embed_gen (re-surfaces for embedding) when body_text changes post-stamp', async () => { + const m = await insertMsg(90004); + await client.query('UPDATE messages SET embed_gen = 12345 WHERE id = $1', [m.id]); // embedded + stamped + await client.query("UPDATE messages SET body_text = 'a late body arrived' WHERE id = $1", [m.id]); // late body + const r = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [m.id]); + expect(r.rows[0].embed_gen).toBeNull(); // the stale subject-only embedding must be re-done + }); + + it('does NOT clear embed_gen on a stamp-only UPDATE (worker stamp) — trigger does not fire', async () => { + const m = await insertMsg(90005); + const before = await client.query('SELECT last_modified FROM messages WHERE id = $1', [m.id]); + await client.query('UPDATE messages SET embed_gen = 777 WHERE id = $1', [m.id]); + const after = await client.query('SELECT embed_gen, last_modified FROM messages WHERE id = $1', [m.id]); + expect(String(after.rows[0].embed_gen)).toBe('777'); // stamp persists + expect(after.rows[0].last_modified.getTime()).toBe(before.rows[0].last_modified.getTime()); // trigger didn't fire + }); + + it('does NOT clear embed_gen on an identical-content UPSERT (unchanged re-sync)', async () => { + const m = await insertMsg(90006); + await client.query('UPDATE messages SET embed_gen = 888 WHERE id = $1', [m.id]); + await client.query( + `INSERT INTO messages (account_id, uid, folder, subject, body_text) + VALUES ($1, 90006, 'INBOX', 'orig subject', NULL) + ON CONFLICT (account_id, uid, folder) + DO UPDATE SET subject = EXCLUDED.subject, body_text = EXCLUDED.body_text`, + [acctId] + ); + const r = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [m.id]); + expect(String(r.rows[0].embed_gen)).toBe('888'); // unchanged content → trigger no-op → stamp survives + }); +}); diff --git a/backend/src/services/embeddings/postStampReembed.integration.test.js b/backend/src/services/embeddings/postStampReembed.integration.test.js new file mode 100644 index 00000000..ebd216fc --- /dev/null +++ b/backend/src/services/embeddings/postStampReembed.integration.test.js @@ -0,0 +1,62 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// Re-verification catch (slice 05 contract): a row embedded subject-only and stamped, +// whose body arrives LATER (phase-2 drainer / on-open fetch — the primary product flow), +// must re-surface for embedding. The 0038 trigger clears embed_gen on a content change, +// so the NULL-only scan finds it and the idempotent upsert replaces the stale chunks. +d('post-stamp late-body re-embed (end-to-end)', () => { + let store, generations, worker, client, acctId, userId, gen, msgId; + const DIM = 4; + // Vector encodes the source length so a re-embed with a longer (subject+body) input is + // observably different from the subject-only embedding. + const fakeClient = { async embed(inputs) { return inputs.map((t) => [t.length, 0, 0, 0]); } }; + + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + generations = await import('./generations.js'); + const { EmbeddingWorker } = await import('./worker.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(DIM); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'restale')); + const m = await client.query(`INSERT INTO messages (account_id, uid, folder, subject, body_text) VALUES ($1, 84001, 'INBOX', 'meeting notes', NULL) RETURNING id`, [acctId]); + msgId = m.rows[0].id; + gen = await generations.createGeneration('fake', DIM, 'fp-restale'); + worker = new EmbeddingWorker({ store, client: fakeClient, generations, preprocessCfg: {}, maxInputChars: 32768, batchSize: 8 }); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('re-embeds a stamped row after a late body arrives (trigger clears embed_gen)', async () => { + // 1. Subject-only embed + stamp. + await worker.runOnce(gen); + const stamped = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [msgId]); + expect(String(stamped.rows[0].embed_gen)).toBe(String(gen)); + const first = await client.query('SELECT source_char_len FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, msgId]); + expect(first.rows.length).toBe(1); + const subjectOnlyLen = first.rows[0].source_char_len; + + // 2. Late body arrives — the trigger must clear embed_gen so the scan re-finds it. + await client.query("UPDATE messages SET body_text = 'a much longer body that arrived after the subject-only embedding was already stamped' WHERE id = $1", [msgId]); + const cleared = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [msgId]); + expect(cleared.rows[0].embed_gen).toBeNull(); + const pending = await store.scanForEmbedding(gen, store.ZERO_UUID, 10); + expect(pending).toContain(msgId); + + // 3. Re-embed replaces the stale chunk (idempotent upsert) with the longer source text. + await worker.runOnce(gen); + const restamped = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [msgId]); + expect(String(restamped.rows[0].embed_gen)).toBe(String(gen)); + const after = await client.query('SELECT source_char_len FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, msgId]); + expect(after.rows.length).toBe(1); // one chunk — replaced, not duplicated + expect(after.rows[0].source_char_len).toBeGreaterThan(subjectOnlyLen); // re-embedded WITH the body + }); +}); diff --git a/backend/src/services/embeddings/preprocess.js b/backend/src/services/embeddings/preprocess.js new file mode 100644 index 00000000..53934641 --- /dev/null +++ b/backend/src/services/embeddings/preprocess.js @@ -0,0 +1,105 @@ +// Port of internal/vector/embed/preprocess.go. IMPORTANT: any change here that shifts +// output for unchanged flags MUST bump PREPROCESS_VERSION in config.js (folds into the +// generation fingerprint). + +const reReplyPreamble = /^On [^\n]+wrote:\s*\n(?:>+[ \t]?.*\n?)+/gm; +const reSigDelim = /\n--\s*\n[\s\S]*$/; +const reQuoteLine = /^>+[ \t]?.*\n?/gm; +const reStyleBlock = /]*>.*?<\/style>/gis; +const reScriptBlock = /]*>.*?<\/script>/gis; +const reHTMLTag = /<\/?[a-z][a-zA-Z0-9-]*(?:\s+[a-zA-Z_:][a-zA-Z0-9_:.-]*\s*=\s*(?:"[^"]{0,400}"|'[^']{0,400}'|[^\s>"']{1,400}))*\s*\/?>/g; +const reDataURI = /data:[a-zA-Z0-9./+-]{0,128};base64,[A-Za-z0-9+/]+={0,2}/gi; +const reBase64Blob = /[A-Za-z0-9+]{200,}={0,2}/g; +const reBase64BlobWithSlash = /[A-Za-z0-9+/]{300,}={0,2}/g; +const reURL = /https?:\/\/[^\s"'<>)]+/g; +const reTrailingHWS = /[ \t]+$/gm; +const reMultiNewline = /\n{3,}/g; +const reHorizontalRun = /[ \t]{2,}/g; + +const TRACKING_PARAMS = new Set([ + 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id', + 'utm_name', 'utm_brand', 'utm_social', 'fbclid', 'gclid', 'dclid', 'gbraid', + 'wbraid', 'msclkid', 'yclid', 'twclid', 'mc_cid', 'mc_eid', 'ml_subscriber', + '_hsenc', '_hsmi', 'hsctatracking', 'vero_conv', 'vero_id', 'ck_subscriber_id', + '_branch_match_id', 'ref', 'ref_src', 's_cid', 'icid', 'spm', +]); + +// Compact analogue of html.UnescapeString: the named entities seen in body_text prose +// plus numeric (decimal/hex) references. PREPROCESS_VERSION pins this behavior. +const NAMED_ENTITIES = { + amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", '#39': "'", nbsp: ' ', + copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', + ndash: '–', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', +}; +function decodeEntities(s) { + return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (m, ent) => { + if (ent[0] === '#') { + const cp = ent[1] === 'x' || ent[1] === 'X' ? parseInt(ent.slice(2), 16) : parseInt(ent.slice(1), 10); + if (Number.isFinite(cp) && cp > 0 && cp <= 0x10ffff) { try { return String.fromCodePoint(cp); } catch { return m; } } + return m; + } + const v = NAMED_ENTITIES[ent]; + return v !== undefined ? v : m; + }); +} + +function stripTrackingParams(s) { + return s.replace(reURL, (raw) => { + let trailing = ''; + while (raw.length && '.,;:!?)]'.includes(raw[raw.length - 1])) { + trailing = raw[raw.length - 1] + trailing; + raw = raw.slice(0, -1); + } + let u; + try { u = new URL(raw); } catch { return raw + trailing; } + if (!u.host) return raw + trailing; + let dropped = false; + for (const k of [...u.searchParams.keys()]) { + if (TRACKING_PARAMS.has(k.toLowerCase())) { u.searchParams.delete(k); dropped = true; } + } + if (!dropped) return raw + trailing; + return u.toString() + trailing; + }); +} + +function capToRunes(s, maxRunes) { + // Cap s to maxRunes code points without materializing a full array for huge inputs. + let count = 0, u16 = 0; + for (const ch of s) { + if (count >= maxRunes) return { text: s.slice(0, u16), truncated: true }; + count++; u16 += ch.length; + } + return { text: s, truncated: false }; +} + +export function preprocess(subject, body, maxChars, cfg = {}) { + let s = String(body).replace(/\r\n/g, '\n'); + let bodyTruncated = false; + + if (cfg.stripBase64) { + s = s.replace(reDataURI, ' ').replace(reBase64Blob, ' ').replace(reBase64BlobWithSlash, ' '); + } + if (cfg.maxBodyRunes > 0) { + const capped = capToRunes(s, cfg.maxBodyRunes); + s = capped.text; bodyTruncated = capped.truncated; + } + if (cfg.stripHTML) { + s = s.replace(reStyleBlock, ' ').replace(reScriptBlock, ' ').replace(reHTMLTag, ' '); + s = decodeEntities(s); + } + if (cfg.stripURLTracking) s = stripTrackingParams(s); + if (cfg.stripQuotes) s = s.replace(reReplyPreamble, '').replace(reQuoteLine, ''); + if (cfg.stripSignatures) s = s.replace(reSigDelim, ''); + if (cfg.collapseWhitespace) { + s = s.replace(reTrailingHWS, '').replace(reHorizontalRun, ' ').replace(reMultiNewline, '\n\n'); + } + s = s.trim(); + + const prefix = subject ? `Subject: ${subject}\n\n` : ''; + const combined = prefix + s; + + if (maxChars <= 0) return { text: combined, truncated: bodyTruncated }; + const cps = Array.from(combined); + if (cps.length <= maxChars) return { text: combined, truncated: bodyTruncated }; + return { text: cps.slice(0, maxChars).join(''), truncated: true }; +} diff --git a/backend/src/services/embeddings/preprocess.test.js b/backend/src/services/embeddings/preprocess.test.js new file mode 100644 index 00000000..76fbafcc --- /dev/null +++ b/backend/src/services/embeddings/preprocess.test.js @@ -0,0 +1,87 @@ +import { describe, it, expect } from 'vitest'; +import { preprocess } from './preprocess.js'; + +const cases = [ + { name: 'PlainBody', subject: 'Hello', body: "Hi there,\n\nLet's chat tomorrow.", maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: true }, want: "Subject: Hello\n\nHi there,\n\nLet's chat tomorrow.", wantTrunc: false }, + { name: 'StripsQuotedPreamble', subject: 'Re: plan', body: 'My reply.\n\nOn 2026-01-01, alice wrote:\n> previous message\n> more quote', maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: true }, want: 'Subject: Re: plan\n\nMy reply.', wantTrunc: false }, + { name: 'StripsStandaloneQuoteLines', subject: '', body: '> nested quote 1\n> nested quote 2\nActual content.', maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: false }, want: 'Actual content.', wantTrunc: false }, + { name: 'StripsSignature', subject: 'Hi', body: 'Body here.\n-- \nBob\nPhone: ...', maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: true }, want: 'Subject: Hi\n\nBody here.', wantTrunc: false }, + { name: 'SignatureWithoutTrailingSpace', subject: 'Hi', body: 'Body.\n--\nBob', maxChars: 1000, + cfg: { stripQuotes: false, stripSignatures: true }, want: 'Subject: Hi\n\nBody.', wantTrunc: false }, + { name: 'Truncates', subject: 'S', body: 'x'.repeat(2000), maxChars: 100, cfg: {}, lenLE: 100, wantTrunc: true }, + { name: 'TruncateAtRuneBoundary', subject: '', body: '☃'.repeat(50), maxChars: 10, cfg: {}, lenLE: 30, wantTrunc: true }, + { name: 'EmptySubjectAndBody', subject: '', body: '', maxChars: 1000, cfg: {}, want: '', wantTrunc: false }, + { name: 'NoConfigPreservesEverything', subject: 'Hi', body: 'Hello\n> not stripped\n-- \nsig kept', maxChars: 1000, + cfg: { stripQuotes: false, stripSignatures: false }, want: 'Subject: Hi\n\nHello\n> not stripped\n-- \nsig kept', wantTrunc: false }, + { name: 'MaxCharsZeroIsUnlimited', subject: 'S', body: 'x'.repeat(100), maxChars: 0, cfg: {}, want: 'Subject: S\n\n' + 'x'.repeat(100), wantTrunc: false }, + { name: 'MultiByteRunesUnderCap', subject: '', body: '☃'.repeat(33), maxChars: 100, cfg: {}, lenLE: 100, wantTrunc: false }, + { name: 'StripsNestedQuotes', subject: '', body: '>> deep quote\n>>> deeper quote\nReal content.', maxChars: 1000, + cfg: { stripQuotes: true }, want: 'Real content.', wantTrunc: false }, + { name: 'StripsQuoteWithoutSpace', subject: '', body: '>no space after caret\n>another\nKept content.', maxChars: 1000, + cfg: { stripQuotes: true }, want: 'Kept content.', wantTrunc: false }, + { name: 'StripsPreambleWithNestedQuotes', subject: 'Re: topic', body: 'My reply.\n\nOn 2026-04-18, bob wrote:\n>> prior reply\n> and response\n>> more', maxChars: 1000, + cfg: { stripQuotes: true }, want: 'Subject: Re: topic\n\nMy reply.', wantTrunc: false }, + { name: 'StripHTMLDropsTagsAndDecodesEntities', subject: '', body: '

Hello & goodbye


End.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Hello & goodbye End.', wantTrunc: false }, + { name: 'StripHTMLPreservesAngleBracketEmailAddress', subject: '', body: 'Please CC John on the next reply.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Please CC John on the next reply.', wantTrunc: false }, + { name: 'StripHTMLPreservesAngleBracketURL', subject: '', body: 'See .', maxChars: 1000, + cfg: { stripHTML: true }, want: 'See .', wantTrunc: false }, + { name: 'StripHTMLPreservesMathPunctuation', subject: '', body: 'Show me rows where x < 3 and y > 4 but z != 0.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Show me rows where x < 3 and y > 4 but z != 0.', wantTrunc: false }, + { name: 'StripHTMLPreservesAngleBracketDatePlaceholder', subject: '', body: 'Schedule for still good?', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Schedule for still good?', wantTrunc: false }, + { name: 'StripHTMLPreservesMailMergePlaceholder', subject: '', body: 'Hi , welcome to .', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Hi , welcome to .', wantTrunc: false }, + { name: 'StripHTMLDropsSingleQuotedAttrValue', subject: '', body: "Click here now.", maxChars: 1000, + cfg: { stripHTML: true }, want: 'Click here now.', wantTrunc: false }, + { name: 'StripHTMLDropsUnquotedAttrValues', subject: '', body: 'Pixel end.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Pixel end.', wantTrunc: false }, + { name: 'StripHTMLDropsStyleBlock', subject: '', body: 'Hello\n\nWorld', maxChars: 1000, + cfg: { stripHTML: true, collapseWhitespace: true }, want: 'Hello\n\nWorld', wantTrunc: false }, + { name: 'StripHTMLDropsScriptBlock', subject: '', body: 'Pre\n\nPost', maxChars: 1000, + cfg: { stripHTML: true, collapseWhitespace: true }, want: 'Pre\n\nPost', wantTrunc: false }, + { name: 'StripBase64DropsDataURI', subject: '', body: 'Before image data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== After image', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Before image After image', wantTrunc: false }, + { name: 'StripBase64DropsBareBlob', subject: '', body: 'Hello ' + 'A'.repeat(250) + ' world', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Hello world', wantTrunc: false }, + { name: 'StripBase64DropsBareBlobWithSlashes', subject: '', body: 'Before ' + 'abcdefghij/'.repeat(30) + ' After', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Before After', wantTrunc: false }, + { name: 'StripBase64KeepsShortAlphanumeric', subject: '', body: 'Token=abc123XYZ check this hash 0123456789abcdef0123456789abcdef.', maxChars: 1000, + cfg: { stripBase64: true }, want: 'Token=abc123XYZ check this hash 0123456789abcdef0123456789abcdef.', wantTrunc: false }, + { name: 'StripURLTrackingDropsKnownParams', subject: '', body: 'Visit https://example.com/page?utm_source=newsletter&utm_medium=email&fbclid=xyz&id=42 today', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'Visit https://example.com/page?id=42 today', wantTrunc: false }, + { name: 'StripURLTrackingHandlesHubSpotMixedCase', subject: '', body: 'Click https://hub.example.com/cta?hsCtaTracking=abc&id=7 now', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'Click https://hub.example.com/cta?id=7 now', wantTrunc: false }, + { name: 'StripURLTrackingPreservesTrailingPunctuation', subject: '', body: 'See https://example.com/x?utm_source=foo&keep=1.', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'See https://example.com/x?keep=1.', wantTrunc: false }, + { name: 'StripURLTrackingLeavesCleanURLAlone', subject: '', body: 'See https://example.com/page?id=42 and ftp://elsewhere/path', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'See https://example.com/page?id=42 and ftp://elsewhere/path', wantTrunc: false }, + { name: 'CollapseWhitespaceShrinksRuns', subject: '', body: 'Line one\n\n\n\nLine two with big gaps', maxChars: 1000, + cfg: { collapseWhitespace: true }, want: 'Line one\n\nLine two with big gaps', wantTrunc: false }, + { name: 'StripBase64KeepsLongURLPath', subject: '', body: 'https://example.com/' + 'a/b'.repeat(80) + '/end', maxChars: 2000, + cfg: { stripBase64: true }, want: 'https://example.com/' + 'a/b'.repeat(80) + '/end', wantTrunc: false }, + { name: 'StripPipelineSweepsOversizedImgTag', subject: '', body: 'Before ' + 'x' + ' After', maxChars: 5000, + cfg: { stripHTML: true, stripBase64: true, collapseWhitespace: true }, want: 'Before After', wantTrunc: false }, + { name: 'CollapseWhitespaceNormalizesCRLF', subject: '', body: 'Line one \r\n\r\n\r\n\r\nLine two', maxChars: 1000, + cfg: { collapseWhitespace: true }, want: 'Line one\n\nLine two', wantTrunc: false }, + { name: 'PipelineRemovesPollutionEndToEnd', subject: 'Newsletter', + body: '

Hello

\n\n\n' + 'data:image/gif;base64,R0lGODlhAQABAAAAACw= ' + '\n\n\nClick ' + 'https://example.com/?utm_source=x&keep=y' + '\n\n-- \nSig', + maxChars: 1000, cfg: { stripQuotes: true, stripSignatures: true, stripHTML: true, stripBase64: true, stripURLTracking: true, collapseWhitespace: true }, + want: 'Subject: Newsletter\n\nHello\n\nClick https://example.com/?keep=y', wantTrunc: false }, +]; + +describe('preprocess (msgvault fixture parity)', () => { + for (const tt of cases) { + it(tt.name, () => { + const { text, truncated } = preprocess(tt.subject, tt.body, tt.maxChars, tt.cfg); + if (tt.want !== undefined) expect(text).toBe(tt.want); + if (tt.lenLE) expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(tt.lenLE); + expect(truncated).toBe(tt.wantTrunc); + }); + } +}); diff --git a/backend/src/services/embeddings/rankingQuality.test.js b/backend/src/services/embeddings/rankingQuality.test.js new file mode 100644 index 00000000..5bc06aa9 --- /dev/null +++ b/backend/src/services/embeddings/rankingQuality.test.js @@ -0,0 +1,83 @@ +import { describe, it, expect, afterAll } from 'vitest'; +import { hasTestDb, withTestDb, closeTestDb } from './__testdb__.js'; +import { fusedSearch } from './vectorStore.js'; + +afterAll(async () => { await closeTestDb(); }); + +const DIM = 8; +function vec(axis, jitter = 0) { const v = new Array(DIM).fill(0); v[axis] = 1; if (jitter) v[(axis + 1) % DIM] = jitter; return v; } + +// 8 concepts. Each has a keyword doc and a synonym-only doc that shares the +// vector neighbourhood but NOT the keyword. Queries ask by keyword; the gold +// hit is the synonym-only doc that lexical alone would miss. +const CONCEPTS = [ + { kw: 'invoice', syn: 'billing statement', axis: 0 }, + { kw: 'flight', syn: 'boarding pass itinerary', axis: 1 }, + { kw: 'invoice2', syn: 'amount due remittance', axis: 2 }, + { kw: 'meeting', syn: 'calendar sync standup', axis: 3 }, + { kw: 'refund', syn: 'money back reversal', axis: 4 }, + { kw: 'password', syn: 'credential reset link', axis: 5 }, + { kw: 'shipment', syn: 'parcel tracking dispatch', axis: 6 }, + { kw: 'contract', syn: 'signed agreement terms', axis: 7 }, +]; +// 20 labeled queries (concepts reused with different jitter for the extra 12). +const QUERIES = []; +for (let i = 0; i < 20; i++) { + const c = CONCEPTS[i % CONCEPTS.length]; + QUERIES.push({ text: c.kw, qvec: vec(c.axis, (i >= 8 ? 0.15 : 0)), goldAxis: c.axis }); +} + +describe.skipIf(!hasTestDb())('ranking quality: hybrid must not lose to lexical', () => { + it('hybrid recall of semantic-only gold hits >= lexical, and never lower', async () => { + await withTestDb(async (pool) => { + const u = (await pool.query(`INSERT INTO users (username,password_hash) VALUES ('r','x') RETURNING id`)).rows[0].id; + const acc = (await pool.query( + `INSERT INTO email_accounts (user_id,name,email_address,enabled) VALUES ($1,'A','a@x.test',true) RETURNING id`, [u])).rows[0].id; + const gen = (await pool.query( + `INSERT INTO index_generations (model,dimension,fingerprint,started_at,state) VALUES ('m',$1,'m:8:test',$2,'active') RETURNING id`, + [DIM, Math.floor(Date.now() / 1000)])).rows[0].id; + + const goldByAxis = {}; + let uid = 0; + for (const c of CONCEPTS) { + // keyword doc (axis vector + keyword in subject) + await insertDoc(pool, acc, gen, ++uid, `${c.kw} notice`, vec(c.axis), DIM); + // synonym-only doc: gold semantic hit, NO keyword token + const goldId = await insertDoc(pool, acc, gen, ++uid, c.syn, vec(c.axis, 0.05), DIM); + goldByAxis[c.axis] = goldId; + } + + let hybridWins = 0, lexicalWins = 0; + for (const q of QUERIES) { + const goldId = goldByAxis[q.goldAxis]; + const lex = await fusedSearch({ ftsQuery: q.text, queryVec: null, generation: { id: gen, dimension: DIM }, + accountIds: [acc], buildFilters: () => [], rrfK: 60, kPerSignal: 100, limit: 10 }, { client: pool }); + const hyb = await fusedSearch({ ftsQuery: q.text, queryVec: q.qvec, generation: { id: gen, dimension: DIM }, + accountIds: [acc], buildFilters: () => [], rrfK: 60, kPerSignal: 100, limit: 10 }, { client: pool }); + const lexHit = lex.hits.some(h => h.message_id === goldId); + const hybHit = hyb.hits.some(h => h.message_id === goldId); + if (hybHit && !lexHit) hybridWins++; + if (lexHit && !hybHit) lexicalWins++; + // Never lose: whatever lexical surfaced, hybrid still surfaces. + for (const h of lex.hits) { + expect(hyb.hits.some(x => x.message_id === h.message_id)).toBe(true); + } + } + // Semantic-only gold hits: hybrid recovers them, lexical cannot. + expect(hybridWins).toBeGreaterThanOrEqual(QUERIES.length); + expect(lexicalWins).toBe(0); + console.log(`[ranking-quality] hybridWins=${hybridWins} lexicalWins=${lexicalWins} of ${QUERIES.length}`); + }); + }); +}); + +async function insertDoc(pool, acc, gen, uid, subject, embedding, dim) { + const id = (await pool.query( + `INSERT INTO messages (account_id,uid,folder,subject,from_name,from_email,date,snippet,body_text,is_deleted) + VALUES ($1,$2,'INBOX',$3,'S','s@x.test',now(),$3,$3,false) RETURNING id`, [acc, uid, subject])).rows[0].id; + await pool.query( + `INSERT INTO embeddings (generation_id,message_id,chunk_index,embedded_at,source_char_len,embedding,dimension) + VALUES ($1,$2,0,$3,$4,$5::vector,$6)`, + [gen, id, Math.floor(Date.now() / 1000), subject.length, `[${embedding.join(',')}]`, dim]); + return id; +} diff --git a/backend/src/services/embeddings/scheduler.js b/backend/src/services/embeddings/scheduler.js new file mode 100644 index 00000000..9afe7c23 --- /dev/null +++ b/backend/src/services/embeddings/scheduler.js @@ -0,0 +1,91 @@ +import { resolveEmbedConfig, generationFingerprint } from './config.js'; +import { isVectorAvailable } from './vectorStore.js'; +import * as store from './vectorStore.js'; +import * as generations from './generations.js'; +const { buildingGeneration, activeGeneration, retireGeneration } = generations; +import { EmbeddingClient } from './client.js'; +import { EmbeddingWorker } from './worker.js'; +import { tryAcquireEmbedRun, releaseEmbedRun } from './embedRunLock.js'; + +let _timer = null; +let _running = false; +let _lastBackstop = 0; +let _backstopIntervalMs = 86_400_000; +// Latch for the once-per-fingerprint "paused" log below — holds the config +// fingerprint we last warned about so a mismatched active generation does not +// spam the log every tick. Cleared whenever a matching run proceeds. +let _pausedForFingerprint = null; + +// One scheduler pass: drive an existing building-or-active generation. No-ops when +// vector is unavailable, embeddings are disabled, or no generation exists. Exported +// for unit testing. +export async function runSchedulerTick(nowMs) { + if (!isVectorAvailable()) return; + let cfg; + try { cfg = await resolveEmbedConfig(); } catch { return; } + if (!cfg || !cfg.enabled || !cfg.endpoint || !cfg.model || !(cfg.dimension > 0)) return; + + const building = await buildingGeneration(); + const gen = building || await activeGeneration(); + if (!gen) return; + + // Single-flight: skip this tick if a manual build — or another run — already + // holds the shared lock, so the scheduler never double-drives a generation. + if (!tryAcquireEmbedRun()) return; + try { + // Fingerprint guard: the worker embeds with a client built from the *current* + // config, so it must only drive a generation whose fingerprint matches that + // config. If the admin changed model/dimension/preprocess mid-build, the + // resolved generation is stale — driving it would embed with the new client + // into the old generation and fail the dimension check forever. + const cfgFingerprint = generationFingerprint(cfg); + if (gen.fingerprint !== cfgFingerprint) { + if (building) { + // A building gen is superseded by the config change — retire it (deletes its + // rows; generations never mix) so a fresh, correctly-fingerprinted build can + // start. Done under the single-flight lock so it can't race an embed run. + await retireGeneration(gen.id); + console.log(`[embed-scheduler] retired building generation ${gen.id}: fingerprint '${gen.fingerprint}' superseded by config '${cfgFingerprint}'`); + } else if (_pausedForFingerprint !== cfgFingerprint) { + // An active gen can't be retired from under live search — pause incremental + // embedding until a rebuild lands. Log once per config fingerprint, not every tick. + _pausedForFingerprint = cfgFingerprint; + console.log(`[embed-scheduler] active generation ${gen.id} fingerprint '${gen.fingerprint}' != config '${cfgFingerprint}' — pausing incremental embedding until a rebuild`); + } + return; + } + _pausedForFingerprint = null; // a matching run clears the paused-log latch + + const client = new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: cfg.dimension }); + // Pass `generations` so the worker can promote a fully-covered building + // generation to active at its shared run-completion seam (worker.js). + const worker = new EmbeddingWorker({ store, client, generations, preprocessCfg: cfg.preprocess, maxInputChars: cfg.maxInputChars, batchSize: cfg.batchSize }); + + if (nowMs - _lastBackstop >= _backstopIntervalMs) { + _lastBackstop = nowMs; + await worker.runBackstop(gen.id); + } else { + await worker.runOnce(gen.id); + } + } finally { + releaseEmbedRun(); + } +} + +export function startEmbeddingScheduler({ intervalMs = 60000, backstopIntervalMs = 86_400_000 } = {}) { + if (_timer) return; + _backstopIntervalMs = backstopIntervalMs; + _lastBackstop = Date.now(); // first backstop one interval out, not on boot + _timer = setInterval(async () => { + if (_running) return; // never overlap ticks + _running = true; + try { await runSchedulerTick(Date.now()); } + catch (err) { console.error(`Embedding scheduler tick error: ${err.message}`); } + finally { _running = false; } + }, intervalMs); + _timer.unref?.(); +} + +export function stopEmbeddingScheduler() { + if (_timer) { clearInterval(_timer); _timer = null; } +} diff --git a/backend/src/services/embeddings/scheduler.test.js b/backend/src/services/embeddings/scheduler.test.js new file mode 100644 index 00000000..0ca9f7b7 --- /dev/null +++ b/backend/src/services/embeddings/scheduler.test.js @@ -0,0 +1,118 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('./config.js', () => ({ resolveEmbedConfig: vi.fn(), generationFingerprint: vi.fn(() => 'fp') })); +vi.mock('./vectorStore.js', () => ({ isVectorAvailable: vi.fn(() => true) })); +vi.mock('./generations.js', () => ({ buildingGeneration: vi.fn(), activeGeneration: vi.fn(), retireGeneration: vi.fn() })); +const runOnce = vi.fn().mockResolvedValue({ claimed: 0 }); +const runBackstop = vi.fn().mockResolvedValue({ claimed: 0 }); +// Regular (constructable) function impls — arrow-function impls throw +// "is not a constructor" when the module under test does `new EmbeddingWorker(...)`. +vi.mock('./worker.js', () => ({ EmbeddingWorker: vi.fn(function () { return { runOnce, runBackstop }; }) })); +vi.mock('./client.js', () => ({ EmbeddingClient: vi.fn(function () {}) })); + +const { resolveEmbedConfig } = await import('./config.js'); +const { isVectorAvailable } = await import('./vectorStore.js'); +const { buildingGeneration, activeGeneration, retireGeneration } = await import('./generations.js'); +const { runSchedulerTick } = await import('./scheduler.js'); +// Real single-flight lock (not mocked) — the scheduler tick must respect it. +const { tryAcquireEmbedRun, releaseEmbedRun, isEmbedRunActive } = await import('./embedRunLock.js'); + +const enabledCfg = { enabled: true, endpoint: 'http://h/v1', model: 'm', dimension: 4, maxInputChars: 32768, batchSize: 32, preprocess: {} }; + +beforeEach(() => { + releaseEmbedRun(); // ensure the shared single-flight lock starts free each test + // Reset the once-queues so an early-returning test can't leave an unconsumed + // mockResolvedValueOnce that poisons a later test's expectations. + resolveEmbedConfig.mockReset(); + buildingGeneration.mockReset(); + activeGeneration.mockReset(); + retireGeneration.mockReset(); + isVectorAvailable.mockReset().mockReturnValue(true); + runOnce.mockClear(); + runBackstop.mockClear(); +}); + +describe('runSchedulerTick', () => { + it('no-ops when vector is unavailable', async () => { + isVectorAvailable.mockReturnValueOnce(false); + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('no-ops when embeddings config is disabled', async () => { + resolveEmbedConfig.mockResolvedValueOnce({ ...enabledCfg, enabled: false }); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('no-ops when there is no building or active generation', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce(null); + activeGeneration.mockResolvedValueOnce(null); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('runs the worker against the building generation', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + expect(runOnce).toHaveBeenCalledWith('9'); + }); + + it('skips when a run is already active (manual build in flight) — C1', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + expect(tryAcquireEmbedRun()).toBe(true); // simulate a manual build holding the lock + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + releaseEmbedRun(); + }); + + it('releases the single-flight lock after running (does not starve later runs)', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + expect(runOnce).toHaveBeenCalledWith('9'); + expect(isEmbedRunActive()).toBe(false); + }); + + // Fingerprint guard: the config's fingerprint must match the generation the worker + // would drive, or a model/dimension change mid-build embeds with the new client into + // the old generation and wedges forever. + it('retires a building generation whose fingerprint no longer matches the config', async () => { + resolveEmbedConfig.mockResolvedValue(enabledCfg); + buildingGeneration.mockResolvedValue({ id: '9', dimension: 4, fingerprint: 'old-fp' }); + await runSchedulerTick(1000); + expect(retireGeneration).toHaveBeenCalledWith('9'); + expect(runOnce).not.toHaveBeenCalled(); + expect(isEmbedRunActive()).toBe(false); // lock released on the retire-and-return path + }); + + it('skips a mismatched active generation and logs once per fingerprint (no retire)', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + resolveEmbedConfig.mockResolvedValue(enabledCfg); + buildingGeneration.mockResolvedValue(null); + // A matching active run first clears the once-per-fingerprint latch so this test + // is independent of whatever ran before it. + activeGeneration.mockResolvedValueOnce({ id: '7', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + runOnce.mockClear(); + + // Two mismatched ticks: skip both, never retire an active gen, log exactly once. + activeGeneration.mockResolvedValue({ id: '7', dimension: 4, fingerprint: 'old-fp' }); + await runSchedulerTick(2000); + await runSchedulerTick(3000); + + expect(runOnce).not.toHaveBeenCalled(); + expect(retireGeneration).not.toHaveBeenCalled(); + expect(isEmbedRunActive()).toBe(false); + const pauseLogs = logSpy.mock.calls.filter(([m]) => /pausing incremental/i.test(String(m))); + expect(pauseLogs).toHaveLength(1); + } finally { + logSpy.mockRestore(); + } + }); +}); diff --git a/backend/src/services/embeddings/stampSkipped.integration.test.js b/backend/src/services/embeddings/stampSkipped.integration.test.js new file mode 100644 index 00000000..41da2215 --- /dev/null +++ b/backend/src/services/embeddings/stampSkipped.integration.test.js @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// L2: stamp (embed_gen) + stale-vector prune must be ONE transaction. store.stampSkipped +// CAS-stamps rows with a last_modified token, unconditionally stamps rows without one, +// and deletes embeddings only for ids whose stamp actually landed — all atomically. +d('store.stampSkipped (single-tx stamp + prune)', () => { + let store, client, acctId, userId, gen; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'stamp')); + const gi = await client.query(`INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) VALUES ('m',4,'fp',0,'building') RETURNING id`); + gen = gi.rows[0].id; + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + async function seedMsgWithVector(uid) { + const m = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1,$2,'INBOX','x') RETURNING id, last_modified::text lm`, [acctId, uid]); + const id = m.rows[0].id; + await client.query(`INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, dimension, embedding) VALUES ($1,$2,0,0,1,4,'[1,0,0,0]')`, [gen, id]); + // Keep message_count honest for the seeded vector so the decrement + // assertions below observe real deltas (upsert normally maintains this). + await client.query('UPDATE index_generations SET message_count = message_count + 1 WHERE id = $1', [gen]); + return { id, lm: m.rows[0].lm }; + } + + const messageCount = async () => + Number((await client.query('SELECT message_count FROM index_generations WHERE id = $1', [gen])).rows[0].message_count); + + it('CAS-stamps and prunes the vector when last_modified matches', async () => { + const { id, lm } = await seedMsgWithVector(70001); + const before = await messageCount(); + const missed = await store.stampSkipped(gen, [{ id, lastModified: lm }], []); + expect(missed).toEqual([]); + const stamp = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [id]); + expect(String(stamp.rows[0].embed_gen)).toBe(String(gen)); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(0); // pruned in the same tx + expect(await messageCount()).toBe(before - 1); // Fix 7: the prune decrements the generation + }); + + it('a CAS miss decrements nothing (the vector stays)', async () => { + const { id } = await seedMsgWithVector(70004); + const before = await messageCount(); + await store.stampSkipped(gen, [{ id, lastModified: '1999-01-01 00:00:00+00' }], []); + expect(await messageCount()).toBe(before); + }); + + it('on a CAS miss leaves BOTH the stamp and the vector untouched', async () => { + const { id } = await seedMsgWithVector(70002); + const missed = await store.stampSkipped(gen, [{ id, lastModified: '1999-01-01 00:00:00+00' }], []); + expect(missed).toEqual([id]); + const stamp = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [id]); + expect(stamp.rows[0].embed_gen).toBeNull(); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(1); // not pruned — the row still needs work + }); + + it('unconditionally stamps + prunes a plain (missing-row) id', async () => { + const { id } = await seedMsgWithVector(70003); + const missed = await store.stampSkipped(gen, [], [id]); + expect(missed).toEqual([]); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(0); + }); +}); diff --git a/backend/src/testSupport/testAccount.js b/backend/src/services/embeddings/testSupport.js similarity index 100% rename from backend/src/testSupport/testAccount.js rename to backend/src/services/embeddings/testSupport.js diff --git a/backend/src/services/embeddings/vectorErrors.js b/backend/src/services/embeddings/vectorErrors.js new file mode 100644 index 00000000..e6516b8d --- /dev/null +++ b/backend/src/services/embeddings/vectorErrors.js @@ -0,0 +1,5 @@ +// Leaf module: imports nothing from vectorStore/hybrid so Phase 3's loadVector +// and Phase 5's MCP handlers can import this without a cycle. +export class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } +} diff --git a/backend/src/services/embeddings/vectorSchema.integration.test.js b/backend/src/services/embeddings/vectorSchema.integration.test.js new file mode 100644 index 00000000..f3501a24 --- /dev/null +++ b/backend/src/services/embeddings/vectorSchema.integration.test.js @@ -0,0 +1,45 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('ensureVectorSchema (pgvector image)', () => { + let store, client; + beforeAll(async () => { + // Point db.js's pool at the test DB via env before importing the module graph. + const u = new URL(DSN); + process.env.DB_HOST = u.hostname; + process.env.DB_PORT = u.port; + process.env.DB_NAME = u.pathname.slice(1); + process.env.DB_USER = u.username; + process.env.DB_PASSWORD = u.password; + store = await import('./vectorStore.js'); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + }); + afterAll(async () => { await client.end(); }); + + it('creates the vector schema and reports available', async () => { + const { vectorAvailable } = await store.ensureVectorSchema(); + expect(vectorAvailable).toBe(true); + expect(store.isVectorAvailable()).toBe(true); + const t = await client.query( + "SELECT to_regclass('embeddings') e, to_regclass('index_generations') g, to_regclass('embed_watermark') w, to_regclass('embed_runs') r" + ); + expect(t.rows[0].e).not.toBeNull(); + expect(t.rows[0].g).not.toBeNull(); + expect(t.rows[0].w).not.toBeNull(); + expect(t.rows[0].r).not.toBeNull(); + }); + + it('is idempotent (second call does not throw)', async () => { + await expect(store.ensureVectorSchema()).resolves.toMatchObject({ vectorAvailable: true }); + }); + + it('builds a partial HNSW index for a dimension on the empty table', async () => { + await store.ensureVectorIndex(4); + const idx = await client.query("SELECT indexname FROM pg_indexes WHERE tablename='embeddings' AND indexname='idx_embeddings_hnsw_d4'"); + expect(idx.rows.length).toBe(1); + }); +}); diff --git a/backend/src/services/embeddings/vectorSchema.unit.test.js b/backend/src/services/embeddings/vectorSchema.unit.test.js new file mode 100644 index 00000000..a1376c68 --- /dev/null +++ b/backend/src/services/embeddings/vectorSchema.unit.test.js @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest'; + +// Mock the module graph so ensureVectorSchema runs without a real DB. The dedicated +// client (withDedicatedClient → new pg.Client) and the pool both resolve; the only +// variable is whether ensureVectorIndex(dim) throws (a non-integer dimension throws +// synchronously), which is exactly the "flag vs return" disagreement path. +const clientMock = { connect: vi.fn().mockResolvedValue(), query: vi.fn().mockResolvedValue({}), end: vi.fn().mockResolvedValue() }; +// Regular (constructable) function impl — `new pg.Client(...)` needs a constructor. +vi.mock('pg', () => ({ default: { Client: vi.fn(function () { return clientMock; }) } })); +vi.mock('../db.js', () => ({ pool: { query: vi.fn().mockResolvedValue({}) }, withTransaction: vi.fn() })); +vi.mock('./config.js', () => ({ resolveEmbedConfig: vi.fn() })); + +const { resolveEmbedConfig } = await import('./config.js'); +const { ensureVectorSchema, isVectorAvailable } = await import('./vectorStore.js'); + +describe('ensureVectorSchema flag/return agreement', () => { + it('reports available and sets the flag when the whole bring-up succeeds', async () => { + resolveEmbedConfig.mockResolvedValueOnce({ dimension: 4, skipExtensionCreate: false }); + const r = await ensureVectorSchema(); + expect(r.vectorAvailable).toBe(true); + expect(isVectorAvailable()).toBe(true); + }); + + it('leaves isVectorAvailable() false when ensureVectorIndex throws after the schema builds', async () => { + // dimension 2.5 passes the `> 0` guard but ensureVectorIndex throws on the + // non-integer, AFTER the extension + schema succeed — the disagreement window. + resolveEmbedConfig.mockResolvedValueOnce({ dimension: 2.5, skipExtensionCreate: false }); + const r = await ensureVectorSchema(); + expect(r.vectorAvailable).toBe(false); + expect(isVectorAvailable()).toBe(false); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.fused.test.js b/backend/src/services/embeddings/vectorStore.fused.test.js new file mode 100644 index 00000000..1f8cf489 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.fused.test.js @@ -0,0 +1,296 @@ +import { describe, it, expect, afterAll } from 'vitest'; +import { hasTestDb, withTestDb, closeTestDb } from './__testdb__.js'; +import { fusedSearch } from './vectorStore.js'; +import { searchLexical } from '../search/lexicalRepo.js'; + +afterAll(async () => { await closeTestDb(); }); + +describe.skipIf(!hasTestDb())('pgvector test DB', () => { + it('boots ensureVectorSchema and exposes the vector extension', async () => { + await withTestDb(async (pool) => { + const { rows } = await pool.query(`SELECT extname FROM pg_extension WHERE extname = 'vector'`); + expect(rows).toHaveLength(1); + }); + }); +}); + +function unitVec(dim, axis) { + const v = new Array(dim).fill(0); + v[axis] = 1; + return v; +} + +// Seeds one user + account + N messages (trigger fills search_fts) and an active +// generation with one chunk per message. Returns { accountId, generation, ids }. +async function seedThree(pool) { + const u = (await pool.query( + `INSERT INTO users (username, password_hash) VALUES ('t','x') RETURNING id`)).rows[0].id; + const acc = (await pool.query( + `INSERT INTO email_accounts (user_id, name, email_address, enabled) + VALUES ($1,'A','a@x.test',true) RETURNING id`, [u])).rows[0].id; + const base = Date.UTC(2025, 0, 15, 12, 0, 0); + const rows = [ + ['alpha quantum project update', 'discussing the quantum roadmap', 0, false], + ['beta vector indexing notes', 'notes about hybrid search and ranking', 1, true], + ['gamma project retrospective', 'retro covering the quantum milestone', 2, false], + ]; + const ids = []; + for (let i = 0; i < rows.length; i++) { + const [subject, body, , hasAtt] = rows[i]; + const id = (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, has_attachments, is_deleted) + VALUES ($1,$2,'INBOX',$3,'Sender','s@x.test',$4,$5,$6,$7,false) RETURNING id`, + [acc, 100 + i, subject, new Date(base + i * 86400000).toISOString(), body, body, hasAtt] + )).rows[0].id; + ids.push(id); + } + const gen = (await pool.query( + `INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) + VALUES ('m', 4, 'm:4:test', $1, 'active') RETURNING id`, + [Math.floor(Date.now() / 1000)])).rows[0].id; + for (let i = 0; i < ids.length; i++) { + await pool.query( + `INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, embedding, dimension) + VALUES ($1,$2,0,$3,4,$4::vector,4)`, + [gen, ids[i], Math.floor(Date.now() / 1000), `[${unitVec(4, i).join(',')}]`]); + } + return { accountId: acc, generation: { id: gen, dimension: 4 }, ids }; +} + +const base = { rrfK: 60, kPerSignal: 10, limit: 10, buildFilters: () => [] }; + +describe.skipIf(!hasTestDb())('fusedSearch', () => { + it('FTS-only: returns the two quantum messages, bm25 set / vector null, RRF descending', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(false); + expect(hits).toHaveLength(2); + const set = new Set(hits.map(h => h.message_id)); + expect(set.has(s.ids[0]) && set.has(s.ids[2])).toBe(true); + for (const h of hits) { + expect(h.vector_score).toBeNull(); + expect(h.bm25_score).not.toBeNull(); + expect(h.rrf_score).toBeGreaterThan(0); + } + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].rrf_score).toBeGreaterThanOrEqual(hits[i].rrf_score); + } + }); + }); + + it('prefix consistency: the fused FTS leg matches the same hit set as the lexical path for a prefix term (Opus review fix-round)', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + // A message whose only match for "invo" is a prefix of "invoice" — none + // of seedThree's quantum-related docs share that prefix, so a correct + // prefix match (and only a correct prefix match) finds exactly this one. + const invoiceId = (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, is_deleted) + VALUES ($1,103,'INBOX','Invoice reminder','Sender','s@x.test',now(),'inv','an invoice is attached',false) + RETURNING id`, + [s.accountId])).rows[0].id; + + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'invo', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const lex = await searchLexical((text, params) => pool.query(text, params), { + parsed: { filters: [], terms: [{ value: 'invo', negate: false }] }, + accountIds: [s.accountId], folderScope: null, folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + + const fusedIds = new Set(hits.map(h => h.message_id)); + const lexIds = new Set(lex.rows.map(r => r.id)); + expect(fusedIds.has(invoiceId)).toBe(true); + expect(fusedIds).toEqual(lexIds); + }); + }); + + it('ANN-only: top hit is the on-axis message, bm25 null / vector set', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, ftsQuery: null, queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(false); + expect(hits[0].message_id).toBe(s.ids[0]); + for (const h of hits) { + expect(h.bm25_score).toBeNull(); + expect(h.vector_score).not.toBeNull(); + } + }); + }); + + it('hybrid: union of the FTS pair and the ANN-only third message (3 hits), RRF descending', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: unitVec(4, 1), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(hits).toHaveLength(3); + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].rrf_score).toBeGreaterThanOrEqual(hits[i].rrf_score); + } + }); + }); + + it('saturation: kPerSignal below the pool size flips poolSaturated and trims', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, kPerSignal: 1, ftsQuery: 'quantum', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(true); + expect(hits).toHaveLength(1); + }); + }); + + it('tenant scope: a different account never leaks into the pool', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: null, generation: s.generation, + accountIds: ['00000000-0000-0000-0000-000000000000'] }, + { client: pool }); + expect(hits).toHaveLength(0); + }); + }); + + it('multi-chunk dedup: a message with a close and a far chunk appears once at its MIN distance', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + // Give the winning (chunk_index=0, on-axis) chunk distinctive offsets so + // Task 5b's best_char_start assertion can tell it apart from the far one. + await pool.query( + `UPDATE embeddings SET chunk_char_start = 6, chunk_char_end = 40 + WHERE generation_id = $1 AND message_id = $2 AND chunk_index = 0`, + [s.generation.id, s.ids[0]]); + // give ids[0] a second, far chunk on axis 2, with DIFFERENT offsets + await pool.query( + `INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, chunk_char_start, chunk_char_end, embedding, dimension) + VALUES ($1,$2,1,$3,4,100,140,$4::vector,4)`, + [s.generation.id, s.ids[0], Math.floor(Date.now() / 1000), `[${unitVec(4, 2).join(',')}]`]); + const { hits } = await fusedSearch( + { ...base, ftsQuery: null, queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const counts = {}; + for (const h of hits) counts[h.message_id] = (counts[h.message_id] || 0) + 1; + expect(counts[s.ids[0]]).toBe(1); + const top = hits.find(h => h.message_id === s.ids[0]); + expect(top.vector_score).toBeCloseTo(1.0, 6); // 1 - MIN(distance)=0 + // The winning (close) chunk's offsets ride through, not the far chunk's. + expect(top.best_chunk_index).toBe(0); + expect(top.best_char_start).toBe(6); + expect(top.best_char_end).toBe(40); + }); + }); + + it('rejects an empty request', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + await expect(fusedSearch( + { ...base, ftsQuery: null, queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool })).rejects.toThrow(); + }); + }); +}); + +// Wave D Fix 1, verified against the live pgvector container: english stopwords +// normalize to an EMPTY tsquery under 'english' (numnode = 0) and `@@ ''` is +// FALSE — pre-fix, ONE stopword in the AND'd term chain zeroed every backfilled +// result ("waiting for invoice" → 0 hits because of "for"). +describe.skipIf(!hasTestDb())('stopword terms never zero a search (Fix 1)', () => { + async function seedWaiting(pool, s) { + return (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, is_deleted) + VALUES ($1,104,'INBOX','Waiting for invoice','Sender','s@x.test',now(),'w', + 'we are waiting for the invoice payment',false) RETURNING id`, + [s.accountId])).rows[0].id; + } + const lex = (pool, terms, s, ordering = 'relevance') => + searchLexical((text, params) => pool.query(text, params), { + parsed: { filters: [], terms }, + accountIds: [s.accountId], folderScope: null, folderFuzzy: false, ordering, limit: 50, offset: 0, + }); + + it('lexical: "waiting for invoice" finds the message despite the stopword', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const res = await lex(pool, [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ], s); + expect(res.rows.map((r) => r.id)).toContain(id); + }); + }); + + it('lexical: a stopword-ONLY query degrades to filter-only/date-order — never zero-by-stopword', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const res = await lex(pool, [{ value: 'the', negate: false }], s); + expect(res.hasCondition).toBe(true); + expect(res.rows).toHaveLength(3); // the whole (trash-excluded) scope + const dates = res.rows.map((r) => new Date(r.date).getTime()); + expect(dates).toEqual([...dates].sort((a, b) => b - a)); // rank 0 ⇒ date tiebreak + }); + }); + + it('lexical: a NEGATED stopword contributes nothing instead of excluding everything', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const res = await lex(pool, [ + { value: 'invoice', negate: false }, + { value: 'the', negate: true }, // body contains "the" — must NOT exclude + ], s); + expect(res.rows.map((r) => r.id)).toContain(id); + }); + }); + + it('fused BM25 leg: "waiting for invoice" matches the same set the lexical path finds', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'waiting for invoice', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const res = await lex(pool, [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ], s); + expect(hits.map((h) => h.message_id)).toContain(id); + expect(new Set(hits.map((h) => h.message_id))).toEqual(new Set(res.rows.map((r) => r.id))); + }); + }); + + it('fused: an ALL-stopword ftsQuery leaves the BM25 leg empty — pure-ANN ranking, silence not noise', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'the for you', queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(hits.length).toBeGreaterThan(0); // ANN still answers + expect(hits.every((h) => h.bm25_score === null)).toBe(true); // FTS contributed nothing + }); + }); + +}); + +describe.skipIf(!hasTestDb())('fusedSearch input validation', () => { + it('rejects a dimension mismatch', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + await expect(fusedSearch( + { ...base, ftsQuery: null, queryVec: [1, 2, 3], generation: s.generation, accountIds: [s.accountId] }, + { client: pool })).rejects.toThrow(); + }); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.fusedSql.test.js b/backend/src/services/embeddings/vectorStore.fusedSql.test.js new file mode 100644 index 00000000..77882196 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.fusedSql.test.js @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest'; +import { fusedSearch, FUSED_ANN_CHUNKS_PER_MESSAGE, HNSW_EF_SEARCH_MAX } from './vectorStore.js'; + +// SQL-text pins for fusedSearch, driven through an injected fake client (a +// plain object — NOT a pg.Pool — so withEfSearch uses it verbatim and every +// statement it issues is recorded in order). +// +// `annPools` scripts the ann_pool_size the fused SELECT reports on each +// successive attempt, so the widening loop's exits are testable. +function fakeDb({ chunkCount = 5000, filteredMessages = 3000, annPools = [999], ftsPool = 5 } = {}) { + const calls = []; + let attempt = -1; + return { + calls, + fusedRuns() { return calls.filter((c) => /^WITH /.test(c.text) && /FROM fused f/.test(c.text)); }, + query: async (text, params) => { + calls.push({ text, params }); + if (/count\(\*\)::int AS n FROM embeddings/.test(text)) return { rows: [{ n: chunkCount }] }; + if (/count\(DISTINCT e\.message_id\)/.test(text)) return { rows: [{ n: filteredMessages }] }; + if (/FROM fused f/.test(text)) { + attempt = Math.min(attempt + 1, annPools.length - 1); + return { + rows: [{ + message_id: 'm1', subject: 's', + fts_pool_size: ftsPool, ann_pool_size: annPools[attempt], + }], + }; + } + return { rows: [] }; + }, + }; +} + +const gen = { id: 7, dimension: 4 }; +const base = { + generation: gen, accountIds: ['a1'], rrfK: 60, kPerSignal: 10, limit: 10, + buildFilters: () => [], +}; + +describe('fusedSearch BM25 leg — stopword-safe combined tsquery (Fix 1)', () => { + it('matches AND ranks with ONE `&&`-combined tsquery so an empty (stopword) operand drops out', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'waiting for invoice', queryVec: [1, 0, 0, 0] }, { client: db }); + const sql = db.fusedRuns()[0].text; + const combined = + "(to_tsquery('english', quote_literal($2) || ':*') && " + + "to_tsquery('english', quote_literal($3) || ':*') && " + + "to_tsquery('english', quote_literal($4) || ':*'))"; + // Single @@ against the combined query — `&&` drops an empty-normalizing + // operand ("for"), so a stopword can no longer zero the whole leg the way + // the old per-term `@@ ... AND @@ ...` chain did. + expect(sql).toContain(`m.search_fts @@ ${combined}`); + // The rank arg is the SAME combined construction (match and rank can't diverge). + expect(sql).toContain(`ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, ${combined}, 32)`); + // No per-term AND chain remains. + expect(sql).not.toMatch(/@@ to_tsquery\('english', quote_literal\(\$\d+\) \|\| ':\*'\) AND/); + const params = db.fusedRuns()[0].params; + expect(params.slice(1, 4)).toEqual(['waiting', 'for', 'invoice']); + }); +}); + +describe('fusedSearch ANN leg — hnsw.ef_search per attempt (Fix 2)', () => { + it('runs each ANN attempt in a transaction that SET LOCALs ef_search to the inner LIMIT', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + const texts = db.calls.map((c) => c.text); + const inner = (base.kPerSignal + 1) * FUSED_ANN_CHUNKS_PER_MESSAGE; // 88 + const begin = texts.indexOf('BEGIN'); + const guc = texts.indexOf(`SET LOCAL hnsw.ef_search = ${inner}`); + const run = texts.findIndex((t) => /FROM fused f/.test(t)); + const commit = texts.indexOf('COMMIT'); + // BEGIN → SET LOCAL → fused SELECT → COMMIT, in that order, on one client. + expect(begin).toBeGreaterThanOrEqual(0); + expect(guc).toBeGreaterThan(begin); + expect(run).toBeGreaterThan(guc); + expect(commit).toBeGreaterThan(run); + }); + + it(`caps the GUC at HNSW_EF_SEARCH_MAX (${HNSW_EF_SEARCH_MAX}) — pgvector rejects larger values`, async () => { + const db = fakeDb(); + // kPerSignal=200 → inner LIMIT (201*8=1608) exceeds the pgvector cap. + await fusedSearch({ ...base, kPerSignal: 200, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + expect(db.calls.some((c) => c.text === `SET LOCAL hnsw.ef_search = ${HNSW_EF_SEARCH_MAX}`)).toBe(true); + expect(db.calls.some((c) => /SET LOCAL hnsw\.ef_search = 1608/.test(c.text))).toBe(false); + }); + + it('re-issues a LARGER ef_search when the widening loop grows the inner LIMIT', async () => { + // First attempt dedups to 5 (< kPerSignal+1 = 11, < filteredCeiling), + // second grows to 11 and exits. + const db = fakeDb({ annPools: [5, 11] }); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + const gucs = db.calls.filter((c) => /^SET LOCAL hnsw\.ef_search = /.test(c.text)).map((c) => c.text); + expect(gucs).toEqual(['SET LOCAL hnsw.ef_search = 88', 'SET LOCAL hnsw.ef_search = 176']); + expect(db.fusedRuns()).toHaveLength(2); + }); + + it('breaks the widening loop when the ann pool stops growing between attempts', async () => { + // The pool sticks at 5 forever; without the no-growth break the loop + // would double 88 → … → 5000 (the chunk ceiling) re-running for nothing. + const db = fakeDb({ annPools: [5, 5] }); + const { poolSaturated } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + expect(db.fusedRuns()).toHaveLength(2); + expect(poolSaturated).toBe(false); + }); + + it('an FTS-only request issues no transaction and no GUC (nothing to tune)', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: null }, { client: db }); + expect(db.calls.some((c) => /hnsw\.ef_search|^BEGIN$/.test(c.text))).toBe(false); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.integration.test.js b/backend/src/services/embeddings/vectorStore.integration.test.js new file mode 100644 index 00000000..fef87f58 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.integration.test.js @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('vectorStore ANN', () => { + let store, client, gen, mA, mB, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(4); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // Re-runnable: clear generation state before seeding. embed_runs FKs + // index_generations without ON DELETE CASCADE (generations are never hard-deleted + // in production), so clear it before the parent to keep the reset re-runnable. + await client.query('DELETE FROM embeddings'); + await client.query('DELETE FROM embed_runs'); + await client.query('DELETE FROM index_generations'); + let acctId; + ({ accountId: acctId, userId } = await seedAccount(client, 'ann')); + const a = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81001, 'INBOX', 'A') RETURNING id`, [acctId]); + const b = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81002, 'INBOX', 'B') RETURNING id`, [acctId]); + mA = a.rows[0].id; mB = b.rows[0].id; + const now = Math.floor(Date.now() / 1000); + const gi = await client.query(`INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) VALUES ('m',4,'fp',$1,'building') RETURNING id`, [now]); + gen = gi.rows[0].id; + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('returns the nearest message first', async () => { + await store.upsert(gen, [ + { messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mB, chunkIndex: 0, vector: [0, 1, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + ]); + const hits = await store.annSearch(gen, [1, 0, 0, 0], 2); + expect(hits[0].messageId).toBe(mA); + expect(hits[0].rank).toBe(1); + expect(hits[0].score).toBeGreaterThan(hits[1].score); + }); + + it('re-upsert is idempotent (PK replace, no duplicate chunks)', async () => { + await store.upsert(gen, [{ messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }]); + const c = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, mA]); + expect(c.rows[0].n).toBe(1); + }); + + it('dedups multi-chunk messages by best (MIN) distance', async () => { + // mB gets a far chunk (0,1,0,0) and a near chunk (1,0,0,0); querying [1,0,0,0] + // should rank mB by its BEST chunk, ahead of a mid message. + await store.upsert(gen, [ + { messageId: mB, chunkIndex: 0, vector: [0, 1, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mB, chunkIndex: 1, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 4, chunkCharEnd: 8, truncated: false }, + ]); + const hits = await store.annSearch(gen, [1, 0, 0, 0], 5); + const mbHit = hits.find((h) => h.messageId === mB); + expect(mbHit.score).toBeGreaterThan(0.9); // best chunk is an exact match + // one hit per message + expect(new Set(hits.map((h) => h.messageId)).size).toBe(hits.length); + }); + + it('filter.accountIds scopes ANN to one account (find_similar_messages needs this)', async () => { + // A second account whose message is an EXACT match for the query — without the + // filter it would surface; with the account filter it must be excluded, and the + // widening still returns the in-scope match. + const other = await seedAccount(client, 'ann-other'); + const oc = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81003, 'INBOX', 'C') RETURNING id`, [other.accountId]); + const mC = oc.rows[0].id; + await store.upsert(gen, [ + { messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mC, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + ]); + const firstAcct = (await client.query('SELECT account_id FROM messages WHERE id = $1', [mA])).rows[0].account_id; + const scoped = await store.annSearch(gen, [1, 0, 0, 0], 5, { filter: { accountIds: [firstAcct] } }); + expect(scoped.some((h) => h.messageId === mC)).toBe(false); + expect(scoped.some((h) => h.messageId === mA)).toBe(true); + const unscoped = await store.annSearch(gen, [1, 0, 0, 0], 5); + expect(unscoped.some((h) => h.messageId === mC)).toBe(true); + await cleanupAccount(client, other.userId); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.js b/backend/src/services/embeddings/vectorStore.js new file mode 100644 index 00000000..6cfb19af --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.js @@ -0,0 +1,731 @@ +import pg from 'pg'; +import { pool, withTransaction } from '../db.js'; +import { resolveEmbedConfig } from './config.js'; +import { LEXICAL_RANK_SQL, ftsTermQueryArg, hasSearchableToken } from '../search/lexicalRepo.js'; + +export const ZERO_UUID = '00000000-0000-0000-0000-000000000000'; + +let _vectorAvailable = false; +export function isVectorAvailable() { return _vectorAvailable; } + +// Build a short-lived connection WITHOUT the pool's `-c statement_timeout=30000` +// startup option, then explicitly disable statement_timeout, so a slow DDL build +// (HNSW over a repopulated table after the alpine→Debian image swap) is not killed +// at 30s. Caller closes nothing — this helper owns the connect/end lifecycle. +async function withDedicatedClient(fn) { + const client = new pg.Client({ + host: process.env.DB_HOST || 'postgres', + port: Number(process.env.DB_PORT) || 5432, + database: process.env.DB_NAME || 'mailflow', + user: process.env.DB_USER || 'mailflow', + password: process.env.DB_PASSWORD, + }); + await client.connect(); + try { + await client.query('SET statement_timeout = 0'); + return await fn(client); + } finally { + await client.end().catch(() => {}); + } +} + +const SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS index_generations ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + model TEXT NOT NULL, + dimension INTEGER NOT NULL, + fingerprint TEXT NOT NULL, + started_at BIGINT NOT NULL, + seeded_at BIGINT, + completed_at BIGINT, + activated_at BIGINT, + state TEXT NOT NULL, + message_count BIGINT NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_generations_active ON index_generations(state) WHERE state = 'active'; +CREATE UNIQUE INDEX IF NOT EXISTS idx_generations_building ON index_generations(state) WHERE state = 'building'; + +CREATE TABLE IF NOT EXISTS embeddings ( + generation_id BIGINT NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE, + message_id UUID NOT NULL, + chunk_index INTEGER NOT NULL DEFAULT 0, + embedded_at BIGINT NOT NULL, + source_char_len INTEGER NOT NULL, + chunk_char_start INTEGER NOT NULL DEFAULT 0, + chunk_char_end INTEGER NOT NULL DEFAULT 0, + truncated BOOLEAN NOT NULL DEFAULT FALSE, + dimension INTEGER NOT NULL, + embedding vector NOT NULL, + PRIMARY KEY (generation_id, message_id, chunk_index) +); +CREATE INDEX IF NOT EXISTS idx_embeddings_msg ON embeddings(message_id); +CREATE INDEX IF NOT EXISTS idx_embeddings_dim ON embeddings(dimension); + +CREATE TABLE IF NOT EXISTS embed_runs ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + generation_id BIGINT NOT NULL REFERENCES index_generations(id), + started_at BIGINT NOT NULL, + ended_at BIGINT, + claimed INTEGER NOT NULL DEFAULT 0, + succeeded INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + truncated INTEGER NOT NULL DEFAULT 0, + error TEXT +); + +CREATE TABLE IF NOT EXISTS embed_watermark ( + generation_id BIGINT PRIMARY KEY, + watermark_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +); +`; + +// Best-effort startup routine (pattern: encryptExistingCredentials in db.js). Never +// throws into boot: on any failure it logs, sets vector_available=false, and returns. +export async function ensureVectorSchema() { + _vectorAvailable = false; + let cfg = null; + try { cfg = await resolveEmbedConfig(); } catch { /* ai_config unreadable — treat as unset */ } + const skipExtension = cfg?.skipExtensionCreate === true; + try { + if (!skipExtension) { + await pool.query('CREATE EXTENSION IF NOT EXISTS vector'); + } + // Apply the schema on a dedicated no-timeout connection: on a legacy populated DB + // the idx_embeddings_* builds can exceed the pool's 30s cap (migrate.go). + await withDedicatedClient((client) => client.query(SCHEMA_SQL)); + if (cfg?.dimension > 0) { + await ensureVectorIndex(cfg.dimension); + } + // Set the flag only after the WHOLE bring-up (extension + schema + per-dimension + // HNSW) succeeds, so isVectorAvailable() never disagrees with the return value — + // e.g. an ensureVectorIndex failure must leave the flag false. + _vectorAvailable = true; + console.log('Vector schema ready — semantic search available'); + return { vectorAvailable: true }; + } catch (err) { + _vectorAvailable = false; + console.warn(`Vector disabled: ${err.message} — lexical search unaffected`); + return { vectorAvailable: false }; + } +} + +// Partial per-dimension HNSW cosine index, created while the table is empty so it is +// maintained incrementally (README invariant; port of migrate.go EnsureVectorIndex). +// The `WHERE dimension = N` guard lets generations of different dims coexist. +export async function ensureVectorIndex(dim) { + if (!Number.isInteger(dim) || dim <= 0) throw new Error(`invalid dimension ${dim}`); + const stmt = `CREATE INDEX IF NOT EXISTS idx_embeddings_hnsw_d${dim} + ON embeddings USING hnsw ((embedding::vector(${dim})) vector_cosine_ops) + WHERE dimension = ${dim}`; + await withDedicatedClient((client) => client.query(stmt)); +} + +const LIVE_MESSAGES_WHERE = 'is_deleted = false'; // Mailflow live-message predicate +const ANN_OVERFETCH = 4; // backend.go annOverFetchFactor + +// pgvector's hnsw.ef_search GUC defaults to 40 and hard-caps at 1000: the HNSW +// scan visits at most ef_search candidates, so any inner ANN `ORDER BY <=> +// LIMIT` above it is silently truncated to ~ef_search rows and widening the +// LIMIT re-runs an identical plan. msgvault sizes a per-connection GUC to 1000 +// (store.go HNSWEfSearch): >= the worst-case fused inner LIMIT, +// (kPerSignal+1)*FUSED_ANN_CHUNKS_PER_MESSAGE ≈ 808 at the default +// kPerSignal=100, with headroom, while keeping per-query latency bounded. +// Mailflow's pool sets no per-connection GUCs, so every ANN-issuing statement +// runs in a transaction that `SET LOCAL`s the GUC to its own inner LIMIT, +// capped here (pgvector REJECTS values above 1000; beyond the cap, recall is +// best-effort — the same trade msgvault documents). +export const HNSW_EF_SEARCH_MAX = 1000; + +// Run fn(client) in a transaction whose hnsw.ef_search covers `efSearch` +// candidates (capped at HNSW_EF_SEARCH_MAX). A pg.Pool is pinned to one +// client first — BEGIN / SET LOCAL / queries must share a session; an +// injected single client (tests) is used as-is. +async function withEfSearch(db, efSearch, fn) { + const pinned = db instanceof pg.Pool ? await db.connect() : db; + try { + await pinned.query('BEGIN'); + await pinned.query(`SET LOCAL hnsw.ef_search = ${Math.min(Math.max(1, Math.floor(efSearch)), HNSW_EF_SEARCH_MAX)}`); + const out = await fn(pinned); + await pinned.query('COMMIT'); + return out; + } catch (err) { + await pinned.query('ROLLBACK').catch(() => {}); + throw err; + } finally { + if (pinned !== db) pinned.release(); + } +} + +export function vectorLiteral(vec) { + return `[${vec.map((f) => Number(f).toString()).join(',')}]`; +} + +// Upsert chunks for one generation. Idempotent per message: clears the message's +// prior chunks (chunk count is not stable across re-embeds) then inserts the new set, +// all in one tx. Maintains index_generations.message_count by distinct-message delta. +export async function upsert(gen, chunks) { + if (!chunks.length) return; + await withTransaction(async (client) => { + // Row-lock the generation (serializes against activate/retire) and read its dim. + const g = await client.query('SELECT dimension, state FROM index_generations WHERE id = $1 FOR UPDATE', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + if (g.rows[0].state === 'retired') { const e = new Error(`generation retired ${gen}`); e.code = 'GEN_RETIRED'; throw e; } + const dim = g.rows[0].dimension; + for (const c of chunks) { + if (c.vector.length !== dim) throw new Error(`dimension mismatch: chunk for msg ${c.messageId} has ${c.vector.length}, gen has ${dim}`); + } + const ids = [...new Set(chunks.map((c) => c.messageId))]; + const pre = await client.query( + 'SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', + [gen, ids], + ); + await client.query('DELETE FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', [gen, ids]); + const now = Math.floor(Date.now() / 1000); + for (const c of chunks) { + await client.query( + `INSERT INTO embeddings + (generation_id, message_id, chunk_index, embedded_at, source_char_len, + chunk_char_start, chunk_char_end, truncated, dimension, embedding) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::vector)`, + [gen, c.messageId, c.chunkIndex, now, c.sourceCharLen, c.chunkCharStart, c.chunkCharEnd, c.truncated, dim, vectorLiteral(c.vector)], + ); + } + const delta = ids.length - pre.rows[0].n; + if (delta !== 0) { + await client.query('UPDATE index_generations SET message_count = message_count + $1 WHERE id = $2', [delta, gen]); + } + }); +} + +export async function chunkCount(gen) { + const r = await pool.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1', [gen]); + return r.rows[0].n; +} + +// ANN search over one generation. Inner ORDER BY <=> LIMIT uses the partial HNSW index +// (dimension embedded as a literal); outer GROUP BY collapses multi-chunk messages to +// their best (MIN) distance. Widens the inner LIMIT until k distinct messages survive the +// dedup. Score = 1 - cosine_distance. +// +// An optional structured `filter` narrows the candidate set by pushing predicates INTO +// the inner liveness EXISTS (a Postgres-native join on messages — the equivalent of +// msgvault's json_each resolved-id set, backend.go filtered path). This is REQUIRED for +// account-scoped callers (e.g. plan-phase5 find_similar_messages in a multi-user DB): +// filtering after k-NN can return zero in-scope rows, whereas widening WITH the filter in +// SQL keeps pulling candidates until k in-scope messages survive (or the generation is +// exhausted). filter fields (all optional, AND-combined): +// accountIds: string[] (UUIDs) → m.account_id = ANY($n::uuid[]) +// after / before: timestamptz-comparable (ISO string or Date) → m.date >= / < $n +// hasAttachment: boolean → m.has_attachments = true/false +// With no filter the SQL + params are byte-identical to the fast path (filter clauses and +// their binds simply don't appear). Note: with a filter present the planner may fall back +// from the HNSW index to a scan within the filtered set — the same trade msgvault accepts. +// Append the structured filter's predicates to `args` (relative to messages alias `m`) +// and return the joined WHERE string. Built with a fresh `args` per query so each +// statement's $N ordinals resolve independently (msgvault's bind-closure pattern). +function buildAnnFilter(filter, args) { + const where = [LIVE_MESSAGES_WHERE]; + if (filter) { + if (filter.accountIds?.length) { args.push(filter.accountIds); where.push(`m.account_id = ANY($${args.length}::uuid[])`); } + if (filter.after) { args.push(filter.after); where.push(`m.date >= $${args.length}`); } + // before is EXCLUSIVE (<) — msgvault filter.go parity, and the same + // bound lexicalRepo's before: operator applies (one convention everywhere). + if (filter.before) { args.push(filter.before); where.push(`m.date < $${args.length}`); } + if (filter.hasAttachment === true) where.push('m.has_attachments = true'); + else if (filter.hasAttachment === false) where.push('m.has_attachments = false'); + } + return where.join(' AND '); +} + +// Doubling-widen the inner ANN LIMIT until at least k distinct messages survive the +// outer dedup, the distinct-message early exit is reached, or the candidate ceiling is +// hit. `ceiling` counts CHUNKS (bounds the inner LIMIT); `distinctEarlyExit` counts the +// distinct MESSAGES that can possibly appear (equals k on the empty-filter path, a +// no-op; equals the filtered distinct-message count on the filtered path, so a +// selective filter stops as soon as every in-scope message is surfaced instead of +// widening up to the whole generation's chunk count — msgvault searchWiden parity). +// Port of backend.go searchWiden. Exported for unit testing. +export async function searchWiden(k, ceiling, distinctEarlyExit, run) { + let innerLimit = Math.max(k * ANN_OVERFETCH, k); + for (;;) { + if (innerLimit > ceiling) innerLimit = ceiling; + const hits = await run(innerLimit); + if (hits.length >= k || hits.length >= distinctEarlyExit || innerLimit >= ceiling) { + return hits.slice(0, k).map((h, i) => ({ ...h, rank: i + 1 })); + } + innerLimit *= 2; + } +} + +export async function annSearch(gen, queryVec, k, { efSearch = 100, filter = null } = {}) { + if (!queryVec.length) throw new Error('annSearch: empty query vector'); + const g = await pool.query('SELECT dimension FROM index_generations WHERE id = $1', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + const dim = g.rows[0].dimension; + if (queryVec.length !== dim) throw new Error(`dimension mismatch: query ${queryVec.length}, gen ${dim}`); + const lit = vectorLiteral(queryVec); + + // Widening bounds. Filtered: count only the in-scope candidate set (chunks = ceiling, + // distinct messages = early exit) with the SAME EXISTS predicate, so the loop stops + // once every in-scope message is surfaced. Unfiltered: whole-generation chunk count as + // the ceiling and k as the (no-op) early exit — byte-identical to the original fast path. + let ceiling, distinctEarlyExit; + if (filter) { + const cArgs = [gen]; + const cWhere = buildAnnFilter(filter, cArgs); + const cnt = await pool.query( + `SELECT COUNT(*)::int chunks, COUNT(DISTINCT e.message_id)::int messages + FROM embeddings e + WHERE e.generation_id = $1 + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${cWhere})`, + cArgs, + ); + ceiling = cnt.rows[0].chunks; + distinctEarlyExit = cnt.rows[0].messages; + } else { + ceiling = await chunkCount(gen); + distinctEarlyExit = k; + } + if (ceiling === 0) return []; + + // $1 = query vector, $2 = generation; filter binds (if any) take $3.. ; the two LIMITs + // are appended per widening run as the trailing two params. + const args = [lit, gen]; + const where = buildAnnFilter(filter, args); + const innerArg = args.length + 1; + const outerArg = args.length + 2; + const sql = ` + SELECT ann.message_id, MIN(ann.distance) AS distance + FROM ( + SELECT e.message_id, (e.embedding::vector(${dim})) <=> $1::vector AS distance + FROM embeddings e + WHERE e.generation_id = $2 AND e.dimension = ${dim} + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${where}) + ORDER BY e.embedding::vector(${dim}) <=> $1::vector + LIMIT $${innerArg} + ) ann + GROUP BY ann.message_id + ORDER BY distance, ann.message_id + LIMIT $${outerArg}`; + const client = await pool.connect(); + try { + await client.query('BEGIN'); + const hits = await searchWiden(k, ceiling, distinctEarlyExit, async (innerLimit) => { + // Re-issue the GUC per widening attempt: ef_search must cover the inner + // LIMIT or the HNSW scan truncates it and the widening loop is a no-op + // (see HNSW_EF_SEARCH_MAX). `efSearch` stays the floor for small limits. + await client.query(`SET LOCAL hnsw.ef_search = ${Math.min(Math.max(Number(efSearch), innerLimit), HNSW_EF_SEARCH_MAX)}`); + const r = await client.query(sql, [...args, innerLimit, k]); + return r.rows.map((row, i) => ({ messageId: row.message_id, score: 1 - Number(row.distance), rank: i + 1 })); + }); + await client.query('COMMIT'); + return hits; + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + throw err; + } finally { + client.release(); + } +} + +// Forward scan by UUID for live messages needing work, resuming above afterId. The +// predicate is `embed_gen IS NULL` (NOT the OR with `embed_gen <> target`) so the +// partial index idx_messages_embed_pending (migration 0039) drives an O(pending) scan +// even on a huge, fully-covered mailbox — an OR forces a full seq scan. This is +// correct because createGeneration resets every live row's stamp to NULL when a NEW +// generation is created, so a rebuild's prior-generation rows surface as NULL here too: +// no live row ever carries a non-null stamp for a generation OTHER than the current +// target. `target` is kept in the signature (callers pass it) but is not needed in the +// predicate given that invariant. +export async function scanForEmbedding(target, afterId, limit) { + const r = await pool.query( + `SELECT id FROM messages + WHERE embed_gen IS NULL AND ${LIVE_MESSAGES_WHERE} AND id > $1 + ORDER BY id LIMIT $2`, + [afterId, limit], + ); + return r.rows.map((row) => row.id); +} + +export async function setEmbedGen(ids, target) { + if (!ids.length) return; + await pool.query('UPDATE messages SET embed_gen = $1 WHERE id = ANY($2::uuid[])', [target, ids]); +} + +// Optimistic CAS stamp: only stamp rows whose last_modified text token is unchanged +// since the worker read it. Returns the ids that MISSED (last_modified moved) — not +// stamped; the backstop recovers them. Bind the token back as ::timestamptz for +// exact-equality (JS Date would lose the microseconds pg stores). +export async function setEmbedGenIfUnchanged(items, target) { + const missed = []; + await withTransaction(async (client) => { + for (const it of items) { + const res = await client.query( + 'UPDATE messages SET embed_gen = $1 WHERE id = $2 AND last_modified = $3::timestamptz', + [target, it.id, it.lastModified], + ); + if (res.rowCount === 0) missed.push(it.id); + } + }); + return missed; +} + +// Skip-mark rows (empty/missing) and prune their now-stale vectors in ONE transaction +// (parity with msgvault worker.go stampSkipped). CAS-stamps `casItems` (rows with a +// last_modified token) and unconditionally stamps `plainIds` (missing rows with no row +// to guard), then deletes embeddings for every id whose stamp actually landed — a +// CAS-missed id keeps both its NULL stamp and its vector so it is re-found later. +// index_generations.message_count is decremented by the DISTINCT messages the delete +// actually removes (msgvault backend.go:1118-1167 counts, deletes, and applies the +// delta under the generation row lock; this used to delete without decrementing, so +// the count drifted upward on every skipped re-embed). Returns the CAS-missed ids. +// Bind the token as ::timestamptz for exact equality. +export async function stampSkipped(gen, casItems, plainIds) { + const missed = []; + if (!casItems.length && !plainIds.length) return missed; + await withTransaction(async (client) => { + // Lock the generation row FIRST — the same order upsert() takes it (and + // msgvault's Delete, which locks before touching embeddings precisely to + // avoid an ABBA asymmetry with those writers) — so the decrement below is + // serialized against concurrent upsert/activate/retire. + const g = await client.query('SELECT id FROM index_generations WHERE id = $1 FOR UPDATE', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + for (const it of casItems) { + const res = await client.query( + 'UPDATE messages SET embed_gen = $1 WHERE id = $2 AND last_modified = $3::timestamptz', + [gen, it.id, it.lastModified], + ); + if (res.rowCount === 0) missed.push(it.id); + } + if (plainIds.length) { + await client.query('UPDATE messages SET embed_gen = $1 WHERE id = ANY($2::uuid[])', [gen, plainIds]); + } + const missedSet = new Set(missed); + const stamped = [...casItems.map((c) => c.id), ...plainIds].filter((id) => !missedSet.has(id)); + if (stamped.length) { + const pre = await client.query( + 'SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', + [gen, stamped], + ); + await client.query('DELETE FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', [gen, stamped]); + if (pre.rows[0].n > 0) { + await client.query('UPDATE index_generations SET message_count = message_count - $1 WHERE id = $2', [pre.rows[0].n, gen]); + } + } + }); + return missed; +} + +export async function getWatermark(gen) { + const r = await pool.query('SELECT watermark_id FROM embed_watermark WHERE generation_id = $1', [gen]); + return r.rows.length ? r.rows[0].watermark_id : ZERO_UUID; +} + +export async function setWatermark(gen, id) { + await pool.query( + `INSERT INTO embed_watermark (generation_id, watermark_id) VALUES ($1, $2) + ON CONFLICT (generation_id) DO UPDATE SET watermark_id = EXCLUDED.watermark_id`, + [gen, id], + ); +} + +export async function resetWatermark(gen) { + await setWatermark(gen, ZERO_UUID); +} + +// Fetch subject + inline bodies + the last_modified CAS token (as text) for a +// batch of message ids. Bodies are inline in messages: no message_bodies table. +export async function fetchForEmbedding(ids) { + if (!ids.length) return []; + const r = await pool.query( + `SELECT id, + COALESCE(subject, '') AS subject, + COALESCE(body_text, '') AS "bodyText", + COALESCE(body_html, '') AS "bodyHtml", + last_modified::text AS "lastModified" + FROM messages WHERE id = ANY($1::uuid[])`, + [ids], + ); + return r.rows; +} + +export async function startEmbedRun(gen) { + const now = Math.floor(Date.now() / 1000); + const r = await pool.query('INSERT INTO embed_runs (generation_id, started_at) VALUES ($1, $2) RETURNING id', [gen, now]); + return r.rows[0].id; +} + +export async function finalizeEmbedRun(runId, res, err) { + if (!runId) return; + const now = Math.floor(Date.now() / 1000); + await pool.query( + `UPDATE embed_runs SET ended_at = $1, claimed = $2, succeeded = $3, failed = $4, truncated = $5, error = $6 WHERE id = $7`, + [now, res.claimed, res.succeeded, res.failed, res.truncated, err ? String(err.message || err) : null, runId], + ).catch(() => {}); +} + +// ── Fused RRF search — port of internal/vector/pgvector/fused.go ── + +export const FUSED_ANN_CHUNKS_PER_MESSAGE = 8; + +const DISPLAY_COLS = ` + m.id AS message_id, m.uid, m.folder, m.subject, m.from_name, m.from_email, + m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, + a.name AS account_name, a.email_address AS account_email, a.color AS account_color`; + +async function fusedChunkCount(db, generation) { + const { rows } = await db.query( + `SELECT count(*)::int AS n FROM embeddings WHERE generation_id = $1 AND dimension = $2`, + [generation.id, generation.dimension]); + return rows[0].n; +} + +async function filteredChunkMessageCount(db, { generation, accountIds, buildFilters }) { + const args = [generation.id, generation.dimension, accountIds]; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + const filters = buildFilters(bind).map(c => ` AND ${c}`).join(''); + const { rows } = await db.query( + `SELECT count(DISTINCT e.message_id)::int AS n + FROM embeddings e + JOIN messages m ON m.id = e.message_id + WHERE e.generation_id = $1 AND e.dimension = $2 + AND m.account_id = ANY($3) AND m.is_deleted = false${filters}`, + args); + return rows[0].n; +} + +// Single-query hybrid RRF fusion: BM25 leg (weighted ts_rank_cd over +// search_fts, reusing LEXICAL_RANK_SQL) + ANN leg (cosine distance +// over one generation's embeddings), FULL OUTER JOIN'd and combined via +// reciprocal-rank fusion. Either leg can be omitted (ftsQuery/queryVec null) +// for lexical-only or vector-only pools. `buildFilters(bind) → string[]` +// applies the SAME structured operator predicates to both legs (README "one +// search seam" — lexicalRepo remains the single owner of those predicates). +export async function fusedSearch(req, { client } = {}) { + const db = client || pool; + const { generation, accountIds, rrfK, kPerSignal, limit } = req; + const buildFilters = req.buildFilters || (() => []); + // Tokenize once (terms don't change across the widening loop below) and + // apply the SAME hygiene the lexical path applies (searchLexical): + // drop sub-2-char and punctuation-only tokens rather than handing Postgres + // a term that would normalize to zero lexemes. This is also what makes + // `useFTS` mean "the FTS leg actually has something to match on", not just + // "a non-empty string was passed" — an all-punctuation ftsQuery degrades to + // ANN-only (or throws below, same as if ftsQuery were absent, if ANN is + // also unavailable). + const ftsTerms = typeof req.ftsQuery === 'string' + ? req.ftsQuery.trim().split(/\s+/).filter(t => t.length >= 2 && hasSearchableToken(t)) + : []; + const useFTS = ftsTerms.length > 0; + const useANN = Array.isArray(req.queryVec) && req.queryVec.length > 0; + if (!useFTS && !useANN) throw new Error('fusedSearch: neither ftsQuery nor queryVec provided'); + if (useANN && req.queryVec.length !== generation.dimension) { + throw new Error(`fusedSearch: dimension mismatch (query ${req.queryVec.length}, generation ${generation.dimension})`); + } + const dim = generation.dimension; + const kPlus1 = kPerSignal + 1; + + let chunkCeiling = 0; + let filteredCeiling = 0; + if (useANN) { + chunkCeiling = await fusedChunkCount(db, generation); + filteredCeiling = await filteredChunkMessageCount(db, { generation, accountIds, buildFilters }); + } + + async function runFused(exec, innerChunks) { + const args = []; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + const accArg = bind(accountIds); + const live = `m.account_id = ANY(${accArg}) AND m.is_deleted = false`; + const filterAnd = buildFilters(bind).map(c => ` AND ${c}`).join(''); + + const ctes = []; + if (useFTS) { + // One bind per term, `&&`-combined into a SINGLE tsquery used for BOTH + // the @@ match and the ts_rank_cd rank — the same per-term prefix-or- + // phrase construction lexicalRepo.js's lexical path uses + // (ftsTermQueryArg), so "invo" matches "invoice" here exactly as it + // does via searchLexical, and msgvault's fused.go shape (one combined + // BuildFTSTerm arg for match and rank alike). `@@ (a && b)` is + // equivalent to `@@ a AND @@ b`, and `&&` DROPS an empty operand + // (verified against pgvector/pg16) — which is what keeps an english + // stopword ("waiting for invoice") from zeroing the whole BM25 leg: + // the stopword's tsquery normalizes empty and simply vanishes from the + // combined query. An ALL-stopword ftsQuery combines to an empty tsquery + // that matches nothing, so the leg contributes silence (not noise) and + // the fused ranking degrades to pure ANN. ftsTermQueryArg wants the raw + // placeholder NUMBER (it prepends its own `$`, matching lexicalRepo.js's + // own internal convention) — push directly onto `args` rather than + // through `bind`, which returns an already-`$`-prefixed string. + const termArgs = ftsTerms.map((term) => { args.push(term); return args.length; }); + const tsquery = `(${ftsTerms.map((term, i) => ftsTermQueryArg(termArgs[i], term)).join(' && ')})`; + const matchWhere = `m.search_fts @@ ${tsquery}`; + const kp1 = bind(kPlus1); + const k = bind(kPerSignal); + const rank = LEXICAL_RANK_SQL('m.search_fts', tsquery); + ctes.push(`fts_pool AS ( + SELECT m.id AS message_id, ${rank} AS bm25 + FROM messages m + WHERE ${matchWhere} + AND ${live}${filterAnd} + ORDER BY bm25 DESC + LIMIT ${kp1} +)`); + ctes.push(`fts_ranked AS ( + SELECT message_id, bm25, + ROW_NUMBER() OVER (ORDER BY bm25 DESC, message_id ASC) AS rnk + FROM fts_pool + ORDER BY bm25 DESC, message_id ASC + LIMIT ${k} +)`); + } + if (useANN) { + const vecArg = bind(vectorLiteral(req.queryVec)); + const genArg = bind(generation.id); + const innerArg = bind(innerChunks); + const kp1 = bind(kPlus1); + const k = bind(kPerSignal); + ctes.push(`ann_pool AS ( + SELECT d.message_id, d.distance, d.chunk_index, d.chunk_char_start, d.chunk_char_end + FROM ( + SELECT DISTINCT ON (ann.message_id) + ann.message_id, ann.distance, ann.chunk_index, ann.chunk_char_start, ann.chunk_char_end + FROM ( + SELECT e.message_id, e.chunk_index, e.chunk_char_start, e.chunk_char_end, + (e.embedding::vector(${dim})) <=> ${vecArg}::vector AS distance + FROM embeddings e + WHERE e.generation_id = ${genArg} AND e.dimension = ${dim} + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${live}${filterAnd}) + ORDER BY e.embedding::vector(${dim}) <=> ${vecArg}::vector + LIMIT ${innerArg} + ) ann + ORDER BY ann.message_id, ann.distance + ) d + ORDER BY d.distance + LIMIT ${kp1} +)`); + ctes.push(`ann_ranked AS ( + SELECT message_id, distance, chunk_index, chunk_char_start, chunk_char_end, + ROW_NUMBER() OVER (ORDER BY distance ASC, message_id ASC) AS rnk + FROM ann_pool + ORDER BY distance ASC, message_id ASC + LIMIT ${k} +)`); + } + + const poolArgsLen = args.length; // rrfk/limit are appended after this + const poolCTEs = ctes.slice(); + const rrfkArg = bind(rrfK); + const limitArg = bind(limit); + + // 1.0 is a `numeric` literal in Postgres; numeric / bigint (rnk) stays + // numeric, and node-postgres returns numeric columns as strings (to avoid + // silent precision loss). Cast to double precision so rrf_score/bm25_score + // come back as JS numbers, matching msgvault's float64 RRF score. + let fused; + if (useFTS && useANN) { + fused = `fused AS ( + SELECT COALESCE(b.message_id, v.message_id) AS message_id, + COALESCE(1.0::float8 / (${rrfkArg} + b.rnk), 0.0) + COALESCE(1.0::float8 / (${rrfkArg} + v.rnk), 0.0) AS rrf_score, + b.bm25::float8 AS bm25_score, + CASE WHEN v.distance IS NULL THEN NULL ELSE 1.0::float8 - v.distance END AS vector_score, + v.chunk_index AS best_chunk_index, v.chunk_char_start AS best_char_start, v.chunk_char_end AS best_char_end + FROM fts_ranked b + FULL OUTER JOIN ann_ranked v USING (message_id) +)`; + } else if (useFTS) { + fused = `fused AS ( + SELECT b.message_id, 1.0::float8 / (${rrfkArg} + b.rnk) AS rrf_score, + b.bm25::float8 AS bm25_score, CAST(NULL AS double precision) AS vector_score, + CAST(NULL AS int) AS best_chunk_index, CAST(NULL AS int) AS best_char_start, CAST(NULL AS int) AS best_char_end + FROM fts_ranked b +)`; + } else { + fused = `fused AS ( + SELECT v.message_id, 1.0::float8 / (${rrfkArg} + v.rnk) AS rrf_score, + CAST(NULL AS double precision) AS bm25_score, 1.0::float8 - v.distance AS vector_score, + v.chunk_index AS best_chunk_index, v.chunk_char_start AS best_char_start, v.chunk_char_end AS best_char_end + FROM ann_ranked v +)`; + } + ctes.push(fused); + + const ftsPoolExpr = useFTS ? '(SELECT count(*) FROM fts_pool)' : '0'; + const annPoolExpr = useANN ? '(SELECT count(*) FROM ann_pool)' : '0'; + + const sql = `WITH ${ctes.join(',\n')} +SELECT ${DISPLAY_COLS}, + f.rrf_score, f.bm25_score, f.vector_score, + f.best_chunk_index, f.best_char_start, f.best_char_end, + ${ftsPoolExpr} AS fts_pool_size, + ${annPoolExpr} AS ann_pool_size + FROM fused f + JOIN messages m ON m.id = f.message_id + JOIN email_accounts a ON a.id = m.account_id + ORDER BY f.rrf_score DESC, f.message_id ASC + LIMIT ${limitArg}`; + + const { rows } = await exec.query(sql, args); + let ftsPoolSize = 0; + let annPoolSize = 0; + if (rows.length > 0) { + ftsPoolSize = rows[0].fts_pool_size; + annPoolSize = rows[0].ann_pool_size; + } else { + // Empty result: the pool-size subqueries never fired (they ride the row + // stream). Re-run a prefix-only count over just the pool CTEs and their + // args (drop the trailing rrfk/limit args). Port of fused.go:322-342. + const prefix = `WITH ${poolCTEs.join(',\n')}\n`; + const prefixArgs = args.slice(0, poolArgsLen); + if (useFTS) { + ftsPoolSize = (await exec.query(prefix + 'SELECT count(*)::int AS n FROM fts_pool', prefixArgs)).rows[0].n; + } + if (useANN) { + annPoolSize = (await exec.query(prefix + 'SELECT count(*)::int AS n FROM ann_pool', prefixArgs)).rows[0].n; + } + } + const hits = rows.map((row) => { + const h = { ...row }; + delete h.fts_pool_size; + delete h.ann_pool_size; + return h; + }); + return { hits, ftsPoolSize, annPoolSize }; + } + + // Candidate-widening loop (port of fused.go:346-380). Start wide enough that + // the common single-chunk case is one query; grow innerChunks (doubling, + // capped by chunkCeiling) only while the ANN dedup collapses the pool below + // kPerSignal+1 and more chunks remain. FTS never collapses, so only ANN drives it. + let innerChunks = kPlus1 * FUSED_ANN_CHUNKS_PER_MESSAGE; + let result; + let prevAnnPool = -1; + for (;;) { + if (useANN && chunkCeiling > 0 && innerChunks > chunkCeiling) innerChunks = chunkCeiling; + // Each ANN attempt sets hnsw.ef_search to its OWN inner LIMIT (capped at + // HNSW_EF_SEARCH_MAX): at the pgvector default of 40 the HNSW scan + // truncated every attempt to ~40 chunks and this loop re-ran an identical + // plan up to the ceiling. FTS-only requests skip the transaction — no ANN + // scan, nothing to tune. + result = useANN + ? await withEfSearch(db, innerChunks, (tx) => runFused(tx, innerChunks)) + : await runFused(db, innerChunks); + if (!useANN || + result.annPoolSize >= kPlus1 || + result.annPoolSize >= filteredCeiling || + innerChunks >= chunkCeiling) break; + // A widened re-run that failed to GROW the pool will never grow it (the + // graph/ef_search budget is saturated) — stop instead of doubling toward + // the ceiling for identical results. + if (result.annPoolSize <= prevAnnPool) break; + prevAnnPool = result.annPoolSize; + let next = innerChunks * 2; + if (chunkCeiling > 0 && next > chunkCeiling) next = chunkCeiling; + if (next === innerChunks) break; + innerChunks = next; + } + + const poolSaturated = result.ftsPoolSize > kPerSignal || result.annPoolSize > kPerSignal; + return { hits: result.hits, poolSaturated, generation }; +} diff --git a/backend/src/services/embeddings/vectorStore.unit.test.js b/backend/src/services/embeddings/vectorStore.unit.test.js new file mode 100644 index 00000000..2fdd838e --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.unit.test.js @@ -0,0 +1,92 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../db.js', () => ({ pool: {}, withTransaction: vi.fn() })); +import { withTransaction } from '../db.js'; +import { vectorLiteral, searchWiden, stampSkipped } from './vectorStore.js'; + +describe('vectorLiteral', () => { + it('formats a float vector as pgvector text', () => { + expect(vectorLiteral([1, 2.5, -3.25])).toBe('[1,2.5,-3.25]'); + }); + it('emits an empty-brackets literal for an empty vector', () => { + expect(vectorLiteral([])).toBe('[]'); + }); +}); + +describe('searchWiden (A1 filtered early-exit)', () => { + it('early-exits once distinctEarlyExit is reached (filtered path, one run)', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return [{ messageId: 'a' }, { messageId: 'b' }]; }; + const hits = await searchWiden(5, 1000, 2, run); // k=5, ceiling=1000, earlyExit=2 + expect(runs).toEqual([20]); // did NOT widen up to the 1000-chunk ceiling + expect(hits.map((h) => h.rank)).toEqual([1, 2]); + }); + + it('widens by doubling until k distinct survive (no-filter: earlyExit=k)', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return Array.from({ length: Math.min(Math.floor(n / 10), 5) }, (_, i) => ({ messageId: 'm' + i })); }; + const hits = await searchWiden(5, 100, 5, run); + expect(runs).toEqual([20, 40, 80]); // 2 < 5, 4 < 5, then 8 → sliced to 5 + expect(hits).toHaveLength(5); + }); + + it('stops at the ceiling even when k is never reached', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return [{ messageId: 'only' }]; }; + const hits = await searchWiden(5, 30, 5, run); + expect(runs).toEqual([20, 30]); // clamps the doubled 40 to the 30 ceiling, then stops + expect(hits).toHaveLength(1); + }); +}); + +describe('stampSkipped (Wave D Fix 7 — message_count decrement)', () => { + beforeEach(() => { withTransaction.mockReset(); }); + + function scriptedClient({ deletedDistinct = 2 } = {}) { + const calls = []; + const client = { + calls, + query: vi.fn(async (text, params) => { + calls.push({ text, params }); + if (/FROM index_generations WHERE id = \$1 FOR UPDATE/.test(text)) return { rows: [{ id: params[0] }] }; + if (/^UPDATE messages SET embed_gen = \$1 WHERE id = \$2 AND/.test(text)) { + return { rowCount: params[1] === 'cas-missed' ? 0 : 1 }; // CAS stamp + } + if (/COUNT\(DISTINCT message_id\)/.test(text)) return { rows: [{ n: deletedDistinct }] }; + return { rows: [], rowCount: 1 }; + }), + }; + withTransaction.mockImplementation(async (fn) => fn(client)); + return client; + } + + it('decrements message_count by the DISTINCT messages actually deleted, in the same tx, under the generation row lock', async () => { + const client = scriptedClient({ deletedDistinct: 2 }); + const missed = await stampSkipped(9, + [{ id: 'cas-ok', lastModified: 't1' }, { id: 'cas-missed', lastModified: 't2' }], + ['plain-1']); + expect(missed).toEqual(['cas-missed']); + const texts = client.calls.map((c) => c.text); + // Generation row locked FIRST — same lock order as upsert()/msgvault Delete. + expect(texts[0]).toMatch(/SELECT id FROM index_generations WHERE id = \$1 FOR UPDATE/); + // Vectors deleted only for ids whose stamp landed (CAS-missed keeps its vector)… + const del = client.calls.find((c) => /DELETE FROM embeddings/.test(c.text)); + expect(del.params).toEqual([9, ['cas-ok', 'plain-1']]); + // …counted BEFORE the delete, and the count decrements the generation. + expect(texts.findIndex((t) => /COUNT\(DISTINCT message_id\)/.test(t))) + .toBeLessThan(texts.findIndex((t) => /DELETE FROM embeddings/.test(t))); + const upd = client.calls.find((c) => /UPDATE index_generations SET message_count = message_count - \$1/.test(c.text)); + expect(upd.params).toEqual([2, 9]); + }); + + it('skips the decrement when the stamped ids had no vectors to delete', async () => { + const client = scriptedClient({ deletedDistinct: 0 }); + await stampSkipped(9, [{ id: 'cas-ok', lastModified: 't1' }], []); + expect(client.calls.some((c) => /UPDATE index_generations SET message_count/.test(c.text))).toBe(false); + expect(client.calls.some((c) => /DELETE FROM embeddings/.test(c.text))).toBe(true); + }); + + it('is a no-op (no transaction) for empty inputs', async () => { + expect(await stampSkipped(9, [], [])).toEqual([]); + expect(withTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/embeddings/worker.integration.test.js b/backend/src/services/embeddings/worker.integration.test.js new file mode 100644 index 00000000..353d2573 --- /dev/null +++ b/backend/src/services/embeddings/worker.integration.test.js @@ -0,0 +1,58 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('worker end-to-end (pgvector)', () => { + let store, generations, worker, client, acctId, gen, userId; + const DIM = 4; + const fakeClient = { async embed(inputs) { return inputs.map((_, i) => [0.1, 0.2, 0.3, 0.4 + i * 1e-6]); } }; + + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + generations = await import('./generations.js'); + const { EmbeddingWorker } = await import('./worker.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(DIM); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // embed_runs FKs index_generations without ON DELETE CASCADE, so clear it first. + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); + // runOnce scans EVERY live embed_gen-IS-NULL message, so stray rows left by + // other IT files (withTestDb truncates before each test, not after the last) + // would inflate the coverage counts — clear messages like the tables above. + await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'wrk')); + for (let i = 0; i < 40; i++) { + await client.query(`INSERT INTO messages (account_id, uid, folder, subject, body_text) VALUES ($1,$2,'INBOX',$3,$4)`, + [acctId, 83000 + i, `subject ${i}`, `body content number ${i}`]); + } + gen = await generations.createGeneration('fake', DIM, 'fp-e2e'); + worker = new EmbeddingWorker({ store, client: fakeClient, generations, preprocessCfg: { stripHTML: true, collapseWhitespace: true }, maxInputChars: 32768, batchSize: 8 }); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('drives the corpus to full coverage and auto-activates the building generation', async () => { + // The worker was constructed with `generations` in its deps (above), so + // draining the scan for the building generation promotes it to active at the + // worker seam — NO manual activateGeneration call here. This is the regression + // guard for the wiring that was missing in production. + const res = await worker.runOnce(gen); + expect(res.succeeded).toBe(40); + const pending = await client.query('SELECT COUNT(*)::int n FROM messages WHERE account_id = $1 AND embed_gen IS DISTINCT FROM $2', [acctId, gen]); + expect(pending.rows[0].n).toBe(0); + const emb = await client.query('SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1', [gen]); + expect(emb.rows[0].n).toBe(40); + expect((await generations.activeGeneration()).id).toBe(gen); + expect(await generations.buildingGeneration()).toBeNull(); // promoted, no longer building + }); + + it('re-run is a no-op (idempotent coverage)', async () => { + const res = await worker.runOnce(gen); + expect(res.claimed).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/worker.js b/backend/src/services/embeddings/worker.js new file mode 100644 index 00000000..40bb2d8c --- /dev/null +++ b/backend/src/services/embeddings/worker.js @@ -0,0 +1,352 @@ +// Port of internal/vector/embed/worker.go RunOnce/RunBackstop (scan-and-fill). +import { preprocess } from './preprocess.js'; +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; +import { isPermanent4xx } from './client.js'; + +export const RAW_BODY_MULT = 16; + +export class EmbeddingWorker { + constructor(deps) { + this.deps = { batchSize: 32, maxConsecutiveFailures: 5, log: console, ...deps }; + } + + runOnce(gen) { return this._run(gen, false); } + runBackstop(gen) { return this._run(gen, true); } + + async _run(gen, backstop) { + const { store } = this.deps; + const res = { claimed: 0, succeeded: 0, failed: 0, truncated: 0 }; + let runId; + let runErr = null; + try { runId = (await store.startEmbedRun?.(gen)) || 0; } catch { runId = 0; } + try { + let consecutiveFailures = 0; + let afterId = backstop ? store.ZERO_UUID : await store.getWatermark(gen); + for (;;) { + const ids = await store.scanForEmbedding(gen, afterId, this.deps.batchSize); + if (!ids.length) { + if (!backstop) await store.resetWatermark(gen); // re-scan from start next tick + // Coverage reached: the shared driver seam (build route + scheduler) that + // promotes a fully-embedded 'building' generation to 'active'. + await this._activateIfBuildingCovered(gen); + return res; + } + res.claimed += ids.length; + const batchMax = ids[ids.length - 1]; + + let eb; + try { + eb = await this._embedBatch(gen, ids); + } catch (err) { + consecutiveFailures++; + this.deps.log.warn?.(`embed batch failed (gen ${gen}): ${err.message}`); + + if (isPermanent4xx(err)) { + // Downshift to size=1 and drain (worker.go RunOnce ErrPermanent4xx branch): + // embed what succeeds, stamp-drop confirmed per-message 4xx offenders, and + // leave rows unstamped only on an all-drop where the endpoint embedded nothing. + this.deps.log.info?.(`embed: downshifting to size=1 to drain failing batch (gen ${gen}, ${ids.length})`); + const dr = await this._downshiftDrain(gen, ids, res); + res.succeeded += dr.embedded; + // Reset the cap on embeddedOK (endpoint embedded+upserted something) — a + // CAS-missed stamp still proves the endpoint is healthy. + if (dr.embeddedOK > 0) consecutiveFailures = 0; + // Advance ONLY past the contiguously-stamped prefix so an unstamped + // straggler is never skipped (safeAdvanceID == batchMax on a clean drain). + if (dr.safeAdvanceID && dr.safeAdvanceID > afterId) { + afterId = dr.safeAdvanceID; + if (!backstop) await store.setWatermark(gen, dr.safeAdvanceID); + } + if (dr.drainErr) { + if (dr.drainErr.code === 'GEN_RETIRED') { this.deps.log.info?.(`generation ${gen} retired mid-drain; stopping`); return res; } + // A transient (non-4xx) error during the drain is a hard abort (an earlier + // singleton may have stamped, so the watermark stopped at the contiguous + // prefix and the next run re-finds the rest). + if (!isPermanent4xx(dr.drainErr)) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${dr.drainErr.message}`, { cause: err }); + } + // All-drop 4xx: the rows stay unstamped; let the failure cap trip. + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${dr.drainErr.message}`, { cause: err }); + } + } + continue; + } + + // Non-4xx (transient) error: leave the batch unstamped, do not advance the + // cursor, so the failure cap short-circuits a persistent fault. + res.failed += ids.length; + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${err.message}`, { cause: err }); + } + continue; // do not advance cursor; next scan re-finds the batch + } + res.truncated += eb.truncated; + const skipIds = [...eb.missing, ...eb.empty]; + + if (!eb.chunks.length) { + if (skipIds.length) await this._stampSkipped(gen, skipIds, eb.lastModified); + consecutiveFailures = 0; + afterId = batchMax; + if (!backstop) await store.setWatermark(gen, batchMax); + continue; + } + + // Step 1: upsert embeddings FIRST (idempotent; crash-safe ordering). + try { + await store.upsert(gen, eb.chunks); + } catch (err) { + if (err.code === 'GEN_RETIRED') { this.deps.log.info?.(`generation ${gen} retired mid-run; stopping`); return res; } + consecutiveFailures++; + res.failed += eb.embeddedIds.length; + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${err.message}`, { cause: err }); + } + continue; + } + + // Step 2: skip-mark empty/missing, then CAS-stamp embedded rows. + if (skipIds.length) await this._stampSkipped(gen, skipIds, eb.lastModified); + const missed = await store.setEmbedGenIfUnchanged( + eb.embeddedIds.map((id) => ({ id, lastModified: eb.lastModified.get(id) })), gen, + ); + if (missed.length) this.deps.log.info?.(`embed_gen CAS misses (concurrent edit): ${missed.length} — backstop recovers`); + res.succeeded += eb.embeddedIds.length - missed.length; + consecutiveFailures = 0; + // Advance the watermark even on a whole-batch CAS miss (backstop recovers misses). + afterId = batchMax; + if (!backstop) await store.setWatermark(gen, batchMax); + this.deps.onProgress?.({ done: res.succeeded, claimed: res.claimed, truncated: res.truncated }); + } + } catch (err) { + runErr = err; + throw err; + } finally { + try { await store.finalizeEmbedRun?.(runId, res, runErr); } catch { /* observability only */ } + } + } + + // Activation-on-coverage seam (worker.go parity — see divergence note below). + // A run reaches here only after scanForEmbedding drains, i.e. no live message + // still needs embedding for `gen` (poison messages count as covered — they are + // stamped, not skipped). When `gen` is the CURRENT building generation, promote + // it to 'active'. This is the sole production caller of activateGeneration: + // without it a completed build stays 'building' forever and hybrid search never + // leaves its index_building fallback. + // + // No-ops when: + // - no generations collaborator is wired (a bare worker with nothing to promote), or + // - `gen` is not the building generation — an active generation's incremental + // or backstop run must never re-activate (README: generations never mix). + // + // Tolerates the two lifecycle races activateGeneration encodes, so a completed + // build is never lost and activation never fails the embed run: + // - "still has messages needing embedding": a late content edit re-NULLed a row + // between the drain and the activate (the coverage gate re-asserted inside the + // activate tx caught it) — leave it building; the next scheduler tick re-drains + // and retries. + // - "not in 'building' state": a concurrent activate/retire already moved it — no-op. + // Any other error is logged, not thrown: activation is a post-coverage promotion, + // not part of the run result. + // + // Divergence from msgvault worker.go: there activation lives in the DRIVERS + // (scheduler embed_job.go + the CLI embed_vector.go each re-check coverage and + // call ActivateGeneration). We fold it into this shared worker seam instead so + // BOTH Mailflow drivers (the fire-and-forget build route and the scheduler tick) + // get it from one place and cannot drift out of sync — the exact failure that + // left the live build wedged in 'building'. The coverage precondition (scan + // drained) and the no-force, gate-re-asserted activate call match worker.go. + async _activateIfBuildingCovered(gen) { + const gens = this.deps.generations; + if (!gens?.activateGeneration || !gens?.buildingGeneration) return; + let building; + try { + building = await gens.buildingGeneration(); + } catch (err) { + this.deps.log.warn?.(`embed: could not read building generation to activate ${gen}: ${err.message}`); + return; + } + if (!building || building.id !== gen) return; // active/incremental run — nothing to promote + try { + await gens.activateGeneration(gen); // no force — activateGeneration re-asserts the coverage gate atomically + this.deps.log.info?.(`embed: generation ${gen} fully covered — activated`); + } catch (err) { + this.deps.log.warn?.(`embed: generation ${gen} not activated (${err.message}); scheduler will retry next tick if still building`); + } + } + + async _embedBatch(gen, ids) { + const { store } = this.deps; + const rows = await store.fetchForEmbedding(ids); + const fetched = new Set(); + const lastModified = new Map(); + const msgs = []; + const empty = []; + for (const r of rows) { + fetched.add(r.id); + lastModified.set(r.id, r.lastModified); + const body = r.bodyText && r.bodyText.trim() !== '' ? r.bodyText : (r.bodyHtml || ''); + const ppCfg = { ...this.deps.preprocessCfg }; + if (!ppCfg.maxBodyRunes && this.deps.maxInputChars > 0) { + ppCfg.maxBodyRunes = this.deps.maxInputChars * MAX_SPANS * RAW_BODY_MULT; + } + const { text, truncated } = preprocess(r.subject || '', body, 0, ppCfg); + if (text.trim() === '') { empty.push(r.id); continue; } + msgs.push({ id: r.id, text, bodyTruncated: truncated }); + } + const missing = ids.filter((id) => !fetched.has(id)); + if (!msgs.length) return { chunks: [], embeddedIds: [], missing, empty, truncated: 0, lastModified }; + + const window = this.deps.maxInputChars; + const overlap = chunkOverlapFor(window); + const pieces = []; + const inputs = []; + const truncatedMsg = new Set(); + for (const m of msgs) { + const { spans, tailDropped } = chunkText(m.text, window, overlap, MAX_SPANS); + const msgTrunc = m.bodyTruncated || tailDropped; + spans.forEach((sp, j) => { + const hardCut = window > 0 && (sp.charEnd - sp.charStart) === window && j < spans.length - 1; + const trunc = msgTrunc || hardCut; + if (trunc) truncatedMsg.add(m.id); + pieces.push({ id: m.id, chunkIndex: j, text: sp.text, chars: sp.charEnd - sp.charStart, charStart: sp.charStart, charEnd: sp.charEnd, trunc }); + inputs.push(sp.text); + }); + } + + const vecs = []; + try { + for (let i = 0; i < inputs.length; i += this.deps.batchSize) { + const got = await this.deps.client.embed(inputs.slice(i, i + this.deps.batchSize)); + vecs.push(...got); + } + } catch (err) { + // Attach the CAS tokens fetched before the embed call so a downshift drain can + // CAS-drop a per-message 4xx offender with the last_modified read at fetch time. + err.lastModified = lastModified; + throw err; + } + if (vecs.length !== pieces.length) throw new Error(`embedder returned ${vecs.length} vectors for ${pieces.length} chunk inputs`); + + const chunks = []; + const embeddedIds = []; + const seen = new Set(); + for (let i = 0; i < pieces.length; i++) { + const p = pieces[i]; + chunks.push({ messageId: p.id, chunkIndex: p.chunkIndex, vector: vecs[i], sourceCharLen: p.chars, chunkCharStart: p.charStart, chunkCharEnd: p.charEnd, truncated: p.trunc }); + if (!seen.has(p.id)) { seen.add(p.id); embeddedIds.push(p.id); } + } + return { chunks, embeddedIds, missing, empty, truncated: truncatedMsg.size, lastModified }; + } + + // Port of worker.go downshiftDrain: walk a 4xx-failing batch one message at a time. + // Embed + upsert + CAS-stamp what succeeds; skip-mark empty/missing; and DEFER + // per-message 4xx offenders — stamp-dropping them only when some sibling embedded + // (proving the endpoint is healthy, so the 4xx is message-specific). On an all-drop + // (endpoint embedded nothing) the deferred ids are left UNSTAMPED and a permanent-4xx + // error is returned so a misconfigured endpoint never silently loses work. Returns + // { embedded, embeddedOK, stamped, safeAdvanceID, drainErr }; safeAdvanceID is the + // highest CONTIGUOUSLY-stamped id (batchMax on a clean drain) so the caller never + // advances the watermark past an unstamped straggler. + async _downshiftDrain(gen, ids, res) { + const { store } = this.deps; + let embedded = 0; + let embeddedOK = 0; + let stamped = 0; + let contiguousStampedID = null; + let brokeContiguity = false; + const deferredDrops = []; + let lastDeferredErr = null; + const lm = new Map(); + const advance = (id, didStamp) => { + if (didStamp) { if (!brokeContiguity) contiguousStampedID = id; } + else brokeContiguity = true; + }; + + for (const id of ids) { + let eb; + try { + eb = await this._embedBatch(gen, [id]); + } catch (e) { + if (isPermanent4xx(e)) { + if (e.lastModified) for (const [k, v] of e.lastModified) lm.set(k, v); + deferredDrops.push(id); + lastDeferredErr = e; + brokeContiguity = true; // a deferred id breaks the stamped-from-start prefix + continue; + } + // Transient error: leave this id unstamped and abort the drain; the watermark + // stays at the contiguous stamped prefix so the next run re-finds it. + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: e }; + } + for (const [k, v] of eb.lastModified) lm.set(k, v); + + if (!eb.chunks.length) { + // Missing/empty singleton — skip-mark it. + const skip = [...eb.missing, ...eb.empty]; + let didStamp = true; + if (skip.length) { + const missed = await this._stampSkipped(gen, skip, eb.lastModified); + const stampedSkip = skip.length - missed.length; + stamped += stampedSkip; + didStamp = stampedSkip > 0; // a CAS-missed skip leaves the row unstamped + } + advance(id, didStamp); + continue; + } + + try { + await store.upsert(gen, eb.chunks); + } catch (uerr) { + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: uerr }; + } + embeddedOK++; // endpoint demonstrably embedded + upserted this singleton + const missed = await store.setEmbedGenIfUnchanged( + eb.embeddedIds.map((mid) => ({ id: mid, lastModified: eb.lastModified.get(mid) })), gen, + ); + const stampedHere = eb.embeddedIds.length - missed.length; + res.truncated += eb.truncated; + embedded += stampedHere; + stamped += stampedHere; + advance(id, stampedHere > 0); + } + + // Clean drain: every id is resolved (stamped, skip-marked, or a deferred 4xx below). + const safeAdvanceID = ids[ids.length - 1]; + + if (!deferredDrops.length) return { embedded, embeddedOK, stamped, safeAdvanceID, drainErr: null }; + + if (embeddedOK > 0) { + // The endpoint embedded something, so the 4xxs are message-specific — stamp-drop them. + for (const id of deferredDrops) { + this.deps.log.warn?.(`stamping (dropping) message after singleton 4xx (gen ${gen}, id ${id}): ${lastDeferredErr?.message}`); + } + const missed = await this._stampSkipped(gen, deferredDrops, lm); + stamped += deferredDrops.length - missed.length; + return { embedded, embeddedOK, stamped, safeAdvanceID, drainErr: null }; + } + + // embeddedOK === 0: endpoint embedded nothing — can't distinguish an endpoint-wide + // failure from a batch where every message is unembeddable. Leave the deferred ids + // UNSTAMPED and surface the 4xx (marked permanent so the caller's cap, not a hard + // abort, governs) so a misconfigured endpoint does not silently drop work. + const err = new Error(`downshift all-drop: every singleton returned non-retryable 4xx (left ${deferredDrops.length} row(s) unstamped): ${lastDeferredErr?.message}`); + err.permanent4xx = true; + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: err }; + } + + // Skip-mark empty/missing rows (drops them from the next scan) and remove any stale + // vectors for the rows that were actually stamped — the stamp + prune happen in ONE + // transaction inside store.stampSkipped (msgvault worker.go parity). CAS-protected + // for rows with a last_modified token; unconditional for missing rows. + async _stampSkipped(gen, ids, lastModified) { + const { store } = this.deps; + const cas = []; + const plain = []; + for (const id of ids) { + if (lastModified.has(id)) cas.push({ id, lastModified: lastModified.get(id) }); + else plain.push(id); + } + return store.stampSkipped(gen, cas, plain); + } +} diff --git a/backend/src/services/embeddings/worker.test.js b/backend/src/services/embeddings/worker.test.js new file mode 100644 index 00000000..e61692db --- /dev/null +++ b/backend/src/services/embeddings/worker.test.js @@ -0,0 +1,250 @@ +import { describe, it, expect, vi } from 'vitest'; +import { EmbeddingWorker } from './worker.js'; + +const ZERO = '00000000-0000-0000-0000-000000000000'; +const uid = (n) => `00000000-0000-0000-0000-${String(n).padStart(12, '0')}`; + +function makeStore(messages, opts = {}) { + const watermark = new Map(); + const state = { upsertCalls: 0, resetCalls: 0 }; + return { + ZERO_UUID: ZERO, + _messages: messages, + _state: state, + async scanForEmbedding(target, afterId, limit) { + return messages + .filter((m) => (m.embedGen == null || m.embedGen !== target) && !m.isDeleted && m.id > afterId) + .sort((a, b) => (a.id < b.id ? -1 : 1)) + .slice(0, limit) + .map((m) => m.id); + }, + async fetchForEmbedding(ids) { + return messages.filter((m) => ids.includes(m.id)).map((m) => ({ + id: m.id, subject: m.subject || '', bodyText: m.bodyText || '', bodyHtml: m.bodyHtml || '', lastModified: m.lastModified, + })); + }, + async upsert() { state.upsertCalls++; }, + async setEmbedGen(ids, target) { for (const id of ids) { const m = messages.find((x) => x.id === id); if (m) m.embedGen = target; } }, + async setEmbedGenIfUnchanged(items, target) { + const missed = []; + for (const it of items) { + const m = messages.find((x) => x.id === it.id); + if (opts.casMiss?.has(it.id)) { missed.push(it.id); continue; } + if (m && m.lastModified === it.lastModified) m.embedGen = target; else missed.push(it.id); + } + return missed; + }, + async stampSkipped(target, casItems, plainIds) { + const missed = []; + for (const it of casItems) { + const m = messages.find((x) => x.id === it.id); + if (opts.casMiss?.has(it.id)) { missed.push(it.id); continue; } + if (m && m.lastModified === it.lastModified) m.embedGen = target; else missed.push(it.id); + } + for (const id of plainIds) { const m = messages.find((x) => x.id === id); if (m) m.embedGen = target; } + return missed; + }, + async getWatermark(gen) { state.getWatermarkCalled = true; return watermark.get(gen) || ZERO; }, + async setWatermark(gen, id) { watermark.set(gen, id); }, + async resetWatermark(gen) { state.resetCalls++; watermark.set(gen, ZERO); }, + }; +} +const fakeClient = { async embed(inputs) { return inputs.map(() => [0.1, 0.2, 0.3, 0.4]); } }; +const deps = (store, over = {}) => ({ store, client: fakeClient, preprocessCfg: {}, maxInputChars: 32768, batchSize: 32, ...over }); + +// A generations collaborator stub for the activation seam. `building` is what +// buildingGeneration() returns (null ⇒ the run's gen is NOT building — an active +// or incremental run); `activate` is the activateGeneration spy. +function makeGenerations({ building = null, activate } = {}) { + return { + buildingGeneration: vi.fn(async () => building), + activateGeneration: activate || vi.fn(async () => {}), + }; +} + +describe('EmbeddingWorker.runOnce', () => { + it('embeds pending messages and stamps embed_gen', async () => { + const msgs = [ + { id: uid(1), subject: 'hello', bodyText: 'world', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'foo', bodyText: 'bar', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs); + const w = new EmbeddingWorker(deps(store)); + const res = await w.runOnce('7'); + expect(res.claimed).toBe(2); + expect(res.succeeded).toBe(2); + expect(msgs.every((m) => m.embedGen === '7')).toBe(true); + }); + + it('excludes a CAS miss from succeeded but still advances the watermark', async () => { + const msgs = [ + { id: uid(1), subject: 's1', bodyText: 'b1', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 's2', bodyText: 'b2', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs, { casMiss: new Set([uid(2)]) }); + const w = new EmbeddingWorker(deps(store)); + const res = await w.runOnce('7'); + expect(res.succeeded).toBe(1); + const m2 = msgs.find((m) => m.id === uid(2)); + expect(m2.embedGen).toBeNull(); // recovered by backstop later + }); + + it('is idempotent — a second run finds nothing', async () => { + const msgs = [{ id: uid(1), subject: 's', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + const w = new EmbeddingWorker(deps(store)); + await w.runOnce('7'); + const res2 = await w.runOnce('7'); + expect(res2.claimed).toBe(0); + }); + + it('resets the watermark on scan exhaustion (P2)', async () => { + const store = makeStore([]); + const w = new EmbeddingWorker(deps(store)); + await w.runOnce('7'); + expect(store._state.resetCalls).toBeGreaterThan(0); + }); + + it('aborts after maxConsecutiveFailures on a persistent embed failure (does not loop)', async () => { + const msgs = Array.from({ length: 10 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const scanSpy = vi.spyOn(store, 'scanForEmbedding'); + const failingClient = { async embed() { throw new Error('unreachable'); } }; + const w = new EmbeddingWorker(deps(store, { client: failingClient, maxConsecutiveFailures: 3 })); + await expect(w.runOnce('7')).rejects.toThrow(/aborting after 3 consecutive failures/); + expect(scanSpy.mock.calls.length).toBe(3); // bounded, not infinite + }); + + it('downshifts on a permanent 4xx: isolates the poison message and embeds the rest (W1)', async () => { + const msgs = [ + { id: uid(1), subject: 'good one', bodyText: 'hello', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'poison here', bodyText: 'x', lastModified: 't2', embedGen: null }, + { id: uid(3), subject: 'good two', bodyText: 'world', lastModified: 't3', embedGen: null }, + ]; + const store = makeStore(msgs); + // 4xx whenever the batch contains the poison text; succeeds otherwise. + const poisonClient = { + async embed(inputs) { + if (inputs.some((t) => t.includes('poison'))) { const e = new Error('unembeddable input'); e.permanent4xx = true; throw e; } + return inputs.map(() => [0.1, 0.2, 0.3, 0.4]); + }, + }; + const w = new EmbeddingWorker(deps(store, { client: poisonClient, maxConsecutiveFailures: 3 })); + const res = await w.runOnce('7'); + expect(msgs.find((m) => m.id === uid(1)).embedGen).toBe('7'); // embedded + expect(msgs.find((m) => m.id === uid(3)).embedGen).toBe('7'); // embedded + expect(msgs.find((m) => m.id === uid(2)).embedGen).toBe('7'); // stamp-dropped (leaves the scan) + expect(res.succeeded).toBe(2); // only the two good ones count as embedded + }); + + it('all-4xx batch leaves rows unstamped and aborts after maxConsecutiveFailures (no silent drop) (W1)', async () => { + const msgs = Array.from({ length: 4 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const scanSpy = vi.spyOn(store, 'scanForEmbedding'); + const allBadClient = { async embed() { const e = new Error('always bad'); e.permanent4xx = true; throw e; } }; + const w = new EmbeddingWorker(deps(store, { client: allBadClient, maxConsecutiveFailures: 3 })); + await expect(w.runOnce('7')).rejects.toThrow(/aborting after 3 consecutive failures/); + expect(msgs.every((m) => m.embedGen === null)).toBe(true); // nothing silently dropped + expect(scanSpy.mock.calls.length).toBe(3); // bounded + }); +}); + +// The activation seam: the SHARED worker-run completion point that both drivers +// (the manual build route and the scheduler) reach. When a run drains the scan +// for the generation that is currently 'building', the worker promotes it to +// 'active' — the production wiring that was missing (activateGeneration had no +// non-test caller, so completed builds stayed 'building' forever). +describe('EmbeddingWorker activation seam', () => { + it('activates the building generation once, without force, after the scan drains', async () => { + const gen = '7'; + const msgs = [ + { id: uid(1), subject: 'a', bodyText: 'b', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'c', bodyText: 'd', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + const res = await w.runOnce(gen); + expect(res.succeeded).toBe(2); + expect(generations.buildingGeneration).toHaveBeenCalled(); + expect(activateSpy).toHaveBeenCalledTimes(1); + expect(activateSpy.mock.calls[0]).toEqual([gen]); // called with the gen only — no force + }); + + it('does not activate when the run aborts before draining the scan', async () => { + const gen = '7'; + const msgs = Array.from({ length: 6 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const failingClient = { async embed() { throw new Error('unreachable'); } }; + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { client: failingClient, maxConsecutiveFailures: 2, generations })); + await expect(w.runOnce(gen)).rejects.toThrow(/aborting after/); + expect(activateSpy).not.toHaveBeenCalled(); // the scan never drained → no coverage → no activation + }); + + it('never activates an active generation on an incremental run (no building generation)', async () => { + const gen = '7'; + const store = makeStore([]); // nothing pending — scan drains immediately + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: null, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runOnce(gen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('never activates an active generation on a backstop run', async () => { + const gen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: null, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runBackstop(gen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('does not activate the running generation when a DIFFERENT generation is building (rebuild in progress)', async () => { + const activeGen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: '8' }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runOnce(activeGen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('swallows the "still has messages" activation race and leaves the run result intact', async () => { + const gen = '7'; + const msgs = [{ id: uid(1), subject: 'a', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + const activateSpy = vi.fn(async () => { throw new Error(`generation ${gen} still has messages needing embedding; pass force to override`); }); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + const res = await w.runOnce(gen); // must NOT reject — activation is a post-coverage promotion + expect(res.succeeded).toBe(1); + expect(activateSpy).toHaveBeenCalledTimes(1); + }); + + it('swallows the "not in building state" activation race (already active/retired)', async () => { + const gen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => { throw new Error(`generation ${gen} not in 'building' state`); }); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await expect(w.runOnce(gen)).resolves.toBeTruthy(); + expect(activateSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('EmbeddingWorker.runBackstop', () => { + it('ignores the watermark and finds a sub-watermark straggler', async () => { + const msgs = [{ id: uid(1), subject: 's', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + await store.setWatermark('7', uid(999)); // high watermark that would hide id 1 + const w = new EmbeddingWorker(deps(store)); + const res = await w.runBackstop('7'); + expect(res.succeeded).toBe(1); + expect(msgs[0].embedGen).toBe('7'); + }); +}); diff --git a/backend/src/services/imapManager.js b/backend/src/services/imapManager.js index f6219422..e1a20f81 100644 --- a/backend/src/services/imapManager.js +++ b/backend/src/services/imapManager.js @@ -3735,7 +3735,7 @@ export class ImapManager { // in the DB and the click returns instantly without a live IMAP round-trip. // Shared fetch+sanitize+snippet+UPDATE step behind every body-materialization path // (opportunistic new-mail prefetch, on-view folder prefetch, and the body-backfill - // drainer). Kept as one helper so the UPDATE shape that fires the search_fts + // drainer). Kept as one helper so the UPDATE shape that fires the search_fts/last_modified // triggers never drifts between callers. Returns true if a body was actually written. // Private (underscore-prefixed, matching this file's convention for internal helpers like // _enqueueFlagPush) — not new public surface, so it doesn't count against the diff --git a/backend/src/services/migrations.collation.test.js b/backend/src/services/migrations.collation.test.js new file mode 100644 index 00000000..d0690543 --- /dev/null +++ b/backend/src/services/migrations.collation.test.js @@ -0,0 +1,58 @@ +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('./db.js', () => ({ pool: { query: vi.fn(), connect: vi.fn() } })); +import { warnOnCollationMismatch } from './migrations.js'; + +const row = (over = {}) => ({ + rows: [{ db: 'mailflow', recorded: '2.36', actual: '2.36', ...over }], +}); + +describe('warnOnCollationMismatch', () => { + it('warns loudly, naming REINDEX DATABASE, when versions diverge', async () => { + const warn = vi.fn(); + const query = vi.fn().mockResolvedValue(row({ recorded: '1.2.38', actual: '2.36' })); + const hit = await warnOnCollationMismatch({ query, warn }); + expect(hit).toBe(true); + expect(warn).toHaveBeenCalledTimes(1); + const msg = warn.mock.calls[0][0]; + expect(msg).toContain('collation version mismatch'); + expect(msg).toContain('1.2.38'); + expect(msg).toContain('2.36'); + expect(msg).toContain('REINDEX DATABASE "mailflow";'); + expect(msg).toContain('REFRESH COLLATION VERSION'); + }); + + it('stays silent when the recorded and actual versions match', async () => { + const warn = vi.fn(); + const hit = await warnOnCollationMismatch({ query: vi.fn().mockResolvedValue(row()), warn }); + expect(hit).toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); + + it('warns on the alpine→glibc signature: NULL recorded version, versioned current libc', async () => { + // Field-verified: musl (postgres:16-alpine) records NO datcollversion, so after + // the swap to pgvector/pgvector:pg16 the row reads recorded=NULL, actual=2.36 — + // and Postgres itself stays silent. This is the primary case to catch. + const warn = vi.fn(); + const query = vi.fn().mockResolvedValue(row({ recorded: null, actual: '2.36' })); + expect(await warnOnCollationMismatch({ query, warn })).toBe(true); + const msg = warn.mock.calls[0][0]; + expect(msg).toContain('reported no collation'); + expect(msg).toContain('REINDEX DATABASE "mailflow";'); + }); + + it('stays silent for versionless locales (NULL actual, e.g. C/POSIX or still on musl)', async () => { + const warn = vi.fn(); + for (const over of [{ actual: null }, { recorded: null, actual: null }]) { + expect(await warnOnCollationMismatch({ query: vi.fn().mockResolvedValue(row(over)), warn })).toBe(false); + } + expect(warn).not.toHaveBeenCalled(); + }); + + it('never throws — a failing query (e.g. PG < 15) only skips the check', async () => { + const warn = vi.fn(); + const query = vi.fn().mockRejectedValue(new Error('column "datcollversion" does not exist')); + await expect(warnOnCollationMismatch({ query, warn })).resolves.toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/migrations.js b/backend/src/services/migrations.js index 38fc9a20..6345fc9d 100644 --- a/backend/src/services/migrations.js +++ b/backend/src/services/migrations.js @@ -94,3 +94,70 @@ export async function runMigrations() { client.release(); } } + +// --- Collation-version drift check (best-effort, never aborts boot) --------------- +// The compose move from postgres:16-alpine (musl libc) to pgvector/pgvector:pg16 +// (Debian, glibc) silently changes how the OS collates text. Postgres records the +// collation version a database was created under (pg_database.datcollversion, PG15+); +// pg_database_collation_actual_version(oid) reports what the running server's libc/ICU +// provides NOW. Two shapes of drift matter here (field-verified on both images): +// - recorded != actual: a versioned libc changed underneath (e.g. a glibc bump). +// Postgres emits its own per-connection WARNING for this, easily lost in logs. +// - recorded IS NULL while actual is not: the database was created under a libc +// that reports NO collation version — exactly what musl/alpine does — and now +// runs under glibc. This is the actual alpine → pgvector upgrade signature, and +// Postgres itself stays completely SILENT about it. +// (actual is NULL for versionless locales like C/POSIX — byte-order collation is +// immune to libc swaps, so there is nothing to warn about.) +// Either way, text indexes built under the old ordering may silently return wrong or +// missing rows until reindexed, so surface it loudly at boot with the remedy. The +// REFRESH COLLATION VERSION step records the current version and silences this +// warning on subsequent boots. Returns whether a mismatch was reported +// (observability only; never throws). +export async function warnOnCollationMismatch(deps = {}) { + const q = deps.query || ((text) => pool.query(text)); + const warn = deps.warn || console.warn; + try { + const { rows } = await q(` + SELECT current_database() AS db, + datcollversion AS recorded, + pg_database_collation_actual_version(oid) AS actual + FROM pg_database + WHERE datname = current_database() + `); + const r = rows[0]; + if (!r || !r.actual || r.recorded === r.actual) return false; + const origin = r.recorded + ? [ + ` Database "${r.db}" was created under collation version ${r.recorded}, but`, + ` the operating system now provides ${r.actual}.`, + ] + : [ + ` Database "${r.db}" was created under a C library that reported no collation`, + ` version (e.g. postgres:16-alpine/musl); the current one provides ${r.actual}.`, + ]; + warn([ + '='.repeat(76), + 'WARNING: database collation version mismatch detected', + '', + ...origin, + ' This happens when the Postgres image\'s C library changes — e.g. the', + ' docker-compose switch from postgres:16-alpine (musl) to', + ' pgvector/pgvector:pg16 (glibc).', + '', + ' Text indexes built under the old collation can silently return wrong or', + ' missing rows. Reindex once, then record the new version:', + '', + ` docker compose exec postgres psql -U mailflow -d ${r.db} \\`, + ` -c 'REINDEX DATABASE "${r.db}";' \\`, + ` -c 'ALTER DATABASE "${r.db}" REFRESH COLLATION VERSION;'`, + '='.repeat(76), + ].join('\n')); + return true; + } catch (err) { + // Postgres < 15 has no datcollversion; a restricted role may not read pg_database. + // The check is observability only — never block or fail the boot on it. + console.log(`Collation version check skipped: ${err.message}`); + return false; + } +} diff --git a/backend/src/services/search/lexicalRepo.js b/backend/src/services/search/lexicalRepo.js index 4d346fce..0b9e7b59 100644 --- a/backend/src/services/search/lexicalRepo.js +++ b/backend/src/services/search/lexicalRepo.js @@ -44,14 +44,17 @@ export function freeTextTermCondition(likeIdx, ftsIdx) { } // BM25-style lexical rank (ts_rank_cd; class weights D,C,B,A → 10:4:1 -// subject:from:rest; length normalization 32). +// subject:from:rest; length normalization 32). Port of pgvector/fused.go:31-36. +// Exported so the fused query reuses it as the lexical leg. export const LEXICAL_RANK_SQL = (vectorExpr, queryExpr) => `ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], ${vectorExpr}, ${queryExpr}, 32)`; // A term counts as searchable text only if it has at least one letter or // digit; msgvault's hasFTSToken drops punctuation-only tokens ("!!!", "***") // the same way, since they'd normalize to zero lexemes and can't usefully -// match or rank anything. +// match or rank anything. Exported so every caller building a free-text +// query (searchLexical, searchService's semantic branch, vectorStore's fused +// BM25 leg) applies the identical hygiene rather than re-deriving it. export function hasSearchableToken(term) { return /[\p{L}\p{N}]/u.test(term); } @@ -73,6 +76,13 @@ function isPhraseTerm(term) { // an ordinary parameter; quote_literal() at query time safely wraps it as a // single tsquery lexeme literal — neutralizing any &, |, !, (, ), ', or : // the term might contain — before ':*' marks it for prefix matching. +// +// Exported (as the raw tsquery expression, not the whole `vectorExpr @@ ...` +// predicate) so `vectorStore.fusedSearch`'s BM25 leg builds its `@@` match +// AND its ts_rank_cd query-arg from this SAME per-term construction, rather +// than forking a second, non-prefix `plainto_tsquery` builder — a review +// caught exactly that fork (msgvault's fused.go reuses BuildFTSTerm for the +// identical reason: one construction, every caller). export function ftsTermQueryArg(ftsIdx, term) { return isPhraseTerm(term) ? `plainto_tsquery('english', $${ftsIdx})` @@ -108,15 +118,18 @@ export function freeTextTermConditionRanked(likeIdx, ftsIdx, term) { // uses, so guard and match can never disagree; a query of ONLY stopwords // degrades to a filter-only, date-ordered search. The relevance rank needs // no guard: `&&` drops an empty tsquery operand and ts_rank_cd over a fully -// empty tsquery is 0. +// empty tsquery is 0 — both verified against pgvector/pg16 (2026-07-17). export function stopwordSafeCondition(ftsIdx, term, condition) { return `(numnode(${ftsTermQueryArg(ftsIdx, term)}) = 0 OR ${condition})`; } -// The one owner of a free-text term's full predicate: ranked FTS match, -// un-backfilled ILIKE fallback, polarity, and stopword vacuity. `bind(value) → -// '$n'` pushes onto the caller's params; the raw ordinals expected by -// freeTextTermConditionRanked are recovered from the placeholders. +// The one owner of a free-text term's FULL metadata-scope predicate — ranked +// FTS match, un-backfilled ILIKE fallback, polarity, stopword vacuity — shared +// by searchLexical, the fused query's NOT-conditions (negatedFreeTextClause), +// and MCP stage_deletion (engineAdapter), so a search preview and a staged +// deletion set can never disagree on what a term matches. `bind(value) → '$n'` +// pushes onto the caller's params; the raw ordinals freeTextTermConditionRanked +// expects are recovered from the placeholders so callers don't juggle them. export function freeTextTermClause(term, negate, bind) { const likeIdx = Number(bind(`%${term}%`).slice(1)); const ftsIdx = Number(bind(term).slice(1)); @@ -124,6 +137,15 @@ export function freeTextTermClause(term, negate, bind) { return stopwordSafeCondition(ftsIdx, term, negate ? negateCond(cond) : cond); } +// Negated free-text NOT-condition for the fused (vector/hybrid) path, built from +// the SAME per-term construction (prefix/phrase match + the un-backfilled ILIKE +// fallback + stopword vacuity) that searchLexical applies, so `invoice -draft` +// excludes drafts identically in every mode. Prefix-vs-phrase semantics follow +// the term, via ftsTermQueryArg, exactly as the positive lexical predicate does. +export function negatedFreeTextClause(term, bind) { + return freeTextTermClause(term, true, bind); +} + // The weighted tsvector written into messages.search_fts. `ref` is the row // reference: 'm' for the backfill UPDATE, 'NEW' for the BEFORE trigger. The // class weights map to ts_rank_cd's D,C,B,A array (10:4:1 subject:from:rest). @@ -138,7 +160,9 @@ export function searchFtsExpr(ref) { // Structured operator predicates (from/to/cc/subject/has/is/after/before), // negation-aware. `bind(value) → '$n'` pushes value onto the caller's param // list and returns its placeholder — the caller owns `params`/the running -// index, so searchLexical remains the single owner of these predicates. +// index, so this stays reusable for both searchLexical (one shared params +// array) and the fused query (its own per-leg bind closure, README "one +// search seam" — lexicalRepo remains the single owner of these predicates). // Excludes free-text terms and folder scope (`in:`), which are not // structured row predicates. Extracted from searchLexical byte-identically — // same branches, same ILIKE-arm reuse, same skip rules. @@ -177,7 +201,10 @@ export function buildOperatorClauses(filters, bind) { return conditions; } -// Folder-scope predicate(s) for the messages table. `folderScope`/`folderFuzzy` come from +// Folder-scope predicate(s) for the messages table, shared by the lexical path +// and the fused (vector/hybrid) path so every mode scopes to the SAME folder — +// semantic search must not leak Sent/Archive/Trash into an Inbox search, and an +// explicit in:sent must apply. `folderScope`/`folderFuzzy` come from // resolveSearchFolderScope: a truthy fuzzy scope (in:) matches the bare // name OR any .../ path; an exact scope (REST ?folder=) matches the full // path; a null scope carries no explicit folder and excludes trash-like folders @@ -214,11 +241,18 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f conditions.push(freeTextTermClause(term.value, term.negate, bind)); } - // A bare in:inbox (or lone folder param) must never dump a whole folder. + // A bare in:inbox (or lone folder param) must never dump a whole folder. (No total + // here: MCP search_metadata pre-checks free text, so a no-condition search — a + // filter-only/empty query — only reaches this path via REST, which ignores total.) if (!conditions.length) return { rows: [], hasCondition: false }; for (const cond of buildFolderScopeClauses(folderScope, folderFuzzy, bind)) conditions.push(cond); + // Snapshot the predicate binds (accountIds + operator/term/folder) BEFORE the + // rank/LIMIT/OFFSET binds are appended, so the metadata COUNT reuses the exact same + // WHERE with the exact same param ordinals (MCP search_metadata needs a real total). + const countParams = params.slice(); + // D5: rank a free-text search by ts_rank_cd over the combined positive terms // (date/id tiebreak); a filter-only search stays date-ordered. The rank arg // must be bound before LIMIT/OFFSET. @@ -229,8 +263,8 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f if (ordering === 'relevance' && positiveTerms.length) { // Rank with the SAME prefix-aware tsquery the MATCH predicate uses // (freeTextTermConditionRanked → ftsTermQueryArg): one bind per term, - // combined with && (ts_rank_cd takes a single tsquery). Reusing the one - // term→tsquery-arg builder is what makes + // combined with && (ts_rank_cd takes a single tsquery), mirroring the fused + // query's BM25 leg. Reusing the one term→tsquery-arg builder is what makes // predicate and rank impossible to diverge — a prefix-only hit ("invo" // matching "invoice") then ranks by ts_rank_cd instead of getting rank 0 // (which COALESCE(...,0) would collapse to date order). Phrases keep @@ -263,5 +297,16 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f LIMIT $${p} OFFSET $${p + 1} `, params); - return { rows: result.rows, hasCondition: true }; + // Bounded metadata total: a COUNT(*) over the SAME predicate (no LIMIT/OFFSET). + const countResult = await client(` + SELECT COUNT(*) AS total + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND ${conditions.join('\n AND ')} + `, countParams); + const total = Number(countResult.rows[0]?.total ?? 0); // COUNT(*) always returns one row in prod + + return { rows: result.rows, hasCondition: true, ...(total !== undefined ? { total } : {}) }; } diff --git a/backend/src/services/search/lexicalRepo.test.js b/backend/src/services/search/lexicalRepo.test.js index 0144e31e..38562b8e 100644 --- a/backend/src/services/search/lexicalRepo.test.js +++ b/backend/src/services/search/lexicalRepo.test.js @@ -10,6 +10,7 @@ import { LEXICAL_RANK_SQL, freeTextTermConditionRanked, freeTextTermClause, + negatedFreeTextClause, stopwordSafeCondition, buildOperatorClauses, buildFolderScopeClauses, @@ -239,7 +240,7 @@ describe('ranked lexical query (slice 02)', () => { }); describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { - // Root cause: + // Root cause (verified against pgvector/pg16, 2026-07-17): // to_tsquery('english', quote_literal('for') || ':*') normalizes to an // EMPTY tsquery (numnode = 0), and `tsvector @@ ` is FALSE for // every row — so once rows were backfilled onto search_fts, one english @@ -294,7 +295,7 @@ describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { expect(text).toContain("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE("); }); - it('freeTextTermClause owns the complete ranked lexical predicate', () => { + it('freeTextTermClause is the one owner searchLexical and the staging path share', () => { const params = []; let p = 2; const bind = (v) => { params.push(v); return `$${p++}`; }; @@ -305,6 +306,13 @@ describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { expect(clause).toContain('m.search_fts IS NULL AND'); }); + it('negatedFreeTextClause carries the guard outside the NOT (fused NOT-conditions)', () => { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const clause = negatedFreeTextClause('the', bind); + expect(clause.startsWith("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE(")).toBe(true); + }); }); describe('buildOperatorClauses (Phase 4 Task 2a — extracted from searchLexical)', () => { @@ -352,7 +360,7 @@ describe('buildOperatorClauses (Phase 4 Task 2a — extracted from searchLexical }); }); -describe('buildFolderScopeClauses', () => { +describe('buildFolderScopeClauses (folder scope — shared by lexical + fused)', () => { function bindHarness(start = 2) { const params = []; let p = start; diff --git a/backend/src/services/search/lexicalRepo.total.test.js b/backend/src/services/search/lexicalRepo.total.test.js new file mode 100644 index 00000000..16efbe1d --- /dev/null +++ b/backend/src/services/search/lexicalRepo.total.test.js @@ -0,0 +1,22 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../db.js', () => ({ query: vi.fn() })); +import { searchLexical } from './lexicalRepo.js'; + +// NOTE: the real searchLexical takes `client` as a FUNCTION (client(sql, params)), +// not an object with a .query method (the plan's Consumes was written earlier — +// real code wins). We adapt the fake accordingly. +const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; + +describe('searchLexical metadata total', () => { + it('returns a real total via a bounded COUNT over the same predicate (no LIMIT/OFFSET)', async () => { + const client = vi.fn() + .mockResolvedValueOnce({ rows: [{ id: 'm1' }] }) // page query + .mockResolvedValueOnce({ rows: [{ total: '42' }] }); // count query + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], limit: 20, offset: 0 }); + expect(out.total).toBe(42); + const countSql = client.mock.calls[1][0]; + expect(countSql).toMatch(/COUNT\(\*\)/i); + expect(countSql).not.toMatch(/\bLIMIT\b/i); + expect(countSql).not.toMatch(/\bOFFSET\b/i); + }); +}); diff --git a/backend/src/services/search/migrations.test.js b/backend/src/services/search/migrations.test.js index 53f2a8c8..feb0f321 100644 --- a/backend/src/services/search/migrations.test.js +++ b/backend/src/services/search/migrations.test.js @@ -70,3 +70,20 @@ describe('0037_search_fts_index.sql', () => { } }); }); + +describe('0039_embed_pending_index.sql', () => { + const sql = read('0039_embed_pending_index.sql'); + + it('runs outside a transaction and builds the partial index CONCURRENTLY', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(true); + expect(sql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending'); + expect(sql).toContain('WHERE embed_gen IS NULL'); + expect(sql).not.toContain('$$'); // no function bodies — safe for the ; splitter + }); + + it('drops the index CONCURRENTLY before creating it (invalid-index retry hazard)', () => { + expect(sql).toContain('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending'); + expect(sql.indexOf('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending')) + .toBeLessThan(sql.indexOf('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending')); + }); +}); diff --git a/backend/src/services/search/searchService.js b/backend/src/services/search/searchService.js index 49e6114d..337f992e 100644 --- a/backend/src/services/search/searchService.js +++ b/backend/src/services/search/searchService.js @@ -1,27 +1,68 @@ import { query } from '../db.js'; -import { searchLexical } from './lexicalRepo.js'; +import { searchLexical, buildOperatorClauses, hasSearchableToken, buildFolderScopeClauses, negatedFreeTextClause } from './lexicalRepo.js'; import { resolveSearchFolderScope } from './queryParser.js'; +import { hybridSearch, isLexicalFallback, MissingFreeTextError } from '../embeddings/hybrid.js'; function clampLimit(limit) { return Math.max(1, Math.min(parseInt(limit) || 50, 200)); } -export async function search(request) { - const { userId, accountId, parsed, folderParam = '', limit = 50, offset = 0 } = request; +// MCP-facing envelope helpers, applied only when the caller opts in +// (REST never sets `explain`/`scope:'body'`, so its response is unaffected). +function withExplainScores(hits) { + for (const h of hits) { + h.score = { + rrf: h.rrf_score, + ...(h.bm25_score != null ? { bm25: h.bm25_score } : {}), + ...(h.vector_score != null ? { vector: h.vector_score } : {}), + subject_boosted: !!h.subject_boosted, + }; + } +} +function attachBestChunk(hits) { + for (const h of hits) { + h.best_chunk = h.best_char_start == null ? null + : { chunk_index: h.best_chunk_index, char_start: h.best_char_start, char_end: h.best_char_end, score: h.vector_score }; + } +} - const cap = clampLimit(limit); - const off = Math.max(0, parseInt(offset) || 0); - const emptyPage = { offset: off, limit: cap, hasMore: false }; +// The vector/hybrid SQL projects the message id as `message_id` +// (vectorStore.fusedSearch DISPLAY_COLS), but every hit consumer — REST +// serialization and MCP hydration alike — keys on `id`. Additively alias `id` +// onto each vector/hybrid hit so the shape is mode-invariant with lexical, leaving +// `message_id` untouched (the REST response is a superset — additive, never +// renamed). searchService.search() is the one seam both modes flow through, so +// this is the single place the alias is applied. +function aliasIdFromMessageId(hits) { + for (const h of hits) h.id = h.message_id; +} +async function resolveAccountIds(request) { + const { userId, accountId } = request; const accountsResult = await query( 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', [userId] ); let accountIds = accountsResult.rows.map(r => r.id); - if (!accountIds.length) return { messages: [], mode: 'lexical', page: emptyPage }; - + if (!accountIds.length) return []; // Optional single-account narrowing, only within the authenticated user's scope. if (accountId && accountIds.includes(accountId)) accountIds = [accountId]; + return accountIds; +} + +// Phase 1's original lexical search, extracted verbatim so the mode dispatch +// below can reuse it both as the default path and as the fallback target when +// a semantic search degrades. Returns the pre-Phase-4 shape (no `mode` field — +// the caller adds that uniformly). +async function runLexical(request, resolvedAccountIds) { + const { parsed, folderParam = '', limit = 50, offset = 0 } = request; + + const cap = clampLimit(limit); + const off = Math.max(0, parseInt(offset) || 0); + const emptyPage = { offset: off, limit: cap, hasMore: false }; + + const accountIds = resolvedAccountIds || await resolveAccountIds(request); + if (!accountIds.length) return { messages: [], page: emptyPage }; const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, folderParam); @@ -30,14 +71,87 @@ export async function search(request) { const hasPositiveText = parsed.terms.some(t => !t.negate && t.value.length >= 2); const ordering = hasPositiveText ? 'relevance' : 'date'; - const { rows, hasCondition } = await searchLexical(query, { + const { rows, total, hasCondition } = await searchLexical(query, { parsed, accountIds, folderScope, folderFuzzy, ordering, limit: cap, offset: off, }); - if (!hasCondition) return { messages: [], mode: 'lexical', page: emptyPage }; + if (!hasCondition) return { messages: [], page: emptyPage }; return { messages: rows, - mode: 'lexical', + ...(total !== undefined ? { total } : {}), page: { offset: off, limit: cap, hasMore: rows.length === cap }, }; } + +// The single search entry point shared by REST and MCP. Owns account scoping, +// mode dispatch (lexical/vector/hybrid), the D5 ordering decision (lexical), +// and result shaping. No HTTP framing here. +export async function search(request) { + const mode = request.mode === 'vector' || request.mode === 'hybrid' ? request.mode : 'lexical'; + if (mode === 'lexical') { + return { ...(await runLexical(request)), mode: 'lexical' }; + } + + const limit = clampLimit(request.limit); + const offset = Math.max(0, parseInt(request.offset) || 0); + // Resolve once, up front, and thread it into a lexical fallback below — a + // fallback must not re-resolve accounts via a second DB round-trip. + const accountIds = await resolveAccountIds(request); + try { + if (accountIds.length === 0) { + return { messages: [], mode, pool_saturated: false, generation: null, + page: { offset, limit, hasMore: false } }; + } + const { parsed } = request; // already parsed by the caller — never re-parse + const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, request.folderParam || ''); + // One buildFilters owner threads the SAME predicates the lexical path applies + // into BOTH fused legs (fusedSearch applies it to the FTS pool and the ANN + // EXISTS alike, on the joined messages table): structured operators, the + // folder scope (so semantic search can't leak Sent/Archive/Trash into an + // Inbox search and an explicit in:sent applies), and negated free-text terms + // as NOT-conditions (so `invoice -draft` excludes drafts in semantic mode + // just as FTS exclusion does in lexical mode). Same term hygiene throughout: + // drop sub-2-char / punctuation-only tokens. + const buildFilters = (bind) => [ + ...buildOperatorClauses(parsed.filters, bind), + ...buildFolderScopeClauses(folderScope, folderFuzzy, bind), + ...parsed.terms + .filter(t => t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => negatedFreeTextClause(t.value, bind)), + ]; + // Only non-negated terms drive the embedding + BM25 leg; negated terms are + // enforced via buildFilters above, never fed to the embedder. + const freeText = parsed.terms + .filter(t => !t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => t.value).join(' '); + + // Ranked pools are bounded (D3); fetch one past the page window so a full + // page can observe whether more hits exist, then slice the page. + const window = offset + limit; + const { hits, poolSaturated, generation } = await hybridSearch({ + mode, freeText, accountIds, buildFilters, limit: window + 1, + }); + const page = hits.slice(offset, offset + limit); + aliasIdFromMessageId(page); // mode-invariant `id` alias (seam contract) + if (request.explain) withExplainScores(page); + if (request.scope === 'body') attachBestChunk(page); + return { + messages: page, + mode, + pool_saturated: poolSaturated, + generation: generation || null, // rich {id,model,dimension,fingerprint,state} object + page: { offset, limit, hasMore: hits.length > window }, + }; + } catch (err) { + if (isLexicalFallback(err)) { + const lexical = { ...(await runLexical(request, accountIds)), mode: 'lexical' }; + // A filter-only query in semantic mode (MissingFreeTextError) is not a + // degradation — there is nothing to embed and the lexical result IS the + // answer, so no fellBack (the UI keys its amber "index building" hint on + // it). Real unavailability (unconfigured/building/stale/ + // embedding_timeout) keeps the flag. + return err instanceof MissingFreeTextError ? lexical : { ...lexical, fellBack: true }; + } + throw err; + } +} diff --git a/backend/src/services/search/searchService.test.js b/backend/src/services/search/searchService.test.js index 4735be64..9ada7260 100644 --- a/backend/src/services/search/searchService.test.js +++ b/backend/src/services/search/searchService.test.js @@ -1,18 +1,28 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; +// Mock inline and import the mocked bindings — referencing an outer `const fn` +// from a vi.mock factory hits Vitest's hoisting temporal-dead-zone error. vi.mock('../db.js', () => ({ query: vi.fn() })); vi.mock('./lexicalRepo.js', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, searchLexical: vi.fn() }; }); +vi.mock('../embeddings/hybrid.js', () => ({ + hybridSearch: vi.fn(), + isLexicalFallback: vi.fn(() => true), + MissingFreeTextError: class extends Error {}, +})); import { query } from '../db.js'; import { searchLexical } from './lexicalRepo.js'; +import * as hybrid from '../embeddings/hybrid.js'; import { search } from './searchService.js'; beforeEach(() => { - query.mockReset(); - searchLexical.mockReset(); + query.mockReset(); searchLexical.mockReset(); + hybrid.hybridSearch.mockReset(); + hybrid.isLexicalFallback.mockReset(); + hybrid.isLexicalFallback.mockReturnValue(true); }); function withAccounts(ids) { @@ -34,6 +44,13 @@ describe('searchService.search', () => { expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a2']); }); + it('passes through an optional total when searchLexical returns one', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'm' }], total: 137, hasCondition: true }); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(res.total).toBe(137); + }); + it('chooses relevance ordering for free text and date ordering for filter-only queries', async () => { withAccounts(['a1']); searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); @@ -65,3 +82,247 @@ describe('searchService.search', () => { expect(res.page.hasMore).toBe(false); }); }); + +describe('searchService mode dispatch (Phase 4 Task 5)', () => { + const hybridReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'quarterly', negate: false }, { value: 'revenue', negate: false }] }, + ...extra, + }); + + it('defaults to lexical and marks the mode', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'hello', negate: false }] } }); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBeUndefined(); + expect(hybrid.hybridSearch).not.toHaveBeenCalled(); + }); + + it('hybrid success returns score-ordered messages with pool_saturated + the rich generation object, no total', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, subject: 's' }], + poolSaturated: true, generation: { id: 7 }, + }); + const res = await search(hybridReq()); + expect(res.mode).toBe('hybrid'); + expect(res.pool_saturated).toBe(true); + // Task 5b supersedes Task 5's bare generation.id with the rich object + // fusedSearch/hybridSearch echoes back. + expect(res.generation).toEqual({ id: 7 }); + expect(res.total).toBeUndefined(); + expect(res.messages[0].message_id).toBe('a'); + }); + + it('additively aliases id onto message_id for BOTH vector and hybrid hits (mode-invariant seam contract), leaving message_id intact', async () => { + // The vector/hybrid SQL (fusedSearch DISPLAY_COLS) projects the message id + // as `message_id`, NOT `id` — the real seam shape the live MCP round-trip + // exposed. The lexical path and every hit consumer (REST serialization, MCP + // hydration via getMessageSummariesByIDs) key on `id`, so the seam must add + // it for both modes. Regression guard for `semantic_search_messages` + // returning 0. + for (const mode of ['vector', 'hybrid']) { + query.mockReset(); hybrid.hybridSearch.mockReset(); + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'real-uuid', uid: 7, folder: 'INBOX', subject: 'Run failed: CD deploy', rrf_score: 0.04 }], + poolSaturated: false, generation: { id: 1 }, + }); + const res = await search({ + userId: 'u1', mode, limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'deploy', negate: false }] }, + }); + expect(res.messages[0].message_id).toBe('real-uuid'); // unchanged — REST reads it + expect(res.messages[0].id).toBe('real-uuid'); // additive alias — MCP hydration + lexical parity + } + }); + + it('drops sub-2-char and punctuation-only terms from the semantic freeText, matching the lexical path\'s hygiene (review MINOR 3)', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + await search(hybridReq({ + parsed: { + filters: [], + terms: [ + { value: 'quarterly', negate: false }, + { value: 'a', negate: false }, // 1-char — dropped + { value: '!!!', negate: false }, // punctuation-only — dropped + { value: 'revenue', negate: false }, + ], + }, + })); + expect(hybrid.hybridSearch.mock.calls[0][0].freeText).toBe('quarterly revenue'); + }); + + it('falls back to lexical (fellBack:true) when hybrid degrades', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'lex1', subject: 'kw' }], hasCondition: true }); + hybrid.hybridSearch.mockRejectedValue(Object.assign(new Error('building'), { code: 'INDEX_BUILDING' })); + const res = await search(hybridReq()); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBe(true); + expect(res.messages[0].id).toBe('lex1'); + }); + + it('serves a filter-only semantic query lexically WITHOUT fellBack — not an "index building" event (Wave D Fix 6)', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'lex1' }], hasCondition: true }); + hybrid.hybridSearch.mockRejectedValue(new hybrid.MissingFreeTextError()); + const res = await search(hybridReq({ parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] } })); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBeUndefined(); // the UI keys an amber degradation hint on fellBack + expect(res.messages[0].id).toBe('lex1'); + }); + + it('propagates an unexpected (non-degradation) error instead of falling back', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockRejectedValue(new Error('boom')); + hybrid.isLexicalFallback.mockReturnValue(false); + await expect(search(hybridReq())).rejects.toThrow('boom'); + }); +}); + +describe('searchService fused folder scope + negation (Fix 2 + Fix 4)', () => { + const semReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + ...extra, + }); + // Run the buildFilters closure the semantic branch hands hybridSearch, so we + // can inspect the SQL predicates + params it applies to BOTH fused legs. + function runBuildFilters() { + const buildFilters = hybrid.hybridSearch.mock.calls[0][0].buildFilters; + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { clauses: buildFilters(bind), params }; + } + + beforeEach(() => { + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + }); + + it('scopes the fused query to an in: for BOTH vector and hybrid (never leaks other folders)', async () => { + for (const mode of ['vector', 'hybrid']) { + query.mockReset(); hybrid.hybridSearch.mockReset(); + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + withAccounts(['a1']); + await search(semReq({ mode, parsed: { filters: [{ key: 'in', value: 'sent', negate: false }], terms: [{ value: 'invoice', negate: false }] } })); + const { clauses, params } = runBuildFilters(); + expect(clauses.some((c) => /m\.folder ILIKE .+ OR m\.folder ILIKE/.test(c))).toBe(true); + expect(params).toContain('sent'); + expect(params).toContain('%/sent'); + } + }); + + it('honors an exact REST folderParam on the fused query', async () => { + withAccounts(['a1']); + await search(semReq({ folderParam: 'INBOX' })); + const { clauses, params } = runBuildFilters(); + expect(clauses.some((c) => c === 'm.folder = $2')).toBe(true); + expect(params).toContain('INBOX'); + }); + + it('defaults the fused query to the trash-excluding scope when no folder is specified', async () => { + withAccounts(['a1']); + await search(semReq()); + const { clauses } = runBuildFilters(); + expect(clauses.some((c) => /NOT EXISTS/.test(c) && /%trash%/.test(c))).toBe(true); + }); + + it('enforces a negated free-text term as a NOT-condition on the fused query (invoice -draft excludes drafts)', async () => { + withAccounts(['a1']); + await search(semReq({ parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'draft', negate: true }] } })); + const { clauses, params } = runBuildFilters(); + // Positive term drives freeText (embedded + BM25 leg); the negated term becomes + // a stopword-guarded NOT COALESCE(...) filter using the SAME prefix-aware + // FTS builder as lexical (guard OUTSIDE the NOT, so a negated stopword + // contributes nothing instead of excluding everything — Fix 1). + expect(hybrid.hybridSearch.mock.calls[0][0].freeText).toBe('invoice'); + const notClause = clauses.find((c) => /OR NOT COALESCE/.test(c)); + expect(notClause).toBeDefined(); + expect(notClause).toMatch(/^\(numnode\(to_tsquery\('english', quote_literal\(\$\d+\) \|\| ':\*'\)\) = 0 OR NOT COALESCE/); + expect(notClause).toContain("m.search_fts @@ to_tsquery('english', quote_literal($"); + expect(params).toContain('draft'); + }); + + it('drops a sub-2-char / punctuation-only negated term from the fused NOT-conditions (lexical hygiene parity)', async () => { + withAccounts(['a1']); + await search(semReq({ parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'x', negate: true }, { value: '!!!', negate: true }] } })); + const { clauses } = runBuildFilters(); + expect(clauses.some((c) => /NOT COALESCE/.test(c))).toBe(false); + }); +}); + +describe('searchService semantic pagination hasMore (Fix 3)', () => { + const semReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 2, offset: 0, + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + ...extra, + }); + + it('fetches window+1 and flags hasMore when a sentinel hit is present', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 3 }, { message_id: 'b', rrf_score: 2 }, { message_id: 'c', rrf_score: 1 }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq()); + expect(hybrid.hybridSearch.mock.calls[0][0].limit).toBe(3); // window(2) + 1 + expect(res.messages).toHaveLength(2); // page sliced to limit + expect(res.page.hasMore).toBe(true); + }); + + it('flags hasMore false when exactly the window is returned', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 2 }, { message_id: 'b', rrf_score: 1 }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq()); + expect(res.messages).toHaveLength(2); + expect(res.page.hasMore).toBe(false); + }); + + it('respects offset: window+1 = offset+limit+1 and slices the page', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a' }, { message_id: 'b' }, { message_id: 'c' }, { message_id: 'd' }, { message_id: 'e' }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq({ offset: 2, limit: 2 })); + expect(hybrid.hybridSearch.mock.calls[0][0].limit).toBe(5); // offset(2)+limit(2)+1 + expect(res.messages.map((m) => m.message_id)).toEqual(['c', 'd']); + expect(res.page.hasMore).toBe(true); // 5 hits > window(4) + }); +}); + +describe('searchService envelope (Task 5b)', () => { + const hybridReq2 = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'q', negate: false }] }, + ...extra, + }); + + it('returns the rich generation object and per-hit score under explain', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, bm25_score: 0.01, vector_score: 0.9, subject_boosted: true, best_char_start: null }], + poolSaturated: false, generation: { id: 7, model: 'm', dimension: 4, fingerprint: 'm:4:x', state: 'active' }, + }); + const res = await search(hybridReq2({ explain: true })); + expect(res.generation).toEqual({ id: 7, model: 'm', dimension: 4, fingerprint: 'm:4:x', state: 'active' }); + expect(res.messages[0].score).toEqual({ rrf: 0.04, bm25: 0.01, vector: 0.9, subject_boosted: true }); + }); + + it('attaches best_chunk metadata under scope=body', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, vector_score: 0.9, best_chunk_index: 2, best_char_start: 6, best_char_end: 40 }], + poolSaturated: false, generation: { id: 1, model: 'm', dimension: 4, fingerprint: 'f', state: 'active' }, + }); + const res = await search(hybridReq2({ scope: 'body' })); + expect(res.messages[0].best_chunk).toEqual({ chunk_index: 2, char_start: 6, char_end: 40, score: 0.9 }); + }); +}); diff --git a/backend/src/services/search/searchService.total.test.js b/backend/src/services/search/searchService.total.test.js new file mode 100644 index 00000000..09fdd1dd --- /dev/null +++ b/backend/src/services/search/searchService.total.test.js @@ -0,0 +1,15 @@ +import { it, expect, vi } from 'vitest'; +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('./lexicalRepo.js', () => ({ searchLexical: vi.fn(), buildOperatorClauses: vi.fn(), hasSearchableToken: vi.fn() })); +import { query } from '../db.js'; +import { searchLexical } from './lexicalRepo.js'; +import { search } from './searchService.js'; + +it('passes the metadata total through to the service result', async () => { + query.mockResolvedValue({ rows: [{ id: 'acc-1' }] }); + searchLexical.mockResolvedValue({ rows: [{ id: 'm1' }], total: 42, hasCondition: true }); + const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; + const r = await search({ mode: 'lexical', userId: 'user-1', parsed, limit: 20, offset: 0 }); + expect(r.total).toBe(42); + expect(searchLexical.mock.calls[0][1]).toMatchObject({ accountIds: ['acc-1'] }); +}); diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 3c51af41..aaeb02e7 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -71,7 +71,7 @@ services: # ── PostgreSQL ─────────────────────────────────────────────────────────────── postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 container_name: mailflow-postgres restart: unless-stopped environment: diff --git a/docker-compose.yml b/docker-compose.yml index 160326f5..23d768e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: # ── PostgreSQL ─────────────────────────────────────────────────────────────── postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 container_name: mailflow-postgres restart: unless-stopped environment: diff --git a/frontend/src/components/AdminPanel.jsx b/frontend/src/components/AdminPanel.jsx index a3141dd3..dd534468 100644 --- a/frontend/src/components/AdminPanel.jsx +++ b/frontend/src/components/AdminPanel.jsx @@ -13,6 +13,7 @@ import SignatureEditor from './SignatureEditor.jsx'; import GtdZeroPet from './GtdZeroPet.jsx'; import { getEffectiveShortcuts, getGroupedActions, ACTION_DEFS, SPECIAL_KEY_LABELS, parseModKey, modLabel } from '../utils/defaultShortcuts.js'; import { DEFAULT_GTD_FOLDERS, GTD_STATES, resolveAccountGtdFolders, diffGtdFolders, findGtdFolderCollisions } from '../utils/gtd.js'; +import { emptyEmbeddingsForm, embeddingsFormFromConfig, buildEmbeddingsPayload, embeddingsDirty, isSameAsChatProvider, reconcileDimension, embeddingsJob, canSaveAiConfig, EMBEDDING_MODEL_HINTS } from '../utils/embeddingsSettings.js'; // ─── Shared field component ─────────────────────────────────────────────────── function Field({ label, required, children }) { @@ -3350,28 +3351,54 @@ function AISection() { const [config, setConfig] = useState(null); const [loading, setLoading] = useState(true); const [form, setForm] = useState({ enabled: true, baseUrl: '', apiKey: '', model: '', features: { compose: true, summarize: true } }); + const [emb, setEmb] = useState(emptyEmbeddingsForm); const [saving, setSaving] = useState(false); const [testing, setTesting] = useState(false); const [msg, setMsg] = useState(null); + // Embeddings block: a background_jobs 'embeddings' row (or null), whether the + // pgvector extension is present, and the two async action flags. + const [job, setJob] = useState(null); + const [vectorAvailable, setVectorAvailable] = useState(true); + const [testingEmb, setTestingEmb] = useState(false); + const [building, setBuilding] = useState(false); + + const loadConfig = async () => { + const { config: cfg } = await api.ai.getConfig(); + setConfig(cfg || null); + if (cfg) { + setForm({ enabled: cfg.enabled !== false, baseUrl: cfg.baseUrl || '', apiKey: cfg.apiKey || '', model: cfg.model || '', features: { compose: cfg.features?.compose !== false, summarize: cfg.features?.summarize !== false } }); + setEmb(embeddingsFormFromConfig(cfg)); + } else { + setEmb(emptyEmbeddingsForm()); + } + }; + + const refreshJob = async () => { + try { + const { jobs } = await api.ai.indexingStatus(); + setJob(embeddingsJob(jobs)); + } catch { /* status is best-effort; leave the last known job in place */ } + }; useEffect(() => { - api.ai.getConfig() - .then(({ config: cfg }) => { - if (cfg) { - setConfig(cfg); - setForm({ enabled: cfg.enabled !== false, baseUrl: cfg.baseUrl || '', apiKey: cfg.apiKey || '', model: cfg.model || '', features: { compose: cfg.features?.compose !== false, summarize: cfg.features?.summarize !== false } }); - } - }) - .catch(console.error) - .finally(() => setLoading(false)); + loadConfig().catch(console.error).finally(() => setLoading(false)); + refreshJob(); + api.ai.status().then(s => setVectorAvailable(s?.vectorAvailable !== false)).catch(() => {}); }, []); + // Poll the indexing status while a build is running so progress ticks live. + useEffect(() => { + if (!job?.active) return; + const id = setInterval(refreshJob, 2000); + return () => clearInterval(id); + }, [job?.active]); + const handleSave = async (e) => { e.preventDefault(); setSaving(true); setMsg(null); try { - await api.ai.saveConfig(form); - setConfig({ ...form }); + await api.ai.saveConfig({ ...form, embeddings: buildEmbeddingsPayload(emb) }); + await loadConfig(); // re-mask keys + reset the dirty baseline setMsg({ type: 'ok', text: t('admin.ai.saved') }); } catch (err) { setMsg({ type: 'error', text: err.message }); @@ -3392,9 +3419,35 @@ function AISection() { await api.ai.deleteConfig(); setConfig(null); setForm({ enabled: true, baseUrl: '', apiKey: '', model: '', features: { compose: true, summarize: true } }); + setEmb(emptyEmbeddingsForm()); setMsg({ type: 'ok', text: t('admin.ai.removed') }); }; + const handleTestEmbeddings = async () => { + setTestingEmb(true); setMsg(null); + try { + const { dimension } = await api.ai.testEmbeddings(); + const { dimension: next, changed } = reconcileDimension(emb.dimension, dimension); + if (changed) setEmb(e => ({ ...e, dimension: String(next) })); + setMsg({ type: 'ok', text: t('admin.ai.emb.testOk', { dimension }) }); + } catch (err) { + setMsg({ type: 'error', text: `${t('admin.ai.testFail')}: ${err.message}` }); + } finally { setTestingEmb(false); } + }; + + const handleBuild = async () => { + setBuilding(true); setMsg(null); + try { + await api.ai.buildEmbeddings(); + setMsg({ type: 'ok', text: t('admin.ai.emb.buildStarted') }); + await refreshJob(); + } catch (err) { + // 409 = a build is already running; still surface progress rather than a dead error. + setMsg({ type: 'error', text: err.message }); + await refreshJob(); + } finally { setBuilding(false); } + }; + const field = (label, key, type = 'text', placeholder = '') => (
@@ -3409,6 +3462,22 @@ function AISection() {
); + // Embeddings-form counterpart of `field` — reads/writes the `emb` state. + const embField = (label, key, type = 'text', placeholder = '', hint = '') => ( +
+ + setEmb(f => ({ ...f, [key]: e.target.value }))} + placeholder={placeholder} + autoComplete={type === 'password' ? 'new-password' : 'off'} + style={{ width: '100%', background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 6, padding: '7px 10px', color: 'var(--text-primary)', fontSize: 13 }} + /> + {hint &&
{hint}
} +
+ ); + const toggle = (label, checked, onChange) => (
+ {/* ── Embeddings (semantic search) ─────────────────────────────────── */} +
+
{t('admin.ai.emb.title')}
+

{t('admin.ai.emb.benefit')}

+

{t('admin.ai.emb.defaultOff')}

+ + {toggle(t('admin.ai.emb.enable'), emb.enabled, () => setEmb(f => ({ ...f, enabled: !f.enabled })))} + + {emb.enabled && ( +
+
+ {t('admin.ai.emb.privacyWarning')} +
+ + {!vectorAvailable && ( +
+ {t('admin.ai.emb.vectorUnavailable')} +
+ )} + +
+ + {form.baseUrl && (isSameAsChatProvider(emb.endpoint, form.baseUrl) + ? {t('admin.ai.emb.sameAsChatActive')} + : )} +
+ setEmb(f => ({ ...f, endpoint: e.target.value }))} + placeholder={t('admin.ai.emb.endpointPh')} + autoComplete="off" + style={{ width: '100%', background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 6, padding: '7px 10px', color: 'var(--text-primary)', fontSize: 13 }} + /> +
{t('admin.ai.emb.endpointHint')}
+ + {embField(t('admin.ai.emb.model'), 'model', 'text', t('admin.ai.emb.modelPh'), modelHint)} + {embField(t('admin.ai.emb.dimension'), 'dimension', 'number', t('admin.ai.emb.dimensionPh'), t('admin.ai.emb.dimensionHint'))} + {embField(t('admin.ai.emb.apiKey'), 'apiKey', 'password', t('admin.ai.apiKeyPh'))} + +
+ {t('admin.ai.emb.localTitle')} +

{t('admin.ai.emb.localHelp')}

+
+ +
+ + +
+ + {embDirty && ( +
{t('admin.ai.emb.saveHint')}
+ )} + + {job && ( +
+ {job.state === 'running' && ( + <> +
+ {t('admin.ai.emb.progressLabel', { processed: job.processed, total: job.total })} +
+
+
+
+ + )} + {job.state === 'done' && ( +
{t('admin.ai.emb.progressDone', { total: job.total })}
+ )} + {job.state === 'error' && ( +
{t('admin.ai.emb.progressError', { error: job.lastError || '' })}
+ )} +
+ )} +
+ )} + +
+ {msgBox} - diff --git a/frontend/src/components/MessageList.jsx b/frontend/src/components/MessageList.jsx index f4665182..714cdb05 100644 --- a/frontend/src/components/MessageList.jsx +++ b/frontend/src/components/MessageList.jsx @@ -19,6 +19,18 @@ import { shortcutBus } from '../utils/shortcutBus.js'; import { createLatestRequest } from '../utils/latestRequest.js'; import { pendingMarkReadMap, completedMarkReadMap, setPending } from '../utils/pendingReads.js'; import { applyDeleteGuard, clearDeleteGuard, clearPendingDelete, setCompletedDelete, setPendingDelete } from '../utils/pendingDeletes.js'; +import { semanticSearchAvailable, semanticToggleState, searchInputRightPad, isCurrentSearchGeneration, LEXICAL_MODE, SEMANTIC_MODE } from '../utils/searchMode.js'; + +// Sparkle-toggle tone → resting/hover glyph colour + background chip. Kept next +// to the presentation (searchMode.js stays framework-free); the tone strings +// come from semanticToggleState(). The chip is what sets ON (faint purple) and +// the amber fallback apart from the greyed OFF state beyond glyph colour alone, +// and the hover variants give the icon a clickable affordance. +const SEMANTIC_TONE = { + off: { color: 'var(--text-secondary)', hoverColor: 'var(--text-primary)', chip: 'transparent', hoverChip: 'var(--bg-hover)' }, + on: { color: 'var(--accent)', hoverColor: 'var(--accent)', chip: 'var(--accent-glow)', hoverChip: 'rgba(124, 106, 247, 0.28)' }, + fallback: { color: 'var(--amber)', hoverColor: 'var(--amber)', chip: 'rgba(251, 191, 36, 0.15)', hoverChip: 'rgba(251, 191, 36, 0.28)' }, +}; // Folder icon for move picker function FolderIcon({ specialUse, size = 13 }) { @@ -116,6 +128,7 @@ export default function MessageList() { categorizationEnabled, categoryCounts, setCategoryCounts, adjustCategoryCount, markReadBehavior, markReadDelay, searchAllFolders, + searchMode, setSearchMode, activeGtdTab, setActiveGtdTab, gtdSections, } = useStore(); // RFC message_id of the open message, so a row highlights when it is a different DB copy @@ -173,6 +186,73 @@ export default function MessageList() { const [searchHasMore, setSearchHasMore] = useState(false); const [searchLoadingMore, setSearchLoadingMore] = useState(false); const searchFetchedOffsetRef = useRef(0); + const [semanticAvailable, setSemanticAvailable] = useState(false); + const [searchFellBack, setSearchFellBack] = useState(false); + // The active mode is searchMode only when the vector-availability probe says + // semantic search is usable; otherwise force lexical regardless of the + // persisted preference (e.g. this Postgres has no pgvector schema). + const effectiveMode = semanticAvailable ? searchMode : LEXICAL_MODE; + + // Right-side control cluster that lives INSIDE the search input, shared by the + // desktop and mobile search boxes. Holds the semantic-search toggle (sparkle + // icon, gated on vector availability) and the clear-query button, in that + // order. The toggle is greyed when off, purple when on, and amber when the + // backend silently fell back to lexical — the fallback hint rides the icon's + // colour + tooltip so no extra row appears below the input. + const renderSearchControls = () => { + const on = searchMode !== LEXICAL_MODE; + const toggle = semanticToggleState({ on, fellBack: searchFellBack, hasQuery: Boolean(searchQuery.trim()) }); + const toggleLabel = t(toggle.titleKey); + const tone = SEMANTIC_TONE[toggle.tone]; + // Shared icon-button shape so the sparkle and clear × read as one cluster + // and their hover chips are the same size. + const iconBtn = { + display: 'flex', alignItems: 'center', justifyContent: 'center', + padding: 4, borderRadius: 6, border: 'none', cursor: 'pointer', + WebkitTapHighlightColor: 'transparent', + transition: 'background 0.15s, color 0.15s', + }; + return ( +
+ {semanticAvailable && ( + + )} + {searchQuery && ( + + )} +
+ ); + }; const listRef = useRef(null); const searchInputRef = useRef(null); // for focusSearch shortcut const pendingDeleteTimers = useRef(new Map()); // id/thread key -> pending delete metadata @@ -204,6 +284,16 @@ export default function MessageList() { window.addEventListener('mailflow:message-opening', markOpening); return () => window.removeEventListener('mailflow:message-opening', markOpening); }, []); + + // Probe vector availability once on mount — gates whether the semantic + // toggle renders at all (Task 10). + useEffect(() => { + let alive = true; + api.ai.status() + .then(s => { if (alive) setSemanticAvailable(semanticSearchAvailable(s)); }) + .catch(() => { if (alive) setSemanticAvailable(false); }); + return () => { alive = false; }; + }, []); const searchTimer = useRef(null); // Category tab scroll arrows @@ -452,23 +542,30 @@ export default function MessageList() { // Search useEffect(() => { clearTimeout(searchTimer.current); + // Bump the request generation on EVERY context change (query, mode, folder, + // account, page size — the dep set below — and clearing the query). Async + // appends (load-more, post-delete prefetch) capture this before their await + // and discard themselves if it moves on, so a stale page can't append onto a + // fresh, different-context list. + const seq = ++searchSeq.current; if (!searchQuery.trim()) { setIsSearching(false); setSearchResults([]); setSearchHasMore(false); + setSearchFellBack(false); searchFetchedOffsetRef.current = 0; return; } setIsSearching(true); setSearchHasMore(false); - const seq = ++searchSeq.current; searchTimer.current = setTimeout(async () => { try { - const data = await api.search(searchQuery, selectedAccountId || undefined, { offset: 0, limit: searchPageSize, folder: searchFolder }); + const data = await api.search(searchQuery, selectedAccountId || undefined, { offset: 0, limit: searchPageSize, folder: searchFolder, mode: effectiveMode }); if (searchSeq.current !== seq) return; searchFetchedOffsetRef.current = data.messages.length; setSearchResults(applyReadGuard(data.messages)); setSearchHasMore(data.messages.length === searchPageSize); + setSearchFellBack(Boolean(data.fellBack)); } catch (err) { if (searchSeq.current === seq) console.error('Search failed:', err); } finally { @@ -476,7 +573,7 @@ export default function MessageList() { } }, 300); return () => clearTimeout(searchTimer.current); - }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchReloadToken, applyReadGuard, setIsSearching, setSearchResults]); + }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchReloadToken, effectiveMode, applyReadGuard, setIsSearching, setSearchResults]); // Re-run an active search (and refresh the folder view) after inbox rules run, since // rules can move messages out of the searched folder and a search snapshot would @@ -497,12 +594,14 @@ export default function MessageList() { const loadMoreSearch = useCallback(async () => { if (searchLoadingMore) return; const qSnapshot = searchQuery; // capture before async gap + const seq = searchSeq.current; // request generation at dispatch time setSearchLoadingMore(true); try { const offset = searchFetchedOffsetRef.current; - const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder }); - // Discard results if the query changed while we were fetching - if (useStore.getState().searchQuery !== qSnapshot) return; + const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder, mode: effectiveMode }); + // Discard if ANY search context (query, mode, folder, account) changed + // while we were fetching — otherwise a stale page appends onto a fresh list. + if (!isCurrentSearchGeneration(seq, searchSeq.current)) return; searchFetchedOffsetRef.current = offset + data.messages.length; const current = useStore.getState().searchResults; useStore.setState({ searchResults: [...current, ...applyReadGuard(data.messages)] }); @@ -512,14 +611,16 @@ export default function MessageList() { } finally { setSearchLoadingMore(false); } - }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchLoadingMore, applyReadGuard]); + }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchLoadingMore, effectiveMode, applyReadGuard]); const prefetchSearchAfterRemoval = useCallback(async (offset) => { const qSnapshot = useStore.getState().searchQuery; if (!qSnapshot.trim()) return; + const seq = searchSeq.current; // request generation at dispatch time try { - const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder }); - if (useStore.getState().searchQuery !== qSnapshot) return; + const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder, mode: effectiveMode }); + // Discard if the search context changed during the fetch (see loadMoreSearch). + if (!isCurrentSearchGeneration(seq, searchSeq.current)) return; searchFetchedOffsetRef.current = Math.max(searchFetchedOffsetRef.current, offset + data.messages.length); const additions = applyReadGuard(data.messages); if (!additions.length) { @@ -535,7 +636,7 @@ export default function MessageList() { } catch (err) { console.error('Search prefetch after delete failed:', err); } - }, [selectedAccountId, searchFolder, searchPageSize, applyReadGuard]); + }, [selectedAccountId, searchFolder, searchPageSize, effectiveMode, applyReadGuard]); // Infinite scroll + scroll-to-top visibility const handleScroll = useCallback(() => { @@ -2672,7 +2773,7 @@ export default function MessageList() { value={searchQuery} onChange={e => setSearchQuery(e.target.value)} style={{ - width: '100%', padding: '8px 10px 8px 32px', + width: '100%', padding: `8px ${searchInputRightPad(semanticAvailable)}px 8px 32px`, background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 8, color: 'var(--text-primary)', fontSize: 13, outline: 'none', boxSizing: 'border-box', @@ -2680,20 +2781,7 @@ export default function MessageList() { onFocus={e => { e.target.style.borderColor = 'var(--accent)'; setSearchFocused(true); }} onBlur={e => { e.target.style.borderColor = 'var(--border)'; setSearchFocused(false); }} /> - {searchQuery && ( - - )} + {renderSearchControls()} {/* Operator hints — shown when focused with an empty query */} {searchFocused && !searchQuery && ( @@ -2760,7 +2848,7 @@ export default function MessageList() { value={searchQuery} onChange={e => setSearchQuery(e.target.value)} style={{ - width: '100%', padding: '8px 10px 8px 32px', + width: '100%', padding: `8px ${searchInputRightPad(semanticAvailable)}px 8px 32px`, background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 8, color: 'var(--text-primary)', fontSize: 13, outline: 'none', boxSizing: 'border-box', @@ -2768,20 +2856,7 @@ export default function MessageList() { onFocus={e => e.target.style.borderColor = 'var(--accent)'} onBlur={e => e.target.style.borderColor = 'var(--border)'} /> - {searchQuery && ( - - )} + {renderSearchControls()}
)} diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.json index d2b85390..0554d466 100644 --- a/frontend/src/locales/de.json +++ b/frontend/src/locales/de.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "Suchen…", + "semanticToggle": "Semantische Suche", + "semanticBuildingHint": "Semantischer Index wird erstellt — Stichwortergebnisse werden angezeigt", "unread": "Ungelesen", "showAll": "Alle anzeigen", "unreadOnly": "Nur ungelesene", @@ -993,7 +995,38 @@ "test": "Verbindung testen", "testing": "Teste…", "notConfigured": "Noch kein KI-Anbieter konfiguriert.", - "remove": "Konfiguration entfernen" + "remove": "Konfiguration entfernen", + "emb": { + "title": "Embeddings (semantische Suche)", + "benefit": "Embeddings machen die Suche schneller und genauer.", + "defaultOff": "Standardmäßig deaktiviert. Es wird nichts gesendet, bis du Embeddings aktivierst und einen Aufbau startest.", + "enable": "Embeddings aktivieren", + "privacyWarning": "Mit einem gehosteten Endpunkt erhält der Embeddings-Anbieter (z. B. OpenAI) deine E-Mail-Inhalte — Betreff und Textkörper. Ab diesem Punkt ist dieser Datenpfad nicht mehr selbst gehostet.", + "vectorUnavailable": "Die pgvector-Erweiterung ist auf dieser Datenbank nicht verfügbar, daher kann die semantische Suche nicht laufen. Die Stichwortsuche funktioniert weiterhin.", + "endpoint": "Endpunkt-URL", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "Der Client hängt /embeddings an diese URL an.", + "sameAsChat": "Wie beim Chat-Anbieter", + "sameAsChatActive": "Endpunkt des Chat-Anbieters wird verwendet", + "model": "Modell", + "modelPh": "text-embedding-3-small", + "dimension": "Dimension", + "dimensionPh": "1536", + "dimensionHint": "Vektorgröße, die das Modell zurückgibt. Führe „Test“ aus, um sie über den Endpunkt zu bestätigen.", + "apiKey": "Embeddings-API-Schlüssel", + "test": "Endpunkt testen", + "testOk": "Endpunkt erreichbar — Dimension {{dimension}}.", + "build": "Index aufbauen", + "rebuild": "Index neu aufbauen", + "building": "Wird aufgebaut…", + "buildStarted": "Aufbau gestartet — die Indexierung läuft im Hintergrund.", + "saveHint": "Speichere deine Änderungen, bevor du testest oder aufbaust.", + "progressLabel": "Indexierung {{processed}} / {{total}} Nachrichten", + "progressDone": "Embeddings auf dem neuesten Stand ({{total}} Nachrichten indexiert).", + "progressError": "Aufbaufehler: {{error}}", + "localTitle": "Embeddings lokal betreiben", + "localHelp": "Mailflow kann Embeddings vollständig auf deiner eigenen Hardware erzeugen, sodass kein Nachrichteninhalt dein Netzwerk verlässt. Auf einem Raspberry Pi betreibst du Ollama mit einem kleinen Modell wie all-minilm (384) oder nomic-embed-text (768) — beide stellen einen OpenAI-kompatiblen Endpunkt bereit, mit dem Mailflow direkt spricht. Pi-CPUs sind langsam: Der erste Indexaufbau für ein großes Postfach kann Stunden bis Tage dauern, läuft aber nur einmal im Hintergrund und wird sicher fortgesetzt; danach ist das Einbetten neuer E-Mails sofort erledigt. Für einen schnelleren ersten Aufbau richtest du den Endpunkt auf eine gehostete API oder einen leistungsstärkeren, dauerhaft laufenden Rechner in deinem LAN." + } }, "aiActions": { "description": "Erstellen Sie eigene KI-Aktionen. Sie erscheinen beim Lesen einer Nachricht im KI-Menü neben „Zusammenfassen“. Der Prompt jeder Aktion wird auf den Nachrichteninhalt angewendet.", diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json index 96d923a1..88bc4d1a 100644 --- a/frontend/src/locales/en.json +++ b/frontend/src/locales/en.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "Search…", + "semanticToggle": "Semantic search", + "semanticBuildingHint": "Semantic index building — showing keyword results", "unread": "Unread", "showAll": "Show all", "unreadOnly": "Unread only", @@ -1060,7 +1062,38 @@ "test": "Test connection", "testing": "Testing…", "notConfigured": "No AI provider configured yet.", - "remove": "Remove configuration" + "remove": "Remove configuration", + "emb": { + "title": "Embeddings (semantic search)", + "benefit": "Embeddings make search faster and more accurate.", + "defaultOff": "Disabled by default. Nothing is sent anywhere until you enable embeddings and start a build.", + "enable": "Enable embeddings", + "privacyWarning": "With a hosted endpoint configured, the embeddings provider (e.g. OpenAI) receives your email content — subject and body text. At that point this data path is no longer self-hosted.", + "vectorUnavailable": "The pgvector extension is not available on this database, so semantic search cannot run. Keyword search still works.", + "endpoint": "Endpoint URL", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "The client appends /embeddings to this URL.", + "sameAsChat": "Same as chat provider", + "sameAsChatActive": "Using the chat provider endpoint", + "model": "Model", + "modelPh": "text-embedding-3-small", + "dimension": "Dimension", + "dimensionPh": "1536", + "dimensionHint": "Vector size the model returns. Run Test to confirm it from the endpoint.", + "apiKey": "Embeddings API key", + "test": "Test endpoint", + "testOk": "Endpoint reachable — dimension {{dimension}}.", + "build": "Build index", + "rebuild": "Rebuild index", + "building": "Building…", + "buildStarted": "Build started — indexing runs in the background.", + "saveHint": "Save your changes before testing or building.", + "progressLabel": "Indexing {{processed}} / {{total}} messages", + "progressDone": "Embeddings up to date ({{total}} messages indexed).", + "progressError": "Build error: {{error}}", + "localTitle": "Running embeddings locally", + "localHelp": "Mailflow can generate embeddings entirely on your own hardware, so no message content leaves your network. On a Raspberry Pi, run Ollama with a small model such as all-minilm (384) or nomic-embed-text (768) — both expose an OpenAI-compatible endpoint Mailflow talks to directly. Pi-class CPUs are slow: the first index build for a large mailbox can take hours to days, but it runs once in the background and resumes safely, and embedding new mail afterwards is instant. For a faster first build, point the endpoint at a hosted API or a more powerful always-on machine on your LAN." + } }, "aiActions": { "description": "Create your own AI actions. They appear in the AI menu when reading a message, alongside Summarize. Each action's prompt is applied to the message content.", diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.json index 2257c4dc..5f6b7a81 100644 --- a/frontend/src/locales/es.json +++ b/frontend/src/locales/es.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "Buscar…", + "semanticToggle": "Búsqueda semántica", + "semanticBuildingHint": "Creando índice semántico — mostrando resultados por palabra clave", "unread": "No leídos", "showAll": "Mostrar todos", "unreadOnly": "Solo no leídos", @@ -1060,7 +1062,38 @@ "test": "Probar conexión", "testing": "Probando…", "notConfigured": "Ningún proveedor de IA configurado aún.", - "remove": "Eliminar configuración" + "remove": "Eliminar configuración", + "emb": { + "title": "Embeddings (búsqueda semántica)", + "benefit": "Los embeddings hacen que la búsqueda sea más rápida y precisa.", + "defaultOff": "Desactivado de forma predeterminada. No se envía nada hasta que actives los embeddings e inicies una compilación.", + "enable": "Activar embeddings", + "privacyWarning": "Con un endpoint alojado configurado, el proveedor de embeddings (p. ej. OpenAI) recibe el contenido de tu correo: el asunto y el cuerpo del texto. En ese momento esta ruta de datos deja de ser autoalojada.", + "vectorUnavailable": "La extensión pgvector no está disponible en esta base de datos, por lo que la búsqueda semántica no puede funcionar. La búsqueda por palabras clave sigue funcionando.", + "endpoint": "URL del endpoint", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "El cliente añade /embeddings a esta URL.", + "sameAsChat": "Igual que el proveedor de chat", + "sameAsChatActive": "Usando el endpoint del proveedor de chat", + "model": "Modelo", + "modelPh": "text-embedding-3-small", + "dimension": "Dimensión", + "dimensionPh": "1536", + "dimensionHint": "Tamaño del vector que devuelve el modelo. Ejecuta «Probar» para confirmarlo desde el endpoint.", + "apiKey": "Clave de API de embeddings", + "test": "Probar endpoint", + "testOk": "Endpoint accesible — dimensión {{dimension}}.", + "build": "Compilar índice", + "rebuild": "Recompilar índice", + "building": "Compilando…", + "buildStarted": "Compilación iniciada: la indexación se ejecuta en segundo plano.", + "saveHint": "Guarda los cambios antes de probar o compilar.", + "progressLabel": "Indexando {{processed}} / {{total}} mensajes", + "progressDone": "Embeddings actualizados ({{total}} mensajes indexados).", + "progressError": "Error de compilación: {{error}}", + "localTitle": "Ejecutar embeddings localmente", + "localHelp": "Mailflow puede generar los embeddings íntegramente en tu propio hardware, de modo que ningún contenido de los mensajes sale de tu red. En una Raspberry Pi, ejecuta Ollama con un modelo pequeño como all-minilm (384) o nomic-embed-text (768); ambos exponen un endpoint compatible con OpenAI con el que Mailflow habla directamente. Las CPU de clase Pi son lentas: la primera compilación del índice de un buzón grande puede tardar de horas a días, pero se ejecuta una sola vez en segundo plano y se reanuda de forma segura; después, incrustar el correo nuevo es instantáneo. Para una primera compilación más rápida, apunta el endpoint a una API alojada o a una máquina más potente y siempre encendida de tu LAN." + } }, "aiActions": { "description": "Crea tus propias acciones de IA. Aparecen en el menú de IA al leer un mensaje, junto a Resumir. El prompt de cada acción se aplica al contenido del mensaje.", diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.json index 7b5b6d3e..014ed98c 100644 --- a/frontend/src/locales/fr.json +++ b/frontend/src/locales/fr.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "Rechercher…", + "semanticToggle": "Recherche sémantique", + "semanticBuildingHint": "Index sémantique en cours de création — affichage des résultats par mot-clé", "unread": "Non lus", "showAll": "Tout afficher", "unreadOnly": "Non lus seulement", @@ -1060,7 +1062,38 @@ "test": "Tester la connexion", "testing": "Test en cours…", "notConfigured": "Aucun fournisseur IA configuré.", - "remove": "Supprimer la configuration" + "remove": "Supprimer la configuration", + "emb": { + "title": "Embeddings (recherche sémantique)", + "benefit": "Les embeddings rendent la recherche plus rapide et plus précise.", + "defaultOff": "Désactivé par défaut. Rien n'est envoyé tant que vous n'activez pas les embeddings et ne lancez pas une génération.", + "enable": "Activer les embeddings", + "privacyWarning": "Avec un point de terminaison hébergé configuré, le fournisseur d'embeddings (p. ex. OpenAI) reçoit le contenu de vos e-mails — objet et corps du texte. À ce moment-là, ce chemin de données n'est plus auto-hébergé.", + "vectorUnavailable": "L'extension pgvector n'est pas disponible sur cette base de données, la recherche sémantique ne peut donc pas fonctionner. La recherche par mots-clés fonctionne toujours.", + "endpoint": "URL du point de terminaison", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "Le client ajoute /embeddings à cette URL.", + "sameAsChat": "Identique au fournisseur de chat", + "sameAsChatActive": "Utilise le point de terminaison du fournisseur de chat", + "model": "Modèle", + "modelPh": "text-embedding-3-small", + "dimension": "Dimension", + "dimensionPh": "1536", + "dimensionHint": "Taille du vecteur renvoyée par le modèle. Lancez « Tester » pour la confirmer depuis le point de terminaison.", + "apiKey": "Clé d'API des embeddings", + "test": "Tester le point de terminaison", + "testOk": "Point de terminaison accessible — dimension {{dimension}}.", + "build": "Générer l'index", + "rebuild": "Régénérer l'index", + "building": "Génération…", + "buildStarted": "Génération lancée — l'indexation s'exécute en arrière-plan.", + "saveHint": "Enregistrez vos modifications avant de tester ou de générer.", + "progressLabel": "Indexation de {{processed}} / {{total}} messages", + "progressDone": "Embeddings à jour ({{total}} messages indexés).", + "progressError": "Erreur de génération : {{error}}", + "localTitle": "Exécuter les embeddings en local", + "localHelp": "Mailflow peut générer les embeddings entièrement sur votre propre matériel, de sorte qu'aucun contenu de message ne quitte votre réseau. Sur un Raspberry Pi, exécutez Ollama avec un petit modèle comme all-minilm (384) ou nomic-embed-text (768) — tous deux exposent un point de terminaison compatible OpenAI auquel Mailflow parle directement. Les processeurs de classe Pi sont lents : la première génération de l'index d'une grande boîte aux lettres peut prendre de quelques heures à quelques jours, mais elle ne s'exécute qu'une fois en arrière-plan et reprend en toute sécurité ; ensuite, l'intégration des nouveaux e-mails est instantanée. Pour une première génération plus rapide, dirigez le point de terminaison vers une API hébergée ou une machine plus puissante et toujours allumée de votre LAN." + } }, "aiActions": { "description": "Créez vos propres actions IA. Elles apparaissent dans le menu IA lors de la lecture d'un message, à côté de Résumer. Le prompt de chaque action est appliqué au contenu du message.", diff --git a/frontend/src/locales/i18n.test.js b/frontend/src/locales/i18n.test.js index 888632e8..7c522958 100644 --- a/frontend/src/locales/i18n.test.js +++ b/frontend/src/locales/i18n.test.js @@ -124,6 +124,9 @@ const SAME_VALUE_ALLOWED = { 'admin.accounts.presetYahoo': 'any', // Yahoo Mail 'admin.accounts.smtpHostPh': 'any', // smtp.gmail.com 'admin.ai.baseUrlPh': 'any', // http://localhost:11434/v1 + 'admin.ai.emb.endpointPh': 'any', // https://api.openai.com/v1 + 'admin.ai.emb.modelPh': 'any', // text-embedding-3-small — model id + 'admin.ai.emb.dimensionPh': 'any', // 1536 — numeric placeholder 'admin.appearance.customCssPlaceholder': 'any', // CSS code snippet, same in all locales 'admin.integrations.microsoft.clientIdPh':'any', // xxxxxxxx-xxxx-… 'admin.integrations.microsoft.title': 'any', // Microsoft 365 / Outlook.com @@ -137,6 +140,8 @@ const SAME_VALUE_ALLOWED = { // ── Specific language groups ─────────────────────────────────────────────── // "Version" — same spelling in de, en, fr 'admin.about.version': [['de', 'en', 'fr']], + // "Dimension" — international technical term, same spelling in de, en, fr + 'admin.ai.emb.dimension': [['de', 'en', 'fr']], // "{{n}} min" — the "min" abbreviation is shared in en, es, fr, it 'admin.lock.autoLockMin': [['en', 'es', 'fr', 'it']], diff --git a/frontend/src/locales/it.json b/frontend/src/locales/it.json index 602b1ccb..be4b5996 100644 --- a/frontend/src/locales/it.json +++ b/frontend/src/locales/it.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "Cerca…", + "semanticToggle": "Ricerca semantica", + "semanticBuildingHint": "Creazione dell'indice semantico — risultati per parola chiave", "unread": "Non letti", "showAll": "Mostra tutto", "unreadOnly": "Solo non letti", @@ -1060,7 +1062,38 @@ "test": "Testa connessione", "testing": "Test in corso…", "notConfigured": "Nessun provider IA configurato.", - "remove": "Rimuovi configurazione" + "remove": "Rimuovi configurazione", + "emb": { + "title": "Embeddings (ricerca semantica)", + "benefit": "Gli embeddings rendono la ricerca più veloce e precisa.", + "defaultOff": "Disattivato per impostazione predefinita. Non viene inviato nulla finché non attivi gli embeddings e avvii una creazione.", + "enable": "Attiva gli embeddings", + "privacyWarning": "Con un endpoint ospitato configurato, il fornitore di embeddings (ad es. OpenAI) riceve il contenuto delle tue email — oggetto e corpo del testo. A quel punto questo percorso dei dati non è più self-hosted.", + "vectorUnavailable": "L'estensione pgvector non è disponibile su questo database, quindi la ricerca semantica non può funzionare. La ricerca per parole chiave funziona ancora.", + "endpoint": "URL dell'endpoint", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "Il client aggiunge /embeddings a questo URL.", + "sameAsChat": "Come il fornitore di chat", + "sameAsChatActive": "Utilizza l'endpoint del fornitore di chat", + "model": "Modello", + "modelPh": "text-embedding-3-small", + "dimension": "Dimensione", + "dimensionPh": "1536", + "dimensionHint": "Dimensione del vettore restituita dal modello. Esegui «Prova» per confermarla dall'endpoint.", + "apiKey": "Chiave API degli embeddings", + "test": "Prova endpoint", + "testOk": "Endpoint raggiungibile — dimensione {{dimension}}.", + "build": "Crea indice", + "rebuild": "Ricrea indice", + "building": "Creazione…", + "buildStarted": "Creazione avviata — l'indicizzazione viene eseguita in background.", + "saveHint": "Salva le modifiche prima di provare o creare.", + "progressLabel": "Indicizzazione di {{processed}} / {{total}} messaggi", + "progressDone": "Embeddings aggiornati ({{total}} messaggi indicizzati).", + "progressError": "Errore di creazione: {{error}}", + "localTitle": "Eseguire gli embeddings in locale", + "localHelp": "Mailflow può generare gli embeddings interamente sul tuo hardware, così nessun contenuto dei messaggi lascia la tua rete. Su un Raspberry Pi, esegui Ollama con un modello piccolo come all-minilm (384) o nomic-embed-text (768) — entrambi espongono un endpoint compatibile con OpenAI con cui Mailflow comunica direttamente. Le CPU di classe Pi sono lente: la prima creazione dell'indice per una casella grande può richiedere da ore a giorni, ma viene eseguita una sola volta in background e riprende in sicurezza; in seguito, l'incorporamento della nuova posta è istantaneo. Per una prima creazione più veloce, punta l'endpoint verso un'API ospitata o una macchina più potente e sempre accesa nella tua LAN." + } }, "aiActions": { "description": "Crea le tue azioni IA. Appaiono nel menu IA durante la lettura di un messaggio, accanto a Riassumi. Il prompt di ogni azione viene applicato al contenuto del messaggio.", diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.json index e22dd961..4d69f40e 100644 --- a/frontend/src/locales/ru.json +++ b/frontend/src/locales/ru.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "Поиск…", + "semanticToggle": "Семантический поиск", + "semanticBuildingHint": "Создаётся семантический индекс — показаны результаты по ключевым словам", "unread": "Непрочитанные", "showAll": "Показать все", "unreadOnly": "Только непрочитанные", @@ -1060,7 +1062,38 @@ "test": "Проверить подключение", "testing": "Проверка…", "notConfigured": "Провайдер ИИ не настроен.", - "remove": "Удалить конфигурацию" + "remove": "Удалить конфигурацию", + "emb": { + "title": "Эмбеддинги (семантический поиск)", + "benefit": "Эмбеддинги делают поиск быстрее и точнее.", + "defaultOff": "Отключено по умолчанию. Ничего не отправляется, пока вы не включите эмбеддинги и не запустите построение.", + "enable": "Включить эмбеддинги", + "privacyWarning": "При настроенной размещённой конечной точке поставщик эмбеддингов (например, OpenAI) получает содержимое ваших писем — тему и текст. С этого момента этот путь данных перестаёт быть самостоятельно размещённым.", + "vectorUnavailable": "Расширение pgvector недоступно в этой базе данных, поэтому семантический поиск не работает. Поиск по ключевым словам по-прежнему работает.", + "endpoint": "URL конечной точки", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "Клиент добавляет /embeddings к этому URL.", + "sameAsChat": "Как у поставщика чата", + "sameAsChatActive": "Используется конечная точка поставщика чата", + "model": "Модель", + "modelPh": "text-embedding-3-small", + "dimension": "Размерность", + "dimensionPh": "1536", + "dimensionHint": "Размер вектора, возвращаемый моделью. Нажмите «Проверить», чтобы подтвердить его через конечную точку.", + "apiKey": "Ключ API эмбеддингов", + "test": "Проверить конечную точку", + "testOk": "Конечная точка доступна — размерность {{dimension}}.", + "build": "Построить индекс", + "rebuild": "Перестроить индекс", + "building": "Построение…", + "buildStarted": "Построение запущено — индексация выполняется в фоне.", + "saveHint": "Сохраните изменения перед проверкой или построением.", + "progressLabel": "Индексация {{processed}} / {{total}} сообщений", + "progressDone": "Эмбеддинги обновлены (проиндексировано сообщений: {{total}}).", + "progressError": "Ошибка построения: {{error}}", + "localTitle": "Запуск эмбеддингов локально", + "localHelp": "Mailflow может создавать эмбеддинги полностью на вашем оборудовании, так что содержимое сообщений не покидает вашу сеть. На Raspberry Pi запустите Ollama с небольшой моделью, например all-minilm (384) или nomic-embed-text (768) — обе предоставляют совместимую с OpenAI конечную точку, с которой Mailflow работает напрямую. Процессоры класса Pi медленные: первое построение индекса для большого почтового ящика может занять от нескольких часов до нескольких дней, но оно выполняется один раз в фоне и безопасно возобновляется; после этого встраивание новой почты происходит мгновенно. Для более быстрого первого построения направьте конечную точку на размещённый API или на более мощную постоянно работающую машину в вашей локальной сети." + } }, "aiActions": { "description": "Создавайте собственные ИИ-действия. Они появляются в меню ИИ при чтении сообщения, рядом с «Кратко». Промпт каждого действия применяется к содержимому сообщения.", diff --git a/frontend/src/locales/zhCN.json b/frontend/src/locales/zhCN.json index 5ed24957..bdd5fb1a 100644 --- a/frontend/src/locales/zhCN.json +++ b/frontend/src/locales/zhCN.json @@ -251,6 +251,8 @@ }, "messageList": { "search": "搜索…", + "semanticToggle": "语义搜索", + "semanticBuildingHint": "正在构建语义索引 — 显示关键词结果", "unread": "未读", "showAll": "显示全部", "unreadOnly": "仅显示未读", @@ -1060,7 +1062,38 @@ "test": "测试连接", "testing": "测试中…", "notConfigured": "尚未配置 AI 提供商。", - "remove": "删除配置" + "remove": "删除配置", + "emb": { + "title": "嵌入（语义搜索）", + "benefit": "嵌入让搜索更快、更准确。", + "defaultOff": "默认关闭。在你启用嵌入并开始构建之前，不会向任何地方发送数据。", + "enable": "启用嵌入", + "privacyWarning": "配置托管端点后，嵌入提供方（例如 OpenAI）会收到你的邮件内容——主题和正文文本。此时该数据路径不再是自托管的。", + "vectorUnavailable": "此数据库上没有可用的 pgvector 扩展，因此无法进行语义搜索。关键词搜索仍然可用。", + "endpoint": "端点 URL", + "endpointPh": "https://api.openai.com/v1", + "endpointHint": "客户端会在该 URL 后追加 /embeddings。", + "sameAsChat": "与聊天提供方相同", + "sameAsChatActive": "正在使用聊天提供方的端点", + "model": "模型", + "modelPh": "text-embedding-3-small", + "dimension": "维度", + "dimensionPh": "1536", + "dimensionHint": "模型返回的向量大小。运行“测试”以从端点确认该值。", + "apiKey": "嵌入 API 密钥", + "test": "测试端点", + "testOk": "端点可访问——维度 {{dimension}}。", + "build": "构建索引", + "rebuild": "重建索引", + "building": "构建中…", + "buildStarted": "构建已开始——索引在后台运行。", + "saveHint": "在测试或构建之前，请先保存更改。", + "progressLabel": "正在索引 {{processed}} / {{total}} 封邮件", + "progressDone": "嵌入已是最新（已索引 {{total}} 封邮件）。", + "progressError": "构建错误：{{error}}", + "localTitle": "在本地运行嵌入", + "localHelp": "Mailflow 可以完全在你自己的硬件上生成嵌入，因此邮件内容不会离开你的网络。在 Raspberry Pi 上，使用小型模型运行 Ollama，例如 all-minilm（384）或 nomic-embed-text（768）——两者都提供与 OpenAI 兼容的端点，Mailflow 可直接与之通信。Pi 级 CPU 较慢：对于大型邮箱，首次索引构建可能需要数小时到数天，但它只在后台运行一次并能安全恢复；此后，为新邮件生成嵌入几乎是即时的。若想加快首次构建，可将端点指向托管 API，或指向你局域网中一台性能更强、始终在线的机器。" + } }, "aiActions": { "description": "创建你自己的 AI 操作。它们会在阅读邮件时出现在 AI 菜单中，与“摘要”并列。每个操作的提示词会应用于邮件内容。", diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index 7c9a33cd..ef44e745 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -6,6 +6,7 @@ import { applyLayout, normalizeLayout } from '../layouts.js'; import { DEFAULT_AI_ACTIONS } from '../aiActions.js'; import { removeGtdThreadFromSections, setGtdThreadReadInSections } from '../utils/gtd.js'; import { clampRightSidebarWidth } from '../utils/rightSidebar.js'; +import { readStoredSearchMode, writeStoredSearchMode } from '../utils/searchMode.js'; import i18n from '../i18n.js'; // Accumulate rapid preference changes and flush at most once per second. @@ -277,6 +278,10 @@ export const useStore = create((set, get) => ({ else localStorage.removeItem('mailflow_search_all_folders'); set({ searchAllFolders: v }); }, + // Search mode: 'lexical' (default) | 'hybrid' | 'vector'. Persisted per device, + // like searchAllFolders. Only meaningful when /api/ai/status reports vector available. + searchMode: readStoredSearchMode(localStorage), + setSearchMode: (mode) => set({ searchMode: writeStoredSearchMode(localStorage, mode) }), swipeActions: (() => { try { return JSON.parse(localStorage.getItem('mailflow_swipe_actions') || 'null') || { left: 'archive', right: 'markRead' }; diff --git a/frontend/src/utils/api.js b/frontend/src/utils/api.js index 38f76db1..dbdf57ae 100644 --- a/frontend/src/utils/api.js +++ b/frontend/src/utils/api.js @@ -209,12 +209,13 @@ export const api = { emptyFolder: (accountId, path) => request('POST', '/mail/folders/empty', { accountId, path }), // Search - search: (q, accountId, { offset = 0, limit, folder } = {}) => { + search: (q, accountId, { offset = 0, limit, folder, mode } = {}) => { const params = new URLSearchParams({ q }); if (accountId) params.set('accountId', accountId); if (limit) params.set('limit', limit); if (folder) params.set('folder', folder); if (offset) params.set('offset', offset); + if (mode && mode !== 'lexical') params.set('mode', mode); return request('GET', `/search?${params}`); }, suggestContacts: (q) => request('GET', `/search/contacts?q=${encodeURIComponent(q)}`), @@ -276,6 +277,10 @@ export const api = { deleteConfig: () => request('DELETE', '/admin/ai'), test: () => request('POST', '/admin/ai/test'), status: () => request('GET', '/ai/status'), + // Embeddings (semantic search) — probe the saved config and kick a build. + testEmbeddings: () => request('POST', '/admin/ai/embeddings/test-embeddings'), + buildEmbeddings: () => request('POST', '/admin/ai/embeddings/build'), + indexingStatus: () => request('GET', '/admin/indexing/status'), }, // Category counts for inbox tab badges @@ -333,4 +338,5 @@ export const api = { getLabels: () => request('GET', '/todoist/labels'), createTask: (data) => request('POST', '/todoist/tasks', data), }, + }; diff --git a/frontend/src/utils/api.search.test.js b/frontend/src/utils/api.search.test.js index a237a775..1f9ff1aa 100644 --- a/frontend/src/utils/api.search.test.js +++ b/frontend/src/utils/api.search.test.js @@ -2,17 +2,16 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { api } from './api.js'; -test('search sends lexical pagination and folder parameters', async () => { - let seen; +test('search adds mode only for a non-default semantic mode', async () => { + const seen = []; const orig = globalThis.fetch; - globalThis.fetch = async (url) => { seen = url; return { ok: true, json: async () => ({ messages: [] }) }; }; + globalThis.fetch = async (url) => { seen.push(url); return { ok: true, json: async () => ({ messages: [] }) }; }; try { - await api.search('quarterly report', 'a1', { offset: 50, limit: 25, folder: 'INBOX' }); + await api.search('hi', undefined, {}); + await api.search('hi', undefined, { mode: 'lexical' }); + await api.search('hi', undefined, { mode: 'hybrid' }); } finally { globalThis.fetch = orig; } - const url = new URL(seen, 'http://localhost'); - assert.equal(url.searchParams.get('q'), 'quarterly report'); - assert.equal(url.searchParams.get('accountId'), 'a1'); - assert.equal(url.searchParams.get('offset'), '50'); - assert.equal(url.searchParams.get('limit'), '25'); - assert.equal(url.searchParams.get('folder'), 'INBOX'); + assert.ok(!seen[0].includes('mode='), 'no mode when absent'); + assert.ok(!seen[1].includes('mode='), 'no mode when lexical'); + assert.ok(seen[2].includes('mode=hybrid'), 'mode=hybrid present'); }); diff --git a/frontend/src/utils/embeddingsSettings.js b/frontend/src/utils/embeddingsSettings.js new file mode 100644 index 00000000..780a5ff9 --- /dev/null +++ b/frontend/src/utils/embeddingsSettings.js @@ -0,0 +1,101 @@ +// Pure helpers for the AI-settings embeddings block. The React component +// (AdminPanel.jsx → AISection) owns the rendering; everything testable lives +// here, mirroring the phase-4 searchMode.js split. + +// The backend masks a stored key as this sentinel on read and, on save, treats it +// as "keep the existing key" — identical handling to the chat apiKey field. +export const EMBEDDINGS_KEY_SENTINEL = '••••••••'; + +// Known model → returned dimension, surfaced as a hint next to the model field. +export const EMBEDDING_MODEL_HINTS = [ + { model: 'text-embedding-3-small', dimension: 1536 }, + { model: 'text-embedding-3-large', dimension: 3072 }, + { model: 'nomic-embed-text', dimension: 768 }, + { model: 'all-minilm', dimension: 384 }, +]; + +export function emptyEmbeddingsForm() { + return { enabled: false, endpoint: '', apiKey: '', model: '', dimension: '' }; +} + +// Map the masked embeddings sub-config from GET /admin/ai to editable form state. +// dimension becomes a string so the numeric stays controlled and empty-able. +export function embeddingsFormFromConfig(cfg) { + const e = cfg?.embeddings; + if (!e) return emptyEmbeddingsForm(); + return { + enabled: e.enabled === true, + endpoint: e.endpoint || '', + apiKey: e.apiKey || '', // already the sentinel when a key is stored + model: e.model || '', + dimension: e.dimension ? String(e.dimension) : '', + }; +} + +// Build the `embeddings` sub-object for the PATCH body. dimension is coerced to a +// number; the masked sentinel is passed through untouched so the backend keeps the +// stored key. Endpoint/model are trimmed (the backend trims again — belt and braces). +export function buildEmbeddingsPayload(form) { + return { + enabled: form.enabled === true, + endpoint: (form.endpoint || '').trim(), + apiKey: form.apiKey || '', + model: (form.model || '').trim(), + dimension: Number(form.dimension) || 0, + }; +} + +// Test and Build probe the SAVED config (resolveEmbedConfig), never request-body +// values — so they must stay gated until the form is persisted. A masked apiKey on +// both sides compares equal (no change); a typed or cleared key reads as dirty. +export function embeddingsDirty(form, savedCfg) { + const saved = embeddingsFormFromConfig(savedCfg); + return form.enabled !== saved.enabled + || (form.endpoint || '') !== (saved.endpoint || '') + || (form.model || '') !== (saved.model || '') + || String(form.dimension || '') !== String(saved.dimension || '') + || (form.apiKey || '') !== (saved.apiKey || ''); +} + +// "Same as chat provider": the stored endpoint mirrors the chat baseUrl exactly +// (trailing slashes ignored, matching the backend's normalization). +export function isSameAsChatProvider(endpoint, chatBaseUrl) { + const norm = (s) => (s || '').trim().replace(/\/+$/, ''); + const c = norm(chatBaseUrl); + return !!c && norm(endpoint) === c; +} + +// After a successful Test probe the endpoint is the source of truth for the real +// dimension, so adopt the probed value when it differs from what was entered. +export function reconcileDimension(current, probed) { + const c = Number(current) || 0; + const changed = probed > 0 && probed !== c; + return { dimension: changed ? probed : c, changed }; +} + +// Extract the embeddings build job from GET /admin/indexing/status → { jobs }. +export function embeddingsJob(jobs) { + const job = (jobs || []).find(j => j.kind === 'embeddings'); + if (!job) return null; + const processed = Number(job.processed) || 0; + const total = Number(job.total) || 0; + return { + state: job.state, // running | done | error + processed, + total, + lastError: job.last_error || null, + percent: total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : 0, + active: job.state === 'running', + }; +} + +// The single PATCH /admin/ai save writes chat + embeddings together, so allow it +// when a complete chat config or an enabled embeddings block is present (an +// embeddings-only install should not need to fill in chat fields). Also allow it +// when the SAVED config already had embeddings on — otherwise toggling embeddings +// off could never be persisted, leaving no way to stop a hosted data path short of +// removing the whole config. +export function canSaveAiConfig(form, emb, savedCfg) { + const savedEmbEnabled = savedCfg?.embeddings?.enabled === true; + return !!(form.baseUrl && form.model) || !!(emb && emb.enabled) || savedEmbEnabled; +} diff --git a/frontend/src/utils/embeddingsSettings.test.js b/frontend/src/utils/embeddingsSettings.test.js new file mode 100644 index 00000000..c195b4f0 --- /dev/null +++ b/frontend/src/utils/embeddingsSettings.test.js @@ -0,0 +1,99 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + EMBEDDINGS_KEY_SENTINEL, + emptyEmbeddingsForm, + embeddingsFormFromConfig, + buildEmbeddingsPayload, + embeddingsDirty, + isSameAsChatProvider, + reconcileDimension, + embeddingsJob, + canSaveAiConfig, +} from './embeddingsSettings.js'; + +test('emptyEmbeddingsForm is disabled with blank fields', () => { + assert.deepEqual(emptyEmbeddingsForm(), { enabled: false, endpoint: '', apiKey: '', model: '', dimension: '' }); +}); + +test('embeddingsFormFromConfig maps the masked sub-config, dimension as a string', () => { + const cfg = { baseUrl: 'x', embeddings: { enabled: true, endpoint: 'https://api.openai.com/v1', apiKey: EMBEDDINGS_KEY_SENTINEL, model: 'text-embedding-3-small', dimension: 1536 } }; + assert.deepEqual(embeddingsFormFromConfig(cfg), { + enabled: true, endpoint: 'https://api.openai.com/v1', apiKey: EMBEDDINGS_KEY_SENTINEL, model: 'text-embedding-3-small', dimension: '1536', + }); +}); + +test('embeddingsFormFromConfig falls back to an empty form when embeddings absent', () => { + assert.deepEqual(embeddingsFormFromConfig(null), emptyEmbeddingsForm()); + assert.deepEqual(embeddingsFormFromConfig({ baseUrl: 'x' }), emptyEmbeddingsForm()); +}); + +test('buildEmbeddingsPayload trims, coerces dimension to a number, passes the key through', () => { + const payload = buildEmbeddingsPayload({ enabled: true, endpoint: ' https://h/v1/ ', apiKey: EMBEDDINGS_KEY_SENTINEL, model: ' m ', dimension: '768' }); + assert.deepEqual(payload, { enabled: true, endpoint: 'https://h/v1/', apiKey: EMBEDDINGS_KEY_SENTINEL, model: 'm', dimension: 768 }); + // The masked sentinel is passed untouched so the backend keeps the stored key. + assert.equal(buildEmbeddingsPayload(emptyEmbeddingsForm()).dimension, 0); + // A freshly typed key is sent verbatim (the backend encrypts it on save). + assert.equal(buildEmbeddingsPayload({ ...emptyEmbeddingsForm(), apiKey: 'sk-new-123' }).apiKey, 'sk-new-123'); +}); + +test('embeddingsDirty is false when the form matches the saved masked config', () => { + const cfg = { embeddings: { enabled: true, endpoint: 'https://h/v1', apiKey: EMBEDDINGS_KEY_SENTINEL, model: 'm', dimension: 768 } }; + const form = embeddingsFormFromConfig(cfg); + assert.equal(embeddingsDirty(form, cfg), false); + assert.equal(embeddingsDirty({ ...form, endpoint: 'https://other/v1' }, cfg), true); + assert.equal(embeddingsDirty({ ...form, apiKey: 'sk-new' }, cfg), true); + assert.equal(embeddingsDirty({ ...form, dimension: '384' }, cfg), true); + assert.equal(embeddingsDirty({ ...form, enabled: false }, cfg), true); +}); + +test('embeddingsDirty treats enabling against no saved config as dirty', () => { + assert.equal(embeddingsDirty(emptyEmbeddingsForm(), null), false); + assert.equal(embeddingsDirty({ ...emptyEmbeddingsForm(), enabled: true, endpoint: 'https://h/v1' }, null), true); +}); + +test('isSameAsChatProvider ignores trailing slashes and empty baseUrls', () => { + assert.equal(isSameAsChatProvider('https://h/v1', 'https://h/v1/'), true); + assert.equal(isSameAsChatProvider('https://h/v1', 'https://other/v1'), false); + assert.equal(isSameAsChatProvider('', ''), false); + assert.equal(isSameAsChatProvider('https://h/v1', ''), false); +}); + +test('reconcileDimension adopts a differing probed dimension', () => { + assert.deepEqual(reconcileDimension('1536', 768), { dimension: 768, changed: true }); + assert.deepEqual(reconcileDimension('768', 768), { dimension: 768, changed: false }); + assert.deepEqual(reconcileDimension('', 384), { dimension: 384, changed: true }); + assert.deepEqual(reconcileDimension('768', 0), { dimension: 768, changed: false }); +}); + +test('embeddingsJob extracts the embeddings row and computes percent', () => { + const jobs = [ + { kind: 'fts', state: 'done', processed: 5, total: 5 }, + { kind: 'embeddings', state: 'running', processed: 25, total: 100, last_error: null }, + ]; + assert.deepEqual(embeddingsJob(jobs), { state: 'running', processed: 25, total: 100, lastError: null, percent: 25, active: true }); + assert.equal(embeddingsJob([{ kind: 'fts', state: 'done' }]), null); + assert.equal(embeddingsJob(null), null); +}); + +test('embeddingsJob surfaces error state and a zero-total percent', () => { + const jobs = [{ kind: 'embeddings', state: 'error', processed: 0, total: 0, last_error: 'connect ECONNREFUSED' }]; + const j = embeddingsJob(jobs); + assert.equal(j.state, 'error'); + assert.equal(j.percent, 0); + assert.equal(j.active, false); + assert.equal(j.lastError, 'connect ECONNREFUSED'); +}); + +test('canSaveAiConfig allows a complete chat config, an enabled block, or turning a saved block off', () => { + assert.equal(canSaveAiConfig({ baseUrl: 'https://h/v1', model: 'm' }, { enabled: false }), true); + assert.equal(canSaveAiConfig({ baseUrl: '', model: '' }, { enabled: true }), true); + // Nothing configured, nothing saved → nothing to persist. + assert.equal(canSaveAiConfig({ baseUrl: '', model: '' }, { enabled: false }), false); + assert.equal(canSaveAiConfig({ baseUrl: 'https://h/v1', model: '' }, { enabled: false }), false); + // Embeddings-only install: the saved config had embeddings on, so toggling it off + // must stay persistable (otherwise stopping the hosted data path is a dead end). + assert.equal(canSaveAiConfig({ baseUrl: '', model: '' }, { enabled: false }, { embeddings: { enabled: true } }), true); + // A saved-but-disabled block does not by itself unlock Save. + assert.equal(canSaveAiConfig({ baseUrl: '', model: '' }, { enabled: false }, { embeddings: { enabled: false } }), false); +}); diff --git a/frontend/src/utils/searchMode.js b/frontend/src/utils/searchMode.js new file mode 100644 index 00000000..c0c9c512 --- /dev/null +++ b/frontend/src/utils/searchMode.js @@ -0,0 +1,80 @@ +export const LEXICAL_MODE = 'lexical'; +export const SEMANTIC_MODE = 'hybrid'; // the toggle flips lexical <-> hybrid +export const SEARCH_MODE_KEY = 'mailflow_search_mode'; + +// Vector availability from GET /api/ai/status, which reports a flat +// `vectorAvailable` boolean (backend/src/routes/ai.js) — not a nested +// `vector.available` shape. Keep the field lookup here so a rename is a +// one-line change. +export function semanticSearchAvailable(aiStatus) { + if (!aiStatus) return false; + return aiStatus.vectorAvailable === true; +} + +export function normalizeSearchMode(mode) { + return mode === SEMANTIC_MODE || mode === 'vector' ? mode : LEXICAL_MODE; +} + +export function readStoredSearchMode(storage) { + try { return normalizeSearchMode(storage.getItem(SEARCH_MODE_KEY)); } + catch { return LEXICAL_MODE; } +} + +export function writeStoredSearchMode(storage, mode) { + const m = normalizeSearchMode(mode); + try { + if (m === LEXICAL_MODE) storage.removeItem(SEARCH_MODE_KEY); + else storage.setItem(SEARCH_MODE_KEY, m); + } catch { /* storage unavailable — in-memory only */ } + return m; +} + +// Visual + aria state for the in-input semantic toggle (the sparkle icon that +// lives on the right of the search box). Pure so it can be unit-tested; the +// component maps `tone` to a CSS colour and `titleKey` to a translated tooltip +// that doubles as the button's aria-label. +// +// off → toggle is off (lexical). Greyed sparkle, tooltip "Semantic search". +// on → semantic active and serving semantic results. Accent (purple) sparkle. +// fallback → semantic active but the backend silently fell back to lexical for +// this query (vector index still building). Amber sparkle + an +// extended tooltip that explains keyword results are showing. This +// re-homes the old fallback pill-row hint onto the icon so no extra +// row appears below the input. +export function semanticToggleState({ on, fellBack, hasQuery }) { + if (!on) return { pressed: false, tone: 'off', titleKey: 'messageList.semanticToggle' }; + if (fellBack && hasQuery) return { pressed: true, tone: 'fallback', titleKey: 'messageList.semanticBuildingHint' }; + return { pressed: true, tone: 'on', titleKey: 'messageList.semanticToggle' }; +} + +// Geometry of the in-input control cluster (sparkle toggle + clear ×). Used to +// reserve enough right-padding on the search input that a long query can never +// slide under the icons. An icon button is a glyph (≤15px) plus 2×4px padding +// ≈ 23px; the cluster adds a 2px gap between the two buttons and sits 6px in +// from the input's right edge. +export const SEARCH_ICON_BOX = 23; +export const SEARCH_CLUSTER_GAP = 2; +export const SEARCH_CLUSTER_OFFSET = 6; + +// Right-padding (px) the search input must reserve. With the semantic toggle +// visible the cluster can hold both the sparkle and the clear ×; without it, +// only the clear × can appear. The trailing +4 is breathing room so the glyph +// never touches the text. +export function searchInputRightPad(semanticAvailable) { + const icons = semanticAvailable ? 2 : 1; + return SEARCH_CLUSTER_OFFSET + icons * SEARCH_ICON_BOX + (icons - 1) * SEARCH_CLUSTER_GAP + 4; +} + +// The whole search context — query text, mode (lexical/hybrid), folder/scope, +// account selection, page size — is identified by a single monotonically +// increasing generation counter that the initial-search effect bumps whenever +// any of those change (or the query is cleared). Every async search response +// (first page, "load more" append, post-delete prefetch) captures the +// generation before its await and must be discarded unless it still matches the +// live one. Keying on the generation instead of comparing individual fields +// guards against every context change at once — e.g. a hybrid page 2 that +// resolves after the user toggled Semantic off must not append onto the fresh +// lexical page — and automatically covers any search param added later. +export function isCurrentSearchGeneration(capturedGeneration, currentGeneration) { + return capturedGeneration === currentGeneration; +} diff --git a/frontend/src/utils/searchMode.test.js b/frontend/src/utils/searchMode.test.js new file mode 100644 index 00000000..bd57269f --- /dev/null +++ b/frontend/src/utils/searchMode.test.js @@ -0,0 +1,87 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + LEXICAL_MODE, SEMANTIC_MODE, SEARCH_MODE_KEY, + semanticSearchAvailable, normalizeSearchMode, readStoredSearchMode, writeStoredSearchMode, + semanticToggleState, searchInputRightPad, + SEARCH_ICON_BOX, SEARCH_CLUSTER_GAP, SEARCH_CLUSTER_OFFSET, + isCurrentSearchGeneration, +} from './searchMode.js'; + +function fakeStorage(init = {}) { + const m = new Map(Object.entries(init)); + return { getItem: (k) => (m.has(k) ? m.get(k) : null), setItem: (k, v) => m.set(k, String(v)), removeItem: (k) => m.delete(k), _m: m }; +} + +test('semanticSearchAvailable reads the vectorAvailable field from GET /api/ai/status', () => { + assert.equal(semanticSearchAvailable({ vectorAvailable: true }), true); + assert.equal(semanticSearchAvailable({ vectorAvailable: false }), false); + assert.equal(semanticSearchAvailable({ enabled: true }), false); + assert.equal(semanticSearchAvailable(null), false); +}); + +test('normalizeSearchMode keeps semantic modes and coerces the rest to lexical', () => { + assert.equal(normalizeSearchMode(SEMANTIC_MODE), 'hybrid'); + assert.equal(normalizeSearchMode('vector'), 'vector'); + assert.equal(normalizeSearchMode('nonsense'), LEXICAL_MODE); + assert.equal(normalizeSearchMode(null), LEXICAL_MODE); +}); + +test('read/write round-trips through storage and clears the key on lexical', () => { + const s = fakeStorage(); + writeStoredSearchMode(s, 'hybrid'); + assert.equal(s._m.get(SEARCH_MODE_KEY), 'hybrid'); + assert.equal(readStoredSearchMode(s), 'hybrid'); + writeStoredSearchMode(s, 'lexical'); + assert.equal(s._m.has(SEARCH_MODE_KEY), false); + assert.equal(readStoredSearchMode(s), 'lexical'); +}); + +test('semanticToggleState maps toggle + fallback into a tone and tooltip key', () => { + // Off — greyed sparkle, default tooltip. Fallback is irrelevant when off. + assert.deepEqual( + semanticToggleState({ on: false, fellBack: false, hasQuery: false }), + { pressed: false, tone: 'off', titleKey: 'messageList.semanticToggle' }); + assert.deepEqual( + semanticToggleState({ on: false, fellBack: true, hasQuery: true }), + { pressed: false, tone: 'off', titleKey: 'messageList.semanticToggle' }); + + // On and serving semantic results — accent sparkle, default tooltip. + assert.deepEqual( + semanticToggleState({ on: true, fellBack: false, hasQuery: true }), + { pressed: true, tone: 'on', titleKey: 'messageList.semanticToggle' }); + + // On but the query is empty — nothing fell back yet, so stay in the plain + // "on" tone even if a stale fellBack flag lingers. + assert.deepEqual( + semanticToggleState({ on: true, fellBack: true, hasQuery: false }), + { pressed: true, tone: 'on', titleKey: 'messageList.semanticToggle' }); + + // On + fell back + a live query — amber sparkle with the extended tooltip. + assert.deepEqual( + semanticToggleState({ on: true, fellBack: true, hasQuery: true }), + { pressed: true, tone: 'fallback', titleKey: 'messageList.semanticBuildingHint' }); +}); + +test('searchInputRightPad reserves room so a long query never slides under the icon cluster', () => { + // With the semantic toggle: room for the sparkle + clear × + gap + edge offset. + const bothIcons = SEARCH_CLUSTER_OFFSET + 2 * SEARCH_ICON_BOX + SEARCH_CLUSTER_GAP; + assert.ok(searchInputRightPad(true) >= bothIcons, + `pad ${searchInputRightPad(true)} must cover the two-icon cluster ${bothIcons}`); + // Without it: room for the clear × alone. + const oneIcon = SEARCH_CLUSTER_OFFSET + SEARCH_ICON_BOX; + assert.ok(searchInputRightPad(false) >= oneIcon, + `pad ${searchInputRightPad(false)} must cover the one-icon cluster ${oneIcon}`); + // Showing the toggle never reserves less space than hiding it. + assert.ok(searchInputRightPad(true) > searchInputRightPad(false)); +}); + +test('isCurrentSearchGeneration only accepts a response from the live generation', () => { + // Same generation captured before and after the await → apply the response. + assert.equal(isCurrentSearchGeneration(3, 3), true); + assert.equal(isCurrentSearchGeneration(0, 0), true); + // Generation moved on during the await (query/mode/folder/account changed, or + // the query was cleared) → the response is stale and must be discarded. + assert.equal(isCurrentSearchGeneration(3, 4), false); + assert.equal(isCurrentSearchGeneration(4, 3), false); +}); From c7361711b89e44e4fc22b19278aee794f570f30b Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:41:05 -0700 Subject: [PATCH 3/4] feat(mcp): bearer-tokened MCP server exposing search to agents Streamable-HTTP /mcp endpoint (SHA-256-hashed tokens minted in Profile, Origin-validated, rate-limited) exposing 12 tools including semantic_search_messages and search_in_message. Every call is scoped to the token owner's accounts; deletion is staged-only. Adds the MCP-only search seams (trusted account scoping, body-scope FTS leg, strictVector, loadVector), the chunk-excerpt read path, migrations 0040-0041, and one new backend dependency: @modelcontextprotocol/sdk. Split 3/3 of #283, stacked on the semantic-embeddings PR. The stacked tree is byte-identical to the reviewed #283 head (3773150). Co-Authored-By: Claude Fable 5 --- backend/migrations/0040_api_tokens.sql | 12 + .../migrations/0041_mcp_deletion_batches.sql | 21 + backend/package-lock.json | 621 +++++++++++++++++- backend/package.json | 1 + backend/src/index.js | 9 + backend/src/mcp/auth.js | 45 ++ backend/src/mcp/auth.test.js | 87 +++ backend/src/mcp/bodyMatch.js | 78 +++ backend/src/mcp/bodyMatch.test.js | 46 ++ backend/src/mcp/engineAdapter.js | 411 ++++++++++++ backend/src/mcp/engineAdapter.test.js | 309 +++++++++ backend/src/mcp/envelope.js | 30 + backend/src/mcp/envelope.test.js | 48 ++ backend/src/mcp/goldenParity.test.js | 319 +++++++++ backend/src/mcp/messageTools.js | 485 ++++++++++++++ backend/src/mcp/messageTools.test.js | 489 ++++++++++++++ backend/src/mcp/result.js | 9 + backend/src/mcp/result.test.js | 29 + backend/src/mcp/searchTools.js | 414 ++++++++++++ backend/src/mcp/searchTools.test.js | 389 +++++++++++ .../mcp/semanticSearchIds.regression.test.js | 89 +++ backend/src/mcp/server.js | 149 +++++ backend/src/mcp/server.test.js | 77 +++ backend/src/mcp/serverGuards.test.js | 267 ++++++++ backend/src/mcp/tools.js | 54 ++ backend/src/mcp/vectorErrors.js | 18 + backend/src/mcp/vectorErrors.test.js | 23 + backend/src/mcp/vectorStats.js | 75 +++ backend/src/mcp/vectorStats.test.js | 98 +++ backend/src/routes/apiTokens.js | 38 ++ backend/src/routes/apiTokens.test.js | 80 +++ backend/src/routes/mcpDeletions.js | 25 + backend/src/routes/mcpDeletions.test.js | 59 ++ backend/src/services/embeddings/chunk.js | 4 + backend/src/services/embeddings/chunk.test.js | 3 + backend/src/services/embeddings/chunkmatch.js | 147 +++++ .../services/embeddings/chunkmatch.test.js | 171 +++++ .../src/services/embeddings/vectorStore.js | 23 + backend/src/services/embeddings/worker.js | 2 + backend/src/services/search/lexicalRepo.js | 41 +- .../src/services/search/lexicalRepo.test.js | 46 ++ .../services/search/lexicalRepo.total.test.js | 9 +- backend/src/services/search/searchService.js | 48 +- .../src/services/search/searchService.test.js | 38 +- .../search/searchService.total.test.js | 7 +- backend/src/testSupport/mockSurface.js | 24 + backend/src/utils/textExcerpt.js | 21 + backend/src/utils/textExcerpt.test.js | 31 + frontend/nginx.conf | 26 + frontend/src/components/ProfileModal.jsx | 96 ++- frontend/src/locales/de.json | 15 +- frontend/src/locales/en.json | 15 +- frontend/src/locales/es.json | 15 +- frontend/src/locales/fr.json | 15 +- frontend/src/locales/it.json | 15 +- frontend/src/locales/ru.json | 15 +- frontend/src/locales/zhCN.json | 15 +- frontend/src/utils/api.js | 14 + 58 files changed, 5716 insertions(+), 44 deletions(-) create mode 100644 backend/migrations/0040_api_tokens.sql create mode 100644 backend/migrations/0041_mcp_deletion_batches.sql create mode 100644 backend/src/mcp/auth.js create mode 100644 backend/src/mcp/auth.test.js create mode 100644 backend/src/mcp/bodyMatch.js create mode 100644 backend/src/mcp/bodyMatch.test.js create mode 100644 backend/src/mcp/engineAdapter.js create mode 100644 backend/src/mcp/engineAdapter.test.js create mode 100644 backend/src/mcp/envelope.js create mode 100644 backend/src/mcp/envelope.test.js create mode 100644 backend/src/mcp/goldenParity.test.js create mode 100644 backend/src/mcp/messageTools.js create mode 100644 backend/src/mcp/messageTools.test.js create mode 100644 backend/src/mcp/result.js create mode 100644 backend/src/mcp/result.test.js create mode 100644 backend/src/mcp/searchTools.js create mode 100644 backend/src/mcp/searchTools.test.js create mode 100644 backend/src/mcp/semanticSearchIds.regression.test.js create mode 100644 backend/src/mcp/server.js create mode 100644 backend/src/mcp/server.test.js create mode 100644 backend/src/mcp/serverGuards.test.js create mode 100644 backend/src/mcp/tools.js create mode 100644 backend/src/mcp/vectorErrors.js create mode 100644 backend/src/mcp/vectorErrors.test.js create mode 100644 backend/src/mcp/vectorStats.js create mode 100644 backend/src/mcp/vectorStats.test.js create mode 100644 backend/src/routes/apiTokens.js create mode 100644 backend/src/routes/apiTokens.test.js create mode 100644 backend/src/routes/mcpDeletions.js create mode 100644 backend/src/routes/mcpDeletions.test.js create mode 100644 backend/src/services/embeddings/chunkmatch.js create mode 100644 backend/src/services/embeddings/chunkmatch.test.js create mode 100644 backend/src/testSupport/mockSurface.js create mode 100644 backend/src/utils/textExcerpt.js create mode 100644 backend/src/utils/textExcerpt.test.js diff --git a/backend/migrations/0040_api_tokens.sql b/backend/migrations/0040_api_tokens.sql new file mode 100644 index 00000000..df17efa9 --- /dev/null +++ b/backend/migrations/0040_api_tokens.sql @@ -0,0 +1,12 @@ +-- MCP API tokens. We store only a SHA-256 hash of the token; the plaintext is +-- shown to the operator exactly once at mint time and is never recoverable. +CREATE TABLE IF NOT EXISTS api_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_used_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_api_tokens_user_id ON api_tokens(user_id); diff --git a/backend/migrations/0041_mcp_deletion_batches.sql b/backend/migrations/0041_mcp_deletion_batches.sql new file mode 100644 index 00000000..cf3b77c4 --- /dev/null +++ b/backend/migrations/0041_mcp_deletion_batches.sql @@ -0,0 +1,21 @@ +-- Staged (not executed) MCP deletions. stage_deletion records a batch; a separate +-- session-authenticated execute step flips messages.is_deleted (soft delete). No +-- tool ever hard-deletes. Renumbered to 0041 (0040 is api_tokens) per the README +-- migration-numbering rule. +CREATE TABLE IF NOT EXISTS mcp_deletion_batches ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + description TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'staged', + message_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + executed_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS mcp_deletion_batch_messages ( + batch_id UUID NOT NULL REFERENCES mcp_deletion_batches(id) ON DELETE CASCADE, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + PRIMARY KEY (batch_id, message_id) +); + +CREATE INDEX IF NOT EXISTS idx_mcp_deletion_batches_user_id ON mcp_deletion_batches(user_id); diff --git a/backend/package-lock.json b/backend/package-lock.json index dd3e503a..3debd3dc 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,6 +8,7 @@ "name": "mailflow-backend", "version": "2.6.0", "dependencies": { + "@modelcontextprotocol/sdk": "^1.29.0", "archiver": "^7.0.1", "bcryptjs": "^2.4.3", "connect-redis": "^7.1.0", @@ -263,6 +264,18 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -432,6 +445,398 @@ "dev": true, "license": "MIT" }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -1117,6 +1522,45 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2270,6 +2714,27 @@ "bare-events": "^2.7.0" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -2335,6 +2800,24 @@ "express": "^4.16.2" } }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/express-session": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.19.0.tgz", @@ -2362,7 +2845,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -2385,6 +2867,22 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-uri": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", + "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-xml-builder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", @@ -2709,6 +3207,15 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.12.30", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", + "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/htmlparser2": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", @@ -2955,6 +3462,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -3029,6 +3542,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -3698,6 +4217,15 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3991,6 +4519,15 @@ "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", "license": "MIT" }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/pngjs": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", @@ -4274,6 +4811,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -4314,6 +4860,55 @@ "@rolldown/binding-win32-x64-msvc": "1.0.3" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/router/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/router/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/router/node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -5164,6 +5759,12 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/ws": { "version": "8.21.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", @@ -5282,6 +5883,24 @@ "engines": { "node": ">= 14" } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/backend/package.json b/backend/package.json index 145125a0..2450e77e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,6 +11,7 @@ "audit:redos": "npm i --no-save --silent eslint-plugin-redos && eslint -c eslint.redos.config.mjs src" }, "dependencies": { + "@modelcontextprotocol/sdk": "^1.29.0", "archiver": "^7.0.1", "bcryptjs": "^2.4.3", "connect-redis": "^7.1.0", diff --git a/backend/src/index.js b/backend/src/index.js index fbf579de..98b51b7f 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -31,6 +31,9 @@ import categoriesRoutes from './routes/categories.js'; import gtdRoutes from './routes/gtd.js'; import carddavRouter from './routes/carddav.js'; import carddavAccountRouter from './routes/carddavAccount.js'; +import { mountMcp } from './mcp/server.js'; +import apiTokensRoutes from './routes/apiTokens.js'; +import mcpDeletionsRoutes from './routes/mcpDeletions.js'; import { startCardavScheduler } from './services/carddavSync.js'; import { scheduleFtsBackfill } from './services/search/ftsBackfill.js'; import { encryptExistingCredentials, query } from './services/db.js'; @@ -181,6 +184,8 @@ app.use('/api/todoist', todoistRoutes); app.use('/api/carddav', carddavAccountRouter); app.use('/api', aiRoutes); app.use('/api', aiEmbeddingsRoutes); +app.use('/api/tokens', apiTokensRoutes); +app.use('/api/mcp-deletions', mcpDeletionsRoutes); app.use('/api', categoriesRoutes); // Mounted at the /api/gtd subtree (not bare /api) so gtd.js's router-level // requireAuth cannot intercept the unauthenticated /api/health and /api/version @@ -192,6 +197,10 @@ app.use('/carddav', carddavRouter); // RFC 6764 well-known redirect — handle all methods so PROPFIND probes also redirect app.all('/.well-known/carddav', (req, res) => res.redirect(308, '/carddav/')); +// MCP Streamable-HTTP endpoint. Bearer-authenticated (mcp/auth.js), intentionally +// outside the /api CSRF gate and session middleware — auth is a token, not a cookie. +mountMcp(app); + app.get('/api/health', (req, res) => res.json({ status: 'ok' })); app.get('/api/version', (_req, res) => res.json({ version: APP_VERSION, sha: process.env.BUILD_SHA || 'dev' })); // Server-side update check (#261). Cached in updateCheck.js so repeated hits never diff --git a/backend/src/mcp/auth.js b/backend/src/mcp/auth.js new file mode 100644 index 00000000..10addcbf --- /dev/null +++ b/backend/src/mcp/auth.js @@ -0,0 +1,45 @@ +import crypto from 'crypto'; +import { query } from '../services/db.js'; + +// Plaintext tokens are shown once at mint time; we persist only the hash. +export function generateToken() { + return 'mcp_' + crypto.randomBytes(32).toString('base64url'); +} + +export function hashToken(plaintext) { + return crypto.createHash('sha256').update(plaintext, 'utf8').digest('hex'); +} + +// The scope object pins multi-user isolation: every MCP tool call is bounded to +// exactly this user's enabled accounts. msgvault is single-archive; Mailflow is not. +export async function resolveScope(userId) { + const { rows } = await query( + 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId], + ); + return { userId, accountIds: rows.map((r) => r.id) }; +} + +export async function mcpBearerAuth(req, res, next) { + try { + const header = req.get('Authorization') || ''; + const m = /^Bearer\s+(.+)$/i.exec(header); + if (!m) return res.status(401).json({ error: 'invalid_token' }); + + const { rows } = await query( + 'SELECT id, user_id FROM api_tokens WHERE token_hash = $1', + [hashToken(m[1].trim())], + ); + if (!rows.length) return res.status(401).json({ error: 'invalid_token' }); + + // Best-effort recency stamp; never block the request on it. + await query('UPDATE api_tokens SET last_used_at = NOW() WHERE id = $1', [rows[0].id]) + .catch(() => {}); + + req.mcpTokenId = rows[0].id; // rate-limit key: per token, not per IP + req.mcpScope = await resolveScope(rows[0].user_id); + next(); + } catch (err) { + next(err); + } +} diff --git a/backend/src/mcp/auth.test.js b/backend/src/mcp/auth.test.js new file mode 100644 index 00000000..9b582740 --- /dev/null +++ b/backend/src/mcp/auth.test.js @@ -0,0 +1,87 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { generateToken, hashToken, resolveScope, mcpBearerAuth } from './auth.js'; + +function mockRes() { + return { statusCode: 200, body: null, status(c){this.statusCode=c;return this;}, json(b){this.body=b;return this;} }; +} + +describe('token helpers', () => { + it('generateToken is prefixed and unique', () => { + const a = generateToken(); const b = generateToken(); + expect(a).toMatch(/^mcp_[A-Za-z0-9_-]{20,}$/); + expect(a).not.toEqual(b); + }); + it('hashToken is deterministic hex SHA-256 and hides the plaintext', () => { + const h = hashToken('mcp_secret'); + expect(h).toMatch(/^[0-9a-f]{64}$/); + expect(h).toEqual(hashToken('mcp_secret')); + expect(h).not.toContain('secret'); + }); +}); + +describe('resolveScope', () => { + it("returns only the token owner's enabled account ids", async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1' }, { id: 'acc-2' }] }); + const scope = await resolveScope('user-1'); + expect(scope).toEqual({ userId: 'user-1', accountIds: ['acc-1', 'acc-2'] }); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/FROM email_accounts WHERE user_id = \$1 AND enabled = true/), + ['user-1'], + ); + }); +}); + +describe('mcpBearerAuth', () => { + beforeEach(() => query.mockReset()); + + it('rejects a request with no Authorization header', async () => { + const req = { get: () => undefined }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(res.body).toEqual({ error: 'invalid_token' }); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects a malformed (non-Bearer) header', async () => { + const req = { get: () => 'Basic abc' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects an unknown / revoked token', async () => { + query.mockResolvedValueOnce({ rows: [] }); // no api_tokens row + const req = { get: () => 'Bearer mcp_revoked' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('accepts a valid token and attaches the user-scoped account ids', async () => { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok-1', user_id: 'user-1' }] }) // token lookup + .mockResolvedValueOnce({ rows: [] }) // last_used_at UPDATE + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); // resolveScope + const req = { get: (h) => (h === 'Authorization' ? 'Bearer mcp_good' : undefined) }; + const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(next).toHaveBeenCalledOnce(); + expect(req.mcpScope).toEqual({ userId: 'user-1', accountIds: ['acc-1'] }); + // token lookup must be by HASH, never the plaintext + expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); + }); + + it('isolates users: the scope carries only the resolved owner', async () => { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok-2', user_id: 'user-2' }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ id: 'acc-9' }] }); + const req = { get: () => 'Bearer mcp_u2' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(req.mcpScope.userId).toBe('user-2'); + expect(req.mcpScope.accountIds).toEqual(['acc-9']); + }); +}); diff --git a/backend/src/mcp/bodyMatch.js b/backend/src/mcp/bodyMatch.js new file mode 100644 index 00000000..a73b4a0b --- /dev/null +++ b/backend/src/mcp/bodyMatch.js @@ -0,0 +1,78 @@ +// All offsets are UTF-8 BYTES (msgvault wire contract). We operate on Buffers, +// never JS string .length (UTF-16 code units). Case-insensitive matching mirrors +// msgvault: find in the lowercased buffer, slice the original at the same offsets +// (ASCII-faithful; identical behavior to strings.ToLower + strings.Index in Go). +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from '../utils/textExcerpt.js'; + +export function contextWindow(bodyLen, pos, termLen, contextChars) { + let start = pos - Math.floor((contextChars - termLen) / 2); + let end = start + contextChars; + if (start < 0) { + start = 0; + end = Math.min(bodyLen, contextChars); + } else if (end > bodyLen) { + end = bodyLen; + start = Math.max(0, end - contextChars); + } + return [start, end]; +} + +export function bodyByteSliceRange(buf, start, end) { + if (start < 0) start = 0; + if (end > buf.length) end = buf.length; + if (start >= buf.length) return { text: '', adjStart: buf.length, adjEnd: buf.length }; + let adjStart = start; + let adjEnd = end <= start ? Math.min(buf.length, start + 1) : end; + while (adjStart < adjEnd && !isRuneStart(buf[adjStart])) adjStart++; + while (adjEnd > adjStart && adjEnd < buf.length && !isRuneStart(buf[adjEnd])) adjEnd--; + return { text: buf.toString('utf8', adjStart, adjEnd), adjStart, adjEnd }; +} + +function bodyByteSlice(buf, start, end) { + return bodyByteSliceRange(buf, start, end).text; +} + +export function findTermMatches(body, term) { + if (!body || !term) return []; + const buf = Buffer.from(body, 'utf8'); + const lower = Buffer.from(body.toLowerCase(), 'utf8'); + const t = Buffer.from(term.toLowerCase(), 'utf8'); + const termLen = t.length; + const matches = []; + let from = 0; + for (;;) { + const idx = lower.indexOf(t, from); + if (idx < 0) break; + from = idx + 1; + const [start, end] = contextWindow(buf.length, idx, termLen, SNIPPET_BYTES); + matches.push({ char_offset: idx, snippet: bodyByteSlice(buf, start, end), line: lineNumberAt(buf, idx) }); + } + return matches; +} + +export function extractContextChar(body, terms, contextChars) { + if (!body || !terms || !terms.length || contextChars <= 0) return null; + const buf = Buffer.from(body, 'utf8'); + const lower = Buffer.from(body.toLowerCase(), 'utf8'); + const spans = []; + for (const term of terms) { + if (!term || term.length < 2) continue; + const t = Buffer.from(term.toLowerCase(), 'utf8'); + let from = 0; + for (;;) { + const idx = lower.indexOf(t, from); + if (idx < 0) break; + from = idx + 1; + spans.push(contextWindow(buf.length, idx, t.length, contextChars)); + } + } + if (!spans.length) return null; + spans.sort((a, b) => (a[0] === b[0] ? a[1] - b[1] : a[0] - b[0])); + const merged = [spans[0].slice()]; + for (const s of spans.slice(1)) { + const last = merged[merged.length - 1]; + if (s[0] <= last[1]) last[1] = Math.max(last[1], s[1]); + else merged.push(s.slice()); + } + return merged.map(([s, e]) => bodyByteSlice(buf, s, e)); +} diff --git a/backend/src/mcp/bodyMatch.test.js b/backend/src/mcp/bodyMatch.test.js new file mode 100644 index 00000000..b494a8b4 --- /dev/null +++ b/backend/src/mcp/bodyMatch.test.js @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; +import { contextWindow, bodyByteSliceRange, extractContextChar, findTermMatches } from './bodyMatch.js'; + +describe('contextWindow', () => { + it('centers within bounds', () => { + expect(contextWindow(1000, 500, 10, 300)).toEqual([355, 655]); + }); + it('clamps at the start', () => { + expect(contextWindow(1000, 5, 10, 300)).toEqual([0, 300]); + }); + it('clamps at the end', () => { + expect(contextWindow(300, 290, 4, 300)).toEqual([0, 300]); + }); +}); + +describe('byte-offset fidelity on multibyte bodies', () => { + // "café — " then the term. 'é' is 2 bytes (0xC3 0xA9), '—' is 3 bytes. + const body = 'café — meeting notes'; + it('findTermMatches reports the BYTE offset of the term, not the code-unit index', () => { + const m = findTermMatches(body, 'meeting'); + // "café — " = c a f é(2) space —(3) space = 1+1+1+2+1+3+1 = 10 bytes + expect(m).toHaveLength(1); + expect(m[0].char_offset).toBe(10); + expect(m[0].line).toBe(1); + expect(m[0].snippet).toContain('meeting'); + }); + it('bodyByteSliceRange never splits a rune', () => { + const buf = Buffer.from(body, 'utf8'); + // Ask for a window that would cut the 'é' (bytes 3-4) in half at byte 4. + const { text } = bodyByteSliceRange(buf, 0, 4); + expect(Buffer.from(text, 'utf8').every((b) => b !== undefined)).toBe(true); + expect(text).toBe('caf'); // é dropped rather than split + }); +}); + +describe('extractContextChar', () => { + it('merges overlapping windows into snippet strings', () => { + const body = 'alpha beta alpha'; + const snippets = extractContextChar(body, ['alpha'], 300); + expect(snippets).toHaveLength(1); // both hits merge into one window + expect(snippets[0]).toContain('alpha'); + }); + it('ignores terms shorter than 2 chars', () => { + expect(extractContextChar('a a a', ['a'], 300)).toBeNull(); + }); +}); diff --git a/backend/src/mcp/engineAdapter.js b/backend/src/mcp/engineAdapter.js new file mode 100644 index 00000000..034bfc03 --- /dev/null +++ b/backend/src/mcp/engineAdapter.js @@ -0,0 +1,411 @@ +// The query.Engine port: all message/aggregate/deletion SQL lives here, scoped by +// accountIds, so MCP handlers never touch db.js (the no-SQL-in-handlers invariant). +// Row → msgvault-shape mappers pin the wire-casing split: MessageSummary/Detail carry +// snake_case json keys, but Address/AttachmentInfo/AccountInfo/AggregateRow/TotalStats +// have NO Go json tags → Go-default CAPITALIZED keys. D6: ids are UUID strings, +// conversation_id = thread_id. +import { query, withTransaction } from '../services/db.js'; +import { buildOperatorClauses, freeTextTermClause, hasSearchableToken } from '../services/search/lexicalRepo.js'; + +const SUMMARY_COLUMNS = ` + m.id, m.account_id, m.message_id, m.thread_id, m.subject, m.snippet, + m.from_email, m.from_name, m.to_addresses, m.cc_addresses, m.date, + m.has_attachments, m.attachments, m.flags, m.folder`; + +const DETAIL_COLUMNS = SUMMARY_COLUMNS + `, m.body_text, m.body_html`; + +export function mapAddrs(jsonb) { + return (jsonb || []).map((a) => ({ Email: a.email || '', Name: a.name || '' })); +} + +// msgvault labels ≈ Gmail labels; Mailflow's closest analogue is the folder plus IMAP flags. +export function mapLabels(row) { + const flags = Array.isArray(row.flags) ? row.flags : []; + return [row.folder, ...flags].filter(Boolean); +} + +function toISO(d) { + return d instanceof Date ? d.toISOString() : (d || ''); +} + +export function rowToMessageSummary(row) { + const atts = Array.isArray(row.attachments) ? row.attachments : []; + const s = { + id: row.id, + source_id: row.account_id, + source_message_id: row.message_id || '', + conversation_id: row.thread_id || '', + source_conversation_id: row.thread_id || '', + subject: row.subject || '', + snippet: row.snippet || '', + from_email: row.from_email || '', + from_name: row.from_name || '', + sent_at: toISO(row.date), + size_estimate: 0, // divergence: Mailflow stores no byte size + has_attachments: !!row.has_attachments, + attachment_count: atts.length, + labels: mapLabels(row), + message_type: 'email', + }; + const to = mapAddrs(row.to_addresses); + const cc = mapAddrs(row.cc_addresses); + if (to.length) s.to = to; // omitempty parity + if (cc.length) s.cc = cc; + return s; +} + +function mapAttachments(atts) { + return (Array.isArray(atts) ? atts : []).map((a, i) => ({ + ID: i, Filename: a.filename || '', MimeType: a.type || '', Size: a.size || 0, + ContentHash: '', URL: '', StoragePath: '', // content tools are non-goals; synthetic index id + })); +} + +export function rowToMessageDetail(row) { + const base = rowToMessageSummary(row); + return { + ...base, + from: row.from_email ? [{ Email: row.from_email, Name: row.from_name || '' }] : [], + to: mapAddrs(row.to_addresses), + cc: mapAddrs(row.cc_addresses), + bcc: [], + body_text: row.body_text || '', + body_html: row.body_html || '', + attachments: mapAttachments(row.attachments), + }; +} + +// Raw detail row (not yet mapped) for one message, scoped. Handlers map/page it. +export async function getMessage(id, accountIds) { + if (!accountIds || !accountIds.length) return null; + const { rows } = await query( + `SELECT ${DETAIL_COLUMNS} FROM messages m WHERE m.id = $1 AND m.account_id = ANY($2)`, + [id, accountIds], + ); + return rows[0] || null; +} + +export async function getMessageSummariesByIDs(ids, accountIds) { + if (!ids || !ids.length) return []; + const { rows } = await query( + `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE m.id = ANY($1) AND m.account_id = ANY($2)`, + [ids, accountIds], + ); + const byId = new Map(rows.map((r) => [r.id, rowToMessageSummary(r)])); + return ids.map((id) => byId.get(id)).filter(Boolean); // preserve input order +} + +export async function getMessageBodiesByIDs(ids, accountIds) { + const out = new Map(); + if (!ids || !ids.length) return out; + const { rows } = await query( + `SELECT m.id, m.body_text, m.body_html FROM messages m WHERE m.id = ANY($1) AND m.account_id = ANY($2)`, + [ids, accountIds], + ); + for (const r of rows) out.set(r.id, { body_text: r.body_text, body_html: r.body_html }); + return out; +} + +export async function listAccounts(accountIds) { + if (!accountIds || !accountIds.length) return []; + const { rows } = await query( + `SELECT id, protocol, email_address, name FROM email_accounts WHERE id = ANY($1) ORDER BY sort_order`, + [accountIds], + ); + return rows.map((a) => ({ ID: a.id, SourceType: a.protocol || 'imap', Identifier: a.email_address, DisplayName: a.name || '' })); +} + +// Resolve an optional `account` email to a single scoped id (msgvault getAccountID). +// Returns { accountIds } narrowed to the match, or { error } when unknown (msgvault +// errors rather than silently widening). Empty account = no narrowing (all of the +// token's accounts). Shared by the search AND message/aggregate/deletion handlers so +// the `account` argument narrows scope everywhere it is advertised. +export async function resolveAccountScope(account, accountIds) { + if (!account) return { accountIds }; + const accounts = await listAccounts(accountIds); + const match = accounts.find((a) => a.Identifier === account); + if (!match) return { error: `account not found: ${account}` }; + return { accountIds: [match.ID] }; +} + +// Cheap owner-scope membership check for a single message id (find_similar seed). +export async function messageInScope(id, accountIds) { + if (!accountIds || !accountIds.length) return false; + const { rows } = await query('SELECT 1 FROM messages WHERE id = $1 AND account_id = ANY($2) LIMIT 1', [id, accountIds]); + return rows.length > 0; +} + +// Port of msgvault getDateArg (internal/mcp/handlers.go): structured after/before +// tool args are strict YYYY-MM-DD (the format every tool schema advertises). An +// unparseable value throws msgvault's exact error message instead of surfacing a +// raw Postgres timestamptz cast failure; a valid one binds as midnight UTC — the +// same instant lexicalRepo's buildOperatorClauses gives query-string dates — so +// structured args and before:/after: operators sit on one clock. Callers apply +// `before` EXCLUSIVELY (<): msgvault uses < and lexicalRepo already does; the +// list/aggregate/domain paths here had drifted to <=. +function parseDateArg(key, value) { + if (!value) return null; + const d = /^\d{4}-\d{2}-\d{2}$/.test(value) ? new Date(`${value}T00:00:00.000Z`) : null; + // Round-trip guard: Date.parse ROLLS an out-of-range day over ("2025-02-31" + // → Mar 2) instead of rejecting it; msgvault's time.Parse rejects. + if (!d || isNaN(d) || !d.toISOString().startsWith(value)) { + throw new Error(`invalid ${key} date "${value}": expected YYYY-MM-DD`); + } + return d.toISOString(); +} + +// Shared by every structured-arg path below: validate both bounds up front, +// then push the >= / < clauses through the caller's bind. +function pushDateClauses(where, bind, after, before) { + const afterISO = parseDateArg('after', after); + const beforeISO = parseDateArg('before', before); + if (afterISO) where.push(`m.date >= ${bind(afterISO)}`); + if (beforeISO) where.push(`m.date < ${bind(beforeISO)}`); +} + +// Newest-first message list, scoped and filtered. Returns MessageSummary[] (mapped +// here so the handler only pages). `from` filters from_email when it looks like an +// address, else from_name (msgvault list_messages semantics). +export async function listMessages({ accountIds, from, to, label, hasAttachment, after, before, conversationId, limit, offset }) { + if (!accountIds || !accountIds.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + if (from) { + where.push(from.includes('@') + ? `m.from_email ILIKE ${bind('%' + from + '%')}` + : `m.from_name ILIKE ${bind('%' + from + '%')}`); + } + if (to) where.push(`m.to_addresses::text ILIKE ${bind('%' + to + '%')}`); + if (label) where.push(`(m.folder = ${bind(label)} OR m.flags @> ${bind(JSON.stringify([label]))}::jsonb)`); + if (hasAttachment) where.push('m.has_attachments = true'); + pushDateClauses(where, bind, after, before); + if (conversationId) where.push(`m.thread_id = ${bind(conversationId)}`); + + const sql = `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} + ORDER BY m.date DESC NULLS LAST LIMIT ${bind(limit)} OFFSET ${bind(offset)}`; + const { rows } = await query(sql, args); + return rows.map(rowToMessageSummary); +} + +const MAX_STAGE_DELETION = 100000; // msgvault maxStageDeletionResults + +// A free-text term contributes a staging predicate only when it is positive, +// ≥2 chars, and carries a searchable token — the same hygiene the lexical and +// semantic paths apply. Negated terms are deliberately NOT enforced on the +// staging path (unlike lexical search's FTS exclusion), so a negation-only +// query yields zero predicates; countEnforceableQueryPredicates below is what +// lets the handler refuse that rather than stage every live message. +function usablePositiveTerm(t) { + return !t.negate && t.value.length >= 2 && hasSearchableToken(t.value); +} + +// Count the enforceable row predicates a parsed query would contribute to a +// staging WHERE: supported structured filters (buildOperatorClauses; in:/ +// unsupported operators contribute nothing) plus usable positive free-text +// terms. Mirrors resolveStageDeletionIds' predicate construction exactly (same +// builder, same term hygiene) so the two can never disagree about whether a +// query is enforceable. Zero here means the WHERE would carry only account + +// liveness — i.e. the query would soft-delete the whole (capped) mailbox — so +// the stage_deletion handler refuses it. This is the final safety net, reached +// only for queries that carry no unsupported operator yet still yield no +// predicate (e.g. all stopwords or punctuation): the handler already rejects any +// unsupported operator up front via unsupportedSearchOperatorMessage (msgvault +// parity, internal/mcp/handlers.go — so a mixed query like `invoice +// label:promotions` is refused, not silently widened to every "invoice" match). +export function countEnforceableQueryPredicates(parsed) { + if (!parsed) return 0; + let n = buildOperatorClauses(parsed.filters, () => '$0').length; + for (const t of parsed.terms || []) if (usablePositiveTerm(t)) n++; + return n; +} + +// Resolve candidate message ids for staging, scoped to accountIds. Query path builds +// its free-text predicates from lexicalRepo's freeTextTermClause — the EXACT builder +// the search path uses (ranked search_fts match + un-backfilled fallback + stopword +// vacuity) — so the search preview and the staged set can never diverge: `the invoice` +// stages the invoice matches the preview showed, not the zero rows the legacy +// ILIKE/plainto fork produced once "the" normalized to an empty tsquery. Structured +// operators share buildOperatorClauses the same way; the structured-filter path +// builds from the discrete filter args. +async function resolveStageDeletionIds({ accountIds, parsed, from, domain, label, hasAttachment, after, before }) { + const params = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { params.push(v); return `$${params.length}`; }; + + if (parsed) { + for (const cond of buildOperatorClauses(parsed.filters, bind)) where.push(cond); + for (const t of parsed.terms || []) { + if (!usablePositiveTerm(t)) continue; + where.push(freeTextTermClause(t.value, false, bind)); + } + } else { + if (from) { const p = bind('%' + from + '%'); where.push(`(m.from_email ILIKE ${p} OR m.from_name ILIKE ${p})`); } + if (domain) where.push(`m.from_email ILIKE ${bind('%@' + domain)}`); + if (label) where.push(`(m.folder = ${bind(label)} OR m.flags @> ${bind(JSON.stringify([label]))}::jsonb)`); + if (hasAttachment) where.push('m.has_attachments = true'); + pushDateClauses(where, bind, after, before); + } + + const sql = `SELECT m.id FROM messages m WHERE ${where.join(' AND ')} LIMIT ${bind(MAX_STAGE_DELETION)}`; + const { rows } = await query(sql, params); + return rows.map((r) => r.id); +} + +// Record a STAGED batch + its members scoped to the token user. NEVER flips +// is_deleted — execution is a separate, session-authed step. +export async function stageDeletion(opts) { + const ids = await resolveStageDeletionIds(opts); + if (!ids.length) return { batchId: null, messageCount: 0 }; + return withTransaction(async (client) => { + const { rows } = await client.query( + "INSERT INTO mcp_deletion_batches (user_id, description, status, message_count) VALUES ($1, $2, 'staged', $3) RETURNING id", + [opts.userId, opts.description || '', ids.length], + ); + const batchId = rows[0].id; + await client.query( + 'INSERT INTO mcp_deletion_batch_messages (batch_id, message_id) SELECT $1, unnest($2::uuid[])', + [batchId, ids], + ); + return { batchId, messageCount: ids.length }; + }); +} + +// Session-authed soft-delete of a STAGED batch, scoped to the owner's accounts. +// Returns the updated row count, or null when the batch is absent/not owned/not staged. +export async function executeDeletionBatch(batchId, userId) { + return withTransaction(async (client) => { + const { rows } = await client.query( + "SELECT id FROM mcp_deletion_batches WHERE id = $1 AND user_id = $2 AND status = 'staged' FOR UPDATE", + [batchId, userId], + ); + if (!rows.length) return null; + const upd = await client.query( + `UPDATE messages SET is_deleted = true + WHERE id IN (SELECT message_id FROM mcp_deletion_batch_messages WHERE batch_id = $1) + AND account_id IN (SELECT id FROM email_accounts WHERE user_id = $2)`, + [batchId, userId], + ); + await client.query( + "UPDATE mcp_deletion_batches SET status = 'executed', executed_at = NOW() WHERE id = $1", + [batchId], + ); + return upd.rowCount; + }); +} + +// Discard a staged batch (owner-scoped). Cascades member rows; never touches messages. +export async function unstageDeletionBatch(batchId, userId) { + const { rowCount } = await query( + 'DELETE FROM mcp_deletion_batches WHERE id = $1 AND user_id = $2', + [batchId, userId], + ); + return rowCount > 0; +} + +// Archive overview scoped to accountIds. TotalStats has NO Go json tags → CAPITALIZED +// keys. TotalSize is a body-byte proxy and AttachmentSize is 0 (Mailflow stores no +// byte sizes — documented divergence). +export async function getTotalStats(accountIds) { + const empty = { + MessageCount: 0, ActiveMessageCount: 0, SourceDeletedMessageCount: 0, + TotalSize: 0, AttachmentCount: 0, AttachmentSize: 0, LabelCount: 0, + AccountCount: (accountIds && accountIds.length) || 0, + }; + if (!accountIds || !accountIds.length) return empty; + const { rows } = await query( + `SELECT + COUNT(*)::bigint AS message_count, + COUNT(*) FILTER (WHERE is_deleted = false)::bigint AS active_count, + COUNT(*) FILTER (WHERE is_deleted = true)::bigint AS deleted_count, + COALESCE(SUM(octet_length(COALESCE(body_text, ''))), 0)::bigint AS total_size, + COALESCE(SUM(jsonb_array_length(COALESCE(attachments, '[]'::jsonb))), 0)::bigint AS attachment_count, + COUNT(DISTINCT folder)::bigint AS label_count + FROM messages WHERE account_id = ANY($1)`, + [accountIds], + ); + const r = rows[0] || {}; + return { + MessageCount: Number(r.message_count || 0), + ActiveMessageCount: Number(r.active_count || 0), + SourceDeletedMessageCount: Number(r.deleted_count || 0), + TotalSize: Number(r.total_size || 0), + AttachmentCount: Number(r.attachment_count || 0), + AttachmentSize: 0, + LabelCount: Number(r.label_count || 0), + AccountCount: accountIds.length, + }; +} + +// group_by → grouping SQL. `recipient` needs a lateral unnest of to_addresses; +// the rest are scalar expressions over messages. `time` buckets by calendar year. +const AGG_KEY_EXPR = { + sender: 'm.from_email', + domain: "split_part(m.from_email, '@', 2)", + label: 'm.folder', + time: "to_char(m.date, 'YYYY')", +}; + +// Grouped statistics (top senders/recipients/domains/labels, or volume by year). +// Returns AggregateRow[] — capitalized keys (no Go json tags). AttachmentSize is 0 +// (Mailflow stores no per-attachment byte totals — documented divergence). +export async function aggregate(groupBy, { accountIds, after, before, limit }) { + if (!accountIds || !accountIds.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + pushDateClauses(where, bind, after, before); + + const measures = ` + COUNT(*)::bigint AS count, + COALESCE(SUM(octet_length(COALESCE(m.body_text, ''))), 0)::bigint AS total_size, + COALESCE(SUM(jsonb_array_length(COALESCE(m.attachments, '[]'::jsonb))), 0)::bigint AS attachment_count, + COUNT(*) OVER ()::bigint AS total_unique`; + + let sql; + if (groupBy === 'recipient') { + sql = `SELECT (ra->>'email') AS key, ${measures} + FROM messages m, LATERAL jsonb_array_elements(COALESCE(m.to_addresses, '[]'::jsonb)) AS ra + WHERE ${where.join(' AND ')} + GROUP BY key ORDER BY count DESC LIMIT ${bind(limit)}`; + } else { + const expr = AGG_KEY_EXPR[groupBy]; + sql = `SELECT ${expr} AS key, ${measures} + FROM messages m + WHERE ${where.join(' AND ')} + GROUP BY key ORDER BY count DESC LIMIT ${bind(limit)}`; + } + const { rows } = await query(sql, args); + return rows.map((r) => ({ + Key: r.key == null ? '' : String(r.key), + Count: Number(r.count || 0), + TotalSize: Number(r.total_size || 0), + AttachmentSize: 0, + AttachmentCount: Number(r.attachment_count || 0), + TotalUnique: Number(r.total_unique || 0), + })); +} + +// Messages where any participant (from/to/cc) belongs to one of the domains. +// Returns MessageSummary[], newest-first, scoped to accountIds. +export async function searchByDomains(domains, after, before, limit, offset, accountIds) { + if (!accountIds || !accountIds.length || !domains.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + const domainConds = domains.map((d) => { + const from = bind('%@' + d); + const to = bind('%' + d + '%'); + const cc = bind('%' + d + '%'); + return `(m.from_email ILIKE ${from} OR m.to_addresses::text ILIKE ${to} OR m.cc_addresses::text ILIKE ${cc})`; + }); + where.push('(' + domainConds.join(' OR ') + ')'); + pushDateClauses(where, bind, after, before); + + const sql = `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} + ORDER BY m.date DESC NULLS LAST LIMIT ${bind(limit)} OFFSET ${bind(offset)}`; + const { rows } = await query(sql, args); + return rows.map(rowToMessageSummary); +} diff --git a/backend/src/mcp/engineAdapter.test.js b/backend/src/mcp/engineAdapter.test.js new file mode 100644 index 00000000..5f352705 --- /dev/null +++ b/backend/src/mcp/engineAdapter.test.js @@ -0,0 +1,309 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +import { query, withTransaction } from '../services/db.js'; +import { rowToMessageSummary, rowToMessageDetail, mapAddrs, getMessageSummariesByIDs, listAccounts, listMessages, getTotalStats, aggregate, searchByDomains, stageDeletion, resolveAccountScope, messageInScope, countEnforceableQueryPredicates } from './engineAdapter.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { freeTextTermClause, searchLexical } from '../services/search/lexicalRepo.js'; + +const row = { + id: '11111111-1111-1111-1111-111111111111', account_id: 'acc-1', message_id: '', + thread_id: 'tid-9', subject: 'Hi', snippet: 's', from_email: 'a@b.com', from_name: 'A', + to_addresses: [{ name: 'C', email: 'c@d.com' }], cc_addresses: [], + date: '2024-01-01T00:00:00.000Z', has_attachments: true, + attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], + flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello', body_html: '

hello

', +}; + +describe('mapAddrs', () => { + it('emits capitalized Email/Name keys (msgvault no-json-tag quirk)', () => { + expect(mapAddrs([{ name: 'C', email: 'c@d.com' }])).toEqual([{ Email: 'c@d.com', Name: 'C' }]); + expect(mapAddrs(null)).toEqual([]); + }); +}); + +describe('rowToMessageSummary', () => { + it('maps thread_id to conversation_id and keeps the uuid id as a string', () => { + const s = rowToMessageSummary(row); + expect(s.id).toBe('11111111-1111-1111-1111-111111111111'); + expect(s.conversation_id).toBe('tid-9'); + expect(s.source_conversation_id).toBe('tid-9'); + expect(s.source_message_id).toBe(''); + expect(s.source_id).toBe('acc-1'); + expect(s.to).toEqual([{ Email: 'c@d.com', Name: 'C' }]); + expect(s.attachment_count).toBe(1); + expect(s.size_estimate).toBe(0); + expect(s.labels).toContain('INBOX'); + expect(s.labels).toContain('\\Seen'); + expect(s.message_type).toBe('email'); + }); + + it('omits empty to/cc (omitempty parity)', () => { + const s = rowToMessageSummary({ ...row, to_addresses: [], cc_addresses: [] }); + expect(s).not.toHaveProperty('to'); + expect(s).not.toHaveProperty('cc'); + }); +}); + +describe('rowToMessageDetail', () => { + it('adds body + capitalized AttachmentInfo', () => { + const d = rowToMessageDetail(row); + expect(d.body_text).toBe('hello'); + expect(d.body_html).toBe('

hello

'); + expect(d.from).toEqual([{ Email: 'a@b.com', Name: 'A' }]); + expect(d.attachments[0]).toEqual({ ID: 0, Filename: 'f.pdf', MimeType: 'application/pdf', Size: 10, ContentHash: '', URL: '', StoragePath: '' }); + }); +}); + +describe('getMessageSummariesByIDs', () => { + beforeEach(() => query.mockReset()); + it('scopes to accountIds and preserves input order', async () => { + query.mockResolvedValueOnce({ rows: [{ ...row, id: 'b' }, { ...row, id: 'a' }] }); + const out = await getMessageSummariesByIDs(['a', 'b'], ['acc-1']); + expect(out.map((m) => m.id)).toEqual(['a', 'b']); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = ANY/); + expect(params[1]).toContain('acc-1'); // accountIds is the array bound to ANY($2) + }); + it('returns [] for empty ids without hitting the DB', async () => { + expect(await getMessageSummariesByIDs([], ['acc-1'])).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('listAccounts', () => { + beforeEach(() => query.mockReset()); + it('maps to capitalized AccountInfo scoped to accountIds', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'Work' }] }); + const out = await listAccounts(['acc-1']); + expect(out).toEqual([{ ID: 'acc-1', SourceType: 'imap', Identifier: 'a@b.com', DisplayName: 'Work' }]); + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); + }); + it('returns [] for empty scope without querying', async () => { + expect(await listAccounts([])).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('resolveAccountScope', () => { + beforeEach(() => query.mockReset()); + it('no account → full scope, no lookup', async () => { + expect(await resolveAccountScope('', ['acc-1', 'acc-2'])).toEqual({ accountIds: ['acc-1', 'acc-2'] }); + expect(query).not.toHaveBeenCalled(); + }); + it('known account email → narrowed to its id', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-2', protocol: 'imap', email_address: 'c@d.com', name: 'W' }] }); + expect(await resolveAccountScope('c@d.com', ['acc-2'])).toEqual({ accountIds: ['acc-2'] }); + }); + it('unknown account email → error (msgvault getAccountID parity)', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'W' }] }); + expect(await resolveAccountScope('nope@x.com', ['acc-1'])).toEqual({ error: 'account not found: nope@x.com' }); + }); +}); + +describe('messageInScope', () => { + beforeEach(() => query.mockReset()); + it('true when the id belongs to one of the accounts, scoped', async () => { + query.mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }); + expect(await messageInScope('m1', ['acc-1'])).toBe(true); + expect(query.mock.calls[0][1]).toEqual(['m1', ['acc-1']]); + }); + it('false for a foreign/absent id', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await messageInScope('m1', ['acc-1'])).toBe(false); + }); + it('false for empty scope without querying', async () => { + expect(await messageInScope('m1', [])).toBe(false); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('getTotalStats', () => { + beforeEach(() => query.mockReset()); + it('returns capitalized TotalStats keys scoped to accountIds', async () => { + query.mockResolvedValueOnce({ rows: [{ message_count: '10', active_count: '9', deleted_count: '1', total_size: '5000', attachment_count: '3', label_count: '2' }] }); + const s = await getTotalStats(['acc-1', 'acc-2']); + expect(s).toEqual({ + MessageCount: 10, ActiveMessageCount: 9, SourceDeletedMessageCount: 1, + TotalSize: 5000, AttachmentCount: 3, AttachmentSize: 0, LabelCount: 2, AccountCount: 2, + }); + expect(query.mock.calls[0][1]).toEqual([['acc-1', 'acc-2']]); + }); + it('returns a zeroed struct for empty scope without querying', async () => { + const s = await getTotalStats([]); + expect(s.MessageCount).toBe(0); + expect(s.AccountCount).toBe(0); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('aggregate', () => { + beforeEach(() => query.mockReset()); + it('groups sender by from_email and maps capitalized AggregateRow keys, scoped', async () => { + query.mockResolvedValueOnce({ rows: [{ key: 'a@b.com', count: '3', total_size: '10', attachment_count: '1', total_unique: '5' }] }); + const out = await aggregate('sender', { accountIds: ['acc-1'], limit: 50 }); + expect(out).toEqual([{ Key: 'a@b.com', Count: 3, TotalSize: 10, AttachmentSize: 0, AttachmentCount: 1, TotalUnique: 5 }]); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/m\.from_email AS key/); + expect(sql).toMatch(/account_id = ANY\(\$1\)/); + expect(params[0]).toEqual(['acc-1']); + }); + + it("time buckets by calendar year (to_char YYYY)", async () => { + query.mockResolvedValueOnce({ rows: [{ key: '2024', count: '9', total_size: '0', attachment_count: '0', total_unique: '2' }] }); + const out = await aggregate('time', { accountIds: ['acc-1'], limit: 50 }); + expect(out[0].Key).toBe('2024'); + expect(query.mock.calls[0][0]).toMatch(/to_char\(m\.date, 'YYYY'\) AS key/); + }); + + it('recipient unnests to_addresses via a lateral join', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await aggregate('recipient', { accountIds: ['acc-1'], limit: 50 }); + expect(query.mock.calls[0][0]).toMatch(/LATERAL jsonb_array_elements/); + }); +}); + +describe('searchByDomains', () => { + beforeEach(() => query.mockReset()); + it('matches any participant at a domain, scoped, and maps capitalized summaries', async () => { + query.mockResolvedValueOnce({ rows: [{ ...row, id: 'm1' }] }); + const out = await searchByDomains(['gobright.com'], null, null, 100, 0, ['acc-1']); + expect(out[0].id).toBe('m1'); + expect(out[0].to).toEqual([{ Email: 'c@d.com', Name: 'C' }]); // capitalized address keys + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/from_email ILIKE/); + expect(sql).toMatch(/to_addresses::text ILIKE/); + expect(sql).toMatch(/cc_addresses::text ILIKE/); + expect(params[0]).toEqual(['acc-1']); + expect(params).toContain('%@gobright.com'); + }); +}); + +describe('stageDeletion', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + it('resolves ids scoped to accountIds, records a STAGED batch + members, never soft-deletes', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'm1' }, { id: 'm2' }] }); // id resolution + const clientCalls = []; + const client = { query: vi.fn(async (text) => { + clientCalls.push(text); + if (/INSERT INTO mcp_deletion_batches/.test(text)) return { rows: [{ id: 'batch-1' }] }; + return { rows: [] }; + }) }; + withTransaction.mockImplementation(async (fn) => fn(client)); + const out = await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], domain: 'linkedin.com', description: 'filter' }); + expect(out).toEqual({ batchId: 'batch-1', messageCount: 2 }); + expect(query.mock.calls[0][1][0]).toEqual(['acc-1']); // id resolution scoped + expect(clientCalls.some((t) => /INSERT INTO mcp_deletion_batches/.test(t) && /'staged'/.test(t))).toBe(true); + expect(clientCalls.some((t) => /INSERT INTO mcp_deletion_batch_messages/.test(t))).toBe(true); + expect(clientCalls.some((t) => /UPDATE messages SET is_deleted/.test(t))).toBe(false); // never hard/soft deletes here + }); + + it('returns a zero count without opening a transaction when nothing matches', async () => { + query.mockResolvedValueOnce({ rows: [] }); + const out = await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], from: 'x' }); + expect(out).toEqual({ batchId: null, messageCount: 0 }); + expect(withTransaction).not.toHaveBeenCalled(); + }); +}); + +describe('structured after/before args (Wave D Fix 5)', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + it('rejects a malformed after/before with msgvault getDateArg wording, before any SQL runs', async () => { + await expect(listMessages({ accountIds: ['a'], after: 'notadate', limit: 50, offset: 0 })) + .rejects.toThrow('invalid after date "notadate": expected YYYY-MM-DD'); + // Strict YYYY-MM-DD (msgvault time.Parse("2006-01-02")) — no loose forms. + await expect(aggregate('sender', { accountIds: ['a'], before: '2025-1-2', limit: 10 })) + .rejects.toThrow('invalid before date "2025-1-2": expected YYYY-MM-DD'); + await expect(searchByDomains(['x.com'], null, '2025-02-31', 10, 0, ['a'])) + .rejects.toThrow('invalid before date "2025-02-31": expected YYYY-MM-DD'); + await expect(stageDeletion({ userId: 'u', accountIds: ['a'], before: 'garbage' })) + .rejects.toThrow('invalid before date "garbage": expected YYYY-MM-DD'); + expect(query).not.toHaveBeenCalled(); + }); + + it('binds valid dates as midnight-UTC ISO with an EXCLUSIVE before (<), matching lexicalRepo', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await listMessages({ accountIds: ['a'], after: '2025-01-02', before: '2025-02-03', limit: 50, offset: 0 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.date >= $2'); + expect(sql).toContain('m.date < $3'); + expect(sql).not.toContain('m.date <='); + expect(params[1]).toBe('2025-01-02T00:00:00.000Z'); + expect(params[2]).toBe('2025-02-03T00:00:00.000Z'); + }); +}); + +describe('stage_deletion / search predicate parity (Wave D Fix 3)', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + // Rebuild the term clauses through lexicalRepo's shared owner with the same + // ordinals both paths allocate ($1 = accountIds, terms from $2). + function expectedClauses(terms) { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { clauses: terms.map((t) => freeTextTermClause(t, false, bind)), params }; + } + + it('builds the staging WHERE from the ranked/search_fts builder the search path uses, stopword guard included', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], parsed: parseQuery('the invoice') }); + const [sql, params] = query.mock.calls[0]; + const expected = expectedClauses(['the', 'invoice']); + for (const clause of expected.clauses) expect(sql).toContain(clause); + expect(params.slice(1, 5)).toEqual(expected.params); + // Ranked construction, not the legacy ILIKE/plainto fork the staging path + // used to build (preview and staged set must agree): search_fts-first match + // with the stopword vacuity guard, so `the invoice` stages the invoice set + // instead of zero rows ("the" normalizes to an empty tsquery). + expect(sql).toContain('m.search_fts @@'); + expect(sql).toContain('numnode('); + }); + + it('emits byte-identical term predicates to a lexical search of the same query', async () => { + const searchCalls = []; + const client = async (text, params) => { searchCalls.push({ text, params }); return { rows: [] }; }; + await searchLexical(client, { + parsed: parseQuery('the invoice'), accountIds: ['acc-1'], + folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + query.mockResolvedValueOnce({ rows: [] }); + await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], parsed: parseQuery('the invoice') }); + const stagingSql = query.mock.calls[0][0]; + const searchSql = searchCalls[0].text; + const expected = expectedClauses(['the', 'invoice']); + for (const clause of expected.clauses) { + expect(searchSql).toContain(clause); // same ordinals in both WHEREs… + expect(stagingSql).toContain(clause); // …so the clause text is identical + } + // And the bind values line up pairwise (like + fts per term). + expect(searchCalls[0].params.slice(1, 5)).toEqual(query.mock.calls[0][1].slice(1, 5)); + }); +}); + +describe('countEnforceableQueryPredicates', () => { + const n = (q) => countEnforceableQueryPredicates(parseQuery(q)); + + it('counts supported structured filters and usable positive free-text terms', () => { + expect(n('from:amazon invoice')).toBe(2); // from: + invoice + expect(n('is:unread')).toBe(1); // one structured predicate + expect(n('invoice urgent')).toBe(2); // two positive terms + }); + + it('returns 0 when every token is discarded (the stage-everything hazard)', () => { + expect(n('label:promotions')).toBe(0); // unsupported operator only + expect(n('-newsletter')).toBe(0); // negation-only (not enforced on staging) + expect(n('a')).toBe(0); // sub-2-char + expect(n('!!!')).toBe(0); // punctuation-only + expect(n('in:inbox')).toBe(0); // folder scope is not a row predicate + }); + + it('mirrors resolveStageDeletionIds: a real term survives alongside an unsupported operator', () => { + expect(n('invoice label:promotions')).toBe(1); + }); + + it('treats a null parsed query (structured-filter path) as zero', () => { + expect(countEnforceableQueryPredicates(null)).toBe(0); + }); +}); diff --git a/backend/src/mcp/envelope.js b/backend/src/mcp/envelope.js new file mode 100644 index 00000000..d143b7fc --- /dev/null +++ b/backend/src/mcp/envelope.js @@ -0,0 +1,30 @@ +export const TOTAL_COUNT_UNKNOWN = -1; + +// msgvault wire timestamps are Go-marshaled RFC3339 with no sub-second part +// (SQLite second-precision sent_at; vector/stats.go formatTime uses +// time.RFC3339 explicitly). engineAdapter's toISO emits Date.toISOString() +// milliseconds — an internal convention REST shares — so the MCP layer +// re-formats at emission instead of touching engineAdapter (Wave D's file). +// TODO(seam): fold into engineAdapter.toISO if it ever becomes wire-final. +export function toRFC3339(value) { + if (typeof value !== 'string' || !value) return value; + return value.replace(/\.\d+(?=(?:Z|[+-]\d\d:\d\d)$)/, ''); +} + +// Re-shape one MessageSummary for the wire: today only sent_at needs +// re-formatting. Non-mutating; tolerates partial summaries from tests. +export function wireSummary(summary) { + if (!summary || typeof summary !== 'object') return summary; + if (typeof summary.sent_at !== 'string' || !summary.sent_at) return summary; + return { ...summary, sent_at: toRFC3339(summary.sent_at) }; +} + +export function newPaginatedResponse(data, total, offset) { + const d = data || []; + return { data: d, total, returned: d.length, offset, has_more: offset + d.length < total }; +} + +export function newPaginatedResponseNoTotal(data, offset, hasMore) { + const d = data || []; + return { data: d, total: TOTAL_COUNT_UNKNOWN, returned: d.length, offset, has_more: hasMore }; +} diff --git a/backend/src/mcp/envelope.test.js b/backend/src/mcp/envelope.test.js new file mode 100644 index 00000000..c4bdb9c4 --- /dev/null +++ b/backend/src/mcp/envelope.test.js @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, TOTAL_COUNT_UNKNOWN, toRFC3339, wireSummary } from './envelope.js'; + +describe('newPaginatedResponse', () => { + it('computes has_more from offset+returned vs total', () => { + expect(newPaginatedResponse([{ id: 'a' }, { id: 'b' }], 10, 0)).toEqual({ + data: [{ id: 'a' }, { id: 'b' }], total: 10, returned: 2, offset: 0, has_more: true, + }); + expect(newPaginatedResponse([{ id: 'a' }], 1, 0).has_more).toBe(false); + }); + it('coerces null data to an empty array', () => { + expect(newPaginatedResponse(null, 0, 0).data).toEqual([]); + }); +}); + +describe('newPaginatedResponseNoTotal', () => { + it('always reports total -1 and echoes has_more', () => { + const r = newPaginatedResponseNoTotal([{ id: 'a' }], 20, true); + expect(r).toEqual({ data: [{ id: 'a' }], total: TOTAL_COUNT_UNKNOWN, returned: 1, offset: 20, has_more: true }); + expect(TOTAL_COUNT_UNKNOWN).toBe(-1); + }); +}); + +describe('toRFC3339', () => { + it('strips fractional seconds (msgvault wire timestamps are Go RFC3339, no millis)', () => { + expect(toRFC3339('2024-01-01T00:00:00.000Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('2024-01-01T00:00:00.123Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('2024-01-01T00:00:00.123456+02:00')).toBe('2024-01-01T00:00:00+02:00'); + }); + it('passes through already-clean, empty, and non-string values', () => { + expect(toRFC3339('2024-01-01T00:00:00Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('')).toBe(''); + expect(toRFC3339(undefined)).toBe(undefined); + }); +}); + +describe('wireSummary', () => { + it('reformats sent_at to RFC3339 without mutating the input', () => { + const s = { id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }; + expect(wireSummary(s)).toEqual({ id: 'm1', sent_at: '2024-01-01T00:00:00Z' }); + expect(s.sent_at).toBe('2024-01-01T00:00:00.000Z'); + }); + it('leaves summaries without a sent_at string untouched', () => { + const s = { id: 'm1' }; + expect(wireSummary(s)).toBe(s); + expect(wireSummary(null)).toBe(null); + }); +}); diff --git a/backend/src/mcp/goldenParity.test.js b/backend/src/mcp/goldenParity.test.js new file mode 100644 index 00000000..2c503b8c --- /dev/null +++ b/backend/src/mcp/goldenParity.test.js @@ -0,0 +1,319 @@ +// Golden-fixture parity: msgvault's wire shapes (internal/query/models.go, +// internal/mcp/handlers.go, internal/vector/stats.go) transcribed as key+type +// shapes and diffed field-for-field against Mailflow's output. Drives REAL code +// end-to-end with only I/O mocked, so the diff pins the true wire shape. +// +// Divergences the diff intentionally accepts: D6 UUID-string ids (token 'uuid' +// matches any string), ≤1 semantic excerpt, and the capitalized-key split +// (Address/AttachmentInfo/AccountInfo/AggregateRow/TotalStats have no Go json +// tags → Go-default caps; MessageSummary/getMessageResponse are snake_case). +// +// Shapes live inline here (not separate fixtures/*.json files) — a deliberate +// consolidation; the diffing is identical and the reference sits beside the test. +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +vi.mock('../services/search/queryParser.js', () => ({ parseQuery: vi.fn(() => ({ filters: [], terms: [{ value: 'hello', negate: false }], unsupported: [], errors: [] })) })); +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +vi.mock('../services/embeddings/generations.js', () => ({ activeGeneration: vi.fn(), buildingGeneration: vi.fn(), chunkCount: vi.fn() })); +vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); +vi.mock('../services/embeddings/config.js', () => ({ generationFingerprint: vi.fn(() => 'fp'), resolveEmbedConfig: vi.fn(async () => ({ enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 })) })); + +import { query, withTransaction } from '../services/db.js'; +import { search } from '../services/search/searchService.js'; +import * as generations from '../services/embeddings/generations.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { rowToMessageSummary, rowToMessageDetail } from './engineAdapter.js'; +import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; +import { + handleGetMessage, handleListMessages, handleGetStats, handleAggregate, + handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, handleSearchByDomains, +} from './messageTools.js'; +import { HANDLERS } from './tools.js'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +// ---- transcribed reference shapes --------------------------------------------- +const Address = { Email: 'string', Name: 'string' }; +const AttachmentInfo = { ID: 'int', Filename: 'string', MimeType: 'string', Size: 'int', ContentHash: 'string', URL: 'string', StoragePath: 'string' }; +const Generation = { id: 'int', model: 'string', dimension: 'int', fingerprint: 'string', state: 'string' }; +const MessageSummary = { + id: 'uuid', 'source_id?': 'uuid', source_message_id: 'string', conversation_id: 'uuid', + source_conversation_id: 'string', subject: 'string', snippet: 'string', + from_email: 'string', from_name: 'string', 'to?': [Address], 'cc?': [Address], + sent_at: 'string', size_estimate: 'int', has_attachments: 'bool', + attachment_count: 'int', labels: ['string'], message_type: 'string', +}; +const Match = { snippet: 'string', 'char_offset?': 'int', 'line?': 'int', 'score?': 'number' }; +// hybridScoreBreakdown (handlers.go:563-572): every field omitempty — rrf only +// fuses in mode=hybrid, subject_boosted only when true. +const HybridScore = { 'rrf?': 'number', 'bm25?': 'number', 'vector?': 'number', 'subject_boosted?': 'bool' }; +// searchMessageItem (handlers.go:331-342): MessageSummary + matches/ +// matches_truncated/score, all three Go-omitempty. +const SearchMessageItem = { ...MessageSummary, 'matches?': [Match], 'matches_truncated?': 'bool', 'score?': HybridScore }; +// searchMessageBodiesResponse (handlers.go:586-595): paginated (no-total) + +// mode/pool_saturated/generation — shared by keyword and vector/hybrid modes. +const SearchBodiesEnvelope = { + data: [SearchMessageItem], total: 'int', returned: 'int', offset: 'int', has_more: 'bool', + mode: 'string', pool_saturated: 'bool', generation: Generation, +}; +const SHAPES = { + message_summary: MessageSummary, + message_detail: { ...MessageSummary, from: [Address], to: [Address], cc: [Address], bcc: [Address], body_text: 'string', body_html: 'string', attachments: [AttachmentInfo] }, + search_metadata: { data: [MessageSummary], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + list_messages: { data: [MessageSummary], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + get_message: { + id: 'uuid', source_message_id: 'string', conversation_id: 'uuid', source_conversation_id: 'string', + subject: 'string', 'message_type?': 'string', snippet: 'string', sent_at: 'string', + size_estimate: 'int', has_attachments: 'bool', from: [Address], to: [Address], cc: [Address], bcc: [Address], + body_text: 'string', body_html: 'string', 'body_format?': 'string', body_length: 'int', + body_returned: 'int', offset: 'int', has_more: 'bool', labels: ['string'], attachments: [AttachmentInfo], + }, + get_stats: { + stats: { MessageCount: 'int', ActiveMessageCount: 'int', SourceDeletedMessageCount: 'int', TotalSize: 'int', AttachmentCount: 'int', AttachmentSize: 'int', LabelCount: 'int', AccountCount: 'int' }, + accounts: [{ ID: 'uuid', SourceType: 'string', Identifier: 'string', DisplayName: 'string' }], + 'vector_search?': { enabled: 'bool', active_generation: { id: 'int', model: 'string', dimension: 'int', fingerprint: 'string', state: 'string', 'activated_at?': 'string', message_count: 'int' }, 'building_generation?': {}, missing_embeddings_total: 'int' }, + }, + aggregate: [{ Key: 'string', Count: 'int', TotalSize: 'int', AttachmentSize: 'int', AttachmentCount: 'int', TotalUnique: 'int' }], + find_similar: { seed_message_id: 'uuid', returned: 'int', generation: Generation, messages: [MessageSummary] }, + search_in_message: { data: [Match], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + stage_deletion: { batch_id: 'uuid', message_count: 'int', status: 'string', next_step: 'string' }, + search_message_bodies: SearchBodiesEnvelope, + semantic_search_messages: SearchBodiesEnvelope, + search_by_domains: [MessageSummary], // raw array, no envelope (handlers.go:1984-1989) + ping: { pong: 'bool' }, // Mailflow-specific health tool — no msgvault counterpart (documented divergence) +}; + +// ---- diffKeys: [] on parity, else a list of divergence paths ------------------- +function typeOk(val, token) { + const t = token.replace(/\?$/, ''); + if (t.includes('uuid')) return typeof val === 'string'; // D6 id divergence + if (t === 'string') return typeof val === 'string'; + if (t === 'int' || t === 'number' || t === 'float') return typeof val === 'number'; + if (t === 'bool') return typeof val === 'boolean'; + return true; +} +function isOptional(shapeKey, shapeVal) { + return shapeKey.endsWith('?') || (typeof shapeVal === 'string' && shapeVal.endsWith('?')); +} +function diffKeys(actual, shape, path = '$') { + const out = []; + if (typeof shape === 'string') { + if (!typeOk(actual, shape)) out.push(`${path}: type mismatch (want ${shape}, got ${typeof actual})`); + return out; + } + if (Array.isArray(shape)) { + if (!Array.isArray(actual)) { out.push(`${path}: want array, got ${typeof actual}`); return out; } + actual.forEach((a, i) => out.push(...diffKeys(a, shape[0], `${path}[${i}]`))); + return out; + } + if (actual === null || typeof actual !== 'object' || Array.isArray(actual)) { out.push(`${path}: want object, got ${actual === null ? 'null' : typeof actual}`); return out; } + const realKeys = new Set(); + for (const sk of Object.keys(shape)) { + const k = sk.endsWith('?') ? sk.slice(0, -1) : sk; + realKeys.add(k); + if (!(k in actual)) { if (!isOptional(sk, shape[sk])) out.push(`${path}.${k}: missing`); continue; } + out.push(...diffKeys(actual[k], shape[sk], `${path}.${k}`)); + } + for (const k of Object.keys(actual)) if (!realKeys.has(k)) out.push(`${path}.${k}: extra (not in msgvault shape)`); + return out; +} + +const realRow = { + id: '11111111-1111-1111-1111-111111111111', account_id: 'acc-1', message_id: '', thread_id: 'tid-9', + subject: 'Hi', snippet: 's', from_email: 'a@b.com', from_name: 'A', + to_addresses: [{ name: 'C', email: 'c@d.com' }], cc_addresses: [{ name: 'E', email: 'e@f.com' }], + date: new Date('2024-01-01T00:00:00Z'), has_attachments: true, + attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], + flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello world', body_html: '

hello world

', +}; +const jsonOf = (r) => JSON.parse(r.content[0].text); +const scope = { userId: 'u', accountIds: ['acc-1'] }; + +beforeEach(() => { + query.mockReset(); withTransaction.mockReset(); search.mockReset(); matchFromChunk.mockReset(); + generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); + resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); +}); + +// Every function this suite vi.mock()s must actually exist on its real module — +// a renamed/never-implemented seam (e.g. generations.chunkCount) otherwise passes +// here while throwing live. Catches the missing/renamed-export drift class only, +// not value-shape drift. +describe('mock-drift guard: mocked seams exist on their real modules', () => { + // hybrid.js is intentionally omitted: importActual runs its module body, which + // references vectorStore.fusedSearch — not in this suite's vectorStore mock. + it.each([ + ['generations', () => generations, '../services/embeddings/generations.js'], + ['vectorStore', () => ({ loadVector, annSearch }), '../services/embeddings/vectorStore.js'], + ['searchService', () => ({ search }), '../services/search/searchService.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); + +describe('golden parity: structural mappers', () => { + it('rowToMessageSummary matches MessageSummary (snake_case + capitalized Address, id divergence excepted)', () => { + // Round-trip through JSON so absent optional keys behave like the wire. + const s = JSON.parse(JSON.stringify(rowToMessageSummary(realRow))); + expect(diffKeys(s, SHAPES.message_summary)).toEqual([]); + }); + it('rowToMessageDetail matches MessageDetail (capitalized AttachmentInfo)', () => { + const d = JSON.parse(JSON.stringify(rowToMessageDetail(realRow))); + expect(diffKeys(d, SHAPES.message_detail)).toEqual([]); + }); +}); + +describe('golden parity: message tool envelopes', () => { + it('get_message → getMessageResponse', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); + const b = jsonOf(await handleGetMessage({ id: realRow.id }, scope)); + expect(diffKeys(b, SHAPES.get_message)).toEqual([]); + }); + + it('list_messages → paginated MessageSummary envelope', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); // listMessages + const b = jsonOf(await handleListMessages({}, scope)); + expect(diffKeys(b, SHAPES.list_messages)).toEqual([]); + }); + + it('aggregate → capitalized AggregateRow array', async () => { + query.mockResolvedValueOnce({ rows: [{ key: 'a@b.com', count: '3', total_size: '10', attachment_count: '1', total_unique: '5' }] }); + const b = jsonOf(await handleAggregate({ group_by: 'sender' }, scope)); + expect(diffKeys(b, SHAPES.aggregate)).toEqual([]); + }); + + it('get_stats → {stats, accounts, vector_search} with capitalized structs', async () => { + query + .mockResolvedValueOnce({ rows: [{ message_count: '10', active_count: '9', deleted_count: '1', total_size: '5000', attachment_count: '3', label_count: '2' }] }) // getTotalStats + .mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'Work' }] }) // listAccounts + .mockResolvedValueOnce({ rows: [{ n: '4' }] }); // collectStats missingCount + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); // epoch seconds → RFC3339 wire + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(1000); + const b = jsonOf(await handleGetStats({}, scope)); + expect(diffKeys(b, SHAPES.get_stats)).toEqual([]); + // RFC3339 without sub-second digits (vector/stats.go:146-153 formatTime). + expect(b.vector_search.active_generation.activated_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('find_similar_messages → seed/returned/generation/messages', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: realRow.id, score: 0.9, rank: 1 }]); + query + .mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }) // messageInScope(seed) + .mockResolvedValueOnce({ rows: [realRow] }); // getMessageSummariesByIDs + const b = jsonOf(await handleFindSimilarMessages({ message_id: realRow.id }, scope)); + expect(diffKeys(b, SHAPES.find_similar)).toEqual([]); + }); + + it('search_in_message (keyword) → Match envelope with byte char_offset', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); // getMessage + const b = jsonOf(await handleSearchInMessage({ id: realRow.id, query: 'hello' }, scope)); + expect(diffKeys(b, SHAPES.search_in_message)).toEqual([]); + expect(b.data[0].char_offset).toBe(Buffer.from(realRow.body_text, 'utf8').indexOf(Buffer.from('hello', 'utf8'))); + }); + + it('stage_deletion → {batch_id, message_count, status, next_step}', async () => { + query.mockResolvedValueOnce({ rows: [{ id: realRow.id }] }); // id resolution + withTransaction.mockImplementation(async (fn) => fn({ query: vi.fn(async (text) => (/INSERT INTO mcp_deletion_batches/.test(text) ? { rows: [{ id: 'batch-1' }] } : { rows: [] })) })); + const b = jsonOf(await handleStageDeletion({ domain: 'linkedin.com' }, scope)); + expect(diffKeys(b, SHAPES.stage_deletion)).toEqual([]); + expect(b.status).toBe('pending'); // msgvault manifest.StatusPending literal (manifest.go:25) + }); +}); + +describe('golden parity: search tool envelopes', () => { + it('search_metadata → paginated MessageSummary envelope (real hydration)', async () => { + search.mockResolvedValue({ messages: [{ id: realRow.id }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + query.mockResolvedValueOnce({ rows: [realRow] }); // getMessageSummariesByIDs hydration + const b = jsonOf(await handleSearchMetadata({ query: 'x' }, scope)); + expect(diffKeys(b, SHAPES.search_metadata)).toEqual([]); + // Go time.Time wire format: RFC3339 with no sub-second digits. + expect(b.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('search_message_bodies → searchMessageItem envelope; matches/matches_truncated omitted when empty/false (handlers.go:339-341)', async () => { + search.mockResolvedValue({ + messages: [ + { id: realRow.id, body_text: 'hello world, hello again' }, // 1 merged excerpt + { id: 'no-hit-1111-1111-1111-111111111111', body_text: 'nothing relevant' }, // 0 excerpts + ], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + query.mockResolvedValueOnce({ rows: [realRow, { ...realRow, id: 'no-hit-1111-1111-1111-111111111111' }] }); // hydration + const b = jsonOf(await handleSearchMessageBodies({ query: 'hello' }, scope)); + expect(diffKeys(b, SHAPES.search_message_bodies)).toEqual([]); + expect(b.mode).toBe('keyword'); + expect(b.total).toBe(-1); // body search never counts + expect(b.generation).toEqual({ id: 0, model: '', dimension: 0, fingerprint: '', state: '' }); + // diffKeys accepts optional keys whether present or absent, so the + // omitempty contract is asserted explicitly on both sides: + expect(b.data[0]).toHaveProperty('matches'); + expect(b.data[0]).not.toHaveProperty('matches_truncated'); // false → omitted + expect(b.data[1]).not.toHaveProperty('matches'); // empty → omitted + expect(b.data[1]).not.toHaveProperty('matches_truncated'); + }); + + it('semantic_search_messages → mode/pool_saturated/generation envelope; explain score omitempty (handlers.go:563-572)', async () => { + search.mockResolvedValue({ + messages: [{ + message_id: realRow.id, id: realRow.id, + best_chunk: { chunk_index: 0, char_start: 0, char_end: 10, score: 0.9 }, + score: { rrf: 0.03, bm25: 1.2, vector: 0.9, subject_boosted: false }, + }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, + pool_saturated: true, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + matchFromChunk.mockResolvedValue({ char_offset: 5, snippet: 'hello', line: 1, score: 0.9 }); + query.mockResolvedValueOnce({ rows: [realRow] }); // hydration + const b = jsonOf(await handleSemanticSearchMessages({ query: 'hello', mode: 'vector', explain: true }, scope)); + expect(diffKeys(b, SHAPES.semantic_search_messages)).toEqual([]); + expect(b.mode).toBe('vector'); + expect(b.pool_saturated).toBe(true); + expect(b.total).toBe(-1); + expect(b.generation).toEqual({ id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }); + // omitempty asserted explicitly: no rrf in mode=vector (nothing to fuse), + // no subject_boosted when false. + expect(b.data[0].score).toEqual({ bm25: 1.2, vector: 0.9 }); + expect(b.data[0].matches).toHaveLength(1); // ≤1 excerpt — documented divergence from msgvault's ≤5 + }); + + it('search_by_domains → raw MessageSummary array, no envelope (handlers.go:1951-1989)', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); + const b = jsonOf(await handleSearchByDomains({ domains: 'b.com' }, scope)); + expect(diffKeys(b, SHAPES.search_by_domains)).toEqual([]); + expect(b[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('ping → {pong:true} (Mailflow-specific health tool)', async () => { + const b = jsonOf(await HANDLERS.ping({}, scope)); + expect(diffKeys(b, SHAPES.ping)).toEqual([]); + expect(b.pong).toBe(true); + }); +}); + +describe('no SQL in MCP handler modules (one-seam invariant)', () => { + const here = dirname(fileURLToPath(import.meta.url)); + const read = (n) => readFileSync(join(here, n), 'utf8'); + for (const file of ['searchTools.js', 'messageTools.js']) { + it(`${file} contains no raw SQL or db.js import`, () => { + const src = read(file); + expect(src).not.toMatch(/\bSELECT\b|\bINSERT\b|\bUPDATE\b|\bDELETE FROM\b/); + expect(src).not.toMatch(/from '\.\.\/services\/db\.js'/); + expect(src).not.toMatch(/\bpool\b/); + }); + } +}); diff --git a/backend/src/mcp/messageTools.js b/backend/src/mcp/messageTools.js new file mode 100644 index 00000000..c5246f41 --- /dev/null +++ b/backend/src/mcp/messageTools.js @@ -0,0 +1,485 @@ +// The msgvault message/aggregate/deletion tool handlers. Every read goes through +// engineAdapter (all SQL lives there); handlers only shape and page. Byte offsets +// throughout are UTF-8 bytes into the raw body (msgvault wire contract). +import { + getMessage, rowToMessageDetail, listMessages, listAccounts, getTotalStats, aggregate, searchByDomains, + getMessageSummariesByIDs, stageDeletion, resolveAccountScope, messageInScope, countEnforceableQueryPredicates, +} from './engineAdapter.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { bodyByteSliceRange, contextWindow, findTermMatches } from './bodyMatch.js'; +import { jsonResult, errorResult } from './result.js'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, toRFC3339, wireSummary } from './envelope.js'; +import { + searchLimitArg, offsetArg, + queryParseErrorMessage, unsupportedSearchOperatorMessage, HYBRID_RANKING_WINDOW, +} from './searchTools.js'; +import { collectStats } from './vectorStats.js'; +import { translateVectorError } from './vectorErrors.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; // phase-4 public face +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; // phase 3 +import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; // phase-5-owned + +const DEFAULT_BODY_CHARS = 2000; +const MAX_BODY_CHARS = 4000; +const MAX_LIMIT = 1000; + +// msgvault limitArg: absent/non-number → default; negative/NaN → 0; clamp to 1000. +// (Distinct from searchLimitArg's 20/50 search clamp.) +function limitArg(args, key, def) { + const raw = args[key]; + if (typeof raw !== 'number') return def; + if (Number.isNaN(raw) || raw < 0) return 0; + if (raw > MAX_LIMIT) return MAX_LIMIT; + return Math.trunc(raw); +} + +// msgvault getDateArg (handlers.go:264-275): an optional after/before arg is a +// strict YYYY-MM-DD; anything else errors at the handler instead of leaking a +// raw Postgres cast error to the wire. Non-strings/empty are "no filter", and +// JS Date rollover (2024-02-31 → Mar 1) is rejected via the Y/M/D round-trip +// (Go time.Parse errors "day out of range"). Returns { value } or { error }. +function dateArg(args, key) { + const v = args[key]; + if (typeof v !== 'string' || v === '') return { value: undefined }; + const err = { error: `invalid ${key} date "${v}": expected YYYY-MM-DD` }; + const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v); + if (!m) return err; + const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])]; + const t = new Date(Date.UTC(y, mo - 1, d)); + if (t.getUTCFullYear() !== y || t.getUTCMonth() !== mo - 1 || t.getUTCDate() !== d) return err; + return { value: v }; +} + +export const getMessageDef = { + name: 'get_message', + description: + 'Get message details including recipients, labels, attachments, and a slice of the message body. ' + + 'Returns plain text when available; HTML-only messages return a body_html slice with body_format=html. ' + + 'Body paging mirrors search pagination: body_length=total bytes, offset=where this chunk starts, body_returned=bytes in this chunk, has_more=more body follows. ' + + 'To read sequentially: call again with offset += body_returned. ' + + 'To jump to a known match location: use center_at= to center the window on that location. ' + + 'Note: snippet is pre-stored source metadata (may be empty for non-Gmail sources).', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'Message ID' }, + offset: { type: 'number', description: 'Byte offset from the start of the selected body to begin reading (default 0). Ignored when center_at is provided.' }, + center_at: { type: 'number', description: 'Byte offset from the start of the selected body to center the window on. Takes precedence over offset.' }, + max_chars: { type: 'number', description: 'Maximum selected-body bytes to return (default 2000, max 4000). Values above 4000 are clamped to 4000; zero or negative values use the default.' }, + body_format: { type: 'string', enum: ['auto', 'text', 'html'], description: 'Which body representation to page: auto (default, plain text when available, HTML fallback), text, or html.' }, + full_body: { type: 'boolean', description: 'Return the complete selected body in one response, ignoring offset, center_at, and max_chars. Use only when the full content is explicitly needed.' }, + }, + required: ['id'], + }, +}; + +export async function handleGetMessage(args, scope) { + const id = args.id; + if (!id || typeof id !== 'string') return errorResult('id parameter is required'); + const row = await getMessage(id, scope.accountIds); + if (!row) return errorResult('message not found'); + const detail = rowToMessageDetail(row); + + let maxChars = Number(args.max_chars); + if (!Number.isFinite(maxChars) || maxChars <= 0) maxChars = DEFAULT_BODY_CHARS; + else if (maxChars > MAX_BODY_CHARS) maxChars = MAX_BODY_CHARS; + + const requested = args.body_format || 'auto'; + let full = detail.body_text; + let bodyFormat = 'text'; + if (requested === 'auto') { + if (!full && detail.body_html) { full = detail.body_html; bodyFormat = 'html'; } + } else if (requested === 'html') { + full = detail.body_html; bodyFormat = 'html'; + } else if (requested !== 'text') { + return errorResult('body_format must be one of auto, text, html'); + } + + const buf = Buffer.from(full || '', 'utf8'); + const bodyLen = buf.length; + + let start, end; + if (args.full_body === true) { + start = 0; end = bodyLen; + } else if (Number.isFinite(Number(args.center_at)) && Number(args.center_at) >= 0) { + [start, end] = contextWindow(bodyLen, Math.trunc(Number(args.center_at)), 0, maxChars); + } else { + start = Math.min(Number.isFinite(Number(args.offset)) ? Math.trunc(Number(args.offset)) : 0, bodyLen); + end = Math.min(start + maxChars, bodyLen); + } + + const { text: slice, adjStart, adjEnd } = bodyByteSliceRange(buf, start, end); + const returnedBytes = Buffer.byteLength(slice, 'utf8'); + + return jsonResult({ + id: detail.id, + source_message_id: detail.source_message_id, + conversation_id: detail.conversation_id, + source_conversation_id: detail.source_conversation_id, + subject: detail.subject, + message_type: detail.message_type, + snippet: detail.snippet, + sent_at: toRFC3339(detail.sent_at), + size_estimate: detail.size_estimate, + has_attachments: detail.has_attachments, + from: detail.from, + to: detail.to, + cc: detail.cc, + bcc: detail.bcc, + body_text: bodyFormat === 'html' ? '' : slice, + body_html: bodyFormat === 'html' ? slice : '', + body_format: bodyFormat, + body_length: bodyLen, + body_returned: returnedBytes, + offset: adjStart, + has_more: adjEnd < bodyLen, + labels: detail.labels, + attachments: detail.attachments, + }); +} + +export const listMessagesDef = { + name: 'list_messages', + description: + 'List messages with optional filters, newest-first. ' + + "Pass conversation_id to enumerate a thread's messages, then call get_message(id) per message to read bodies — " + + 'there is deliberately no bulk body fetch, to avoid loading huge threads into the context window. ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, total, returned, offset, has_more. ' + + 'total=-1 because the full count is not computed; use has_more for paging.', + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + from: { type: 'string', description: 'Filter by sender email address' }, + to: { type: 'string', description: 'Filter by recipient email address' }, + label: { type: 'string', description: 'Filter by Gmail label' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + conversation_id: { type: 'string', description: 'Filter by conversation/thread ID' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + }, +}; + +export async function handleListMessages(args, scope) { + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Over-fetch one to compute has_more without a count query (msgvault limit+1). + let rows = await listMessages({ + accountIds: acc.accountIds, + from: args.from, to: args.to, label: args.label, + hasAttachment: args.has_attachment === true, + after: after.value, before: before.value, + conversationId: args.conversation_id, + limit: limit + 1, offset, + }); + const hasMore = rows.length > limit; + if (hasMore) rows = rows.slice(0, limit); + return jsonResult(newPaginatedResponseNoTotal(rows.map(wireSummary), offset, hasMore)); +} + +export const getStatsDef = { + name: 'get_stats', + description: 'Get archive overview: total messages, size, attachment count, and accounts.', + inputSchema: { type: 'object', properties: {} }, +}; + +// A broken vector sub-query must never blank the whole stats response (msgvault +// best-effort semantics); on any failure the vector_search field is simply omitted. +async function safeCollectStats(accountIds) { + try { return await collectStats(accountIds); } catch { return null; } +} + +export async function handleGetStats(_args, scope) { + const stats = await getTotalStats(scope.accountIds); + const accounts = await listAccounts(scope.accountIds); + const resp = { stats, accounts }; + const vs = await safeCollectStats(scope.accountIds); + if (vs) resp.vector_search = vs; // omitempty: present only when vector search is enabled + return jsonResult(resp); +} + +const AGGREGATE_GROUP_BY = ['sender', 'recipient', 'domain', 'label', 'time']; + +export const aggregateDef = { + name: 'aggregate', + description: + 'Get grouped statistics (top senders, recipients, domains, labels, or message volume by calendar year). ' + + 'Returns a JSON array of objects with fields Key, Count, TotalSize, AttachmentSize, AttachmentCount, and TotalUnique.', + inputSchema: { + type: 'object', + properties: { + group_by: { type: 'string', enum: ['sender', 'recipient', 'domain', 'label', 'time'], description: 'Dimension to group by. When \'time\', buckets are by calendar year only (Key is a year string like "2024").' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 50)' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + }, + required: ['group_by'], + }, +}; + +export async function handleAggregate(args, scope) { + const groupBy = args.group_by || ''; + if (!groupBy) return errorResult('group_by parameter is required'); + if (!AGGREGATE_GROUP_BY.includes(groupBy)) return errorResult('invalid group_by: ' + groupBy); + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const rows = await aggregate(groupBy, { + accountIds: acc.accountIds, + after: after.value, before: before.value, + limit: limitArg(args, 'limit', 50), + }); + return jsonResult(rows); // raw array, no envelope (msgvault parity) +} + +export const searchByDomainsDef = { + name: 'search_by_domains', + description: 'Find emails where any participant (from, to, or cc) belongs to one of the given domains. Useful for finding all communication with a company regardless of direction.', + inputSchema: { + type: 'object', + properties: { + domains: { type: 'string', description: "Comma-separated domain names (e.g. 'gobright.com,ascentae.com')" }, + limit: { type: 'number', description: 'Maximum results to return (default 100)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + }, + required: ['domains'], + }, +}; + +export async function handleSearchByDomains(args, scope) { + const domainsStr = (args.domains || '').trim(); + if (!domainsStr) return errorResult('domains is required'); + const domains = domainsStr.split(',').map((d) => d.trim()).filter(Boolean); + if (!domains.length) return errorResult('at least one domain is required'); + const limit = limitArg(args, 'limit', 100); + const offset = limitArg(args, 'offset', 0); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const results = await searchByDomains(domains, after.value, before.value, limit, offset, scope.accountIds); + return jsonResult(results.map(wireSummary)); // raw array (msgvault parity) +} + +export const findSimilarMessagesDef = { + name: 'find_similar_messages', + description: 'Find messages whose embeddings are closest to the given message. Requires vector search to be configured and an active index generation.', + inputSchema: { + type: 'object', + properties: { + message_id: { type: 'string', description: 'Seed message ID; its embedding is used as the query vector' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + message_type: { type: 'string', description: 'Restrict results to one message type, such as email, sms, mms, fbmessenger, or calendar_event' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + }, + required: ['message_id'], + }, +}; + +export async function handleFindSimilarMessages(args, scope) { + const seedId = args.message_id; + if (!seedId || typeof seedId !== 'string') return errorResult('message_id parameter is required'); + let limit = Number(args.limit); + if (!Number.isFinite(limit) || limit < 1) limit = 20; + // msgvault clamps to the hybrid page cap (handlers.go:885-888). + if (limit > HYBRID_RANKING_WINDOW) limit = HYBRID_RANKING_WINDOW; + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const messageType = typeof args.message_type === 'string' ? args.message_type.trim().toLowerCase() : ''; + + try { + const { generation: gen } = await resolveActiveGenerationFromConfig(); + + // Owner-scope the seed: loadVector is not account-aware, so a foreign/unknown + // seed id must behave like get_message on a foreign id (contract: every tool + // call is owner-scoped; also closes an embedding-existence oracle). + if (!(await messageInScope(seedId, acc.accountIds))) return errorResult('message not found'); + + let seed; + try { seed = await loadVector(seedId); } + catch (e) { return errorResult(`load seed vector: ${e.message}`); } + + const filter = { accountIds: acc.accountIds }; + if (after.value) filter.after = after.value; + if (before.value) filter.before = before.value; + if (args.has_attachment === true) filter.hasAttachment = true; + // phase-3 annSearch takes the generation ID (not the object) and returns + // [{messageId, score, rank}] rank-ordered. +1 to drop the seed without coming up short. + const hits = await annSearch(gen.id, seed, limit + 1, { filter }); + + const ids = []; + for (const h of hits) { + if (h.messageId === seedId) continue; + if (ids.length >= limit) break; + ids.push(h.messageId); + } + let messages = await getMessageSummariesByIDs(ids, acc.accountIds); + // msgvault applies message_type inside the vector backend filter + // (handlers.go:1042-1044); Mailflow's annSearch filter has no such leg, so + // the advertised filter is applied on the hydrated summaries (every + // Mailflow message is 'email' today — a non-email filter returns zero). + if (messageType) messages = messages.filter((m) => m.message_type === messageType); + return jsonResult({ + seed_message_id: seedId, + returned: messages.length, + generation: { id: gen.id, model: gen.model, dimension: gen.dimension, fingerprint: gen.fingerprint, state: gen.state }, + messages: messages.map(wireSummary), + }); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + throw err; + } +} + +export const searchInMessageDef = { + name: 'search_in_message', + description: + 'Find matches within one message body. Default mode=keyword finds literal term occurrences. ' + + 'mode=vector scores each embedded chunk by semantic similarity to the query (best first, with score on each match). ' + + 'Keyword matches include raw-body char_offset and line. Vector matches always include snippet and score; char_offset and line may be omitted after preprocessing. ' + + 'Use a present char_offset with get_message center_at to read a larger window around the match.', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'Message ID' }, + query: { type: 'string', description: 'Search query (keyword term, or semantic query when mode=vector)' }, + limit: { type: 'number', description: 'Maximum matches to return (default 10)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + mode: { type: 'string', enum: ['keyword', 'vector'], description: 'Search mode: keyword (default, literal term) or vector (semantic chunk scoring)' }, + min_score: { type: 'number', description: 'Minimum chunk similarity score (0–1) when mode=vector (default 0)' }, + }, + required: ['id', 'query'], + }, +}; + +export async function handleSearchInMessage(args, scope) { + const id = args.id; + if (!id || typeof id !== 'string') return errorResult('id parameter is required'); + const q = (args.query || '').trim(); + if (!q) return errorResult('query parameter is required'); + const limit = limitArg(args, 'limit', 10); + const offset = limitArg(args, 'offset', 0); + const mode = args.mode || 'keyword'; + + if (mode === 'vector') { + try { + const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 0; + // matchesInMessage is scope-aware: it resolves the message under + // scope.accountIds and throws VectorUnavailableError on stock Postgres. + const all = await matchesInMessage(id, q, minScore, { accountIds: scope.accountIds }); + const page = all.slice(offset, offset + limit); + return jsonResult(newPaginatedResponse(page, all.length, offset)); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + throw err; + } + } + if (mode !== 'keyword') return errorResult(`invalid mode "${mode}": must be keyword (default) or vector`); + + const row = await getMessage(id, scope.accountIds); + if (!row) return errorResult('message not found'); + const all = findTermMatches(row.body_text || '', q); // byte-offset keyword matches (real total) + const page = all.slice(offset, offset + limit); + return jsonResult(newPaginatedResponse(page, all.length, offset)); +} + +export const stageDeletionDef = { + name: 'stage_deletion', + description: "Stage messages for deletion. Use EITHER 'query' (Gmail-style search) OR structured filters (from, domain, label, etc.), not both. Does NOT delete immediately - execution is a separate, explicitly-authorized step.", + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + query: { type: 'string', description: "Gmail-style search query (e.g. 'from:linkedin subject:job alert'). Cannot be combined with structured filters." }, + from: { type: 'string', description: 'Filter by sender email address' }, + domain: { type: 'string', description: "Filter by sender domain (e.g. 'linkedin.com')" }, + label: { type: 'string', description: "Filter by Gmail label (e.g. 'CATEGORY_PROMOTIONS')" }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + }, + }, +}; + +// A parsed query that survives validation but produces zero enforceable +// predicates would stage the entire (capped) mailbox — refuse it. Parse errors +// and unsupported operators are already rejected before this runs (msgvault +// ordering), so the only remaining cause is all-discarded terms. +function noEnforceableFiltersMessage() { + return 'query produced no enforceable filters (all query terms were discarded as too short, ' + + 'punctuation-only, or negation-only); refusing to stage deletions'; +} + +export async function handleStageDeletion(args, scope) { + const q = (args.query || '').trim(); + const hasQuery = q !== ''; + const structured = !!(args.from || args.domain || args.label || args.has_attachment || args.after || args.before); + if (hasQuery && structured) return errorResult("use either 'query' or structured filters (from, domain, label, etc.), not both"); + if (!hasQuery && !structured) return errorResult("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + + const parsed = hasQuery ? parseQuery(q) : null; + if (parsed) { + // Parse-value errors and unsupported operators reject the staging query + // outright (msgvault handlers.go:1818-1821). Deletion is where silent + // widening bites hardest: dropping `label:promotions` from + // `invoice label:promotions` would stage a SUPERSET of what was asked. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + // Guard the "stage EVERYTHING" hazard: a query whose tokens are ALL + // discarded (negation-only, sub-2-char/punctuation-only terms) leaves only + // account+liveness in the WHERE. Refuse before staging. + if (countEnforceableQueryPredicates(parsed) === 0) { + return errorResult(noEnforceableFiltersMessage()); + } + } + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + + const { batchId, messageCount } = await stageDeletion({ + userId: scope.userId, accountIds: acc.accountIds, + parsed, + from: args.from, domain: args.domain, label: args.label, + hasAttachment: args.has_attachment === true, after: after.value, before: before.value, + description: hasQuery ? `query: ${q}`.slice(0, 50) : 'filter', + }); + if (!messageCount) return errorResult('no messages match the specified criteria'); + return jsonResult({ + batch_id: batchId, + message_count: messageCount, + // Wire literal is msgvault's manifest.StatusPending ("pending", + // deletion/manifest.go:25, surfaced at handlers.go:1932). The DB row keeps + // Mailflow's internal 'staged' state (engineAdapter.stageDeletion). + status: 'pending', + next_step: `POST /api/mcp-deletions/${batchId}/execute to soft-delete, or DELETE /api/mcp-deletions/${batchId} to cancel`, + }); +} diff --git a/backend/src/mcp/messageTools.test.js b/backend/src/mcp/messageTools.test.js new file mode 100644 index 00000000..a112fcaa --- /dev/null +++ b/backend/src/mcp/messageTools.test.js @@ -0,0 +1,489 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getMessage: vi.fn(), listMessages: vi.fn(), listAccounts: vi.fn(), getTotalStats: vi.fn(), aggregate: vi.fn(), searchByDomains: vi.fn(), + getMessageSummariesByIDs: vi.fn(), stageDeletion: vi.fn(), + resolveAccountScope: vi.fn(), messageInScope: vi.fn(), + }; +}); +vi.mock('./vectorStats.js', () => ({ collectStats: vi.fn() })); +vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchesInMessage: vi.fn() })); +import * as adapter from './engineAdapter.js'; +import { resolveAccountScope, messageInScope } from './engineAdapter.js'; +import { collectStats } from './vectorStats.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; +import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; + +// The one vector-availability gate returns { cfg, generation }; a VectorUnavailableError +// (name + reason) thrown from it is what every degraded/disabled path surfaces. +class VUE extends Error { constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } } +import { + handleGetMessage, handleListMessages, handleGetStats, handleAggregate, + handleSearchByDomains, handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, +} from './messageTools.js'; + +const scope = { userId: 'u', accountIds: ['acc-1'] }; +const detailRow = { + id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', + from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), + has_attachments: false, attachments: [], flags: [], folder: 'INBOX', + body_text: 'café — meeting notes and a much longer body '.repeat(100), body_html: '', +}; + +beforeEach(() => { + adapter.getMessage.mockReset(); + resolveAccountScope.mockReset(); + messageInScope.mockReset(); + // Defaults: no account narrowing; seed messages are in scope unless a test says otherwise. + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); + messageInScope.mockResolvedValue(true); +}); + +describe('get_message', () => { + it('404s a missing message with a string id', async () => { + adapter.getMessage.mockResolvedValue(null); + const r = await handleGetMessage({ id: 'nope' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + }); + + it('pages by BYTES, never splitting a rune, with correct body_length', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const total = Buffer.byteLength(detailRow.body_text, 'utf8'); + const r = await handleGetMessage({ id: 'm1', max_chars: 20 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_length).toBe(total); + expect(b.offset).toBe(0); + expect(b.body_returned).toBeLessThanOrEqual(20); + expect(b.has_more).toBe(true); + // returned slice is valid UTF-8 (no replacement char from a split é) + expect(b.body_text).not.toContain('�'); + expect(b.body_format).toBe('text'); + expect(b.id).toBe('m1'); // uuid string + // scoped read + expect(adapter.getMessage).toHaveBeenCalledWith('m1', ['acc-1']); + }); + + it('center_at round-trips a byte offset into a window around it', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const r = await handleGetMessage({ id: 'm1', center_at: 200, max_chars: 100 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.offset).toBeLessThanOrEqual(200); + expect(b.offset + b.body_returned).toBeGreaterThanOrEqual(200); + }); + + it('clamps max_chars to 4000', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: 'z'.repeat(9000) }); + const r = await handleGetMessage({ id: 'm1', max_chars: 99999 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_returned).toBe(4000); + }); + + it('full_body returns the whole body ignoring paging', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: 'z'.repeat(5000) }); + const r = await handleGetMessage({ id: 'm1', full_body: true }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_returned).toBe(5000); + expect(b.has_more).toBe(false); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const b = JSON.parse((await handleGetMessage({ id: 'm1' }, scope)).content[0].text); + expect(b.sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('html-only message pages the html body with body_format=html under auto', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: '', body_html: '

hello world

' }); + const r = await handleGetMessage({ id: 'm1' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_format).toBe('html'); + expect(b.body_html).toContain('hello'); + expect(b.body_text).toBe(''); + }); +}); + +describe('list_messages', () => { + beforeEach(() => { adapter.listMessages.mockReset(); adapter.listAccounts.mockReset(); }); + + it('returns a newest-first envelope with total=-1 and pages via has_more (limit+1 over-fetch)', async () => { + // 21 rows for a default limit of 20 -> has_more true, sliced to 20. + adapter.listMessages.mockResolvedValue(Array.from({ length: 21 }, (_, i) => ({ id: `m${i}` }))); + const r = await handleListMessages({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.total).toBe(-1); + expect(b.returned).toBe(20); + expect(b.has_more).toBe(true); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ accountIds: ['acc-1'], limit: 21, offset: 0 })); + }); + + it('threads a string conversation_id filter through to listMessages', async () => { + adapter.listMessages.mockResolvedValue([]); + await handleListMessages({ conversation_id: 'tid-1' }, scope); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ conversationId: 'tid-1' })); + }); + + it('resolves an account email to its id and narrows the scope', async () => { + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + adapter.listMessages.mockResolvedValue([]); + await handleListMessages({ account: 'c@d.com' }, scope); + expect(resolveAccountScope).toHaveBeenCalledWith('c@d.com', ['acc-1']); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ accountIds: ['acc-2'] })); + }); + + it('404-style errors an unknown account (msgvault getAccountID parity)', async () => { + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleListMessages({ account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('account not found: nope@x.com'); + }); + + it('rejects malformed after/before dates before touching the engine (msgvault getDateArg, handlers.go:265-275)', async () => { + for (const [key, bad] of [['after', '13/45/2024'], ['before', '2024-02-31']]) { + const r = await handleListMessages({ [key]: bad }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe(`invalid ${key} date "${bad}": expected YYYY-MM-DD`); + } + expect(adapter.listMessages).not.toHaveBeenCalled(); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + adapter.listMessages.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const r = await handleListMessages({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('get_stats', () => { + beforeEach(() => { adapter.getTotalStats.mockReset(); adapter.listAccounts.mockReset(); collectStats.mockReset(); }); + + const stats = { MessageCount: 10, TotalSize: 500, AttachmentCount: 2, AttachmentSize: 0, LabelCount: 3, AccountCount: 1, ActiveMessageCount: 10, SourceDeletedMessageCount: 0 }; + const accounts = [{ ID: 'acc-1', SourceType: 'imap', Identifier: 'a@b.com', DisplayName: 'Work' }]; + + it('omits vector_search when vector search is disabled (collectStats null)', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + collectStats.mockResolvedValue(null); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.stats).toEqual(stats); + expect(b.accounts).toEqual(accounts); + expect(b).not.toHaveProperty('vector_search'); + expect(adapter.getTotalStats).toHaveBeenCalledWith(['acc-1']); + }); + + it('includes the StatsView when vector search is enabled', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + const vs = { enabled: true, active_generation: null, missing_embeddings_total: 4 }; + collectStats.mockResolvedValue(vs); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.vector_search).toEqual(vs); + }); + + it('survives a broken vector sub-query (omits vector_search, keeps stats)', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + collectStats.mockRejectedValue(new Error('boom')); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.stats).toEqual(stats); + expect(b).not.toHaveProperty('vector_search'); + }); +}); + +describe('aggregate', () => { + beforeEach(() => { adapter.aggregate.mockReset(); adapter.listAccounts.mockReset(); }); + + it('requires group_by', async () => { + const r = await handleAggregate({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('group_by parameter is required'); + }); + + it('rejects an invalid group_by', async () => { + const r = await handleAggregate({ group_by: 'colour' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid group_by: colour'); + }); + + it('returns the raw AggregateRow array (no envelope), default limit 50, scoped', async () => { + const rows = [{ Key: 'a@b.com', Count: 3, TotalSize: 10, AttachmentSize: 0, AttachmentCount: 1, TotalUnique: 5 }]; + adapter.aggregate.mockResolvedValue(rows); + const r = await handleAggregate({ group_by: 'sender' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual(rows); + expect(adapter.aggregate).toHaveBeenCalledWith('sender', expect.objectContaining({ accountIds: ['acc-1'], limit: 50 })); + }); + + it('rejects malformed after/before dates instead of leaking a raw PG error (msgvault getDateArg)', async () => { + const r = await handleAggregate({ group_by: 'sender', after: 'notadate' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid after date "notadate": expected YYYY-MM-DD'); + expect(adapter.aggregate).not.toHaveBeenCalled(); + }); +}); + +describe('search_by_domains', () => { + beforeEach(() => adapter.searchByDomains.mockReset()); + + it('requires domains', async () => { + const r = await handleSearchByDomains({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('domains is required'); + }); + + it('splits/trims the CSV, default limit 100, scoped, returns the raw array', async () => { + const rows = [{ id: 'm1', from_email: 'x@gobright.com' }]; + adapter.searchByDomains.mockResolvedValue(rows); + const r = await handleSearchByDomains({ domains: ' gobright.com , ascentae.com ' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual(rows); + expect(adapter.searchByDomains).toHaveBeenCalledWith(['gobright.com', 'ascentae.com'], undefined, undefined, 100, 0, ['acc-1']); + }); + + it('rejects malformed after/before dates and emits sent_at as RFC3339 (Go wire format)', async () => { + const bad = await handleSearchByDomains({ domains: 'x.com', before: '2024-1-1' }, scope); + expect(bad.isError).toBe(true); + expect(bad.content[0].text).toBe('invalid before date "2024-1-1": expected YYYY-MM-DD'); + expect(adapter.searchByDomains).not.toHaveBeenCalled(); + + adapter.searchByDomains.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const ok = await handleSearchByDomains({ domains: 'x.com' }, scope); + expect(JSON.parse(ok.content[0].text)[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('find_similar_messages', () => { + beforeEach(() => { + resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); + adapter.getMessageSummariesByIDs.mockReset(); adapter.listAccounts.mockReset(); + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + }); + + it('returns vector_not_enabled when embeddings are disabled (config check before the resolver)', async () => { + resolveActiveGenerationFromConfig.mockRejectedValueOnce(new VUE('vector_not_enabled')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('maps a VectorUnavailableError reason from the resolver to its wire string', async () => { + resolveActiveGenerationFromConfig.mockRejectedValue(new VUE('no_active_generation')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^no_active_generation:/); + }); + + it('excludes the seed, hydrates in rank order via annSearch(gen.id, ...), scoped', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([ + { messageId: 'm1', score: 0.9, rank: 1 }, + { messageId: 'seed', score: 1, rank: 0 }, + { messageId: 'm2', score: 0.7, rank: 2 }, + ]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1' }, { id: 'm2' }]); + const r = await handleFindSimilarMessages({ message_id: 'seed', limit: 20 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.seed_message_id).toBe('seed'); + expect(b.returned).toBe(2); + expect(b.generation).toEqual({ id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }); + expect(b.messages).toEqual([{ id: 'm1' }, { id: 'm2' }]); + // real annSearch takes the generation ID and the {filter} with accountIds + expect(annSearch).toHaveBeenCalledWith(3, [0.1, 0.2], 21, { filter: { accountIds: ['acc-1'] } }); + expect(adapter.getMessageSummariesByIDs).toHaveBeenCalledWith(['m1', 'm2'], ['acc-1']); + }); + + it('reports a readable error when the seed has no embedding', async () => { + loadVector.mockRejectedValue(new Error('no embedding for message seed')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('load seed vector: no embedding for message seed'); + }); + + it('rejects a foreign/out-of-scope seed id without loading its vector (owner-scope isolation)', async () => { + messageInScope.mockResolvedValue(false); // seed belongs to another user + const r = await handleFindSimilarMessages({ message_id: 'foreign-seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + expect(loadVector).not.toHaveBeenCalled(); + expect(annSearch).not.toHaveBeenCalled(); + expect(messageInScope).toHaveBeenCalledWith('foreign-seed', ['acc-1']); + }); + + it('applies the advertised message_type filter to hydrated results (msgvault handlers.go:1042-1044)', async () => { + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: 'm1', score: 0.9, rank: 1 }]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1', message_type: 'email' }]); + const none = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed', message_type: 'sms' }, scope)).content[0].text); + expect(none.messages).toEqual([]); + expect(none.returned).toBe(0); + const all = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed', message_type: 'email' }, scope)).content[0].text); + expect(all.messages).toHaveLength(1); + }); + + it('clamps limit to the hybrid page cap (msgvault handlers.go:885-888)', async () => { + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([]); + adapter.getMessageSummariesByIDs.mockResolvedValue([]); + await handleFindSimilarMessages({ message_id: 'seed', limit: 5000 }, scope); + expect(annSearch).toHaveBeenCalledWith(3, [0.1, 0.2], 101, expect.anything()); // 100 + 1 seed over-fetch + }); + + it('rejects malformed after/before dates and emits sent_at as RFC3339 (Go wire format)', async () => { + const bad = await handleFindSimilarMessages({ message_id: 'seed', after: '2024-13-45' }, scope); + expect(bad.isError).toBe(true); + expect(bad.content[0].text).toBe('invalid after date "2024-13-45": expected YYYY-MM-DD'); + expect(annSearch).not.toHaveBeenCalled(); + + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: 'm1', score: 0.9, rank: 1 }]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const ok = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed' }, scope)).content[0].text); + expect(ok.messages[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('search_in_message', () => { + const body = 'café note — the budget is set here'; + const detail = { id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), has_attachments: false, attachments: [], flags: [], folder: 'INBOX', body_text: body, body_html: '' }; + beforeEach(() => { adapter.getMessage.mockReset(); matchesInMessage.mockReset(); }); + + it('keyword mode returns a byte char_offset + real total; the offset round-trips into get_message center_at', async () => { + adapter.getMessage.mockResolvedValue(detail); + const r = await handleSearchInMessage({ id: 'm1', query: 'budget' }, scope); + const b = JSON.parse(r.content[0].text); + const byteOffset = Buffer.from(body, 'utf8').indexOf(Buffer.from('budget', 'utf8')); + expect(b.total).toBe(1); + expect(b.data[0].char_offset).toBe(byteOffset); // BYTE offset (not code-point index) + expect(b.data[0]).not.toHaveProperty('score'); // keyword = no score + const g = JSON.parse((await handleGetMessage({ id: 'm1', center_at: byteOffset, max_chars: 100 }, scope)).content[0].text); + expect(g.offset).toBeLessThanOrEqual(byteOffset); + expect(g.offset + g.body_returned).toBeGreaterThanOrEqual(byteOffset); + }); + + it('404s a missing message in keyword mode', async () => { + adapter.getMessage.mockResolvedValue(null); + const r = await handleSearchInMessage({ id: 'x', query: 'q' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + }); + + it('vector mode maps VectorUnavailableError to vector_not_enabled (stock Postgres)', async () => { + class VUE extends Error { constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } } + matchesInMessage.mockRejectedValue(new VUE('vector_not_enabled')); + const r = await handleSearchInMessage({ id: 'm1', query: 'travel', mode: 'vector' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('vector mode pages scored matches from matchesInMessage (scoped)', async () => { + matchesInMessage.mockResolvedValue([{ snippet: 'a', score: 0.9 }, { snippet: 'b', score: 0.8 }]); + const r = await handleSearchInMessage({ id: 'm1', query: 'travel', mode: 'vector', limit: 1 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.total).toBe(2); + expect(b.returned).toBe(1); + expect(b.has_more).toBe(true); + expect(matchesInMessage).toHaveBeenCalledWith('m1', 'travel', 0, { accountIds: ['acc-1'] }); + }); + + it('rejects an unknown mode', async () => { + const r = await handleSearchInMessage({ id: 'm1', query: 'q', mode: 'fuzzy' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid mode "fuzzy": must be keyword (default) or vector'); + }); +}); + +describe('stage_deletion', () => { + beforeEach(() => adapter.stageDeletion.mockReset()); + + it('rejects query + structured filters together', async () => { + const r = await handleStageDeletion({ query: 'from:x', from: 'y@z.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("use either 'query' or structured filters (from, domain, label, etc.), not both"); + }); + + it('rejects neither query nor filters', async () => { + const r = await handleStageDeletion({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + }); + + it('rejects an all-whitespace query with no structured filters (empty/missing query guard)', async () => { + const r = await handleStageDeletion({ query: ' ' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('rejects a query with an unsupported operator via the taxonomy error — never stages the whole mailbox', async () => { + const r = await handleStageDeletion({ query: 'label:promotions' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('refuses a negation-only / sub-2-char / punctuation-only query (all tokens discarded)', async () => { + for (const query of ['-newsletter', 'a', '!!!']) { + adapter.stageDeletion.mockReset(); + const r = await handleStageDeletion({ query }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('query produced no enforceable filters (all query terms were discarded as too short, punctuation-only, or negation-only); refusing to stage deletions'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + } + }); + + it('rejects a mixed query too — dropping label: would stage a SUPERSET of what was asked (msgvault handlers.go:1818-1821)', async () => { + const r = await handleStageDeletion({ query: 'invoice label:promotions' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim (msgvault q.Err front-door rule)', async () => { + const r = await handleStageDeletion({ query: 'invoice older_than:xyz' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('rejects malformed structured after/before dates (msgvault getDateArg, handlers.go:1793-1800)', async () => { + const r = await handleStageDeletion({ after: '13/45/2024' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid after date "13/45/2024": expected YYYY-MM-DD'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('errors when no messages match', async () => { + adapter.stageDeletion.mockResolvedValue({ batchId: null, messageCount: 0 }); + const r = await handleStageDeletion({ from: 'linkedin.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('no messages match the specified criteria'); + }); + + it('stages a batch with wire status "pending" (msgvault manifest.StatusPending, manifest.go:25) — never soft-deletes', async () => { + adapter.stageDeletion.mockResolvedValue({ batchId: 'batch-9', messageCount: 5 }); + const r = await handleStageDeletion({ domain: 'linkedin.com' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual({ + batch_id: 'batch-9', message_count: 5, status: 'pending', + next_step: 'POST /api/mcp-deletions/batch-9/execute to soft-delete, or DELETE /api/mcp-deletions/batch-9 to cancel', + }); + // batch is scoped to the token user (cross-user isolation) and never flips is_deleted here + expect(adapter.stageDeletion).toHaveBeenCalledWith(expect.objectContaining({ userId: 'u', accountIds: ['acc-1'], domain: 'linkedin.com' })); + }); +}); diff --git a/backend/src/mcp/result.js b/backend/src/mcp/result.js new file mode 100644 index 00000000..f416103a --- /dev/null +++ b/backend/src/mcp/result.js @@ -0,0 +1,9 @@ +// Mirror msgvault's mcp.NewToolResultText / mcp.NewToolResultError: every tool +// returns a single text content block; errors additionally set isError. +export function jsonResult(obj) { + return { content: [{ type: 'text', text: JSON.stringify(obj) }] }; +} + +export function errorResult(msg) { + return { content: [{ type: 'text', text: msg }], isError: true }; +} diff --git a/backend/src/mcp/result.test.js b/backend/src/mcp/result.test.js new file mode 100644 index 00000000..a9e118b0 --- /dev/null +++ b/backend/src/mcp/result.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { jsonResult, errorResult } from './result.js'; +import { HANDLERS, TOOL_DEFS } from './tools.js'; + +describe('result helpers', () => { + it('jsonResult wraps a JSON string in a text content block', () => { + expect(jsonResult({ ok: true })).toEqual({ + content: [{ type: 'text', text: '{"ok":true}' }], + }); + }); + it('errorResult marks isError and carries the message verbatim', () => { + expect(errorResult('boom')).toEqual({ + content: [{ type: 'text', text: 'boom' }], + isError: true, + }); + }); +}); + +describe('ping tool', () => { + it('is registered with a JSON-schema input', () => { + const def = TOOL_DEFS.find((t) => t.name === 'ping'); + expect(def).toBeTruthy(); + expect(def.inputSchema).toEqual({ type: 'object', properties: {} }); + }); + it('echoes pong and is scope-agnostic', async () => { + const r = await HANDLERS.ping({}, { userId: 'u', accountIds: [] }); + expect(r.content[0].text).toBe('{"pong":true}'); + }); +}); diff --git a/backend/src/mcp/searchTools.js b/backend/src/mcp/searchTools.js new file mode 100644 index 00000000..463a7e6f --- /dev/null +++ b/backend/src/mcp/searchTools.js @@ -0,0 +1,414 @@ +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; +import { jsonResult, errorResult } from './result.js'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, wireSummary } from './envelope.js'; +import { extractContextChar } from './bodyMatch.js'; +import { translateVectorError } from './vectorErrors.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; + +// The search seam returns raw REST-shaped rows (subject/from/date/snippet/… under a +// column set REST froze); MCP must emit msgvault MessageSummary. We hydrate the hit +// ids through engineAdapter (recipients, thread_id, attachments — data the ranked row +// set doesn't carry) so every search tool speaks the same wire shape as the message +// tools. Order is preserved by getMessageSummariesByIDs; ids are already in scope. +// wireSummary re-formats sent_at to Go RFC3339 (no millis) at the wire. +async function hydrateSummaries(messages, accountIds) { + const ids = (messages || []).map((m) => m.id); + const summaries = await getMessageSummariesByIDs(ids, accountIds); + return new Map(summaries.map((s) => [s.id, wireSummary(s)])); +} + +// msgvault front doors reject queries whose known operators carry unparseable +// values (q.Err(), internal/search/parser.go:45-52; returned verbatim at +// handlers.go:373-375) instead of silently dropping the filter and returning +// wider-than-requested results. parsed.errors carries the verbatim messages; +// errors.Join separates with newlines. +export function queryParseErrorMessage(parsed) { + const errs = (parsed && parsed.errors) || []; + return errs.length ? errs.join('\n') : ''; +} + +// Port of unsupportedSearchOperatorMessage (msgvault handlers.go:408-428) with +// per-operator reasons. msgvault's parser-level unsupported set is only +// list:/list-id: (Gmail-only, parser.go:270-273); Mailflow additionally cannot +// serve label:/l:, bcc:, larger:, smaller: — msgvault supports those four, but +// Mailflow's messages schema stores no labels, BCC recipients, or byte sizes +// (documented divergence). The taxonomy prefix is verbatim. +const UNSUPPORTED_OPERATOR_REASONS = { + list: 'is Gmail-only syntax (this server does not index List-ID); use supported operators instead', + 'list-id': 'is Gmail-only syntax (this server does not index List-ID); use supported operators instead', + label: 'is not supported on this server (Mailflow stores no Gmail labels)', + bcc: 'is not supported on this server (Mailflow does not store BCC recipients)', + larger: 'is not supported on this server (Mailflow does not store message sizes)', + smaller: 'is not supported on this server (Mailflow does not store message sizes)', +}; + +// list:/list-id: live in msgvault's parser-level unsupported set; Mailflow's +// queryParser (Wave D's file) currently leaves them as literal free-text +// terms, so recognize them here at the handler seam. +// TODO(seam): consolidate into queryParser's unsupported set with Wave D. +function unsupportedListOperators(parsed) { + const out = []; + for (const t of (parsed && parsed.terms) || []) { + const m = /^(list|list-id):/i.exec(t.value || ''); + if (m) out.push(m[1].toLowerCase()); + } + return out; +} + +export function unsupportedSearchOperatorMessage(parsed) { + const names = []; + const seen = new Set(); + const push = (name) => { if (name && !seen.has(name)) { seen.add(name); names.push(name); } }; + for (const u of (parsed && parsed.unsupported) || []) push(u.key); + for (const n of unsupportedListOperators(parsed)) push(n); + if (!names.length) return ''; + // Group operators sharing a reason, msgvault-style ("name:, name2: "), + // preserving first-appearance order. + const groups = []; + const byReason = new Map(); + for (const n of names) { + const reason = UNSUPPORTED_OPERATOR_REASONS[n] || 'is not supported on this server'; + let g = byReason.get(reason); + if (!g) { g = { reason, ops: [] }; byReason.set(reason, g); groups.push(g); } + g.ops.push(`${n}:`); + } + return 'unsupported_search_operator: ' + + groups.map((g) => `${g.ops.join(', ')} ${g.reason}`).join('; '); +} + +// msgvault clamps hybrid paging to [vector.search].max_page_size_hybrid +// (default 50; vector/config.go:196-205,339-342, enforced at handlers.go:651-668). +// Mailflow has no config knob; its effective ranking window is the fused +// per-signal candidate cap (hybrid.js K_PER_SIGNAL = 100) — offsets past it +// can only return silently-empty pages, so reject them the msgvault way. +// TODO(seam): read this from hybrid.js if Wave D exports the per-signal cap. +export const HYBRID_RANKING_WINDOW = 100; + +// Tool descriptions/schemas follow msgvault internal/mcp/server.go, with the +// Mailflow divergences spelled out in the text: no label:/bcc:/larger:/smaller: +// (schema lacks the data — msgvault supports them), negation IS supported +// (msgvault does not), free-text ordering is relevance-ranked (D5), and +// semantic hits carry at most 1 excerpt. README D6: only id-bearing fields +// diverge to UUID strings — search tools have none. +const SEARCH_METADATA_OPERATOR_DOC = + 'Supported operators: from:, to:, cc:, subject:, has:attachment, ' + + 'before:/after: (YYYY-MM-DD), older_than:/newer_than: (e.g. 7d, 2w, 1m, 1y). ' + + 'Bare domains on from:/to: match any address at that domain. Multiple terms are ANDed. ' + + 'Rejected as unsupported on this server (divergence from msgvault, which supports them): ' + + 'label: (or l:), bcc:, larger:, smaller: — Mailflow stores no labels, BCC recipients, or message sizes; ' + + 'list:/list-id: are Gmail-only and also rejected. ' + + 'Negation with a leading - (e.g. -from:alice, -invoice) IS supported (divergence from msgvault); ' + + 'OR and parentheses grouping are not.'; +const SEARCH_METADATA_FREETEXT_DOC = + 'Free text matches subject, snippet, and sender/recipient metadata only (not bodies). ' + + 'Use search_message_bodies for body keywords or semantic_search_messages for vector/hybrid search.'; +const SEARCH_METADATA_PAGINATION_DOC = + 'Free-text results are relevance-ranked (divergence from msgvault); filter-only queries ' + + 'are ordered newest-first (by sent date). There is no sort parameter — ' + + 'use before:/after: to scope a date range. ' + + 'Paginate with offset/limit (default limit 20, max 50). ' + + 'Response: data, total, returned, offset, has_more.'; + +export const searchMetadataDef = { + name: 'search_metadata', + description: + 'Search message metadata using a subset of Gmail query syntax (not full Gmail compatibility). ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + SEARCH_METADATA_FREETEXT_DOC + ' ' + + SEARCH_METADATA_PAGINATION_DOC + + 'For body keywords use search_message_bodies; for vector/hybrid search use semantic_search_messages.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: "Search query (e.g. 'from:alice subject:meeting after:2024-01-01'). See tool description for supported operators and limitations." }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + required: ['query'], + }, +}; + +const DEFAULT_SEARCH_LIMIT = 20; +const MAX_SEARCH_LIMIT = 50; +export function searchLimitArg(args) { + const v = Number(args.limit); + if (!Number.isFinite(v) || v <= 0) return DEFAULT_SEARCH_LIMIT; + return Math.min(Math.trunc(v), MAX_SEARCH_LIMIT); +} +export function offsetArg(args) { + const v = Number(args.offset); + if (!Number.isFinite(v) || v < 0) return 0; + return Math.trunc(v); +} + +export async function handleSearchMetadata(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:372-378): parse-value errors verbatim, + // then unsupported operators — never silently widen the result set. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Narrow to a single account when `account` is given (msgvault getAccountID) — + // searchService trusts a pre-resolved accountIds as-is, so the narrowing must + // happen here (it never re-reads the `account` email). + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const result = await search({ + mode: 'lexical', scope: 'metadata', + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit, offset, + }); + const byId = await hydrateSummaries(result.messages, acc.accountIds); + const data = (result.messages || []).map((m) => byId.get(m.id)).filter(Boolean); + // total is ALWAYS present on this envelope (msgvault SearchFastCount is a + // real count, handlers.go:400-405). The seam omits it on degenerate queries + // whose terms were all dropped — those return zero rows, so total is 0. + const total = Number.isFinite(result.total) ? result.total : 0; + return jsonResult(newPaginatedResponse(data, total, offset)); +} + +export const searchMessageBodiesDef = { + name: 'search_message_bodies', + description: + 'Keyword full-text search over message bodies. ' + + 'Returns messages whose body text contains the query terms, relevance-ranked, ' + + 'each with matches — up to 5 excerpt snippets centered on matched terms. ' + + 'Backend excerpts may omit char_offset and line when efficient source locations are unavailable; use search_in_message when exact locations are needed. ' + + 'When matches_truncated is true on a hit, more than 5 excerpts matched — use search_in_message or get_message to read the full body. ' + + 'Known Gmail operators (from:, subject:, etc.) apply as metadata filters only and do not satisfy the free-text requirement. ' + + 'Filter-only queries such as from:alice are rejected — use search_metadata for filter-only queries. ' + + 'Unrecognized word:value tokens (e.g. RXD2:V2) are treated as literal body text, not filters. ' + + 'Query syntax: space-separated words are ANDed (each must appear somewhere in the body); ' + + 'a double-quoted phrase is one exact phrase (e.g. "RXD2 V2"); negation with a leading - excludes a term ' + + '(divergence from msgvault); OR is not supported. ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + + 'Results are relevance-ranked, best lexical match first (divergence from msgvault, which orders newest-first). ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more. ' + + 'Body search does not return a total; use has_more to detect more pages.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Body search query with at least one free-text term (bare word or quoted phrase). Gmail operators (from:, subject:, etc.) are metadata filters, not body search — subject:test alone is rejected; combine with body terms (from:alice budget) or use search_metadata for filter-only queries. Unrecognized word:value tokens (RXD2:V2) are literal text. Space-separated words are ANDed; double quotes match an exact phrase; a leading - negates a term; OR unsupported.' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + required: ['query'], + }, +}; + +const EMPTY_GENERATION = { id: 0, model: '', dimension: 0, fingerprint: '', state: '' }; +const MAX_CONTEXT_SNIPPETS = 5; +const SEARCH_CONTEXT_CHARS = 300; + +function freeTextTerms(parsed) { + return (parsed.terms || []).filter((t) => !t.negate).map((t) => t.value); +} + +// hybridScoreBreakdown omitempty parity (msgvault handlers.go:563-572): all +// signal fields are pointer-typed with omitempty — rrf is omitted in +// mode=vector (one signal, nothing to fuse) and subject_boosted when false. +// The seam's explain object always carries rrf + a boolean subject_boosted. +function wireScore(score, mode) { + const out = {}; + if (mode === 'hybrid' && score.rrf != null) out.rrf = score.rrf; + if (score.bm25 != null) out.bm25 = score.bm25; + if (score.vector != null) out.vector = score.vector; + if (score.subject_boosted) out.subject_boosted = true; + return out; +} + +export async function handleSearchMessageBodies(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + // Keyword-only tool: an explicit vector/hybrid mode is rejected, not + // ignored (msgvault handlers.go:442-457 — same two wordings). + const mode = args.mode || 'keyword'; + if (mode === 'vector' || mode === 'hybrid') { + return errorResult( + `invalid mode "${mode}": search_message_bodies is keyword-only; use semantic_search_messages for vector or hybrid search`, + ); + } + if (mode !== 'keyword') { + return errorResult( + `invalid mode "${mode}": search_message_bodies only supports keyword search; use semantic_search_messages for vector or hybrid search`, + ); + } + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:459-465): parse errors, then unsupported + // operators, before the free-text requirement. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const terms = freeTextTerms(parsed); + if (!terms.length) { + return errorResult( + 'search_message_bodies requires at least one free-text term (bare word or quoted phrase); ' + + 'Gmail operators such as from: or subject: are metadata filters and do not count — ' + + 'use search_metadata for filter-only queries', + ); + } + const limit = searchLimitArg(args); + const offset = offsetArg(args); + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + // Over-fetch one to compute has_more without a count query (msgvault limit+1). + const result = await search({ + mode: 'lexical', scope: 'body', + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit: limit + 1, offset, + }); + let hits = result.messages || []; + const hasMore = hits.length > limit; + if (hasMore) hits = hits.slice(0, limit); + + const byId = await hydrateSummaries(hits, acc.accountIds); + const data = hits.map((m) => { + const summary = byId.get(m.id); + if (!summary) return null; + const snippets = extractContextChar(m.body_text || '', terms, SEARCH_CONTEXT_CHARS) || []; + const capped = snippets.slice(0, MAX_CONTEXT_SNIPPETS); + // Go omitempty parity (msgvault handlers.go:339-341): matches is omitted + // when empty and matches_truncated when false — never emitted as []/false. + const item = { ...summary }; + if (capped.length) item.matches = capped.map((snippet) => ({ snippet })); + if (snippets.length > MAX_CONTEXT_SNIPPETS) item.matches_truncated = true; + return item; + }).filter(Boolean); + + return jsonResult({ + ...newPaginatedResponseNoTotal(data, offset, hasMore), + mode: 'keyword', + pool_saturated: false, + generation: EMPTY_GENERATION, + }); +} + +export const semanticSearchMessagesDef = { + name: 'semantic_search_messages', + description: + 'Semantic (embedding) search over each preprocessed message subject and body. ' + + 'Returns messages ranked by similarity to the query — there is no exact total, so page on has_more. ' + + 'Each hit includes matches — at most 1 best-matching embedded subject/body chunk excerpt with a score (divergence from msgvault, which returns up to 5). ' + + 'Vector char_offset and line locations may be omitted because preprocessing usually prevents exact raw-body mapping; use snippet terms with search_in_message keyword mode when navigation is needed. ' + + 'min_score filters chunk excerpts only; it does not remove or reorder ranked messages. ' + + 'Requires at least one free-text term (used to embed); filter-only queries must use search_metadata. ' + + 'Known Gmail operators (from:, subject:, etc.) apply as metadata filters only. ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + + 'mode=vector for pure semantic search or mode=hybrid to fuse BM25 and vector ranking via RRF. ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more, mode, pool_saturated, generation. ' + + 'total is not available; use has_more to page.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Free-text query to embed (requires at least one free-text term). Gmail operators are metadata filters, not body search; combine with body terms or use search_metadata for filter-only queries.' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + mode: { type: 'string', enum: ['vector', 'hybrid'], description: 'Search mode: vector (semantic only) or hybrid (BM25 + vector fused via RRF). Defaults to hybrid when omitted.' }, + explain: { type: 'boolean', description: 'Include per-signal scores in the response (for debugging or ranking inspection)' }, + min_score: { type: 'number', description: 'Minimum chunk similarity score for included match excerpts (default 0); does not filter ranked messages' }, + }, + required: ['query'], + }, +}; + +export async function handleSemanticSearchMessages(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + + const mode = args.mode || 'hybrid'; + if (mode !== 'vector' && mode !== 'hybrid') { + return errorResult( + `invalid mode "${mode}": must be vector or hybrid (default hybrid); use search_message_bodies for keyword search`, + ); + } + + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:552-558): parse errors, then unsupported + // operators, before the free-text requirement. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const terms = freeTextTerms(parsed); + if (!terms.length) { + return errorResult( + `missing_free_text: mode=${mode} requires at least one free-text term; use search_metadata for filter-only queries`, + ); + } + + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Offsets past the ranked window cannot be served — reject them instead of + // returning silently-empty pages (msgvault handlers.go:656-663 wording). + if (offset >= HYBRID_RANKING_WINDOW) { + return errorResult( + `pagination_limit: offset ${offset} exceeds hybrid ranking window (max ${HYBRID_RANKING_WINDOW}); ` + + 'use search_metadata or search_message_bodies for deeper pagination', + ); + } + const explain = args.explain === true; + const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 0; + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + + // strictVector: the seam rethrows VectorUnavailableError (msgvault taxonomy) + // instead of the REST silent-lexical fallback. + let result; + try { + result = await search({ + mode, scope: 'body', strictVector: true, + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit, offset, explain, minScore, + }); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + if (err.name === 'MissingFreeTextError') { + // The seam applies stricter term hygiene (sub-2-char / punctuation-only + // tokens never embed) than the raw-terms pre-check above; surface the + // same msgvault wording (handlers.go:631-635) instead of letting it + // escape the tool call as `internal error: missing_free_text`. + return errorResult( + `missing_free_text: mode=${mode} requires at least one free-text term; use search_metadata for filter-only queries`, + ); + } + throw err; + } + + // Phase 4 surfaces best_chunk per hit (chunk_index + code-point char_start/char_end + // into preprocessed text + score). Phase 5 owns the snippet + raw-body byte offsets: + // build the wire matches[] (≤1 excerpt — documented divergence from msgvault's ≤5) + // from best_chunk via chunkmatch.matchFromChunk. + const byId = await hydrateSummaries(result.messages, acc.accountIds); + const data = (await Promise.all((result.messages || []).map(async (m) => { + const summary = byId.get(m.id); + if (!summary) return null; + const item = { ...summary }; + if (m.best_chunk) { + const match = await matchFromChunk(m.id, m.best_chunk, { accountIds: acc.accountIds }); + if (match && match.score >= minScore) item.matches = [match]; + } + if (explain && m.score) item.score = wireScore(m.score, mode); + return item; + }))).filter(Boolean); + + return jsonResult({ + ...newPaginatedResponseNoTotal(data, offset, result.page?.hasMore || false), + mode, + pool_saturated: result.pool_saturated || false, + generation: result.generation || EMPTY_GENERATION, + }); +} diff --git a/backend/src/mcp/searchTools.test.js b/backend/src/mcp/searchTools.test.js new file mode 100644 index 00000000..45f18c98 --- /dev/null +++ b/backend/src/mcp/searchTools.test.js @@ -0,0 +1,389 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/search/queryParser.js', () => ({ parseQuery: vi.fn() })); +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +// The search seam returns raw rows; MCP hydrates hit ids into MessageSummary shape. +vi.mock('./engineAdapter.js', () => ({ getMessageSummariesByIDs: vi.fn(), resolveAccountScope: vi.fn() })); +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; + +class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } +} + +const scope = { userId: 'u1', accountIds: ['acc-1'] }; +function payload(r) { return JSON.parse(r.content[0].text); } +// Hydration echoes each requested id as a minimal summary unless a test overrides it. +function echoSummaries(rows) { + getMessageSummariesByIDs.mockImplementation(async (ids) => ids.map((id) => rows.find((r) => r.id === id)).filter(Boolean)); +} + +beforeEach(() => { + parseQuery.mockReset(); search.mockReset(); matchFromChunk.mockReset(); getMessageSummariesByIDs.mockReset(); + resolveAccountScope.mockReset(); + // Default: no `account` narrowing — pass the token's full scope through. + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); +}); + +describe('search_metadata', () => { + it('requires a query', async () => { + const r = await handleSearchMetadata({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('query parameter is required'); + }); + + it('hydrates hits into the msgvault envelope with a real total, scoped to the token accounts', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + search.mockResolvedValue({ + messages: [{ id: 'm1' }], // raw seam row — only the id is load-bearing for hydration + total: 5, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: true }, + }); + echoSummaries([{ id: 'm1', subject: 'Hi', to: [{ Email: 'c@d.com', Name: 'C' }] }]); + const r = await handleSearchMetadata({ query: 'from:alice' }, scope); + const body = payload(r); + expect(body).toEqual({ data: [{ id: 'm1', subject: 'Hi', to: [{ Email: 'c@d.com', Name: 'C' }] }], total: 5, returned: 1, offset: 0, has_more: true }); + // scope + parsed handed to the seam; MCP never builds SQL; hydration is scoped + expect(search).toHaveBeenCalledWith(expect.objectContaining({ mode: 'lexical', scope: 'metadata', accountIds: ['acc-1'], limit: 20, offset: 0 })); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-1']); + }); + + it('clamps limit to 50', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + search.mockResolvedValue({ messages: [], total: 0, mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + getMessageSummariesByIDs.mockResolvedValue([]); + await handleSearchMetadata({ query: 'x', limit: 999 }, scope); + expect(search.mock.calls[0][0].limit).toBe(50); + }); + + it('narrows scope to a resolved account id (the account arg is no longer a silent no-op)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + search.mockResolvedValue({ messages: [{ id: 'm1' }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', subject: 'Hi' }]); + await handleSearchMetadata({ query: 'x', account: 'work@x.com' }, scope); + expect(resolveAccountScope).toHaveBeenCalledWith('work@x.com', ['acc-1']); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); // narrowed, not the full token scope + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); // hydration narrowed too + }); + + it('rejects an unknown account without querying (msgvault getAccountID parity)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSearchMetadata({ query: 'x', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('account not found: nope@x.com'); + expect(search).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim instead of silently widening (msgvault q.Err, handlers.go:373-375)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [], + errors: ['invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'], + }); + const r = await handleSearchMetadata({ query: 'older_than:xyz' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects schema-unsupported operators with the unsupported_search_operator taxonomy (handlers.go:408-428)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'x', negate: false }], + unsupported: [{ key: 'label', token: 'label:promos' }, { key: 'larger', token: 'larger:5M' }], + errors: [], + }); + const r = await handleSearchMetadata({ query: 'x label:promos larger:5M' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(r.content[0].text).toContain('larger:'); + expect(search).not.toHaveBeenCalled(); + }); + + it('treats literal list:/list-id: terms as unsupported (msgvault parser set, parser.go:270-273, hoisted to the handler)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'list-id:foo', negate: false }], unsupported: [], errors: [] }); + const r = await handleSearchMetadata({ query: 'list-id:foo' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: list-id: is Gmail-only syntax/); + expect(search).not.toHaveBeenCalled(); + }); + + it('always carries a numeric total, even when the seam omits it (degenerate all-terms-dropped query)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'a', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); // no total key + getMessageSummariesByIDs.mockResolvedValue([]); + const body = payload(await handleSearchMetadata({ query: 'a' }, scope)); + expect(body.total).toBe(0); + expect(body.has_more).toBe(false); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ messages: [{ id: 'm1' }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const body = payload(await handleSearchMetadata({ query: 'x' }, scope)); + expect(body.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('search_message_bodies', () => { + it('rejects filter-only queries (no free-text term)', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + const r = await handleSearchMessageBodies({ query: 'from:alice' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toContain('requires at least one free-text term'); + }); + + it('returns keyword envelope: total -1, mode keyword, empty generation, snippet-only matches on a hydrated summary', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + search.mockResolvedValue({ + messages: [{ id: 'm1', body_text: 'the budget is approved. budget notes.' }], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + echoSummaries([{ id: 'm1', subject: 'Q3', from_email: 'a@b.com' }]); + const r = await handleSearchMessageBodies({ query: 'budget' }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.total).toBe(-1); + expect(body.mode).toBe('keyword'); + expect(body.pool_saturated).toBe(false); + expect(body.generation).toEqual({ id: 0, model: '', dimension: 0, fingerprint: '', state: '' }); + expect(body.data[0].subject).toBe('Q3'); // hydrated summary, not the raw row + expect(body.data[0]).not.toHaveProperty('body_text'); // body is not leaked to the wire + expect(body.data[0].matches[0]).toHaveProperty('snippet'); + expect(body.data[0].matches[0]).not.toHaveProperty('char_offset'); // keyword body = snippet only + }); + + it('narrows scope to a resolved account id', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + search.mockResolvedValue({ messages: [{ id: 'm1', body_text: 'budget' }], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', subject: 'Q3' }]); + await handleSearchMessageBodies({ query: 'budget', account: 'work@x.com' }, scope); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); + }); + + it('rejects an unknown account', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSearchMessageBodies({ query: 'budget', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects explicit mode=vector/hybrid — keyword-only tool (msgvault handlers.go:447-452)', async () => { + for (const mode of ['vector', 'hybrid']) { + const r = await handleSearchMessageBodies({ query: 'budget', mode }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + `invalid mode "${mode}": search_message_bodies is keyword-only; use semantic_search_messages for vector or hybrid search`, + ); + } + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects unknown modes with the only-supports wording (msgvault handlers.go:453-457)', async () => { + const r = await handleSearchMessageBodies({ query: 'budget', mode: 'fuzzy' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + 'invalid mode "fuzzy": search_message_bodies only supports keyword search; use semantic_search_messages for vector or hybrid search', + ); + }); + + it('accepts an explicit mode=keyword', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + getMessageSummariesByIDs.mockResolvedValue([]); + const r = await handleSearchMessageBodies({ query: 'budget', mode: 'keyword' }, scope); + expect(r.isError).toBeFalsy(); + }); + + it('returns parser value errors and unsupported operators before the free-text check (msgvault ordering)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [{ key: 'bcc', token: 'bcc:x@y.com' }], errors: [], + }); + const r = await handleSearchMessageBodies({ query: 'bcc:x@y.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: bcc: /); + + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [], + errors: ['invalid value "5X" for smaller: — expected a size like 5M, 100K, or 1G'], + }); + const r2 = await handleSearchMessageBodies({ query: 'smaller:5X' }, scope); + expect(r2.content[0].text).toBe('invalid value "5X" for smaller: — expected a size like 5M, 100K, or 1G'); + expect(search).not.toHaveBeenCalled(); + }); + + it('OMITS matches/matches_truncated when empty/false and emits matches_truncated past 5 excerpts (Go omitempty, handlers.go:339-341)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + // m1: term absent from the body → no snippets; m2: 7 occurrences spaced + // past the 300-byte context window (no merge) → 5 capped + truncated. + const spread = Array.from({ length: 7 }, () => 'budget').join(' filler'.repeat(120)); + search.mockResolvedValue({ + messages: [{ id: 'm1', body_text: 'nothing relevant here' }, { id: 'm2', body_text: spread }], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + echoSummaries([{ id: 'm1', subject: 'A' }, { id: 'm2', subject: 'B' }]); + const body = payload(await handleSearchMessageBodies({ query: 'budget' }, scope)); + expect(body.data[0]).not.toHaveProperty('matches'); + expect(body.data[0]).not.toHaveProperty('matches_truncated'); + expect(body.data[1].matches).toHaveLength(5); + expect(body.data[1].matches_truncated).toBe(true); + }); +}); + +describe('semantic_search_messages', () => { + it('rejects filter-only queries with missing_free_text', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + const r = await handleSemanticSearchMessages({ query: 'from:alice' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^missing_free_text: mode=hybrid/); + }); + + it('rejects mode=keyword', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + const r = await handleSemanticSearchMessages({ query: 'x', mode: 'keyword' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/invalid mode "keyword"/); + }); + + it('returns vector_not_enabled when the seam is strict-unavailable', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + search.mockRejectedValue(new VectorUnavailableError('vector_not_enabled')); + const r = await handleSemanticSearchMessages({ query: 'travel plans' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('builds matches from best_chunk via matchFromChunk on a hydrated summary, with mode/pool_saturated/generation and per-signal scores when explain', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + matchFromChunk.mockResolvedValue({ char_offset: 42, snippet: 'flights', line: 3, score: 0.9 }); + search.mockResolvedValue({ + // Real hybrid/vector seam shape: keyed on message_id, with the additive + // `id` alias searchService now emits (mode-invariant with the lexical path). + messages: [{ message_id: 'm1', id: 'm1', + best_chunk: { chunk_index: 2, char_start: 100, char_end: 180, score: 0.9 }, + score: { rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: true } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, + pool_saturated: true, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const r = await handleSemanticSearchMessages({ query: 'travel plans', explain: true }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.total).toBe(-1); + expect(body.mode).toBe('hybrid'); + expect(body.pool_saturated).toBe(true); + expect(body.generation).toEqual({ id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }); + expect(body.data[0].subject).toBe('Trip'); // hydrated summary + expect(body.data[0].score).toEqual({ rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: true }); + expect(body.data[0].matches).toEqual([{ char_offset: 42, snippet: 'flights', line: 3, score: 0.9 }]); + expect(body.data[0]).not.toHaveProperty('best_chunk'); + expect(matchFromChunk).toHaveBeenCalledWith('m1', { chunk_index: 2, char_start: 100, char_end: 180, score: 0.9 }, { accountIds: scope.accountIds }); + }); + + it('drops the excerpt when its score is below min_score (ranking unaffected)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + matchFromChunk.mockResolvedValue({ snippet: 'weak', score: 0.1 }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', best_chunk: { chunk_index: 0, char_start: 10, char_end: 20, score: 0.1 } }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const r = await handleSemanticSearchMessages({ query: 'travel', mode: 'vector', min_score: 0.5 }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.data[0]).not.toHaveProperty('matches'); // excerpt dropped, message still returned + expect(body.mode).toBe('vector'); + }); + + it('narrows scope to a resolved account id (search + hydration + matchFromChunk)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + matchFromChunk.mockResolvedValue({ snippet: 'x', score: 0.9 }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', best_chunk: { chunk_index: 0, char_start: 0, char_end: 5, score: 0.9 } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + await handleSemanticSearchMessages({ query: 'travel', account: 'work@x.com' }, scope); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); + expect(matchFromChunk).toHaveBeenCalledWith('m1', expect.any(Object), { accountIds: ['acc-2'] }); + }); + + it('rejects an unknown account', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSemanticSearchMessages({ query: 'travel', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(search).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim and unsupported operators as taxonomy errors', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], + errors: ['invalid value "13" for newer_than: — expected a relative age like 7d, 2w, 1m, or 1y'], + }); + const r = await handleSemanticSearchMessages({ query: 'travel newer_than:13' }, scope); + expect(r.content[0].text).toBe('invalid value "13" for newer_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'travel', negate: false }], + unsupported: [{ key: 'smaller', token: 'smaller:1M' }], errors: [], + }); + const r2 = await handleSemanticSearchMessages({ query: 'travel smaller:1M' }, scope); + expect(r2.content[0].text).toMatch(/^unsupported_search_operator: smaller: /); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects offsets past the hybrid ranking window with pagination_limit (msgvault handlers.go:656-663)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [], errors: [] }); + const r = await handleSemanticSearchMessages({ query: 'x', offset: 100 }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + 'pagination_limit: offset 100 exceeds hybrid ranking window (max 100); use search_metadata or search_message_bodies for deeper pagination', + ); + expect(search).not.toHaveBeenCalled(); + }); + + it('maps a seam MissingFreeTextError to the missing_free_text result — never `internal error:` (msgvault handlers.go:631-635)', async () => { + // 'a' survives the handler's raw-terms pre-check, but the seam's stricter + // hygiene (sub-2-char tokens do not embed) throws MissingFreeTextError. + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'a', negate: false }], unsupported: [], errors: [] }); + class MissingFreeTextError extends Error { constructor() { super('missing_free_text'); this.name = 'MissingFreeTextError'; } } + search.mockRejectedValue(new MissingFreeTextError()); + const r = await handleSemanticSearchMessages({ query: 'a' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('missing_free_text: mode=hybrid requires at least one free-text term; use search_metadata for filter-only queries'); + }); + + it('explain omits rrf in mode=vector and subject_boosted when false (Go omitempty, handlers.go:563-572)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', score: { rrf: 0.5, vector: 0.8, subject_boosted: false } }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const body = payload(await handleSemanticSearchMessages({ query: 'travel', mode: 'vector', explain: true }, scope)); + expect(body.data[0].score).toEqual({ vector: 0.8 }); // no rrf (one signal), no false subject_boosted + }); + + it('explain keeps rrf in mode=hybrid but still omits a false subject_boosted', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', score: { rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: false } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const body = payload(await handleSemanticSearchMessages({ query: 'travel', explain: true }, scope)); + expect(body.data[0].score).toEqual({ rrf: 0.5, bm25: 1.2, vector: 0.8 }); + }); +}); diff --git a/backend/src/mcp/semanticSearchIds.regression.test.js b/backend/src/mcp/semanticSearchIds.regression.test.js new file mode 100644 index 00000000..a71a815a --- /dev/null +++ b/backend/src/mcp/semanticSearchIds.regression.test.js @@ -0,0 +1,89 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Regression for the live MCP bug: `semantic_search_messages` returned 0 because +// hybrid/vector seam hits carry `message_id` (fusedSearch DISPLAY_COLS), not `id`, +// so the handler's `byId.get(m.id)` hydration mapped every hit to null. +// +// Unlike searchTools.test.js (which mocks the searchService seam directly), this +// test drives the REAL seam and mocks only the DEEPER dependency (hybridSearch) +// with the REAL fused-row field inventory the running container exposed +// (message_id/uid/folder/…/rrf_score/best_char_*). That exercises the actual +// `id`-alias fix in searchService plus the handler's hydration + excerpt + explain +// assembly end-to-end: it fails (returned:0) without the seam fix and passes with it. +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn(), pool: {} })); +vi.mock('../services/embeddings/hybrid.js', () => ({ + hybridSearch: vi.fn(), + isLexicalFallback: () => false, + MissingFreeTextError: class MissingFreeTextError extends Error {}, + VectorUnavailableError: class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } + }, + resolveActiveGeneration: vi.fn(), +})); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +vi.mock('./engineAdapter.js', () => ({ getMessageSummariesByIDs: vi.fn(), resolveAccountScope: vi.fn() })); + +import { hybridSearch } from '../services/embeddings/hybrid.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { handleSemanticSearchMessages } from './searchTools.js'; + +const scope = { userId: 'u1', accountIds: ['acc-1'] }; +const payload = (r) => JSON.parse(r.content[0].text); + +// The exact key inventory the deployed container returned for a hybrid/vector hit — +// keyed on message_id, with NO `id` (that is the bug the seam fix repairs). +function realFusedHit(overrides = {}) { + return { + message_id: 'uuid-1', uid: 42, folder: 'INBOX', + subject: 'Run failed: CD deploy', from_name: 'GitHub', from_email: 'notifications@github.com', + date: new Date('2026-07-15T00:00:00Z'), snippet: 'deployment failing', is_read: false, + is_starred: false, has_attachments: false, account_id: 'acc-1', + account_name: 'Work', account_email: 'me@work.com', account_color: '#fff', + rrf_score: 0.033, bm25_score: 1.2, vector_score: 0.88, subject_boosted: false, + best_chunk_index: 0, best_char_start: 5, best_char_end: 40, + ...overrides, + }; +} + +beforeEach(() => { + hybridSearch.mockReset(); matchFromChunk.mockReset(); + getMessageSummariesByIDs.mockReset(); resolveAccountScope.mockReset(); + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); + // Hydration echoes each requested id back as a minimal summary. + getMessageSummariesByIDs.mockImplementation(async (ids) => + ids.map((id) => ({ id, subject: 'Run failed: CD deploy', from_email: 'notifications@github.com' }))); +}); + +describe('semantic_search_messages end-to-end over the real seam (message_id → id alias)', () => { + for (const mode of ['hybrid', 'vector']) { + it(`mode=${mode}: hydrates real message_id-keyed seam hits into data (was returned:0)`, async () => { + hybridSearch.mockResolvedValue({ + hits: [realFusedHit()], + poolSaturated: false, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + matchFromChunk.mockResolvedValue({ char_offset: 5, snippet: 'deploy', score: 0.88 }); + + const r = await handleSemanticSearchMessages({ query: 'server deployment failing', mode, explain: true }, scope); + const body = payload(r); + + expect(r.isError).toBeFalsy(); + expect(body.returned).toBe(1); // the bug returned 0 + expect(body.mode).toBe(mode); + expect(body.data[0].id).toBe('uuid-1'); // hydrated via the aliased id + expect(body.data[0].subject).toBe('Run failed: CD deploy'); + // explain surfaces the real per-signal fields with Go omitempty parity + // (msgvault handlers.go:563-572): rrf only fuses in hybrid, and a false + // subject_boosted is omitted from the wire. + expect(body.data[0].score).toEqual( + mode === 'hybrid' ? { rrf: 0.033, bm25: 1.2, vector: 0.88 } : { bm25: 1.2, vector: 0.88 }, + ); + // matches assembled from best_chunk, keyed on the aliased id + expect(body.data[0].matches).toEqual([{ char_offset: 5, snippet: 'deploy', score: 0.88 }]); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['uuid-1'], ['acc-1']); + expect(matchFromChunk).toHaveBeenCalledWith( + 'uuid-1', { chunk_index: 0, char_start: 5, char_end: 40, score: 0.88 }, { accountIds: ['acc-1'] }); + }); + } +}); diff --git a/backend/src/mcp/server.js b/backend/src/mcp/server.js new file mode 100644 index 00000000..9fb25a32 --- /dev/null +++ b/backend/src/mcp/server.js @@ -0,0 +1,149 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import { mcpBearerAuth } from './auth.js'; +import { TOOL_DEFS, HANDLERS } from './tools.js'; +import { errorResult } from './result.js'; + +// --- Origin validation (DNS-rebinding protection) -------------------------------- +// The MCP Streamable HTTP spec REQUIRES servers to validate the Origin header so a +// malicious web page cannot drive a local MCP endpoint from a victim's browser via +// DNS rebinding. Our SDK (@modelcontextprotocol/sdk 1.29) can enforce this in the +// transport (`enableDnsRebindingProtection` + `allowedOrigins` on +// WebStandardStreamableHTTPServerTransport) but ships with it DISABLED by default +// (GHSA-w48q-cv73-mx4w) and compares origins by exact string match. We enforce at +// the Express layer instead: one owner, normalized (URL.origin, lowercased) +// comparison, and rejection happens before bearer auth ever touches the database. +// +// Policy (mirrors websocket.js): a request with NO Origin header passes — MCP +// clients are non-browser processes and send none. A request WITH an Origin must +// resolve to an allowlisted origin: APP_URL, FRONTEND_URL, any localhost / +// 127.0.0.1 / [::1] origin on any port (a DNS-rebinding page always presents the +// attacker's hostname as Origin, never localhost), or an operator-supplied extra +// via MCP_ALLOWED_ORIGINS (comma-separated URLs, e.g. "https://lan-host:8087"). +const LOCAL_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]']); + +export function buildAllowedOrigins(env = process.env) { + const allowed = new Set(); + const add = (raw) => { + if (!raw || !raw.trim()) return; + try { allowed.add(new URL(raw.trim()).origin.toLowerCase()); } + catch { /* malformed allowlist entry — skip it rather than fail the boot */ } + }; + add(env.APP_URL); + add(env.FRONTEND_URL); + for (const entry of (env.MCP_ALLOWED_ORIGINS || '').split(',')) add(entry); + return allowed; +} + +export function mcpOriginGuard(allowed = buildAllowedOrigins()) { + return (req, res, next) => { + const raw = req.get('Origin'); + if (!raw) return next(); // non-browser MCP client + try { + const url = new URL(raw); + if (allowed.has(url.origin.toLowerCase()) || LOCAL_HOSTNAMES.has(url.hostname.toLowerCase())) { + return next(); + } + } catch { /* unparseable Origin (including the literal "null") — reject below */ } + // Same JSON-RPC error envelope the SDK transport uses for its own HTTP-level + // rejections (webStandardStreamableHttp.js createJsonErrorResponse). + res.status(403).json({ + jsonrpc: '2.0', + error: { code: -32000, message: `Origin not allowed: ${raw}` }, + id: null, + }); + }; +} + +// --- Per-token tool-call rate limit ----------------------------------------------- +// REST search is limited per user (routes/search.js); the MCP surface reuses the same +// in-memory bucket pattern, keyed by the api_tokens row id — NOT the client IP, since +// many agents legitimately share one egress. Only tools/call requests count, so the +// initialize / tools-list handshake a stateless client repeats is never throttled. +// Over-limit requests get an HTTP 429 with Retry-After BEFORE the transport, carrying +// the SDK's JSON-RPC error envelope shape. Default 60 tool calls per minute per token, +// overridable via MCP_RATE_LIMIT_PER_MIN. +export function countToolCalls(body) { + if (Array.isArray(body)) return body.filter((m) => m && m.method === 'tools/call').length; + return body && body.method === 'tools/call' ? 1 : 0; +} + +export function createMcpRateLimiter({ limit, windowMs = 60_000, now = Date.now } = {}) { + const envLimit = Number(process.env.MCP_RATE_LIMIT_PER_MIN); + const max = limit ?? (envLimit > 0 ? envLimit : 60); + const buckets = new Map(); + const sweeper = setInterval(() => { + const t = now(); + for (const [k, b] of buckets) if (t > b.resetAt) buckets.delete(k); + }, windowMs); + sweeper.unref?.(); // observability sweeper must never keep the process alive + + return (req, res, next) => { + const calls = countToolCalls(req.body); + if (!calls) return next(); + const t = now(); + let b = buckets.get(req.mcpTokenId); + if (!b || t > b.resetAt) { + b = { count: 0, resetAt: t + windowMs }; + buckets.set(req.mcpTokenId, b); + } + if (b.count + calls > max) { + res.setHeader('Retry-After', Math.ceil((b.resetAt - t) / 1000)); + return res.status(429).json({ + jsonrpc: '2.0', + error: { code: -32000, message: `Rate limit exceeded: at most ${max} tool calls per minute per token — retry shortly` }, + id: Array.isArray(req.body) ? null : (req.body?.id ?? null), + }); + } + b.count += calls; + next(); + }; +} + +// Build a fresh Server bound to one request's scope. Stateless: no session store, +// one Server+transport per HTTP request, matching msgvault's daemon-less posture. +function buildServer(scope) { + const server = new Server( + { name: 'mailflow', version: '1.0.0' }, + { capabilities: { tools: {} } }, + ); + + server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: TOOL_DEFS, + })); + + server.setRequestHandler(CallToolRequestSchema, async (req) => { + const handler = HANDLERS[req.params.name]; + if (!handler) return errorResult(`unknown tool: ${req.params.name}`); + try { + return await handler(req.params.arguments || {}, scope); + } catch (err) { + // Tool-level failures flow as isError results, not JSON-RPC errors, + // so the client sees a readable message (msgvault convention). + return errorResult(`internal error: ${err.message}`); + } + }); + + return server; +} + +export function mountMcp(app) { + const handle = async (req, res) => { + const server = buildServer(req.mcpScope); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + res.on('close', () => { transport.close(); server.close(); }); + await server.connect(transport); + // Global express.json already parsed the body; hand it to the transport. + await transport.handleRequest(req, res, req.body); + }; + + const originGuard = mcpOriginGuard(buildAllowedOrigins()); + const rateLimiter = createMcpRateLimiter(); + + // Tool calls only arrive as POST bodies; GET (SSE open) and DELETE (session + // teardown) carry none, so the rate limiter guards POST alone. + app.post('/mcp', originGuard, mcpBearerAuth, rateLimiter, handle); + app.get('/mcp', originGuard, mcpBearerAuth, handle); + app.delete('/mcp', originGuard, mcpBearerAuth, handle); +} diff --git a/backend/src/mcp/server.test.js b/backend/src/mcp/server.test.js new file mode 100644 index 00000000..38301342 --- /dev/null +++ b/backend/src/mcp/server.test.js @@ -0,0 +1,77 @@ +import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; +import express from 'express'; +import { createServer } from 'http'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { hashToken } from './auth.js'; +import { mountMcp } from './server.js'; + +let server, base; + +beforeAll(async () => { + const app = express(); + app.use(express.json()); + mountMcp(app); + server = createServer(app); + await new Promise((r) => server.listen(0, r)); + base = `http://127.0.0.1:${server.address().port}`; +}); +afterAll(() => new Promise((r) => server.close(r))); + +// Every authed request: token lookup -> last_used_at update -> resolveScope. +function primeAuth(userId = 'user-1', accountIds = ['acc-1']) { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: userId }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: accountIds.map((id) => ({ id })) }); +} + +async function rpc(method, params, { auth = true } = {}) { + query.mockReset(); + if (auth) primeAuth(); + const headers = { 'Content-Type': 'application/json', Accept: 'application/json, text/event-stream' }; + if (auth) headers.Authorization = 'Bearer mcp_good'; + const res = await fetch(`${base}/mcp`, { + method: 'POST', + headers, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), + }); + return res; +} + +describe('/mcp transport', () => { + it('rejects unauthenticated requests with 401', async () => { + const res = await rpc('tools/list', {}, { auth: false }); + expect(res.status).toBe(401); + }); + + it('completes initialize', async () => { + const res = await rpc('initialize', { + protocolVersion: '2025-06-18', + capabilities: {}, + clientInfo: { name: 'test', version: '0' }, + }); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).toContain('"serverInfo"'); + expect(text).toContain('mailflow'); + }); + + it('lists the ping tool', async () => { + const res = await rpc('tools/list', {}); + const text = await res.text(); + expect(text).toContain('"ping"'); + }); + + it('calls ping and returns pong', async () => { + const res = await rpc('tools/call', { name: 'ping', arguments: {} }); + const text = await res.text(); + expect(text).toContain('{\\"pong\\":true}'); + }); + + it('verifies the token by hash, not plaintext', async () => { + await rpc('tools/list', {}); + expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); + }); +}); diff --git a/backend/src/mcp/serverGuards.test.js b/backend/src/mcp/serverGuards.test.js new file mode 100644 index 00000000..7329d819 --- /dev/null +++ b/backend/src/mcp/serverGuards.test.js @@ -0,0 +1,267 @@ +import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; +import express from 'express'; +import { createServer } from 'http'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { + buildAllowedOrigins, mcpOriginGuard, countToolCalls, createMcpRateLimiter, mountMcp, +} from './server.js'; + +function mockReq({ origin, body, tokenId } = {}) { + return { + get: (h) => (h.toLowerCase() === 'origin' ? origin : undefined), + body, + mcpTokenId: tokenId, + }; +} + +function mockRes() { + const res = { statusCode: 200, headers: {}, body: null }; + res.setHeader = (k, v) => { res.headers[k] = v; }; + res.status = (c) => { res.statusCode = c; return res; }; + res.json = (b) => { res.body = b; return res; }; + return res; +} + +describe('buildAllowedOrigins', () => { + it('derives normalized origins from APP_URL, FRONTEND_URL, and MCP_ALLOWED_ORIGINS', () => { + const allowed = buildAllowedOrigins({ + APP_URL: 'https://Mail.Example.com/', // trailing slash + case normalize away + FRONTEND_URL: 'http://localhost:5173', + MCP_ALLOWED_ORIGINS: 'https://lan-host:8087, http://mail.internal', + }); + expect(allowed).toEqual(new Set([ + 'https://mail.example.com', + 'http://localhost:5173', + 'https://lan-host:8087', + 'http://mail.internal', + ])); + }); + + it('skips malformed and empty entries instead of throwing', () => { + const allowed = buildAllowedOrigins({ APP_URL: 'not a url', MCP_ALLOWED_ORIGINS: ' ,, ' }); + expect(allowed.size).toBe(0); + }); +}); + +describe('mcpOriginGuard', () => { + const guard = mcpOriginGuard(buildAllowedOrigins({ APP_URL: 'https://mail.example.com' })); + + it('passes requests with no Origin header (non-browser MCP clients)', () => { + const next = vi.fn(); + guard(mockReq(), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('passes an allowlisted Origin', () => { + const next = vi.fn(); + guard(mockReq({ origin: 'https://mail.example.com' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('normalizes Origin casing before comparing', () => { + const next = vi.fn(); + guard(mockReq({ origin: 'HTTPS://MAIL.EXAMPLE.COM' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('passes localhost variants on any port', () => { + for (const origin of ['http://localhost:8087', 'http://127.0.0.1:3000', 'http://[::1]:8087', 'https://localhost']) { + const next = vi.fn(); + guard(mockReq({ origin }), mockRes(), next); + expect(next, origin).toHaveBeenCalledTimes(1); + } + }); + + it('rejects a non-allowlisted Origin with a 403 JSON-RPC error', () => { + const next = vi.fn(); + const res = mockRes(); + guard(mockReq({ origin: 'http://evil.example' }), res, next); + expect(next).not.toHaveBeenCalled(); + expect(res.statusCode).toBe(403); + expect(res.body).toEqual({ + jsonrpc: '2.0', + error: { code: -32000, message: 'Origin not allowed: http://evil.example' }, + id: null, + }); + }); + + it('rejects an attacker hostname that merely resolves to this host (DNS rebinding)', () => { + const res = mockRes(); + guard(mockReq({ origin: 'http://rebind.attacker.net:8087' }), res, vi.fn()); + expect(res.statusCode).toBe(403); + }); + + it('rejects unparseable Origins, including the literal "null"', () => { + for (const origin of ['null', 'not a url']) { + const res = mockRes(); + guard(mockReq({ origin }), res, vi.fn()); + expect(res.statusCode, origin).toBe(403); + } + }); +}); + +describe('countToolCalls', () => { + it('counts a single tools/call body as 1 and other methods as 0', () => { + expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/call', id: 1 })).toBe(1); + expect(countToolCalls({ jsonrpc: '2.0', method: 'initialize', id: 1 })).toBe(0); + expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/list', id: 1 })).toBe(0); + expect(countToolCalls(undefined)).toBe(0); + }); + + it('counts tools/call entries inside a batch array', () => { + expect(countToolCalls([ + { method: 'tools/call' }, { method: 'notifications/initialized' }, { method: 'tools/call' }, + ])).toBe(2); + }); +}); + +describe('createMcpRateLimiter', () => { + let clock; + const now = () => clock; + const call = (id = 1) => ({ jsonrpc: '2.0', method: 'tools/call', id }); + + beforeEach(() => { clock = 1_000_000; }); + + it('allows up to the limit then 429s with Retry-After and a JSON-RPC error', () => { + const limiter = createMcpRateLimiter({ limit: 2, now }); + for (let i = 0; i < 2; i++) { + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + } + const res = mockRes(); + const next = vi.fn(); + limiter(mockReq({ body: call(42), tokenId: 'tok-1' }), res, next); + expect(next).not.toHaveBeenCalled(); + expect(res.statusCode).toBe(429); + expect(res.headers['Retry-After']).toBe(60); + expect(res.body.jsonrpc).toBe('2.0'); + expect(res.body.error.code).toBe(-32000); + expect(res.body.error.message).toMatch(/rate limit/i); + expect(res.body.id).toBe(42); // correlates with the throttled request + }); + + it('keys buckets per token, not globally', () => { + const limiter = createMcpRateLimiter({ limit: 1, now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-2' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); // a different token has its own budget + }); + + it('never throttles the initialize/tools-list handshake', () => { + const limiter = createMcpRateLimiter({ limit: 1, now }); + for (const method of ['initialize', 'notifications/initialized', 'tools/list', 'tools/list']) { + const next = vi.fn(); + limiter(mockReq({ body: { jsonrpc: '2.0', method, id: 1 }, tokenId: 'tok-1' }), mockRes(), next); + expect(next, method).toHaveBeenCalledTimes(1); + } + }); + + it('resets the budget after the window elapses', () => { + const limiter = createMcpRateLimiter({ limit: 1, now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const blocked = mockRes(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), blocked, vi.fn()); + expect(blocked.statusCode).toBe(429); + clock += 60_001; + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('reads the default limit from MCP_RATE_LIMIT_PER_MIN', () => { + vi.stubEnv('MCP_RATE_LIMIT_PER_MIN', '1'); + try { + const limiter = createMcpRateLimiter({ now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const res = mockRes(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), res, vi.fn()); + expect(res.statusCode).toBe(429); + } finally { + vi.unstubAllEnvs(); + } + }); +}); + +// --- End-to-end through the live Express mount (same harness as server.test.js) --- +describe('mounted /mcp guards', () => { + const servers = []; + afterAll(async () => { + for (const s of servers) await new Promise((r) => s.close(r)); + }); + + async function mountApp(env) { + for (const [k, v] of Object.entries(env)) vi.stubEnv(k, v); + const app = express(); + app.use(express.json()); + mountMcp(app); // captures env-derived allowlist + limit at mount time + vi.unstubAllEnvs(); + const server = createServer(app); + servers.push(server); + await new Promise((r) => server.listen(0, r)); + return `http://127.0.0.1:${server.address().port}`; + } + + // Every authed request: token lookup -> last_used_at update -> resolveScope. + function primeAuth() { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: 'user-1' }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); + } + + async function rpc(base, method, params, { origin } = {}) { + query.mockReset(); + primeAuth(); + const headers = { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + Authorization: 'Bearer mcp_good', + }; + if (origin) headers.Origin = origin; + return fetch(`${base}/mcp`, { + method: 'POST', + headers, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), + }); + } + + it('403s a cross-origin browser request before auth, passes the app origin and no-Origin clients', async () => { + const base = await mountApp({ APP_URL: 'https://mail.example.com' }); + + query.mockReset(); + const evil = await fetch(`${base}/mcp`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Origin: 'http://evil.example' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }), + }); + expect(evil.status).toBe(403); + expect((await evil.json()).error.code).toBe(-32000); + expect(query).not.toHaveBeenCalled(); // rejected before the token ever hits the DB + + const sameOrigin = await rpc(base, 'tools/list', {}, { origin: 'https://mail.example.com' }); + expect(sameOrigin.status).toBe(200); + + const headless = await rpc(base, 'tools/list', {}); + expect(headless.status).toBe(200); + }); + + it('429s tool calls over the per-token budget but leaves tools/list untouched', async () => { + const base = await mountApp({ MCP_RATE_LIMIT_PER_MIN: '1' }); + + const first = await rpc(base, 'tools/call', { name: 'ping', arguments: {} }); + expect(first.status).toBe(200); + + const second = await rpc(base, 'tools/call', { name: 'ping', arguments: {} }); + expect(second.status).toBe(429); + expect(second.headers.get('Retry-After')).toMatch(/^\d+$/); + const body = await second.json(); + expect(body.error.message).toMatch(/rate limit/i); + + const list = await rpc(base, 'tools/list', {}); + expect(list.status).toBe(200); + }); +}); diff --git a/backend/src/mcp/tools.js b/backend/src/mcp/tools.js new file mode 100644 index 00000000..83779aa9 --- /dev/null +++ b/backend/src/mcp/tools.js @@ -0,0 +1,54 @@ +import { jsonResult } from './result.js'; +import { + searchMetadataDef, handleSearchMetadata, + searchMessageBodiesDef, handleSearchMessageBodies, + semanticSearchMessagesDef, handleSemanticSearchMessages, +} from './searchTools.js'; +import { + getMessageDef, handleGetMessage, + listMessagesDef, handleListMessages, + getStatsDef, handleGetStats, + aggregateDef, handleAggregate, + searchByDomainsDef, handleSearchByDomains, + findSimilarMessagesDef, handleFindSimilarMessages, + searchInMessageDef, handleSearchInMessage, + stageDeletionDef, handleStageDeletion, +} from './messageTools.js'; + +// TOOL_DEFS drives tools/list; HANDLERS drives tools/call. Slices 09 and 10 +// append to both. Keep names, descriptions, and inputSchema field names verbatim +// from internal/mcp/server.go (README D6: id-bearing fields diverge to strings). +export const TOOL_DEFS = [ + { + name: 'ping', + description: 'Health check: returns {"pong":true}. Proves transport + auth round-trip.', + inputSchema: { type: 'object', properties: {} }, + }, + searchMetadataDef, + searchMessageBodiesDef, + semanticSearchMessagesDef, + getMessageDef, + listMessagesDef, + getStatsDef, + aggregateDef, + searchByDomainsDef, + findSimilarMessagesDef, + searchInMessageDef, + stageDeletionDef, +]; + +export const HANDLERS = { + // eslint-disable-next-line no-unused-vars + ping: async (_args, _scope) => jsonResult({ pong: true }), + search_metadata: (a, s) => handleSearchMetadata(a, s), + search_message_bodies: (a, s) => handleSearchMessageBodies(a, s), + semantic_search_messages: (a, s) => handleSemanticSearchMessages(a, s), + get_message: (a, s) => handleGetMessage(a, s), + list_messages: (a, s) => handleListMessages(a, s), + get_stats: (a, s) => handleGetStats(a, s), + aggregate: (a, s) => handleAggregate(a, s), + search_by_domains: (a, s) => handleSearchByDomains(a, s), + find_similar_messages: (a, s) => handleFindSimilarMessages(a, s), + search_in_message: (a, s) => handleSearchInMessage(a, s), + stage_deletion: (a, s) => handleStageDeletion(a, s), +}; diff --git a/backend/src/mcp/vectorErrors.js b/backend/src/mcp/vectorErrors.js new file mode 100644 index 00000000..13b658d5 --- /dev/null +++ b/backend/src/mcp/vectorErrors.js @@ -0,0 +1,18 @@ +// Code prefixes are verbatim from msgvault (handlers.go translateVectorErr). +// Remediation hints are Mailflow-specific (msgvault references a CLI we don't +// ship) — a documented divergence; golden diffing asserts the prefix only. +const MESSAGES = { + vector_not_enabled: 'vector_not_enabled: vector search is not configured on this server', + index_stale: 'index_stale: the vector index does not match the configured model; reconfigure embeddings to rebuild', + index_building: 'index_building: the initial vector index is still being built', + no_active_generation: 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + // msgvault handlers.go:225-229; remediation names Mailflow's knob (the + // embedding client timeout) instead of msgvault's [vector.embeddings].timeout TOML. + embedding_timeout: 'embedding_timeout: the embedding endpoint did not respond in time; retry, or raise the embedding client timeout in settings', +}; + +export const VECTOR_ERROR_CODES = Object.keys(MESSAGES); + +export function translateVectorError(reason) { + return MESSAGES[reason] || MESSAGES.vector_not_enabled; +} diff --git a/backend/src/mcp/vectorErrors.test.js b/backend/src/mcp/vectorErrors.test.js new file mode 100644 index 00000000..334a07cd --- /dev/null +++ b/backend/src/mcp/vectorErrors.test.js @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; +import { translateVectorError } from './vectorErrors.js'; + +describe('translateVectorError', () => { + it('maps each reason to its verbatim code prefix', () => { + expect(translateVectorError('vector_not_enabled')).toMatch(/^vector_not_enabled: /); + expect(translateVectorError('index_stale')).toMatch(/^index_stale: /); + expect(translateVectorError('index_building')).toMatch(/^index_building: /); + expect(translateVectorError('no_active_generation')).toMatch(/^no_active_generation: /); + expect(translateVectorError('embedding_timeout')).toMatch(/^embedding_timeout: /); + }); + it('index_stale matches msgvault prefix-verbatim (handlers.go:206-210 — no inserted "embedding")', () => { + expect(translateVectorError('index_stale')) + .toMatch(/^index_stale: the vector index does not match the configured model; /); + }); + it('embedding_timeout ports msgvault wording (handlers.go:225-229)', () => { + expect(translateVectorError('embedding_timeout')) + .toMatch(/^embedding_timeout: the embedding endpoint did not respond in time; retry, /); + }); + it('falls back to vector_not_enabled for an unknown reason', () => { + expect(translateVectorError('mystery')).toMatch(/^vector_not_enabled: /); + }); +}); diff --git a/backend/src/mcp/vectorStats.js b/backend/src/mcp/vectorStats.js new file mode 100644 index 00000000..dc90f2e3 --- /dev/null +++ b/backend/src/mcp/vectorStats.js @@ -0,0 +1,75 @@ +import { query } from '../services/db.js'; +import * as generations from '../services/embeddings/generations.js'; // phase 3 + +// Epoch SECONDS (index_generations.activated_at/started_at, bigint — node-pg +// returns it as a string) → RFC3339 UTC string, matching msgvault CollectStats +// (vector/stats.go:146-153: time.Unix(sec,0).UTC().Format(time.RFC3339) — no +// sub-second digits, unlike Date.toISOString()). "" for a missing/zero/ +// unparsable epoch; callers omit the field entirely then (Go omitempty on +// activated_at/started_at, stats.go:47,:56). +function epochToISO(sec) { + const n = Number(sec); + if (!Number.isFinite(n) || n <= 0) return ''; + return new Date(n * 1000).toISOString().replace('.000Z', 'Z'); +} + +// Live-message count still needing embedding, SCOPED to the caller's accounts +// (documented divergence from msgvault's single-archive count — avoids a cross-user +// cardinality leak). Frozen invariant: `embed_gen IS NULL ⟺ needs embedding` — +// createGeneration resets every live row's stamp to NULL on a rebuild, so the count +// is generation-agnostic and the old `OR embed_gen <> gen` arm was dead/double-counting. +async function missingCount(accountIds) { + if (!accountIds || !accountIds.length) return 0; + const { rows } = await query( + `SELECT COUNT(*)::bigint AS n FROM messages + WHERE account_id = ANY($1) AND is_deleted = false AND embed_gen IS NULL`, + [accountIds], + ); + return Number(rows[0].n); +} + +// Port of vector.CollectStats. Returns null when vector search is disabled (the +// generation queries throw on stock Postgres without the vector schema); an +// enabled archive with no active generation yet reports active_generation: null. +// Generation metadata is archive-global; missing_embeddings_total is account-scoped. +export async function collectStats(accountIds) { + let active; + try { + active = await generations.activeGeneration(); // null = none yet; throw = disabled + } catch { + return null; + } + + // Sub-query failures degrade to partial data (msgvault stats.go:69-78: one + // broken sub-query never blanks the whole stats envelope) — every leg below + // catches and falls back rather than throwing the block away. + const out = { enabled: true, active_generation: null, missing_embeddings_total: 0 }; + if (active) { + const messageCount = await generations.chunkCount(active.id).catch(() => 0); + const activatedAt = epochToISO(active.activatedAt); + out.active_generation = { + id: active.id, model: active.model, dimension: active.dimension, + fingerprint: active.fingerprint, state: active.state, + ...(activatedAt ? { activated_at: activatedAt } : {}), // omitempty (stats.go:47) + message_count: messageCount, + }; + } + + const building = await generations.buildingGeneration().catch(() => null); + if (building) { + // A rebuild in flight is the actionable coverage target; active-generation + // top-ups are frozen until activation (msgvault CollectStats semantics). + const done = await generations.chunkCount(building.id).catch(() => 0); + const pending = await missingCount(accountIds).catch(() => 0); + const startedAt = epochToISO(building.startedAt); + out.building_generation = { + id: building.id, model: building.model, dimension: building.dimension, + ...(startedAt ? { started_at: startedAt } : {}), // omitempty (stats.go:56) + progress: { done, total: done + pending }, + }; + out.missing_embeddings_total = pending; + } else if (active) { + out.missing_embeddings_total = await missingCount(accountIds).catch(() => 0); + } + return out; +} diff --git a/backend/src/mcp/vectorStats.test.js b/backend/src/mcp/vectorStats.test.js new file mode 100644 index 00000000..cbc093ee --- /dev/null +++ b/backend/src/mcp/vectorStats.test.js @@ -0,0 +1,98 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../services/embeddings/generations.js', () => ({ + activeGeneration: vi.fn(), buildingGeneration: vi.fn(), chunkCount: vi.fn(), +})); +import { query } from '../services/db.js'; +import * as generations from '../services/embeddings/generations.js'; +import { collectStats } from './vectorStats.js'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +beforeEach(() => { query.mockReset(); generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); }); + +describe('mock-drift guard', () => { + it('every mocked generations key exists as a function on the real module', async () => { + // Regression guard: chunkCount was mocked here (and in goldenParity) while the + // real generations.js never implemented it — collectStats threw live. + const real = await vi.importActual('../services/embeddings/generations.js'); + expect(mockSurfaceDrift(generations, real)).toEqual([]); + }); +}); + +describe('collectStats', () => { + it('returns null when vector search is disabled', async () => { + generations.activeGeneration.mockRejectedValue(new Error('disabled')); + expect(await collectStats(['a'])).toBeNull(); + }); + + it('reports the active generation and a scoped missing count', async () => { + // activatedAt is epoch SECONDS (bigint) as generationByState now returns it; + // the wire field is RFC3339 UTC WITHOUT sub-second digits (msgvault + // vector/stats.go:146-153 formatTime uses time.RFC3339, never millis). + // 1704067200 = 2024-01-01T00:00:00Z. + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(1000); + query.mockResolvedValueOnce({ rows: [{ n: '7' }] }); // missing count + const vs = await collectStats(['acc-1']); + expect(vs.enabled).toBe(true); + expect(vs.active_generation).toEqual({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activated_at: '2024-01-01T00:00:00Z', message_count: 1000 }); + expect(vs.missing_embeddings_total).toBe(7); + // chunkCount is keyed by generation id (real signature). The missing count is scoped to + // accountIds and keys off `embed_gen IS NULL` only (frozen invariant — no dead OR arm). + expect(generations.chunkCount).toHaveBeenCalledWith(2); + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); + const sql = query.mock.calls[0][0]; + expect(sql).toMatch(/embed_gen IS NULL/); + expect(sql).not.toMatch(/embed_gen\s*<>/); + }); + + it('enabled with no active generation yet (first build) reports a null active_generation', async () => { + generations.activeGeneration.mockResolvedValue(null); + generations.buildingGeneration.mockResolvedValue(null); + const vs = await collectStats([]); + expect(vs).toEqual({ enabled: true, active_generation: null, missing_embeddings_total: 0 }); + expect(query).not.toHaveBeenCalled(); // empty scope skips the count query + }); + + it('OMITS activated_at when the epoch is absent or zero (msgvault omitempty, stats.go:47)', async () => { + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active' }); // no activatedAt + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(3); + query.mockResolvedValueOnce({ rows: [{ n: '0' }] }); + const vs = await collectStats(['acc-1']); + expect(vs.active_generation).not.toHaveProperty('activated_at'); + }); + + it('reports a building generation with scoped progress and freezes missing on the build target', async () => { + // startedAt is epoch SECONDS → RFC3339 (no millis) wire. 1706745600 = 2024-02-01T00:00:00Z. + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active' }); + generations.buildingGeneration.mockResolvedValue({ id: 3, model: 'm2', dimension: 8, startedAt: 1706745600 }); + generations.chunkCount.mockImplementation(async (id) => (id === 2 ? 100 : 40)); // active done, building done + query.mockResolvedValueOnce({ rows: [{ n: '10' }] }); // missing for building gen 3 + const vs = await collectStats(['acc-1']); + expect(vs.building_generation).toEqual({ id: 3, model: 'm2', dimension: 8, started_at: '2024-02-01T00:00:00Z', progress: { done: 40, total: 50 } }); + expect(vs.missing_embeddings_total).toBe(10); // building coverage is the actionable target + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); // generation-agnostic (embed_gen IS NULL) + }); + + it('OMITS started_at when the building epoch is absent (msgvault omitempty, stats.go:56)', async () => { + generations.activeGeneration.mockResolvedValue(null); + generations.buildingGeneration.mockResolvedValue({ id: 3, model: 'm2', dimension: 8 }); // no startedAt + generations.chunkCount.mockResolvedValue(40); + query.mockResolvedValueOnce({ rows: [{ n: '10' }] }); + const vs = await collectStats(['acc-1']); + expect(vs.building_generation).not.toHaveProperty('started_at'); + }); + + it('degrades to partial data when the missing-count sub-query fails (msgvault stats.go:69-78 best-effort)', async () => { + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(3); + query.mockRejectedValueOnce(new Error('relation vanished')); // missingCount blows up + const vs = await collectStats(['acc-1']); + expect(vs).not.toBeNull(); // one broken sub-query must not blank the whole block + expect(vs.active_generation.id).toBe(2); + expect(vs.missing_embeddings_total).toBe(0); + }); +}); diff --git a/backend/src/routes/apiTokens.js b/backend/src/routes/apiTokens.js new file mode 100644 index 00000000..598975b9 --- /dev/null +++ b/backend/src/routes/apiTokens.js @@ -0,0 +1,38 @@ +import { Router } from 'express'; +import { query } from '../services/db.js'; +import { requireAuth } from '../middleware/auth.js'; +import { generateToken, hashToken } from '../mcp/auth.js'; + +const router = Router(); +router.use(requireAuth); + +router.post('/', async (req, res) => { + const name = (req.body?.name || '').trim(); + if (!name) return res.status(400).json({ error: 'name is required' }); + const token = generateToken(); + const { rows } = await query( + 'INSERT INTO api_tokens (user_id, token_hash, name) VALUES ($1, $2, $3) RETURNING id, name', + [req.session.userId, hashToken(token), name], + ); + // Plaintext returned exactly once; only the hash was persisted. + res.status(201).json({ id: rows[0].id, name: rows[0].name, token }); +}); + +router.get('/', async (req, res) => { + const { rows } = await query( + 'SELECT id, name, created_at, last_used_at FROM api_tokens WHERE user_id = $1 ORDER BY created_at DESC', + [req.session.userId], + ); + res.json({ tokens: rows }); +}); + +router.delete('/:id', async (req, res) => { + const { rowCount } = await query( + 'DELETE FROM api_tokens WHERE id = $1 AND user_id = $2', + [req.params.id, req.session.userId], + ); + if (!rowCount) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/apiTokens.test.js b/backend/src/routes/apiTokens.test.js new file mode 100644 index 00000000..92309b2e --- /dev/null +++ b/backend/src/routes/apiTokens.test.js @@ -0,0 +1,80 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +// Stub auth to inject a fixed session user. +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { req.session = { userId: 'user-1' }; next(); }, +})); +import { query } from '../services/db.js'; +import router from './apiTokens.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/tokens', router); + return app; +} +async function call(app, method, path, body) { + const { createServer } = await import('http'); + const server = createServer(app); + await new Promise((r) => server.listen(0, r)); + const base = `http://127.0.0.1:${server.address().port}`; + const res = await fetch(base + path, { + method, + headers: body ? { 'Content-Type': 'application/json' } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + server.close(); + return { status: res.status, body: text ? JSON.parse(text) : null }; +} + +beforeEach(() => query.mockReset()); + +describe('POST /api/tokens', () => { + it('mints a token, returns the plaintext once, and stores only the hash', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop' }] }); + const { status, body } = await call(appWith(), 'POST', '/api/tokens', { name: 'laptop' }); + expect(status).toBe(201); + expect(body.token).toMatch(/^mcp_/); + expect(body).toMatchObject({ id: 'tok-1', name: 'laptop' }); + // INSERT bound values: [user_id, token_hash, name] — never the plaintext. + const params = query.mock.calls[0][1]; + expect(params[0]).toBe('user-1'); + expect(params[1]).toMatch(/^[0-9a-f]{64}$/); + expect(params[2]).toBe('laptop'); + expect(params).not.toContain(body.token); + }); + it('rejects a missing name', async () => { + const { status } = await call(appWith(), 'POST', '/api/tokens', {}); + expect(status).toBe(400); + }); +}); + +describe('GET /api/tokens', () => { + it('lists tokens without hashes or plaintext', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', created_at: 't', last_used_at: null }] }); + const { status, body } = await call(appWith(), 'GET', '/api/tokens'); + expect(status).toBe(200); + expect(body.tokens[0]).toEqual({ id: 'tok-1', name: 'laptop', created_at: 't', last_used_at: null }); + expect(JSON.stringify(body)).not.toContain('token_hash'); + }); +}); + +describe('DELETE /api/tokens/:id', () => { + it('revokes only within the session user', async () => { + query.mockResolvedValueOnce({ rowCount: 1, rows: [] }); + const { status } = await call(appWith(), 'DELETE', '/api/tokens/tok-1'); + expect(status).toBe(204); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/DELETE FROM api_tokens WHERE id = \$1 AND user_id = \$2/), + ['tok-1', 'user-1'], + ); + }); + it('404s when the token is absent or not owned', async () => { + query.mockResolvedValueOnce({ rowCount: 0, rows: [] }); + const { status } = await call(appWith(), 'DELETE', '/api/tokens/nope'); + expect(status).toBe(404); + }); +}); diff --git a/backend/src/routes/mcpDeletions.js b/backend/src/routes/mcpDeletions.js new file mode 100644 index 00000000..c7cf924c --- /dev/null +++ b/backend/src/routes/mcpDeletions.js @@ -0,0 +1,25 @@ +import { Router } from 'express'; +import { requireAuth } from '../middleware/auth.js'; +import { executeDeletionBatch, unstageDeletionBatch } from '../mcp/engineAdapter.js'; + +// Session-authenticated execute/unstage endpoints for MCP deletion batches. The +// stage_deletion tool only RECORDS a batch; flipping messages.is_deleted (Mailflow's +// soft delete) happens here, behind a browser session, never from the token. Both +// operations are scoped to req.session.userId, so a user cannot execute/cancel +// another user's batch. +const router = Router(); +router.use(requireAuth); + +router.post('/:id/execute', async (req, res) => { + const n = await executeDeletionBatch(req.params.id, req.session.userId); + if (n === null) return res.status(404).json({ error: 'not found' }); + res.json({ batch_id: req.params.id, status: 'executed', deleted: n }); +}); + +router.delete('/:id', async (req, res) => { + const ok = await unstageDeletionBatch(req.params.id, req.session.userId); + if (!ok) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/mcpDeletions.test.js b/backend/src/routes/mcpDeletions.test.js new file mode 100644 index 00000000..1a4eff25 --- /dev/null +++ b/backend/src/routes/mcpDeletions.test.js @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; + +vi.mock('../mcp/engineAdapter.js', () => ({ executeDeletionBatch: vi.fn(), unstageDeletionBatch: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { req.session = { userId: 'user-1' }; next(); }, +})); +import { executeDeletionBatch, unstageDeletionBatch } from '../mcp/engineAdapter.js'; +import router from './mcpDeletions.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/mcp-deletions', router); + return app; +} +async function call(app, method, path) { + const { createServer } = await import('http'); + const server = createServer(app); + await new Promise((r) => server.listen(0, r)); + const base = `http://127.0.0.1:${server.address().port}`; + const res = await fetch(base + path, { method }); + const text = await res.text(); + server.close(); + return { status: res.status, body: text ? JSON.parse(text) : null }; +} + +beforeEach(() => { executeDeletionBatch.mockReset(); unstageDeletionBatch.mockReset(); }); + +describe('POST /api/mcp-deletions/:id/execute', () => { + it('soft-deletes the owner batch and reports the count, scoped to the session user', async () => { + executeDeletionBatch.mockResolvedValue(3); + const { status, body } = await call(appWith(), 'POST', '/api/mcp-deletions/b1/execute'); + expect(status).toBe(200); + expect(body).toEqual({ batch_id: 'b1', status: 'executed', deleted: 3 }); + expect(executeDeletionBatch).toHaveBeenCalledWith('b1', 'user-1'); + }); + + it('404s a batch not owned / not found (cross-user isolation, no update)', async () => { + executeDeletionBatch.mockResolvedValue(null); + const { status } = await call(appWith(), 'POST', '/api/mcp-deletions/other/execute'); + expect(status).toBe(404); + }); +}); + +describe('DELETE /api/mcp-deletions/:id', () => { + it('unstages only the owner batch', async () => { + unstageDeletionBatch.mockResolvedValue(true); + const { status } = await call(appWith(), 'DELETE', '/api/mcp-deletions/b1'); + expect(status).toBe(204); + expect(unstageDeletionBatch).toHaveBeenCalledWith('b1', 'user-1'); + }); + + it('404s when absent or not owned', async () => { + unstageDeletionBatch.mockResolvedValue(false); + const { status } = await call(appWith(), 'DELETE', '/api/mcp-deletions/nope'); + expect(status).toBe(404); + }); +}); diff --git a/backend/src/services/embeddings/chunk.js b/backend/src/services/embeddings/chunk.js index 54d7a594..fb400758 100644 --- a/backend/src/services/embeddings/chunk.js +++ b/backend/src/services/embeddings/chunk.js @@ -3,6 +3,10 @@ const SENTENCE_TERMS = [['.', ' '], ['?', ' '], ['!', ' '], ['.', '\n'], ['?', '\n'], ['!', '\n']]; +// Chunking policy shared by the embed worker (write path) and chunkmatch (read path): +// the per-message span cap and the maxRunes→overlap rule. This module owns them so the +// two paths cannot drift — a mismatch would silently misalign re-chunked read offsets +// from the stored write offsets. Changing either MUST bump EMBED_POLICY_VERSION. export const MAX_SPANS = 64; // worker.go maxSpansPerMessage export function chunkOverlapFor(maxRunes) { diff --git a/backend/src/services/embeddings/chunk.test.js b/backend/src/services/embeddings/chunk.test.js index 1693e98f..96032f2a 100644 --- a/backend/src/services/embeddings/chunk.test.js +++ b/backend/src/services/embeddings/chunk.test.js @@ -3,6 +3,9 @@ import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; const cp = (s) => Array.from(s).length; +// Pin the shared chunking policy chunk.js owns. worker.js (write path) and chunkmatch.js +// (read path) both import MAX_SPANS + chunkOverlapFor from here, so they cannot drift — +// a mismatch would silently misalign re-chunked read offsets from stored write offsets. describe('shared chunking policy (one owner)', () => { it('exports MAX_SPANS = 64 (worker.go maxSpansPerMessage)', () => { expect(MAX_SPANS).toBe(64); diff --git a/backend/src/services/embeddings/chunkmatch.js b/backend/src/services/embeddings/chunkmatch.js new file mode 100644 index 00000000..33ed9826 --- /dev/null +++ b/backend/src/services/embeddings/chunkmatch.js @@ -0,0 +1,147 @@ +// Port of internal/vector/chunkmatch/matches.go. Converts stored vector chunk +// offsets (code points into the PREPROCESSED subject+body) into API-safe match +// excerpts: a snippet plus, only when the preprocessed chunk maps exactly and +// uniquely into the raw body, a raw-body BYTE char_offset and 1-based line. +// +// Two exports, both consumed by the MCP handler layer: +// matchesInMessage — search_in_message mode=vector: re-embeds the query and the +// message's freshly re-chunked text, scores, and builds excerpts. Gates on +// resolveActiveGenerationFromConfig (throws VectorUnavailableError on stock/degraded PG). +// matchFromChunk — semantic_search_messages: phase 4 already ranked and surfaced +// the single best_chunk per hit, so no embedding — just re-derive the snippet +// and map it back to a raw-body byte offset. +import { query } from '../db.js'; +import { resolveActiveGenerationFromConfig } from './hybrid.js'; // phase-4 public face +import { EmbeddingClient } from './client.js'; // phase 3 +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; // phase 3 (shared chunking policy) +import { preprocess } from './preprocess.js'; // phase 3 +import { RAW_BODY_MULT } from './worker.js'; // phase 3 (write-path body cap) +import { resolveEmbedConfig } from './config.js'; // phase 1 +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from '../../utils/textExcerpt.js'; + +// Preprocess options with the derived raw-body cap the worker applies at write time +// (worker.js _embedBatch). Without the same cap, a pathological body preprocesses to +// different text here than what was embedded, silently misaligning chunk offsets. +function derivedPreprocessCfg(cfg) { + const pp = { ...((cfg && cfg.preprocess) || {}) }; + if (!pp.maxBodyRunes && cfg && cfg.maxInputChars > 0) { + pp.maxBodyRunes = cfg.maxInputChars * MAX_SPANS * RAW_BODY_MULT; + } + return pp; +} + +// First maxBytes of a UTF-8 buffer, never splitting a rune (msgvault bytePrefix). +function bytePrefix(buf, maxBytes) { + if (buf.length <= maxBytes) return buf.toString('utf8'); + let end = maxBytes; + while (end > 0 && !isRuneStart(buf[end])) end--; + return buf.toString('utf8', 0, end); +} + +function cosine(a, b) { + let dot = 0, na = 0, nb = 0; + for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; } + return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0; +} + +// Byte offset of chunk in the raw body, only if it occurs exactly once (msgvault uniqueBodyOffset). +function uniqueByteOffset(bodyBuf, chunkBuf) { + const first = bodyBuf.indexOf(chunkBuf); + if (first < 0 || bodyBuf.lastIndexOf(chunkBuf) !== first) return null; + return first; +} + +// The subject-prefix rune count preprocess.js prepends ("Subject: \n\n"), or 0 +// when the subject is empty (preprocess adds no prefix then). A chunk starting at +// or past this offset lies in the body region, so a raw-body offset is meaningful. +function subjectPrefixRunes(subject) { + return subject ? [...('Subject: ' + subject + '\n\n')].length : 0; +} + +// Scoped message load. Body selection matches worker.js (body_text unless blank, +// then body_html) so the preprocessed text — and thus the chunk offsets — align +// with what was embedded. Returns null when the message is out of scope. +async function loadMessage(messageId, accountIds) { + const { rows } = await query( + 'SELECT subject, body_text, body_html FROM messages WHERE id = $1 AND account_id = ANY($2)', + [messageId, accountIds], + ); + if (!rows.length) return null; + const subject = rows[0].subject || ''; + const bodyText = rows[0].body_text; + const rawBody = bodyText && bodyText.trim() !== '' ? bodyText : (rows[0].body_html || ''); + return { subject, rawBody }; +} + +export async function matchesInMessage(messageId, queryText, minScore, { accountIds }) { + const q = (queryText || '').trim(); + if (!q) return []; + + const { cfg } = await resolveActiveGenerationFromConfig(); + + const msg = await loadMessage(messageId, accountIds); + if (!msg) return []; + const { subject, rawBody } = msg; + if (!rawBody) return []; + + const { text: preprocessed } = preprocess(subject, rawBody, 0, derivedPreprocessCfg(cfg)); + const prefixRunes = subjectPrefixRunes(subject); + const window = cfg.maxInputChars || 0; + const { spans } = chunkText(preprocessed, window, chunkOverlapFor(window), MAX_SPANS); + if (!spans.length) return []; + + // Re-embed the query plus every chunk. Embeddings are deterministic for a given + // model+input, so re-embedding the chunk text yields the same vectors the stored + // chunks carry — the equivalent of msgvault's ScoreMessageChunks. The inputs + // (1 query + up to MAX_SPANS chunks) are embedded in slices of cfg.batchSize, + // the same per-call budget the worker's write path honors. + const client = new EmbeddingClient(cfg); + const inputs = [q, ...spans.map((s) => s.text)]; + const batchSize = cfg.batchSize > 0 ? cfg.batchSize : 32; + const vecs = []; + for (let i = 0; i < inputs.length; i += batchSize) { + vecs.push(...await client.embed(inputs.slice(i, i + batchSize))); + } + const queryVec = vecs[0]; + const bodyBuf = Buffer.from(rawBody, 'utf8'); + const min = Number.isFinite(minScore) ? minScore : 0; + + return spans + .map((span, i) => ({ span, score: cosine(queryVec, vecs[i + 1]) })) + .filter((s) => s.score >= min) + .sort((a, b) => b.score - a.score) + .map(({ span, score }) => { + const chunkBuf = Buffer.from(span.text, 'utf8'); + const m = { snippet: bytePrefix(chunkBuf, SNIPPET_BYTES), score }; + if (span.charStart >= prefixRunes) { + const off = uniqueByteOffset(bodyBuf, chunkBuf); + if (off !== null) { m.char_offset = off; m.line = lineNumberAt(bodyBuf, off); } + } + return m; + }); +} + +export async function matchFromChunk(messageId, bestChunk, { accountIds }) { + if (!bestChunk) return null; + const msg = await loadMessage(messageId, accountIds); + if (!msg) return null; + const { subject, rawBody } = msg; + + const cfg = await resolveEmbedConfig(); + const { text: preprocessed } = preprocess(subject, rawBody, 0, derivedPreprocessCfg(cfg)); + const prefixRunes = subjectPrefixRunes(subject); + const cps = [...preprocessed]; // code points — the domain phase-4 char_start/char_end index into + const start = Math.max(0, bestChunk.char_start | 0); + const end = Math.min(cps.length, bestChunk.char_end | 0); + if (end <= start) return null; + + const chunkStr = cps.slice(start, end).join(''); + const chunkBuf = Buffer.from(chunkStr, 'utf8'); + const match = { snippet: bytePrefix(chunkBuf, SNIPPET_BYTES), score: bestChunk.score }; + if (start >= prefixRunes && rawBody) { + const bodyBuf = Buffer.from(rawBody, 'utf8'); + const off = uniqueByteOffset(bodyBuf, chunkBuf); + if (off !== null) { match.char_offset = off; match.line = lineNumberAt(bodyBuf, off); } + } + return match; +} diff --git a/backend/src/services/embeddings/chunkmatch.test.js b/backend/src/services/embeddings/chunkmatch.test.js new file mode 100644 index 00000000..ecec9279 --- /dev/null +++ b/backend/src/services/embeddings/chunkmatch.test.js @@ -0,0 +1,171 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Real module shapes (verified against source, not the plan's Consumes sketch): +// preprocess(subject, body, maxChars, cfg) -> { text, truncated } +// chunkText(text, maxRunes, overlap, maxSpans) -> { spans: [{text, charStart, charEnd}] } +// client.js exports the EmbeddingClient class (no bare `embed`) — mirror hybrid.js: +// new EmbeddingClient(cfg).embed([...]) -> number[][] +// resolveEmbedConfig() is async; generationFingerprint(cfg) needs the full config. +const { mockEmbed } = vi.hoisted(() => ({ mockEmbed: vi.fn() })); + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('./hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('./client.js', () => ({ EmbeddingClient: class { embed(...a) { return mockEmbed(...a); } } })); +vi.mock('./chunk.js', async (importOriginal) => ({ ...(await importOriginal()), chunkText: vi.fn() })); +vi.mock('./preprocess.js', () => ({ preprocess: vi.fn() })); +vi.mock('./config.js', () => ({ + generationFingerprint: vi.fn(() => 'fp'), + resolveEmbedConfig: vi.fn(async () => ({ + enabled: true, endpoint: 'http://x', apiKey: null, model: 'm', dimension: 2, + maxInputChars: 100, preprocess: {}, + })), +})); + +import { query } from '../db.js'; +import { resolveActiveGenerationFromConfig } from './hybrid.js'; +import { chunkText } from './chunk.js'; +import { preprocess } from './preprocess.js'; +import { resolveEmbedConfig } from './config.js'; +import { matchesInMessage, matchFromChunk } from './chunkmatch.js'; + +class VectorUnavailableError extends Error { + constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } +} + +const cfg = { + enabled: true, endpoint: 'http://x', apiKey: null, model: 'm', dimension: 2, + maxInputChars: 100, preprocess: {}, +}; + +beforeEach(() => { + query.mockReset(); + resolveActiveGenerationFromConfig.mockReset(); + chunkText.mockReset(); + preprocess.mockReset(); + mockEmbed.mockReset(); + resolveEmbedConfig.mockReset(); + resolveEmbedConfig.mockResolvedValue(cfg); + // matchesInMessage gets its cfg + generation from the one vector-availability gate. + resolveActiveGenerationFromConfig.mockResolvedValue({ cfg, generation: { id: 1 } }); +}); + +describe('matchesInMessage', () => { + it('throws vector_not_enabled when embeddings are disabled', async () => { + resolveActiveGenerationFromConfig.mockRejectedValueOnce(new VectorUnavailableError('vector_not_enabled')); + await expect(matchesInMessage('m1', 'hi', 0, { accountIds: ['a'] })) + .rejects.toHaveProperty('reason', 'vector_not_enabled'); + }); + + it('propagates VectorUnavailableError on stock Postgres', async () => { + resolveActiveGenerationFromConfig.mockRejectedValue(new VectorUnavailableError('no_active_generation')); + await expect(matchesInMessage('m1', 'hi', 0, { accountIds: ['a'] })) + .rejects.toHaveProperty('reason', 'no_active_generation'); + }); + + it('scores body chunks, returns byte char_offset for a unique body chunk, filters by minScore', async () => { + const subject = 'Trip'; + const body = 'café — the flight itinerary is attached and confirmed'; + query.mockResolvedValueOnce({ rows: [{ subject, body_text: body, body_html: '' }] }); + // preprocessed = "Subject: Trip\n\n" + body; prefix rune count decides subject vs body. + preprocess.mockReturnValue({ text: 'Subject: Trip\n\n' + body }); + // one subject chunk, one body chunk that appears verbatim & uniquely in the raw body. + chunkText.mockReturnValue({ spans: [ + { text: 'Subject: Trip', charStart: 0, charEnd: 13 }, + { text: 'the flight itinerary is attached', charStart: 15, charEnd: 47 }, + ] }); + // embed([query, chunk0, chunk1]) -> query aligns with chunk1 (score 1), chunk0 score 0. + mockEmbed.mockResolvedValue([[1, 0], [0, 1], [1, 0]]); + const out = await matchesInMessage('m1', 'flight itinerary', 0.5, { accountIds: ['a'] }); + expect(out).toHaveLength(1); // chunk0 filtered by minScore + const byteOffset = Buffer.from(body, 'utf8').indexOf(Buffer.from('the flight itinerary is attached', 'utf8')); + expect(out[0].char_offset).toBe(byteOffset); // BYTE offset into raw body + expect(out[0].line).toBe(1); + expect(out[0].score).toBeCloseTo(1); + expect(out[0].snippet).toContain('flight itinerary'); + // message load was scoped to accountIds + expect(query.mock.calls[0][1]).toEqual(['m1', ['a']]); + // query embedded first, then each chunk text + expect(mockEmbed).toHaveBeenCalledWith(['flight itinerary', 'Subject: Trip', 'the flight itinerary is attached']); + }); + + it('returns [] when the message is not in scope', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await matchesInMessage('m1', 'x', 0, { accountIds: ['a'] })).toEqual([]); + }); + + it('slices embed calls by cfg.batchSize and keeps query/chunk alignment across slices', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ cfg: { ...cfg, batchSize: 2 }, generation: { id: 1 } }); + query.mockResolvedValueOnce({ rows: [{ subject: '', body_text: 'b', body_html: '' }] }); + preprocess.mockReturnValue({ text: 'b' }); + chunkText.mockReturnValue({ spans: [ + { text: 'c0', charStart: 0, charEnd: 2 }, + { text: 'c1', charStart: 2, charEnd: 4 }, + { text: 'c2', charStart: 4, charEnd: 6 }, + ] }); + // 4 inputs (query + 3 chunks) with batchSize 2 → two calls of 2. + mockEmbed + .mockResolvedValueOnce([[1, 0], [0, 1]]) // [query, c0] + .mockResolvedValueOnce([[1, 0], [0, 1]]); // [c1, c2] — c1 aligns with the query + const out = await matchesInMessage('m1', 'q', 0.5, { accountIds: ['a'] }); + expect(mockEmbed.mock.calls.map((c) => c[0])).toEqual([['q', 'c0'], ['c1', 'c2']]); + expect(out).toHaveLength(1); + expect(out[0].snippet).toBe('c1'); // slice order preserved: vecs[2] is still chunk 1 + }); + + it('preprocesses with the worker\'s derived maxBodyRunes cap so offsets align', async () => { + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'b', body_html: '' }] }); + preprocess.mockReturnValue({ text: '' }); + chunkText.mockReturnValue({ spans: [] }); + await matchesInMessage('m1', 'q', 0, { accountIds: ['a'] }); + // worker.js _embedBatch: maxInputChars(100) * MAX_SPANS(64) * RAW_BODY_MULT(16) + expect(preprocess).toHaveBeenCalledWith('S', 'b', 0, { maxBodyRunes: 102400 }); + }); + + it('never overrides an explicitly configured maxBodyRunes', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { ...cfg, preprocess: { maxBodyRunes: 7 } }, generation: { id: 1 }, + }); + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'b', body_html: '' }] }); + preprocess.mockReturnValue({ text: '' }); + chunkText.mockReturnValue({ spans: [] }); + await matchesInMessage('m1', 'q', 0, { accountIds: ['a'] }); + expect(preprocess).toHaveBeenCalledWith('S', 'b', 0, { maxBodyRunes: 7 }); + }); +}); + +describe('matchFromChunk', () => { + it('re-derives snippet + raw-body byte char_offset from best_chunk, without embedding', async () => { + const subject = 'Trip'; + const body = 'the flight itinerary is attached and confirmed'; + query.mockResolvedValueOnce({ rows: [{ subject, body_text: body, body_html: '' }] }); + preprocess.mockReturnValue({ text: 'Subject: Trip\n\n' + body }); + const prefixLen = [...'Subject: Trip\n\n'].length; // code points before the body + const startCp = prefixLen + body.indexOf('flight itinerary'); + const endCp = startCp + 'flight itinerary'.length; + const m = await matchFromChunk('m1', { chunk_index: 0, char_start: startCp, char_end: endCp, score: 0.87 }, { accountIds: ['acc-1'] }); + expect(m.snippet).toBe('flight itinerary'); + expect(m.score).toBe(0.87); + expect(m.char_offset).toBe(Buffer.from(body, 'utf8').indexOf(Buffer.from('flight itinerary', 'utf8'))); + expect(m.line).toBe(1); + expect(mockEmbed).not.toHaveBeenCalled(); // matchFromChunk never embeds — phase 4 already scored + expect(query.mock.calls[0][1]).toEqual(['m1', ['acc-1']]); // scoped to the token's accounts + }); + + it('returns null for an out-of-scope message', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await matchFromChunk('m1', { char_start: 0, char_end: 5, score: 1 }, { accountIds: ['acc-1'] })).toBeNull(); + }); + + it('returns null when the code-point range is empty', async () => { + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'body', body_html: '' }] }); + preprocess.mockReturnValue({ text: 'Subject: S\n\nbody' }); + expect(await matchFromChunk('m1', { char_start: 5, char_end: 5, score: 1 }, { accountIds: ['acc-1'] })).toBeNull(); + }); + + it('preprocesses with the worker\'s derived maxBodyRunes cap so stored offsets align', async () => { + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'body', body_html: '' }] }); + preprocess.mockReturnValue({ text: 'Subject: S\n\nbody' }); + await matchFromChunk('m1', { char_start: 12, char_end: 16, score: 1 }, { accountIds: ['acc-1'] }); + expect(preprocess).toHaveBeenCalledWith('S', 'body', 0, { maxBodyRunes: 102400 }); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.js b/backend/src/services/embeddings/vectorStore.js index 6cfb19af..a19fe50f 100644 --- a/backend/src/services/embeddings/vectorStore.js +++ b/backend/src/services/embeddings/vectorStore.js @@ -328,6 +328,18 @@ export async function annSearch(gen, queryVec, k, { efSearch = 100, filter = nul } } +// Return the chunk_index=0 vector for messageId in the active generation. +export async function loadVector(messageId) { + const active = await pool.query("SELECT id, dimension FROM index_generations WHERE state = 'active'"); + if (!active.rows.length) throw new Error('no active generation'); + const r = await pool.query( + 'SELECT embedding::text lit FROM embeddings WHERE generation_id = $1 AND message_id = $2 AND chunk_index = 0', + [active.rows[0].id, messageId], + ); + if (!r.rows.length) throw new Error(`no embedding for message ${messageId}`); + return r.rows[0].lit.slice(1, -1).split(',').map(Number); +} + // Forward scan by UUID for live messages needing work, resuming above afterId. The // predicate is `embed_gen IS NULL` (NOT the OR with `embed_gen <> target`) so the // partial index idx_messages_embed_pending (migration 0039) drives an O(pending) scan @@ -501,6 +513,11 @@ async function filteredChunkMessageCount(db, { generation, accountIds, buildFilt // for lexical-only or vector-only pools. `buildFilters(bind) → string[]` // applies the SAME structured operator predicates to both legs (README "one // search seam" — lexicalRepo remains the single owner of those predicates). +// Each hit also carries best_chunk_index/best_char_start/best_char_end — +// the ANN leg's winning (min-distance) chunk's offsets, null on an +// FTS-only hit. These are CODE POINTS into the PREPROCESSED text (README +// Unicode contract), not raw body_text byte offsets — chunkmatch.js owns +// turning them into a raw-body byte snippet. export async function fusedSearch(req, { client } = {}) { const db = client || pool; const { generation, accountIds, rrfK, kPerSignal, limit } = req; @@ -586,6 +603,12 @@ export async function fusedSearch(req, { client } = {}) { const innerArg = bind(innerChunks); const kp1 = bind(kPlus1); const k = bind(kPerSignal); + // DISTINCT ON (message_id), ordered by distance, picks the SAME min-distance + // chunk per message as the MIN(distance)/GROUP BY form — but also + // keeps that winning chunk's offsets, which the excerpt seam + // needs. Offsets are code points into the PREPROCESSED text, not raw + // body_text (README Unicode contract) — phase 5's chunkmatch.js owns + // turning them into a raw-body byte snippet. ctes.push(`ann_pool AS ( SELECT d.message_id, d.distance, d.chunk_index, d.chunk_char_start, d.chunk_char_end FROM ( diff --git a/backend/src/services/embeddings/worker.js b/backend/src/services/embeddings/worker.js index 40bb2d8c..8ea2caf2 100644 --- a/backend/src/services/embeddings/worker.js +++ b/backend/src/services/embeddings/worker.js @@ -3,6 +3,8 @@ import { preprocess } from './preprocess.js'; import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; import { isPermanent4xx } from './client.js'; +// worker.go rawBodyMultiplier. Exported for chunkmatch.js, whose read-path re-preprocess +// must derive the identical maxBodyRunes cap or chunk offsets misalign on huge bodies. export const RAW_BODY_MULT = 16; export class EmbeddingWorker { diff --git a/backend/src/services/search/lexicalRepo.js b/backend/src/services/search/lexicalRepo.js index 0b9e7b59..967f4747 100644 --- a/backend/src/services/search/lexicalRepo.js +++ b/backend/src/services/search/lexicalRepo.js @@ -146,6 +146,15 @@ export function negatedFreeTextClause(term, bind) { return freeTextTermClause(term, true, bind); } +// Body-only free-text match for scope:'body' (MCP search_message_bodies leg): +// matches ONLY the message body, length-capped at the SAME FTS_BODY_CHAR_CAP as +// the stored FTS body cap and the body_text return cap (frozen contract — never +// a second cap, so every FTS body match is locatable in the returned text). +// Query-time (not GIN-served); bodies are sparse and this pool is bounded (D3). +export function bodyTermCondition(ftsIdx, term) { + return ftsMatchExpr(`to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))`, ftsIdx, term); +} + // The weighted tsvector written into messages.search_fts. `ref` is the row // reference: 'm' for the backfill UPDATE, 'NEW' for the BEFORE trigger. The // class weights map to ts_rank_cd's D,C,B,A array (10:4:1 subject:from:rest). @@ -225,7 +234,7 @@ export function buildFolderScopeClauses(folderScope, folderFuzzy, bind) { return conditions; } -export async function searchLexical(client, { parsed, accountIds, folderScope, folderFuzzy, ordering, limit, offset }) { +export async function searchLexical(client, { parsed, accountIds, folderScope, folderFuzzy, ordering, scope = 'metadata', limit, offset }) { const { filters, terms } = parsed; const params = [accountIds]; let p = 2; @@ -238,7 +247,14 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f // hasFTSToken drops it the same way rather than handing Postgres a // token that would normalize to zero lexemes. if (term.value.length < 2 || !hasSearchableToken(term.value)) continue; - conditions.push(freeTextTermClause(term.value, term.negate, bind)); + if (scope === 'body') { + params.push(term.value); + const ftsIdx = p++; + const cond = bodyTermCondition(ftsIdx, term.value); + conditions.push(stopwordSafeCondition(ftsIdx, term.value, term.negate ? negateCond(cond) : cond)); + } else { + conditions.push(freeTextTermClause(term.value, term.negate, bind)); + } } // A bare in:inbox (or lone folder param) must never dump a whole folder. (No total @@ -280,6 +296,15 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f orderBy = `ORDER BY COALESCE(${rankExpr}, 0) DESC, m.date DESC, m.id DESC`; } + // scope:'body' returns the body so MCP can compute keyword excerpts without a + // second query. The return cap is FTS_BODY_CHAR_CAP — the SAME constant as the + // FTS body cap (frozen contract: never a second cap), so every FTS body match + // is locatable in the returned text. Metadata scope keeps the historical + // column list (REST response byte-identical). + const bodyCol = scope === 'body' + ? `,\n LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}) AS body_text` + : ''; + params.push(limit); params.push(offset); @@ -287,7 +312,7 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f SELECT m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, - a.name as account_name, a.email_address as account_email, a.color as account_color + a.name as account_name, a.email_address as account_email, a.color as account_color${bodyCol} FROM messages m JOIN email_accounts a ON m.account_id = a.id WHERE m.account_id = ANY($1) @@ -297,8 +322,11 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f LIMIT $${p} OFFSET $${p + 1} `, params); - // Bounded metadata total: a COUNT(*) over the SAME predicate (no LIMIT/OFFSET). - const countResult = await client(` + // Bounded metadata total: a COUNT(*) over the SAME predicate (no LIMIT/OFFSET), for + // scope:'metadata' only. body/semantic stay total:-1 upstream. + let total; + if (scope === 'metadata') { + const countResult = await client(` SELECT COUNT(*) AS total FROM messages m JOIN email_accounts a ON m.account_id = a.id @@ -306,7 +334,8 @@ export async function searchLexical(client, { parsed, accountIds, folderScope, f AND m.is_deleted = false AND ${conditions.join('\n AND ')} `, countParams); - const total = Number(countResult.rows[0]?.total ?? 0); // COUNT(*) always returns one row in prod + total = Number(countResult.rows[0]?.total ?? 0); // COUNT(*) always returns one row in prod + } return { rows: result.rows, hasCondition: true, ...(total !== undefined ? { total } : {}) }; } diff --git a/backend/src/services/search/lexicalRepo.test.js b/backend/src/services/search/lexicalRepo.test.js index 38562b8e..8db21d7a 100644 --- a/backend/src/services/search/lexicalRepo.test.js +++ b/backend/src/services/search/lexicalRepo.test.js @@ -5,6 +5,7 @@ import { negateCond, trashFolderExclusionCondition, freeTextTermCondition, + bodyTermCondition, searchFtsExpr, searchLexical, LEXICAL_RANK_SQL, @@ -32,6 +33,24 @@ describe('SQL fragment builders', () => { expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); }); + it('bodyTermCondition prefix-matches a single-word term (msgvault BuildFTSArg parity)', () => { + const cond = bodyTermCondition(4, 'invoice'); + expect(cond).toBe( + "to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ to_tsquery('english', quote_literal($4) || ':*')" + ); + expect(cond).not.toContain('ILIKE'); + expect(cond).not.toContain('search_vector'); + expect(cond).not.toContain('plainto_tsquery'); + }); + + it('bodyTermCondition keeps non-prefix phrase matching for a quoted multi-word term', () => { + const cond = bodyTermCondition(4, 'weekly report'); + expect(cond).toBe( + "to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ plainto_tsquery('english', $4)" + ); + expect(cond).not.toContain('quote_literal'); + }); + it('negateCond wraps a positive condition to also match NULL columns', () => { expect(negateCond('m.is_read = true')).toBe('NOT COALESCE((m.is_read = true), false)'); }); @@ -103,6 +122,22 @@ describe('searchLexical', () => { expect(calls[0].params).toEqual([['a1'], '%boss%', 'INBOX', 50, 0]); }); + it('scope:body matches only the capped body branch and returns body_text at the SAME cap', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', scope: 'body', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // Body-only free-text leg (no sender/subject ILIKE, no search_vector), + // prefix-matched (msgvault BuildFTSArg parity — "invoic" still finds "invoice"). + expect(text).toContain("to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ to_tsquery('english', quote_literal($2) || ':*')"); + expect(text).not.toContain('ILIKE $2'); + // body_text returned at FTS_BODY_CHAR_CAP (600000) — same cap as the FTS body match (frozen contract). + expect(text).toContain("LEFT(coalesce(m.body_text,''), 600000) AS body_text"); + // Body scope pushes only the fts param per term (no %like%): [accountIds, term, folder, limit, offset]. + expect(params).toEqual([['a1'], 'invoice', 'INBOX', 50, 0]); + }); }); describe('ranked lexical query (slice 02)', () => { @@ -295,6 +330,17 @@ describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { expect(text).toContain("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE("); }); + it('guards the body-scope term condition with the same construction', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', scope: 'body', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain( + "(numnode(to_tsquery('english', quote_literal($2) || ':*')) = 0 OR to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@" + ); + }); + it('freeTextTermClause is the one owner searchLexical and the staging path share', () => { const params = []; let p = 2; diff --git a/backend/src/services/search/lexicalRepo.total.test.js b/backend/src/services/search/lexicalRepo.total.test.js index 16efbe1d..065201a8 100644 --- a/backend/src/services/search/lexicalRepo.total.test.js +++ b/backend/src/services/search/lexicalRepo.total.test.js @@ -12,11 +12,18 @@ describe('searchLexical metadata total', () => { const client = vi.fn() .mockResolvedValueOnce({ rows: [{ id: 'm1' }] }) // page query .mockResolvedValueOnce({ rows: [{ total: '42' }] }); // count query - const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], limit: 20, offset: 0 }); + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], scope: 'metadata', limit: 20, offset: 0 }); expect(out.total).toBe(42); const countSql = client.mock.calls[1][0]; expect(countSql).toMatch(/COUNT\(\*\)/i); expect(countSql).not.toMatch(/\bLIMIT\b/i); expect(countSql).not.toMatch(/\bOFFSET\b/i); }); + + it("does not COUNT for scope:'body' (no total)", async () => { + const client = vi.fn().mockResolvedValueOnce({ rows: [] }); // page query only + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], scope: 'body', limit: 20, offset: 0 }); + expect(out.total).toBeUndefined(); + expect(client.mock.calls.every((c) => !/COUNT\(\*\)/i.test(c[0]))).toBe(true); + }); }); diff --git a/backend/src/services/search/searchService.js b/backend/src/services/search/searchService.js index 337f992e..9ed9a70e 100644 --- a/backend/src/services/search/searchService.js +++ b/backend/src/services/search/searchService.js @@ -37,31 +37,44 @@ function aliasIdFromMessageId(hits) { for (const h of hits) h.id = h.message_id; } +// A caller that already resolved its scope (e.g. the MCP handler, from the +// bearer-token owner's enabled accounts) passes accountIds directly and it is +// trusted as-is; otherwise derive it from the session userId. The REST route +// never forwards a raw accountIds, so a browser client cannot widen its scope. +// Shared by both the lexical and semantic branches. async function resolveAccountIds(request) { - const { userId, accountId } = request; - const accountsResult = await query( - 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', - [userId] - ); - let accountIds = accountsResult.rows.map(r => r.id); - if (!accountIds.length) return []; - // Optional single-account narrowing, only within the authenticated user's scope. - if (accountId && accountIds.includes(accountId)) accountIds = [accountId]; - return accountIds; + const { userId, accountIds: providedScope, accountId } = request; + let scopeIds; + if (providedScope) { + scopeIds = providedScope; + } else { + const accountsResult = await query( + 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId] + ); + scopeIds = accountsResult.rows.map(r => r.id); + } + if (!scopeIds.length) return []; + // Optional single-account narrowing (REST ?accountId=), only within scope. + return accountId && scopeIds.includes(accountId) ? [accountId] : scopeIds; } // Phase 1's original lexical search, extracted verbatim so the mode dispatch // below can reuse it both as the default path and as the fallback target when // a semantic search degrades. Returns the pre-Phase-4 shape (no `mode` field — // the caller adds that uniformly). -async function runLexical(request, resolvedAccountIds) { - const { parsed, folderParam = '', limit = 50, offset = 0 } = request; +async function runLexical(request) { + const { parsed, folderParam = '', limit = 50, offset = 0, scope } = request; + + // Frozen contract: scope ∈ 'metadata' | 'body' (default metadata). Anything + // else coerces to metadata (today's behavior); phase 5's body tool passes 'body'. + const searchScope = scope === 'body' ? 'body' : 'metadata'; const cap = clampLimit(limit); const off = Math.max(0, parseInt(offset) || 0); const emptyPage = { offset: off, limit: cap, hasMore: false }; - const accountIds = resolvedAccountIds || await resolveAccountIds(request); + const accountIds = await resolveAccountIds(request); if (!accountIds.length) return { messages: [], page: emptyPage }; const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, folderParam); @@ -72,7 +85,7 @@ async function runLexical(request, resolvedAccountIds) { const ordering = hasPositiveText ? 'relevance' : 'date'; const { rows, total, hasCondition } = await searchLexical(query, { - parsed, accountIds, folderScope, folderFuzzy, ordering, limit: cap, offset: off, + parsed, accountIds, folderScope, folderFuzzy, ordering, scope: searchScope, limit: cap, offset: off, }); if (!hasCondition) return { messages: [], page: emptyPage }; @@ -97,6 +110,7 @@ export async function search(request) { // Resolve once, up front, and thread it into a lexical fallback below — a // fallback must not re-resolve accounts via a second DB round-trip. const accountIds = await resolveAccountIds(request); + const scopedRequest = { ...request, accountIds }; try { if (accountIds.length === 0) { return { messages: [], mode, pool_saturated: false, generation: null, @@ -144,7 +158,11 @@ export async function search(request) { }; } catch (err) { if (isLexicalFallback(err)) { - const lexical = { ...(await runLexical(request, accountIds)), mode: 'lexical' }; + if (request.strictVector) { + if (err instanceof MissingFreeTextError) throw err; // invalid input, not unavailability + throw err; // VectorUnavailableError already carries .reason + } + const lexical = { ...(await runLexical(scopedRequest)), mode: 'lexical' }; // A filter-only query in semantic mode (MissingFreeTextError) is not a // degradation — there is nothing to embed and the lexical result IS the // answer, so no fellBack (the UI keys its amber "index building" hint on diff --git a/backend/src/services/search/searchService.test.js b/backend/src/services/search/searchService.test.js index 9ada7260..eaacb0c8 100644 --- a/backend/src/services/search/searchService.test.js +++ b/backend/src/services/search/searchService.test.js @@ -17,6 +17,7 @@ import { query } from '../db.js'; import { searchLexical } from './lexicalRepo.js'; import * as hybrid from '../embeddings/hybrid.js'; import { search } from './searchService.js'; +import { VectorUnavailableError } from '../embeddings/vectorErrors.js'; beforeEach(() => { query.mockReset(); searchLexical.mockReset(); @@ -44,6 +45,13 @@ describe('searchService.search', () => { expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a2']); }); + it('trusts a caller-provided accountIds scope and skips the DB account lookup (MCP path)', async () => { + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ accountIds: ['a1', 'a2'], parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(query).not.toHaveBeenCalled(); // no user_id → email_accounts lookup + expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a1', 'a2']); + }); + it('passes through an optional total when searchLexical returns one', async () => { withAccounts(['a1']); searchLexical.mockResolvedValue({ rows: [{ id: 'm' }], total: 137, hasCondition: true }); @@ -51,6 +59,25 @@ describe('searchService.search', () => { expect(res.total).toBe(137); }); + it('passes scope through to searchLexical, defaulting to metadata and coercing unknown values', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('metadata'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', scope: 'body', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('body'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', scope: 'nonsense', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('metadata'); + }); + it('chooses relevance ordering for free text and date ordering for filter-only queries', async () => { withAccounts(['a1']); searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); @@ -298,13 +325,22 @@ describe('searchService semantic pagination hasMore (Fix 3)', () => { }); }); -describe('searchService envelope (Task 5b)', () => { +describe('searchService strictVector + envelope (Task 5b)', () => { const hybridReq2 = (extra = {}) => ({ userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, parsed: { filters: [], terms: [{ value: 'q', negate: false }] }, ...extra, }); + it('rethrows the VectorUnavailableError (no fallback) under strictVector', async () => { + withAccounts(['a1']); + const unavail = new VectorUnavailableError('index_building'); + hybrid.hybridSearch.mockRejectedValue(unavail); + const err = await search(hybridReq2({ strictVector: true })).catch(e => e); + expect(err).toBe(unavail); + expect(err.reason).toBe('index_building'); + }); + it('returns the rich generation object and per-hit score under explain', async () => { withAccounts(['a1']); hybrid.hybridSearch.mockResolvedValue({ diff --git a/backend/src/services/search/searchService.total.test.js b/backend/src/services/search/searchService.total.test.js index 09fdd1dd..f7c5bf8b 100644 --- a/backend/src/services/search/searchService.total.test.js +++ b/backend/src/services/search/searchService.total.test.js @@ -1,15 +1,12 @@ import { it, expect, vi } from 'vitest'; -vi.mock('../db.js', () => ({ query: vi.fn() })); vi.mock('./lexicalRepo.js', () => ({ searchLexical: vi.fn(), buildOperatorClauses: vi.fn(), hasSearchableToken: vi.fn() })); -import { query } from '../db.js'; import { searchLexical } from './lexicalRepo.js'; import { search } from './searchService.js'; it('passes the metadata total through to the service result', async () => { - query.mockResolvedValue({ rows: [{ id: 'acc-1' }] }); searchLexical.mockResolvedValue({ rows: [{ id: 'm1' }], total: 42, hasCondition: true }); const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; - const r = await search({ mode: 'lexical', userId: 'user-1', parsed, limit: 20, offset: 0 }); + const r = await search({ mode: 'lexical', scope: 'metadata', parsed, accountIds: ['acc-1'], limit: 20, offset: 0 }); expect(r.total).toBe(42); - expect(searchLexical.mock.calls[0][1]).toMatchObject({ accountIds: ['acc-1'] }); + expect(searchLexical.mock.calls[0][1]).toMatchObject({ scope: 'metadata', accountIds: ['acc-1'] }); }); diff --git a/backend/src/testSupport/mockSurface.js b/backend/src/testSupport/mockSurface.js new file mode 100644 index 00000000..aa3fc448 --- /dev/null +++ b/backend/src/testSupport/mockSurface.js @@ -0,0 +1,24 @@ +import { vi } from 'vitest'; + +// Mock-drift guard. Given a mocked module namespace (`import * as ns` from a +// vi.mock'd module, or a hand-built `{ fnA, fnB }` of its named imports) and the +// real module (`await vi.importActual(path)`), return the names that are mocked +// as functions but do NOT exist as functions on the real module. A non-empty +// result means a suite invented a seam the production module never implemented — +// e.g. `generations.chunkCount`, mocked as a vi.fn() in two suites while the real +// module never exported it, which made live `collectStats` throw and silently drop +// get_stats' vector_search block. +// +// Scope note: this catches the missing/renamed EXPORT drift class only, not +// value-shape drift (a mock whose fn returns the wrong row shape still passes — +// the earlier message_id/id bug would not have been caught here). Pair it with +// real-shape fixtures where the return shape is load-bearing. +export function mockSurfaceDrift(mockedNs, realNs) { + const drift = []; + for (const [name, value] of Object.entries(mockedNs)) { + if (vi.isMockFunction(value) && typeof realNs[name] !== 'function') { + drift.push(name); + } + } + return drift; +} diff --git a/backend/src/utils/textExcerpt.js b/backend/src/utils/textExcerpt.js new file mode 100644 index 00000000..7c9f92ca --- /dev/null +++ b/backend/src/utils/textExcerpt.js @@ -0,0 +1,21 @@ +// Low-level UTF-8 / line primitives shared by the excerpt builders on both search +// paths: mcp/bodyMatch.js (keyword matches) and services/embeddings/chunkmatch.js +// (vector chunk matches). All offsets are UTF-8 BYTES (msgvault wire contract). The +// higher-level keyword-vs-chunk assembly stays in each owner; only these primitives +// live here so they cannot drift. + +// Default snippet width in bytes for a match excerpt. +export const SNIPPET_BYTES = 300; + +// True when `byte` is a UTF-8 leading byte (not a 10xxxxxx continuation byte), so a +// slice boundary at it does not split a multi-byte rune. +export const isRuneStart = (byte) => (byte & 0xc0) !== 0x80; + +// 1-based line number at a byte offset: one plus the count of '\n' bytes before it. +export function lineNumberAt(buf, byteOffset) { + if (byteOffset <= 0) return 1; + const o = Math.min(byteOffset, buf.length); + let n = 1; + for (let i = 0; i < o; i++) if (buf[i] === 0x0a) n++; + return n; +} diff --git a/backend/src/utils/textExcerpt.test.js b/backend/src/utils/textExcerpt.test.js new file mode 100644 index 00000000..036046bf --- /dev/null +++ b/backend/src/utils/textExcerpt.test.js @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from './textExcerpt.js'; + +describe('SNIPPET_BYTES', () => { + it('is the shared 300-byte excerpt width', () => { + expect(SNIPPET_BYTES).toBe(300); + }); +}); + +describe('isRuneStart', () => { + it('flags UTF-8 leading bytes, not continuation bytes', () => { + const buf = Buffer.from('é', 'utf8'); // 0xC3 0xA9 + expect(isRuneStart(buf[0])).toBe(true); // 0xC3 lead byte + expect(isRuneStart(buf[1])).toBe(false); // 0xA9 continuation byte + expect(isRuneStart(0x61)).toBe(true); // ASCII 'a' + }); +}); + +describe('lineNumberAt', () => { + it('counts newlines before the byte offset (1-based)', () => { + const buf = Buffer.from('a\nb\nc', 'utf8'); + expect(lineNumberAt(buf, 0)).toBe(1); + expect(lineNumberAt(buf, 2)).toBe(2); + expect(lineNumberAt(buf, 4)).toBe(3); + }); + it('clamps a negative or over-long offset', () => { + const buf = Buffer.from('a\nb', 'utf8'); + expect(lineNumberAt(buf, -1)).toBe(1); + expect(lineNumberAt(buf, 999)).toBe(2); + }); +}); diff --git a/frontend/nginx.conf b/frontend/nginx.conf index c4ca0c89..5d486404 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -87,6 +87,19 @@ server { proxy_read_timeout 60s; } + # MCP endpoint (Streamable HTTP — long-lived SSE responses, no buffering) + location /mcp { + proxy_pass http://backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_cache off; + } + # WebSocket proxy location /ws { proxy_pass http://backend:3000; @@ -215,6 +228,19 @@ server { proxy_read_timeout 60s; } + # MCP endpoint (Streamable HTTP — long-lived SSE responses, no buffering) + location /mcp { + proxy_pass http://backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_cache off; + } + # WebSocket proxy location /ws { proxy_pass http://backend:3000; diff --git a/frontend/src/components/ProfileModal.jsx b/frontend/src/components/ProfileModal.jsx index 1470fc2a..01fe6142 100644 --- a/frontend/src/components/ProfileModal.jsx +++ b/frontend/src/components/ProfileModal.jsx @@ -1,4 +1,4 @@ -import { useState, useRef } from 'react'; +import { useState, useRef, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; import { useStore } from '../store/index.js'; import { api } from '../utils/api.js'; @@ -37,6 +37,45 @@ export default function ProfileModal({ onClose }) { const [saving, setSaving] = useState(false); const [error, setError] = useState(''); + // MCP API tokens (per-user bearer tokens for the /mcp endpoint). + const [tokens, setTokens] = useState([]); + const [tokenName, setTokenName] = useState(''); + const [mintedToken, setMintedToken] = useState(''); + const [tokenBusy, setTokenBusy] = useState(false); + + useEffect(() => { + let alive = true; + api.tokens.list().then((r) => { if (alive) setTokens(r.tokens || []); }).catch(() => {}); + return () => { alive = false; }; + }, []); + + async function handleCreateToken() { + const name = tokenName.trim(); + if (!name || tokenBusy) return; + setTokenBusy(true); + setError(''); + try { + const { token } = await api.tokens.create(name); + setMintedToken(token); // shown once — never retrievable again + setTokenName(''); + const r = await api.tokens.list(); + setTokens(r.tokens || []); + } catch (e) { + setError(e.message || t('profile.tokens.createFailed')); + } finally { + setTokenBusy(false); + } + } + + async function handleRevokeToken(id) { + try { + await api.tokens.revoke(id); + setTokens((prev) => prev.filter((tok) => tok.id !== id)); + } catch (e) { + setError(e.message || t('profile.tokens.revokeFailed')); + } + } + async function handleFileChange(e) { const file = e.target.files[0]; if (!file) return; @@ -193,6 +232,61 @@ export default function ProfileModal({ onClose }) {
+ {/* API tokens (MCP) — bearer tokens for the Streamable-HTTP /mcp endpoint */} +
+ + {tokens.length > 0 && ( +
+ {tokens.map((tok) => ( +
+ + {tok.name} + + {tok.last_used_at + ? t('profile.tokens.usedOn', { date: new Date(tok.last_used_at).toLocaleDateString() }) + : t('profile.tokens.neverUsed')} + + + +
+ ))} +
+ )} +
+ setTokenName(e.target.value)} + onKeyDown={e => e.key === 'Enter' && handleCreateToken()} + placeholder={t('profile.tokens.namePlaceholder')} + maxLength={80} + style={{ flex: 1, padding: '7px 10px', borderRadius: 8, border: '1px solid var(--border)', background: 'var(--bg-primary)', color: 'var(--text-primary)', fontSize: 13, outline: 'none', boxSizing: 'border-box' }} + /> + +
+ {mintedToken && ( +
+ {mintedToken} + + {t('profile.tokens.copyOnce')} + +
+ )} +
+ {error && (
{error} diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.json index 0554d466..a4431cf8 100644 --- a/frontend/src/locales/de.json +++ b/frontend/src/locales/de.json @@ -587,7 +587,18 @@ "uploadPhoto": "Foto hochladen", "removePhoto": "Foto entfernen", "errorNotImage": "Bitte eine Bilddatei auswählen.", - "errorProcess": "Bild konnte nicht verarbeitet werden." + "errorProcess": "Bild konnte nicht verarbeitet werden.", + "tokens": { + "label": "API-Tokens (MCP)", + "namePlaceholder": "Token-Name (z. B. Laptop)", + "create": "Erstellen", + "revoke": "Widerrufen", + "usedOn": "verwendet {{date}}", + "neverUsed": "nie verwendet", + "copyOnce": "Jetzt kopieren — du siehst ihn nicht noch einmal.", + "createFailed": "Token konnte nicht erstellt werden", + "revokeFailed": "Token konnte nicht widerrufen werden" + } }, "admin": { "title": "Einstellungen", @@ -998,7 +1009,7 @@ "remove": "Konfiguration entfernen", "emb": { "title": "Embeddings (semantische Suche)", - "benefit": "Embeddings machen die Suche schneller und genauer.", + "benefit": "Embeddings machen die Suche schneller und genauer — für dich oder für einen über MCP verbundenen Agenten.", "defaultOff": "Standardmäßig deaktiviert. Es wird nichts gesendet, bis du Embeddings aktivierst und einen Aufbau startest.", "enable": "Embeddings aktivieren", "privacyWarning": "Mit einem gehosteten Endpunkt erhält der Embeddings-Anbieter (z. B. OpenAI) deine E-Mail-Inhalte — Betreff und Textkörper. Ab diesem Punkt ist dieser Datenpfad nicht mehr selbst gehostet.", diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json index 88bc4d1a..abb6a2de 100644 --- a/frontend/src/locales/en.json +++ b/frontend/src/locales/en.json @@ -587,7 +587,18 @@ "uploadPhoto": "Upload photo", "removePhoto": "Remove photo", "errorNotImage": "Please select an image file.", - "errorProcess": "Could not process image." + "errorProcess": "Could not process image.", + "tokens": { + "label": "API tokens (MCP)", + "namePlaceholder": "Token name (e.g. laptop)", + "create": "Create", + "revoke": "Revoke", + "usedOn": "used {{date}}", + "neverUsed": "never used", + "copyOnce": "Copy this now — you won't see it again.", + "createFailed": "Failed to create token", + "revokeFailed": "Failed to revoke token" + } }, "admin": { "title": "Settings", @@ -1065,7 +1076,7 @@ "remove": "Remove configuration", "emb": { "title": "Embeddings (semantic search)", - "benefit": "Embeddings make search faster and more accurate.", + "benefit": "Embeddings make search faster and more accurate — for you, or for an agent connected over MCP.", "defaultOff": "Disabled by default. Nothing is sent anywhere until you enable embeddings and start a build.", "enable": "Enable embeddings", "privacyWarning": "With a hosted endpoint configured, the embeddings provider (e.g. OpenAI) receives your email content — subject and body text. At that point this data path is no longer self-hosted.", diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.json index 5f6b7a81..3e3b9151 100644 --- a/frontend/src/locales/es.json +++ b/frontend/src/locales/es.json @@ -587,7 +587,18 @@ "uploadPhoto": "Subir foto", "removePhoto": "Eliminar foto", "errorNotImage": "Por favor selecciona un archivo de imagen.", - "errorProcess": "No se pudo procesar la imagen." + "errorProcess": "No se pudo procesar la imagen.", + "tokens": { + "label": "Tokens de API (MCP)", + "namePlaceholder": "Nombre del token (p. ej. portátil)", + "create": "Crear", + "revoke": "Revocar", + "usedOn": "usado {{date}}", + "neverUsed": "nunca usado", + "copyOnce": "Cópialo ahora: no volverás a verlo.", + "createFailed": "No se pudo crear el token", + "revokeFailed": "No se pudo revocar el token" + } }, "admin": { "title": "Ajustes", @@ -1065,7 +1076,7 @@ "remove": "Eliminar configuración", "emb": { "title": "Embeddings (búsqueda semántica)", - "benefit": "Los embeddings hacen que la búsqueda sea más rápida y precisa.", + "benefit": "Los embeddings hacen que la búsqueda sea más rápida y precisa, para ti o para un agente conectado por MCP.", "defaultOff": "Desactivado de forma predeterminada. No se envía nada hasta que actives los embeddings e inicies una compilación.", "enable": "Activar embeddings", "privacyWarning": "Con un endpoint alojado configurado, el proveedor de embeddings (p. ej. OpenAI) recibe el contenido de tu correo: el asunto y el cuerpo del texto. En ese momento esta ruta de datos deja de ser autoalojada.", diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.json index 014ed98c..b52e6341 100644 --- a/frontend/src/locales/fr.json +++ b/frontend/src/locales/fr.json @@ -587,7 +587,18 @@ "uploadPhoto": "Téléverser une photo", "removePhoto": "Supprimer la photo", "errorNotImage": "Veuillez sélectionner un fichier image.", - "errorProcess": "Impossible de traiter l'image." + "errorProcess": "Impossible de traiter l'image.", + "tokens": { + "label": "Jetons API (MCP)", + "namePlaceholder": "Nom du jeton (p. ex. ordinateur)", + "create": "Créer", + "revoke": "Révoquer", + "usedOn": "utilisé {{date}}", + "neverUsed": "jamais utilisé", + "copyOnce": "Copiez-le maintenant — vous ne le reverrez plus.", + "createFailed": "Échec de la création du jeton", + "revokeFailed": "Échec de la révocation du jeton" + } }, "admin": { "title": "Paramètres", @@ -1065,7 +1076,7 @@ "remove": "Supprimer la configuration", "emb": { "title": "Embeddings (recherche sémantique)", - "benefit": "Les embeddings rendent la recherche plus rapide et plus précise.", + "benefit": "Les embeddings rendent la recherche plus rapide et plus précise — pour vous ou pour un agent connecté via MCP.", "defaultOff": "Désactivé par défaut. Rien n'est envoyé tant que vous n'activez pas les embeddings et ne lancez pas une génération.", "enable": "Activer les embeddings", "privacyWarning": "Avec un point de terminaison hébergé configuré, le fournisseur d'embeddings (p. ex. OpenAI) reçoit le contenu de vos e-mails — objet et corps du texte. À ce moment-là, ce chemin de données n'est plus auto-hébergé.", diff --git a/frontend/src/locales/it.json b/frontend/src/locales/it.json index be4b5996..78e24ba9 100644 --- a/frontend/src/locales/it.json +++ b/frontend/src/locales/it.json @@ -587,7 +587,18 @@ "uploadPhoto": "Carica foto", "removePhoto": "Rimuovi foto", "errorNotImage": "Seleziona un file immagine.", - "errorProcess": "Impossibile elaborare l'immagine." + "errorProcess": "Impossibile elaborare l'immagine.", + "tokens": { + "label": "Token API (MCP)", + "namePlaceholder": "Nome del token (es. laptop)", + "create": "Crea", + "revoke": "Revoca", + "usedOn": "usato {{date}}", + "neverUsed": "mai usato", + "copyOnce": "Copialo ora: non lo vedrai più.", + "createFailed": "Impossibile creare il token", + "revokeFailed": "Impossibile revocare il token" + } }, "admin": { "title": "Impostazioni", @@ -1065,7 +1076,7 @@ "remove": "Rimuovi configurazione", "emb": { "title": "Embeddings (ricerca semantica)", - "benefit": "Gli embeddings rendono la ricerca più veloce e precisa.", + "benefit": "Gli embeddings rendono la ricerca più veloce e precisa — per te o per un agente connesso tramite MCP.", "defaultOff": "Disattivato per impostazione predefinita. Non viene inviato nulla finché non attivi gli embeddings e avvii una creazione.", "enable": "Attiva gli embeddings", "privacyWarning": "Con un endpoint ospitato configurato, il fornitore di embeddings (ad es. OpenAI) riceve il contenuto delle tue email — oggetto e corpo del testo. A quel punto questo percorso dei dati non è più self-hosted.", diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.json index 4d69f40e..522a7f42 100644 --- a/frontend/src/locales/ru.json +++ b/frontend/src/locales/ru.json @@ -587,7 +587,18 @@ "uploadPhoto": "Загрузить фото", "removePhoto": "Удалить фото", "errorNotImage": "Выберите файл изображения.", - "errorProcess": "Не удалось обработать изображение." + "errorProcess": "Не удалось обработать изображение.", + "tokens": { + "label": "API-токены (MCP)", + "namePlaceholder": "Имя токена (напр. ноутбук)", + "create": "Создать", + "revoke": "Отозвать", + "usedOn": "использован {{date}}", + "neverUsed": "не использован", + "copyOnce": "Скопируйте сейчас — больше вы его не увидите.", + "createFailed": "Не удалось создать токен", + "revokeFailed": "Не удалось отозвать токен" + } }, "admin": { "title": "Настройки", @@ -1065,7 +1076,7 @@ "remove": "Удалить конфигурацию", "emb": { "title": "Эмбеддинги (семантический поиск)", - "benefit": "Эмбеддинги делают поиск быстрее и точнее.", + "benefit": "Эмбеддинги делают поиск быстрее и точнее — для вас или для агента, подключённого через MCP.", "defaultOff": "Отключено по умолчанию. Ничего не отправляется, пока вы не включите эмбеддинги и не запустите построение.", "enable": "Включить эмбеддинги", "privacyWarning": "При настроенной размещённой конечной точке поставщик эмбеддингов (например, OpenAI) получает содержимое ваших писем — тему и текст. С этого момента этот путь данных перестаёт быть самостоятельно размещённым.", diff --git a/frontend/src/locales/zhCN.json b/frontend/src/locales/zhCN.json index bdd5fb1a..463c2dee 100644 --- a/frontend/src/locales/zhCN.json +++ b/frontend/src/locales/zhCN.json @@ -587,7 +587,18 @@ "uploadPhoto": "上传头像", "removePhoto": "移除头像", "errorNotImage": "请选择一个图片文件。", - "errorProcess": "无法处理该图片。" + "errorProcess": "无法处理该图片。", + "tokens": { + "label": "API 令牌 (MCP)", + "namePlaceholder": "令牌名称（例如 笔记本）", + "create": "创建", + "revoke": "撤销", + "usedOn": "使用于 {{date}}", + "neverUsed": "从未使用", + "copyOnce": "立即复制 — 你将无法再次看到它。", + "createFailed": "创建令牌失败", + "revokeFailed": "撤销令牌失败" + } }, "admin": { "title": "设置", @@ -1065,7 +1076,7 @@ "remove": "删除配置", "emb": { "title": "嵌入（语义搜索）", - "benefit": "嵌入让搜索更快、更准确。", + "benefit": "嵌入让搜索更快、更准确——无论是为你自己，还是为通过 MCP 连接的智能体。", "defaultOff": "默认关闭。在你启用嵌入并开始构建之前，不会向任何地方发送数据。", "enable": "启用嵌入", "privacyWarning": "配置托管端点后，嵌入提供方（例如 OpenAI）会收到你的邮件内容——主题和正文文本。此时该数据路径不再是自托管的。", diff --git a/frontend/src/utils/api.js b/frontend/src/utils/api.js index dbdf57ae..25038ffb 100644 --- a/frontend/src/utils/api.js +++ b/frontend/src/utils/api.js @@ -339,4 +339,18 @@ export const api = { createTask: (data) => request('POST', '/todoist/tasks', data), }, + // MCP API tokens — per-user bearer tokens for the Streamable-HTTP /mcp endpoint. + // create() returns the plaintext token exactly once; only its hash is stored. + tokens: { + list: () => request('GET', '/tokens'), + create: (name) => request('POST', '/tokens', { name }), + revoke: async (id) => { + // DELETE responds 204 with no body — don't run it through request()/res.json(). + const res = await fetch(`${BASE}/tokens/${id}`, { + method: 'DELETE', credentials: 'include', + headers: { [CSRF_HEADER]: CSRF_VALUE }, + }); + if (!res.ok) throw new Error('Failed to revoke token'); + }, + }, }; From 025b53113ebf1bf9507a3ff27d1728a6d7b94538 Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:55:13 -0700 Subject: [PATCH 4/4] =?UTF-8?q?feat(mcp):=20write=20tools=20=E2=80=94=20co?= =?UTF-8?q?mpose,=20undo-send,=20mailbox=20ops,=20triage,=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the read-only MCP server (#296) to the full mail client: - Token scopes (read/write/send/settings) enforced fail-closed at one dispatch choke point; tools/list filtered per token; existing tokens stay read-only. Per-class rate limits plus a daily send cap. - 33 new tools (58 total): drafts, send/reply/reply-all/forward with hard alias validation and receipts, unsend via a real outbox (0-120s window, SKIP LOCKED worker, race-safe cancel), honest best-effort recall, folders/ move/archive/trash/flags/spam/snooze/category/GTD with refuse-unbounded guards, an inbox-triage loop (triage_inbox / get_triage_context / mark_triaged) with embedding-derived signals, and settings tools including a staged add_account that never accepts secrets over MCP. - Undo-send frontend: countdown toast with Undo, user preference (0-120s, default off so upgrades never change send behavior). - Service extractions shared by REST and MCP (send/draft/reply/mailbox), fixing real bugs en route: alias silent-fallback, the References chain (thread_references was never used by compose), drafts dropping threading headers, and a new synchronous account test-connection endpoint. - Migrations 0042-0046. Every tool def carries MCP annotations. Co-Authored-By: Claude Fable 5 --- README.md | 166 +++ backend/migrations/0042_api_token_scopes.sql | 13 + backend/migrations/0043_outbox_messages.sql | 30 + backend/migrations/0044_message_triage.sql | 13 + .../migrations/0045_mcp_account_stages.sql | 12 + .../0046_message_triage_feed_index.sql | 12 + backend/src/index.js | 35 +- backend/src/mcp/accountAdapter.js | 493 +++++++ backend/src/mcp/accountAdapter.phase4.test.js | 338 +++++ backend/src/mcp/accountAdapter.test.js | 257 ++++ backend/src/mcp/accountTools.js | 501 +++++++ backend/src/mcp/accountTools.test.js | 457 +++++++ backend/src/mcp/auth.js | 25 +- backend/src/mcp/auth.test.js | 58 +- backend/src/mcp/composeTools.js | 258 ++++ backend/src/mcp/composeTools.test.js | 559 ++++++++ backend/src/mcp/draftTools.js | 432 ++++++ backend/src/mcp/draftTools.test.js | 467 +++++++ backend/src/mcp/engineAdapter.js | 4 +- backend/src/mcp/goldenParity.test.js | 655 ++++++++- backend/src/mcp/mailboxTools.js | 748 ++++++++++ backend/src/mcp/mailboxTools.test.js | 1146 ++++++++++++++++ backend/src/mcp/messageTools.js | 117 +- backend/src/mcp/messageTools.test.js | 3 +- .../src/mcp/messageTriage.integration.test.js | 158 +++ backend/src/mcp/searchTools.js | 21 + backend/src/mcp/searchTools.test.js | 3 +- backend/src/mcp/sendTools.js | 484 +++++++ backend/src/mcp/sendTools.test.js | 1024 ++++++++++++++ backend/src/mcp/server.js | 160 ++- backend/src/mcp/server.test.js | 6 +- backend/src/mcp/serverGuards.test.js | 371 ++++- backend/src/mcp/tools.js | 219 +++ backend/src/mcp/tools.test.js | 42 + .../src/mcp/triageAdapter.integration.test.js | 145 ++ backend/src/mcp/triageAdapter.js | 342 +++++ backend/src/mcp/triageAdapter.test.js | 344 +++++ backend/src/mcp/triageProbes.js | 70 + backend/src/mcp/triageProbes.test.js | 96 ++ backend/src/mcp/triageTools.js | 757 +++++++++++ backend/src/mcp/triageTools.test.js | 1117 +++++++++++++++ backend/src/mcp/writeResult.js | 72 + backend/src/mcp/writeResult.test.js | 106 ++ backend/src/routes/accounts.js | 167 +-- backend/src/routes/accounts.test.js | 123 ++ backend/src/routes/apiTokens.js | 23 +- backend/src/routes/apiTokens.test.js | 85 +- backend/src/routes/auth.js | 7 +- backend/src/routes/auth.preferences.test.js | 72 + backend/src/routes/draft.js | 241 +--- backend/src/routes/draft.test.js | 88 +- backend/src/routes/gtd.js | 277 +--- backend/src/routes/mail.js | 1201 ++--------------- backend/src/routes/mail.test.js | 311 +++++ backend/src/routes/mcpAccountStages.js | 43 + backend/src/routes/mcpAccountStages.test.js | 165 +++ backend/src/routes/rules.js | 27 +- backend/src/routes/send.js | 636 ++------- backend/src/routes/send.outbox.test.js | 250 ++++ backend/src/routes/send.test.js | 185 +++ backend/src/services/accountFields.js | 19 + backend/src/services/accountFields.test.js | 42 + backend/src/services/accountService.js | 225 +++ backend/src/services/accountService.test.js | 302 +++++ backend/src/services/connectionErrors.js | 38 + backend/src/services/connectionErrors.test.js | 63 + backend/src/services/connectionTest.js | 74 + backend/src/services/connectionTest.test.js | 159 +++ backend/src/services/draftService.js | 201 +++ backend/src/services/draftService.test.js | 282 ++++ backend/src/services/emailSanitizer.js | 5 + backend/src/services/emailSanitizer.test.js | 17 + backend/src/services/gtd/actions.js | 182 +++ backend/src/services/gtd/actions.test.js | 146 ++ backend/src/services/imapManager.js | 78 +- backend/src/services/imapManager.test.js | 49 + backend/src/services/inboxRules.js | 44 +- backend/src/services/mail/addresses.js | 51 + backend/src/services/mail/addresses.test.js | 69 + backend/src/services/mail/identity.js | 52 + backend/src/services/mail/identity.test.js | 91 ++ backend/src/services/mail/mimeBuilder.js | 84 ++ backend/src/services/mail/mimeBuilder.test.js | 103 ++ backend/src/services/mail/sentCopy.js | 176 +++ backend/src/services/mail/sentCopy.test.js | 158 +++ backend/src/services/mail/smtp.js | 90 ++ backend/src/services/mail/smtp.test.js | 122 ++ backend/src/services/mailbox/archive.js | 207 +++ backend/src/services/mailbox/archive.test.js | 247 ++++ backend/src/services/mailbox/batch.js | 11 + backend/src/services/mailbox/batch.test.js | 47 + backend/src/services/mailbox/category.js | 22 + backend/src/services/mailbox/category.test.js | 42 + backend/src/services/mailbox/flags.js | 183 +++ backend/src/services/mailbox/flags.test.js | 157 +++ backend/src/services/mailbox/folders.js | 102 ++ backend/src/services/mailbox/folders.test.js | 136 ++ backend/src/services/mailbox/move.js | 172 +++ backend/src/services/mailbox/move.test.js | 214 +++ backend/src/services/mailbox/snooze.js | 298 ++++ .../services/mailbox/snooze.restore.test.js | 122 ++ backend/src/services/mailbox/snooze.test.js | 269 ++++ backend/src/services/mailbox/spamLabel.js | 177 +++ .../src/services/mailbox/spamLabel.test.js | 114 ++ backend/src/services/mailbox/trash.js | 280 ++++ backend/src/services/mailbox/trash.test.js | 230 ++++ backend/src/services/outboxService.js | 216 +++ backend/src/services/outboxService.test.js | 295 ++++ backend/src/services/outboxWorker.test.js | 315 +++++ backend/src/services/replyService.js | 396 ++++++ backend/src/services/replyService.test.js | 760 +++++++++++ backend/src/services/sendService.js | 401 ++++++ backend/src/services/sendService.test.js | 459 +++++++ backend/src/utils/validation.js | 11 + backend/src/utils/validation.test.js | 29 + frontend/src/components/AdminPanel.jsx | 37 +- frontend/src/components/ComposeModal.jsx | 125 +- frontend/src/components/MailApp.jsx | 29 + frontend/src/components/MessageList.jsx | 30 +- .../src/components/NotificationToasts.jsx | 43 +- frontend/src/components/ProfileModal.jsx | 63 +- frontend/src/locales/de.json | 22 +- frontend/src/locales/en.json | 22 +- frontend/src/locales/es.json | 22 +- frontend/src/locales/fr.json | 22 +- frontend/src/locales/i18n.test.js | 25 + frontend/src/locales/it.json | 22 +- frontend/src/locales/ru.json | 22 +- frontend/src/locales/zhCN.json | 22 +- frontend/src/store/index.js | 12 + frontend/src/utils/api.js | 12 +- frontend/src/utils/api.outbox.test.js | 69 + frontend/src/utils/composeFromMessage.js | 31 + frontend/src/utils/composeFromMessage.test.js | 42 +- frontend/src/utils/outboxNotifications.js | 56 + .../src/utils/outboxNotifications.test.js | 107 ++ 136 files changed, 23384 insertions(+), 2522 deletions(-) create mode 100644 backend/migrations/0042_api_token_scopes.sql create mode 100644 backend/migrations/0043_outbox_messages.sql create mode 100644 backend/migrations/0044_message_triage.sql create mode 100644 backend/migrations/0045_mcp_account_stages.sql create mode 100644 backend/migrations/0046_message_triage_feed_index.sql create mode 100644 backend/src/mcp/accountAdapter.js create mode 100644 backend/src/mcp/accountAdapter.phase4.test.js create mode 100644 backend/src/mcp/accountAdapter.test.js create mode 100644 backend/src/mcp/accountTools.js create mode 100644 backend/src/mcp/accountTools.test.js create mode 100644 backend/src/mcp/composeTools.js create mode 100644 backend/src/mcp/composeTools.test.js create mode 100644 backend/src/mcp/draftTools.js create mode 100644 backend/src/mcp/draftTools.test.js create mode 100644 backend/src/mcp/mailboxTools.js create mode 100644 backend/src/mcp/mailboxTools.test.js create mode 100644 backend/src/mcp/messageTriage.integration.test.js create mode 100644 backend/src/mcp/sendTools.js create mode 100644 backend/src/mcp/sendTools.test.js create mode 100644 backend/src/mcp/tools.test.js create mode 100644 backend/src/mcp/triageAdapter.integration.test.js create mode 100644 backend/src/mcp/triageAdapter.js create mode 100644 backend/src/mcp/triageAdapter.test.js create mode 100644 backend/src/mcp/triageProbes.js create mode 100644 backend/src/mcp/triageProbes.test.js create mode 100644 backend/src/mcp/triageTools.js create mode 100644 backend/src/mcp/triageTools.test.js create mode 100644 backend/src/mcp/writeResult.js create mode 100644 backend/src/mcp/writeResult.test.js create mode 100644 backend/src/routes/accounts.test.js create mode 100644 backend/src/routes/auth.preferences.test.js create mode 100644 backend/src/routes/mail.test.js create mode 100644 backend/src/routes/mcpAccountStages.js create mode 100644 backend/src/routes/mcpAccountStages.test.js create mode 100644 backend/src/routes/send.outbox.test.js create mode 100644 backend/src/routes/send.test.js create mode 100644 backend/src/services/accountFields.js create mode 100644 backend/src/services/accountFields.test.js create mode 100644 backend/src/services/accountService.js create mode 100644 backend/src/services/accountService.test.js create mode 100644 backend/src/services/connectionErrors.js create mode 100644 backend/src/services/connectionErrors.test.js create mode 100644 backend/src/services/connectionTest.js create mode 100644 backend/src/services/connectionTest.test.js create mode 100644 backend/src/services/draftService.js create mode 100644 backend/src/services/draftService.test.js create mode 100644 backend/src/services/gtd/actions.js create mode 100644 backend/src/services/gtd/actions.test.js create mode 100644 backend/src/services/mail/addresses.js create mode 100644 backend/src/services/mail/addresses.test.js create mode 100644 backend/src/services/mail/identity.js create mode 100644 backend/src/services/mail/identity.test.js create mode 100644 backend/src/services/mail/mimeBuilder.js create mode 100644 backend/src/services/mail/mimeBuilder.test.js create mode 100644 backend/src/services/mail/sentCopy.js create mode 100644 backend/src/services/mail/sentCopy.test.js create mode 100644 backend/src/services/mail/smtp.js create mode 100644 backend/src/services/mail/smtp.test.js create mode 100644 backend/src/services/mailbox/archive.js create mode 100644 backend/src/services/mailbox/archive.test.js create mode 100644 backend/src/services/mailbox/batch.js create mode 100644 backend/src/services/mailbox/batch.test.js create mode 100644 backend/src/services/mailbox/category.js create mode 100644 backend/src/services/mailbox/category.test.js create mode 100644 backend/src/services/mailbox/flags.js create mode 100644 backend/src/services/mailbox/flags.test.js create mode 100644 backend/src/services/mailbox/folders.js create mode 100644 backend/src/services/mailbox/folders.test.js create mode 100644 backend/src/services/mailbox/move.js create mode 100644 backend/src/services/mailbox/move.test.js create mode 100644 backend/src/services/mailbox/snooze.js create mode 100644 backend/src/services/mailbox/snooze.restore.test.js create mode 100644 backend/src/services/mailbox/snooze.test.js create mode 100644 backend/src/services/mailbox/spamLabel.js create mode 100644 backend/src/services/mailbox/spamLabel.test.js create mode 100644 backend/src/services/mailbox/trash.js create mode 100644 backend/src/services/mailbox/trash.test.js create mode 100644 backend/src/services/outboxService.js create mode 100644 backend/src/services/outboxService.test.js create mode 100644 backend/src/services/outboxWorker.test.js create mode 100644 backend/src/services/replyService.js create mode 100644 backend/src/services/replyService.test.js create mode 100644 backend/src/services/sendService.js create mode 100644 backend/src/services/sendService.test.js create mode 100644 backend/src/utils/validation.js create mode 100644 backend/src/utils/validation.test.js create mode 100644 frontend/src/utils/api.outbox.test.js create mode 100644 frontend/src/utils/outboxNotifications.js create mode 100644 frontend/src/utils/outboxNotifications.test.js diff --git a/README.md b/README.md index 4188a1bd..60d89704 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,172 @@ configurable per account, and accounts with GTD off behave exactly as before. --- +## MCP server + +MailFlow exposes a Model Context Protocol endpoint at `POST /mcp`. Authenticate +with an API token in the `Authorization: Bearer ` header. Tokens have one or +more scopes: + +- `read` — search, inspect messages, and list mailbox state +- `write` — change mailbox state; also implies `read` +- `send` — send, queue, cancel, or recall mail; also implies `read` +- `settings` — manage accounts, aliases, and rules; also implies `read` + +`stage_deletion` requires `write`, even though execution is a separate, +session-authorized step. `run_rules` requires both `settings` and `write` because +rule actions can move, archive, or delete messages. + +### Tools + +#### Search and read + +| Tool | Scope | Description | +|---|---|---| +| `ping` | `read` | Check transport and authentication health. | +| `search_metadata` | `read` | Search message metadata with the supported Gmail-style query subset. | +| `search_message_bodies` | `read` | Run keyword full-text search over message bodies. | +| `semantic_search_messages` | `read` | Run vector or hybrid semantic search over indexed messages. | +| `get_message` | `read` | Get message details and a pageable body slice. | +| `list_messages` | `read` | List messages with account, participant, date, attachment, and conversation filters. | +| `get_stats` | `read` | Get archive totals, attachment statistics, and accounts. | +| `aggregate` | `read` | Group message statistics by sender, recipient, domain, label, or year. | +| `search_by_domains` | `read` | Find messages involving any of a set of domains. | +| `find_similar_messages` | `read` | Find messages closest to a seed message in the vector index. | +| `search_in_message` | `read` | Find keyword or semantic matches inside one message. | + +#### Drafts + +| Tool | Scope | Description | +|---|---|---| +| `create_draft` | `write` | Create a draft without sending it. | +| `update_draft` | `write` | Replace a draft and return its new IMAP UID. | +| `list_drafts` | `read` | List drafts newest-first. | +| `get_draft` | `read` | Get draft bodies, threading headers, and attachment metadata. | +| `delete_draft` | `write` | Permanently delete a draft from IMAP. | + +#### Send, outbox, and undo-send + +| Tool | Scope | Description | +|---|---|---| +| `send_email` | `send` | Send immediately or queue within an undo-send window. | +| `send_draft` | `send` | Send a stored draft and delete it only after delivery or enqueue succeeds. | +| `reply_email` | `send` | Reply to a message sender. | +| `reply_all_email` | `send` | Reply to the sender and stored To/Cc recipients. | +| `forward_email` | `send` | Forward a message, including server-side attachment references. | +| `unsend_email` | `send` | Cancel a queued message before its `send_at` time. | +| `list_outbox` | `read` | List messages still waiting in the undo-send outbox. | +| `recall_email` | `send` | Cancel a queued send, or clean up a Sent copy and prepare a follow-up draft. | + +#### Mailbox and folders + +| Tool | Scope | Description | +|---|---|---| +| `list_folders` | `read` | List scoped folders and live message counts. | +| `create_folder` | `write` | Create a folder in one account. | +| `rename_folder` | `write` | Rename the final component of a folder path. | +| `delete_folder` | `write` | Delete a folder after confirming its live message count. | +| `move_messages` | `write` | Move messages and return replacement IDs. | +| `archive_messages` | `write` | Archive messages and return replacement IDs. | +| `trash_messages` | `write` | Move messages to Trash without silently expunging existing Trash items. | +| `mark_read` / `mark_unread` | `write` | Change read state for explicit message IDs. | +| `star_message` / `unstar_message` | `write` | Change starred state for explicit message IDs. | +| `mark_spam` / `mark_not_spam` | `write` | Move a message into or out of the configured spam folder. | +| `snooze_message` / `unsnooze_message` | `write` | Snooze or restore a message and its reply-chain siblings. | +| `set_category` | `write` | Set the category of one message. | +| `gtd_classify` | `write` | Apply or remove one GTD state label. | +| `gtd_done` | `write` | Remove GTD labels and archive the Inbox copy. | +| `stage_deletion` | `write` | Stage a filtered deletion for separate session-authorized execution. | + +#### Triage + +| Tool | Scope | Description | +|---|---|---| +| `triage_inbox` | `read` | Page through untriaged Inbox messages with sender, thread, category, and optional semantic signals. | +| `get_triage_context` | `read` | Get thread, sender-history, similar-message, and matched-rule context. | +| `mark_triaged` | `write` | Checkpoint messages so later triage runs skip them. | + +#### Accounts and settings + +| Tool | Scope | Description | +|---|---|---| +| `list_accounts` | `read` | List connected accounts and aliases without credentials. | +| `add_account` | `settings` | Stage non-secret account configuration and return a `stage_id`. | +| `update_account_settings` | `settings` | Update non-secret account settings. | +| `test_account_connection` | `settings` | Test stored IMAP and SMTP credentials without sending mail. | +| `create_alias` / `update_alias` / `delete_alias` | `settings` | Manage send-as aliases on scoped accounts. | +| `list_rules` | `settings` | List global and account-specific inbox rules. | +| `create_rule` / `update_rule` / `delete_rule` | `settings` | Manage user-owned inbox rules. | +| `run_rules` | `settings` + `write` | Run enabled rules against current Inbox messages. | + +### Undo-send and recall + +`send_email` and `send_draft` accept an undo window from 0 to 120 seconds. A +zero-second window hands the message to SMTP immediately. A positive window puts +it in the outbox until `send_at`; use `list_outbox` to inspect queued messages and +`unsend_email` to cancel one before handoff. + +Recall is deliberately honest. `recall_email` can withdraw a message only while +it is still queued. SMTP cannot claw back mail already delivered; for an +already-sent message, recall can delete MailFlow's Sent copy and prepare a +"please disregard" follow-up draft, but it never pretends recipients lost their +copy and never sends the follow-up automatically. + +### Adding an account + +`add_account` accepts non-secret connection configuration only. It returns a +`stage_id`; a human then supplies fresh credentials in **Settings → Accounts**, +or through the session-authenticated +`POST /api/mcp-account-stages/:id/execute` endpoint. + +Passwords, OAuth access tokens, and OAuth refresh tokens never cross the MCP +bearer-token channel. + +### Agent cookbook + +A morning triage loop can page oldest-first and checkpoint every disposition: + +```text +07:00 triage_inbox { limit: 25, unread_only: true } + → items, cursor, has_more, counts.untriaged_unread + + Obvious newsletters/promotions: + archive_messages { message_ids: ["m1", "m2", ...] } + → archived: [{ id: "m1", new_id: "n1", ... }, ...] + mark_triaged { + message_ids: ["n1", "n2", ...], + action: "archived" + } + + Ambiguous message with a known sender and active thread: + get_triage_context { message_id: "m17" } + gtd_classify { message_id: "m17", state: "todo" } + star_message { message_ids: ["m17"] } + mark_triaged { + message_ids: ["m17"], + action: "flagged_todo", + note: "Needs a reply" + } + + Snooze a reply chain: + mark_triaged { message_ids: ["m22"], action: "snoozed" } + snooze_message { + message_id: "m22", + until: "2026-07-31T08:00:00Z" + } + → { ok: true, moved_count: 3, sibling_ids: [...], folder: "Snoozed" } + +07:04 triage_inbox { cursor: "", limit: 25 } + → repeat until has_more is false +``` + +**Mark before move:** call `mark_triaged` before archiving, moving, or snoozing, +or use each successful move/archive receipt's `new_id`. Message IDs change during +a move; a stale pre-move ID resolves to `skipped`. Re-running `triage_inbox` +without a cursor is safe: the checkpoint table, not the cursor, is what prevents +already-triaged messages from being offered again. + +--- + ## Screenshots diff --git a/backend/migrations/0042_api_token_scopes.sql b/backend/migrations/0042_api_token_scopes.sql new file mode 100644 index 00000000..6aca6d7d --- /dev/null +++ b/backend/migrations/0042_api_token_scopes.sql @@ -0,0 +1,13 @@ +-- Scope MCP bearer tokens. Existing tokens were minted when the MCP surface was +-- read-only, so 'read' is both the default and the safe upgrade backfill. +ALTER TABLE api_tokens + ADD COLUMN IF NOT EXISTS scopes TEXT[] NOT NULL DEFAULT ARRAY['read']::TEXT[]; + +-- Keep every storage writer constrained to the scopes the server can enforce. +ALTER TABLE api_tokens + DROP CONSTRAINT IF EXISTS api_tokens_scopes_valid; +ALTER TABLE api_tokens + ADD CONSTRAINT api_tokens_scopes_valid CHECK ( + scopes <@ ARRAY['read','write','send','settings']::TEXT[] + AND COALESCE(array_length(scopes, 1), 0) >= 1 + ); diff --git a/backend/migrations/0043_outbox_messages.sql b/backend/migrations/0043_outbox_messages.sql new file mode 100644 index 00000000..bcd9a1f0 --- /dev/null +++ b/backend/migrations/0043_outbox_messages.sql @@ -0,0 +1,30 @@ +-- Deferred sends. A row is a fully-resolved compose payload waiting out its undo +-- window; the worker claims it at send_at and hands it to sendService. Rows are +-- short-lived by construction (max window is 120s) — payload is NULLed on any +-- terminal status so delivered mail bodies do not accumulate at rest. +CREATE TABLE IF NOT EXISTS outbox_messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + payload JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','claimed','sent','cancelled','failed')), + send_at TIMESTAMPTZ NOT NULL, + claimed_at TIMESTAMPTZ, + attempts INT NOT NULL DEFAULT 0, + subject TEXT, + to_preview JSONB NOT NULL DEFAULT '[]', + message_id TEXT, + sent_message_id TEXT, + error TEXT, + idempotency_key TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_outbox_due + ON outbox_messages(send_at) WHERE status = 'pending'; +CREATE INDEX IF NOT EXISTS idx_outbox_user + ON outbox_messages(user_id, created_at DESC); +CREATE UNIQUE INDEX IF NOT EXISTS idx_outbox_idem + ON outbox_messages(user_id, idempotency_key) WHERE idempotency_key IS NOT NULL; diff --git a/backend/migrations/0044_message_triage.sql b/backend/migrations/0044_message_triage.sql new file mode 100644 index 00000000..0ba80ce5 --- /dev/null +++ b/backend/migrations/0044_message_triage.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS message_triage ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + message_id_header TEXT NOT NULL, + triaged_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + action TEXT, -- free-form: archived | replied | snoozed | left | … + note TEXT, + source TEXT NOT NULL DEFAULT 'mcp', + token_id UUID REFERENCES api_tokens(id) ON DELETE SET NULL, + UNIQUE (account_id, message_id_header) +); +CREATE INDEX idx_message_triage_user_time ON message_triage (user_id, triaged_at DESC); diff --git a/backend/migrations/0045_mcp_account_stages.sql b/backend/migrations/0045_mcp_account_stages.sql new file mode 100644 index 00000000..dbde0ae5 --- /dev/null +++ b/backend/migrations/0045_mcp_account_stages.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS mcp_account_stages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'staged', -- staged | completed | discarded + payload JSONB NOT NULL, -- name, sender_name, email_address, color, protocol, + -- imap_host, imap_port, imap_skip_tls_verify, + -- smtp_host, smtp_port, smtp_tls, auth_user, signature + -- NEVER auth_pass / oauth_* tokens + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_account_id UUID REFERENCES email_accounts(id) ON DELETE SET NULL +); +CREATE INDEX IF NOT EXISTS idx_mcp_account_stages_user_id ON mcp_account_stages(user_id); diff --git a/backend/migrations/0046_message_triage_feed_index.sql b/backend/migrations/0046_message_triage_feed_index.sql new file mode 100644 index 00000000..9273f867 --- /dev/null +++ b/backend/migrations/0046_message_triage_feed_index.sql @@ -0,0 +1,12 @@ +-- no-transaction +-- +-- This partial index supports the oldest-first inbox triage feed on the existing +-- messages table. CONCURRENTLY avoids blocking mailbox writes while it is built. +-- +-- DROP before CREATE makes retries safe after a cancelled or crashed concurrent +-- build: PostgreSQL can leave an invalid index with the target name, which plain +-- IF NOT EXISTS would otherwise skip and then record as successfully migrated. +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_triage_feed; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_triage_feed + ON messages (account_id, date, message_id) + WHERE folder = 'INBOX' AND is_deleted = false; diff --git a/backend/src/index.js b/backend/src/index.js index 98b51b7f..d86bc1aa 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -11,7 +11,7 @@ import { redisClient } from './services/redis.js'; import sendRoutes from './routes/send.js'; import draftRoutes from './routes/draft.js'; -import oauthRoutes from './routes/oauth.js'; +import oauthRoutes, { refreshMicrosoftToken } from './routes/oauth.js'; import integrationsRoutes, { loadIntegrationConfigs } from './routes/integrations.js'; import authRoutes from './routes/auth.js'; import accountRoutes from './routes/accounts.js'; @@ -31,9 +31,14 @@ import categoriesRoutes from './routes/categories.js'; import gtdRoutes from './routes/gtd.js'; import carddavRouter from './routes/carddav.js'; import carddavAccountRouter from './routes/carddavAccount.js'; -import { mountMcp } from './mcp/server.js'; +import { + entityTooLargeResponse, + mcpBodyLimit, + mountMcp, +} from './mcp/server.js'; import apiTokensRoutes from './routes/apiTokens.js'; import mcpDeletionsRoutes from './routes/mcpDeletions.js'; +import mcpAccountStagesRoutes from './routes/mcpAccountStages.js'; import { startCardavScheduler } from './services/carddavSync.js'; import { scheduleFtsBackfill } from './services/search/ftsBackfill.js'; import { encryptExistingCredentials, query } from './services/db.js'; @@ -45,6 +50,9 @@ import { reloadAuthSettings } from './services/authLimiter.js'; import { setupWebSocket } from './services/websocket.js'; import { ImapManager } from './services/imapManager.js'; import { getUpdateStatus } from './services/updateCheck.js'; +import * as sendService from './services/sendService.js'; +import * as outboxService from './services/outboxService.js'; +import * as draftService from './services/draftService.js'; const packageMeta = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8')); let buildMeta = {}; @@ -120,15 +128,15 @@ app.use((req, res, next) => { // 25 MB attachment limit → ~34 MB base64 on the wire; add headroom for the rest of the payload. app.use('/api/mail/send', express.json({ limit: '35mb' })); app.use('/api/mail/draft', express.json({ limit: '35mb' })); +app.use('/mcp', mcpBodyLimit()); // A pet-import body carries a base64 spritesheet (~33% larger than the 5 MB sheet cap // enforced after decode in gtdPet.importPet), so it needs more than the global 1 MB. app.use('/api/gtd/pet/import', express.json({ limit: '8mb' })); app.use(express.json({ limit: '1mb' })); // Return a clean JSON error when the body parser rejects an oversized payload. app.use((err, req, res, next) => { - if (err.type === 'entity.too.large') { - return res.status(413).json({ error: 'Request too large. Total attachment size must not exceed 25 MB.' }); - } + const response = entityTooLargeResponse(err, req.path); + if (response) return res.status(response.status).json(response.body); next(err); }); app.use(sessionMiddleware); @@ -186,6 +194,7 @@ app.use('/api', aiRoutes); app.use('/api', aiEmbeddingsRoutes); app.use('/api/tokens', apiTokensRoutes); app.use('/api/mcp-deletions', mcpDeletionsRoutes); +app.use('/api/mcp-account-stages', mcpAccountStagesRoutes); app.use('/api', categoriesRoutes); // Mounted at the /api/gtd subtree (not bare /api) so gtd.js's router-level // requireAuth cannot intercept the unauthenticated /api/health and /api/version @@ -199,7 +208,17 @@ app.all('/.well-known/carddav', (req, res) => res.redirect(308, '/carddav/')); // MCP Streamable-HTTP endpoint. Bearer-authenticated (mcp/auth.js), intentionally // outside the /api CSRF gate and session middleware — auth is a token, not a cookie. -mountMcp(app); +mountMcp(app, { + imapManager, + refreshMicrosoftToken, + redisClient, + sendService, + outboxService, + draftService, + // outboxService reads the DB through its deps (query / withTransaction) rather + // than importing db.js, so the MCP mount must supply them like the REST routes do. + query, +}); app.get('/api/health', (req, res) => res.json({ status: 'ok' })); app.get('/api/version', (_req, res) => res.json({ version: APP_VERSION, sha: process.env.BUILD_SHA || 'dev' })); @@ -268,6 +287,10 @@ startEmbeddingScheduler(); // Start background snooze watcher — polls every 60 seconds to restore snoozed messages imapManager.startSnoozeWatcher(); +outboxService.startOutboxWorker( + { imapManager, refreshMicrosoftToken, redisClient, query }, + { tickMs: 5000 }, +); // Schedule periodic CardDAV contact sync for any connected accounts. startCardavScheduler(); diff --git a/backend/src/mcp/accountAdapter.js b/backend/src/mcp/accountAdapter.js new file mode 100644 index 00000000..870fa4bd --- /dev/null +++ b/backend/src/mcp/accountAdapter.js @@ -0,0 +1,493 @@ +// Account/alias/compose/outbox SQL seam for MCP write tools. Callers pass account +// ids or a user id already resolved from the bearer token; every lookup keeps +// that boundary in SQL. +import { query } from '../services/db.js'; +import { safeAccount, SAFE_FIELDS } from '../services/accountFields.js'; +import { sanitizeSignature } from '../services/emailSanitizer.js'; +import { + DEFAULT_GTD_FOLDERS, + findGtdFolderCollisions, + invalidateGtdConfigCache, + sanitizeGtdFoldersDetailed, +} from '../services/gtdConfig.js'; +import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; +import { applyInboxRules, toRuleMessage } from '../services/inboxRules.js'; +import { DETAIL_COLUMNS } from './engineAdapter.js'; + +const COMPOSE_SOURCE_COLUMNS = DETAIL_COLUMNS + + ', m.uid, m.reply_to, m.in_reply_to, m.thread_references'; + +export async function getAccountRow(accountId, accountIds) { + if (!accountIds?.includes(accountId)) return null; + const { rows } = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND id = ANY($2)', + [accountId, accountIds], + ); + return rows[0] || null; +} + +export async function getAccountByEmail(email, accountIds) { + if (!accountIds?.length) return { error: `account_not_found: ${email}` }; + const { rows } = await query( + 'SELECT * FROM email_accounts WHERE email_address = $1 AND id = ANY($2)', + [email, accountIds], + ); + return rows[0] || { error: `account_not_found: ${email}` }; +} + +export async function listAliases(accountId) { + const { rows } = await query( + 'SELECT * FROM account_aliases WHERE account_id = $1 ORDER BY created_at', + [accountId], + ); + return rows; +} + +export async function resolveAlias(accountId, aliasEmail) { + // Keep the same predicate as mail/identity.js intentionally: this adapter lets + // MCP tools preflight an email selector as row-or-null, while identity.js must + // still hard-fail aliasId/aliasEmail selectors for existing REST service callers. + const { rows } = await query( + 'SELECT * FROM account_aliases WHERE account_id = $1 AND LOWER(email) = LOWER($2) LIMIT 1', + [accountId, aliasEmail], + ); + return rows[0] || null; +} + +export async function getComposeSource(messageId, accountIds) { + if (!accountIds?.length) return null; + const { rows } = await query( + `SELECT ${COMPOSE_SOURCE_COLUMNS} + FROM messages m + WHERE m.id = $1 AND m.account_id = ANY($2)`, + [messageId, accountIds], + ); + return rows[0] || null; +} + +export async function getOutboxRowByMessageId(messageId, userId) { + const { rows } = await query( + `SELECT * + FROM outbox_messages + WHERE message_id = $1 AND user_id = $2 AND status = 'pending' + LIMIT 1`, + [messageId, userId], + ); + return rows[0] || null; +} + +export async function deleteMessageRow(accountId, uid, folder) { + const result = await query( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + [accountId, uid, folder], + ); + return result.rowCount; +} + +export async function listDraftRows(accountId, { limit, offset, folder } = {}) { + const params = [accountId]; + const where = ['account_id = $1', 'is_deleted = false']; + if (folder) { + params.push(folder); + where.push(`folder = $${params.length}`); + } + params.push(limit, offset); + const limitParam = `$${params.length - 1}`; + const offsetParam = `$${params.length}`; + const { rows } = await query( + `SELECT * FROM messages + WHERE ${where.join(' AND ')} + ORDER BY date DESC NULLS LAST + LIMIT ${limitParam} OFFSET ${offsetParam}`, + params, + ); + return rows; +} + +export async function getDraftRow(accountId, folder, uid) { + const { rows } = await query( + `SELECT * FROM messages + WHERE account_id = $1 AND folder = $2 AND uid = $3 AND is_deleted = false + LIMIT 1`, + [accountId, folder, uid], + ); + return rows[0] || null; +} + +export async function getUserPreferences(userId) { + const { rows } = await query( + 'SELECT preferences FROM users WHERE id = $1', + [userId], + ); + return rows[0]?.preferences || {}; +} + +export async function listAccountsSafe(accountIds) { + if (!accountIds?.length) return []; + const { rows } = await query( + `SELECT ${SAFE_FIELDS.join(', ')} + FROM email_accounts + WHERE id = ANY($1) + ORDER BY sort_order, created_at`, + [accountIds], + ); + + const aliasesByAccount = new Map(); + if (rows.length) { + const aliases = await query( + `SELECT id, account_id, name, email, reply_to, signature, created_at + FROM account_aliases WHERE account_id = ANY($1) ORDER BY created_at`, + [rows.map((account) => account.id)], + ); + for (const alias of aliases.rows) { + if (!aliasesByAccount.has(alias.account_id)) aliasesByAccount.set(alias.account_id, []); + aliasesByAccount.get(alias.account_id).push({ + ...alias, + signature: alias.signature ? sanitizeSignature(alias.signature) : alias.signature, + }); + } + } + + return rows.map((row) => ({ + ...safeAccount(row), + aliases: aliasesByAccount.get(row.id) || [], + })); +} + +const MCP_ACCOUNT_UPDATE_FIELDS = [ + 'name', + 'sender_name', + 'color', + 'sort_order', + 'folder_mappings', + 'signature', + 'categorization_enabled', + 'gtd_enabled', + 'gtd_folders', + 'enabled', +]; + +export async function updateAccountSettings({ accountId, accountIds, updates }) { + if (!accountIds?.includes(accountId)) return { error: 'Account not found', status: 404 }; + const check = await query( + 'SELECT id, gtd_folders FROM email_accounts WHERE id = $1 AND id = ANY($2)', + [accountId, accountIds], + ); + if (!check.rows.length) return { error: 'Account not found', status: 404 }; + + let gtdFoldersValue; + let gtdRejected = []; + let gtdFoldersChanged = false; + if ('gtd_folders' in updates) { + const { folders, rejected, reserved } = sanitizeGtdFoldersDetailed(updates.gtd_folders); + if (reserved.length) { + return { + error: 'A GTD state cannot map to a reserved system folder', + reserved, + status: 400, + }; + } + const collisions = findGtdFolderCollisions({ + ...DEFAULT_GTD_FOLDERS, + ...folders, + }); + if (collisions.length) { + return { + error: 'Two GTD states cannot map to the same folder', + collisions, + status: 400, + }; + } + gtdFoldersValue = folders; + gtdRejected = rejected; + const before = sanitizeGtdFoldersDetailed(check.rows[0].gtd_folders).folders; + gtdFoldersChanged = JSON.stringify(before) !== JSON.stringify(folders); + } + + const sets = []; + const values = []; + for (const key of MCP_ACCOUNT_UPDATE_FIELDS) { + if (!(key in updates)) continue; + sets.push(`${key} = $${values.length + 1}`); + const value = key === 'signature' + ? sanitizeSignature(updates[key]) || null + : key === 'gtd_enabled' + ? !!updates[key] + : key === 'gtd_folders' + ? gtdFoldersValue + : updates[key]; + values.push(value); + } + if (!sets.length) return { error: 'No valid fields to update', status: 400 }; + + const idParam = values.length + 1; + const scopeParam = values.length + 2; + values.push(accountId, accountIds); + const result = await query( + `UPDATE email_accounts SET ${sets.join(', ')} + WHERE id = $${idParam} AND id = ANY($${scopeParam}) RETURNING *`, + values, + ); + if (!result.rows.length) return { error: 'Account not found', status: 404 }; + + const updated = result.rows[0]; + const account = safeAccount(updated); + if ('gtd_folders' in updates) account.gtd_folders_rejected = gtdRejected; + if ('gtd_enabled' in updates || 'gtd_folders' in updates) { + invalidateGtdConfigCache(accountId); + } + if ('enabled' in updates || 'gtd_enabled' in updates || 'gtd_folders' in updates) { + const { reconcileConnectionState } = await import('../services/accountService.js'); + reconcileConnectionState({ + id: accountId, + updates, + before: { gtdFoldersChanged }, + updated, + }); + } + return { account }; +} + +async function accountOwnedByUser(accountId, accountIds, userId) { + if (!accountIds?.includes(accountId)) return false; + const { rows } = await query( + 'SELECT id FROM email_accounts WHERE id = $1 AND user_id = $2 AND id = ANY($3)', + [accountId, userId, accountIds], + ); + return rows.length > 0; +} + +export async function createAlias({ + accountId, + accountIds, + userId, + fields, +}) { + if (!(await accountOwnedByUser(accountId, accountIds, userId))) return null; + const result = await query( + `INSERT INTO account_aliases (account_id, name, email, reply_to, signature) + VALUES ($1, $2, $3, $4, $5) RETURNING *`, + [ + accountId, + fields.name, + fields.email, + fields.reply_to || null, + sanitizeSignature(fields.signature) || null, + ], + ); + invalidateOwnerAddressesCache(accountId); + return result.rows[0]; +} + +async function ownedAlias(accountId, accountIds, userId, aliasId) { + if (!accountIds?.includes(accountId)) return null; + const { rows } = await query( + `SELECT a.id, a.account_id FROM account_aliases a + JOIN email_accounts e ON a.account_id = e.id + WHERE a.id = $1 AND e.user_id = $2 AND e.id = $3 AND e.id = ANY($4)`, + [aliasId, userId, accountId, accountIds], + ); + return rows[0] || null; +} + +export async function updateAlias({ + accountId, + accountIds, + userId, + aliasId, + fields, +}) { + const owned = await ownedAlias(accountId, accountIds, userId, aliasId); + if (!owned) return null; + const result = await query( + `UPDATE account_aliases + SET name = $1, email = $2, reply_to = $3, signature = $4 + WHERE id = $5 RETURNING *`, + [ + fields.name, + fields.email, + fields.reply_to || null, + sanitizeSignature(fields.signature) || null, + aliasId, + ], + ); + invalidateOwnerAddressesCache(accountId); + return result.rows[0] || null; +} + +export async function deleteAlias({ + accountId, + accountIds, + userId, + aliasId, +}) { + const owned = await ownedAlias(accountId, accountIds, userId, aliasId); + if (!owned) return false; + await query('DELETE FROM account_aliases WHERE id = $1', [aliasId]); + invalidateOwnerAddressesCache(accountId); + return true; +} + +export async function listRules({ userId, accountId }) { + const params = [userId]; + const accountFilter = accountId + ? ' AND (account_id IS NULL OR account_id = $2)' + : ''; + if (accountId) params.push(accountId); + const { rows } = await query( + `SELECT * FROM inbox_rules WHERE user_id = $1${accountFilter} + ORDER BY priority, created_at`, + params, + ); + return rows; +} + +async function moveDestinationError(accountId, actions) { + const moveAction = actions.find( + (action) => action.type === 'move' && action.value?.trim(), + ); + if (!moveAction || !accountId) return null; + const { rows } = await query( + `SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE path = $2) AS match + FROM folders WHERE account_id = $1`, + [accountId, moveAction.value.trim()], + ); + const { total, match } = rows[0]; + if (parseInt(total, 10) > 0 && parseInt(match, 10) === 0) { + return 'Move destination folder not found for this account'; + } + return null; +} + +export async function createRule({ + userId, + accountId, + name, + conditionLogic, + conditions, + actions, + enabled, + stopProcessing, +}) { + const folderError = await moveDestinationError(accountId, actions); + if (folderError) return { error: folderError, status: 400 }; + const countResult = await query( + 'SELECT COUNT(*) AS cnt FROM inbox_rules WHERE user_id = $1', + [userId], + ); + const priority = parseInt(countResult.rows[0].cnt, 10); + const result = await query( + `INSERT INTO inbox_rules + (user_id, account_id, name, enabled, stop_processing, priority, condition_logic, conditions, actions) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + RETURNING *`, + [ + userId, + accountId || null, + name || '', + enabled !== false, + !!stopProcessing, + priority, + conditionLogic === 'OR' ? 'OR' : 'AND', + JSON.stringify(conditions), + JSON.stringify(actions), + ], + ); + return result.rows[0]; +} + +export async function updateRule({ + userId, + ruleId, + accountId, + name, + conditionLogic, + conditions, + actions, + enabled, + stopProcessing, +}) { + const folderError = await moveDestinationError(accountId, actions); + if (folderError) return { error: folderError, status: 400 }; + const result = await query( + `UPDATE inbox_rules + SET name = $1, account_id = $2, enabled = $3, stop_processing = $4, + condition_logic = $5, conditions = $6, actions = $7, updated_at = NOW() + WHERE id = $8 AND user_id = $9 + RETURNING *`, + [ + name || '', + accountId || null, + enabled !== false, + !!stopProcessing, + conditionLogic === 'OR' ? 'OR' : 'AND', + JSON.stringify(conditions), + JSON.stringify(actions), + ruleId, + userId, + ], + ); + return result.rows[0] || null; +} + +export async function deleteRule({ userId, ruleId }) { + const result = await query( + 'DELETE FROM inbox_rules WHERE id = $1 AND user_id = $2 RETURNING id', + [ruleId, userId], + ); + return result.rows.length > 0; +} + +export async function runRules({ userId, accountIds, imapManager }) { + let processed = 0; + let matched = 0; + for (const accountId of accountIds || []) { + try { + const rulesCheck = await query( + `SELECT COUNT(*) AS cnt FROM inbox_rules + WHERE user_id = $1 AND enabled = true + AND (account_id IS NULL OR account_id = $2)`, + [userId, accountId], + ); + if (parseInt(rulesCheck.rows[0].cnt, 10) === 0) continue; + + const accountResult = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND id = ANY($2)', + [accountId, accountIds], + ); + const account = accountResult.rows[0]; + if (!account) continue; + + const batchSize = 500; + let lastId = null; + while (true) { + const messageResult = await query( + `SELECT id, uid, folder, from_email, from_name, to_addresses, + subject, has_attachments, is_read + FROM messages + WHERE account_id = $1 AND lower(folder) = 'inbox' + ${lastId ? 'AND id > $3' : ''} + ORDER BY id + LIMIT $2`, + lastId + ? [accountId, batchSize, lastId] + : [accountId, batchSize], + ); + if (!messageResult.rows.length) break; + lastId = messageResult.rows.at(-1).id; + const messages = messageResult.rows.map(toRuleMessage); + const { remaining } = await applyInboxRules( + messages, + account, + imapManager, + ); + processed += messages.length; + matched += messages.length - remaining.length; + if (messageResult.rows.length < batchSize) break; + } + } catch (error) { + console.error(`MCP run_rules error for account ${accountId}:`, error.message); + } + } + return { processed, matched }; +} diff --git a/backend/src/mcp/accountAdapter.phase4.test.js b/backend/src/mcp/accountAdapter.phase4.test.js new file mode 100644 index 00000000..360d421e --- /dev/null +++ b/backend/src/mcp/accountAdapter.phase4.test.js @@ -0,0 +1,338 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SAFE_FIELDS } from '../services/accountFields.js'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../services/accountService.js', () => ({ + reconcileConnectionState: vi.fn(), +})); +vi.mock('../services/gtdConfig.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + invalidateGtdConfigCache: vi.fn(), + }; +}); +vi.mock('../services/gtdTransitions.js', () => ({ + invalidateOwnerAddressesCache: vi.fn(), +})); +vi.mock('../services/inboxRules.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + applyInboxRules: vi.fn(), + toRuleMessage: vi.fn(actual.toRuleMessage), + }; +}); + +import { query } from '../services/db.js'; +import { reconcileConnectionState } from '../services/accountService.js'; +import { invalidateGtdConfigCache } from '../services/gtdConfig.js'; +import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; +import { applyInboxRules } from '../services/inboxRules.js'; +import * as accountAdapter from './accountAdapter.js'; + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const SECOND_ACCOUNT_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; +const accountIds = [ACCOUNT_ID, SECOND_ACCOUNT_ID]; + +function exported(name) { + expect(accountAdapter[name], `${name} must be exported`).toBeTypeOf('function'); + return accountAdapter[name]; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('listAccountsSafe', () => { + it('passes an exact SAFE_FIELDS SQL column list and attaches aliases in one query', async () => { + const account = Object.fromEntries(SAFE_FIELDS.map((field) => [field, null])); + account.id = ACCOUNT_ID; + account.email_address = 'owner@example.com'; + query + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ + rows: [{ + id: 'alias-1', + account_id: ACCOUNT_ID, + name: 'Alias', + email: 'alias@example.com', + signature: 'safe', + }], + }); + + const result = await exported('listAccountsSafe')([ACCOUNT_ID]); + + expect(result).toEqual([expect.objectContaining({ + id: ACCOUNT_ID, + email_address: 'owner@example.com', + aliases: [expect.objectContaining({ + id: 'alias-1', + signature: 'safe', + })], + })]); + const accountSql = query.mock.calls[0][0]; + const columns = accountSql + .slice(accountSql.indexOf('SELECT') + 6, accountSql.indexOf('FROM email_accounts')) + .split(',') + .map((column) => column.trim()) + .filter(Boolean); + expect(columns).toEqual(SAFE_FIELDS); + expect(accountSql).not.toMatch(/\bauth_pass\b|\boauth_access_token\b|\boauth_refresh_token\b/); + expect(query.mock.calls[1][0]).toMatch( + /SELECT id, account_id, name, email, reply_to, signature, created_at[\s\S]*account_id = ANY\(\$1\)/, + ); + expect(query.mock.calls[1][1]).toEqual([[ACCOUNT_ID]]); + }); + + it('does not query for an empty account scope', async () => { + await expect(exported('listAccountsSafe')([])).resolves.toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('updateAccountSettings', () => { + it('rejects reserved and colliding GTD folder mappings before UPDATE', async () => { + query.mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID, gtd_folders: {} }] }); + const reserved = await exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { gtd_folders: { todo: 'INBOX' } }, + }); + expect(reserved).toEqual({ + error: 'A GTD state cannot map to a reserved system folder', + reserved: ['todo'], + status: 400, + }); + expect(query).toHaveBeenCalledTimes(1); + + query.mockReset(); + query.mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID, gtd_folders: {} }] }); + const collision = await exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { gtd_folders: { todo: 'Work', watch: 'Work' } }, + }); + expect(collision.error).toBe('Two GTD states cannot map to the same folder'); + expect(collision.collisions).toEqual([{ folder: 'Work', states: ['todo', 'watch'] }]); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('returns a safe row, reports rejected folders, invalidates GTD, and reconciles once', async () => { + const updated = Object.fromEntries(SAFE_FIELDS.map((field) => [field, null])); + updated.id = ACCOUNT_ID; + updated.protocol = 'imap'; + updated.enabled = true; + updated.gtd_enabled = true; + query + .mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID, gtd_folders: {} }] }) + .mockResolvedValueOnce({ rows: [updated] }); + + const result = await exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { + name: 'Updated', + signature: 'safe', + gtd_enabled: true, + gtd_folders: { todo: 'Work', watch: '../bad' }, + }, + }); + + expect(result.account).toEqual(expect.objectContaining({ + id: ACCOUNT_ID, + signature: null, + gtd_folders_rejected: ['watch'], + })); + const [sql, values] = query.mock.calls[1]; + expect(sql).toMatch(/UPDATE email_accounts SET/); + expect(sql).toMatch(/WHERE id = \$\d+ AND id = ANY\(\$\d+\) RETURNING \*/); + expect(values).toContain('safe'); + expect(invalidateGtdConfigCache).toHaveBeenCalledTimes(1); + expect(invalidateGtdConfigCache).toHaveBeenCalledWith(ACCOUNT_ID); + expect(reconcileConnectionState).toHaveBeenCalledTimes(1); + expect(reconcileConnectionState).toHaveBeenCalledWith(expect.objectContaining({ + id: ACCOUNT_ID, + updates: expect.objectContaining({ gtd_enabled: true }), + before: { gtdFoldersChanged: true }, + updated, + })); + }); + + it('returns Account not found without updating when the scoped row is absent', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { name: 'Nope' }, + })).resolves.toEqual({ error: 'Account not found', status: 404 }); + expect(query).toHaveBeenCalledTimes(1); + expect(reconcileConnectionState).not.toHaveBeenCalled(); + }); +}); + +describe('alias mutations', () => { + const fields = { + name: 'Alias', + email: 'alias@example.com', + reply_to: 'reply@example.com', + signature: 'Sig', + }; + + it.each([ + ['createAlias', { fields }], + ['updateAlias', { aliasId: 'alias-1', fields }], + ['deleteAlias', { aliasId: 'alias-1' }], + ])('%s rejects ownership misses before mutation', async (name, extra) => { + query.mockResolvedValueOnce({ rows: [] }); + + const result = await exported(name)({ + accountId: ACCOUNT_ID, + accountIds, + userId: 'wrong-user', + ...extra, + }); + + expect(result === null || result === false).toBe(true); + expect(query).toHaveBeenCalledTimes(1); + expect(invalidateOwnerAddressesCache).not.toHaveBeenCalled(); + }); + + it.each([ + ['createAlias', { + extra: { fields }, + responses: [{ rows: [{ id: ACCOUNT_ID }] }, { rows: [{ id: 'alias-1' }] }], + expected: { id: 'alias-1' }, + }], + ['updateAlias', { + extra: { aliasId: 'alias-1', fields }, + responses: [ + { rows: [{ id: 'alias-1', account_id: ACCOUNT_ID }] }, + { rows: [{ id: 'alias-1' }] }, + ], + expected: { id: 'alias-1' }, + }], + ['deleteAlias', { + extra: { aliasId: 'alias-1' }, + responses: [ + { rows: [{ id: 'alias-1', account_id: ACCOUNT_ID }] }, + { rows: [], rowCount: 1 }, + ], + expected: true, + }], + ])('%s invalidates owner addresses exactly once after success', async (name, spec) => { + for (const response of spec.responses) query.mockResolvedValueOnce(response); + + await expect(exported(name)({ + accountId: ACCOUNT_ID, + accountIds, + userId: 'user-1', + ...spec.extra, + })).resolves.toEqual(spec.expected); + expect(invalidateOwnerAddressesCache).toHaveBeenCalledTimes(1); + expect(invalidateOwnerAddressesCache).toHaveBeenCalledWith(ACCOUNT_ID); + }); +}); + +describe('rule persistence', () => { + it('lists global and account-specific rules for an optional account filter', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'rule-1' }] }); + + await expect(exported('listRules')({ + userId: 'user-1', + accountId: ACCOUNT_ID, + })).resolves.toEqual([{ id: 'rule-1' }]); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/user_id = \$1 AND \(account_id IS NULL OR account_id = \$2\).*ORDER BY priority, created_at/s), + ['user-1', ACCOUNT_ID], + ); + }); + + it('creates a user-owned rule with REST defaults and priority', async () => { + query + .mockResolvedValueOnce({ rows: [{ cnt: '4' }] }) + .mockResolvedValueOnce({ rows: [{ id: 'rule-1' }] }); + + await expect(exported('createRule')({ + userId: 'user-1', + accountId: ACCOUNT_ID, + name: 'Rule', + conditionLogic: 'OR', + conditions: [{ field: 'from', value: 'example.com' }], + actions: [{ type: 'archive' }], + enabled: true, + stopProcessing: false, + })).resolves.toEqual({ id: 'rule-1' }); + expect(query.mock.calls[1][0]).toMatch(/INSERT INTO inbox_rules/); + expect(query.mock.calls[1][1]).toEqual([ + 'user-1', + ACCOUNT_ID, + 'Rule', + true, + false, + 4, + 'OR', + JSON.stringify([{ field: 'from', value: 'example.com' }]), + JSON.stringify([{ type: 'archive' }]), + ]); + }); + + it('updates and deletes only rules owned by the scoped user', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await expect(exported('updateRule')({ + userId: 'wrong-user', + ruleId: 'rule-1', + accountId: null, + name: 'Rule', + conditionLogic: 'AND', + conditions: [], + actions: [], + })).resolves.toBeNull(); + expect(query.mock.calls[0][0]).toMatch(/WHERE id = \$8 AND user_id = \$9/); + + query.mockReset(); + query.mockResolvedValueOnce({ rows: [] }); + await expect(exported('deleteRule')({ + userId: 'wrong-user', + ruleId: 'rule-1', + })).resolves.toBe(false); + expect(query).toHaveBeenCalledWith( + 'DELETE FROM inbox_rules WHERE id = $1 AND user_id = $2 RETURNING id', + ['rule-1', 'wrong-user'], + ); + }); +}); + +describe('runRules', () => { + it('skips accounts with no enabled rules and batches inbox messages for the rest', async () => { + const imapManager = { marker: 'injected' }; + const account = { id: SECOND_ACCOUNT_ID, email_address: 'second@example.com' }; + query + .mockResolvedValueOnce({ rows: [{ cnt: '0' }] }) + .mockResolvedValueOnce({ rows: [{ cnt: '1' }] }) + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ + rows: [ + { id: 'm1', uid: 1, folder: 'INBOX', subject: 'One' }, + { id: 'm2', uid: 2, folder: 'INBOX', subject: 'Two' }, + ], + }); + applyInboxRules.mockResolvedValue({ remaining: [{ id: 'm2' }] }); + + const result = await exported('runRules')({ + userId: 'user-1', + accountIds, + imapManager, + }); + + expect(result).toEqual({ processed: 2, matched: 1 }); + expect(applyInboxRules).toHaveBeenCalledTimes(1); + expect(applyInboxRules.mock.calls[0][1]).toEqual(account); + expect(applyInboxRules.mock.calls[0][2]).toBe(imapManager); + const messageSql = query.mock.calls[3][0]; + expect(messageSql).toMatch(/lower\(folder\) = 'inbox'/); + expect(messageSql).toMatch(/ORDER BY id[\s\S]*LIMIT \$2/); + expect(query.mock.calls[3][1]).toEqual([SECOND_ACCOUNT_ID, 500]); + }); +}); diff --git a/backend/src/mcp/accountAdapter.test.js b/backend/src/mcp/accountAdapter.test.js new file mode 100644 index 00000000..59cd9da8 --- /dev/null +++ b/backend/src/mcp/accountAdapter.test.js @@ -0,0 +1,257 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); + +import { query } from '../services/db.js'; +import { + deleteMessageRow, + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + getOutboxRowByMessageId, + getUserPreferences, + listAliases, + listDraftRows, + resolveAlias, +} from './accountAdapter.js'; + +describe('outbox row lookups', () => { + beforeEach(() => query.mockReset()); + + it('finds a pending outbox row by message id within the caller user scope', async () => { + const row = { + id: 'outbox-1', + user_id: 'user-1', + message_id: '', + status: 'pending', + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect( + getOutboxRowByMessageId('', 'user-1'), + ).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/message_id = \$1/); + expect(sql).toMatch(/user_id = \$2/); + expect(sql).toMatch(/status = 'pending'/); + expect(params).toEqual(['', 'user-1']); + }); + + it('returns null when no pending row matches that message and user', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect( + getOutboxRowByMessageId('', 'user-1'), + ).resolves.toBeNull(); + }); +}); + +describe('deleteMessageRow', () => { + beforeEach(() => query.mockReset()); + + it('uses the same account/uid/folder delete tuple as draftService', async () => { + query.mockResolvedValueOnce({ rowCount: 1, rows: [] }); + + await expect( + deleteMessageRow('account-1', 42, 'Sent'), + ).resolves.toBe(1); + expect(query).toHaveBeenCalledWith( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + ['account-1', 42, 'Sent'], + ); + }); +}); + +describe('getAccountRow', () => { + beforeEach(() => query.mockReset()); + + it('returns a full account row only when its id is in the caller account scope', async () => { + const row = { + id: 'acc-1', + email_address: 'owner@example.com', + smtp_host: 'smtp.example.com', + auth_pass: 'encrypted-secret', + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getAccountRow('acc-1', ['acc-1', 'acc-2'])).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/SELECT \* FROM email_accounts/); + expect(sql).toMatch(/id = \$1 AND id = ANY\(\$2\)/); + expect(params).toEqual(['acc-1', ['acc-1', 'acc-2']]); + }); + + it('returns null for an out-of-scope account without querying', async () => { + await expect(getAccountRow('acc-foreign', ['acc-1'])).resolves.toBeNull(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('getAccountByEmail', () => { + beforeEach(() => query.mockReset()); + + it('returns the account row matched by email within the caller account scope', async () => { + const row = { id: 'acc-2', email_address: 'owner@example.com', auth_pass: 'secret' }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getAccountByEmail('owner@example.com', ['acc-1', 'acc-2'])).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/email_address = \$1/); + expect(sql).toMatch(/id = ANY\(\$2\)/); + expect(params).toEqual(['owner@example.com', ['acc-1', 'acc-2']]); + }); + + it('returns account_not_found on a scoped lookup miss', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getAccountByEmail('missing@example.com', ['acc-1'])).resolves.toEqual({ + error: 'account_not_found: missing@example.com', + }); + }); + + it('returns account_not_found for an empty account scope without querying', async () => { + await expect(getAccountByEmail('owner@example.com', [])).resolves.toEqual({ + error: 'account_not_found: owner@example.com', + }); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('alias lookups', () => { + beforeEach(() => query.mockReset()); + + it('lists aliases only for the supplied scoped account id', async () => { + const rows = [{ id: 'alias-1', account_id: 'acc-1', email: 'alias@example.com' }]; + query.mockResolvedValueOnce({ rows }); + + await expect(listAliases('acc-1')).resolves.toBe(rows); + expect(query.mock.calls[0][0]).toMatch(/account_id = \$1/); + expect(query.mock.calls[0][1]).toEqual(['acc-1']); + }); + + it('resolves an alias email only within the supplied scoped account id', async () => { + const alias = { id: 'alias-1', account_id: 'acc-1', email: 'Alias@Example.com' }; + query.mockResolvedValueOnce({ rows: [alias] }); + + await expect(resolveAlias('acc-1', 'alias@example.com')).resolves.toBe(alias); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1/); + expect(sql).toMatch(/LOWER\(email\) = LOWER\(\$2\)/); + expect(params).toEqual(['acc-1', 'alias@example.com']); + }); + + it('returns null when the alias email is not configured on that account', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(resolveAlias('acc-1', 'missing@example.com')).resolves.toBeNull(); + }); +}); + +describe('getComposeSource', () => { + beforeEach(() => query.mockReset()); + + it('returns a compose source selected through the caller account scope', async () => { + const row = { + id: 'message-1', + account_id: 'acc-1', + uid: 42, + folder: 'INBOX', + message_id: '', + reply_to: [{ email: 'reply@example.com' }], + in_reply_to: '', + thread_references: ' ', + body_text: 'Hello', + body_html: '

Hello

', + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getComposeSource('message-1', ['acc-1'])).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/m\.account_id = ANY\(\$2\)/); + for (const column of ['uid', 'reply_to', 'in_reply_to', 'thread_references', 'body_text', 'body_html']) { + expect(sql).toContain(`m.${column}`); + } + expect(params).toEqual(['message-1', ['acc-1']]); + }); + + it('returns null when the message is outside the caller account scope', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getComposeSource('message-foreign', ['acc-1'])).resolves.toBeNull(); + }); + + it('returns null for an empty account scope without querying', async () => { + await expect(getComposeSource('message-1', [])).resolves.toBeNull(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('draft row lookups', () => { + beforeEach(() => query.mockReset()); + + it('lists live draft rows only for the supplied scoped account and folder', async () => { + const rows = [{ account_id: 'acc-1', folder: 'Drafts', uid: 7 }]; + query.mockResolvedValueOnce({ rows }); + + await expect(listDraftRows('acc-1', { + limit: 20, + offset: 5, + folder: 'Drafts', + })).resolves.toBe(rows); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1/); + expect(sql).toMatch(/folder = \$2/); + expect(sql).toMatch(/is_deleted = false/); + expect(sql).toMatch(/LIMIT \$3 OFFSET \$4/); + expect(params).toEqual(['acc-1', 'Drafts', 20, 5]); + }); + + it('lists live draft rows for a scoped account without widening through a folder predicate', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await listDraftRows('acc-1', { limit: 10, offset: 0 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1/); + expect(sql).not.toMatch(/folder = \$2/); + expect(params).toEqual(['acc-1', 10, 0]); + }); + + it('gets a draft row by scoped account, folder, and uid', async () => { + const row = { account_id: 'acc-1', folder: 'Drafts', uid: 7 }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getDraftRow('acc-1', 'Drafts', 7)).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1 AND folder = \$2 AND uid = \$3/); + expect(sql).toMatch(/is_deleted = false/); + expect(params).toEqual(['acc-1', 'Drafts', 7]); + }); + + it('returns null when a draft lookup misses its scoped account/folder/uid tuple', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getDraftRow('acc-1', 'Drafts', 999)).resolves.toBeNull(); + }); +}); + +describe('getUserPreferences', () => { + beforeEach(() => query.mockReset()); + + it('returns preferences only for the supplied user id', async () => { + const preferences = { plaintextEmail: true, undoSendSeconds: 30 }; + query.mockResolvedValueOnce({ rows: [{ preferences }] }); + + await expect(getUserPreferences('user-1')).resolves.toBe(preferences); + expect(query.mock.calls[0]).toEqual([ + 'SELECT preferences FROM users WHERE id = $1', + ['user-1'], + ]); + }); + + it('returns an empty object when the scoped user row is absent', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getUserPreferences('user-missing')).resolves.toEqual({}); + }); +}); diff --git a/backend/src/mcp/accountTools.js b/backend/src/mcp/accountTools.js new file mode 100644 index 00000000..13a4a988 --- /dev/null +++ b/backend/src/mcp/accountTools.js @@ -0,0 +1,501 @@ +import { + createAlias, + createRule, + deleteAlias, + deleteRule, + getAccountRow, + listAccountsSafe, + listRules, + runRules, + updateAccountSettings, + updateAlias, + updateRule, +} from './accountAdapter.js'; +import { resolveAccountScope } from './engineAdapter.js'; +import { errorResult, jsonResult } from './result.js'; +import { + hasHeaderInjectionChars, +} from '../services/emailSanitizer.js'; +import { + normalizeActions, + validateConditions, +} from '../routes/rules.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); +const CREATE_ANNOTATIONS = annotations(); +const IDEMPOTENT_WRITE_ANNOTATIONS = annotations({ idempotentHint: true }); +const DESTRUCTIVE_ANNOTATIONS = annotations({ destructiveHint: true }); +const DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS = annotations({ + destructiveHint: true, + idempotentHint: true, +}); + +export const listAccountsDef = { + name: 'list_accounts', + description: 'List the connected email accounts and their aliases. Never returns credentials.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { type: 'object', properties: {} }, +}; + +export const addAccountDef = { + name: 'add_account', + description: + 'Stage a new email account for setup. Does NOT accept passwords or OAuth tokens — the user must complete authentication in the Mailflow settings UI. Returns a stage_id.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['name', 'email_address'], + properties: { + name: { type: 'string' }, + email_address: { type: 'string' }, + sender_name: { type: 'string' }, + color: { type: 'string' }, + protocol: { type: 'string', enum: ['imap'] }, + imap_host: { type: 'string' }, + imap_port: { type: 'integer' }, + smtp_host: { type: 'string' }, + smtp_port: { type: 'integer' }, + smtp_tls: { type: 'string', enum: ['STARTTLS', 'SSL', 'none'] }, + auth_user: { type: 'string' }, + signature: { type: 'string' }, + }, + }, +}; + +export const updateAccountSettingsDef = { + name: 'update_account_settings', + description: + 'Update non-secret account settings. Credentials, hosts, ports, and TLS settings must be changed in the Mailflow settings UI.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + name: { type: 'string' }, + sender_name: { type: 'string' }, + color: { type: 'string' }, + sort_order: { type: 'integer' }, + folder_mappings: { type: 'object' }, + signature: { type: 'string' }, + categorization_enabled: { type: 'boolean' }, + gtd_enabled: { type: 'boolean' }, + gtd_folders: { type: 'object' }, + enabled: { type: 'boolean' }, + }, + }, +}; + +export const testAccountConnectionDef = { + name: 'test_account_connection', + description: + 'Test IMAP and SMTP connectivity for an existing account using its stored credentials. Does not send mail.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account'], + properties: { + account: { + type: 'string', + description: 'Account email address (use list_accounts)', + }, + }, + }, +}; + +const aliasFields = { + name: { type: 'string' }, + email: { type: 'string' }, + reply_to: { type: 'string' }, + signature: { type: 'string' }, +}; + +export const createAliasDef = { + name: 'create_alias', + description: 'Create a send-as alias on a scoped account.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'name', 'email'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + ...aliasFields, + }, + }, +}; + +export const updateAliasDef = { + name: 'update_alias', + description: 'Update a send-as alias owned by a scoped account.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'alias_id', 'name', 'email'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + alias_id: { type: 'string' }, + ...aliasFields, + }, + }, +}; + +export const deleteAliasDef = { + name: 'delete_alias', + description: 'Delete a send-as alias owned by a scoped account.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'alias_id'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + alias_id: { type: 'string' }, + }, + }, +}; + +export const listRulesDef = { + name: 'list_rules', + description: 'List inbox rules owned by the token user, optionally including only global and one account’s rules.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Optional account email address' }, + }, + }, +}; + +const ruleFields = { + name: { type: 'string' }, + account_id: { + type: 'string', + description: 'Optional account email address; the field name matches the REST rule shape', + }, + condition_logic: { type: 'string', enum: ['AND', 'OR'] }, + conditions: { type: 'array', items: { type: 'object' } }, + actions: { type: 'array', items: { type: 'object' } }, + enabled: { type: 'boolean' }, + stop_processing: { type: 'boolean' }, +}; + +export const createRuleDef = { + name: 'create_rule', + description: 'Create an inbox rule for all accounts or one scoped account.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['name', 'conditions', 'actions'], + properties: ruleFields, + }, +}; + +export const updateRuleDef = { + name: 'update_rule', + description: 'Replace the settings of a user-owned inbox rule.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['rule_id', 'name', 'conditions', 'actions'], + properties: { + rule_id: { type: 'string' }, + ...ruleFields, + }, + }, +}; + +export const deleteRuleDef = { + name: 'delete_rule', + description: 'Delete a user-owned inbox rule.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['rule_id'], + properties: { + rule_id: { type: 'string' }, + }, + }, +}; + +export const runRulesDef = { + name: 'run_rules', + description: + 'Run enabled inbox rules against current INBOX messages for all scoped accounts or one account. Rule actions can move, archive, or delete messages.', + annotations: DESTRUCTIVE_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Optional account email address' }, + }, + }, +}; + +async function resolvedAccountId(account, scope) { + const resolved = await resolveAccountScope(account, scope.accountIds); + if (resolved.error) return resolved; + const accountId = resolved.accountIds?.[0]; + if (!accountId) return { error: `account not found: ${account}` }; + return { accountId }; +} + +export async function handleListAccounts(_args, scope) { + return jsonResult({ accounts: await listAccountsSafe(scope.accountIds) }); +} + +const SECRET_ACCOUNT_FIELDS = [ + 'auth_pass', + 'oauth_access_token', + 'oauth_refresh_token', +]; +const SECRET_REJECTION = + 'Passwords and OAuth tokens cannot be sent over MCP; finish authentication in the Mailflow settings UI'; + +export async function handleAddAccount(args, scope) { + if (SECRET_ACCOUNT_FIELDS.some((field) => Object.prototype.hasOwnProperty.call(args, field))) { + return errorResult(SECRET_REJECTION); + } + try { + const { stageAccount } = await import('../services/accountService.js'); + const result = await stageAccount({ userId: scope.userId, payload: args }); + if (result?.error) return errorResult(result.error); + return jsonResult({ + stage_id: result.id, + next_step: + 'Open Mailflow Settings > Accounts to finish setup, or POST /api/mcp-account-stages/:id/execute (session-authed) with credentials', + }); + } catch (error) { + return errorResult(error.message); + } +} + +const ACCOUNT_UPDATE_FIELDS = new Set([ + 'name', + 'sender_name', + 'color', + 'sort_order', + 'folder_mappings', + 'signature', + 'categorization_enabled', + 'gtd_enabled', + 'gtd_folders', + 'enabled', +]); +const UI_ONLY_ACCOUNT_FIELDS = new Set([ + 'auth_user', + 'auth_pass', + 'imap_host', + 'imap_port', + 'imap_tls', + 'imap_skip_tls_verify', + 'smtp_host', + 'smtp_port', + 'smtp_tls', +]); + +export async function handleUpdateAccountSettings(args, scope) { + for (const field of Object.keys(args)) { + if (UI_ONLY_ACCOUNT_FIELDS.has(field)) { + return errorResult( + `${field} cannot be changed over MCP; use the Mailflow settings UI`, + ); + } + if (field !== 'account' && !ACCOUNT_UPDATE_FIELDS.has(field)) { + return errorResult(`${field} is not an account setting supported over MCP`); + } + } + if ('name' in args && hasHeaderInjectionChars(args.name)) { + return errorResult('Name cannot contain control characters'); + } + if ( + 'sender_name' in args + && args.sender_name + && hasHeaderInjectionChars(args.sender_name) + ) { + return errorResult('Sender name cannot contain control characters'); + } + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const updates = Object.fromEntries( + Object.entries(args).filter(([field]) => ACCOUNT_UPDATE_FIELDS.has(field)), + ); + const result = await updateAccountSettings({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + updates, + }); + if (result.error) return errorResult(result.error); + return jsonResult(result.account); +} + +export async function handleTestAccountConnection(args, scope) { + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const account = await getAccountRow(resolved.accountId, scope.accountIds); + if (!account) return errorResult(`account not found: ${args.account}`); + const { testConnection } = await import('../services/connectionTest.js'); + return jsonResult({ + account: args.account, + ...await testConnection(account), + }); +} + +function aliasFieldsFrom(args) { + return { + name: args.name, + email: args.email, + reply_to: args.reply_to, + signature: args.signature, + }; +} + +function aliasValidation(args) { + if (!args.name || !args.email) return 'Name and email required'; + if ( + hasHeaderInjectionChars(args.name) + || hasHeaderInjectionChars(args.email) + || hasHeaderInjectionChars(args.reply_to) + ) { + return 'Fields cannot contain control characters'; + } + return null; +} + +export async function handleCreateAlias(args, scope) { + const validation = aliasValidation(args); + if (validation) return errorResult(validation); + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const alias = await createAlias({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + userId: scope.userId, + fields: aliasFieldsFrom(args), + }); + if (!alias) return errorResult('Account not found'); + return jsonResult(alias); +} + +export async function handleUpdateAlias(args, scope) { + const validation = aliasValidation(args); + if (validation) return errorResult(validation); + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const alias = await updateAlias({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + userId: scope.userId, + aliasId: args.alias_id, + fields: aliasFieldsFrom(args), + }); + if (!alias) return errorResult('Alias not found'); + return jsonResult(alias); +} + +export async function handleDeleteAlias(args, scope) { + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const deleted = await deleteAlias({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + userId: scope.userId, + aliasId: args.alias_id, + }); + if (!deleted) return errorResult('Alias not found'); + return jsonResult({ ok: true }); +} + +export async function handleListRules(args, scope) { + let accountId; + if (args.account) { + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + accountId = resolved.accountId; + } + return jsonResult({ + rules: await listRules({ userId: scope.userId, accountId }), + }); +} + +async function normalizedRuleInput(args, scope) { + if (!Array.isArray(args.conditions) || !Array.isArray(args.actions)) { + return { error: 'conditions and actions must be arrays' }; + } + const conditionError = validateConditions(args.conditions); + if (conditionError) return { error: conditionError }; + let accountId = null; + if (args.account_id) { + const resolved = await resolvedAccountId(args.account_id, scope); + if (resolved.error) return resolved; + accountId = resolved.accountId; + } + const actions = normalizeActions(args.actions) + .filter((action) => accountId || action.type !== 'move'); + return { + accountId, + name: args.name, + conditionLogic: args.condition_logic, + conditions: args.conditions, + actions, + enabled: args.enabled, + stopProcessing: args.stop_processing, + }; +} + +export async function handleCreateRule(args, scope) { + const input = await normalizedRuleInput(args, scope); + if (input.error) return errorResult(input.error); + const rule = await createRule({ userId: scope.userId, ...input }); + if (rule?.error) return errorResult(rule.error); + return jsonResult(rule); +} + +export async function handleUpdateRule(args, scope) { + const input = await normalizedRuleInput(args, scope); + if (input.error) return errorResult(input.error); + const rule = await updateRule({ + userId: scope.userId, + ruleId: args.rule_id, + ...input, + }); + if (rule?.error) return errorResult(rule.error); + if (!rule) return errorResult('Rule not found'); + return jsonResult(rule); +} + +export async function handleDeleteRule(args, scope) { + const deleted = await deleteRule({ + userId: scope.userId, + ruleId: args.rule_id, + }); + if (!deleted) return errorResult('Rule not found'); + return jsonResult({ ok: true }); +} + +export async function handleRunRules(args, scope, deps = {}) { + let accountIds = scope.accountIds; + if (args.account) { + const resolved = await resolveAccountScope(args.account, scope.accountIds); + if (resolved.error) return errorResult(resolved.error); + accountIds = resolved.accountIds; + } + return jsonResult(await runRules({ + userId: scope.userId, + accountIds, + imapManager: deps.imapManager, + })); +} diff --git a/backend/src/mcp/accountTools.test.js b/backend/src/mcp/accountTools.test.js new file mode 100644 index 00000000..cc825eef --- /dev/null +++ b/backend/src/mcp/accountTools.test.js @@ -0,0 +1,457 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; +import { SAFE_FIELDS } from '../services/accountFields.js'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('./accountAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getAccountRow: vi.fn(), + listAccountsSafe: vi.fn(), + updateAccountSettings: vi.fn(), + createAlias: vi.fn(), + updateAlias: vi.fn(), + deleteAlias: vi.fn(), + listRules: vi.fn(), + createRule: vi.fn(), + updateRule: vi.fn(), + deleteRule: vi.fn(), + runRules: vi.fn(), + }; +}); +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { ...actual, resolveAccountScope: vi.fn() }; +}); +vi.mock('../services/accountService.js', () => ({ + stageAccount: vi.fn(), + reconcileConnectionState: vi.fn(), +})); +vi.mock('../services/connectionTest.js', () => ({ testConnection: vi.fn() })); +vi.mock('../services/gtdConfig.js', () => ({ + DEFAULT_GTD_FOLDERS: { + todo: 'Todo', + watch: 'Watch', + delegated: 'Delegated', + someday: 'Someday', + reference: 'Reference', + }, + GTD_STATES: ['todo', 'watch', 'delegated', 'someday', 'reference'], + sanitizeGtdFoldersDetailed: vi.fn((input) => ({ + folders: input || {}, + rejected: [], + reserved: [], + })), + findGtdFolderCollisions: vi.fn(() => []), + invalidateGtdConfigCache: vi.fn(), + getGtdFolderSet: vi.fn(async () => new Set()), +})); +vi.mock('../services/gtdTransitions.js', () => ({ + invalidateOwnerAddressesCache: vi.fn(), +})); +vi.mock('../services/inboxRules.js', () => ({ + applyInboxRules: vi.fn(), + isDangerousRegex: vi.fn(() => false), + matchingRules: vi.fn(() => []), + toRuleMessage: vi.fn((message) => message), +})); +vi.mock('../routes/rules.js', () => ({ + validateConditions: vi.fn(), + normalizeActions: vi.fn(), +})); + +const db = await import('../services/db.js'); +const accountAdapter = await import('./accountAdapter.js'); +const realAccountAdapter = await vi.importActual('./accountAdapter.js'); +const engineAdapter = await import('./engineAdapter.js'); +const accountService = await import('../services/accountService.js'); +const connectionTest = await import('../services/connectionTest.js'); +const rulesRoute = await import('../routes/rules.js'); +const accountTools = await import('./accountTools.js').catch(() => ({})); +const registeredTools = await import('./tools.js'); + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const OTHER_ACCOUNT_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; +const ACCOUNT_EMAIL = 'owner@example.com'; +const scope = { + userId: 'user-1', + accountIds: [ACCOUNT_ID, OTHER_ACCOUNT_ID], + scopes: ['read', 'write', 'settings'], +}; +const deps = { imapManager: { marker: 'injected-imap-manager' } }; + +function handler(name) { + expect(accountTools[name], `${name} must be exported`).toBeTypeOf('function'); + return accountTools[name]; +} + +function jsonOf(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + engineAdapter.resolveAccountScope.mockImplementation(async (account, accountIds) => ( + account + ? { accountIds: [ACCOUNT_ID] } + : { accountIds } + )); + rulesRoute.validateConditions.mockReturnValue(null); + rulesRoute.normalizeActions.mockImplementation((actions) => actions); +}); + +describe('account/settings tool definitions and registration', () => { + it('registers all twelve tools with their required scopes and annotation hints', () => { + const expected = { + list_accounts: ['read', true, false, true], + add_account: ['settings', false, false, false], + update_account_settings: ['settings', false, false, true], + test_account_connection: ['settings', true, false, true], + create_alias: ['settings', false, false, false], + update_alias: ['settings', false, false, true], + delete_alias: ['settings', false, true, true], + list_rules: ['settings', true, false, true], + create_rule: ['settings', false, false, false], + update_rule: ['settings', false, false, true], + delete_rule: ['settings', false, true, true], + run_rules: [['settings', 'write'], false, true, false], + }; + const defs = new Map(registeredTools.TOOL_DEFS.map((def) => [def.name, def])); + + for (const [name, [requiredScope, readOnlyHint, destructiveHint, idempotentHint]] of Object.entries(expected)) { + expect(defs.get(name)?.annotations).toEqual({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); + expect(registeredTools.TOOL_SCOPES[name]).toEqual(requiredScope); + expect(registeredTools.HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('documents and schemas the no-secrets staged account flow', () => { + const def = registeredTools.TOOL_DEFS.find(({ name }) => name === 'add_account'); + expect(def?.description).toMatch(/does not accept passwords or OAuth tokens/i); + expect(def?.description).toMatch(/stage_id/i); + expect(def?.inputSchema.required).toEqual(['name', 'email_address']); + expect(def?.inputSchema.properties).not.toHaveProperty('auth_pass'); + expect(def?.inputSchema.properties).not.toHaveProperty('oauth_access_token'); + expect(def?.inputSchema.properties).not.toHaveProperty('oauth_refresh_token'); + }); +}); + +describe('list_accounts', () => { + it('returns safe accounts with aliases through the adapter seam', async () => { + const accounts = [{ + id: ACCOUNT_ID, + email_address: ACCOUNT_EMAIL, + aliases: [{ id: 'alias-1', email: 'alias@example.com' }], + }]; + accountAdapter.listAccountsSafe.mockResolvedValue(accounts); + + const result = await handler('handleListAccounts')({}, scope, deps); + + expect(jsonOf(result)).toEqual({ accounts }); + expect(accountAdapter.listAccountsSafe).toHaveBeenCalledWith(scope.accountIds); + }); + + it('passes exactly SAFE_FIELDS to the account query and never selects credentials', async () => { + const account = Object.fromEntries(SAFE_FIELDS.map((field) => [field, null])); + account.id = ACCOUNT_ID; + db.query + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ rows: [] }); + + await realAccountAdapter.listAccountsSafe([ACCOUNT_ID]); + + const accountSql = db.query.mock.calls[0][0]; + const columns = accountSql + .slice(accountSql.indexOf('SELECT') + 6, accountSql.indexOf('FROM email_accounts')) + .split(',') + .map((column) => column.trim()) + .filter(Boolean); + expect(columns).toEqual(SAFE_FIELDS); + expect(accountSql).not.toMatch(/\bauth_pass\b|\boauth_access_token\b|\boauth_refresh_token\b/); + expect(db.query.mock.calls[1][0]).toMatch(/account_aliases WHERE account_id = ANY\(\$1\)/); + }); +}); + +describe('add_account', () => { + it.each(['auth_pass', 'oauth_access_token', 'oauth_refresh_token'])( + 'rejects %s even when its value is empty before staging', + async (secretField) => { + const result = await handler('handleAddAccount')({ + name: 'Mail', + email_address: ACCOUNT_EMAIL, + [secretField]: '', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'Passwords and OAuth tokens cannot be sent over MCP; finish authentication in the Mailflow settings UI', + ); + expect(accountService.stageAccount).not.toHaveBeenCalled(); + }, + ); + + it('returns the stage id and the exact human-authentication next step', async () => { + accountService.stageAccount.mockResolvedValue({ id: 'stage-1' }); + + const result = await handler('handleAddAccount')({ + name: 'Mail', + email_address: ACCOUNT_EMAIL, + imap_host: 'imap.example.com', + }, scope, deps); + + expect(jsonOf(result)).toEqual({ + stage_id: 'stage-1', + next_step: 'Open Mailflow Settings > Accounts to finish setup, or POST /api/mcp-account-stages/:id/execute (session-authed) with credentials', + }); + expect(accountService.stageAccount).toHaveBeenCalledWith({ + userId: scope.userId, + payload: { + name: 'Mail', + email_address: ACCOUNT_EMAIL, + imap_host: 'imap.example.com', + }, + }); + }); + + it('surfaces host/port validation errors from stageAccount', async () => { + accountService.stageAccount.mockResolvedValue({ + error: 'IMAP: Port 25 is not allowed. Allowed: 143, 993', + status: 400, + }); + + const result = await handler('handleAddAccount')({ + name: 'Mail', + email_address: ACCOUNT_EMAIL, + imap_port: 25, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('IMAP: Port 25 is not allowed. Allowed: 143, 993'); + }); +}); + +describe('update_account_settings', () => { + it.each([ + 'auth_user', + 'auth_pass', + 'imap_host', + 'imap_port', + 'imap_tls', + 'imap_skip_tls_verify', + 'smtp_host', + 'smtp_port', + 'smtp_tls', + ])('hard-errors on excluded field %s instead of silently dropping it', async (field) => { + const result = await handler('handleUpdateAccountSettings')({ + account: ACCOUNT_EMAIL, + [field]: field.endsWith('port') ? 993 : 'value', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + `${field} cannot be changed over MCP; use the Mailflow settings UI`, + ); + expect(engineAdapter.resolveAccountScope).not.toHaveBeenCalled(); + expect(accountAdapter.updateAccountSettings).not.toHaveBeenCalled(); + }); + + it('resolves the account and returns the updated safe row', async () => { + const account = { id: ACCOUNT_ID, email_address: ACCOUNT_EMAIL, enabled: false }; + accountAdapter.updateAccountSettings.mockResolvedValue({ account }); + + const result = await handler('handleUpdateAccountSettings')({ + account: ACCOUNT_EMAIL, + enabled: false, + signature: 'Bye', + }, scope, deps); + + expect(jsonOf(result)).toEqual(account); + expect(accountAdapter.updateAccountSettings).toHaveBeenCalledWith({ + accountId: ACCOUNT_ID, + accountIds: scope.accountIds, + updates: { enabled: false, signature: 'Bye' }, + }); + }); +}); + +describe('test_account_connection', () => { + it('loads the scoped full account row and delegates both probes', async () => { + const fullRow = { + id: ACCOUNT_ID, + email_address: ACCOUNT_EMAIL, + auth_pass: 'encrypted-secret', + }; + accountAdapter.getAccountRow.mockResolvedValue(fullRow); + connectionTest.testConnection.mockResolvedValue({ + imap: { ok: true }, + smtp: { ok: false, error: 'SMTP authentication failed' }, + }); + + const result = await handler('handleTestAccountConnection')({ + account: ACCOUNT_EMAIL, + }, scope, deps); + + expect(jsonOf(result)).toEqual({ + account: ACCOUNT_EMAIL, + imap: { ok: true }, + smtp: { ok: false, error: 'SMTP authentication failed' }, + }); + expect(accountAdapter.getAccountRow).toHaveBeenCalledWith(ACCOUNT_ID, scope.accountIds); + expect(connectionTest.testConnection).toHaveBeenCalledWith(fullRow); + }); + + it('returns the account-scope error before touching the full-row adapter', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: foreign@example.com', + }); + + const result = await handler('handleTestAccountConnection')({ + account: 'foreign@example.com', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: foreign@example.com'); + expect(accountAdapter.getAccountRow).not.toHaveBeenCalled(); + expect(connectionTest.testConnection).not.toHaveBeenCalled(); + }); +}); + +describe('alias CRUD', () => { + it.each([ + ['handleCreateAlias', 'createAlias', { account: 'foreign@example.com', name: 'Alias', email: 'alias@example.com' }], + ['handleUpdateAlias', 'updateAlias', { account: 'foreign@example.com', alias_id: 'alias-1', name: 'Alias', email: 'alias@example.com' }], + ['handleDeleteAlias', 'deleteAlias', { account: 'foreign@example.com', alias_id: 'alias-1' }], + ])('%s rejects an out-of-scope account before the mutation adapter', async (handlerName, adapterName, args) => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: foreign@example.com', + }); + + const result = await handler(handlerName)(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: foreign@example.com'); + expect(accountAdapter[adapterName]).not.toHaveBeenCalled(); + }); + +}); + +describe('rules tools', () => { + it('list_rules resolves an optional account and returns user-owned rules', async () => { + const rules = [{ id: 'rule-1', account_id: ACCOUNT_ID }]; + accountAdapter.listRules.mockResolvedValue(rules); + + const result = await handler('handleListRules')({ account: ACCOUNT_EMAIL }, scope, deps); + + expect(jsonOf(result)).toEqual({ rules }); + expect(accountAdapter.listRules).toHaveBeenCalledWith({ + userId: scope.userId, + accountId: ACCOUNT_ID, + }); + }); + + it.each([ + ['handleCreateRule', 'createRule'], + ['handleUpdateRule', 'updateRule'], + ])('%s calls the imported condition validator and action normalizer', async (handlerName, adapterName) => { + const conditions = [{ field: 'from', operator: 'contains', value: 'example.com' }]; + const actions = [{ type: 'archive' }, { type: 'move', value: 'Archive ' }]; + const normalized = [{ type: 'archive' }]; + rulesRoute.normalizeActions.mockReturnValue(normalized); + accountAdapter[adapterName].mockResolvedValue({ id: 'rule-1' }); + const args = { + name: 'Archive example mail', + account_id: ACCOUNT_EMAIL, + conditions, + actions, + condition_logic: 'OR', + enabled: true, + stop_processing: true, + }; + if (handlerName === 'handleUpdateRule') args.rule_id = 'rule-1'; + + const result = await handler(handlerName)(args, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(rulesRoute.validateConditions).toHaveBeenCalledWith(conditions); + expect(rulesRoute.normalizeActions).toHaveBeenCalledWith(actions); + expect(accountAdapter[adapterName]).toHaveBeenCalledWith(expect.objectContaining({ + userId: scope.userId, + accountId: ACCOUNT_ID, + conditions, + actions: normalized, + conditionLogic: 'OR', + enabled: true, + stopProcessing: true, + })); + }); + + it('surfaces validation errors before normalizing or writing', async () => { + rulesRoute.validateConditions.mockReturnValue('Condition value cannot be empty'); + + const result = await handler('handleCreateRule')({ + name: 'Bad', + conditions: [{ field: 'from', value: '' }], + actions: [], + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Condition value cannot be empty'); + expect(rulesRoute.normalizeActions).not.toHaveBeenCalled(); + expect(accountAdapter.createRule).not.toHaveBeenCalled(); + }); + + it('delete_rule returns an error for a missing user-owned rule', async () => { + accountAdapter.deleteRule.mockResolvedValue(false); + + const result = await handler('handleDeleteRule')({ rule_id: 'missing' }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Rule not found'); + }); +}); + +describe('run_rules', () => { + it('uses every scoped account when account is omitted and threads deps.imapManager', async () => { + accountAdapter.runRules.mockResolvedValue({ processed: 7, matched: 3 }); + + const result = await handler('handleRunRules')({}, scope, deps); + + expect(jsonOf(result)).toEqual({ processed: 7, matched: 3 }); + expect(accountAdapter.runRules).toHaveBeenCalledWith({ + userId: scope.userId, + accountIds: scope.accountIds, + imapManager: deps.imapManager, + }); + }); + + it('returns an account scope error without invoking the run loop', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: foreign@example.com', + }); + + const result = await handler('handleRunRules')({ + account: 'foreign@example.com', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: foreign@example.com'); + expect(accountAdapter.runRules).not.toHaveBeenCalled(); + }); + +}); + +describe('mock-drift guard: mocked seams exist on their real modules', () => { + it.each([ + ['accountAdapter', () => accountAdapter, './accountAdapter.js'], + ['engineAdapter', () => engineAdapter, './engineAdapter.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); diff --git a/backend/src/mcp/auth.js b/backend/src/mcp/auth.js index 10addcbf..b2e3964e 100644 --- a/backend/src/mcp/auth.js +++ b/backend/src/mcp/auth.js @@ -1,6 +1,23 @@ import crypto from 'crypto'; import { query } from '../services/db.js'; +export const ALL_SCOPES = ['read', 'write', 'send', 'settings']; + +export function expandScopes(scopes) { + const expanded = new Set(Array.isArray(scopes) && scopes.length ? scopes : ['read']); + if (expanded.has('write') || expanded.has('send') || expanded.has('settings')) { + expanded.add('read'); + } + return [...expanded]; +} + +export function hasScope(scope, required) { + if (!required) return true; + const requiredScopes = Array.isArray(required) ? required : [required]; + const grantedScopes = scope.scopes || []; + return requiredScopes.every((requiredScope) => grantedScopes.includes(requiredScope)); +} + // Plaintext tokens are shown once at mint time; we persist only the hash. export function generateToken() { return 'mcp_' + crypto.randomBytes(32).toString('base64url'); @@ -12,12 +29,12 @@ export function hashToken(plaintext) { // The scope object pins multi-user isolation: every MCP tool call is bounded to // exactly this user's enabled accounts. msgvault is single-archive; Mailflow is not. -export async function resolveScope(userId) { +export async function resolveScope(userId, scopes) { const { rows } = await query( 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', [userId], ); - return { userId, accountIds: rows.map((r) => r.id) }; + return { userId, accountIds: rows.map((r) => r.id), scopes: expandScopes(scopes) }; } export async function mcpBearerAuth(req, res, next) { @@ -27,7 +44,7 @@ export async function mcpBearerAuth(req, res, next) { if (!m) return res.status(401).json({ error: 'invalid_token' }); const { rows } = await query( - 'SELECT id, user_id FROM api_tokens WHERE token_hash = $1', + 'SELECT id, user_id, scopes FROM api_tokens WHERE token_hash = $1', [hashToken(m[1].trim())], ); if (!rows.length) return res.status(401).json({ error: 'invalid_token' }); @@ -37,7 +54,7 @@ export async function mcpBearerAuth(req, res, next) { .catch(() => {}); req.mcpTokenId = rows[0].id; // rate-limit key: per token, not per IP - req.mcpScope = await resolveScope(rows[0].user_id); + req.mcpScope = await resolveScope(rows[0].user_id, rows[0].scopes); next(); } catch (err) { next(err); diff --git a/backend/src/mcp/auth.test.js b/backend/src/mcp/auth.test.js index 9b582740..691b3070 100644 --- a/backend/src/mcp/auth.test.js +++ b/backend/src/mcp/auth.test.js @@ -2,13 +2,50 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; vi.mock('../services/db.js', () => ({ query: vi.fn() })); import { query } from '../services/db.js'; -import { generateToken, hashToken, resolveScope, mcpBearerAuth } from './auth.js'; +import { + ALL_SCOPES, + expandScopes, + generateToken, + hasScope, + hashToken, + resolveScope, + mcpBearerAuth, +} from './auth.js'; function mockRes() { return { statusCode: 200, body: null, status(c){this.statusCode=c;return this;}, json(b){this.body=b;return this;} }; } describe('token helpers', () => { + it('exports the complete set of supported scopes', () => { + expect(ALL_SCOPES).toEqual(['read', 'write', 'send', 'settings']); + }); + + it('adds implied read access to elevated scopes', () => { + expect(expandScopes(['write'])).toEqual(['write', 'read']); + expect(expandScopes(['send'])).toEqual(['send', 'read']); + expect(expandScopes(['settings'])).toEqual(['settings', 'read']); + }); + + it('defaults empty or non-array scope values to read', () => { + expect(expandScopes([])).toEqual(['read']); + expect(expandScopes()).toEqual(['read']); + expect(expandScopes('send')).toEqual(['read']); + }); + + it('checks a single required scope', () => { + const scope = { scopes: ['read', 'write'] }; + expect(hasScope(scope, 'write')).toBe(true); + expect(hasScope(scope, 'send')).toBe(false); + expect(hasScope(scope)).toBe(true); + }); + + it('requires every scope when given an array', () => { + const scope = { scopes: ['read', 'write', 'settings'] }; + expect(hasScope(scope, ['settings', 'write'])).toBe(true); + expect(hasScope(scope, ['settings', 'send'])).toBe(false); + }); + it('generateToken is prefixed and unique', () => { const a = generateToken(); const b = generateToken(); expect(a).toMatch(/^mcp_[A-Za-z0-9_-]{20,}$/); @@ -23,10 +60,14 @@ describe('token helpers', () => { }); describe('resolveScope', () => { - it("returns only the token owner's enabled account ids", async () => { + it("returns only the token owner's enabled account ids and expanded scopes", async () => { query.mockResolvedValueOnce({ rows: [{ id: 'acc-1' }, { id: 'acc-2' }] }); - const scope = await resolveScope('user-1'); - expect(scope).toEqual({ userId: 'user-1', accountIds: ['acc-1', 'acc-2'] }); + const scope = await resolveScope('user-1', ['send']); + expect(scope).toEqual({ + userId: 'user-1', + accountIds: ['acc-1', 'acc-2'], + scopes: ['send', 'read'], + }); expect(query).toHaveBeenCalledWith( expect.stringMatching(/FROM email_accounts WHERE user_id = \$1 AND enabled = true/), ['user-1'], @@ -62,15 +103,20 @@ describe('mcpBearerAuth', () => { it('accepts a valid token and attaches the user-scoped account ids', async () => { query - .mockResolvedValueOnce({ rows: [{ id: 'tok-1', user_id: 'user-1' }] }) // token lookup + .mockResolvedValueOnce({ rows: [{ id: 'tok-1', user_id: 'user-1', scopes: ['write'] }] }) // token lookup .mockResolvedValueOnce({ rows: [] }) // last_used_at UPDATE .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); // resolveScope const req = { get: (h) => (h === 'Authorization' ? 'Bearer mcp_good' : undefined) }; const res = mockRes(); const next = vi.fn(); await mcpBearerAuth(req, res, next); expect(next).toHaveBeenCalledOnce(); - expect(req.mcpScope).toEqual({ userId: 'user-1', accountIds: ['acc-1'] }); + expect(req.mcpScope).toEqual({ + userId: 'user-1', + accountIds: ['acc-1'], + scopes: ['write', 'read'], + }); // token lookup must be by HASH, never the plaintext + expect(query.mock.calls[0][0]).toMatch(/SELECT id, user_id, scopes FROM api_tokens/); expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); }); diff --git a/backend/src/mcp/composeTools.js b/backend/src/mcp/composeTools.js new file mode 100644 index 00000000..4ef212ba --- /dev/null +++ b/backend/src/mcp/composeTools.js @@ -0,0 +1,258 @@ +import { + getAccountRow, + getComposeSource, + getUserPreferences, + listAliases, +} from './accountAdapter.js'; +import { jsonResult } from './result.js'; +import { buildWriteReceipt, writeError } from './writeResult.js'; +import { normalizeUndoWindow } from '../services/outboxService.js'; +import * as replyService from '../services/replyService.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const replyProperties = { + message_id: { + type: 'string', + description: 'Id of the message being replied to (from search/list/get_message)', + }, + body: { type: 'string' }, + body_html: { type: 'string' }, + to: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed To. Displaced recipients move to Cc rather than being dropped. Mutually exclusive with to_add.', + }, + cc: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed Cc.', + }, + bcc: { type: 'array', items: { type: 'string' } }, + to_add: { + type: 'array', + items: { type: 'string' }, + description: 'Appends to the computed To.', + }, + cc_add: { type: 'array', items: { type: 'string' } }, + bcc_add: { type: 'array', items: { type: 'string' } }, + remove: { + type: 'array', + items: { type: 'string' }, + description: 'Email addresses to drop from To/Cc/Bcc after computation.', + }, + no_quote: { + type: 'boolean', + description: 'Omit the quoted original.', + }, + include_inline_images: { + type: 'boolean', + description: 'Re-fetch cid: images from the original so the quote renders (default: replaced with [inline image: name]).', + }, + alias: { type: 'string' }, + attachments: {}, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, +}; + +export const replyEmailDef = { + name: 'reply_email', + description: 'Reply to the sender of a message.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: annotations(false, true, false), +}; + +export const replyAllEmailDef = { + name: 'reply_all_email', + description: 'Reply to the sender and all original To/Cc recipients. Bcc recipients of the original are not recoverable (they are not stored). Your own addresses and aliases are excluded from the recipients automatically.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: annotations(false, true, false), +}; + +export const forwardEmailDef = { + name: 'forward_email', + description: "Forward a message. Carries the original's attachments by default via forwardedAttachments: [{messageId, part}] — re-fetched from IMAP inside sendService, never round-tripped through the MCP wire.", + inputSchema: { + type: 'object', + required: ['message_id', 'to'], + properties: { + message_id: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + note: { type: 'string' }, + skip_attachments: { type: 'boolean' }, + alias: { type: 'string' }, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, + }, + }, + annotations: annotations(false, true, false), +}; + +function errorFrom(err) { + if (err?.code) return writeError(err.code, err.message); + return writeError('invalid_arguments', err?.message || 'compose operation failed'); +} + +async function composeContext(messageId, scope) { + const message = await getComposeSource(messageId, scope.accountIds); + if (!message) { + return { error: writeError('message_not_found', messageId) }; + } + const account = await getAccountRow(message.account_id, scope.accountIds); + if (!account) { + return { error: writeError('account_not_found', message.account_id) }; + } + const aliases = await listAliases(account.id); + return { message, account, aliases }; +} + +function recipientsComputed(message, account, aliases) { + return { + reply_target: replyService.pickReplyTarget(message).email || '', + excluded_self: [...replyService.selfAddressSet(account, aliases)].sort(), + }; +} + +function receiptForResult(result, compose, forwarded) { + const receipt = { + ...result.receipt, + ...(compose.inReplyTo !== undefined ? { inReplyTo: compose.inReplyTo } : {}), + ...(compose.references !== undefined ? { references: compose.references } : {}), + }; + if (forwarded) { + receipt.attachments = (receipt.attachments || []).map(attachment => ({ + ...attachment, + source: 'forwarded', + })); + } + return receipt; +} + +function sendResult(result, compose, resultFields = {}, forwarded = false) { + if (result.queued) { + return jsonResult(buildWriteReceipt({ + subject: compose.subject, + inReplyTo: compose.inReplyTo, + references: compose.references, + }, { + queued: true, + outboxId: result.outboxId, + sendAt: result.sendAt, + undoSeconds: result.undoSeconds, + ...resultFields, + note: 'Cancel with unsend_email before send_at.', + })); + } + return jsonResult(buildWriteReceipt( + receiptForResult(result, compose, forwarded), + { sent: true, ...resultFields }, + )); +} + +async function sendCompose(compose, args, scope, deps, resultFields, forwarded = false) { + const preferences = await getUserPreferences(scope.userId); + const result = await deps.sendService.sendOrEnqueue({ + ...compose, + undoSeconds: normalizeUndoWindow( + args.undo_send_seconds, + preferences?.undoSendSeconds, + ), + idempotencyKey: args.idempotency_key, + }, deps); + return sendResult(result, compose, resultFields, forwarded); +} + +function replyInput(args, context, replyAll) { + const bodyIsHtml = args.body_html !== undefined; + return { + message: context.message, + account: context.account, + aliases: context.aliases, + replyAll, + body: bodyIsHtml ? args.body_html : (args.body || ''), + bodyIsHtml, + to: args.to, + cc: args.cc, + bcc: args.bcc, + toAdd: args.to_add, + ccAdd: args.cc_add, + bccAdd: args.bcc_add, + remove: args.remove, + noQuote: args.no_quote, + includeInlineImages: args.include_inline_images, + alias: args.alias, + }; +} + +async function handleReply(args, scope, deps, replyAll) { + if (!deps?.sendService) { + return writeError('unsupported', 'compose tools require sendService'); + } + try { + const context = await composeContext(args.message_id, scope); + if (context.error) return context.error; + if ( + args.include_inline_images && + !args.no_quote && + !deps.imapManager?.fetchAttachment + ) { + return writeError('unsupported', 'include_inline_images requires imapManager'); + } + const compose = await replyService.buildReply( + replyInput(args, context, replyAll), + deps, + ); + return sendCompose(compose, args, scope, deps, { + recipientsComputed: recipientsComputed( + context.message, + context.account, + context.aliases, + ), + }); + } catch (err) { + return errorFrom(err); + } +} + +export async function handleReplyEmail(args, scope, deps = {}) { + return handleReply(args, scope, deps, false); +} + +export async function handleReplyAllEmail(args, scope, deps = {}) { + return handleReply(args, scope, deps, true); +} + +export async function handleForwardEmail(args, scope, deps = {}) { + if (!deps?.sendService) { + return writeError('unsupported', 'compose tools require sendService'); + } + try { + const context = await composeContext(args.message_id, scope); + if (context.error) return context.error; + const compose = await replyService.buildForward({ + message: context.message, + account: context.account, + aliases: context.aliases, + to: args.to, + note: args.note, + skipAttachments: args.skip_attachments, + alias: args.alias, + }, deps); + return sendCompose(compose, args, scope, deps, {}, true); + } catch (err) { + return errorFrom(err); + } +} diff --git a/backend/src/mcp/composeTools.test.js b/backend/src/mcp/composeTools.test.js new file mode 100644 index 00000000..d8fb181a --- /dev/null +++ b/backend/src/mcp/composeTools.test.js @@ -0,0 +1,559 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + getAccountRow: vi.fn(), + getComposeSource: vi.fn(), + getUserPreferences: vi.fn(), + listAliases: vi.fn(), + }; +}); + +import { + getAccountRow, + getComposeSource, + getUserPreferences, + listAliases, +} from './accountAdapter.js'; +import * as composeTools from './composeTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +const replyProperties = { + message_id: { + type: 'string', + description: 'Id of the message being replied to (from search/list/get_message)', + }, + body: { type: 'string' }, + body_html: { type: 'string' }, + to: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed To. Displaced recipients move to Cc rather than being dropped. Mutually exclusive with to_add.', + }, + cc: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed Cc.', + }, + bcc: { type: 'array', items: { type: 'string' } }, + to_add: { + type: 'array', + items: { type: 'string' }, + description: 'Appends to the computed To.', + }, + cc_add: { type: 'array', items: { type: 'string' } }, + bcc_add: { type: 'array', items: { type: 'string' } }, + remove: { + type: 'array', + items: { type: 'string' }, + description: 'Email addresses to drop from To/Cc/Bcc after computation.', + }, + no_quote: { + type: 'boolean', + description: 'Omit the quoted original.', + }, + include_inline_images: { + type: 'boolean', + description: 'Re-fetch cid: images from the original so the quote renders (default: replaced with [inline image: name]).', + }, + alias: { type: 'string' }, + attachments: {}, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, +}; + +const sendAnnotations = { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, +}; + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['send'], +}; +const account = { + id: 'account-1', + user_id: 'user-1', + email_address: 'me@example.com', + sender_name: 'Me', +}; +const aliases = [{ + id: 'alias-1', + email: 'team@example.com', + reply_to: 'alias-replies@example.com', +}]; +const message = { + id: '11111111-1111-4111-8111-111111111111', + account_id: 'account-1', + uid: 42, + folder: 'INBOX', + subject: 'Topic', + from_name: 'Sender', + from_email: 'sender@example.com', + reply_to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + to_addresses: [ + { name: 'Me', email: 'me@example.com' }, + { name: 'Team', email: 'team@example.com' }, + { name: 'Other', email: 'other@example.com' }, + ], + cc_addresses: [ + { name: 'Colleague', email: 'colleague@example.com' }, + { name: 'Alias Reply', email: 'alias-replies@example.com' }, + ], + body_text: 'Original text', + body_html: '

Original HTML

', + attachments: [{ + part: '2', + filename: 'deck.pdf', + type: 'application/pdf', + size: 2_144_000, + }], + message_id: '', + thread_references: '', +}; +const replyReceipt = { + from: { name: 'Team', email: 'team@example.com' }, + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [], + bcc: [], + subject: 'Re: Topic', + attachments: [], + messageId: '', + sentCopySaved: true, + folder: 'Sent', +}; + +function deps(overrides = {}) { + return { + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: replyReceipt, + }), + }, + ...overrides, + }; +} + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + getComposeSource.mockResolvedValue(message); + getAccountRow.mockResolvedValue(account); + listAliases.mockResolvedValue(aliases); + getUserPreferences.mockResolvedValue({}); +}); + +describe('compose tool definitions and registration', () => { + it('publishes the Phase 1 schemas, descriptions, annotations, scopes, and handlers', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + + expect(defs.get('reply_email')).toEqual({ + name: 'reply_email', + description: 'Reply to the sender of a message.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: sendAnnotations, + }); + expect(defs.get('reply_all_email')).toEqual({ + name: 'reply_all_email', + description: 'Reply to the sender and all original To/Cc recipients. Bcc recipients of the original are not recoverable (they are not stored). Your own addresses and aliases are excluded from the recipients automatically.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: sendAnnotations, + }); + expect(defs.get('forward_email')).toEqual({ + name: 'forward_email', + description: "Forward a message. Carries the original's attachments by default via forwardedAttachments: [{messageId, part}] — re-fetched from IMAP inside sendService, never round-tripped through the MCP wire.", + inputSchema: { + type: 'object', + required: ['message_id', 'to'], + properties: { + message_id: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + note: { type: 'string' }, + skip_attachments: { type: 'boolean' }, + alias: { type: 'string' }, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, + }, + }, + annotations: sendAnnotations, + }); + + for (const [name, handler] of [ + ['reply_email', composeTools.handleReplyEmail], + ['reply_all_email', composeTools.handleReplyAllEmail], + ['forward_email', composeTools.handleForwardEmail], + ]) { + expect(TOOL_SCOPES[name]).toBe('send'); + expect(HANDLERS[name]).toBe(handler); + } + }); +}); + +describe('reply_email', () => { + it('returns message_not_found without disclosing an out-of-scope message', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(null); + + const result = await composeTools.handleReplyEmail({ + message_id: 'foreign-message', + body: 'Reply', + }, scope, service); + + expect(getComposeSource).toHaveBeenCalledWith('foreign-message', ['account-1']); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_not_found: foreign-message'); + expect(getAccountRow).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('returns account_not_found if the compose source account disappears', async () => { + const service = deps(); + getAccountRow.mockResolvedValue(null); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + }, scope, service); + + expect(getAccountRow).toHaveBeenCalledWith('account-1', ['account-1']); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account_not_found: account-1'); + expect(listAliases).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('preserves alias_not_found from replyService', async () => { + const service = deps(); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + alias: 'missing@example.com', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it.each([ + [{ + to: Array.from({ length: 101 }, (_, i) => `person-${i}@example.com`), + }, 'too_many_recipients: Too many recipients (max 100)'], + [{ + to: ['replacement@example.com'], + to_add: ['additional@example.com'], + }, 'invalid_arguments: to and toAdd are mutually exclusive'], + ])('preserves recipient computation errors for %j', async (recipientArgs, expected) => { + const service = deps(); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + ...recipientArgs, + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(expected); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('sends immediately and returns threading plus recipients_computed', async () => { + const service = deps(); + + const wireResult = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Thanks', + no_quote: true, + undo_send_seconds: 0, + idempotency_key: 'reply-1', + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(listAliases).toHaveBeenCalledWith('account-1'); + expect(getUserPreferences).toHaveBeenCalledWith('user-1'); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith({ + account, + aliasId: 'alias-1', + userId: 'user-1', + to: ['Reply Desk '], + cc: [], + bcc: [], + subject: 'Re: Topic', + body: 'Thanks', + bodyIsHtml: false, + quotedBody: '', + quotedBodyHtml: null, + inReplyTo: '', + references: ' ', + undoSeconds: 0, + idempotencyKey: 'reply-1', + }, service); + expect(payload(wireResult)).toEqual({ + sent: true, + recipients_computed: { + reply_target: 'reply@example.com', + excluded_self: [ + 'alias-replies@example.com', + 'me@example.com', + 'team@example.com', + ], + }, + message_id: '', + in_reply_to: '', + references: ' ', + from: { name: 'Team', email: 'team@example.com' }, + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [], + bcc: [], + subject: 'Re: Topic', + attachments: [], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('uses HTML body selection and passes additive recipient adjustments to replyService', async () => { + const service = deps(); + + await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Plain fallback', + body_html: '

HTML reply

', + to_add: ['additional@example.com'], + cc_add: ['added-copy@example.com'], + bcc_add: ['added-blind@example.com'], + remove: ['reply@example.com'], + no_quote: true, + }, scope, service); + + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + body: '

HTML reply

', + bodyIsHtml: true, + to: ['additional@example.com'], + cc: ['added-copy@example.com'], + bcc: ['added-blind@example.com'], + }), + service, + ); + }); + + it('maps a missing inline-image fetch dependency to unsupported', async () => { + const service = deps(); + getComposeSource.mockResolvedValue({ + ...message, + body_html: '

Original

', + attachments: [{ + part: '2', + cid: 'image-1', + filename: 'inline.png', + type: 'image/png', + }], + }); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + include_inline_images: true, + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'unsupported: include_inline_images requires imapManager', + ); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('does not require imapManager when no_quote disables inline-image fetching', async () => { + const service = deps(); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + no_quote: true, + include_inline_images: true, + }, scope, service); + + expect(result.isError).not.toBe(true); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledOnce(); + }); + + it('returns the queued write shape with threading and recipient computation', async () => { + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + queued: true, + outboxId: 'outbox-reply-1', + sendAt: new Date('2026-07-28T10:00:30.000Z'), + undoSeconds: 30, + }), + }, + }); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 30 }); + + const wireResult = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Queued reply', + }, scope, service); + + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + undoSeconds: 30, + idempotencyKey: undefined, + }), + service, + ); + expect(payload(wireResult)).toEqual({ + queued: true, + outbox_id: 'outbox-reply-1', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + recipients_computed: { + reply_target: 'reply@example.com', + excluded_self: [ + 'alias-replies@example.com', + 'me@example.com', + 'team@example.com', + ], + }, + in_reply_to: '', + references: ' ', + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Re: Topic', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); +}); + +describe('reply_all_email', () => { + it('excludes account and alias identities while preserving non-self recipients', async () => { + const service = deps(); + + const result = await composeTools.handleReplyAllEmail({ + message_id: message.id, + body: 'Reply all', + no_quote: true, + }, scope, service); + + const [input] = service.sendService.sendOrEnqueue.mock.calls[0]; + expect(input.to).toEqual(['Reply Desk ']); + expect(input.cc).toEqual([ + 'Other ', + 'Colleague ', + ]); + expect(payload(result).recipients_computed).toEqual({ + reply_target: 'reply@example.com', + excluded_self: [ + 'alias-replies@example.com', + 'me@example.com', + 'team@example.com', + ], + }); + }); +}); + +describe('forward_email', () => { + it('passes forward options and tags delivered attachments as forwarded', async () => { + const forwardReceipt = { + ...replyReceipt, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + subject: 'Fwd: Topic', + attachments: [{ filename: 'deck.pdf', size: 2_144_000 }], + messageId: '', + }; + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: forwardReceipt, + }), + }, + }); + + const wireResult = await composeTools.handleForwardEmail({ + message_id: message.id, + to: ['Recipient '], + note: 'Please review.', + alias: 'team@example.com', + undo_send_seconds: 0, + idempotency_key: 'forward-1', + }, scope, service); + + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + account, + aliasId: 'alias-1', + userId: 'user-1', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Fwd: Topic', + body: 'Please review.', + bodyIsHtml: false, + forwardedAttachments: [{ + messageId: message.id, + part: '2', + }], + undoSeconds: 0, + idempotencyKey: 'forward-1', + }), + service, + ); + expect(payload(wireResult).attachments).toEqual([{ + filename: 'deck.pdf', + size: 2_144_000, + source: 'forwarded', + }]); + }); +}); + +describe('missing dependencies', () => { + it.each([ + ['reply_email', composeTools.handleReplyEmail, { + message_id: message.id, + body: 'Reply', + }], + ['reply_all_email', composeTools.handleReplyAllEmail, { + message_id: message.id, + body: 'Reply all', + }], + ['forward_email', composeTools.handleForwardEmail, { + message_id: message.id, + to: ['recipient@example.com'], + }], + ])('%s degrades to unsupported when sendService is missing', async (_name, handler, args) => { + const result = await handler(args, scope, {}); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'unsupported: compose tools require sendService', + ); + expect(getComposeSource).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/mcp/draftTools.js b/backend/src/mcp/draftTools.js new file mode 100644 index 00000000..5e742c81 --- /dev/null +++ b/backend/src/mcp/draftTools.js @@ -0,0 +1,432 @@ +import { + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + listDraftRows, +} from './accountAdapter.js'; +import { newPaginatedResponse, toRFC3339 } from './envelope.js'; +import { errorResult, jsonResult } from './result.js'; +import { buildWriteReceipt, writeError } from './writeResult.js'; +import { mapRecipientList } from '../services/mail/addresses.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; +import { buildReferences } from '../services/replyService.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const attachmentSchema = { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, +}; + +const draftComposeProperties = { + account: { + type: 'string', + description: 'Account email address (see get_stats for available accounts)', + }, + to: { + type: 'array', + items: { type: 'string' }, + description: "Recipients as 'Name ' or bare 'email'", + }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string', description: 'Plain-text body' }, + body_html: { + type: 'string', + description: 'HTML body. When set, takes precedence over body for the HTML part.', + }, + reply_to_message_id: { + type: 'string', + description: 'Message id to thread this draft under; sets In-Reply-To and References.', + }, + alias: { + type: 'string', + description: 'Send-as alias email. Must be a configured alias on the account — errors if unknown, never silently falls back.', + }, + attachments: attachmentSchema, +}; + +export const createDraftDef = { + name: 'create_draft', + description: "Create a draft email in the account's Drafts folder. Does NOT send. Returns the draft's uid and folder for later update_draft/send_draft/delete_draft calls.", + inputSchema: { + type: 'object', + required: ['account'], + properties: draftComposeProperties, + }, + annotations: annotations(false, false, false), +}; + +export const updateDraftDef = { + name: 'update_draft', + description: 'Update an existing draft. IMAP has no update-in-place for messages: this appends a new draft and deletes the old one, so the draft_uid CHANGES. Use the returned draft_uid for subsequent calls.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + ...draftComposeProperties, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + }, + }, + annotations: annotations(false, true, false), +}; + +export const listDraftsDef = { + name: 'list_drafts', + description: 'List drafts, newest first. Paginate with offset/limit (default limit 20, max 100).', + inputSchema: { + type: 'object', + properties: { + account: { + type: 'string', + description: 'Filter by account email address (use get_stats to list available accounts)', + }, + limit: { type: 'integer', default: 20, maximum: 100 }, + offset: { type: 'integer', default: 0, minimum: 0 }, + }, + }, + annotations: annotations(true, false, true), +}; + +export const getDraftDef = { + name: 'get_draft', + description: 'Get a draft including body text, body HTML, threading headers, and attachment metadata.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { + type: 'string', + description: 'Account email address (see get_stats for available accounts)', + }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + }, + }, + annotations: annotations(true, false, true), +}; + +export const deleteDraftDef = { + name: 'delete_draft', + description: 'PERMANENTLY deletes the draft from IMAP — it does not go to Trash and cannot be recovered.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { + type: 'string', + description: 'Account email address (see get_stats for available accounts)', + }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + }, + }, + annotations: annotations(false, true, true), +}; + +function unsupported(deps) { + if (deps?.draftService) return null; + return writeError('unsupported', 'draft tools require draftService'); +} + +function draftFolder(account, requestedFolder) { + return requestedFolder || account?.folder_mappings?.drafts || 'Drafts'; +} + +async function accountForEmail(email, scope) { + const account = await getAccountByEmail(email, scope.accountIds); + return account?.error ? { error: errorResult(account.error) } : { account }; +} + +function addressString(address) { + if (typeof address === 'string') return address; + if (!address?.email) return ''; + return address.name ? `${address.name} <${address.email}>` : address.email; +} + +function addressStrings(value) { + return (Array.isArray(value) ? value : []).map(addressString).filter(Boolean); +} + +function attachmentInput(attachment) { + return { + filename: attachment.filename, + content: attachment.content, + contentType: attachment.content_type ?? attachment.contentType, + }; +} + +function attachmentSize(attachment) { + if (Number.isFinite(Number(attachment?.size))) return Number(attachment.size); + if (typeof attachment?.content !== 'string') return 0; + try { + return Buffer.from(attachment.content, 'base64').length; + } catch { + return 0; + } +} + +function receiptForDraft(identity, compose, saved) { + return buildWriteReceipt({ + messageId: saved.messageId, + from: { + name: identity?.fromName || '', + email: identity?.fromEmail || '', + }, + to: mapRecipientList(compose.to), + cc: mapRecipientList(compose.cc), + bcc: mapRecipientList(compose.bcc), + subject: compose.subject || '', + attachments: (compose.attachments || []).map(attachment => ({ + filename: attachment.filename, + size: attachmentSize(attachment), + })), + }); +} + +function draftResult(saved, identity, compose) { + return jsonResult({ + draft_uid: saved.uid, + folder: saved.folder, + message_id: saved.messageId, + receipt: receiptForDraft(identity, compose, saved), + }); +} + +function errorFrom(exception) { + if (exception?.code) return writeError(exception.code, exception.message); + if (/drafts folder/i.test(exception?.message || '')) { + return writeError('no_drafts_folder', exception.message); + } + return writeError('invalid_arguments', exception?.message || 'draft operation failed'); +} + +async function threadingFor(messageId, scope) { + if (!messageId) return {}; + const source = await getComposeSource(messageId, scope.accountIds); + if (!source) return { error: writeError('message_not_found', messageId) }; + return buildReferences(source); +} + +export async function handleCreateDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const identity = await resolveFromIdentity( + resolved.account, + { aliasEmail: args.alias }, + deps, + ); + const threading = await threadingFor(args.reply_to_message_id, scope); + if (threading.error) return threading.error; + const compose = { + userId: scope.userId, + account: resolved.account, + aliasEmail: args.alias, + to: args.to || [], + cc: args.cc || [], + bcc: args.bcc || [], + subject: args.subject || '', + body: args.body_html !== undefined ? args.body_html : (args.body || ''), + bodyIsHtml: args.body_html !== undefined, + attachments: (args.attachments || []).map(attachmentInput), + inReplyTo: threading.inReplyTo, + references: threading.references, + }; + const saved = await deps.draftService.saveDraft(compose, deps); + return draftResult(saved, identity, compose); + } catch (err) { + return errorFrom(err); + } +} + +function hasOwn(object, key) { + return Object.prototype.hasOwnProperty.call(object, key); +} + +function mergedCompose(args, existing, account) { + const explicitBody = hasOwn(args, 'body'); + const explicitHtml = hasOwn(args, 'body_html'); + const useHtml = explicitHtml || (!explicitBody && Boolean(existing.body_html)); + const body = explicitHtml + ? args.body_html + : explicitBody + ? args.body + : useHtml + ? existing.body_html + : (existing.body_text || ''); + return { + userId: undefined, + account, + aliasEmail: hasOwn(args, 'alias') + ? args.alias + : (existing.from_email && existing.from_email !== account.email_address + ? existing.from_email + : undefined), + to: hasOwn(args, 'to') ? args.to : addressStrings(existing.to_addresses), + cc: hasOwn(args, 'cc') ? args.cc : addressStrings(existing.cc_addresses), + bcc: hasOwn(args, 'bcc') ? args.bcc : addressStrings(existing.bcc_addresses), + subject: hasOwn(args, 'subject') ? args.subject : (existing.subject || ''), + body, + bodyIsHtml: useHtml, + attachments: hasOwn(args, 'attachments') + ? (args.attachments || []).map(attachmentInput) + : (existing.attachments || []).map(attachmentInput), + inReplyTo: existing.in_reply_to || undefined, + references: existing.thread_references || undefined, + }; +} + +export async function handleUpdateDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const folder = draftFolder(resolved.account, args.folder); + const existing = await getDraftRow(resolved.account.id, folder, args.draft_uid); + if (!existing) return writeError('draft_not_found', args.draft_uid); + + const compose = mergedCompose(args, existing, resolved.account); + compose.userId = scope.userId; + if (args.reply_to_message_id) { + const threading = await threadingFor(args.reply_to_message_id, scope); + if (threading.error) return threading.error; + compose.inReplyTo = threading.inReplyTo; + compose.references = threading.references; + } + const identity = await resolveFromIdentity( + resolved.account, + { aliasEmail: compose.aliasEmail }, + deps, + ); + compose.existingUid = existing.uid; + compose.existingFolder = existing.folder; + const saved = await deps.draftService.saveDraft(compose, deps); + return draftResult(saved, identity, compose); + } catch (err) { + return errorFrom(err); + } +} + +function draftSummary(row) { + return { + draft_uid: row.uid, + folder: row.folder, + subject: row.subject || '', + to: row.to_addresses || [], + cc: row.cc_addresses || [], + snippet: row.snippet || '', + date: toRFC3339(row.date), + has_attachments: Boolean( + row.has_attachments || (Array.isArray(row.attachments) && row.attachments.length), + ), + }; +} + +async function listAccounts(args, scope) { + if (args.account) { + const resolved = await accountForEmail(args.account, scope); + return resolved.error ? resolved : { accounts: [resolved.account] }; + } + const accounts = []; + for (const accountId of scope.accountIds || []) { + const account = await getAccountRow(accountId, scope.accountIds); + if (account) accounts.push(account); + } + return { accounts }; +} + +export async function handleListDrafts(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await listAccounts(args, scope); + if (resolved.error) return resolved.error; + const rows = []; + for (const account of resolved.accounts) { + const accountRows = await listDraftRows(account.id, { + limit: Number.MAX_SAFE_INTEGER, + offset: 0, + folder: draftFolder(account), + }); + rows.push(...accountRows); + } + rows.sort((a, b) => String(b.date || '').localeCompare(String(a.date || ''))); + const limit = Math.min(Math.max(Number.parseInt(args.limit, 10) || 20, 1), 100); + const offset = Math.max(Number.parseInt(args.offset, 10) || 0, 0); + const page = rows.slice(offset, offset + limit).map(draftSummary); + return jsonResult(newPaginatedResponse(page, rows.length, offset)); + } catch (err) { + return errorFrom(err); + } +} + +function fullDraft(row) { + return { + draft_uid: row.uid, + folder: row.folder, + subject: row.subject || '', + to: row.to_addresses || [], + cc: row.cc_addresses || [], + bcc: row.bcc_addresses || [], + body_text: row.body_text || '', + body_html: row.body_html || '', + in_reply_to: row.in_reply_to || null, + references: row.thread_references || null, + attachments: row.attachments || [], + }; +} + +export async function handleGetDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const folder = draftFolder(resolved.account, args.folder); + const row = await getDraftRow(resolved.account.id, folder, args.draft_uid); + if (!row) return writeError('draft_not_found', args.draft_uid); + return jsonResult(fullDraft(row)); + } catch (err) { + return errorFrom(err); + } +} + +export async function handleDeleteDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const folder = draftFolder(resolved.account, args.folder); + const row = await getDraftRow(resolved.account.id, folder, args.draft_uid); + if (!row) return writeError('draft_not_found', args.draft_uid); + await deps.draftService.deleteDraft({ + account: resolved.account, + uid: row.uid, + folder: row.folder, + }, deps); + return jsonResult({ deleted: true, draft_uid: row.uid, folder: row.folder }); + } catch (err) { + return errorFrom(err); + } +} diff --git a/backend/src/mcp/draftTools.test.js b/backend/src/mcp/draftTools.test.js new file mode 100644 index 00000000..200f88c7 --- /dev/null +++ b/backend/src/mcp/draftTools.test.js @@ -0,0 +1,467 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', () => ({ + getAccountByEmail: vi.fn(), + getAccountRow: vi.fn(), + getComposeSource: vi.fn(), + getDraftRow: vi.fn(), + listDraftRows: vi.fn(), +})); +vi.mock('../services/mail/identity.js', () => ({ + resolveFromIdentity: vi.fn(), +})); +vi.mock('../services/replyService.js', () => ({ + buildReferences: vi.fn(), +})); + +import { + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + listDraftRows, +} from './accountAdapter.js'; +import { + createDraftDef, + deleteDraftDef, + getDraftDef, + handleCreateDraft, + handleDeleteDraft, + handleGetDraft, + handleListDrafts, + handleUpdateDraft, + listDraftsDef, + updateDraftDef, +} from './draftTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; +import { buildReferences } from '../services/replyService.js'; + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['read', 'write'], +}; +const account = { + id: 'account-1', + email_address: 'sender@example.com', + sender_name: 'Sender', + folder_mappings: { drafts: 'Drafts' }, +}; +const identity = { + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: null, + signature: null, + aliasId: null, +}; + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +function deps(overrides = {}) { + return { + draftService: { + saveDraft: vi.fn().mockResolvedValue({ + uid: 42, + folder: 'Drafts', + messageId: '', + }), + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + }, + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + getAccountByEmail.mockResolvedValue(account); + getAccountRow.mockResolvedValue(account); + resolveFromIdentity.mockResolvedValue(identity); + buildReferences.mockReturnValue({ + inReplyTo: '', + references: ' ', + }); +}); + +describe('draft tool definitions and registration', () => { + it('publishes the plan schemas, descriptions, annotations, scopes, and handlers', () => { + expect(createDraftDef).toEqual({ + name: 'create_draft', + description: "Create a draft email in the account's Drafts folder. Does NOT send. Returns the draft's uid and folder for later update_draft/send_draft/delete_draft calls.", + inputSchema: { + type: 'object', + required: ['account'], + properties: { + account: { type: 'string', description: 'Account email address (see get_stats for available accounts)' }, + to: { type: 'array', items: { type: 'string' }, description: "Recipients as 'Name ' or bare 'email'" }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string', description: 'Plain-text body' }, + body_html: { type: 'string', description: 'HTML body. When set, takes precedence over body for the HTML part.' }, + reply_to_message_id: { type: 'string', description: 'Message id to thread this draft under; sets In-Reply-To and References.' }, + alias: { type: 'string', description: 'Send-as alias email. Must be a configured alias on the account — errors if unknown, never silently falls back.' }, + attachments: { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: false, + }, + }); + expect(updateDraftDef.description).toContain( + 'IMAP has no update-in-place for messages: this appends a new draft and deletes the old one, so the draft_uid CHANGES. Use the returned draft_uid for subsequent calls.', + ); + expect(updateDraftDef.inputSchema.required).toEqual(['account', 'draft_uid']); + expect(updateDraftDef.inputSchema.properties).toEqual(expect.objectContaining({ + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + })); + expect(listDraftsDef.inputSchema.properties.limit).toEqual({ + type: 'integer', + default: 20, + maximum: 100, + }); + expect(getDraftDef.inputSchema.required).toEqual(['account', 'draft_uid']); + expect(deleteDraftDef.description).toContain( + 'PERMANENTLY deletes the draft from IMAP — it does not go to Trash and cannot be recovered.', + ); + + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, requiredScope, handler] of [ + ['create_draft', 'write', handleCreateDraft], + ['update_draft', 'write', handleUpdateDraft], + ['list_drafts', 'read', handleListDrafts], + ['get_draft', 'read', handleGetDraft], + ['delete_draft', 'write', handleDeleteDraft], + ]) { + expect(defs.get(name)).toBeTruthy(); + expect(TOOL_SCOPES[name]).toBe(requiredScope); + expect(HANDLERS[name]).toBe(handler); + } + }); +}); + +describe('create_draft', () => { + it('resolves the scoped account and alias, threads the draft, saves it, and returns a write receipt', async () => { + const service = deps(); + getComposeSource.mockResolvedValue({ + id: 'message-1', + message_id: '', + thread_references: '', + }); + resolveFromIdentity.mockResolvedValue({ + ...identity, + fromName: 'Team', + fromEmail: 'team@example.com', + aliasId: 'alias-1', + }); + + const result = payload(await handleCreateDraft({ + account: 'sender@example.com', + to: ['Recipient '], + cc: ['copy@example.com'], + subject: 'Subject', + body: 'plain fallback', + body_html: '

Hello

', + reply_to_message_id: 'message-1', + alias: 'team@example.com', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + content_type: 'text/plain', + }], + }, scope, service)); + + expect(getAccountByEmail).toHaveBeenCalledWith('sender@example.com', ['account-1']); + expect(resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + service, + ); + expect(getComposeSource).toHaveBeenCalledWith('message-1', ['account-1']); + expect(buildReferences).toHaveBeenCalledWith(expect.objectContaining({ id: 'message-1' })); + expect(service.draftService.saveDraft).toHaveBeenCalledWith({ + userId: 'user-1', + account, + aliasEmail: 'team@example.com', + to: ['Recipient '], + cc: ['copy@example.com'], + bcc: [], + subject: 'Subject', + body: '

Hello

', + bodyIsHtml: true, + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + contentType: 'text/plain', + }], + inReplyTo: '', + references: ' ', + }, service); + expect(result).toEqual({ + draft_uid: 42, + folder: 'Drafts', + message_id: '', + receipt: { + message_id: '', + from: { name: 'Team', email: 'team@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [{ name: '', email: 'copy@example.com' }], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + }, + }); + }); + + it('does not reveal an out-of-scope account and never calls the service', async () => { + const service = deps(); + getAccountByEmail.mockResolvedValue({ error: 'account_not_found: foreign@example.com' }); + + const result = await handleCreateDraft( + { account: 'foreign@example.com' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account_not_found: foreign@example.com'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); + + it('hard-fails an unknown alias and preserves its stable error code', async () => { + const service = deps(); + resolveFromIdentity.mockRejectedValue( + Object.assign(new Error('Alias not found'), { code: 'alias_not_found' }), + ); + + const result = await handleCreateDraft({ + account: 'sender@example.com', + alias: 'unknown@example.com', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); + + it('returns message_not_found for an out-of-scope reply source', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(null); + + const result = await handleCreateDraft({ + account: 'sender@example.com', + reply_to_message_id: 'foreign-message', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_not_found: foreign-message'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); +}); + +describe('update_draft', () => { + it('carries over every unsupplied compose field and returns the replacement uid', async () => { + const service = deps(); + getDraftRow.mockResolvedValue({ + uid: 9, + folder: 'Drafts', + to_addresses: [{ name: 'Original', email: 'original@example.com' }], + cc_addresses: [{ name: '', email: 'copy@example.com' }], + bcc_addresses: [{ name: '', email: 'blind@example.com' }], + subject: 'Original subject', + body_text: 'Original body', + body_html: null, + in_reply_to: '', + thread_references: ' ', + attachments: [], + }); + + const result = payload(await handleUpdateDraft({ + account: 'sender@example.com', + draft_uid: 9, + body: 'Edited body only', + }, scope, service)); + + expect(getDraftRow).toHaveBeenCalledWith('account-1', 'Drafts', 9); + expect(service.draftService.saveDraft).toHaveBeenCalledWith(expect.objectContaining({ + account, + to: ['Original '], + cc: ['copy@example.com'], + bcc: ['blind@example.com'], + subject: 'Original subject', + body: 'Edited body only', + bodyIsHtml: false, + inReplyTo: '', + references: ' ', + existingUid: 9, + existingFolder: 'Drafts', + }), service); + expect(result.draft_uid).toBe(42); + expect(result.draft_uid).not.toBe(9); + expect(result.receipt.to).toEqual([{ name: 'Original', email: 'original@example.com' }]); + }); + + it('returns draft_not_found without saving when the existing row is absent', async () => { + const service = deps(); + getDraftRow.mockResolvedValue(null); + + const result = await handleUpdateDraft({ + account: 'sender@example.com', + draft_uid: 404, + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('draft_not_found: 404'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); +}); + +describe('list_drafts and get_draft', () => { + it('lists shaped draft summaries with the shared pagination envelope', async () => { + listDraftRows.mockResolvedValue([ + { + uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to_addresses: [{ name: '', email: 'to@example.com' }], + cc_addresses: [], + snippet: 'Draft snippet', + date: '2026-07-28T10:00:00.000Z', + has_attachments: true, + }, + ]); + + const result = payload(await handleListDrafts({ + account: 'sender@example.com', + limit: 20, + offset: 0, + }, scope, deps())); + + expect(listDraftRows).toHaveBeenCalledWith('account-1', { + limit: Number.MAX_SAFE_INTEGER, + offset: 0, + folder: 'Drafts', + }); + expect(result).toEqual({ + data: [{ + draft_uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to: [{ name: '', email: 'to@example.com' }], + cc: [], + snippet: 'Draft snippet', + date: '2026-07-28T10:00:00Z', + has_attachments: true, + }], + total: 1, + returned: 1, + offset: 0, + has_more: false, + }); + }); + + it('uses every scoped account when list_drafts omits account', async () => { + const multiScope = { ...scope, accountIds: ['account-1', 'account-2'] }; + getAccountRow + .mockResolvedValueOnce(account) + .mockResolvedValueOnce({ + ...account, + id: 'account-2', + email_address: 'other@example.com', + }); + listDraftRows + .mockResolvedValueOnce([{ uid: 1, folder: 'Drafts', date: '2026-01-01T00:00:00Z' }]) + .mockResolvedValueOnce([{ uid: 2, folder: 'Drafts', date: '2026-02-01T00:00:00Z' }]); + + const result = payload(await handleListDrafts({}, multiScope, deps())); + + expect(getAccountRow).toHaveBeenCalledTimes(2); + expect(result.data.map(row => row.draft_uid)).toEqual([2, 1]); + expect(result.total).toBe(2); + }); + + it('returns a full shaped draft without leaking storage-only fields', async () => { + getDraftRow.mockResolvedValue({ + uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to_addresses: [{ name: '', email: 'to@example.com' }], + cc_addresses: [], + bcc_addresses: [], + body_text: 'Hello', + body_html: '

Hello

', + in_reply_to: '', + thread_references: ' ', + attachments: [{ filename: 'note.txt', size: 5, part: '2' }], + account_id: 'account-1', + auth_pass: 'must-not-leak', + }); + + const result = payload(await handleGetDraft({ + account: 'sender@example.com', + draft_uid: 7, + }, scope, deps())); + + expect(result).toEqual({ + draft_uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to: [{ name: '', email: 'to@example.com' }], + cc: [], + bcc: [], + body_text: 'Hello', + body_html: '

Hello

', + in_reply_to: '', + references: ' ', + attachments: [{ filename: 'note.txt', size: 5, part: '2' }], + }); + }); +}); + +describe('delete_draft and missing dependencies', () => { + it('permanently deletes a scoped draft and returns its identifier', async () => { + const service = deps(); + getDraftRow.mockResolvedValue({ uid: 7, folder: 'Drafts' }); + + const result = payload(await handleDeleteDraft({ + account: 'sender@example.com', + draft_uid: 7, + }, scope, service)); + + expect(service.draftService.deleteDraft).toHaveBeenCalledWith({ + account, + uid: 7, + folder: 'Drafts', + }, service); + expect(result).toEqual({ deleted: true, draft_uid: 7, folder: 'Drafts' }); + }); + + it.each([ + ['create', handleCreateDraft, { account: 'sender@example.com' }], + ['update', handleUpdateDraft, { account: 'sender@example.com', draft_uid: 1 }], + ['list', handleListDrafts, { account: 'sender@example.com' }], + ['get', handleGetDraft, { account: 'sender@example.com', draft_uid: 1 }], + ['delete', handleDeleteDraft, { account: 'sender@example.com', draft_uid: 1 }], + ])('%s degrades to unsupported when draftService is absent', async (_name, handler, args) => { + const result = await handler(args, scope, {}); + expect(result.isError).toBe(true); + expect(result.content[0].text).toMatch(/^unsupported: /); + }); +}); diff --git a/backend/src/mcp/engineAdapter.js b/backend/src/mcp/engineAdapter.js index 034bfc03..c4c56fce 100644 --- a/backend/src/mcp/engineAdapter.js +++ b/backend/src/mcp/engineAdapter.js @@ -7,12 +7,12 @@ import { query, withTransaction } from '../services/db.js'; import { buildOperatorClauses, freeTextTermClause, hasSearchableToken } from '../services/search/lexicalRepo.js'; -const SUMMARY_COLUMNS = ` +export const SUMMARY_COLUMNS = ` m.id, m.account_id, m.message_id, m.thread_id, m.subject, m.snippet, m.from_email, m.from_name, m.to_addresses, m.cc_addresses, m.date, m.has_attachments, m.attachments, m.flags, m.folder`; -const DETAIL_COLUMNS = SUMMARY_COLUMNS + `, m.body_text, m.body_html`; +export const DETAIL_COLUMNS = SUMMARY_COLUMNS + `, m.body_text, m.body_html`; export function mapAddrs(jsonb) { return (jsonb || []).map((a) => ({ Email: a.email || '', Name: a.name || '' })); diff --git a/backend/src/mcp/goldenParity.test.js b/backend/src/mcp/goldenParity.test.js index 2c503b8c..2e20086b 100644 --- a/backend/src/mcp/goldenParity.test.js +++ b/backend/src/mcp/goldenParity.test.js @@ -23,6 +23,30 @@ vi.mock('../services/embeddings/generations.js', () => ({ activeGeneration: vi.f vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); vi.mock('../services/embeddings/config.js', () => ({ generationFingerprint: vi.fn(() => 'fp'), resolveEmbedConfig: vi.fn(async () => ({ enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 })) })); +vi.mock('../services/mailbox/move.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + bulkMoveToFolder: vi.fn(), + resolveMovedIds: vi.fn(), + }; +}); +vi.mock('../services/mailbox/archive.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkArchive: vi.fn() }; +}); +vi.mock('../services/mailbox/trash.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkTrash: vi.fn() }; +}); +vi.mock('../services/mailbox/snooze.js', async (orig) => { + const actual = await orig(); + return { ...actual, snoozeConversation: vi.fn() }; +}); +vi.mock('../services/gtd/actions.js', async (orig) => { + const actual = await orig(); + return { ...actual, gtdDone: vi.fn() }; +}); import { query, withTransaction } from '../services/db.js'; import { search } from '../services/search/searchService.js'; @@ -30,12 +54,38 @@ import * as generations from '../services/embeddings/generations.js'; import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { ALL_SCOPES } from './auth.js'; import { rowToMessageSummary, rowToMessageDetail } from './engineAdapter.js'; import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; import { handleGetMessage, handleListMessages, handleGetStats, handleAggregate, handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, handleSearchByDomains, } from './messageTools.js'; +import { + handleCreateDraft, handleDeleteDraft, handleUpdateDraft, +} from './draftTools.js'; +import { + handleListOutbox, handleRecallEmail, handleSendDraft, handleSendEmail, + handleUnsendEmail, +} from './sendTools.js'; +import { + handleForwardEmail, handleReplyAllEmail, handleReplyEmail, +} from './composeTools.js'; +import { + handleArchiveMessages, + handleGtdDone, + handleMoveMessages, + handleSnoozeMessage, + handleTrashMessages, +} from './mailboxTools.js'; +import { + bulkMoveToFolder, + resolveMovedIds, +} from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { snoozeConversation } from '../services/mailbox/snooze.js'; +import { gtdDone } from '../services/gtd/actions.js'; import { HANDLERS } from './tools.js'; import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; @@ -63,6 +113,40 @@ const SearchBodiesEnvelope = { data: [SearchMessageItem], total: 'int', returned: 'int', offset: 'int', has_more: 'bool', mode: 'string', pool_saturated: 'bool', generation: Generation, }; +const WriteAddress = { name: 'string', email: 'string' }; +const WriteAttachment = { filename: 'string', size: 'int', 'source?': 'string' }; +const WriteReceipt = { + from: WriteAddress, + to: [WriteAddress], + cc: [WriteAddress], + bcc: [WriteAddress], + subject: 'string', + attachments: [WriteAttachment], +}; +const ImmediateSend = { + sent: 'bool', + message_id: 'string', + ...WriteReceipt, + sent_copy_saved: 'bool', + folder: 'string', +}; +const QueuedSend = { + queued: 'bool', + outbox_id: 'string', + send_at: 'string', + undo_seconds: 'int', + from: {}, + to: [WriteAddress], + cc: [WriteAddress], + bcc: [WriteAddress], + subject: 'string', + attachments: [WriteAttachment], + note: 'string', +}; +const RecipientsComputed = { + reply_target: 'string', + excluded_self: ['string'], +}; const SHAPES = { message_summary: MessageSummary, message_detail: { ...MessageSummary, from: [Address], to: [Address], cc: [Address], bcc: [Address], body_text: 'string', body_html: 'string', attachments: [AttachmentInfo] }, @@ -88,6 +172,60 @@ const SHAPES = { semantic_search_messages: SearchBodiesEnvelope, search_by_domains: [MessageSummary], // raw array, no envelope (handlers.go:1984-1989) ping: { pong: 'bool' }, // Mailflow-specific health tool — no msgvault counterpart (documented divergence) + write_receipt: WriteReceipt, + create_or_update_draft: { + draft_uid: 'int', + folder: 'string', + message_id: 'string', + receipt: { message_id: 'string', ...WriteReceipt }, + }, + delete_draft: { deleted: 'bool', draft_uid: 'int', folder: 'string' }, + immediate_send: ImmediateSend, + queued_send: QueuedSend, + reply_send: { + sent: 'bool', + recipients_computed: RecipientsComputed, + message_id: 'string', + in_reply_to: 'string', + references: 'string', + ...WriteReceipt, + sent_copy_saved: 'bool', + folder: 'string', + }, + forward_send: { + ...ImmediateSend, + attachments: [{ filename: 'string', size: 'int', source: 'string' }], + }, + unsend_email: { + cancelled: 'bool', + outbox_id: 'string', + subject: 'string', + to: ['string'], + }, + list_outbox: { + data: [{ + id: 'string', + subject: 'string', + to_preview: ['string'], + send_at: 'string', + }], + total: 'int', + returned: 'int', + offset: 'int', + has_more: 'bool', + }, + recall_cancelled_before_send: { + recalled: 'string', + outbox_id: 'string', + subject: 'string', + to: ['string'], + }, + recall_not_possible: { + recalled: 'string', + note: 'string', + sent_copy_deleted: 'bool', + followup_draft: { draft_uid: 'int', folder: 'string' }, + }, }; // ---- diffKeys: [] on parity, else a list of divergence paths ------------------- @@ -133,13 +271,88 @@ const realRow = { attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello world', body_html: '

hello world

', }; +const writeAccount = { + id: 'acc-1', + user_id: 'u', + email_address: 'sender@example.com', + sender_name: 'Sender', + signature: null, + folder_mappings: { drafts: 'Drafts' }, +}; +const composeSource = { + id: '22222222-2222-4222-8222-222222222222', + account_id: 'acc-1', + uid: 42, + folder: 'Sent', + message_id: '', + thread_references: '', + subject: 'Topic', + from_name: 'Original Sender', + from_email: 'original@example.com', + reply_to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + to_addresses: [{ name: 'Sender', email: 'sender@example.com' }], + cc_addresses: [{ name: 'Colleague', email: 'colleague@example.com' }], + body_text: 'Original body', + body_html: '

Original body

', + attachments: [{ part: '2', filename: 'deck.pdf', size: 2_144_000 }], + date: new Date('2026-07-28T09:00:00Z'), +}; +const immediateReceipt = { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', +}; const jsonOf = (r) => JSON.parse(r.content[0].text); -const scope = { userId: 'u', accountIds: ['acc-1'] }; +const scope = { userId: 'u', accountIds: ['acc-1'], scopes: ALL_SCOPES }; + +function writeDeps(overrides = {}) { + return { + draftService: { + saveDraft: vi.fn().mockResolvedValue({ + uid: 42, + folder: 'Drafts', + messageId: '', + }), + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + }, + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: immediateReceipt, + }), + }, + outboxService: { + normalizeUndoWindow: vi.fn((requested, preference) => requested ?? preference ?? 0), + cancel: vi.fn().mockResolvedValue({ cancelled: true }), + listPending: vi.fn().mockResolvedValue([]), + }, + imapManager: { + permanentDeleteMessage: vi.fn().mockResolvedValue(undefined), + }, + ...overrides, + }; +} + +function mockQueryRows(...rowSets) { + for (const rows of rowSets) { + query.mockResolvedValueOnce({ rows, rowCount: rows.length }); + } +} beforeEach(() => { query.mockReset(); withTransaction.mockReset(); search.mockReset(); matchFromChunk.mockReset(); generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); + bulkMoveToFolder.mockReset(); resolveMovedIds.mockReset(); bulkArchive.mockReset(); + bulkTrash.mockReset(); snoozeConversation.mockReset(); gtdDone.mockReset(); }); // Every function this suite vi.mock()s must actually exist on its real module — @@ -172,6 +385,8 @@ describe('golden parity: structural mappers', () => { }); describe('golden parity: message tool envelopes', () => { + // These unchanged get/list cases are the read-wire regression guard for the + // write-handler `(args, scope, deps)` stack. it('get_message → getMessageResponse', async () => { query.mockResolvedValueOnce({ rows: [realRow] }); const b = jsonOf(await handleGetMessage({ id: realRow.id }, scope)); @@ -305,10 +520,444 @@ describe('golden parity: search tool envelopes', () => { }); }); -describe('no SQL in MCP handler modules (one-seam invariant)', () => { +describe('golden parity: write tool envelopes', () => { + it('create_draft → draft identifier plus nested write receipt', async () => { + const deps = writeDeps(); + mockQueryRows([writeAccount]); + + const b = jsonOf(await handleCreateDraft({ + account: 'sender@example.com', + to: ['Recipient '], + subject: 'Subject', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + content_type: 'text/plain', + }], + }, scope, deps)); + + expect(diffKeys(b, SHAPES.create_or_update_draft)).toEqual([]); + }); + + it('update_draft → replacement identifier plus nested write receipt', async () => { + const deps = writeDeps(); + deps.draftService.saveDraft.mockResolvedValue({ + uid: 43, + folder: 'Drafts', + messageId: '', + }); + mockQueryRows( + [writeAccount], + [{ + uid: 42, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Subject', + body_text: 'Original body', + body_html: null, + attachments: [], + }], + ); + + const b = jsonOf(await handleUpdateDraft({ + account: 'sender@example.com', + draft_uid: 42, + body: 'Updated body', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.create_or_update_draft)).toEqual([]); + expect(b.draft_uid).toBe(43); + }); + + it('delete_draft → permanent-deletion identifier envelope', async () => { + const deps = writeDeps(); + mockQueryRows( + [writeAccount], + [{ uid: 42, folder: 'Drafts' }], + ); + + const b = jsonOf(await handleDeleteDraft({ + account: 'sender@example.com', + draft_uid: 42, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.delete_draft)).toEqual([]); + }); + + it('send_email immediate → full write receipt', async () => { + const deps = writeDeps(); + mockQueryRows([writeAccount], [{ preferences: { undoSendSeconds: 0 } }]); + + const b = jsonOf(await handleSendEmail({ + account: 'sender@example.com', + to: ['Recipient '], + subject: 'Subject', + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.immediate_send)).toEqual([]); + }); + + it('send_email queued → placeholder receipt with undo metadata', async () => { + const deps = writeDeps(); + deps.outboxService.normalizeUndoWindow.mockReturnValue(30); + deps.sendService.sendOrEnqueue.mockResolvedValue({ + queued: true, + outboxId: 'outbox-1', + sendAt: new Date('2026-07-28T10:00:30.000Z'), + undoSeconds: 30, + }); + mockQueryRows([writeAccount], [{ preferences: { undoSendSeconds: 30 } }]); + + const b = jsonOf(await handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + subject: 'Queued subject', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.queued_send)).toEqual([]); + expect(b.from).toEqual({}); + expect(b.to).toEqual([]); + }); + + it('send_draft immediate → same full write receipt as send_email', async () => { + const deps = writeDeps(); + mockQueryRows( + [writeAccount], + [{ + uid: 42, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Subject', + body_text: 'Draft body', + body_html: '', + }], + [{ preferences: { undoSendSeconds: 0 } }], + ); + + const b = jsonOf(await handleSendDraft({ + account: 'sender@example.com', + draft_uid: 42, + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.immediate_send)).toEqual([]); + }); + + it.each([ + ['reply_email', handleReplyEmail], + ['reply_all_email', handleReplyAllEmail], + ])('%s → threading plus recipients_computed receipt', async (_name, handler) => { + const deps = writeDeps(); + deps.sendService.sendOrEnqueue.mockResolvedValue({ + ok: true, + receipt: { + ...immediateReceipt, + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + subject: 'Re: Topic', + attachments: [], + messageId: '', + }, + }); + mockQueryRows( + [composeSource], + [writeAccount], + [], + [{ preferences: { undoSendSeconds: 0 } }], + ); + + const b = jsonOf(await handler({ + message_id: composeSource.id, + body: 'Thanks', + no_quote: true, + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.reply_send)).toEqual([]); + expect(b.recipients_computed.reply_target).toBe('reply@example.com'); + }); + + it('forward_email → immediate receipt with forwarded attachment source', async () => { + const deps = writeDeps(); + deps.sendService.sendOrEnqueue.mockResolvedValue({ + ok: true, + receipt: { + ...immediateReceipt, + subject: 'Fwd: Topic', + attachments: [{ filename: 'deck.pdf', size: 2_144_000 }], + messageId: '', + }, + }); + mockQueryRows( + [composeSource], + [writeAccount], + [], + [{ preferences: { undoSendSeconds: 0 } }], + ); + + const b = jsonOf(await handleForwardEmail({ + message_id: composeSource.id, + to: ['Recipient '], + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.forward_send)).toEqual([]); + expect(b.attachments[0].source).toBe('forwarded'); + }); + + it('unsend_email → cancelled outbox receipt', async () => { + const deps = writeDeps(); + deps.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const b = jsonOf(await handleUnsendEmail({ + outbox_id: 'outbox-1', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.unsend_email)).toEqual([]); + }); + + it('list_outbox → no-total pagination envelope', async () => { + const deps = writeDeps(); + deps.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const b = jsonOf(await handleListOutbox({}, scope, deps)); + + expect(diffKeys(b, SHAPES.list_outbox)).toEqual([]); + expect(b.total).toBe(-1); + }); + + it('recall_email pending → cancelled_before_send envelope', async () => { + const deps = writeDeps(); + deps.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + }]); + + const b = jsonOf(await handleRecallEmail({ + outbox_id: 'outbox-1', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.recall_cancelled_before_send)).toEqual([]); + expect(b.recalled).toBe('cancelled_before_send'); + }); + + it('recall_email delivered → not_possible envelope with follow-up draft', async () => { + const deps = writeDeps(); + deps.draftService.saveDraft.mockResolvedValue({ + uid: 57, + folder: 'Drafts', + messageId: '', + }); + mockQueryRows( + [], + [composeSource], + [writeAccount], + ); + query.mockResolvedValueOnce({ rows: [], rowCount: 1 }); + + const b = jsonOf(await handleRecallEmail({ + message_id: composeSource.id, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.recall_not_possible)).toEqual([]); + expect(b.recalled).toBe('not_possible'); + expect(deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); +}); + +describe('golden parity: mailbox tool envelopes', () => { + const id = '33333333-3333-4333-8333-333333333333'; + const secondId = '44444444-4444-4444-8444-444444444444'; + const newId = '55555555-5555-4555-8555-555555555555'; + const deps = { imapManager: {} }; + + it('pins move_messages receipt keys', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + movedDetails: [{ id, accountId: 'acc-1', uid: 110 }], + failed: [], + skippedAccounts: [], + }); + resolveMovedIds.mockResolvedValue([{ id: newId, uid: 110 }]); + + expect(jsonOf(await handleMoveMessages({ + message_ids: [id], + folder: 'Projects', + }, scope, deps))).toEqual({ + ok: true, + moved: [{ id, new_id: newId, uid: 110, folder: 'Projects' }], + failed: [], + skipped_accounts: [], + resync_pending: false, + note: 'message ids change on move; use new_id for follow-up calls', + }); + }); + + it('pins archive_messages receipt keys including destination_untracked', async () => { + bulkArchive.mockResolvedValue({ + ok: true, + archivedDetails: [ + { + id, + accountId: 'acc-1', + folder: 'Archive', + uid: 110, + destinationUntracked: false, + }, + { + id: secondId, + accountId: 'acc-1', + folder: '[Gmail]/All Mail', + uid: 111, + destinationUntracked: true, + }, + ], + failed: [], + noArchiveFolder: [], + }); + resolveMovedIds.mockResolvedValue([{ id: newId, uid: 110 }]); + + expect(jsonOf(await handleArchiveMessages({ + message_ids: [id, secondId], + }, scope, deps))).toEqual({ + ok: true, + archived: [ + { + id, + new_id: newId, + uid: 110, + folder: 'Archive', + destination_untracked: false, + }, + { + id: secondId, + new_id: null, + uid: 111, + folder: '[Gmail]/All Mail', + destination_untracked: true, + }, + ], + failed: [], + no_archive_folder: [], + resync_pending: false, + note: 'message ids change on archive; use new_id for follow-up calls', + }); + }); + + it('pins trash_messages receipt keys including the refusal partition', async () => { + bulkTrash.mockResolvedValue({ + ok: true, + trashedDetails: [{ + id, + accountId: 'acc-1', + folder: 'Trash', + uid: 110, + }], + failed: [], + refused: [{ + id: secondId, + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + }); + resolveMovedIds.mockResolvedValue([{ id: newId, uid: 110 }]); + + expect(jsonOf(await handleTrashMessages({ + message_ids: [id, secondId], + }, scope, deps))).toEqual({ + ok: true, + trashed: [{ id, new_id: newId, folder: 'Trash' }], + failed: [], + refused: [{ + id: secondId, + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + resync_pending: false, + next_step: 'use stage_deletion for permanent removal', + }); + }); + + it('pins snooze_message receipt keys', async () => { + snoozeConversation.mockResolvedValue({ + ok: true, + movedCount: 2, + movedIds: [id, secondId], + folder: 'Snoozed', + }); + + expect(jsonOf(await handleSnoozeMessage({ + message_id: id, + until: new Date(Date.now() + 60_000).toISOString(), + }, scope, deps))).toEqual({ + ok: true, + moved_count: 2, + sibling_ids: [secondId], + folder: 'Snoozed', + }); + }); + + it('pins gtd_done partial-success receipt keys', async () => { + gtdDone.mockResolvedValue({ + ok: true, + removed: ['Watch'], + archived: false, + noArchiveFolder: false, + archiveFailed: true, + }); + + const result = await handleGtdDone({ + message_id: id, + states: ['watch'], + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual({ + ok: true, + removed: ['Watch'], + archived: false, + no_archive_folder: false, + archive_failed: true, + }); + }); +}); + +describe('SQL is confined to MCP adapter seams', () => { const here = dirname(fileURLToPath(import.meta.url)); const read = (n) => readFileSync(join(here, n), 'utf8'); - for (const file of ['searchTools.js', 'messageTools.js']) { + for (const file of ['engineAdapter.js', 'accountAdapter.js']) { + it(`${file} is an explicitly allowed SQL seam`, () => { + expect(read(file)).toMatch(/from '\.\.\/services\/db\.js'/); + }); + } + for (const file of [ + 'searchTools.js', + 'messageTools.js', + 'mailboxTools.js', + 'triageTools.js', + 'composeTools.js', + 'draftTools.js', + 'sendTools.js', + 'writeResult.js', + 'accountTools.js', + ]) { it(`${file} contains no raw SQL or db.js import`, () => { const src = read(file); expect(src).not.toMatch(/\bSELECT\b|\bINSERT\b|\bUPDATE\b|\bDELETE FROM\b/); diff --git a/backend/src/mcp/mailboxTools.js b/backend/src/mcp/mailboxTools.js new file mode 100644 index 00000000..abb6600d --- /dev/null +++ b/backend/src/mcp/mailboxTools.js @@ -0,0 +1,748 @@ +import { getAccountRow } from './accountAdapter.js'; +import { errorResult, jsonResult } from './result.js'; +import { + countMessagesIn, + createFolder, + deleteFolder, + listFolders, + renameFolder, +} from '../services/mailbox/folders.js'; +import { + bulkMoveToFolder, + resolveMovedIds, +} from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { bulkSetRead, setStarred } from '../services/mailbox/flags.js'; +import { runInBatches } from '../services/mailbox/batch.js'; +import { markNotSpam, markSpam } from '../services/mailbox/spamLabel.js'; +import { + snoozeConversation, + unsnoozeConversation, +} from '../services/mailbox/snooze.js'; +import { setCategory } from '../services/mailbox/category.js'; +import { + gtdClassify, + gtdDone, + gtdUnclassify, +} from '../services/gtd/actions.js'; +import { GTD_STATES } from '../services/gtdConfig.js'; +import { + areValidUUIDs, + isValidFolderName, + UUID_RE, +} from '../utils/validation.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); +const CREATE_ANNOTATIONS = annotations(); +const IDEMPOTENT_WRITE_ANNOTATIONS = annotations({ idempotentHint: true }); +const DESTRUCTIVE_WRITE_ANNOTATIONS = annotations({ destructiveHint: true }); +const DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS = annotations({ + destructiveHint: true, + idempotentHint: true, +}); + +export function messageIdsArg(args) { + const ids = args.message_ids; + if (!Array.isArray(ids) || ids.length === 0) { + return { error: 'message_ids must contain at least one id' }; + } + if (ids.length > 500) return { error: 'Too many ids — maximum 500 per request' }; + if (!areValidUUIDs(ids)) return { error: 'Invalid message id format' }; + return { value: ids }; +} + +export function messageIdArg(args) { + const id = args.message_id; + if (!id || typeof id !== 'string') return { error: 'message_id parameter is required' }; + if (!UUID_RE.test(id)) return { error: 'Invalid message id format' }; + return { value: id }; +} + +export const listFoldersDef = { + name: 'list_folders', + description: 'List folders and live message counts for scoped accounts. Pass account to narrow the result to one account.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string' }, + }, + }, +}; + +export const createFolderDef = { + name: 'create_folder', + description: 'Create a folder in one scoped account. The name must be an explicit valid folder component.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'name'], + properties: { + account: { type: 'string' }, + name: { type: 'string' }, + parent_path: { type: 'string' }, + }, + }, +}; + +export const renameFolderDef = { + name: 'rename_folder', + description: 'Rename the final component of an existing folder path. Both the account and source path must be explicit.', + annotations: DESTRUCTIVE_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'path', 'new_name'], + properties: { + account: { type: 'string' }, + path: { type: 'string' }, + new_name: { type: 'string' }, + }, + }, +}; + +export const deleteFolderDef = { + name: 'delete_folder', + description: 'Delete a folder and its tracked messages. The live message count must exactly match expected_message_count.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'path', 'expected_message_count'], + properties: { + account: { type: 'string' }, + path: { type: 'string' }, + expected_message_count: { type: 'integer', minimum: 0 }, + }, + }, +}; + +const messageIdsSchema = { + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, +}; + +export const moveMessagesDef = { + name: 'move_messages', + description: 'Move explicit messages to a destination folder and return their replacement ids. Message ids change on move, and non-UIDPLUS servers may report resync_pending.', + annotations: DESTRUCTIVE_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids', 'folder'], + properties: { + message_ids: messageIdsSchema, + folder: { type: 'string' }, + }, + }, +}; + +export const archiveMessagesDef = { + name: 'archive_messages', + description: 'Archive explicit messages and return their replacement ids. Gmail All Mail destinations are reported as destination_untracked.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + }, + }, +}; + +export const trashMessagesDef = { + name: 'trash_messages', + description: 'Move explicit messages to Trash and return their replacement ids. Messages requiring permanent deletion are refused.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + }, + }, +}; + +function messageFlagDef(name, description) { + return { + name, + description, + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + }, + }, + }; +} + +export const markReadDef = messageFlagDef( + 'mark_read', + 'Mark explicit messages as read. The updated receipt excludes messages already read.', +); +export const markUnreadDef = messageFlagDef( + 'mark_unread', + 'Mark explicit messages as unread. The updated receipt excludes messages already unread.', +); +export const starMessageDef = messageFlagDef( + 'star_message', + 'Star explicit messages. The updated receipt excludes messages already starred.', +); +export const unstarMessageDef = messageFlagDef( + 'unstar_message', + 'Unstar explicit messages. The updated receipt excludes messages already unstarred.', +); + +function singleMessageDef(name, description, extraProperties = {}, extraRequired = []) { + return { + name, + description, + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_id', ...extraRequired], + properties: { + message_id: { type: 'string' }, + ...extraProperties, + }, + }, + }; +} + +export const markSpamDef = singleMessageDef( + 'mark_spam', + 'Mark one explicit message as spam and move it to the configured spam folder.', +); +export const markNotSpamDef = singleMessageDef( + 'mark_not_spam', + 'Mark one explicit spam-folder message as not spam and move it to Inbox.', +); +export const snoozeMessageDef = singleMessageDef( + 'snooze_message', + 'Snooze one message and its reply-chain siblings to the Snoozed folder for up to 30 days.', + { until: { type: 'string', format: 'date-time' } }, + ['until'], +); +export const unsnoozeMessageDef = singleMessageDef( + 'unsnooze_message', + 'Restore one snoozed message and its reply-chain siblings to their original folder.', + { mark_unread: { type: 'boolean', default: false } }, +); +const CATEGORIES = [ + 'primary', + 'newsletter', + 'promotion', + 'automated', + 'social', +]; +export const setCategoryDef = singleMessageDef( + 'set_category', + 'Set the category of one explicit message.', + { category: { type: 'string', enum: CATEGORIES } }, + ['category'], +); +export const gtdClassifyDef = singleMessageDef( + 'gtd_classify', + 'Apply or remove one GTD state label from an explicit message.', + { + state: { type: 'string', enum: GTD_STATES }, + remove: { type: 'boolean', default: false }, + }, + ['state'], +); +export const gtdDoneDef = singleMessageDef( + 'gtd_done', + 'Remove GTD state labels and archive the Inbox copy of one explicit message.', + { + states: { + type: 'array', + items: { type: 'string', enum: GTD_STATES }, + minItems: 1, + }, + }, +); + +function folderWire(row) { + return { + path: row.path, + name: row.name, + delimiter: row.delimiter, + special_use: row.special_use, + total_count: Number(row.total_count || 0), + unread_count: Number(row.unread_count || 0), + message_count: Number(row.total_count || 0), + }; +} + +async function scopedAccountIds(accountId, scope) { + if (!accountId) return scope.accountIds; + const account = await getAccountRow(accountId, scope.accountIds); + return account ? [account.id] : null; +} + +async function requireAccount(accountId, scope) { + if (!accountId || typeof accountId !== 'string') { + return { error: 'account parameter is required' }; + } + const account = await getAccountRow(accountId, scope.accountIds); + if (!account) return { error: `account not found: ${accountId}` }; + return { account }; +} + +function serviceError(result) { + return errorResult(result.error || 'Mailbox operation failed'); +} + +export async function handleListFolders(args, scope, deps) { + const accountIds = await scopedAccountIds(args.account, scope); + if (!accountIds) return errorResult(`account not found: ${args.account}`); + + const results = await Promise.all(accountIds.map(accountId => listFolders(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId, + }))); + const failed = results.find(result => !result.ok); + if (failed) return errorResult(failed.error); + return jsonResult({ folders: results.flatMap(result => result.folders.map(folderWire)) }); +} + +export async function handleCreateFolder(args, scope, deps) { + if (!isValidFolderName(args.name?.trim())) return errorResult('Invalid folder name'); + if (args.parent_path !== undefined && !isValidFolderName(args.parent_path)) { + return errorResult('Invalid parent folder path'); + } + const scoped = await requireAccount(args.account, scope); + if (scoped.error) return errorResult(scoped.error); + const result = await createFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + name: args.name.trim(), + parentPath: args.parent_path, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, path: result.path }); +} + +export async function handleRenameFolder(args, scope, deps) { + if (!isValidFolderName(args.path)) return errorResult('Invalid folder path'); + if (!isValidFolderName(args.new_name?.trim())) return errorResult('Invalid folder name'); + const scoped = await requireAccount(args.account, scope); + if (scoped.error) return errorResult(scoped.error); + const result = await renameFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + oldPath: args.path, + newName: args.new_name.trim(), + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, old_path: args.path, new_path: result.newPath }); +} + +export async function handleDeleteFolder(args, scope, deps) { + if (!isValidFolderName(args.path)) return errorResult('Invalid folder path'); + if (!Number.isInteger(args.expected_message_count) || args.expected_message_count < 0) { + return errorResult('expected_message_count must be a non-negative integer'); + } + const scoped = await requireAccount(args.account, scope); + if (scoped.error) return errorResult(scoped.error); + + const listed = await listFolders(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + }); + if (!listed.ok) return serviceError(listed); + if (!listed.folders.some(folder => folder.path === args.path)) { + return errorResult(`folder not found: ${args.path}`); + } + + const count = await countMessagesIn(scoped.account.id, args.path); + if (args.expected_message_count !== count) { + return errorResult( + `folder "${args.path}" holds ${count} messages, not ${args.expected_message_count}; ` + + 're-check with list_folders and pass the current count to confirm', + ); + } + + const result = await deleteFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + path: args.path, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, deleted: args.path, message_count: count }); +} + +export async function handleMoveMessages(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + if (!isValidFolderName(args.folder)) return errorResult('Invalid destination folder'); + + const result = await bulkMoveToFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + folder: args.folder, + }); + if (!result.ok) return serviceError(result); + + const movedDetails = result.movedDetails || []; + const failed = result.failed || []; + const skippedAccounts = result.skippedAccounts || []; + if (movedDetails.length === 0 && failed.length === 0 && skippedAccounts.length > 0) { + const accountIds = skippedAccounts.map(item => item.account_id).join(', '); + return errorResult( + `destination folder not found for account ${accountIds}: ${args.folder}`, + ); + } + + const resolvedByAccount = new Map(); + for (const detail of movedDetails) { + if (detail.uid == null) continue; + if (!resolvedByAccount.has(detail.accountId)) resolvedByAccount.set(detail.accountId, []); + resolvedByAccount.get(detail.accountId).push(detail.uid); + } + const newIds = new Map(); + await Promise.all([...resolvedByAccount].map(async ([accountId, uids]) => { + const rows = await resolveMovedIds(accountId, args.folder, uids); + for (const row of rows) newIds.set(`${accountId}:${String(row.uid)}`, row.id); + })); + + const moved = movedDetails.map(detail => ({ + id: detail.id, + new_id: detail.uid == null + ? null + : (newIds.get(`${detail.accountId}:${String(detail.uid)}`) || null), + uid: detail.uid, + folder: args.folder, + })); + const resyncPending = movedDetails.some((detail, index) => ( + detail.uid == null || moved[index].new_id == null + )); + return jsonResult({ + ok: true, + moved, + failed, + skipped_accounts: skippedAccounts, + resync_pending: resyncPending, + note: 'message ids change on move; use new_id for follow-up calls', + }); +} + +async function resolveDestinationIds(details) { + const grouped = new Map(); + for (const detail of details) { + if (detail.uid == null || detail.destinationUntracked) continue; + const key = `${detail.accountId}:${detail.folder}`; + if (!grouped.has(key)) { + grouped.set(key, { + accountId: detail.accountId, + folder: detail.folder, + uids: [], + }); + } + grouped.get(key).uids.push(detail.uid); + } + + const resolved = new Map(); + await Promise.all([...grouped.values()].map(async ({ accountId, folder, uids }) => { + const rows = await resolveMovedIds(accountId, folder, uids); + for (const row of rows) { + resolved.set(`${accountId}:${folder}:${String(row.uid)}`, row.id); + } + })); + return resolved; +} + +export async function handleArchiveMessages(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + + const result = await bulkArchive(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + }); + if (!result.ok) return serviceError(result); + + const details = result.archivedDetails || []; + const resolved = await resolveDestinationIds(details); + const archived = details.map(detail => ({ + id: detail.id, + new_id: detail.destinationUntracked || detail.uid == null + ? null + : (resolved.get( + `${detail.accountId}:${detail.folder}:${String(detail.uid)}`, + ) || null), + uid: detail.uid, + folder: detail.folder, + destination_untracked: Boolean(detail.destinationUntracked), + })); + const resyncPending = details.some((detail, index) => ( + !detail.destinationUntracked + && (detail.uid == null || archived[index].new_id == null) + )); + + return jsonResult({ + ok: true, + archived, + failed: result.failed || [], + no_archive_folder: result.noArchiveFolder || [], + resync_pending: resyncPending, + note: 'message ids change on archive; use new_id for follow-up calls', + }); +} + +export async function handleTrashMessages(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + + const result = await bulkTrash(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + allowPermanent: false, + }); + if (!result.ok) return serviceError(result); + + const details = result.trashedDetails || []; + const resolved = await resolveDestinationIds(details); + const trashed = details.map(detail => ({ + id: detail.id, + new_id: detail.uid == null + ? null + : (resolved.get( + `${detail.accountId}:${detail.folder}:${String(detail.uid)}`, + ) || null), + folder: detail.folder, + })); + const resyncPending = details.some((detail, index) => ( + detail.uid == null || trashed[index].new_id == null + )); + + return jsonResult({ + ok: true, + trashed, + failed: result.failed || [], + refused: result.refused || [], + resync_pending: resyncPending, + next_step: 'use stage_deletion for permanent removal', + }); +} + +function createFlagHandler({ kind, value }) { + return async function handleFlag(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + + if (kind === 'read') { + const result = await bulkSetRead(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + read: value, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, updated: result.updated || [] }); + } + + const results = await runInBatches(ids.value, 3, id => setStarred( + deps.imapManager, + { + userId: scope.userId, + accountIds: scope.accountIds, + id, + starred: value, + }, + )); + const updated = []; + results.forEach((result, index) => { + if ( + result.status === 'fulfilled' + && result.value.ok + && result.value.updated + ) { + updated.push(ids.value[index]); + } + }); + return jsonResult({ ok: true, updated }); + }; +} + +export const handleMarkRead = createFlagHandler({ kind: 'read', value: true }); +export const handleMarkUnread = createFlagHandler({ kind: 'read', value: false }); +export const handleStarMessage = createFlagHandler({ kind: 'starred', value: true }); +export const handleUnstarMessage = createFlagHandler({ kind: 'starred', value: false }); + +function createSpamHandler(service) { + return async function handleSpamLabel(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + + const result = await service(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + folder: result.body.folder, + new_uid: result.body.newUid ?? null, + already_in_folder: Boolean(result.body.alreadyInFolder), + }); + }; +} + +export const handleMarkSpam = createSpamHandler(markSpam); +export const handleMarkNotSpam = createSpamHandler(markNotSpam); + +export async function handleSnoozeMessage(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if (!args.until) return errorResult('until is required'); + + const until = new Date(args.until); + if (Number.isNaN(until.getTime())) { + return errorResult('until must be a valid ISO date'); + } + const now = Date.now(); + if (until.getTime() <= now) return errorResult('until must be in the future'); + if (until.getTime() > now + 30 * 86_400_000) { + return errorResult('until must be within 30 days'); + } + + const result = await snoozeConversation(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + until, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + moved_count: result.movedCount, + sibling_ids: (result.movedIds || []).filter(movedId => movedId !== id.value), + folder: result.folder, + }); +} + +export async function handleUnsnoozeMessage(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if ( + args.mark_unread !== undefined + && typeof args.mark_unread !== 'boolean' + ) { + return errorResult('mark_unread must be a boolean'); + } + + const result = await unsnoozeConversation(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + markUnread: args.mark_unread ?? false, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + restored: result.restored, + folder: result.folder, + }); +} + +export async function handleSetCategory(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if (!CATEGORIES.includes(args.category)) return errorResult('Invalid category'); + + const result = await setCategory(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + category: args.category, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, category: result.category }); +} + +export async function handleGtdClassify(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if (!GTD_STATES.includes(args.state)) { + return errorResult(`Unknown GTD state: ${args.state}`); + } + if (args.remove !== undefined && typeof args.remove !== 'boolean') { + return errorResult('remove must be a boolean'); + } + + const remove = args.remove ?? false; + const service = remove ? gtdUnclassify : gtdClassify; + const result = await service(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + messageId: id.value, + state: args.state, + }); + if (!result.ok) return serviceError(result); + + const receipt = { + ok: true, + state: args.state, + folder: result.folder ?? null, + }; + if (remove) receipt.removed = Boolean(result.removed); + return jsonResult(receipt); +} + +export async function handleGtdDone(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + + let states = 'all'; + if (args.states !== undefined) { + if (!Array.isArray(args.states) || args.states.length === 0) { + return errorResult('states must be a non-empty array'); + } + const unknown = args.states.find(state => !GTD_STATES.includes(state)); + if (unknown) return errorResult(`Unknown GTD state: ${unknown}`); + states = args.states; + } + + const result = await gtdDone(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + states, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + removed: result.removed, + archived: result.archived, + no_archive_folder: result.noArchiveFolder, + archive_failed: result.archiveFailed, + }); +} diff --git a/backend/src/mcp/mailboxTools.test.js b/backend/src/mcp/mailboxTools.test.js new file mode 100644 index 00000000..6d823d54 --- /dev/null +++ b/backend/src/mcp/mailboxTools.test.js @@ -0,0 +1,1146 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', async (orig) => { + const actual = await orig(); + return { ...actual, getAccountRow: vi.fn() }; +}); +vi.mock('../services/mailbox/folders.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + listFolders: vi.fn(), + createFolder: vi.fn(), + renameFolder: vi.fn(), + deleteFolder: vi.fn(), + countMessagesIn: vi.fn(), + }; +}); +vi.mock('../services/mailbox/move.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + bulkMoveToFolder: vi.fn(), + resolveMovedIds: vi.fn(), + }; +}); +vi.mock('../services/mailbox/archive.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkArchive: vi.fn() }; +}); +vi.mock('../services/mailbox/trash.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkTrash: vi.fn() }; +}); +vi.mock('../services/mailbox/flags.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + bulkSetRead: vi.fn(), + setStarred: vi.fn(), + }; +}); +vi.mock('../services/mailbox/batch.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + runInBatches: vi.fn(actual.runInBatches), + }; +}); +vi.mock('../services/mailbox/spamLabel.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + markSpam: vi.fn(), + markNotSpam: vi.fn(), + }; +}); +vi.mock('../services/mailbox/snooze.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + snoozeConversation: vi.fn(), + unsnoozeConversation: vi.fn(), + }; +}); +vi.mock('../services/mailbox/category.js', async (orig) => { + const actual = await orig(); + return { ...actual, setCategory: vi.fn() }; +}); +vi.mock('../services/gtd/actions.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + gtdClassify: vi.fn(), + gtdUnclassify: vi.fn(), + gtdDone: vi.fn(), + }; +}); + +import { getAccountRow } from './accountAdapter.js'; +import { + countMessagesIn, + createFolder, + deleteFolder, + listFolders, + renameFolder, +} from '../services/mailbox/folders.js'; +import { + bulkMoveToFolder, + resolveMovedIds, +} from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { bulkSetRead, setStarred } from '../services/mailbox/flags.js'; +import { runInBatches } from '../services/mailbox/batch.js'; +import { markNotSpam, markSpam } from '../services/mailbox/spamLabel.js'; +import { + snoozeConversation, + unsnoozeConversation, +} from '../services/mailbox/snooze.js'; +import { setCategory } from '../services/mailbox/category.js'; +import { + gtdClassify, + gtdDone, + gtdUnclassify, +} from '../services/gtd/actions.js'; +import * as mailboxTools from './mailboxTools.js'; +import { handleListFolders } from './mailboxTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const scope = { userId: 'user-1', accountIds: [ACCOUNT_ID] }; +const deps = { imapManager: { marker: 'imap' } }; + +beforeEach(() => { + vi.clearAllMocks(); + getAccountRow.mockResolvedValue({ id: ACCOUNT_ID }); +}); + +describe('mailbox tool definitions and registration', () => { + it('registers list_folders as a read tool', () => { + expect(TOOL_DEFS.map(def => def.name)).toContain('list_folders'); + expect(TOOL_SCOPES.list_folders).toBe('read'); + expect(HANDLERS.list_folders).toBeTypeOf('function'); + }); + + it('registers all folder mutations with write scope', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, required] of [ + ['create_folder', ['account', 'name']], + ['rename_folder', ['account', 'path', 'new_name']], + ['delete_folder', ['account', 'path', 'expected_message_count']], + ]) { + expect(defs.get(name)?.inputSchema.required).toEqual(required); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers move_messages with explicit bulk ids and destination folder', () => { + const def = TOOL_DEFS.find(entry => entry.name === 'move_messages'); + expect(def.inputSchema.required).toEqual(['message_ids', 'folder']); + expect(def.inputSchema.properties.message_ids).toEqual({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, + }); + expect(def.inputSchema.properties).not.toHaveProperty('label'); + expect(TOOL_SCOPES.move_messages).toBe('write'); + expect(HANDLERS.move_messages).toBeTypeOf('function'); + }); + + it('registers archive_messages and trash_messages without a delete_messages alias', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of ['archive_messages', 'trash_messages']) { + expect(defs.get(name)?.inputSchema.required).toEqual(['message_ids']); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + expect(defs.has('delete_messages')).toBe(false); + expect(HANDLERS).not.toHaveProperty('delete_messages'); + }); + + it('registers all four message flag tools with explicit bulk ids and write scope', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of [ + 'mark_read', + 'mark_unread', + 'star_message', + 'unstar_message', + ]) { + expect(defs.get(name)?.inputSchema.required).toEqual(['message_ids']); + expect(defs.get(name)?.inputSchema.properties.message_ids).toEqual({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, + }); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers mark_spam and mark_not_spam with one explicit message id', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of ['mark_spam', 'mark_not_spam']) { + expect(defs.get(name)?.inputSchema.required).toEqual(['message_id']); + expect(defs.get(name)?.inputSchema.properties.message_id).toEqual({ + type: 'string', + }); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers snooze_message and unsnooze_message with their explicit arguments', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + expect(defs.get('snooze_message')?.inputSchema.required).toEqual([ + 'message_id', + 'until', + ]); + expect(defs.get('unsnooze_message')?.inputSchema.required).toEqual([ + 'message_id', + ]); + expect( + defs.get('unsnooze_message')?.inputSchema.properties.mark_unread, + ).toEqual({ type: 'boolean', default: false }); + for (const name of ['snooze_message', 'unsnooze_message']) { + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers set_category, gtd_classify, and gtd_done with write scope', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + expect(defs.get('set_category')?.inputSchema.required).toEqual([ + 'message_id', + 'category', + ]); + expect(defs.get('gtd_classify')?.inputSchema.required).toEqual([ + 'message_id', + 'state', + ]); + expect(defs.get('gtd_done')?.inputSchema.required).toEqual(['message_id']); + for (const name of ['set_category', 'gtd_classify', 'gtd_done']) { + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); +}); + +describe('mailbox tool metadata completeness', () => { + const expected = { + list_folders: [true, false, true, false], + create_folder: [false, false, false, false], + rename_folder: [false, true, false, false], + delete_folder: [false, true, true, false], + move_messages: [false, true, false, false], + archive_messages: [false, false, true, false], + trash_messages: [false, true, true, false], + mark_read: [false, false, true, false], + mark_unread: [false, false, true, false], + star_message: [false, false, true, false], + unstar_message: [false, false, true, false], + mark_spam: [false, false, true, false], + mark_not_spam: [false, false, true, false], + snooze_message: [false, false, true, false], + unsnooze_message: [false, false, true, false], + set_category: [false, false, true, false], + gtd_classify: [false, false, true, false], + gtd_done: [false, false, true, false], + }; + + it('pins all 18 mailbox annotation rows from the protocol table', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + expect(Object.keys(expected)).toHaveLength(18); + for (const [name, [ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint, + ]] of Object.entries(expected)) { + expect(defs.get(name)?.annotations).toEqual({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint, + }); + } + }); + + it('gives every mailbox write definition a scope and four boolean hints', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of Object.keys(expected).filter(name => name !== 'list_folders')) { + expect(TOOL_SCOPES[name]).toBe('write'); + const annotations = defs.get(name)?.annotations; + expect(Object.keys(annotations || {}).sort()).toEqual([ + 'destructiveHint', + 'idempotentHint', + 'openWorldHint', + 'readOnlyHint', + ]); + expect(Object.values(annotations).every(value => typeof value === 'boolean')).toBe(true); + } + }); +}); + +describe('shared mailbox validators', () => { + it('enforces explicit 1-500 UUID message_ids arrays', () => { + expect(mailboxTools.messageIdsArg({})).toEqual({ + error: 'message_ids must contain at least one id', + }); + expect(mailboxTools.messageIdsArg({ message_ids: [] })).toEqual({ + error: 'message_ids must contain at least one id', + }); + expect(mailboxTools.messageIdsArg({ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + })).toEqual({ error: 'Too many ids — maximum 500 per request' }); + expect(mailboxTools.messageIdsArg({ message_ids: ['not-a-uuid'] })).toEqual({ + error: 'Invalid message id format', + }); + expect(mailboxTools.messageIdsArg({ message_ids: [ACCOUNT_ID] })).toEqual({ + value: [ACCOUNT_ID], + }); + }); + + it('requires a single UUID message_id', () => { + expect(mailboxTools.messageIdArg({})).toEqual({ + error: 'message_id parameter is required', + }); + expect(mailboxTools.messageIdArg({ message_id: 'not-a-uuid' })).toEqual({ + error: 'Invalid message id format', + }); + expect(mailboxTools.messageIdArg({ message_id: ACCOUNT_ID })).toEqual({ + value: ACCOUNT_ID, + }); + }); +}); + +describe('list_folders', () => { + it('scope-checks the account and projects the curated folder shape', async () => { + listFolders.mockResolvedValue({ + ok: true, + folders: [{ + account_id: ACCOUNT_ID, + path: 'INBOX', + name: 'Inbox', + delimiter: '/', + special_use: '\\Inbox', + total_count: 9, + unread_count: 2, + ignored_column: 'not-on-wire', + }], + }); + + const result = await handleListFolders({ account: ACCOUNT_ID }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + folders: [{ + path: 'INBOX', + name: 'Inbox', + delimiter: '/', + special_use: '\\Inbox', + total_count: 9, + unread_count: 2, + message_count: 9, + }], + }); + expect(getAccountRow).toHaveBeenCalledWith(ACCOUNT_ID, scope.accountIds); + expect(listFolders).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + }); + }); + + it('rejects an out-of-scope account before calling the folder service', async () => { + getAccountRow.mockResolvedValue(null); + + const result = await handleListFolders({ account: 'outside' }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: outside'); + expect(listFolders).not.toHaveBeenCalled(); + }); +}); + +describe('folder mutations', () => { + it('creates a scoped folder and returns the resolved path', async () => { + createFolder.mockResolvedValue({ ok: true, path: 'Projects/Client' }); + + const result = await mailboxTools.handleCreateFolder({ + account: ACCOUNT_ID, + name: 'Client', + parent_path: 'Projects', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + path: 'Projects/Client', + }); + expect(createFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + name: 'Client', + parentPath: 'Projects', + }); + }); + + it('rejects an invalid folder name before calling the service', async () => { + const result = await mailboxTools.handleCreateFolder({ + account: ACCOUNT_ID, + name: 'bad\u0000name', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Invalid folder name'); + expect(createFolder).not.toHaveBeenCalled(); + }); + + it('renames the final folder component and names both paths in the receipt', async () => { + renameFolder.mockResolvedValue({ ok: true, newPath: 'Projects/New' }); + + const result = await mailboxTools.handleRenameFolder({ + account: ACCOUNT_ID, + path: 'Projects/Old', + new_name: 'New', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + old_path: 'Projects/Old', + new_path: 'Projects/New', + }); + expect(renameFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + oldPath: 'Projects/Old', + newName: 'New', + }); + }); + + it('refuses delete_folder when the live count differs from confirmation', async () => { + listFolders.mockResolvedValue({ + ok: true, + folders: [{ path: 'Projects', total_count: 7 }], + }); + countMessagesIn.mockResolvedValue(7); + + const result = await mailboxTools.handleDeleteFolder({ + account: ACCOUNT_ID, + path: 'Projects', + expected_message_count: 3, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'folder "Projects" holds 7 messages, not 3; re-check with list_folders and pass the current count to confirm', + ); + expect(getAccountRow).toHaveBeenCalledWith(ACCOUNT_ID, scope.accountIds); + expect(countMessagesIn).toHaveBeenCalledWith(ACCOUNT_ID, 'Projects'); + expect(deleteFolder).not.toHaveBeenCalled(); + }); + + it('checks folder existence before reading its live count', async () => { + listFolders.mockResolvedValue({ ok: true, folders: [] }); + + const result = await mailboxTools.handleDeleteFolder({ + account: ACCOUNT_ID, + path: 'Missing', + expected_message_count: 0, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('folder not found: Missing'); + expect(countMessagesIn).not.toHaveBeenCalled(); + expect(deleteFolder).not.toHaveBeenCalled(); + }); + + it('deletes only after the scope, existence, and count guards pass', async () => { + listFolders.mockResolvedValue({ + ok: true, + folders: [{ path: 'Projects', total_count: 2 }], + }); + countMessagesIn.mockResolvedValue(2); + deleteFolder.mockResolvedValue({ ok: true }); + + const result = await mailboxTools.handleDeleteFolder({ + account: ACCOUNT_ID, + path: 'Projects', + expected_message_count: 2, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + deleted: 'Projects', + message_count: 2, + }); + expect(deleteFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + path: 'Projects', + }); + }); +}); + +describe('move_messages', () => { + const SECOND_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + const NEW_ID = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc'; + + it('rejects malformed ids before calling the move service', async () => { + const result = await mailboxTools.handleMoveMessages({ + message_ids: ['not-a-uuid'], + folder: 'Projects', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Invalid message id format'); + expect(bulkMoveToFolder).not.toHaveBeenCalled(); + }); + + it('resolves UIDPLUS destination ids and reports non-UIDPLUS resyncs', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + moved: [ACCOUNT_ID, SECOND_ID], + movedDetails: [ + { id: ACCOUNT_ID, accountId: ACCOUNT_ID, uid: 110 }, + { id: SECOND_ID, accountId: ACCOUNT_ID, uid: null }, + ], + failed: [], + skippedAccounts: [], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleMoveMessages({ + message_ids: [ACCOUNT_ID, SECOND_ID], + folder: 'Projects', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + moved: [ + { id: ACCOUNT_ID, new_id: NEW_ID, uid: 110, folder: 'Projects' }, + { id: SECOND_ID, new_id: null, uid: null, folder: 'Projects' }, + ], + failed: [], + skipped_accounts: [], + resync_pending: true, + note: 'message ids change on move; use new_id for follow-up calls', + }); + expect(bulkMoveToFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ACCOUNT_ID, SECOND_ID], + folder: 'Projects', + }); + expect(resolveMovedIds).toHaveBeenCalledWith(ACCOUNT_ID, 'Projects', [110]); + }); + + it('turns a fully skipped destination into an error', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + moved: [], + movedDetails: [], + failed: [], + skippedAccounts: [{ + account_id: ACCOUNT_ID, + reason: 'folder_not_found', + }], + }); + + const result = await mailboxTools.handleMoveMessages({ + message_ids: [ACCOUNT_ID], + folder: 'Missing', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + `destination folder not found for account ${ACCOUNT_ID}: Missing`, + ); + expect(resolveMovedIds).not.toHaveBeenCalled(); + }); + + it('keeps failed and skipped partitions on a partial success receipt', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + moved: [ACCOUNT_ID], + movedDetails: [{ id: ACCOUNT_ID, accountId: ACCOUNT_ID, uid: 110 }], + failed: [{ id: SECOND_ID, reason: 'IMAP move failed' }], + skippedAccounts: [{ account_id: 'other-account', reason: 'folder_not_found' }], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleMoveMessages({ + message_ids: [ACCOUNT_ID, SECOND_ID], + folder: 'Projects', + }, scope, deps); + + const receipt = JSON.parse(result.content[0].text); + expect(result.isError).toBeUndefined(); + expect(receipt.failed).toEqual([{ id: SECOND_ID, reason: 'IMAP move failed' }]); + expect(receipt.skipped_accounts).toEqual([ + { account_id: 'other-account', reason: 'folder_not_found' }, + ]); + }); +}); + +describe('archive_messages', () => { + const NEW_ID = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc'; + const ALL_MAIL_ID = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'; + + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + ])('rejects invalid explicit ids before calling the archive service', async (args, error) => { + const result = await mailboxTools.handleArchiveMessages(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(bulkArchive).not.toHaveBeenCalled(); + }); + + it('returns durable ids and marks Gmail All Mail destinations as untracked', async () => { + bulkArchive.mockResolvedValue({ + ok: true, + archived: [ACCOUNT_ID, ALL_MAIL_ID], + archivedDetails: [ + { + id: ACCOUNT_ID, + accountId: ACCOUNT_ID, + folder: 'Archive', + uid: 110, + destinationUntracked: false, + }, + { + id: ALL_MAIL_ID, + accountId: ACCOUNT_ID, + folder: '[Gmail]/All Mail', + uid: 111, + destinationUntracked: true, + }, + ], + failed: [], + noArchiveFolder: ['account-without-archive'], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleArchiveMessages({ + message_ids: [ACCOUNT_ID, ALL_MAIL_ID], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + archived: [ + { + id: ACCOUNT_ID, + new_id: NEW_ID, + uid: 110, + folder: 'Archive', + destination_untracked: false, + }, + { + id: ALL_MAIL_ID, + new_id: null, + uid: 111, + folder: '[Gmail]/All Mail', + destination_untracked: true, + }, + ], + failed: [], + no_archive_folder: ['account-without-archive'], + resync_pending: false, + note: 'message ids change on archive; use new_id for follow-up calls', + }); + expect(bulkArchive).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ACCOUNT_ID, ALL_MAIL_ID], + }); + expect(resolveMovedIds).toHaveBeenCalledWith(ACCOUNT_ID, 'Archive', [110]); + }); +}); + +describe('trash_messages', () => { + const NEW_ID = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc'; + + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + ])('rejects invalid explicit ids before calling the trash service', async (args, error) => { + const result = await mailboxTools.handleTrashMessages(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(bulkTrash).not.toHaveBeenCalled(); + }); + + it('never enables permanent deletion and preserves the refusal partition', async () => { + bulkTrash.mockResolvedValue({ + ok: true, + deleted: [ACCOUNT_ID], + trashedDetails: [{ + id: ACCOUNT_ID, + accountId: ACCOUNT_ID, + folder: 'Trash', + uid: 110, + }], + failed: [], + refused: [{ + id: 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleTrashMessages({ + message_ids: [ + ACCOUNT_ID, + 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + ], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + trashed: [{ + id: ACCOUNT_ID, + new_id: NEW_ID, + folder: 'Trash', + }], + failed: [], + refused: [{ + id: 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + resync_pending: false, + next_step: 'use stage_deletion for permanent removal', + }); + expect(bulkTrash).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ + ACCOUNT_ID, + 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + ], + allowPermanent: false, + }); + }); +}); + +describe('mark_read and mark_unread', () => { + it.each([ + ['handleMarkRead', true], + ['handleMarkUnread', false], + ])('returns only ids changed by %s', async (handlerName, read) => { + bulkSetRead.mockResolvedValue({ ok: true, updated: [ACCOUNT_ID] }); + + const result = await mailboxTools[handlerName]({ + message_ids: [ACCOUNT_ID], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + updated: [ACCOUNT_ID], + }); + expect(bulkSetRead).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ACCOUNT_ID], + read, + }); + }); +}); + +describe('star_message and unstar_message', () => { + const SECOND_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + + it.each([ + ['handleStarMessage', true], + ['handleUnstarMessage', false], + ])('runs %s through bounded single-message writes and excludes no-ops', async ( + handlerName, + starred, + ) => { + setStarred + .mockResolvedValueOnce({ ok: true, is_starred: starred, updated: true }) + .mockResolvedValueOnce({ ok: true, is_starred: starred, updated: false }); + + const result = await mailboxTools[handlerName]({ + message_ids: [ACCOUNT_ID, SECOND_ID], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + updated: [ACCOUNT_ID], + }); + expect(runInBatches).toHaveBeenCalledWith( + [ACCOUNT_ID, SECOND_ID], + 3, + expect.any(Function), + ); + expect(setStarred).toHaveBeenNthCalledWith(1, deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + starred, + }); + expect(setStarred).toHaveBeenNthCalledWith(2, deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: SECOND_ID, + starred, + }); + }); +}); + +describe.each([ + ['handleMarkRead', bulkSetRead], + ['handleMarkUnread', bulkSetRead], + ['handleStarMessage', setStarred], + ['handleUnstarMessage', setStarred], +])('%s explicit-id validation', (handlerName, service) => { + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + ])('rejects invalid ids before any service call', async (args, error) => { + const result = await mailboxTools[handlerName](args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(service).not.toHaveBeenCalled(); + }); +}); + +describe('mark_spam and mark_not_spam', () => { + it.each([ + ['handleMarkSpam', markSpam, 'Junk', 110], + ['handleMarkNotSpam', markNotSpam, 'INBOX', null], + ])('%s forwards scope and projects the destination receipt', async ( + handlerName, + service, + folder, + newUid, + ) => { + service.mockResolvedValue({ + ok: true, + status: 200, + body: { + ok: true, + folder, + newUid, + alreadyInFolder: false, + }, + }); + + const result = await mailboxTools[handlerName]({ + message_id: ACCOUNT_ID, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + folder, + new_uid: newUid, + already_in_folder: false, + }); + expect(service).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + }); + }); + + it.each([ + ['handleMarkSpam', markSpam], + ['handleMarkNotSpam', markNotSpam], + ])('%s rejects a missing or malformed id before the service call', async ( + handlerName, + service, + ) => { + for (const [args, error] of [ + [{}, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid' }, 'Invalid message id format'], + ]) { + const result = await mailboxTools[handlerName](args, scope, deps); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + } + expect(service).not.toHaveBeenCalled(); + }); + + it('preserves a spam service refusal as an MCP error', async () => { + markSpam.mockResolvedValue({ + ok: false, + status: 422, + error: 'No spam folder configured for this account', + }); + + const result = await mailboxTools.handleMarkSpam({ + message_id: ACCOUNT_ID, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'No spam folder configured for this account', + ); + }); +}); + +describe('snooze_message', () => { + it('reports the whole-conversation move count and sibling ids', async () => { + const siblingId = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + snoozeConversation.mockResolvedValue({ + ok: true, + movedCount: 2, + movedIds: [ACCOUNT_ID, siblingId], + folder: 'Snoozed', + }); + const until = new Date(Date.now() + 60_000).toISOString(); + + const result = await mailboxTools.handleSnoozeMessage({ + message_id: ACCOUNT_ID, + until, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + moved_count: 2, + sibling_ids: [siblingId], + folder: 'Snoozed', + }); + expect(snoozeConversation).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + until: new Date(until), + }); + }); + + it.each([ + [{ until: new Date(Date.now() + 60_000).toISOString() }, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid', until: new Date(Date.now() + 60_000).toISOString() }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID }, 'until is required'], + [{ message_id: ACCOUNT_ID, until: 'not-a-date' }, 'until must be a valid ISO date'], + [{ message_id: ACCOUNT_ID, until: new Date(Date.now() - 60_000).toISOString() }, 'until must be in the future'], + [{ message_id: ACCOUNT_ID, until: new Date(Date.now() + 31 * 86_400_000).toISOString() }, 'until must be within 30 days'], + ])('rejects invalid arguments before calling the snooze service', async (args, error) => { + const result = await mailboxTools.handleSnoozeMessage(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(snoozeConversation).not.toHaveBeenCalled(); + }); +}); + +describe('unsnooze_message', () => { + it.each([ + [undefined, false], + [true, true], + ])('restores the conversation with mark_unread=%s', async ( + markUnreadArg, + markUnread, + ) => { + unsnoozeConversation.mockResolvedValue({ + ok: true, + restored: 2, + folder: 'INBOX', + }); + const args = { message_id: ACCOUNT_ID }; + if (markUnreadArg !== undefined) args.mark_unread = markUnreadArg; + + const result = await mailboxTools.handleUnsnoozeMessage(args, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + restored: 2, + folder: 'INBOX', + }); + expect(unsnoozeConversation).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + markUnread, + }); + }); + + it.each([ + [{}, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID, mark_unread: 'yes' }, 'mark_unread must be a boolean'], + ])('rejects invalid arguments before calling the unsnooze service', async (args, error) => { + const result = await mailboxTools.handleUnsnoozeMessage(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(unsnoozeConversation).not.toHaveBeenCalled(); + }); +}); + +describe('set_category', () => { + it('sets one validated category in the enabled-account scope', async () => { + setCategory.mockResolvedValue({ ok: true, category: 'newsletter' }); + + const result = await mailboxTools.handleSetCategory({ + message_id: ACCOUNT_ID, + category: 'newsletter', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + category: 'newsletter', + }); + expect(setCategory).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + category: 'newsletter', + }); + }); + + it.each([ + [{ category: 'primary' }, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid', category: 'primary' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID }, 'Invalid category'], + [{ message_id: ACCOUNT_ID, category: 'other' }, 'Invalid category'], + ])('rejects invalid arguments before the category service', async (args, error) => { + const result = await mailboxTools.handleSetCategory(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(setCategory).not.toHaveBeenCalled(); + }); +}); + +describe('gtd_classify', () => { + it.each([ + [false, gtdClassify, { ok: true, folder: 'Todo' }, { + ok: true, + state: 'todo', + folder: 'Todo', + }], + [true, gtdUnclassify, { + ok: true, + removed: true, + folder: 'Todo', + }, { + ok: true, + state: 'todo', + folder: 'Todo', + removed: true, + }], + ])('dispatches remove=%s to the correct scoped GTD action', async ( + remove, + service, + serviceResult, + receipt, + ) => { + service.mockResolvedValue(serviceResult); + + const result = await mailboxTools.handleGtdClassify({ + message_id: ACCOUNT_ID, + state: 'todo', + remove, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual(receipt); + expect(service).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + messageId: ACCOUNT_ID, + state: 'todo', + }); + }); + + it.each([ + [{ state: 'todo' }, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid', state: 'todo' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID }, 'Unknown GTD state: undefined'], + [{ message_id: ACCOUNT_ID, state: 'inbox' }, 'Unknown GTD state: inbox'], + [{ message_id: ACCOUNT_ID, state: 'todo', remove: 'yes' }, 'remove must be a boolean'], + ])('rejects invalid arguments before either GTD classify service', async (args, error) => { + const result = await mailboxTools.handleGtdClassify(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(gtdClassify).not.toHaveBeenCalled(); + expect(gtdUnclassify).not.toHaveBeenCalled(); + }); +}); + +describe('gtd_done', () => { + it('preserves archive failure as a non-error partial-success receipt', async () => { + gtdDone.mockResolvedValue({ + ok: true, + removed: ['Watch'], + archived: false, + noArchiveFolder: false, + archiveFailed: true, + }); + + const result = await mailboxTools.handleGtdDone({ + message_id: ACCOUNT_ID, + states: ['watch'], + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + removed: ['Watch'], + archived: false, + no_archive_folder: false, + archive_failed: true, + }); + expect(gtdDone).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + states: ['watch'], + }); + }); + + it('defaults omitted states to all', async () => { + gtdDone.mockResolvedValue({ + ok: true, + removed: [], + archived: false, + noArchiveFolder: true, + archiveFailed: false, + }); + + await mailboxTools.handleGtdDone({ message_id: ACCOUNT_ID }, scope, deps); + + expect(gtdDone).toHaveBeenCalledWith( + deps.imapManager, + expect.objectContaining({ states: 'all' }), + ); + }); + + it.each([ + [{}, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID, states: [] }, 'states must be a non-empty array'], + [{ message_id: ACCOUNT_ID, states: ['inbox'] }, 'Unknown GTD state: inbox'], + ])('rejects invalid arguments before gtd_done', async (args, error) => { + const result = await mailboxTools.handleGtdDone(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(gtdDone).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/mcp/messageTools.js b/backend/src/mcp/messageTools.js index c5246f41..96cc186f 100644 --- a/backend/src/mcp/messageTools.js +++ b/backend/src/mcp/messageTools.js @@ -19,6 +19,25 @@ import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; // phase 3 import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; // phase-5-owned +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); +const NON_IDEMPOTENT_WRITE_ANNOTATIONS = annotations(); + const DEFAULT_BODY_CHARS = 2000; const MAX_BODY_CHARS = 4000; const MAX_LIMIT = 1000; @@ -59,6 +78,7 @@ export const getMessageDef = { 'To read sequentially: call again with offset += body_returned. ' + 'To jump to a known match location: use center_at= to center the window on that location. ' + 'Note: snippet is pre-stored source metadata (may be empty for non-Gmail sources).', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -146,6 +166,7 @@ export const listMessagesDef = { 'there is deliberately no bulk body fetch, to avoid loading huge threads into the context window. ' + 'Paginate with offset/limit (default limit 20, max 50). Response: data, total, returned, offset, has_more. ' + 'total=-1 because the full count is not computed; use has_more for paging.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -189,6 +210,7 @@ export async function handleListMessages(args, scope) { export const getStatsDef = { name: 'get_stats', description: 'Get archive overview: total messages, size, attachment count, and accounts.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: {} }, }; @@ -214,6 +236,7 @@ export const aggregateDef = { description: 'Get grouped statistics (top senders, recipients, domains, labels, or message volume by calendar year). ' + 'Returns a JSON array of objects with fields Key, Count, TotalSize, AttachmentSize, AttachmentCount, and TotalUnique.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -248,6 +271,7 @@ export async function handleAggregate(args, scope) { export const searchByDomainsDef = { name: 'search_by_domains', description: 'Find emails where any participant (from, to, or cc) belongs to one of the given domains. Useful for finding all communication with a company regardless of direction.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -279,6 +303,7 @@ export async function handleSearchByDomains(args, scope) { export const findSimilarMessagesDef = { name: 'find_similar_messages', description: 'Find messages whose embeddings are closest to the given message. Requires vector search to be configured and an active index generation.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -294,6 +319,53 @@ export const findSimilarMessagesDef = { }, }; +function findSimilarError(message) { + const error = new Error(message); + error.name = 'FindSimilarError'; + return error; +} + +export async function findSimilarSummaries(seedId, { + accountIds, + limit = 20, + after, + before, + hasAttachment = false, +} = {}) { + const { generation } = await resolveActiveGenerationFromConfig(); + + // loadVector is not account-aware, so scope membership must be established + // before the vector is loaded (and before its existence can be observed). + if (!(await messageInScope(seedId, accountIds))) { + throw findSimilarError('message not found'); + } + + let seed; + try { + seed = await loadVector(seedId); + } catch (error) { + throw findSimilarError(`load seed vector: ${error.message}`); + } + + const filter = { accountIds }; + if (after) filter.after = after; + if (before) filter.before = before; + if (hasAttachment) filter.hasAttachment = true; + + // annSearch is generation-specific and rank-ordered. Over-fetch one so the + // seed can be removed without shortening the requested result set. + const hits = await annSearch(generation.id, seed, limit + 1, { filter }); + const ids = []; + for (const hit of hits) { + if (hit.messageId === seedId) continue; + if (ids.length >= limit) break; + ids.push(hit.messageId); + } + + const messages = await getMessageSummariesByIDs(ids, accountIds); + return { generation, messages }; +} + export async function handleFindSimilarMessages(args, scope) { const seedId = args.message_id; if (!seedId || typeof seedId !== 'string') return errorResult('message_id parameter is required'); @@ -311,32 +383,14 @@ export async function handleFindSimilarMessages(args, scope) { const messageType = typeof args.message_type === 'string' ? args.message_type.trim().toLowerCase() : ''; try { - const { generation: gen } = await resolveActiveGenerationFromConfig(); - - // Owner-scope the seed: loadVector is not account-aware, so a foreign/unknown - // seed id must behave like get_message on a foreign id (contract: every tool - // call is owner-scoped; also closes an embedding-existence oracle). - if (!(await messageInScope(seedId, acc.accountIds))) return errorResult('message not found'); - - let seed; - try { seed = await loadVector(seedId); } - catch (e) { return errorResult(`load seed vector: ${e.message}`); } - - const filter = { accountIds: acc.accountIds }; - if (after.value) filter.after = after.value; - if (before.value) filter.before = before.value; - if (args.has_attachment === true) filter.hasAttachment = true; - // phase-3 annSearch takes the generation ID (not the object) and returns - // [{messageId, score, rank}] rank-ordered. +1 to drop the seed without coming up short. - const hits = await annSearch(gen.id, seed, limit + 1, { filter }); - - const ids = []; - for (const h of hits) { - if (h.messageId === seedId) continue; - if (ids.length >= limit) break; - ids.push(h.messageId); - } - let messages = await getMessageSummariesByIDs(ids, acc.accountIds); + const result = await findSimilarSummaries(seedId, { + accountIds: acc.accountIds, + limit, + after: after.value, + before: before.value, + hasAttachment: args.has_attachment === true, + }); + let messages = result.messages; // msgvault applies message_type inside the vector backend filter // (handlers.go:1042-1044); Mailflow's annSearch filter has no such leg, so // the advertised filter is applied on the hydrated summaries (every @@ -345,11 +399,18 @@ export async function handleFindSimilarMessages(args, scope) { return jsonResult({ seed_message_id: seedId, returned: messages.length, - generation: { id: gen.id, model: gen.model, dimension: gen.dimension, fingerprint: gen.fingerprint, state: gen.state }, + generation: { + id: result.generation.id, + model: result.generation.model, + dimension: result.generation.dimension, + fingerprint: result.generation.fingerprint, + state: result.generation.state, + }, messages: messages.map(wireSummary), }); } catch (err) { if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + if (err.name === 'FindSimilarError') return errorResult(err.message); throw err; } } @@ -361,6 +422,7 @@ export const searchInMessageDef = { 'mode=vector scores each embedded chunk by semantic similarity to the query (best first, with score on each match). ' + 'Keyword matches include raw-body char_offset and line. Vector matches always include snippet and score; char_offset and line may be omitted after preprocessing. ' + 'Use a present char_offset with get_message center_at to read a larger window around the match.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -409,6 +471,7 @@ export async function handleSearchInMessage(args, scope) { export const stageDeletionDef = { name: 'stage_deletion', description: "Stage messages for deletion. Use EITHER 'query' (Gmail-style search) OR structured filters (from, domain, label, etc.), not both. Does NOT delete immediately - execution is a separate, explicitly-authorized step.", + annotations: NON_IDEMPOTENT_WRITE_ANNOTATIONS, inputSchema: { type: 'object', properties: { diff --git a/backend/src/mcp/messageTools.test.js b/backend/src/mcp/messageTools.test.js index a112fcaa..072a176a 100644 --- a/backend/src/mcp/messageTools.test.js +++ b/backend/src/mcp/messageTools.test.js @@ -18,6 +18,7 @@ import { collectStats } from './vectorStats.js'; import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; +import { ALL_SCOPES } from './auth.js'; // The one vector-availability gate returns { cfg, generation }; a VectorUnavailableError // (name + reason) thrown from it is what every degraded/disabled path surfaces. @@ -27,7 +28,7 @@ import { handleSearchByDomains, handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, } from './messageTools.js'; -const scope = { userId: 'u', accountIds: ['acc-1'] }; +const scope = { userId: 'u', accountIds: ['acc-1'], scopes: ALL_SCOPES }; const detailRow = { id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), diff --git a/backend/src/mcp/messageTriage.integration.test.js b/backend/src/mcp/messageTriage.integration.test.js new file mode 100644 index 00000000..4974c24e --- /dev/null +++ b/backend/src/mcp/messageTriage.integration.test.js @@ -0,0 +1,158 @@ +import { randomUUID } from 'crypto'; +import { readFile } from 'fs/promises'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from '../services/embeddings/testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const integrationDescribe = DSN ? describe : describe.skip; +const migrationsUrl = new URL('../../migrations/', import.meta.url); + +describe('message triage migration sources', () => { + it('keeps table creation transactional and the feed index in a later concurrent migration', async () => { + const tableSql = await readFile(new URL('0044_message_triage.sql', migrationsUrl), 'utf8').catch(() => ''); + const feedSql = await readFile(new URL('0046_message_triage_feed_index.sql', migrationsUrl), 'utf8').catch(() => ''); + + expect(tableSql).toContain('CREATE TABLE IF NOT EXISTS message_triage'); + expect(tableSql).not.toMatch(/^--\s*no-transaction\b/i); + expect(tableSql).not.toContain('CREATE INDEX CONCURRENTLY'); + expect(feedSql).toMatch(/^--\s*no-transaction\b/i); + expect(feedSql).toContain('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_triage_feed'); + expect(feedSql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_triage_feed'); + }); +}); + +integrationDescribe('message_triage migration', () => { + let client; + const users = []; + + beforeAll(async () => { + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + }); + + afterAll(async () => { + await client.end(); + }); + + beforeEach(() => { + users.length = 0; + }); + + afterEach(async () => { + for (const userId of users) await cleanupAccount(client, userId); + }); + + async function account(label) { + const seeded = await seedAccount(client, label); + users.push(seeded.userId); + return seeded; + } + + async function token(userId) { + const result = await client.query( + `INSERT INTO api_tokens (user_id, token_hash, name) + VALUES ($1, $2, 'message-triage-it') + RETURNING id`, + [userId, `triage-it-${randomUUID()}`], + ); + return result.rows[0].id; + } + + async function triage({ userId, accountId, tokenId = null, header = `<${randomUUID()}@example.com>` }) { + const result = await client.query( + `INSERT INTO message_triage + (user_id, account_id, message_id_header, token_id) + VALUES ($1, $2, $3, $4) + RETURNING id`, + [userId, accountId, header, tokenId], + ); + return { id: result.rows[0].id, header }; + } + + it('has the required columns and constraints', async () => { + const columns = await client.query( + `SELECT column_name, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'message_triage' + ORDER BY ordinal_position`, + ); + expect(columns.rows.map(row => row.column_name)).toEqual([ + 'id', + 'user_id', + 'account_id', + 'message_id_header', + 'triaged_at', + 'action', + 'note', + 'source', + 'token_id', + ]); + expect(columns.rows.find(row => row.column_name === 'message_id_header')?.is_nullable).toBe('NO'); + expect(columns.rows.find(row => row.column_name === 'source')?.column_default).toContain("'mcp'"); + + const constraints = await client.query( + `SELECT c.contype, pg_get_constraintdef(c.oid) AS definition + FROM pg_constraint c + WHERE c.conrelid = 'message_triage'::regclass`, + ); + const definitions = constraints.rows.map(row => row.definition); + expect(definitions).toContain('PRIMARY KEY (id)'); + expect(definitions).toContain('UNIQUE (account_id, message_id_header)'); + expect(definitions).toContain('FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE'); + expect(definitions).toContain('FOREIGN KEY (account_id) REFERENCES email_accounts(id) ON DELETE CASCADE'); + expect(definitions).toContain('FOREIGN KEY (token_id) REFERENCES api_tokens(id) ON DELETE SET NULL'); + expect(definitions.some(definition => definition.includes('message_id_header') && definition.includes('FOREIGN KEY'))).toBe(false); + }); + + it('enforces uniqueness by account and Message-ID header', async () => { + const { userId, accountId } = await account('triage-unique'); + const first = await triage({ userId, accountId }); + + await expect(triage({ userId, accountId, header: first.header })).rejects.toMatchObject({ code: '23505' }); + }); + + it('cascades when an email account is deleted', async () => { + const { userId, accountId } = await account('triage-account-cascade'); + const row = await triage({ userId, accountId }); + + await client.query('DELETE FROM email_accounts WHERE id = $1', [accountId]); + + const result = await client.query('SELECT id FROM message_triage WHERE id = $1', [row.id]); + expect(result.rows).toEqual([]); + }); + + it('cascades when a user is deleted', async () => { + const { userId, accountId } = await account('triage-user-cascade'); + const row = await triage({ userId, accountId }); + + await client.query('DELETE FROM users WHERE id = $1', [userId]); + users.splice(users.indexOf(userId), 1); + + const result = await client.query('SELECT id FROM message_triage WHERE id = $1', [row.id]); + expect(result.rows).toEqual([]); + }); + + it('sets token_id to null when the API token is deleted', async () => { + const { userId, accountId } = await account('triage-token-null'); + const tokenId = await token(userId); + const row = await triage({ userId, accountId, tokenId }); + + await client.query('DELETE FROM api_tokens WHERE id = $1', [tokenId]); + + const result = await client.query('SELECT token_id FROM message_triage WHERE id = $1', [row.id]); + expect(result.rows[0].token_id).toBeNull(); + }); + + it('has the partial inbox feed index', async () => { + const result = await client.query( + `SELECT indexdef + FROM pg_indexes + WHERE schemaname = 'public' AND indexname = 'idx_messages_triage_feed'`, + ); + expect(result.rows).toHaveLength(1); + expect(result.rows[0].indexdef).toContain('(account_id, date, message_id)'); + expect(result.rows[0].indexdef).toContain("folder = 'INBOX'"); + expect(result.rows[0].indexdef).toContain('(is_deleted = false)'); + }); +}); diff --git a/backend/src/mcp/searchTools.js b/backend/src/mcp/searchTools.js index 463a7e6f..d602dbaa 100644 --- a/backend/src/mcp/searchTools.js +++ b/backend/src/mcp/searchTools.js @@ -7,6 +7,24 @@ import { translateVectorError } from './vectorErrors.js'; import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); + // The search seam returns raw REST-shaped rows (subject/from/date/snippet/… under a // column set REST froze); MCP must emit msgvault MessageSummary. We hydrate the hit // ids through engineAdapter (recipients, thread_id, attachments — data the ranked row @@ -118,6 +136,7 @@ export const searchMetadataDef = { SEARCH_METADATA_OPERATOR_DOC + ' ' + SEARCH_METADATA_FREETEXT_DOC + ' ' + SEARCH_METADATA_PAGINATION_DOC + 'For body keywords use search_message_bodies; for vector/hybrid search use semantic_search_messages.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -193,6 +212,7 @@ export const searchMessageBodiesDef = { 'Results are relevance-ranked, best lexical match first (divergence from msgvault, which orders newest-first). ' + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more. ' + 'Body search does not return a total; use has_more to detect more pages.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { @@ -308,6 +328,7 @@ export const semanticSearchMessagesDef = { 'mode=vector for pure semantic search or mode=hybrid to fuse BM25 and vector ranking via RRF. ' + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more, mode, pool_saturated, generation. ' + 'total is not available; use has_more to page.', + annotations: READ_ONLY_ANNOTATIONS, inputSchema: { type: 'object', properties: { diff --git a/backend/src/mcp/searchTools.test.js b/backend/src/mcp/searchTools.test.js index 45f18c98..367da772 100644 --- a/backend/src/mcp/searchTools.test.js +++ b/backend/src/mcp/searchTools.test.js @@ -9,13 +9,14 @@ import { parseQuery } from '../services/search/queryParser.js'; import { search } from '../services/search/searchService.js'; import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { ALL_SCOPES } from './auth.js'; import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; class VectorUnavailableError extends Error { constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } } -const scope = { userId: 'u1', accountIds: ['acc-1'] }; +const scope = { userId: 'u1', accountIds: ['acc-1'], scopes: ALL_SCOPES }; function payload(r) { return JSON.parse(r.content[0].text); } // Hydration echoes each requested id as a minimal summary unless a test overrides it. function echoSummaries(rows) { diff --git a/backend/src/mcp/sendTools.js b/backend/src/mcp/sendTools.js new file mode 100644 index 00000000..88a70e91 --- /dev/null +++ b/backend/src/mcp/sendTools.js @@ -0,0 +1,484 @@ +import { + deleteMessageRow, + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + getOutboxRowByMessageId, + getUserPreferences, +} from './accountAdapter.js'; +import { newPaginatedResponseNoTotal, toRFC3339 } from './envelope.js'; +import { errorResult, jsonResult } from './result.js'; +import { buildWriteReceipt, writeError } from './writeResult.js'; +import { normalizeRecipients } from '../services/mail/addresses.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const attachmentSchema = { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, +}; + +const undoSendSecondsSchema = { + type: 'integer', + minimum: 0, + maximum: 120, + description: "Cancellation window in seconds (max 120). Defaults to the user's undo-send preference.", +}; + +const idempotencyKeySchema = { + type: 'string', + description: 'Stable key; a retry with the same key returns the original result instead of sending twice.', +}; + +export const sendEmailDef = { + name: 'send_email', + description: 'Send an email. When undo_send_seconds > 0 the message is QUEUED and can be cancelled with unsend_email until send_at; when 0 it is delivered immediately. Returns a receipt of exactly what was sent.', + inputSchema: { + type: 'object', + required: ['account', 'to'], + properties: { + account: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string' }, + body_html: { type: 'string' }, + alias: { + type: 'string', + description: 'Send-as alias email; must be configured on the account (hard error if not)', + }, + priority: { type: 'string', enum: ['high', 'normal', 'low'] }, + attachments: attachmentSchema, + undo_send_seconds: undoSendSecondsSchema, + idempotency_key: idempotencyKeySchema, + }, + }, + annotations: annotations(false, true, false), +}; + +export const sendDraftDef = { + name: 'send_draft', + description: 'Reads the draft via `getDraftRow`, reconstructs the compose input (recipients from `to_addresses`/`cc_addresses`, body from `body_text`/`body_html`, threading from `in_reply_to`/`thread_references`), sends, and **deletes the draft only after delivery succeeds** (or after enqueue succeeds, with a `delete_draft_on_send` flag on the outbox payload the worker honors). Errors `draft_not_found` when the uid is absent or not a draft.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { type: 'string' }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + undo_send_seconds: undoSendSecondsSchema, + idempotency_key: idempotencyKeySchema, + }, + }, + annotations: annotations(false, true, false), +}; + +export const unsendEmailDef = { + name: 'unsend_email', + description: 'Cancel a queued email before it is delivered. Only works while the message is still in its undo window (see the send_at returned by send_email).', + inputSchema: { + type: 'object', + required: ['outbox_id'], + properties: { + outbox_id: { type: 'string' }, + }, + }, + annotations: annotations(false, false, true), +}; + +export const listOutboxDef = { + name: 'list_outbox', + description: 'List emails still queued in the undo-send outbox. Entries can be cancelled with unsend_email before send_at.', + inputSchema: { + type: 'object', + properties: {}, + }, + annotations: annotations(true, false, true), +}; + +export const recallEmailDef = { + name: 'recall_email', + description: "Best-effort recall of an already-sent message. SMTP CANNOT retract delivered mail — recipients already have it. This tool (1) cancels the send if it is still queued, otherwise (2) deletes your Sent copy and (3) prepares a 'please disregard' follow-up DRAFT addressed to the original recipients, which it never sends automatically.", + inputSchema: { + type: 'object', + properties: { + message_id: { + type: 'string', + description: 'Id of the sent message (from search or the Sent folder)', + }, + outbox_id: { + type: 'string', + description: "Alternative: a queued message's outbox id", + }, + delete_sent_copy: { type: 'boolean', default: true }, + draft_followup: { type: 'boolean', default: true }, + followup_note: { + type: 'string', + description: "Body of the follow-up draft; defaults to a short 'please disregard' note.", + }, + }, + }, + annotations: annotations(false, true, false), +}; + +function attachmentInput(attachment) { + return { + filename: attachment.filename, + content: attachment.content, + contentType: attachment.content_type ?? attachment.contentType, + }; +} + +function addressString(address) { + if (typeof address === 'string') return address; + if (!address?.email) return ''; + return address.name ? `${address.name} <${address.email}>` : address.email; +} + +function addressStrings(value) { + return (Array.isArray(value) ? value : []).map(addressString).filter(Boolean); +} + +function normalizedRecipients(args) { + try { + const normalized = { + to: normalizeRecipients(args.to, 'to'), + cc: normalizeRecipients(args.cc || [], 'cc'), + bcc: normalizeRecipients(args.bcc || [], 'bcc'), + }; + if (normalized.to.length + normalized.cc.length + normalized.bcc.length > 100) { + throw Object.assign(new Error('Too many recipients (max 100)'), { + code: 'too_many_recipients', + }); + } + return normalized; + } catch (err) { + err.code ||= 'invalid_recipient'; + throw err; + } +} + +function sendResult(result, subject) { + if (result.queued) { + return jsonResult(buildWriteReceipt({ subject }, { + queued: true, + outboxId: result.outboxId, + sendAt: result.sendAt, + undoSeconds: result.undoSeconds, + note: 'Cancel with unsend_email before send_at.', + })); + } + return jsonResult(buildWriteReceipt(result.receipt, { sent: true })); +} + +function errorFrom(err) { + if (err?.code) return writeError(err.code, err.message); + return writeError('invalid_arguments', err?.message || 'send failed'); +} + +export async function handleSendEmail(args, scope, deps = {}) { + if (!deps.sendService || !deps.outboxService?.normalizeUndoWindow) { + return writeError('unsupported', 'send tools require sendService and outboxService'); + } + try { + const account = await getAccountByEmail(args.account, scope.accountIds); + if (account?.error) return errorResult(account.error); + let identity; + if (args.alias) { + identity = await resolveFromIdentity(account, { aliasEmail: args.alias }, deps); + } + const recipients = normalizedRecipients(args); + const preferences = await getUserPreferences(scope.userId); + const undoSeconds = deps.outboxService.normalizeUndoWindow( + args.undo_send_seconds, + preferences.undoSendSeconds, + ); + const bodyIsHtml = args.body_html !== undefined; + const result = await deps.sendService.sendOrEnqueue({ + userId: scope.userId, + account, + aliasId: identity?.aliasId, + aliasEmail: args.alias, + ...recipients, + subject: args.subject || '', + body: bodyIsHtml ? args.body_html : (args.body || ''), + bodyIsHtml, + priority: args.priority, + attachments: (args.attachments || []).map(attachmentInput), + undoSeconds, + idempotencyKey: args.idempotency_key, + }, deps); + return sendResult(result, args.subject || ''); + } catch (err) { + return errorFrom(err); + } +} + +export async function handleSendDraft(args, scope, deps = {}) { + if ( + !deps.sendService || + !deps.draftService || + !deps.outboxService?.normalizeUndoWindow + ) { + return writeError( + 'unsupported', + 'send_draft requires sendService, draftService, and outboxService', + ); + } + try { + const account = await getAccountByEmail(args.account, scope.accountIds); + if (account?.error) return errorResult(account.error); + const folder = args.folder || account?.folder_mappings?.drafts || 'Drafts'; + const draft = await getDraftRow(account.id, folder, args.draft_uid); + if (!draft) return writeError('draft_not_found', args.draft_uid); + const aliasEmail = draft.from_email && draft.from_email !== account.email_address + ? draft.from_email + : undefined; + const identity = aliasEmail + ? await resolveFromIdentity(account, { aliasEmail }, deps) + : undefined; + const preferences = await getUserPreferences(scope.userId); + const undoSeconds = deps.outboxService.normalizeUndoWindow( + args.undo_send_seconds, + preferences.undoSendSeconds, + ); + const bodyIsHtml = Boolean(draft.body_html); + const result = await deps.sendService.sendOrEnqueue({ + userId: scope.userId, + account, + aliasId: identity?.aliasId, + aliasEmail, + to: addressStrings(draft.to_addresses), + cc: addressStrings(draft.cc_addresses), + bcc: addressStrings(draft.bcc_addresses), + subject: draft.subject || '', + body: bodyIsHtml ? draft.body_html : (draft.body_text || ''), + bodyIsHtml, + inReplyTo: draft.in_reply_to || undefined, + references: draft.thread_references || undefined, + undoSeconds, + idempotencyKey: args.idempotency_key, + ...(undoSeconds + ? { deleteDraftOnSend: { uid: draft.uid, folder: draft.folder } } + : {}), + }, deps); + if (!result.queued) { + try { + await deps.draftService.deleteDraft({ + account, + uid: draft.uid, + folder: draft.folder, + }, deps); + } catch (err) { + console.error('Draft cleanup after send failed:', err.message); + } + } + return sendResult(result, draft.subject || ''); + } catch (err) { + return errorFrom(err); + } +} + +function cancelledOutbox(outboxId, row) { + return { + cancelled: true, + outbox_id: outboxId, + subject: row?.subject || '', + to: row?.to_preview || [], + }; +} + +function recalledOutbox(outboxId, row) { + return { + recalled: 'cancelled_before_send', + outbox_id: outboxId, + subject: row?.subject || '', + to: row?.to_preview || [], + }; +} + +function cancelFailure(outboxId, reason) { + if (reason === 'already_sent') { + return writeError( + 'already_sent', + `message ${outboxId} is no longer pending and cannot be unsent`, + ); + } + return writeError('outbox_not_found', outboxId); +} + +async function pendingOutboxById(outboxId, scope, deps) { + if (!deps.outboxService?.listPending) return null; + const rows = await deps.outboxService.listPending({ userId: scope.userId }, deps); + return rows.find(row => row.id === outboxId) || null; +} + +async function cancelOutbox(outboxId, row, scope, deps, resultBuilder) { + const result = await deps.outboxService.cancel({ + id: outboxId, + userId: scope.userId, + }, deps); + if (result.cancelled || result.reason === 'cancelled') { + return jsonResult(resultBuilder(outboxId, row)); + } + return cancelFailure(outboxId, result.reason); +} + +export async function handleUnsendEmail(args, scope, deps = {}) { + if (!deps.outboxService?.cancel) { + return writeError('unsupported', 'unsend_email requires outboxService'); + } + try { + const row = await pendingOutboxById(args.outbox_id, scope, deps); + return await cancelOutbox( + args.outbox_id, + row, + scope, + deps, + cancelledOutbox, + ); + } catch (err) { + return errorFrom(err); + } +} + +function wireOutboxRow(row) { + const sendAt = row?.send_at instanceof Date + ? row.send_at.toISOString() + : row?.send_at; + return { + ...row, + ...(sendAt !== undefined ? { send_at: toRFC3339(sendAt) } : {}), + }; +} + +export async function handleListOutbox(_args, scope, deps = {}) { + if (!deps.outboxService?.listPending) { + return writeError('unsupported', 'list_outbox requires outboxService'); + } + try { + const rows = await deps.outboxService.listPending({ userId: scope.userId }, deps); + return jsonResult(newPaginatedResponseNoTotal( + rows.map(wireOutboxRow), + 0, + false, + )); + } catch (err) { + return errorFrom(err); + } +} + +function followupSubject(subject) { + const value = subject || ''; + return /^re:/i.test(value) ? value : `Re: ${value}`; +} + +async function recallDelivered(args, message, account, scope, deps) { + const deleteSentCopy = args.delete_sent_copy !== false; + const draftFollowup = args.draft_followup !== false; + if (deleteSentCopy && !deps.imapManager?.permanentDeleteMessage) { + return writeError('unsupported', 'recall_email requires imapManager'); + } + if (draftFollowup && !deps.draftService?.saveDraft) { + return writeError('unsupported', 'recall_email requires draftService'); + } + + if (deleteSentCopy) { + await deps.imapManager.permanentDeleteMessage( + account, + message.uid, + message.folder, + ); + await deleteMessageRow(message.account_id, message.uid, message.folder); + } + + let followupDraft; + if (draftFollowup) { + const saved = await deps.draftService.saveDraft({ + userId: scope.userId, + account, + to: addressStrings(message.to_addresses), + cc: addressStrings(message.cc_addresses), + bcc: [], + subject: followupSubject(message.subject), + body: args.followup_note ?? 'Please disregard my previous email.', + bodyIsHtml: false, + attachments: [], + inReplyTo: message.message_id, + }, deps); + followupDraft = { + draft_uid: saved.uid, + folder: saved.folder, + }; + } + + return jsonResult({ + recalled: 'not_possible', + note: 'SMTP cannot retract a delivered message. Recipients already received it; deleting your Sent copy does not affect their mailboxes.', + sent_copy_deleted: deleteSentCopy, + ...(followupDraft ? { followup_draft: followupDraft } : {}), + }); +} + +export async function handleRecallEmail(args, scope, deps = {}) { + if (!deps.outboxService?.cancel) { + return writeError('unsupported', 'recall_email requires outboxService'); + } + try { + if (!args.outbox_id && !args.message_id) { + return writeError( + 'invalid_arguments', + 'recall_email requires message_id or outbox_id', + ); + } + + if (args.outbox_id) { + const row = await pendingOutboxById(args.outbox_id, scope, deps); + return await cancelOutbox( + args.outbox_id, + row, + scope, + deps, + recalledOutbox, + ); + } + + const pending = await getOutboxRowByMessageId( + args.message_id, + scope.userId, + ); + if (pending) { + return await cancelOutbox( + pending.id, + pending, + scope, + deps, + recalledOutbox, + ); + } + + const message = await getComposeSource(args.message_id, scope.accountIds); + if (!message) return writeError('message_not_found', args.message_id); + const account = await getAccountRow(message.account_id, scope.accountIds); + if (!account) return writeError('account_not_found', message.account_id); + return recallDelivered(args, message, account, scope, deps); + } catch (err) { + return errorFrom(err); + } +} diff --git a/backend/src/mcp/sendTools.test.js b/backend/src/mcp/sendTools.test.js new file mode 100644 index 00000000..cbed9938 --- /dev/null +++ b/backend/src/mcp/sendTools.test.js @@ -0,0 +1,1024 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', () => ({ + deleteMessageRow: vi.fn(), + getAccountByEmail: vi.fn(), + getAccountRow: vi.fn(), + getComposeSource: vi.fn(), + getDraftRow: vi.fn(), + getOutboxRowByMessageId: vi.fn(), + getUserPreferences: vi.fn(), +})); +vi.mock('../services/mail/identity.js', () => ({ + resolveFromIdentity: vi.fn(), +})); + +import { + deleteMessageRow, + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + getOutboxRowByMessageId, + getUserPreferences, +} from './accountAdapter.js'; +import * as sendTools from './sendTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['send'], +}; +const account = { + id: 'account-1', + email_address: 'sender@example.com', + sender_name: 'Sender', + folder_mappings: { drafts: 'Drafts' }, +}; +const identity = { + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: null, + signature: null, + aliasId: null, +}; +const immediateReceipt = { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', +}; + +function deps(overrides = {}) { + return { + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: immediateReceipt, + }), + }, + outboxService: { + normalizeUndoWindow: vi.fn((requested, preference) => requested ?? preference ?? 0), + cancel: vi.fn().mockResolvedValue({ cancelled: true }), + listPending: vi.fn().mockResolvedValue([]), + }, + draftService: { + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + saveDraft: vi.fn().mockResolvedValue({ + uid: 57, + folder: 'Drafts', + messageId: '', + }), + }, + imapManager: { + permanentDeleteMessage: vi.fn().mockResolvedValue(undefined), + }, + ...overrides, + }; +} + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + getAccountByEmail.mockResolvedValue(account); + getAccountRow.mockResolvedValue(account); + getComposeSource.mockResolvedValue(null); + getOutboxRowByMessageId.mockResolvedValue(null); + getUserPreferences.mockResolvedValue({}); + deleteMessageRow.mockResolvedValue(undefined); + resolveFromIdentity.mockResolvedValue(identity); +}); + +describe('send tool definitions and registration', () => { + it('publishes the plan schemas, descriptions, annotations, scopes, and handlers', () => { + expect(sendTools.sendEmailDef).toEqual({ + name: 'send_email', + description: 'Send an email. When undo_send_seconds > 0 the message is QUEUED and can be cancelled with unsend_email until send_at; when 0 it is delivered immediately. Returns a receipt of exactly what was sent.', + inputSchema: { + type: 'object', + required: ['account', 'to'], + properties: { + account: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string' }, + body_html: { type: 'string' }, + alias: { + type: 'string', + description: 'Send-as alias email; must be configured on the account (hard error if not)', + }, + priority: { type: 'string', enum: ['high', 'normal', 'low'] }, + attachments: { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, + }, + undo_send_seconds: { + type: 'integer', + minimum: 0, + maximum: 120, + description: "Cancellation window in seconds (max 120). Defaults to the user's undo-send preference.", + }, + idempotency_key: { + type: 'string', + description: 'Stable key; a retry with the same key returns the original result instead of sending twice.', + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }); + + expect(sendTools.sendDraftDef).toEqual({ + name: 'send_draft', + description: 'Reads the draft via `getDraftRow`, reconstructs the compose input (recipients from `to_addresses`/`cc_addresses`, body from `body_text`/`body_html`, threading from `in_reply_to`/`thread_references`), sends, and **deletes the draft only after delivery succeeds** (or after enqueue succeeds, with a `delete_draft_on_send` flag on the outbox payload the worker honors). Errors `draft_not_found` when the uid is absent or not a draft.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { type: 'string' }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + undo_send_seconds: { + type: 'integer', + minimum: 0, + maximum: 120, + description: "Cancellation window in seconds (max 120). Defaults to the user's undo-send preference.", + }, + idempotency_key: { + type: 'string', + description: 'Stable key; a retry with the same key returns the original result instead of sending twice.', + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }); + + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, handler] of [ + ['send_email', sendTools.handleSendEmail], + ['send_draft', sendTools.handleSendDraft], + ]) { + expect(defs.get(name)).toBeTruthy(); + expect(TOOL_SCOPES[name]).toBe('send'); + expect(HANDLERS[name]).toBe(handler); + } + }); + + it('publishes and registers the unsend, outbox-list, and recall tools', () => { + expect(sendTools.unsendEmailDef).toEqual({ + name: 'unsend_email', + description: 'Cancel a queued email before it is delivered. Only works while the message is still in its undo window (see the send_at returned by send_email).', + inputSchema: { + type: 'object', + required: ['outbox_id'], + properties: { + outbox_id: { type: 'string' }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }); + expect(sendTools.listOutboxDef).toEqual({ + name: 'list_outbox', + description: 'List emails still queued in the undo-send outbox. Entries can be cancelled with unsend_email before send_at.', + inputSchema: { type: 'object', properties: {} }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }); + expect(sendTools.recallEmailDef).toEqual({ + name: 'recall_email', + description: "Best-effort recall of an already-sent message. SMTP CANNOT retract delivered mail — recipients already have it. This tool (1) cancels the send if it is still queued, otherwise (2) deletes your Sent copy and (3) prepares a 'please disregard' follow-up DRAFT addressed to the original recipients, which it never sends automatically.", + inputSchema: { + type: 'object', + properties: { + message_id: { + type: 'string', + description: 'Id of the sent message (from search or the Sent folder)', + }, + outbox_id: { + type: 'string', + description: "Alternative: a queued message's outbox id", + }, + delete_sent_copy: { type: 'boolean', default: true }, + draft_followup: { type: 'boolean', default: true }, + followup_note: { + type: 'string', + description: "Body of the follow-up draft; defaults to a short 'please disregard' note.", + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }); + + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, requiredScope, handler] of [ + ['unsend_email', 'send', sendTools.handleUnsendEmail], + ['list_outbox', 'read', sendTools.handleListOutbox], + ['recall_email', 'send', sendTools.handleRecallEmail], + ]) { + expect(defs.get(name)).toBeTruthy(); + expect(TOOL_SCOPES[name]).toBe(requiredScope); + expect(HANDLERS[name]).toBe(handler); + } + }); +}); + +describe('send_email', () => { + it('does not reveal an out-of-scope account and never calls the service', async () => { + const service = deps(); + getAccountByEmail.mockResolvedValue({ error: 'account_not_found: foreign@example.com' }); + + const result = await sendTools.handleSendEmail({ + account: 'foreign@example.com', + to: ['recipient@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account_not_found: foreign@example.com'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('hard-fails an unknown alias and preserves its stable error code', async () => { + const service = deps(); + resolveFromIdentity.mockRejectedValue( + Object.assign(new Error('Alias not found'), { code: 'alias_not_found' }), + ); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + alias: 'unknown@example.com', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('maps malformed recipients to invalid_recipient before sending', async () => { + const service = deps(); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['victim@example.com\r\nBcc: attacker@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toMatch(/^invalid_recipient: /); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('rejects more than 100 total recipients as too_many_recipients', async () => { + const service = deps(); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: Array.from({ length: 99 }, (_, i) => `to-${i}@example.com`), + cc: ['copy@example.com'], + bcc: ['blind@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'too_many_recipients: Too many recipients (max 100)', + ); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('sends immediately with normalized input and returns the exact write receipt', async () => { + const service = deps(); + const aliasIdentity = { + ...identity, + fromName: 'Team', + fromEmail: 'team@example.com', + aliasId: 'alias-1', + }; + resolveFromIdentity.mockResolvedValue(aliasIdentity); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 60 }); + + const wireResult = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: [' Recipient '], + cc: [], + bcc: [], + subject: 'Subject', + body: 'Plain fallback', + body_html: '

Body

', + alias: 'team@example.com', + priority: 'high', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + content_type: 'text/plain', + }], + undo_send_seconds: 0, + idempotency_key: 'send-1', + }, scope, service); + expect(wireResult.isError).not.toBe(true); + const result = payload(wireResult); + + expect(getAccountByEmail).toHaveBeenCalledWith('sender@example.com', ['account-1']); + expect(resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + service, + ); + expect(getUserPreferences).toHaveBeenCalledWith('user-1'); + expect(service.outboxService.normalizeUndoWindow).toHaveBeenCalledWith(0, 60); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith({ + userId: 'user-1', + account, + aliasId: 'alias-1', + aliasEmail: 'team@example.com', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Subject', + body: '

Body

', + bodyIsHtml: true, + priority: 'high', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + contentType: 'text/plain', + }], + undoSeconds: 0, + idempotencyKey: 'send-1', + }, service); + expect(result).toEqual({ + sent: true, + message_id: '', + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('uses the preference undo window and returns the exact queued result shape', async () => { + const sendAt = new Date('2026-07-28T10:00:30.000Z'); + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + queued: true, + outboxId: 'outbox-1', + sendAt, + undoSeconds: 30, + }), + }, + }); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 30 }); + + const wireResult = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + subject: 'Subject', + idempotency_key: 'queued-1', + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(service.outboxService.normalizeUndoWindow).toHaveBeenCalledWith(undefined, 30); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + undoSeconds: 30, + idempotencyKey: 'queued-1', + }), + service, + ); + expect(payload(wireResult)).toEqual({ + queued: true, + outbox_id: 'outbox-1', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); + + it('defaults the undo window to zero when no argument or preference exists', async () => { + const service = deps(); + + await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + }, scope, service); + + expect(service.outboxService.normalizeUndoWindow).toHaveBeenCalledWith( + undefined, + undefined, + ); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ undoSeconds: 0 }), + service, + ); + }); + + it.each([ + ['invalid_recipient', 'Recipient is invalid'], + ['too_many_recipients', 'Too many recipients'], + ])('preserves an underlying %s service error', async (code, message) => { + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockRejectedValue( + Object.assign(new Error(message), { code }), + ), + }, + }); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(`${code}: ${message}`); + }); +}); + +describe('send_draft', () => { + it('returns draft_not_found for an absent folder and uid match', async () => { + const service = deps(); + getDraftRow.mockResolvedValue(null); + + const result = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 404, + folder: 'Other Drafts', + }, scope, service); + + expect(getDraftRow).toHaveBeenCalledWith('account-1', 'Other Drafts', 404); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('draft_not_found: 404'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(service.draftService.deleteDraft).not.toHaveBeenCalled(); + }); + + it('reconstructs and immediately sends a draft before deleting it', async () => { + const service = deps(); + const draft = { + uid: 7, + folder: 'Drafts', + from_email: 'team@example.com', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [{ name: '', email: 'copy@example.com' }], + bcc_addresses: [{ name: '', email: 'blind@example.com' }], + subject: 'Draft subject', + body_text: 'Plain fallback', + body_html: '

Draft body

', + in_reply_to: '', + thread_references: ' ', + }; + getDraftRow.mockResolvedValue(draft); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 60 }); + resolveFromIdentity.mockResolvedValue({ + ...identity, + fromName: 'Team', + fromEmail: 'team@example.com', + aliasId: 'alias-1', + }); + + const wireResult = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 7, + undo_send_seconds: 0, + idempotency_key: 'draft-7', + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + service, + ); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith({ + userId: 'user-1', + account, + aliasId: 'alias-1', + aliasEmail: 'team@example.com', + to: ['Recipient '], + cc: ['copy@example.com'], + bcc: ['blind@example.com'], + subject: 'Draft subject', + body: '

Draft body

', + bodyIsHtml: true, + inReplyTo: '', + references: ' ', + undoSeconds: 0, + idempotencyKey: 'draft-7', + }, service); + expect(service.draftService.deleteDraft).toHaveBeenCalledWith({ + account, + uid: 7, + folder: 'Drafts', + }, service); + expect( + service.sendService.sendOrEnqueue.mock.invocationCallOrder[0], + ).toBeLessThan(service.draftService.deleteDraft.mock.invocationCallOrder[0]); + expect(payload(wireResult)).toEqual({ + sent: true, + message_id: '', + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('queues a draft with a delete-on-send marker and does not delete synchronously', async () => { + const sendAt = new Date('2026-07-28T10:00:30.000Z'); + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + queued: true, + outboxId: 'outbox-draft-7', + sendAt, + undoSeconds: 30, + }), + }, + }); + getDraftRow.mockResolvedValue({ + uid: 7, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: '', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Queued draft', + body_text: 'Queued body', + body_html: '', + in_reply_to: null, + thread_references: null, + }); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 30 }); + + const wireResult = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 7, + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + undoSeconds: 30, + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }), + service, + ); + expect(service.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(payload(wireResult)).toEqual({ + queued: true, + outbox_id: 'outbox-draft-7', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Queued draft', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); + + it('keeps a successful immediate send successful when draft cleanup fails', async () => { + const service = deps(); + service.draftService.deleteDraft.mockRejectedValue(new Error('IMAP delete failed')); + getDraftRow.mockResolvedValue({ + uid: 7, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: '', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Draft subject', + body_text: 'Draft body', + body_html: '', + }); + vi.spyOn(console, 'error').mockImplementation(() => {}); + + const result = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 7, + undo_send_seconds: 0, + }, scope, service); + + expect(result.isError).not.toBe(true); + expect(console.error).toHaveBeenCalledWith( + 'Draft cleanup after send failed:', + 'IMAP delete failed', + ); + }); +}); + +describe('queued draft payload transport', () => { + it('preserves deleteDraftOnSend in the credential-free outbox payload', async () => { + const { sendOrEnqueue } = await import('../services/sendService.js'); + const outboxService = { + enqueue: vi.fn().mockResolvedValue({ + outbox_id: 'outbox-draft-7', + send_at: new Date('2026-07-28T10:00:30.000Z'), + undo_seconds: 30, + }), + }; + const service = { + outboxService, + resolveFromIdentity: vi.fn().mockResolvedValue(identity), + randomBytes: vi.fn(() => Buffer.alloc(16, 1)), + }; + + await sendOrEnqueue({ + userId: 'user-1', + account, + to: ['recipient@example.com'], + cc: [], + bcc: [], + subject: 'Queued draft', + body: 'Queued body', + bodyIsHtml: false, + undoSeconds: 30, + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }, service); + + expect(outboxService.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ + payload: expect.objectContaining({ + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }), + }), + service, + ); + }); +}); + +describe('unsend_email', () => { + it('cancels a pending outbox row and returns its prefetched metadata', async () => { + const service = deps(); + service.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-1' }, + scope, + service, + ); + + expect(service.outboxService.listPending).toHaveBeenCalledWith( + { userId: 'user-1' }, + service, + ); + expect(service.outboxService.cancel).toHaveBeenCalledWith( + { id: 'outbox-1', userId: 'user-1' }, + service, + ); + expect(payload(result)).toEqual({ + cancelled: true, + outbox_id: 'outbox-1', + subject: 'Queued subject', + to: ['recipient@example.com'], + }); + }); + + it('returns already_sent when the undo window has closed', async () => { + const service = deps(); + service.outboxService.cancel.mockResolvedValue({ + cancelled: false, + reason: 'already_sent', + }); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-sent' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'already_sent: message outbox-sent is no longer pending and cannot be unsent', + ); + }); + + it('returns outbox_not_found for an absent or out-of-scope id', async () => { + const service = deps(); + service.outboxService.cancel.mockResolvedValue({ + cancelled: false, + reason: 'not_found', + }); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-missing' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('outbox_not_found: outbox-missing'); + }); + + it('treats an already-cancelled row as an idempotent success', async () => { + const service = deps(); + service.outboxService.cancel.mockResolvedValue({ + cancelled: false, + reason: 'cancelled', + }); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-cancelled' }, + scope, + service, + ); + + expect(payload(result)).toEqual({ + cancelled: true, + outbox_id: 'outbox-cancelled', + subject: '', + to: [], + }); + }); +}); + +describe('list_outbox', () => { + it('returns pending rows in a no-total pagination envelope', async () => { + const service = deps(); + service.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const result = await sendTools.handleListOutbox({}, scope, service); + + expect(service.outboxService.listPending).toHaveBeenCalledWith( + { userId: 'user-1' }, + service, + ); + expect(payload(result)).toEqual({ + data: [{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: '2026-07-28T10:00:30Z', + }], + total: -1, + returned: 1, + offset: 0, + has_more: false, + }); + }); + + it('degrades to unsupported without outboxService.listPending', async () => { + const result = await sendTools.handleListOutbox({}, scope, {}); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'unsupported: list_outbox requires outboxService', + ); + }); +}); + +describe('recall_email', () => { + const delivered = { + id: 'message-1', + account_id: 'account-1', + uid: 42, + folder: 'Sent', + message_id: '', + subject: 'Project update', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [{ name: '', email: 'copy@example.com' }], + }; + + it('cancels a pending row found by message id before touching a Sent copy', async () => { + const service = deps(); + getOutboxRowByMessageId.mockResolvedValue({ + id: 'outbox-1', + message_id: '', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + }); + + const result = await sendTools.handleRecallEmail( + { message_id: '' }, + scope, + service, + ); + + expect(getOutboxRowByMessageId).toHaveBeenCalledWith( + '', + 'user-1', + ); + expect(service.outboxService.cancel).toHaveBeenCalledWith( + { id: 'outbox-1', userId: 'user-1' }, + service, + ); + expect(payload(result)).toEqual({ + recalled: 'cancelled_before_send', + outbox_id: 'outbox-1', + subject: 'Queued subject', + to: ['recipient@example.com'], + }); + expect(getComposeSource).not.toHaveBeenCalled(); + expect(service.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('deletes the Sent copy and creates, but never sends, a threaded follow-up draft', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(delivered); + + const result = await sendTools.handleRecallEmail( + { + message_id: 'message-1', + followup_note: 'Please disregard the previous update.', + }, + scope, + service, + ); + + expect(getComposeSource).toHaveBeenCalledWith('message-1', ['account-1']); + expect(getAccountRow).toHaveBeenCalledWith('account-1', ['account-1']); + expect(service.imapManager.permanentDeleteMessage).toHaveBeenCalledWith( + account, + 42, + 'Sent', + ); + expect(deleteMessageRow).toHaveBeenCalledWith('account-1', 42, 'Sent'); + expect(service.draftService.saveDraft).toHaveBeenCalledWith({ + userId: 'user-1', + account, + to: ['Recipient '], + cc: ['copy@example.com'], + bcc: [], + subject: 'Re: Project update', + body: 'Please disregard the previous update.', + bodyIsHtml: false, + attachments: [], + inReplyTo: '', + }, service); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(payload(result)).toEqual({ + recalled: 'not_possible', + note: 'SMTP cannot retract a delivered message. Recipients already received it; deleting your Sent copy does not affect their mailboxes.', + sent_copy_deleted: true, + followup_draft: { + draft_uid: 57, + folder: 'Drafts', + }, + }); + }); + + it('keeps the Sent copy when delete_sent_copy is false', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(delivered); + + const result = await sendTools.handleRecallEmail( + { message_id: 'message-1', delete_sent_copy: false }, + scope, + service, + ); + + expect(service.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(deleteMessageRow).not.toHaveBeenCalled(); + expect(service.draftService.saveDraft).toHaveBeenCalled(); + expect(payload(result)).toMatchObject({ + recalled: 'not_possible', + sent_copy_deleted: false, + followup_draft: { draft_uid: 57, folder: 'Drafts' }, + }); + }); + + it('does not create a follow-up when draft_followup is false', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(delivered); + + const result = await sendTools.handleRecallEmail( + { message_id: 'message-1', draft_followup: false }, + scope, + service, + ); + + expect(service.imapManager.permanentDeleteMessage).toHaveBeenCalled(); + expect(deleteMessageRow).toHaveBeenCalled(); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(payload(result)).toEqual({ + recalled: 'not_possible', + note: 'SMTP cannot retract a delivered message. Recipients already received it; deleting your Sent copy does not affect their mailboxes.', + sent_copy_deleted: true, + }); + }); + + it('returns message_not_found without destructive effects', async () => { + const service = deps(); + + const result = await sendTools.handleRecallEmail( + { message_id: 'message-missing' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_not_found: message-missing'); + expect(service.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(deleteMessageRow).not.toHaveBeenCalled(); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); +}); + +describe('missing dependencies', () => { + it.each([ + ['send_email/sendService', sendTools.handleSendEmail, { + account: 'sender@example.com', + to: ['recipient@example.com'], + }, { + outboxService: { normalizeUndoWindow: vi.fn() }, + }], + ['send_draft/sendService', sendTools.handleSendDraft, { + account: 'sender@example.com', + draft_uid: 7, + }, { + draftService: { deleteDraft: vi.fn() }, + outboxService: { normalizeUndoWindow: vi.fn() }, + }], + ['send_draft/draftService', sendTools.handleSendDraft, { + account: 'sender@example.com', + draft_uid: 7, + }, { + sendService: { sendOrEnqueue: vi.fn() }, + outboxService: { normalizeUndoWindow: vi.fn() }, + }], + ['unsend_email/outboxService', sendTools.handleUnsendEmail, { + outbox_id: 'outbox-1', + }, {}], + ['recall_email/outboxService', sendTools.handleRecallEmail, { + message_id: 'message-1', + }, {}], + ])('%s degrades to unsupported', async (_name, handler, args, service) => { + const result = await handler(args, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toMatch(/^unsupported: /); + }); +}); diff --git a/backend/src/mcp/server.js b/backend/src/mcp/server.js index 9fb25a32..c5f40fcf 100644 --- a/backend/src/mcp/server.js +++ b/backend/src/mcp/server.js @@ -1,9 +1,38 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js'; -import { mcpBearerAuth } from './auth.js'; -import { TOOL_DEFS, HANDLERS } from './tools.js'; +import express from 'express'; +import { hasScope, mcpBearerAuth } from './auth.js'; +import { TOOL_DEFS, TOOL_SCOPES, HANDLERS } from './tools.js'; import { errorResult } from './result.js'; +import { listRules } from './accountAdapter.js'; + +// Attachments are base64-inlined in MCP tool arguments, so this parser needs the +// same headroom as the REST send/draft routes. index.js mounts it before the +// global 1 MB parser. +export function mcpBodyLimit() { + return express.json({ limit: process.env.MCP_BODY_LIMIT || '35mb' }); +} + +export function entityTooLargeResponse(err, path) { + if (err?.type !== 'entity.too.large') return null; + if (path.startsWith('/mcp')) { + return { + status: 413, + body: { + jsonrpc: '2.0', + error: { code: -32000, message: 'Request too large (max 35 MB).' }, + id: null, + }, + }; + } + return { + status: 413, + body: { + error: 'Request too large. Total attachment size must not exceed 25 MB.', + }, + }; +} // --- Origin validation (DNS-rebinding protection) -------------------------------- // The MCP Streamable HTTP spec REQUIRES servers to validate the Origin header so a @@ -56,68 +85,136 @@ export function mcpOriginGuard(allowed = buildAllowedOrigins()) { }; } -// --- Per-token tool-call rate limit ----------------------------------------------- +// --- Per-token tool-call rate limits ---------------------------------------------- // REST search is limited per user (routes/search.js); the MCP surface reuses the same // in-memory bucket pattern, keyed by the api_tokens row id — NOT the client IP, since // many agents legitimately share one egress. Only tools/call requests count, so the // initialize / tools-list handshake a stateless client repeats is never throttled. // Over-limit requests get an HTTP 429 with Retry-After BEFORE the transport, carrying -// the SDK's JSON-RPC error envelope shape. Default 60 tool calls per minute per token, -// overridable via MCP_RATE_LIMIT_PER_MIN. +// the SDK's JSON-RPC error envelope shape. Read/write/send/settings have independent +// per-minute budgets, and send also has a per-token daily cap. export function countToolCalls(body) { if (Array.isArray(body)) return body.filter((m) => m && m.method === 'tools/call').length; return body && body.method === 'tools/call' ? 1 : 0; } -export function createMcpRateLimiter({ limit, windowMs = 60_000, now = Date.now } = {}) { - const envLimit = Number(process.env.MCP_RATE_LIMIT_PER_MIN); - const max = limit ?? (envLimit > 0 ? envLimit : 60); +const RATE_CLASSES = ['read', 'write', 'send', 'settings']; + +export function classifyToolCalls(body) { + const counts = { read: 0, write: 0, send: 0, settings: 0 }; + const messages = Array.isArray(body) ? body : [body]; + for (const message of messages) { + if (!message || message.method !== 'tools/call') continue; + const required = TOOL_SCOPES[message.params?.name]; + // Array scopes use AND semantics at authorization. For rate limiting, the + // first-listed scope is the primary/most-restrictive class. No current tool + // has an array scope; this pins the forward-looking tie-break explicitly. + const primary = Array.isArray(required) ? required[0] : required; + const rateClass = RATE_CLASSES.includes(primary) ? primary : 'read'; + counts[rateClass]++; + } + return counts; +} + +function positiveEnv(name, fallback) { + const value = Number(process.env[name]); + return value > 0 ? value : fallback; +} + +export function createMcpRateLimiter({ limits, windowMs = 60_000, now = Date.now } = {}) { + const maximums = { + read: limits?.read ?? positiveEnv('MCP_RATE_LIMIT_PER_MIN', 60), + write: limits?.write ?? positiveEnv('MCP_WRITE_RATE_LIMIT_PER_MIN', 30), + send: limits?.send ?? positiveEnv('MCP_SEND_RATE_LIMIT_PER_MIN', 10), + settings: limits?.settings ?? positiveEnv('MCP_SETTINGS_RATE_LIMIT_PER_MIN', 10), + }; + const dailySendMax = positiveEnv('MCP_SEND_DAILY_CAP', 200); + const dayMs = 24 * 60 * 60 * 1000; const buckets = new Map(); const sweeper = setInterval(() => { const t = now(); - for (const [k, b] of buckets) if (t > b.resetAt) buckets.delete(k); + for (const [k, b] of buckets) if (t >= b.resetAt) buckets.delete(k); }, windowMs); sweeper.unref?.(); // observability sweeper must never keep the process alive + const bucketFor = (key, duration, t) => { + let bucket = buckets.get(key); + if (!bucket || t >= bucket.resetAt) { + bucket = { count: 0, resetAt: t + duration }; + buckets.set(key, bucket); + } + return bucket; + }; + + const reject = (req, res, bucket, max, rateClass, period, t) => { + res.setHeader('Retry-After', Math.ceil((bucket.resetAt - t) / 1000)); + return res.status(429).json({ + jsonrpc: '2.0', + error: { + code: -32000, + message: `Rate limit exceeded: at most ${max} ${rateClass} tool calls per ${period} per token`, + }, + id: Array.isArray(req.body) ? null : (req.body?.id ?? null), + }); + }; + return (req, res, next) => { - const calls = countToolCalls(req.body); - if (!calls) return next(); + const calls = classifyToolCalls(req.body); + if (!RATE_CLASSES.some((rateClass) => calls[rateClass] > 0)) return next(); const t = now(); - let b = buckets.get(req.mcpTokenId); - if (!b || t > b.resetAt) { - b = { count: 0, resetAt: t + windowMs }; - buckets.set(req.mcpTokenId, b); + const admissions = []; + for (const rateClass of RATE_CLASSES) { + if (!calls[rateClass]) continue; + const bucket = bucketFor(`${req.mcpTokenId}:${rateClass}`, windowMs, t); + if (bucket.count + calls[rateClass] > maximums[rateClass]) { + return reject(req, res, bucket, maximums[rateClass], rateClass, 'minute', t); + } + admissions.push([bucket, calls[rateClass]]); } - if (b.count + calls > max) { - res.setHeader('Retry-After', Math.ceil((b.resetAt - t) / 1000)); - return res.status(429).json({ - jsonrpc: '2.0', - error: { code: -32000, message: `Rate limit exceeded: at most ${max} tool calls per minute per token — retry shortly` }, - id: Array.isArray(req.body) ? null : (req.body?.id ?? null), - }); + + if (calls.send) { + const daily = bucketFor(`${req.mcpTokenId}:send:day`, dayMs, t); + if (daily.count + calls.send > dailySendMax) { + return reject(req, res, daily, dailySendMax, 'send', 'day', t); + } + admissions.push([daily, calls.send]); } - b.count += calls; + + // Admit only after every affected bucket passes, so a rejected batch consumes + // no budget in any class. + for (const [bucket, count] of admissions) bucket.count += count; next(); }; } // Build a fresh Server bound to one request's scope. Stateless: no session store, // one Server+transport per HTTP request, matching msgvault's daemon-less posture. -function buildServer(scope) { +export function buildServer(scope, deps = {}) { const server = new Server( { name: 'mailflow', version: '1.0.0' }, { capabilities: { tools: {} } }, ); server.setRequestHandler(ListToolsRequestSchema, async () => ({ - tools: TOOL_DEFS, + tools: TOOL_DEFS.filter((definition) => hasScope(scope, TOOL_SCOPES[definition.name])), })); server.setRequestHandler(CallToolRequestSchema, async (req) => { - const handler = HANDLERS[req.params.name]; - if (!handler) return errorResult(`unknown tool: ${req.params.name}`); + const name = req.params.name; + const handler = HANDLERS[name]; + if (!handler) return errorResult(`unknown tool: ${name}`); + const requiredScope = TOOL_SCOPES[name]; + if (!requiredScope) { + return errorResult(`permission_denied: tool "${name}" has no scope classification`); + } + if (!hasScope(scope, requiredScope)) { + return errorResult( + `permission_denied: tool "${name}" requires the "${requiredScope}" scope; ` + + `this token has [${(scope.scopes || []).join(', ')}]`, + ); + } try { - return await handler(req.params.arguments || {}, scope); + return await handler(req.params.arguments || {}, scope, deps); } catch (err) { // Tool-level failures flow as isError results, not JSON-RPC errors, // so the client sees a readable message (msgvault convention). @@ -128,9 +225,12 @@ function buildServer(scope) { return server; } -export function mountMcp(app) { +export function mountMcp(app, deps = {}) { + // get_triage_context's matched-rules section needs a read-only rules loader; + // the adapter's scoped listRules is the natural default, overridable in tests. + const resolvedDeps = { loadInboxRules: listRules, ...deps }; const handle = async (req, res) => { - const server = buildServer(req.mcpScope); + const server = buildServer(req.mcpScope, resolvedDeps); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); res.on('close', () => { transport.close(); server.close(); }); await server.connect(transport); diff --git a/backend/src/mcp/server.test.js b/backend/src/mcp/server.test.js index 38301342..6934ec01 100644 --- a/backend/src/mcp/server.test.js +++ b/backend/src/mcp/server.test.js @@ -4,7 +4,7 @@ import { createServer } from 'http'; vi.mock('../services/db.js', () => ({ query: vi.fn() })); import { query } from '../services/db.js'; -import { hashToken } from './auth.js'; +import { ALL_SCOPES, hashToken } from './auth.js'; import { mountMcp } from './server.js'; let server, base; @@ -20,9 +20,9 @@ beforeAll(async () => { afterAll(() => new Promise((r) => server.close(r))); // Every authed request: token lookup -> last_used_at update -> resolveScope. -function primeAuth(userId = 'user-1', accountIds = ['acc-1']) { +function primeAuth(userId = 'user-1', accountIds = ['acc-1'], scopes = ALL_SCOPES) { query - .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: userId }] }) + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: userId, scopes }] }) .mockResolvedValueOnce({ rows: [] }) .mockResolvedValueOnce({ rows: accountIds.map((id) => ({ id })) }); } diff --git a/backend/src/mcp/serverGuards.test.js b/backend/src/mcp/serverGuards.test.js index 7329d819..cd4c38ff 100644 --- a/backend/src/mcp/serverGuards.test.js +++ b/backend/src/mcp/serverGuards.test.js @@ -1,12 +1,23 @@ -import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; import express from 'express'; import { createServer } from 'http'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('./accountAdapter.js', async (orig) => { + const actual = await orig(); + return { ...actual, runRules: vi.fn() }; +}); import { query } from '../services/db.js'; +import { runRules } from './accountAdapter.js'; +import { ALL_SCOPES } from './auth.js'; import { - buildAllowedOrigins, mcpOriginGuard, countToolCalls, createMcpRateLimiter, mountMcp, + buildAllowedOrigins, buildServer, mcpBodyLimit, mcpOriginGuard, + classifyToolCalls, countToolCalls, createMcpRateLimiter, + entityTooLargeResponse, mountMcp, } from './server.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; function mockReq({ origin, body, tokenId } = {}) { return { @@ -24,6 +35,144 @@ function mockRes() { return res; } +async function withMcpClient(scope, callback, deps = {}) { + const server = buildServer(scope, deps); + const client = new Client({ name: 'scope-test', version: '1.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + try { + return await callback(client); + } finally { + await client.close(); + } +} + +describe('MCP dependency injection', () => { + it('passes the buildServer dependency bag to a three-argument tool handler', async () => { + const scope = { userId: 'u', accountIds: ['acc-1'], scopes: ['read'] }; + const deps = { marker: Symbol('deps') }; + const handler = vi.fn(async () => ({ + content: [{ type: 'text', text: JSON.stringify({ ok: true }) }], + })); + HANDLERS.__dependency_probe = handler; + TOOL_SCOPES.__dependency_probe = 'read'; + + try { + await withMcpClient(scope, async (client) => { + await client.callTool({ + name: '__dependency_probe', + arguments: { value: 42 }, + }); + }, deps); + expect(handler).toHaveBeenCalledWith({ value: 42 }, scope, deps); + } finally { + delete HANDLERS.__dependency_probe; + delete TOOL_SCOPES.__dependency_probe; + } + }); + + it('mounts both with a dependency bag and without one', () => { + expect(() => mountMcp(express(), { marker: true })).not.toThrow(); + expect(() => mountMcp(express())).not.toThrow(); + }); +}); + +describe('tool scope classifications', () => { + it('classifies every listed tool and handler', () => { + const definedNames = TOOL_DEFS.map(({ name }) => name).sort(); + expect(Object.keys(HANDLERS).sort()).toEqual(definedNames); + expect(Object.keys(TOOL_SCOPES).sort()).toEqual(definedNames); + }); +}); + +describe('tool scope enforcement', () => { + it('omits tools from tools/list when the token lacks their scope', async () => { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const { tools } = await client.listTools(); + expect(tools.map(({ name }) => name)).toContain('ping'); + expect(tools.map(({ name }) => name)).not.toContain('stage_deletion'); + }); + }); + + it('refuses tools/call when the token lacks the required scope', async () => { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const result = await client.callTool({ + name: 'stage_deletion', + arguments: { from: 'sender@example.com' }, + }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain( + 'permission_denied: tool "stage_deletion" requires the "write" scope; this token has [read]', + ); + }); + }); + + it('refuses tools/call when the tool has no scope classification', async () => { + const requiredScope = TOOL_SCOPES.ping; + delete TOOL_SCOPES.ping; + try { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const result = await client.callTool({ name: 'ping', arguments: {} }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain( + 'permission_denied: tool "ping" has no scope classification', + ); + }); + } finally { + TOOL_SCOPES.ping = requiredScope; + } + }); + + it('allows tools/call when the token has the required scope', async () => { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const result = await client.callTool({ name: 'ping', arguments: {} }); + expect(JSON.parse(result.content[0].text)).toEqual({ pong: true }); + }); + }); + + it('requires both settings and write before run_rules reaches its adapter', async () => { + const deps = { imapManager: { marker: 'injected' } }; + runRules.mockReset().mockResolvedValue({ processed: 0, matched: 0 }); + + await withMcpClient({ + userId: 'u', + accountIds: ['acc-1'], + scopes: ['settings'], + }, async (client) => { + const result = await client.callTool({ + name: 'run_rules', + arguments: {}, + }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain('permission_denied'); + expect(runRules).not.toHaveBeenCalled(); + }, deps); + + await withMcpClient({ + userId: 'u', + accountIds: ['acc-1'], + scopes: ['settings', 'write', 'read'], + }, async (client) => { + const result = await client.callTool({ + name: 'run_rules', + arguments: {}, + }); + expect(result.isError).toBeUndefined(); + expect(JSON.parse(result.content[0].text)).toEqual({ + processed: 0, + matched: 0, + }); + expect(runRules).toHaveBeenCalledTimes(1); + expect(runRules).toHaveBeenCalledWith({ + userId: 'u', + accountIds: ['acc-1'], + imapManager: deps.imapManager, + }); + }, deps); + }); +}); + describe('buildAllowedOrigins', () => { it('derives normalized origins from APP_URL, FRONTEND_URL, and MCP_ALLOWED_ORIGINS', () => { const allowed = buildAllowedOrigins({ @@ -102,6 +251,62 @@ describe('mcpOriginGuard', () => { }); }); +describe('mcpBodyLimit', () => { + it('parses a 2 MB MCP body before the global 1 MB parser', async () => { + const app = express(); + app.use('/mcp', mcpBodyLimit()); + app.use(express.json({ limit: '1mb' })); + app.post('/mcp', (req, res) => res.json({ size: req.body.payload.length })); + const server = createServer(app); + await new Promise((resolve) => server.listen(0, resolve)); + + try { + const payload = 'x'.repeat(2 * 1024 * 1024); + const response = await fetch(`http://127.0.0.1:${server.address().port}/mcp`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ payload }), + }); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ size: payload.length }); + } finally { + await new Promise((resolve) => server.close(resolve)); + } + }); +}); + +describe('entityTooLargeResponse', () => { + it('returns a JSON-RPC-shaped MCP 413 response', () => { + expect(entityTooLargeResponse( + { type: 'entity.too.large' }, + '/mcp', + )).toEqual({ + status: 413, + body: { + jsonrpc: '2.0', + error: { code: -32000, message: 'Request too large (max 35 MB).' }, + id: null, + }, + }); + }); + + it('preserves the existing REST attachment 413 response', () => { + expect(entityTooLargeResponse( + { type: 'entity.too.large' }, + '/api/mail/send', + )).toEqual({ + status: 413, + body: { + error: 'Request too large. Total attachment size must not exceed 25 MB.', + }, + }); + }); + + it('returns null for other errors so Express can forward them', () => { + expect(entityTooLargeResponse(new Error('boom'), '/mcp')).toBeNull(); + }); +}); + describe('countToolCalls', () => { it('counts a single tools/call body as 1 and other methods as 0', () => { expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/call', id: 1 })).toBe(1); @@ -117,34 +322,97 @@ describe('countToolCalls', () => { }); }); +describe('classifyToolCalls', () => { + it('counts tool calls by the tool scope class and ignores handshake methods', () => { + TOOL_SCOPES.__send_probe = 'send'; + TOOL_SCOPES.__settings_probe = 'settings'; + try { + expect(classifyToolCalls([ + { method: 'initialize' }, + { method: 'tools/call', params: { name: 'ping' } }, + { method: 'tools/call', params: { name: 'stage_deletion' } }, + { method: 'tools/call', params: { name: '__send_probe' } }, + { method: 'tools/call', params: { name: '__settings_probe' } }, + ])).toEqual({ read: 1, write: 1, send: 1, settings: 1 }); + } finally { + delete TOOL_SCOPES.__send_probe; + delete TOOL_SCOPES.__settings_probe; + } + }); + + it('uses the first listed scope as an array-scoped tool primary class', () => { + TOOL_SCOPES.__array_probe = ['send', 'write']; + try { + expect(classifyToolCalls({ + method: 'tools/call', + params: { name: '__array_probe' }, + })).toEqual({ read: 0, write: 0, send: 1, settings: 0 }); + } finally { + delete TOOL_SCOPES.__array_probe; + } + }); + + it('charges unknown tool names to read and returns zeros for non-call bodies', () => { + expect(classifyToolCalls({ + method: 'tools/call', + params: { name: 'typo_tool' }, + })).toEqual({ read: 1, write: 0, send: 0, settings: 0 }); + expect(classifyToolCalls({ method: 'tools/list' })) + .toEqual({ read: 0, write: 0, send: 0, settings: 0 }); + expect(classifyToolCalls(undefined)) + .toEqual({ read: 0, write: 0, send: 0, settings: 0 }); + }); +}); + describe('createMcpRateLimiter', () => { let clock; const now = () => clock; - const call = (id = 1) => ({ jsonrpc: '2.0', method: 'tools/call', id }); + const call = (name = 'ping', id = 1) => ({ + jsonrpc: '2.0', + method: 'tools/call', + params: { name }, + id, + }); + const limits = { read: 1, write: 1, send: 1, settings: 1 }; - beforeEach(() => { clock = 1_000_000; }); + beforeEach(() => { + clock = 1_000_000; + TOOL_SCOPES.__send_probe = 'send'; + TOOL_SCOPES.__settings_probe = 'settings'; + }); + afterEach(() => { + delete TOOL_SCOPES.__send_probe; + delete TOOL_SCOPES.__settings_probe; + vi.unstubAllEnvs(); + }); + + it.each([ + ['read', 'ping'], + ['write', 'stage_deletion'], + ['send', '__send_probe'], + ['settings', '__settings_probe'], + ])('enforces the %s per-minute bucket independently', (rateClass, toolName) => { + const limiter = createMcpRateLimiter({ limits, now }); + const first = vi.fn(); + limiter(mockReq({ body: call(toolName), tokenId: 'tok-1' }), mockRes(), first); + expect(first).toHaveBeenCalledTimes(1); - it('allows up to the limit then 429s with Retry-After and a JSON-RPC error', () => { - const limiter = createMcpRateLimiter({ limit: 2, now }); - for (let i = 0; i < 2; i++) { - const next = vi.fn(); - limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), next); - expect(next).toHaveBeenCalledTimes(1); - } const res = mockRes(); - const next = vi.fn(); - limiter(mockReq({ body: call(42), tokenId: 'tok-1' }), res, next); - expect(next).not.toHaveBeenCalled(); + limiter(mockReq({ body: call(toolName, 42), tokenId: 'tok-1' }), res, vi.fn()); expect(res.statusCode).toBe(429); expect(res.headers['Retry-After']).toBe(60); - expect(res.body.jsonrpc).toBe('2.0'); - expect(res.body.error.code).toBe(-32000); - expect(res.body.error.message).toMatch(/rate limit/i); - expect(res.body.id).toBe(42); // correlates with the throttled request + expect(res.body).toEqual({ + jsonrpc: '2.0', + error: { + code: -32000, + message: `Rate limit exceeded: at most 1 ${rateClass} tool calls per minute per token`, + }, + id: 42, + }); }); it('keys buckets per token, not globally', () => { - const limiter = createMcpRateLimiter({ limit: 1, now }); + const limiter = createMcpRateLimiter({ limits, now }); limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); const next = vi.fn(); limiter(mockReq({ body: call(), tokenId: 'tok-2' }), mockRes(), next); @@ -152,7 +420,7 @@ describe('createMcpRateLimiter', () => { }); it('never throttles the initialize/tools-list handshake', () => { - const limiter = createMcpRateLimiter({ limit: 1, now }); + const limiter = createMcpRateLimiter({ limits, now }); for (const method of ['initialize', 'notifications/initialized', 'tools/list', 'tools/list']) { const next = vi.fn(); limiter(mockReq({ body: { jsonrpc: '2.0', method, id: 1 }, tokenId: 'tok-1' }), mockRes(), next); @@ -161,7 +429,7 @@ describe('createMcpRateLimiter', () => { }); it('resets the budget after the window elapses', () => { - const limiter = createMcpRateLimiter({ limit: 1, now }); + const limiter = createMcpRateLimiter({ limits, now }); limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); const blocked = mockRes(); limiter(mockReq({ body: call(), tokenId: 'tok-1' }), blocked, vi.fn()); @@ -172,18 +440,63 @@ describe('createMcpRateLimiter', () => { expect(next).toHaveBeenCalledTimes(1); }); - it('reads the default limit from MCP_RATE_LIMIT_PER_MIN', () => { + it('reads every default class limit from its environment variable', () => { vi.stubEnv('MCP_RATE_LIMIT_PER_MIN', '1'); - try { - const limiter = createMcpRateLimiter({ now }); - limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + vi.stubEnv('MCP_WRITE_RATE_LIMIT_PER_MIN', '1'); + vi.stubEnv('MCP_SEND_RATE_LIMIT_PER_MIN', '1'); + vi.stubEnv('MCP_SETTINGS_RATE_LIMIT_PER_MIN', '1'); + const limiter = createMcpRateLimiter({ now }); + for (const toolName of ['ping', 'stage_deletion', '__send_probe', '__settings_probe']) { + limiter(mockReq({ body: call(toolName), tokenId: 'tok-1' }), mockRes(), vi.fn()); const res = mockRes(); - limiter(mockReq({ body: call(), tokenId: 'tok-1' }), res, vi.fn()); + limiter(mockReq({ body: call(toolName), tokenId: 'tok-1' }), res, vi.fn()); expect(res.statusCode).toBe(429); - } finally { - vi.unstubAllEnvs(); } }); + + it('rejects an over-budget batch atomically with a null JSON-RPC id', () => { + const limiter = createMcpRateLimiter({ limits, now }); + const batch = [ + call('ping', 1), + call('stage_deletion', 2), + call('stage_deletion', 3), + ]; + const res = mockRes(); + limiter(mockReq({ body: batch, tokenId: 'tok-1' }), res, vi.fn()); + expect(res.statusCode).toBe(429); + expect(res.body.id).toBeNull(); + expect(res.body.error.message).toContain('write tool calls'); + + // Rejection admitted none of the batch, including its otherwise-valid read. + const next = vi.fn(); + limiter(mockReq({ body: call('ping'), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('enforces and resets the 24-hour send cap alongside the minute bucket', () => { + vi.stubEnv('MCP_SEND_DAILY_CAP', '2'); + const limiter = createMcpRateLimiter({ + limits: { ...limits, send: 10 }, + now, + }); + for (let i = 0; i < 2; i++) { + const next = vi.fn(); + limiter(mockReq({ body: call('__send_probe'), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + } + + const blocked = mockRes(); + limiter(mockReq({ body: call('__send_probe', 9), tokenId: 'tok-1' }), blocked, vi.fn()); + expect(blocked.statusCode).toBe(429); + expect(blocked.headers['Retry-After']).toBe(86_400); + expect(blocked.body.error.message) + .toBe('Rate limit exceeded: at most 2 send tool calls per day per token'); + + clock += 86_400_001; + const next = vi.fn(); + limiter(mockReq({ body: call('__send_probe'), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); }); // --- End-to-end through the live Express mount (same harness as server.test.js) --- @@ -208,7 +521,7 @@ describe('mounted /mcp guards', () => { // Every authed request: token lookup -> last_used_at update -> resolveScope. function primeAuth() { query - .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: 'user-1' }] }) + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: 'user-1', scopes: ALL_SCOPES }] }) .mockResolvedValueOnce({ rows: [] }) .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); } diff --git a/backend/src/mcp/tools.js b/backend/src/mcp/tools.js index 83779aa9..0860b143 100644 --- a/backend/src/mcp/tools.js +++ b/backend/src/mcp/tools.js @@ -14,6 +14,64 @@ import { searchInMessageDef, handleSearchInMessage, stageDeletionDef, handleStageDeletion, } from './messageTools.js'; +import { + createDraftDef, handleCreateDraft, + updateDraftDef, handleUpdateDraft, + listDraftsDef, handleListDrafts, + getDraftDef, handleGetDraft, + deleteDraftDef, handleDeleteDraft, +} from './draftTools.js'; +import { + sendEmailDef, handleSendEmail, + sendDraftDef, handleSendDraft, + unsendEmailDef, handleUnsendEmail, + listOutboxDef, handleListOutbox, + recallEmailDef, handleRecallEmail, +} from './sendTools.js'; +import { + replyEmailDef, handleReplyEmail, + replyAllEmailDef, handleReplyAllEmail, + forwardEmailDef, handleForwardEmail, +} from './composeTools.js'; +import { + listFoldersDef, handleListFolders, + createFolderDef, handleCreateFolder, + renameFolderDef, handleRenameFolder, + deleteFolderDef, handleDeleteFolder, + moveMessagesDef, handleMoveMessages, + archiveMessagesDef, handleArchiveMessages, + trashMessagesDef, handleTrashMessages, + markReadDef, handleMarkRead, + markUnreadDef, handleMarkUnread, + starMessageDef, handleStarMessage, + unstarMessageDef, handleUnstarMessage, + markSpamDef, handleMarkSpam, + markNotSpamDef, handleMarkNotSpam, + snoozeMessageDef, handleSnoozeMessage, + unsnoozeMessageDef, handleUnsnoozeMessage, + setCategoryDef, handleSetCategory, + gtdClassifyDef, handleGtdClassify, + gtdDoneDef, handleGtdDone, +} from './mailboxTools.js'; +import { + markTriagedDef, handleMarkTriaged, + triageInboxDef, handleTriageInbox, + getTriageContextDef, handleGetTriageContext, +} from './triageTools.js'; +import { + listAccountsDef, handleListAccounts, + addAccountDef, handleAddAccount, + updateAccountSettingsDef, handleUpdateAccountSettings, + testAccountConnectionDef, handleTestAccountConnection, + createAliasDef, handleCreateAlias, + updateAliasDef, handleUpdateAlias, + deleteAliasDef, handleDeleteAlias, + listRulesDef, handleListRules, + createRuleDef, handleCreateRule, + updateRuleDef, handleUpdateRule, + deleteRuleDef, handleDeleteRule, + runRulesDef, handleRunRules, +} from './accountTools.js'; // TOOL_DEFS drives tools/list; HANDLERS drives tools/call. Slices 09 and 10 // append to both. Keep names, descriptions, and inputSchema field names verbatim @@ -22,6 +80,12 @@ export const TOOL_DEFS = [ { name: 'ping', description: 'Health check: returns {"pong":true}. Proves transport + auth round-trip.', + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, inputSchema: { type: 'object', properties: {} }, }, searchMetadataDef, @@ -35,8 +99,117 @@ export const TOOL_DEFS = [ findSimilarMessagesDef, searchInMessageDef, stageDeletionDef, + createDraftDef, + updateDraftDef, + listDraftsDef, + getDraftDef, + deleteDraftDef, + sendEmailDef, + sendDraftDef, + unsendEmailDef, + listOutboxDef, + recallEmailDef, + replyEmailDef, + replyAllEmailDef, + forwardEmailDef, + listFoldersDef, + createFolderDef, + renameFolderDef, + deleteFolderDef, + moveMessagesDef, + archiveMessagesDef, + trashMessagesDef, + markReadDef, + markUnreadDef, + starMessageDef, + unstarMessageDef, + markSpamDef, + markNotSpamDef, + snoozeMessageDef, + unsnoozeMessageDef, + setCategoryDef, + gtdClassifyDef, + gtdDoneDef, + markTriagedDef, + triageInboxDef, + getTriageContextDef, + listAccountsDef, + addAccountDef, + updateAccountSettingsDef, + testAccountConnectionDef, + createAliasDef, + updateAliasDef, + deleteAliasDef, + listRulesDef, + createRuleDef, + updateRuleDef, + deleteRuleDef, + runRulesDef, ]; +// Name → required scope. Keep this map separate from TOOL_DEFS because those +// definitions are serialized verbatim onto the tools/list wire response. +export const TOOL_SCOPES = { + ping: 'read', + search_metadata: 'read', + search_message_bodies: 'read', + semantic_search_messages: 'read', + get_message: 'read', + list_messages: 'read', + get_stats: 'read', + aggregate: 'read', + search_by_domains: 'read', + find_similar_messages: 'read', + search_in_message: 'read', + stage_deletion: 'write', + create_draft: 'write', + update_draft: 'write', + list_drafts: 'read', + get_draft: 'read', + delete_draft: 'write', + send_email: 'send', + send_draft: 'send', + unsend_email: 'send', + list_outbox: 'read', + recall_email: 'send', + reply_email: 'send', + reply_all_email: 'send', + forward_email: 'send', + list_folders: 'read', + create_folder: 'write', + rename_folder: 'write', + delete_folder: 'write', + move_messages: 'write', + archive_messages: 'write', + trash_messages: 'write', + mark_read: 'write', + mark_unread: 'write', + star_message: 'write', + unstar_message: 'write', + mark_spam: 'write', + mark_not_spam: 'write', + snooze_message: 'write', + unsnooze_message: 'write', + set_category: 'write', + gtd_classify: 'write', + gtd_done: 'write', + mark_triaged: 'write', + triage_inbox: 'read', + get_triage_context: 'read', + list_accounts: 'read', + add_account: 'settings', + update_account_settings: 'settings', + test_account_connection: 'settings', + create_alias: 'settings', + update_alias: 'settings', + delete_alias: 'settings', + list_rules: 'settings', + create_rule: 'settings', + update_rule: 'settings', + delete_rule: 'settings', + run_rules: ['settings', 'write'], +}; + export const HANDLERS = { // eslint-disable-next-line no-unused-vars ping: async (_args, _scope) => jsonResult({ pong: true }), @@ -51,4 +224,50 @@ export const HANDLERS = { find_similar_messages: (a, s) => handleFindSimilarMessages(a, s), search_in_message: (a, s) => handleSearchInMessage(a, s), stage_deletion: (a, s) => handleStageDeletion(a, s), + create_draft: handleCreateDraft, + update_draft: handleUpdateDraft, + list_drafts: handleListDrafts, + get_draft: handleGetDraft, + delete_draft: handleDeleteDraft, + send_email: handleSendEmail, + send_draft: handleSendDraft, + unsend_email: handleUnsendEmail, + list_outbox: handleListOutbox, + recall_email: handleRecallEmail, + reply_email: handleReplyEmail, + reply_all_email: handleReplyAllEmail, + forward_email: handleForwardEmail, + list_folders: handleListFolders, + create_folder: handleCreateFolder, + rename_folder: handleRenameFolder, + delete_folder: handleDeleteFolder, + move_messages: handleMoveMessages, + archive_messages: handleArchiveMessages, + trash_messages: handleTrashMessages, + mark_read: handleMarkRead, + mark_unread: handleMarkUnread, + star_message: handleStarMessage, + unstar_message: handleUnstarMessage, + mark_spam: handleMarkSpam, + mark_not_spam: handleMarkNotSpam, + snooze_message: handleSnoozeMessage, + unsnooze_message: handleUnsnoozeMessage, + set_category: handleSetCategory, + gtd_classify: handleGtdClassify, + gtd_done: handleGtdDone, + mark_triaged: handleMarkTriaged, + triage_inbox: handleTriageInbox, + get_triage_context: handleGetTriageContext, + list_accounts: handleListAccounts, + add_account: handleAddAccount, + update_account_settings: handleUpdateAccountSettings, + test_account_connection: handleTestAccountConnection, + create_alias: handleCreateAlias, + update_alias: handleUpdateAlias, + delete_alias: handleDeleteAlias, + list_rules: handleListRules, + create_rule: handleCreateRule, + update_rule: handleUpdateRule, + delete_rule: handleDeleteRule, + run_rules: handleRunRules, }; diff --git a/backend/src/mcp/tools.test.js b/backend/src/mcp/tools.test.js new file mode 100644 index 00000000..c0821841 --- /dev/null +++ b/backend/src/mcp/tools.test.js @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +const VALID_SCOPES = new Set(['read', 'write', 'send', 'settings']); +const ANNOTATION_KEYS = [ + 'readOnlyHint', + 'destructiveHint', + 'idempotentHint', + 'openWorldHint', +]; + +describe('MCP tool registry invariants', () => { + it('gives every tool definition all four boolean annotation hints', () => { + for (const definition of TOOL_DEFS) { + expect(definition.annotations, definition.name).toBeTypeOf('object'); + for (const key of ANNOTATION_KEYS) { + expect(definition.annotations, `${definition.name}.${key}`).toHaveProperty(key); + expect(typeof definition.annotations[key], `${definition.name}.${key}`).toBe('boolean'); + } + } + }); + + it('classifies every definition with one or more valid scopes', () => { + for (const definition of TOOL_DEFS) { + const required = TOOL_SCOPES[definition.name]; + if (Array.isArray(required)) { + expect(required.length, definition.name).toBeGreaterThanOrEqual(1); + for (const scope of required) { + expect(VALID_SCOPES.has(scope), `${definition.name}: ${scope}`).toBe(true); + } + } else { + expect(VALID_SCOPES.has(required), `${definition.name}: ${required}`).toBe(true); + } + } + }); + + it('keeps definitions, scope classifications, and handlers in exact three-way sync', () => { + const definitions = TOOL_DEFS.map(({ name }) => name).sort(); + expect(Object.keys(TOOL_SCOPES).sort()).toEqual(definitions); + expect(Object.keys(HANDLERS).sort()).toEqual(definitions); + }); +}); diff --git a/backend/src/mcp/triageAdapter.integration.test.js b/backend/src/mcp/triageAdapter.integration.test.js new file mode 100644 index 00000000..329e5800 --- /dev/null +++ b/backend/src/mcp/triageAdapter.integration.test.js @@ -0,0 +1,145 @@ +import { randomUUID } from 'crypto'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from '../services/embeddings/testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const integrationDescribe = DSN ? describe : describe.skip; + +integrationDescribe('triageAdapter', () => { + let client; + let adapter; + let db; + let userId; + let accountId; + let nextUid = 700000; + + beforeAll(async () => { + const url = new URL(DSN); + process.env.DB_HOST = url.hostname; + process.env.DB_PORT = url.port; + process.env.DB_NAME = url.pathname.slice(1); + process.env.DB_USER = url.username; + process.env.DB_PASSWORD = url.password; + + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + ({ userId, accountId } = await seedAccount(client, 'triage-adapter')); + adapter = await import('./triageAdapter.js'); + db = await import('../services/db.js'); + }); + + afterAll(async () => { + await cleanupAccount(client, userId); + await client.end(); + await db.pool.end(); + }); + + async function insertMessage({ + date, + header = `<${randomUUID()}@example.com>`, + folder = 'INBOX', + isRead = false, + threadId = randomUUID(), + }) { + const result = await client.query( + `INSERT INTO messages + (account_id, uid, folder, message_id, subject, from_email, date, is_read, thread_id) + VALUES ($1, $2, $3, $4, $5, 'sender@example.com', $6, $7, $8) + RETURNING id, message_id`, + [accountId, nextUid++, folder, header, `message ${nextUid}`, date, isRead, threadId], + ); + return result.rows[0]; + } + + it('keyset-pages a seeded inbox without duplicates or gaps under a mid-stream insert', async () => { + const expectedIds = []; + for (let index = 0; index < 60; index++) { + const message = await insertMessage({ + date: new Date(Date.UTC(2026, 0, 1, 0, index)).toISOString(), + header: ``, + }); + expectedIds.push(message.id); + } + + const seen = []; + let page = await adapter.listTriageCandidates({ + accountIds: [accountId], + limit: 20, + unreadOnly: true, + }); + seen.push(...page.rows.map(row => row.id)); + + const inserted = await insertMessage({ + date: new Date(Date.UTC(2026, 0, 1, 0, 25, 30)).toISOString(), + header: '', + }); + expectedIds.push(inserted.id); + + while (page.hasMore) { + page = await adapter.listTriageCandidates({ + accountIds: [accountId], + cursor: page.cursor, + limit: 20, + unreadOnly: true, + }); + seen.push(...page.rows.map(row => row.id)); + } + + expect(new Set(seen).size).toBe(seen.length); + expect(new Set(seen)).toEqual(new Set(expectedIds)); + }); + + it('marks idempotently and distinguishes INSERT from conflict UPDATE with xmax=0', async () => { + const message = await insertMessage({ date: '2026-02-01T00:00:00.000Z' }); + const args = { + userId, + accountIds: [accountId], + messageIds: [message.id], + action: 'archived', + note: 'integration', + tokenId: null, + }; + + await expect(adapter.markTriaged(args)).resolves.toMatchObject({ + marked: 1, + newly_marked: 1, + already_triaged: 0, + }); + await expect(adapter.markTriaged(args)).resolves.toMatchObject({ + marked: 1, + newly_marked: 0, + already_triaged: 1, + }); + }); + + it('skips null headers and cascades checkpoints when the account is deleted', async () => { + const noHeader = await insertMessage({ + date: '2026-02-02T00:00:00.000Z', + header: null, + }); + const durable = await insertMessage({ date: '2026-02-03T00:00:00.000Z' }); + + await expect(adapter.markTriaged({ + userId, + accountIds: [accountId], + messageIds: [noHeader.id], + })).resolves.toMatchObject({ + marked: 0, + skipped: [{ id: noHeader.id, reason: 'no_message_id_header' }], + }); + + await adapter.markTriaged({ + userId, + accountIds: [accountId], + messageIds: [durable.id], + }); + await client.query('DELETE FROM email_accounts WHERE id = $1', [accountId]); + + const result = await client.query( + 'SELECT id FROM message_triage WHERE account_id = $1', + [accountId], + ); + expect(result.rows).toEqual([]); + }); +}); diff --git a/backend/src/mcp/triageAdapter.js b/backend/src/mcp/triageAdapter.js new file mode 100644 index 00000000..1314c2bf --- /dev/null +++ b/backend/src/mcp/triageAdapter.js @@ -0,0 +1,342 @@ +// SQL owner for the inbox-triage MCP surface. Handlers validate and shape wire +// envelopes; this adapter keeps every read/checkpoint query scoped to accountIds. +import { query } from '../services/db.js'; + +function decodeCursor(cursor) { + if (!cursor) return null; + try { + const parsed = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8')); + if ( + !parsed + || typeof parsed.d !== 'string' + || !parsed.d + || typeof parsed.h !== 'string' + || !parsed.h + ) { + throw new Error('invalid shape'); + } + return parsed; + } catch { + throw new Error('invalid triage cursor'); + } +} + +function encodeCursor(row) { + if (!row) return null; + const date = row.date instanceof Date ? row.date.toISOString() : row.date; + return Buffer.from(JSON.stringify({ d: date, h: row.message_id }), 'utf8').toString('base64'); +} + +export async function listTriageCandidates({ + accountIds, + cursor, + limit = 25, + unreadOnly = true, + includeTriaged = false, + categories, + since, +}) { + const pageLimit = Number(limit); + if (!accountIds?.length) return { rows: [], hasMore: false, cursor: null }; + + const params = [accountIds]; + const bind = value => { + params.push(value); + return `$${params.length}`; + }; + const where = [ + 'm.account_id = ANY($1)', + "m.folder = 'INBOX'", + 'm.is_deleted = false', + 'm.message_id IS NOT NULL', + ]; + + if (unreadOnly) where.push('m.is_read = false'); + if (!includeTriaged) { + where.push(`NOT EXISTS ( + SELECT 1 + FROM message_triage mt + WHERE mt.account_id = m.account_id + AND mt.message_id_header = m.message_id + )`); + } + if (categories?.length) { + where.push(`COALESCE(m.category, 'primary') = ANY(${bind(categories)})`); + } + if (since) where.push(`m.date >= ${bind(since)}`); + + const decodedCursor = decodeCursor(cursor); + if (decodedCursor) { + const dateParam = bind(decodedCursor.d); + const headerParam = bind(decodedCursor.h); + where.push(`(m.date, m.message_id) > (${dateParam}, ${headerParam})`); + } + + const sql = ` + WITH sender_history AS ( + SELECT + lower(mh.from_email) AS sender_email, + COUNT(*)::int AS received_count, + MIN(mh.date) AS first_received, + MAX(mh.date) AS last_received + FROM messages mh + WHERE mh.account_id = ANY($1) + AND mh.is_deleted = false + AND mh.from_email IS NOT NULL + GROUP BY lower(mh.from_email) + ) + SELECT + m.id, + m.account_id, + a.email_address AS account, + m.message_id, + m.thread_key AS conversation_id, + m.subject, + m.snippet, + m.from_email, + m.from_name, + m.date, + m.is_read, + m.is_starred, + m.has_attachments, + COALESCE(m.category, 'primary') AS category, + COALESCE(m.is_bulk, false) AS is_bulk, + (m.list_unsubscribe IS NOT NULL) AS has_unsubscribe, + m.spam_verdict, + thread_state.message_count AS thread_message_count, + thread_state.last_activity AS thread_last_activity, + thread_state.i_replied, + COALESCE(sh.received_count, 0)::int AS received_count, + sh.first_received, + sh.last_received, + c.id AS contact_id, + c.display_name AS contact_name, + COALESCE(c.send_count, 0)::int AS send_count, + c.last_sent, + c.is_auto, + (c.id IS NOT NULL) AS contact_known + FROM messages m + JOIN email_accounts a ON a.id = m.account_id + LEFT JOIN LATERAL ( + SELECT + COUNT(*)::int AS message_count, + MAX(t.date) AS last_activity, + EXISTS ( + SELECT 1 + FROM messages sent + WHERE sent.account_id = m.account_id + AND sent.thread_key = m.thread_key + AND sent.is_deleted = false + AND ( + sent.folder = COALESCE(a.folder_mappings->>'sent', '') + OR sent.folder ~* '(^|[./ ])sent($|[./ ])' + OR EXISTS ( + SELECT 1 + FROM folders sf + WHERE sf.account_id = sent.account_id + AND sf.path = sent.folder + AND sf.special_use = '\\Sent' + ) + ) + ) AS i_replied + FROM messages t + WHERE t.account_id = m.account_id + AND t.thread_key = m.thread_key + AND t.is_deleted = false + ) thread_state ON true + LEFT JOIN sender_history sh ON sh.sender_email = lower(m.from_email) + LEFT JOIN contacts c + ON c.user_id = a.user_id + AND c.primary_email = lower(m.from_email) + WHERE ${where.join('\n AND ')} + ORDER BY m.date ASC, m.message_id ASC + LIMIT ${bind(pageLimit + 1)} + `; + + const result = await query(sql, params); + const hasMore = result.rows.length > pageLimit; + const rows = result.rows.slice(0, pageLimit); + return { + rows, + hasMore, + cursor: encodeCursor(rows.at(-1)), + }; +} + +// Backlog total for triage_inbox's counts.untriaged_unread: unread INBOX +// messages not yet checkpointed in message_triage, regardless of paging. +export async function countUntriagedUnread(accountIds) { + if (!accountIds?.length) return 0; + const { rows } = await query( + `SELECT COUNT(*) AS total + FROM messages m + WHERE m.account_id = ANY($1) + AND m.folder = 'INBOX' + AND m.is_deleted = false + AND m.message_id IS NOT NULL + AND m.is_read = false + AND NOT EXISTS ( + SELECT 1 + FROM message_triage mt + WHERE mt.account_id = m.account_id + AND mt.message_id_header = m.message_id + )`, + [accountIds], + ); + return parseInt(rows[0]?.total, 10) || 0; +} + +export async function senderHistory(fromEmail, accountIds) { + if (!fromEmail || !accountIds?.length) return null; + const { rows } = await query( + `WITH sender_history AS ( + SELECT + COUNT(*)::int AS received_count, + MIN(m.date) AS first_received, + MAX(m.date) AS last_received + FROM messages m + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND lower(m.from_email) = lower($2) + ) + SELECT + sh.received_count, + sh.first_received, + sh.last_received, + c.id AS contact_id, + c.display_name AS contact_name, + c.primary_email, + COALESCE(c.send_count, 0)::int AS send_count, + c.last_sent, + c.is_auto, + (c.id IS NOT NULL) AS contact_known + FROM sender_history sh + LEFT JOIN contacts c + ON c.user_id IN ( + SELECT a.user_id + FROM email_accounts a + WHERE a.id = ANY($1) + ) + AND c.primary_email = lower($2) + LIMIT 1`, + [accountIds, fromEmail], + ); + return rows[0] || null; +} + +// Resolved Sent folder for disposition classification: the account's explicit +// folder_mappings.sent wins, else the IMAP \Sent special-use folder. Mirrors +// services/mail/sentCopy.js resolveSentFolder but keyed by accountId so the +// signals path never needs a full account row. +export async function sentFolderForAccount(accountId) { + if (!accountId) return null; + const { rows } = await query( + `SELECT COALESCE( + a.folder_mappings->>'sent', + (SELECT f.path FROM folders f + WHERE f.account_id = a.id AND f.special_use = '\\Sent' + LIMIT 1) + ) AS path + FROM email_accounts a + WHERE a.id = $1`, + [accountId], + ); + return rows[0]?.path || null; +} + +export async function triageActionsForMessages(pairs) { + if (!pairs?.length) return []; + const { rows } = await query( + `SELECT + mt.account_id, + mt.message_id_header, + mt.action, + mt.triaged_at + FROM unnest($1::uuid[], $2::text[]) AS input(account_id, message_id_header) + JOIN message_triage mt + ON mt.account_id = input.account_id + AND mt.message_id_header = input.message_id_header`, + [ + pairs.map(pair => pair.accountId), + pairs.map(pair => pair.messageIdHeader), + ], + ); + return rows; +} + +export async function resolveHeadersForIds(ids, accountIds) { + if (!ids?.length || !accountIds?.length) return []; + const { rows } = await query( + `SELECT id, account_id, message_id AS message_id_header + FROM messages + WHERE id = ANY($1::uuid[]) + AND account_id = ANY($2::uuid[])`, + [ids, accountIds], + ); + return rows; +} + +export async function markTriaged({ + userId, + accountIds, + messageIds, + action = null, + note = null, + tokenId = null, +}) { + const resolved = await resolveHeadersForIds(messageIds, accountIds); + const byId = new Map(resolved.map(row => [row.id, row])); + const skipped = []; + const durableByKey = new Map(); + + for (const id of messageIds || []) { + const row = byId.get(id); + if (!row) { + skipped.push({ id, reason: 'not_found_or_out_of_scope' }); + } else if (!row.message_id_header) { + skipped.push({ id, reason: 'no_message_id_header' }); + } else { + durableByKey.set(`${row.account_id}\0${row.message_id_header}`, row); + } + } + + const durable = [...durableByKey.values()]; + if (!durable.length) { + return { + ok: true, + marked: 0, + newly_marked: 0, + already_triaged: 0, + skipped, + }; + } + + const { rows } = await query( + `INSERT INTO message_triage + (user_id, account_id, message_id_header, action, note, source, token_id) + SELECT $1, input.account_id, input.message_id_header, $4, $5, 'mcp', $6 + FROM unnest($2::uuid[], $3::text[]) AS input(account_id, message_id_header) + ON CONFLICT (account_id, message_id_header) DO UPDATE + SET triaged_at = NOW(), + action = EXCLUDED.action, + note = EXCLUDED.note + RETURNING account_id, message_id_header, (xmax = 0) AS inserted`, + [ + userId, + durable.map(row => row.account_id), + durable.map(row => row.message_id_header), + action, + note, + tokenId, + ], + ); + + const newlyMarked = rows.filter(row => row.inserted === true).length; + return { + ok: true, + marked: rows.length, + newly_marked: newlyMarked, + already_triaged: rows.length - newlyMarked, + skipped, + }; +} diff --git a/backend/src/mcp/triageAdapter.test.js b/backend/src/mcp/triageAdapter.test.js new file mode 100644 index 00000000..da6a5646 --- /dev/null +++ b/backend/src/mcp/triageAdapter.test.js @@ -0,0 +1,344 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); + +import { query } from '../services/db.js'; + +const adapterPromise = import('./triageAdapter.js').catch(() => null); + +function cursor(date = '2026-07-28T08:00:00.000Z', header = '') { + return Buffer.from(JSON.stringify({ d: date, h: header }), 'utf8').toString('base64'); +} + +describe('listTriageCandidates', () => { + beforeEach(() => query.mockReset()); + + it('uses one scoped, enriched inbox query and excludes triaged messages by default', async () => { + const adapter = await adapterPromise; + expect(adapter?.listTriageCandidates).toBeTypeOf('function'); + query.mockResolvedValueOnce({ rows: [] }); + + await adapter.listTriageCandidates({ + accountIds: ['acc-1'], + limit: 25, + unreadOnly: true, + includeTriaged: false, + }); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($1)'); + expect(sql).toContain("m.folder = 'INBOX'"); + expect(sql).toContain('m.is_deleted = false'); + expect(sql).toContain('m.message_id IS NOT NULL'); + expect(sql).toContain('m.is_read = false'); + expect(sql).toMatch(/NOT EXISTS\s*\(\s*SELECT 1\s+FROM message_triage mt/i); + expect(sql).toContain('LEFT JOIN LATERAL'); + expect(sql).toContain('t.thread_key = m.thread_key'); + expect(sql).toContain('AS i_replied'); + expect(sql).toContain('WITH sender_history AS'); + expect(sql).toContain('LEFT JOIN sender_history'); + expect(sql).toContain('LEFT JOIN contacts'); + expect(sql).toContain('received_count'); + expect(sql).not.toContain('receive_count'); + expect(sql).toContain('ORDER BY m.date ASC, m.message_id ASC'); + expect(params[0]).toEqual(['acc-1']); + expect(params.at(-1)).toBe(26); + }); + + it('adds the tuple keyset predicate only when a cursor is supplied', async () => { + const adapter = await adapterPromise; + query.mockResolvedValue({ rows: [] }); + + await adapter.listTriageCandidates({ accountIds: ['acc-1'], limit: 10 }); + expect(query.mock.calls[0][0]).not.toMatch(/\(m\.date,\s*m\.message_id\)\s*>/); + + query.mockClear(); + const date = '2026-07-28T08:00:00.000Z'; + const header = ''; + await adapter.listTriageCandidates({ + accountIds: ['acc-1'], + cursor: cursor(date, header), + limit: 10, + }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/\(m\.date,\s*m\.message_id\)\s*>\s*\(\$\d+,\s*\$\d+\)/); + expect(params).toContain(date); + expect(params).toContain(header); + }); + + it('honors includeTriaged, unreadOnly, categories, and since without losing scope', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ rows: [] }); + + await adapter.listTriageCandidates({ + accountIds: ['acc-1', 'acc-2'], + limit: 5, + unreadOnly: false, + includeTriaged: true, + categories: ['primary', 'newsletter'], + since: '2026-07-01T00:00:00.000Z', + }); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($1)'); + expect(sql).not.toContain('FROM message_triage mt'); + expect(sql).not.toContain('m.is_read = false'); + expect(sql).toContain("COALESCE(m.category, 'primary') = ANY"); + expect(sql).toMatch(/m\.date >= \$\d+/); + expect(params).toContainEqual(['primary', 'newsletter']); + expect(params).toContain('2026-07-01T00:00:00.000Z'); + }); + + it('returns limit rows plus hasMore and a header-based cursor from limit+1 results', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ + rows: [ + { id: 'm1', date: new Date('2026-07-01T00:00:00Z'), message_id: '' }, + { id: 'm2', date: new Date('2026-07-02T00:00:00Z'), message_id: '' }, + { id: 'm3', date: new Date('2026-07-03T00:00:00Z'), message_id: '' }, + ], + }); + + const page = await adapter.listTriageCandidates({ accountIds: ['acc-1'], limit: 2 }); + + expect(page.rows.map(row => row.id)).toEqual(['m1', 'm2']); + expect(page.hasMore).toBe(true); + expect(JSON.parse(Buffer.from(page.cursor, 'base64').toString('utf8'))).toEqual({ + d: '2026-07-02T00:00:00.000Z', + h: '', + }); + }); +}); + +describe('senderHistory', () => { + beforeEach(() => query.mockReset()); + + it('returns computed receive history and the matching scoped contact in one query', async () => { + const adapter = await adapterPromise; + const row = { + received_count: 4, + first_received: '2026-01-01T00:00:00Z', + last_received: '2026-07-01T00:00:00Z', + send_count: 2, + last_sent: '2026-06-01T00:00:00Z', + is_auto: false, + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(adapter.senderHistory('Sender@Example.com', ['acc-1'])).resolves.toEqual(row); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($1)'); + expect(sql).toContain('COUNT(*)::int AS received_count'); + expect(sql).toContain('LEFT JOIN contacts'); + expect(sql).toContain('c.primary_email = lower($2)'); + expect(params).toEqual([['acc-1'], 'Sender@Example.com']); + }); +}); + +describe('resolveHeadersForIds', () => { + beforeEach(() => query.mockReset()); + + it('resolves message row ids to durable headers within account scope', async () => { + const adapter = await adapterPromise; + const rows = [{ id: 'm1', account_id: 'acc-1', message_id_header: '' }]; + query.mockResolvedValueOnce({ rows }); + + await expect(adapter.resolveHeadersForIds(['m1'], ['acc-1'])).resolves.toEqual(rows); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('id = ANY($1'); + expect(sql).toContain('account_id = ANY($2'); + expect(sql).toContain('message_id AS message_id_header'); + expect(params).toEqual([['m1'], ['acc-1']]); + }); +}); + +describe('triageActionsForMessages', () => { + beforeEach(() => query.mockReset()); + + it('reads actions for account/header pairs with one pairwise scoped query', async () => { + const adapter = await adapterPromise; + const pairs = [ + { accountId: 'acc-1', messageIdHeader: '' }, + { accountId: 'acc-2', messageIdHeader: '' }, + ]; + const rows = [ + { + account_id: 'acc-1', + message_id_header: '', + action: 'archived', + triaged_at: '2026-07-28T08:00:00Z', + }, + ]; + query.mockResolvedValueOnce({ rows }); + + await expect(adapter.triageActionsForMessages(pairs)).resolves.toEqual(rows); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch( + /FROM unnest\(\$1::uuid\[\], \$2::text\[\]\) AS input\(account_id, message_id_header\)/, + ); + expect(sql).toContain('JOIN message_triage mt'); + expect(sql).toContain('mt.account_id = input.account_id'); + expect(sql).toContain('mt.message_id_header = input.message_id_header'); + expect(sql).toContain('mt.action'); + expect(sql).toContain('mt.triaged_at'); + expect(params).toEqual([ + ['acc-1', 'acc-2'], + ['', ''], + ]); + }); + + it('returns no actions without querying when the pair list is empty', async () => { + const adapter = await adapterPromise; + + await expect(adapter.triageActionsForMessages([])).resolves.toEqual([]); + + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('sentFolderForAccount', () => { + beforeEach(() => query.mockReset()); + + it('resolves the mapped sent folder with a \\Sent special-use fallback in one query', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ rows: [{ path: 'Custom/Outgoing' }] }); + + await expect(adapter.sentFolderForAccount('acc-1')).resolves.toBe('Custom/Outgoing'); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain("a.folder_mappings->>'sent'"); + expect(sql).toContain("f.special_use = '\\Sent'"); + expect(sql).toContain('WHERE a.id = $1'); + expect(params).toEqual(['acc-1']); + }); + + it('returns null without querying when no account id is given', async () => { + const adapter = await adapterPromise; + + await expect(adapter.sentFolderForAccount(null)).resolves.toBeNull(); + + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('markTriaged', () => { + beforeEach(() => query.mockReset()); + + it('skips a null-header id without issuing an UPSERT for it', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ + rows: [{ id: 'm-null', account_id: 'acc-1', message_id_header: null }], + }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['m-null'], + action: 'left', + note: null, + tokenId: 'token-1', + }); + + expect(query).toHaveBeenCalledTimes(1); + expect(result).toEqual({ + ok: true, + marked: 0, + newly_marked: 0, + already_triaged: 0, + skipped: [{ id: 'm-null', reason: 'no_message_id_header' }], + }); + }); + + it('uses xmax=0 to distinguish new checkpoints from updated checkpoints', async () => { + const adapter = await adapterPromise; + query + .mockResolvedValueOnce({ + rows: [ + { id: 'm-new', account_id: 'acc-1', message_id_header: '' }, + { id: 'm-old', account_id: 'acc-1', message_id_header: '' }, + ], + }) + .mockResolvedValueOnce({ + rows: [ + { account_id: 'acc-1', message_id_header: '', inserted: true }, + { account_id: 'acc-1', message_id_header: '', inserted: false }, + ], + }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['m-new', 'm-old'], + action: 'archived', + note: 'morning pass', + tokenId: 'token-1', + }); + + expect(result).toEqual({ + ok: true, + marked: 2, + newly_marked: 1, + already_triaged: 1, + skipped: [], + }); + const [sql, params] = query.mock.calls[1]; + expect(sql).toContain('ON CONFLICT (account_id, message_id_header) DO UPDATE'); + expect(sql).toContain('triaged_at = NOW()'); + expect(sql).toContain('action = EXCLUDED.action'); + expect(sql).toContain('note = EXCLUDED.note'); + expect(sql).toContain('RETURNING'); + expect(sql).toContain('(xmax = 0) AS inserted'); + expect(params).toEqual([ + 'user-1', + ['acc-1', 'acc-1'], + ['', ''], + 'archived', + 'morning pass', + 'token-1', + ]); + }); + + it('reports unresolved or out-of-scope ids as skipped', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ rows: [] }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['foreign-id'], + }); + + expect(result.skipped).toEqual([{ id: 'foreign-id', reason: 'not_found_or_out_of_scope' }]); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('deduplicates folder-copy rows that share one durable checkpoint key', async () => { + const adapter = await adapterPromise; + query + .mockResolvedValueOnce({ + rows: [ + { id: 'm-inbox', account_id: 'acc-1', message_id_header: '' }, + { id: 'm-label', account_id: 'acc-1', message_id_header: '' }, + ], + }) + .mockResolvedValueOnce({ + rows: [{ account_id: 'acc-1', message_id_header: '', inserted: true }], + }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['m-inbox', 'm-label'], + }); + + expect(result).toMatchObject({ marked: 1, newly_marked: 1, already_triaged: 0 }); + expect(query.mock.calls[1][1][1]).toEqual(['acc-1']); + expect(query.mock.calls[1][1][2]).toEqual(['']); + }); +}); diff --git a/backend/src/mcp/triageProbes.js b/backend/src/mcp/triageProbes.js new file mode 100644 index 00000000..a41db53b --- /dev/null +++ b/backend/src/mcp/triageProbes.js @@ -0,0 +1,70 @@ +import { EmbeddingClient } from '../services/embeddings/client.js'; + +// Fixed v1 probes: they are not user-tunable. Raw cosine scores are not +// calibrated across requests or users; only ranking candidates within one +// response is meaningful, so callers must not treat them as global thresholds. +export const TRIAGE_PROBES = { + urgent: 'urgent, needs action today, deadline, time sensitive', + needs_reply: 'a direct question addressed to me that expects a reply', + financial: 'invoice, payment due, receipt, billing, subscription charge', + scheduling: 'meeting request, calendar invite, reschedule, availability', + bulk: 'newsletter, marketing promotion, unsubscribe, mass mailing', +}; + +const vectorsByFingerprint = new Map(); + +export async function getTriageProbeVectors(cfg, generation) { + const fingerprint = generation?.fingerprint; + if (!fingerprint) throw new Error('embedding generation fingerprint is required'); + if (vectorsByFingerprint.has(fingerprint)) { + return vectorsByFingerprint.get(fingerprint); + } + + const pending = (async () => { + const names = Object.keys(TRIAGE_PROBES); + const vectors = await new EmbeddingClient(cfg).embed(Object.values(TRIAGE_PROBES)); + if (!Array.isArray(vectors) || vectors.length !== names.length) { + throw new Error(`embedder returned ${vectors?.length} probe vectors, want ${names.length}`); + } + return Object.fromEntries(names.map((name, index) => [name, vectors[index]])); + })(); + vectorsByFingerprint.set(fingerprint, pending); + + try { + return await pending; + } catch (error) { + if (vectorsByFingerprint.get(fingerprint) === pending) { + vectorsByFingerprint.delete(fingerprint); + } + throw error; + } +} + +function cosineSimilarity(left, right) { + if (!Array.isArray(left) || !Array.isArray(right)) { + throw new Error('cosine similarity requires vector arrays'); + } + if (left.length !== right.length) { + throw new Error(`probe vector dimension mismatch: candidate=${left.length}, probe=${right.length}`); + } + + let dot = 0; + let leftSquared = 0; + let rightSquared = 0; + for (let index = 0; index < left.length; index++) { + dot += left[index] * right[index]; + leftSquared += left[index] * left[index]; + rightSquared += right[index] * right[index]; + } + if (leftSquared === 0 || rightSquared === 0) return 0; + return dot / (Math.sqrt(leftSquared) * Math.sqrt(rightSquared)); +} + +export function scoreTriageProbes(candidateVector, probeVectors) { + return Object.fromEntries( + Object.keys(TRIAGE_PROBES).map(name => [ + name, + cosineSimilarity(candidateVector, probeVectors?.[name]), + ]), + ); +} diff --git a/backend/src/mcp/triageProbes.test.js b/backend/src/mcp/triageProbes.test.js new file mode 100644 index 00000000..4d935361 --- /dev/null +++ b/backend/src/mcp/triageProbes.test.js @@ -0,0 +1,96 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const clientMocks = vi.hoisted(() => ({ + construct: vi.fn(), + embed: vi.fn(), +})); + +vi.mock('../services/embeddings/client.js', () => ({ + EmbeddingClient: class { + constructor(cfg) { + clientMocks.construct(cfg); + } + + embed(inputs) { + return clientMocks.embed(inputs); + } + }, +})); + +const probesPromise = import('./triageProbes.js').catch(() => null); + +const cfg = { + endpoint: 'https://embeddings.example.test/v1', + apiKey: 'secret', + model: 'embed-model', + dimension: 2, +}; + +const vectors = [ + [1, 0], + [0.8, 0.2], + [0, 1], + [-0.5, 0.5], + [-1, 0], +]; + +describe('getTriageProbeVectors', () => { + beforeEach(() => { + clientMocks.construct.mockReset(); + clientMocks.embed.mockReset(); + }); + + it('embeds all five fixed probes in one client call', async () => { + const probes = await probesPromise; + expect(probes?.getTriageProbeVectors).toBeTypeOf('function'); + clientMocks.embed.mockResolvedValueOnce(vectors); + + const result = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-all-five' }); + + expect(clientMocks.construct).toHaveBeenCalledOnce(); + expect(clientMocks.construct).toHaveBeenCalledWith(cfg); + expect(clientMocks.embed).toHaveBeenCalledOnce(); + expect(clientMocks.embed).toHaveBeenCalledWith(Object.values(probes.TRIAGE_PROBES)); + expect(result).toEqual(Object.fromEntries( + Object.keys(probes.TRIAGE_PROBES).map((name, index) => [name, vectors[index]]), + )); + }); + + it('reuses one cached result for a fingerprint and re-embeds for a new generation', async () => { + const probes = await probesPromise; + clientMocks.embed + .mockResolvedValueOnce(vectors) + .mockResolvedValueOnce(vectors.map(vector => [...vector].reverse())); + + const first = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-cache-a' }); + const second = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-cache-a' }); + + expect(second).toBe(first); + expect(clientMocks.embed).toHaveBeenCalledTimes(1); + + const changed = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-cache-b' }); + expect(changed).not.toBe(first); + expect(clientMocks.embed).toHaveBeenCalledTimes(2); + }); +}); + +describe('scoreTriageProbes', () => { + it('ranks a hand-built candidate closer to the aligned probe', async () => { + const probes = await probesPromise; + expect(probes?.scoreTriageProbes).toBeTypeOf('function'); + + const scores = probes.scoreTriageProbes([1, 0], { + urgent: [0.9, 0.1], + needs_reply: [0.5, 0.5], + financial: [-1, 0], + scheduling: [0.2, 0.8], + bulk: [0, 1], + }); + + expect(Object.keys(scores)).toEqual(Object.keys(probes.TRIAGE_PROBES)); + expect(scores.urgent).toBeGreaterThan(scores.needs_reply); + expect(scores.needs_reply).toBeGreaterThan(scores.bulk); + expect(scores.bulk).toBeGreaterThan(scores.financial); + expect(scores.urgent).toBeCloseTo(0.9939, 4); + }); +}); diff --git a/backend/src/mcp/triageTools.js b/backend/src/mcp/triageTools.js new file mode 100644 index 00000000..26fc6ef2 --- /dev/null +++ b/backend/src/mcp/triageTools.js @@ -0,0 +1,757 @@ +import { + countUntriagedUnread, + listTriageCandidates, + markTriaged, + senderHistory, + sentFolderForAccount, + triageActionsForMessages, +} from './triageAdapter.js'; +import { + getMessage, + getMessageSummariesByIDs, + listMessages, + resolveAccountScope, +} from './engineAdapter.js'; +import { findSimilarSummaries } from './messageTools.js'; +import { errorResult, jsonResult } from './result.js'; +import { toRFC3339, wireSummary } from './envelope.js'; +import { translateVectorError } from './vectorErrors.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { annSearch, loadVector } from '../services/embeddings/vectorStore.js'; +import { runInBatches } from '../services/mailbox/batch.js'; +import { + getTriageProbeVectors, + scoreTriageProbes, +} from './triageProbes.js'; +import { + resolveAllSpamPaths, + resolveAllTrashPaths, + resolveArchiveFolder, +} from '../utils/mailUtils.js'; +import { getGtdFolderSet } from '../services/gtdConfig.js'; +import { matchingRules, toRuleMessage } from '../services/inboxRules.js'; +import { areValidUUIDs } from '../utils/validation.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const IDEMPOTENT_WRITE_ANNOTATIONS = annotations({ idempotentHint: true }); +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); + +const messageIdsSchema = { + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, +}; + +function messageIdsArg(args) { + const ids = args.message_ids; + if (!Array.isArray(ids) || ids.length === 0) { + return { error: 'message_ids must contain at least one id' }; + } + if (ids.length > 500) return { error: 'Too many ids — maximum 500 per request' }; + if (!areValidUUIDs(ids)) return { error: 'Invalid message id format' }; + return { value: ids }; +} + +export const markTriagedDef = { + name: 'mark_triaged', + description: + 'Checkpoint messages as triaged so future triage_inbox runs skip them. Idempotent: re-marking updates the timestamp and action rather than erroring. ' + + "Call mark_triaged before a move/archive, or use the move receipt's new_id, because marking after a move with the stale id silently resolves to skipped.", + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + action: { type: 'string' }, + note: { type: 'string', maxLength: 500 }, + }, + }, +}; + +export const triageInboxDef = { + name: 'triage_inbox', + description: + 'One-call inbox triage feed across scoped accounts, oldest first, enriched with category, sender history, and thread state. ' + + 'The cursor is for paging only; the message_triage table (via NOT EXISTS) guarantees correctness and prevents duplicate skipping, so re-running with no cursor is safe and cheap. ' + + 'Call mark_triaged after acting so later no-cursor runs skip checkpointed messages. ' + + 'Optional similar-message and fixed v1 urgency-probe signals are included only at limit 25 or below. Raw cosine scores are not calibrated: rank within this response, not threshold across responses.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string' }, + cursor: { type: 'string' }, + limit: { type: 'number', minimum: 1, maximum: 50, default: 25 }, + unread_only: { type: 'boolean', default: true }, + include_triaged: { type: 'boolean', default: false }, + categories: { type: 'array', items: { type: 'string' } }, + since: { type: 'string' }, + include_signals: { type: 'boolean', default: true }, + }, + }, +}; + +export const getTriageContextDef = { + name: 'get_triage_context', + description: + 'Get independently degradable thread, sender-history, similar-message, and matched-rule context for one scoped message. ' + + 'Matched rules are report only: this tool never executes rule actions, and body/header rules are reported as unevaluated when those fields are not loaded.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_id'], + properties: { + message_id: { type: 'string' }, + thread_limit: { type: 'number', minimum: 1, maximum: 50 }, + similar_limit: { type: 'number', minimum: 1, maximum: 50 }, + }, + }, +}; + +function dateArg(args, key) { + const value = args[key]; + if (typeof value !== 'string' || value === '') return { value: undefined }; + const error = { error: `invalid ${key} date "${value}": expected YYYY-MM-DD` }; + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value); + if (!match) return error; + const [year, month, day] = [ + Number(match[1]), + Number(match[2]), + Number(match[3]), + ]; + const parsed = new Date(Date.UTC(year, month - 1, day)); + if ( + parsed.getUTCFullYear() !== year + || parsed.getUTCMonth() !== month - 1 + || parsed.getUTCDate() !== day + ) { + return error; + } + return { value }; +} + +function triageLimit(args) { + const raw = args.limit; + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 1) return 25; + return Math.min(Math.trunc(raw), 50); +} + +function wireDate(value) { + const iso = value instanceof Date ? value.toISOString() : value; + return toRFC3339(iso); +} + +function wireCandidate(row, includeSignals) { + const item = { + id: row.id, + message_id: row.message_id, + account: row.account, + conversation_id: row.conversation_id, + subject: row.subject, + snippet: row.snippet, + from_email: row.from_email, + from_name: row.from_name, + date: wireDate(row.date), + is_read: !!row.is_read, + is_starred: !!row.is_starred, + has_attachments: !!row.has_attachments, + category: row.category, + is_bulk: !!row.is_bulk, + has_unsubscribe: !!row.has_unsubscribe, + spam_verdict: row.spam_verdict ?? null, + thread: { + message_count: Number(row.thread_message_count || 0), + last_activity: wireDate(row.thread_last_activity), + i_replied: !!row.i_replied, + }, + contact: { + known: !!row.contact_known, + send_count: Number(row.send_count || 0), + last_sent: wireDate(row.last_sent), + received_count: Number(row.received_count || 0), + first_received: wireDate(row.first_received), + last_received: wireDate(row.last_received), + }, + }; + if (includeSignals) item.signals = { similar: [], probes: {} }; + return item; +} + +const SIMILAR_SIGNAL_LIMIT = 5; + +function emptySignals(reason) { + const signals = { similar: [], probes: {} }; + if (reason) signals.reason = reason; + return signals; +} + +function messageFolder(summary) { + return Array.isArray(summary?.labels) && typeof summary.labels[0] === 'string' + ? summary.labels[0] + : ''; +} + +function hasFlag(summary, flag) { + const wanted = flag.toLowerCase(); + return (Array.isArray(summary?.labels) ? summary.labels : []) + .some(label => typeof label === 'string' && label.toLowerCase() === wanted); +} + +// Name-based fallback only — used when the account has neither a sent mapping +// nor a \Sent special-use folder for sentFolderForAccount to resolve. +function isSentFolder(folder) { + return /(^|[./ ])sent($|[./ ])/i.test(folder || ''); +} + +async function resolvedFolderClasses(accountId, cache) { + if (!accountId) { + return { + archive: null, + trash: new Set(), + spam: new Set(), + gtd: new Set(), + }; + } + if (!cache.has(accountId)) { + const pending = (async () => { + const [archive, trash, spam, gtd, sent] = await Promise.allSettled([ + resolveArchiveFolder(accountId), + resolveAllTrashPaths(accountId), + resolveAllSpamPaths(accountId), + getGtdFolderSet(accountId), + sentFolderForAccount(accountId), + ]); + return { + archive: archive.status === 'fulfilled' ? archive.value : null, + trash: trash.status === 'fulfilled' ? trash.value : new Set(), + spam: spam.status === 'fulfilled' ? spam.value : new Set(), + gtd: gtd.status === 'fulfilled' ? gtd.value : new Set(), + sent: sent.status === 'fulfilled' ? sent.value : null, + }; + })(); + cache.set(accountId, pending); + } + return cache.get(accountId); +} + +function folderClass(folder, resolved) { + if (!folder) return 'unknown'; + if (folder.toLowerCase() === 'inbox') return 'inbox'; + if (resolved.trash?.has(folder)) return 'trashed'; + if (resolved.spam?.has(folder)) return 'spam'; + if (resolved.sent ? folder === resolved.sent : isSentFolder(folder)) return 'sent'; + if (resolved.archive && folder === resolved.archive) return 'archived'; + if (resolved.gtd?.has(folder)) return 'labelled'; + return 'labelled'; +} + +async function similarWithDisposition( + candidate, + vector, + generation, + accountIds, + folderCache, +) { + const hits = await annSearch( + generation.id, + vector, + SIMILAR_SIGNAL_LIMIT + 1, + { + filter: { + accountIds, + before: candidate.date, + }, + }, + ); + const keptHits = hits + .filter(hit => hit.messageId !== candidate.id) + .slice(0, SIMILAR_SIGNAL_LIMIT); + const hitIds = keptHits.map(hit => hit.messageId); + const summaries = await getMessageSummariesByIDs(hitIds, accountIds); + const scoreById = new Map(keptHits.map(hit => [hit.messageId, hit.score])); + const classified = await Promise.all(summaries.map(async summary => { + const resolved = await resolvedFolderClasses(summary.source_id, folderCache); + return { + summary, + folderClass: folderClass(messageFolder(summary), resolved), + }; + })); + const repliedThreads = new Set( + classified + .filter(entry => entry.folderClass === 'sent') + .map(entry => entry.summary.conversation_id) + .filter(Boolean), + ); + + return classified.map(({ summary, folderClass: classification }) => ({ + ...wireSummary(summary), + score: scoreById.get(summary.id), + disposition: { + folder_class: classification, + was_read: hasFlag(summary, '\\Seen'), + was_starred: hasFlag(summary, '\\Flagged'), + was_replied: !!( + summary.conversation_id + && repliedThreads.has(summary.conversation_id) + ), + }, + })); +} + +async function candidateSignals( + candidate, + { + generation, + probeVectors, + accountIds, + folderCache, + }, +) { + let vector; + try { + vector = await loadVector(candidate.id); + } catch { + return emptySignals('not_embedded'); + } + + let probes = {}; + let probeFailed = false; + if (probeVectors) { + try { + probes = scoreTriageProbes(vector, probeVectors); + } catch { + probeFailed = true; + } + } + + try { + const similar = await similarWithDisposition( + candidate, + vector, + generation, + accountIds, + folderCache, + ); + const signals = { similar, probes }; + if (probeFailed) signals.reason = 'probe_error'; + return signals; + } catch (error) { + return { + similar: [], + probes, + reason: error?.message || 'error', + }; + } +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function probeCalibration(signalsByCandidate) { + const byProbe = new Map(); + for (const signals of signalsByCandidate) { + for (const [name, score] of Object.entries(signals?.probes || {})) { + if (!Number.isFinite(score)) continue; + if (!byProbe.has(name)) byProbe.set(name, []); + byProbe.get(name).push(score); + } + } + return Object.fromEntries( + [...byProbe].map(([name, values]) => [ + name, + { + min: Math.min(...values), + median: median(values), + max: Math.max(...values), + }, + ]), + ); +} + +function triageMessageKey(accountId, messageIdHeader) { + return `${accountId}\0${messageIdHeader}`; +} + +async function pageSignals(rows, accountIds) { + if (!rows.length) { + return { + signals: [], + available: true, + reason: null, + calibration: {}, + }; + } + + let cfg; + let generation; + try { + ({ cfg, generation } = await resolveActiveGenerationFromConfig()); + } catch (error) { + return { + signals: rows.map(() => emptySignals()), + available: false, + reason: error?.name === 'VectorUnavailableError' + ? translateVectorError(error.reason) + : 'error', + calibration: {}, + }; + } + + let probeVectors = null; + try { + probeVectors = await getTriageProbeVectors(cfg, generation); + } catch { + // Similar-message dispositions remain useful when only probe embedding fails. + } + + const folderCache = new Map(); + const settled = await runInBatches(rows, 4, candidate => candidateSignals( + candidate, + { + generation, + probeVectors, + accountIds, + folderCache, + }, + )); + const signals = settled.map(result => ( + result.status === 'fulfilled' + ? result.value + : emptySignals(result.reason?.message || 'error') + )); + const pairsByKey = new Map(); + for (const signal of signals) { + for (const summary of signal.similar) { + if (!summary.source_message_id) continue; + const pair = { + accountId: summary.source_id, + messageIdHeader: summary.source_message_id, + }; + pairsByKey.set(triageMessageKey(pair.accountId, pair.messageIdHeader), pair); + } + } + + let actionRows = []; + try { + actionRows = await triageActionsForMessages([...pairsByKey.values()]); + } catch { + // Similar-message signals remain useful when checkpoint history is unavailable. + } + const actionByKey = new Map(actionRows.map(row => [ + triageMessageKey(row.account_id, row.message_id_header), + row.action, + ])); + const enrichedSignals = signals.map(signal => ({ + ...signal, + similar: signal.similar.map(summary => ({ + ...summary, + disposition: { + ...summary.disposition, + triage_action: actionByKey.get( + triageMessageKey(summary.source_id, summary.source_message_id), + ) ?? null, + }, + })), + })); + return { + signals: enrichedSignals, + available: true, + reason: null, + calibration: probeCalibration(enrichedSignals), + }; +} + +function contextLimit(raw) { + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 1) return 20; + return Math.min(Math.trunc(raw), 50); +} + +function sectionError(error) { + return { + available: false, + reason: 'error', + detail: error?.message || String(error), + }; +} + +async function threadSection(seed, accountIds, limit) { + try { + if (!seed.thread_id) { + return { available: false, reason: 'conversation_not_available' }; + } + const messages = await listMessages({ + accountIds, + conversationId: seed.thread_id, + limit, + }); + return { + available: true, + messages: messages.map(wireSummary), + }; + } catch (error) { + return sectionError(error); + } +} + +async function senderHistorySection(seed, accountIds) { + try { + const history = await senderHistory(seed.from_email, accountIds); + return { available: true, history }; + } catch (error) { + return sectionError(error); + } +} + +async function similarSection(seedId, accountIds, limit) { + try { + const result = await findSimilarSummaries(seedId, { + accountIds, + limit, + }); + return { + available: true, + generation: result.generation, + messages: result.messages.map(wireSummary), + }; + } catch (error) { + if (error?.name === 'VectorUnavailableError') { + return { + available: false, + reason: translateVectorError(error.reason), + }; + } + return sectionError(error); + } +} + +function arrayField(value) { + if (Array.isArray(value)) return value; + if (typeof value !== 'string') return []; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +function normalizedRule(rule) { + return { + ...rule, + conditions: arrayField(rule?.conditions), + actions: arrayField(rule?.actions), + }; +} + +function reportMatchingRules(rules, seed) { + const message = toRuleMessage(seed); + const reports = []; + for (const rawRule of Array.isArray(rules) ? rules : []) { + const rule = normalizedRule(rawRule); + const needsUnavailableField = rule.conditions.some( + condition => condition?.field === 'body' || condition?.field === 'header', + ); + if (needsUnavailableField) { + reports.push({ + id: rule.id, + name: rule.name, + evaluated: false, + reason: 'body_not_loaded', + actions: rule.actions, + }); + continue; + } + const match = matchingRules([rule], message)[0]; + if (match) reports.push(match); + } + return reports; +} + +async function matchedRulesSection(seed, scope, deps) { + if (typeof deps?.loadInboxRules !== 'function') { + return { + available: false, + reason: 'rules_loader_unavailable', + }; + } + try { + const loaded = await deps.loadInboxRules({ + userId: scope.userId, + accountId: seed.account_id, + accountIds: scope.accountIds, + }); + const rules = Array.isArray(loaded) ? loaded : loaded?.rows; + return { + available: true, + rules: reportMatchingRules(rules, seed), + }; + } catch (error) { + return sectionError(error); + } +} + +export async function handleMarkTriaged(args, scope) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + if (args.action !== undefined && typeof args.action !== 'string') { + return errorResult('action must be a string'); + } + if (args.note !== undefined && typeof args.note !== 'string') { + return errorResult('note must be a string'); + } + if (args.note?.length > 500) { + return errorResult('note must be at most 500 characters'); + } + + const result = await markTriaged({ + userId: scope.userId, + accountIds: scope.accountIds, + messageIds: ids.value, + action: args.action ?? null, + note: args.note ?? null, + }); + return jsonResult(result); +} + +export async function handleTriageInbox(args, scope) { + if (args.cursor !== undefined && typeof args.cursor !== 'string') { + return errorResult('cursor must be a string'); + } + if ( + args.categories !== undefined + && ( + !Array.isArray(args.categories) + || args.categories.some(category => typeof category !== 'string') + ) + ) { + return errorResult('categories must be an array of strings'); + } + + const since = dateArg(args, 'since'); + if (since.error) return errorResult(since.error); + const account = await resolveAccountScope(args.account, scope.accountIds); + if (account.error) return errorResult(account.error); + + const limit = triageLimit(args); + const includeSignals = args.include_signals !== false; + try { + const [page, untriagedUnread] = await Promise.all([ + listTriageCandidates({ + accountIds: account.accountIds, + cursor: args.cursor, + limit, + unreadOnly: args.unread_only !== false, + includeTriaged: args.include_triaged === true, + categories: args.categories, + since: since.value, + }), + countUntriagedUnread(account.accountIds), + ]); + let signalState; + if (!includeSignals) { + signalState = { + signals: [], + available: false, + reason: 'disabled', + calibration: {}, + }; + } else if (limit > 25) { + signalState = { + signals: page.rows.map(() => emptySignals()), + available: false, + reason: 'limit_exceeds_25', + calibration: {}, + }; + } else { + signalState = await pageSignals(page.rows, account.accountIds); + } + + const items = page.rows.map((row, index) => { + const item = wireCandidate(row, includeSignals); + if (includeSignals) { + item.signals = signalState.signals[index] || emptySignals(); + } + return item; + }); + return jsonResult({ + items, + cursor: page.cursor, + has_more: page.hasMore, + counts: { + untriaged_unread: untriagedUnread, + returned: items.length, + }, + signals_available: signalState.available, + signals_reason: signalState.reason, + probe_calibration: signalState.calibration, + }); + } catch (error) { + if (error?.message === 'invalid triage cursor') { + return errorResult(error.message); + } + throw error; + } +} + +export async function handleGetTriageContext(args, scope, deps = {}) { + const seedId = args.message_id; + if (!seedId || typeof seedId !== 'string') { + return errorResult('message_id parameter is required'); + } + + let seed; + try { + seed = await getMessage(seedId, scope.accountIds); + } catch (error) { + const unavailable = { + available: false, + reason: 'seed_lookup_failed', + detail: error?.message || String(error), + }; + return jsonResult({ + message_id: seedId, + thread: { ...unavailable }, + sender_history: { ...unavailable }, + similar: { ...unavailable }, + matched_rules: { ...unavailable }, + }); + } + if (!seed) return errorResult('message not found'); + + const [thread, senderHistoryResult, similar, matchedRules] = await Promise.all([ + threadSection(seed, scope.accountIds, contextLimit(args.thread_limit)), + senderHistorySection(seed, scope.accountIds), + similarSection(seedId, scope.accountIds, contextLimit(args.similar_limit)), + matchedRulesSection(seed, scope, deps), + ]); + + return jsonResult({ + message_id: seedId, + thread, + sender_history: senderHistoryResult, + similar, + matched_rules: matchedRules, + }); +} diff --git a/backend/src/mcp/triageTools.test.js b/backend/src/mcp/triageTools.test.js new file mode 100644 index 00000000..308bbd5d --- /dev/null +++ b/backend/src/mcp/triageTools.test.js @@ -0,0 +1,1117 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +vi.mock('./triageAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + countUntriagedUnread: vi.fn(async () => 0), + listTriageCandidates: vi.fn(), + markTriaged: vi.fn(), + senderHistory: vi.fn(), + sentFolderForAccount: vi.fn(async () => null), + triageActionsForMessages: vi.fn(), + }; +}); +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getMessage: vi.fn(), + getMessageSummariesByIDs: vi.fn(), + listMessages: vi.fn(), + resolveAccountScope: vi.fn(), + }; +}); +vi.mock('../services/embeddings/hybrid.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + resolveActiveGenerationFromConfig: vi.fn(), + }; +}); +vi.mock('../services/embeddings/vectorStore.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + annSearch: vi.fn(), + loadVector: vi.fn(), + }; +}); +vi.mock('../services/mailbox/batch.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + runInBatches: vi.fn(actual.runInBatches), + }; +}); +vi.mock('./triageProbes.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getTriageProbeVectors: vi.fn(), + scoreTriageProbes: vi.fn(), + }; +}); +vi.mock('../utils/mailUtils.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + resolveArchiveFolder: vi.fn(), + resolveAllTrashPaths: vi.fn(), + resolveAllSpamPaths: vi.fn(), + }; +}); +vi.mock('../services/gtdConfig.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getGtdFolderSet: vi.fn(), + }; +}); +vi.mock('./messageTools.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + findSimilarSummaries: vi.fn(), + }; +}); +vi.mock('../services/inboxRules.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + matchingRules: vi.fn(actual.matchingRules), + toRuleMessage: vi.fn(actual.toRuleMessage), + }; +}); + +const triageAdapter = await import('./triageAdapter.js'); +const engineAdapter = await import('./engineAdapter.js'); +const hybrid = await import('../services/embeddings/hybrid.js'); +const vectorStore = await import('../services/embeddings/vectorStore.js'); +const batch = await import('../services/mailbox/batch.js'); +const triageProbes = await import('./triageProbes.js'); +const mailUtils = await import('../utils/mailUtils.js'); +const gtdConfig = await import('../services/gtdConfig.js'); +const messageTools = await import('./messageTools.js'); +const inboxRules = await import('../services/inboxRules.js'); +const triageTools = await import('./triageTools.js').catch(() => ({})); +const registeredTools = await import('./tools.js').catch(() => ({})); + +const MESSAGE_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const scope = { + userId: 'user-1', + accountIds: ['bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'], + scopes: ['read', 'write'], +}; +const deps = { ignored: true }; +const defaultProbeScores = { + urgent: 0.2, + needs_reply: 0.1, + financial: 0.05, + scheduling: 0.15, + bulk: 0.3, +}; + +function jsonOf(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + engineAdapter.resolveAccountScope.mockImplementation(async (_account, accountIds) => ({ + accountIds, + })); + engineAdapter.getMessageSummariesByIDs.mockReset().mockResolvedValue([]); + triageAdapter.triageActionsForMessages.mockReset().mockResolvedValue([]); + hybrid.resolveActiveGenerationFromConfig.mockReset().mockResolvedValue({ + cfg: { enabled: true }, + generation: { + id: 7, + fingerprint: 'fingerprint-1', + }, + }); + vectorStore.loadVector.mockReset().mockResolvedValue([1, 0]); + vectorStore.annSearch.mockReset().mockResolvedValue([]); + triageProbes.getTriageProbeVectors.mockReset().mockResolvedValue({ + urgent: [1, 0], + }); + triageProbes.scoreTriageProbes.mockReset().mockReturnValue(defaultProbeScores); + mailUtils.resolveArchiveFolder.mockReset().mockResolvedValue('Archive'); + mailUtils.resolveAllTrashPaths.mockReset().mockResolvedValue(new Set(['Trash'])); + mailUtils.resolveAllSpamPaths.mockReset().mockResolvedValue(new Set(['Junk'])); + gtdConfig.getGtdFolderSet.mockReset().mockResolvedValue(new Set(['Todo'])); +}); + +describe('mark_triaged definition and registration', () => { + it('registers an idempotent write tool with the move-ordering warning', () => { + const def = registeredTools.TOOL_DEFS?.find(entry => entry.name === 'mark_triaged'); + + expect(def?.inputSchema.required).toEqual(['message_ids']); + expect(def?.inputSchema.properties.message_ids).toEqual({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, + }); + expect(def?.annotations).toEqual({ + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + expect(def?.description).toMatch(/before (?:a )?move\/archive/i); + expect(def?.description).toMatch(/new_id/); + expect(def?.description).toMatch(/stale id.*skipped/i); + expect(registeredTools.TOOL_SCOPES?.mark_triaged).toBe('write'); + expect(registeredTools.HANDLERS?.mark_triaged).toBe(triageTools.handleMarkTriaged); + }); +}); + +describe('mark_triaged handler', () => { + it('passes scoped ids and optional receipt metadata to the adapter without inventing tokenId', async () => { + const receipt = { + ok: true, + marked: 1, + newly_marked: 1, + already_triaged: 0, + skipped: [], + }; + triageAdapter.markTriaged.mockResolvedValue(receipt); + + const result = await triageTools.handleMarkTriaged({ + message_ids: [MESSAGE_ID], + action: 'archived', + note: 'Morning pass', + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual(receipt); + expect(triageAdapter.markTriaged).toHaveBeenCalledWith({ + userId: scope.userId, + accountIds: scope.accountIds, + messageIds: [MESSAGE_ID], + action: 'archived', + note: 'Morning pass', + }); + }); + + it('passes through the adapter skip reason for a message without a Message-ID header', async () => { + const receipt = { + ok: true, + marked: 0, + newly_marked: 0, + already_triaged: 0, + skipped: [{ id: MESSAGE_ID, reason: 'no_message_id_header' }], + }; + triageAdapter.markTriaged.mockResolvedValue(receipt); + + const result = await triageTools.handleMarkTriaged({ + message_ids: [MESSAGE_ID], + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual(receipt); + }); + + it('passes through an idempotent re-mark receipt under already_triaged', async () => { + const receipt = { + ok: true, + marked: 1, + newly_marked: 0, + already_triaged: 1, + skipped: [], + }; + triageAdapter.markTriaged.mockResolvedValue(receipt); + + const result = await triageTools.handleMarkTriaged({ + message_ids: [MESSAGE_ID], + action: 'left', + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual(receipt); + }); + + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ message_ids: Array.from({ length: 501 }, () => MESSAGE_ID) }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + [{ message_ids: [MESSAGE_ID], action: 42 }, 'action must be a string'], + [{ message_ids: [MESSAGE_ID], note: 42 }, 'note must be a string'], + [{ message_ids: [MESSAGE_ID], note: 'x'.repeat(501) }, 'note must be at most 500 characters'], + ])('rejects invalid arguments before checkpointing', async (args, message) => { + const result = await triageTools.handleMarkTriaged(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(message); + expect(triageAdapter.markTriaged).not.toHaveBeenCalled(); + }); +}); + +const candidateRow = { + id: MESSAGE_ID, + account_id: scope.accountIds[0], + message_id: '', + account: 'me@example.com', + conversation_id: 'thread-1', + subject: 'Quarterly planning', + snippet: 'Could you review the agenda?', + from_email: 'alice@example.com', + from_name: 'Alice', + date: new Date('2026-07-28T06:11:00.123Z'), + is_read: false, + is_starred: true, + has_attachments: true, + category: 'primary', + is_bulk: false, + has_unsubscribe: false, + spam_verdict: null, + thread_message_count: 3, + thread_last_activity: new Date('2026-07-28T07:12:00.456Z'), + i_replied: true, + contact_known: true, + send_count: 12, + last_sent: new Date('2026-07-27T10:00:00.789Z'), + received_count: 47, + first_received: new Date('2025-01-01T00:00:00.111Z'), + last_received: new Date('2026-07-28T06:11:00.123Z'), +}; + +describe('triage_inbox definition and registration', () => { + it('registers an idempotent read tool and documents cursor correctness', () => { + const def = registeredTools.TOOL_DEFS?.find(entry => entry.name === 'triage_inbox'); + + expect(def?.annotations).toEqual({ + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + expect(def?.inputSchema.properties).toEqual(expect.objectContaining({ + account: { type: 'string' }, + cursor: { type: 'string' }, + limit: expect.objectContaining({ maximum: 50 }), + unread_only: expect.objectContaining({ type: 'boolean', default: true }), + include_triaged: expect.objectContaining({ type: 'boolean', default: false }), + categories: { type: 'array', items: { type: 'string' } }, + since: { type: 'string' }, + include_signals: expect.objectContaining({ type: 'boolean', default: true }), + })); + expect(def?.description).toMatch(/cursor is for paging only/i); + expect(def?.description).toMatch(/message_triage.*NOT EXISTS/i); + expect(def?.description).toMatch(/re-running with no cursor is safe and cheap/i); + expect(def?.description).toMatch(/raw cosine.*not calibrated/i); + expect(def?.description).toMatch(/rank.*not threshold/i); + expect(def?.description).toMatch(/fixed.*v1/i); + expect(registeredTools.TOOL_SCOPES?.triage_inbox).toBe('read'); + expect(registeredTools.HANDLERS?.triage_inbox).toBe(triageTools.handleTriageInbox); + }); +}); + +describe('triage_inbox handler', () => { + it('maps one enriched adapter row to the snake_case feed envelope', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: 'cursor-one', + }); + triageAdapter.countUntriagedUnread.mockResolvedValueOnce(118); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual({ + items: [{ + id: MESSAGE_ID, + message_id: '', + account: 'me@example.com', + conversation_id: 'thread-1', + subject: 'Quarterly planning', + snippet: 'Could you review the agenda?', + from_email: 'alice@example.com', + from_name: 'Alice', + date: '2026-07-28T06:11:00Z', + is_read: false, + is_starred: true, + has_attachments: true, + category: 'primary', + is_bulk: false, + has_unsubscribe: false, + spam_verdict: null, + thread: { + message_count: 3, + last_activity: '2026-07-28T07:12:00Z', + i_replied: true, + }, + contact: { + known: true, + send_count: 12, + last_sent: '2026-07-27T10:00:00Z', + received_count: 47, + first_received: '2025-01-01T00:00:00Z', + last_received: '2026-07-28T06:11:00Z', + }, + signals: { similar: [], probes: defaultProbeScores }, + }], + cursor: 'cursor-one', + has_more: false, + counts: { untriaged_unread: 118, returned: 1 }, + signals_available: true, + signals_reason: null, + probe_calibration: { + urgent: { min: 0.2, median: 0.2, max: 0.2 }, + needs_reply: { min: 0.1, median: 0.1, max: 0.1 }, + financial: { min: 0.05, median: 0.05, max: 0.05 }, + scheduling: { min: 0.15, median: 0.15, max: 0.15 }, + bulk: { min: 0.3, median: 0.3, max: 0.3 }, + }, + }); + expect(triageAdapter.listTriageCandidates).toHaveBeenCalledWith({ + accountIds: scope.accountIds, + cursor: undefined, + limit: 25, + unreadOnly: true, + includeTriaged: false, + categories: undefined, + since: undefined, + }); + }); + + it('round-trips the opaque returned cursor into the next adapter call', async () => { + triageAdapter.listTriageCandidates + .mockResolvedValueOnce({ rows: [candidateRow], hasMore: true, cursor: 'opaque-page-1' }) + .mockResolvedValueOnce({ rows: [], hasMore: false, cursor: null }); + + const first = jsonOf(await triageTools.handleTriageInbox({ include_signals: false }, scope, deps)); + const second = jsonOf(await triageTools.handleTriageInbox({ + cursor: first.cursor, + include_signals: false, + }, scope, deps)); + + expect(first.cursor).toBe('opaque-page-1'); + expect(second.cursor).toBeNull(); + expect(triageAdapter.listTriageCandidates).toHaveBeenLastCalledWith( + expect.objectContaining({ cursor: 'opaque-page-1' }), + ); + }); + + it('surfaces adapter limit+1 paging as has_more while returning only the page rows', async () => { + const rows = Array.from({ length: 25 }, (_, index) => ({ + ...candidateRow, + id: `message-${index}`, + })); + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows, + hasMore: true, + cursor: 'next-page', + }); + + const body = jsonOf(await triageTools.handleTriageInbox({ + limit: 25, + include_signals: false, + }, scope, deps)); + + expect(body.items).toHaveLength(25); + expect(body.counts.returned).toBe(25); + expect(body.has_more).toBe(true); + expect(body.cursor).toBe('next-page'); + expect(triageAdapter.listTriageCandidates).toHaveBeenCalledWith( + expect.objectContaining({ limit: 25 }), + ); + }); + + it('turns a malformed adapter cursor failure into a clean errorResult', async () => { + triageAdapter.listTriageCandidates.mockRejectedValue(new Error('invalid triage cursor')); + + const result = await triageTools.handleTriageInbox({ cursor: 'not-a-cursor' }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('invalid triage cursor'); + }); + + it('narrows account scope and threads feed filters with a max-50 limit', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ accountIds: ['account-2'] }); + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [], + hasMore: false, + cursor: null, + }); + + await triageTools.handleTriageInbox({ + account: 'other@example.com', + limit: 99, + unread_only: false, + include_triaged: true, + categories: ['newsletter', 'promotion'], + since: '2026-07-01', + include_signals: false, + }, scope, deps); + + expect(engineAdapter.resolveAccountScope).toHaveBeenCalledWith( + 'other@example.com', + scope.accountIds, + ); + expect(triageAdapter.listTriageCandidates).toHaveBeenCalledWith({ + accountIds: ['account-2'], + cursor: undefined, + limit: 50, + unreadOnly: false, + includeTriaged: true, + categories: ['newsletter', 'promotion'], + since: '2026-07-01', + }); + }); + + it('reports explicitly disabled signals without attaching per-item placeholders', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + + const body = jsonOf(await triageTools.handleTriageInbox({ + include_signals: false, + }, scope, deps)); + + expect(body.signals_available).toBe(false); + expect(body.signals_reason).toBe('disabled'); + expect(body.items[0]).not.toHaveProperty('signals'); + expect(hybrid.resolveActiveGenerationFromConfig).not.toHaveBeenCalled(); + expect(vectorStore.loadVector).not.toHaveBeenCalled(); + }); + + it('silently disables signals above the 25-item cost bound', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + + const result = await triageTools.handleTriageInbox({ + limit: 30, + }, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(false); + expect(body.signals_reason).toBe('limit_exceeds_25'); + expect(body.items[0].signals).toEqual({ similar: [], probes: {} }); + expect(hybrid.resolveActiveGenerationFromConfig).not.toHaveBeenCalled(); + expect(vectorStore.loadVector).not.toHaveBeenCalled(); + }); + + it('computes similar dispositions at concurrency 4 and calibrates page probe scores', async () => { + const secondCandidate = { + ...candidateRow, + id: 'candidate-two', + message_id: '', + date: new Date('2026-07-28T08:00:00.000Z'), + }; + const rows = [candidateRow, secondCandidate]; + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows, + hasMore: false, + cursor: null, + }); + vectorStore.loadVector.mockImplementation(async id => ( + id === MESSAGE_ID ? [0.2] : [0.8] + )); + triageProbes.scoreTriageProbes.mockImplementation(vector => ({ + urgent: vector[0], + bulk: 1 - vector[0], + })); + vectorStore.annSearch.mockImplementation(async (_generationId, _vector, _k, options) => [ + { + messageId: options.filter.before.getUTCHours() === 6 + ? MESSAGE_ID + : 'candidate-two', + score: 1, + }, + { messageId: 'past-archived', score: 0.9 }, + { messageId: 'past-sent', score: 0.8 }, + ]); + const hydrated = { + 'past-archived': { + id: 'past-archived', + source_id: scope.accountIds[0], + source_message_id: 'past-archived', + conversation_id: 'past-thread', + subject: 'Past archived message', + sent_at: '2026-06-01T10:00:00.123Z', + labels: ['Archive', '\\Seen', '\\Flagged'], + }, + 'past-sent': { + id: 'past-sent', + source_id: scope.accountIds[0], + source_message_id: 'past-sent', + conversation_id: 'past-thread', + subject: 'My reply', + sent_at: '2026-06-01T11:00:00.456Z', + labels: ['Sent'], + }, + }; + engineAdapter.getMessageSummariesByIDs.mockImplementation(async ids => ( + ids.map(id => hydrated[id]).filter(Boolean) + )); + triageAdapter.triageActionsForMessages.mockResolvedValue([{ + account_id: scope.accountIds[0], + message_id_header: 'past-archived', + action: 'archived', + triaged_at: '2026-07-28T08:00:00Z', + }]); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(true); + expect(body.signals_reason).toBeNull(); + expect(body.probe_calibration).toEqual({ + urgent: { min: 0.2, median: 0.5, max: 0.8 }, + bulk: { min: 0.19999999999999996, median: 0.5, max: 0.8 }, + }); + expect(batch.runInBatches).toHaveBeenCalledWith(rows, 4, expect.any(Function)); + expect(vectorStore.annSearch).toHaveBeenCalledWith( + 7, + [0.2], + 6, + { + filter: { + accountIds: scope.accountIds, + before: candidateRow.date, + }, + }, + ); + expect(engineAdapter.getMessageSummariesByIDs).toHaveBeenCalledWith( + ['past-archived', 'past-sent'], + scope.accountIds, + ); + expect(body.items[0].signals.probes).toEqual({ urgent: 0.2, bulk: 0.8 }); + expect(body.items[0].signals.similar[0]).toEqual(expect.objectContaining({ + id: 'past-archived', + sent_at: '2026-06-01T10:00:00Z', + score: 0.9, + disposition: { + folder_class: 'archived', + was_read: true, + was_starred: true, + was_replied: true, + triage_action: 'archived', + }, + })); + expect(body.items[0].signals.similar[1].disposition).toEqual(expect.objectContaining({ + folder_class: 'sent', + triage_action: null, + })); + expect(triageAdapter.triageActionsForMessages).toHaveBeenCalledTimes(1); + expect(triageAdapter.triageActionsForMessages).toHaveBeenCalledWith([ + { accountId: scope.accountIds[0], messageIdHeader: 'past-archived' }, + { accountId: scope.accountIds[0], messageIdHeader: 'past-sent' }, + ]); + expect(mailUtils.resolveArchiveFolder).toHaveBeenCalledTimes(1); + expect(mailUtils.resolveAllTrashPaths).toHaveBeenCalledTimes(1); + expect(mailUtils.resolveAllSpamPaths).toHaveBeenCalledTimes(1); + expect(gtdConfig.getGtdFolderSet).toHaveBeenCalledTimes(1); + expect(triageAdapter.sentFolderForAccount).toHaveBeenCalledTimes(1); + }); + + it('prefers the resolved sent folder over the name heuristic when one exists', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + triageAdapter.sentFolderForAccount.mockResolvedValue('Custom/Outgoing'); + vectorStore.annSearch.mockResolvedValue([ + { messageId: 'past-outgoing', score: 0.9 }, + { messageId: 'past-sent-items', score: 0.8 }, + ]); + engineAdapter.getMessageSummariesByIDs.mockResolvedValue([ + { + id: 'past-outgoing', + source_id: scope.accountIds[0], + source_message_id: 'past-outgoing', + subject: 'Reply from the mapped folder', + sent_at: '2026-06-01T10:00:00.123Z', + labels: ['Custom/Outgoing'], + }, + { + id: 'past-sent-items', + source_id: scope.accountIds[0], + source_message_id: 'past-sent-items', + subject: 'Name-matches sent but is not the resolved folder', + sent_at: '2026-06-01T11:00:00.456Z', + labels: ['Sent Items'], + }, + ]); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + const byId = Object.fromEntries( + body.items[0].signals.similar.map(entry => [entry.id, entry]), + ); + expect(byId['past-outgoing'].disposition.folder_class).toBe('sent'); + expect(byId['past-sent-items'].disposition.folder_class).toBe('labelled'); + }); + + it('degrades a rejected page triage-action lookup to null dispositions', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + vectorStore.annSearch.mockResolvedValue([ + { messageId: 'past-archived', score: 0.9 }, + { messageId: 'past-sent', score: 0.8 }, + ]); + engineAdapter.getMessageSummariesByIDs.mockResolvedValue([ + { + id: 'past-archived', + source_id: scope.accountIds[0], + source_message_id: 'past-archived', + subject: 'Past archived message', + sent_at: '2026-06-01T10:00:00.123Z', + labels: ['Archive'], + }, + { + id: 'past-sent', + source_id: scope.accountIds[0], + source_message_id: 'past-sent', + subject: 'My reply', + sent_at: '2026-06-01T11:00:00.456Z', + labels: ['Sent'], + }, + ]); + triageAdapter.triageActionsForMessages.mockRejectedValue( + new Error('triage lookup unavailable'), + ); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(triageAdapter.triageActionsForMessages).toHaveBeenCalledTimes(1); + expect(body.items[0].signals.similar).toHaveLength(2); + expect(body.items[0].signals.similar.every( + similar => similar.disposition.triage_action === null, + )).toBe(true); + }); + + it('degrades one unembedded candidate without affecting the rest of the page', async () => { + const missingCandidate = { + ...candidateRow, + id: 'missing-vector', + message_id: '', + }; + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow, missingCandidate], + hasMore: false, + cursor: null, + }); + vectorStore.loadVector.mockImplementation(async id => { + if (id === 'missing-vector') throw new Error('no embedding for message'); + return [1, 0]; + }); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(true); + expect(body.items[0].signals).toEqual({ + similar: [], + probes: defaultProbeScores, + }); + expect(body.items[1].signals).toEqual({ + similar: [], + probes: {}, + reason: 'not_embedded', + }); + }); + + it('turns page-level VectorUnavailableError into empty signals without isError', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + const error = new Error('no active generation'); + error.name = 'VectorUnavailableError'; + error.reason = 'no_active_generation'; + hybrid.resolveActiveGenerationFromConfig.mockRejectedValue(error); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(false); + expect(body.signals_reason).toBe( + 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + ); + expect(body.items[0].signals).toEqual({ similar: [], probes: {} }); + expect(vectorStore.loadVector).not.toHaveBeenCalled(); + }); + + it.each([ + [{ since: '2026-02-30' }, 'invalid since date "2026-02-30": expected YYYY-MM-DD'], + [{ since: '07/28/2026' }, 'invalid since date "07/28/2026": expected YYYY-MM-DD'], + [{ categories: 'primary' }, 'categories must be an array of strings'], + [{ categories: ['primary', 42] }, 'categories must be an array of strings'], + [{ cursor: 42 }, 'cursor must be a string'], + ])('rejects malformed filters before reading the feed', async (args, message) => { + const result = await triageTools.handleTriageInbox(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(message); + expect(triageAdapter.listTriageCandidates).not.toHaveBeenCalled(); + }); + + it('returns an unknown-account error without widening scope', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: missing@example.com', + }); + + const result = await triageTools.handleTriageInbox({ + account: 'missing@example.com', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: missing@example.com'); + expect(triageAdapter.listTriageCandidates).not.toHaveBeenCalled(); + }); +}); + +const contextSeed = { + id: MESSAGE_ID, + account_id: scope.accountIds[0], + uid: 10, + folder: 'INBOX', + message_id: '', + thread_id: 'thread-1', + subject: 'Quarterly planning', + snippet: 'Could you review the agenda?', + from_email: 'alice@example.com', + from_name: 'Alice', + to_addresses: [{ email: 'me@example.com', name: 'Me' }], + date: new Date('2026-07-28T06:11:00.123Z'), + has_attachments: false, + is_read: false, +}; +const threadSummary = { + id: 'thread-message-1', + subject: 'Re: Quarterly planning', + sent_at: '2026-07-28T05:00:00.123Z', +}; +const similarSummary = { + id: 'similar-message-1', + subject: 'Prior quarterly planning', + sent_at: '2026-04-01T05:00:00.456Z', +}; +const senderHistoryRow = { + received_count: 8, + first_received: '2025-01-01T00:00:00Z', + last_received: '2026-07-28T06:11:00Z', + contact_id: 'contact-1', + contact_name: 'Alice', + primary_email: 'alice@example.com', + send_count: 3, + last_sent: '2026-07-20T12:00:00Z', + is_auto: false, + contact_known: true, +}; +const generation = { + id: 7, + model: 'embed-model', + dimension: 3, + fingerprint: 'fingerprint-1', + state: 'active', +}; + +function contextDeps(rules = []) { + return { loadInboxRules: vi.fn().mockResolvedValue(rules) }; +} + +function arrangeContextSuccess() { + engineAdapter.getMessage.mockResolvedValue(contextSeed); + engineAdapter.listMessages.mockResolvedValue([threadSummary]); + triageAdapter.senderHistory.mockResolvedValue(senderHistoryRow); + messageTools.findSimilarSummaries.mockResolvedValue({ + generation, + messages: [similarSummary], + }); +} + +describe('get_triage_context definition and registration', () => { + it('registers an idempotent read-only report tool', () => { + const def = registeredTools.TOOL_DEFS?.find( + entry => entry.name === 'get_triage_context', + ); + + expect(def?.inputSchema.required).toEqual(['message_id']); + expect(def?.inputSchema.properties).toEqual({ + message_id: { type: 'string' }, + thread_limit: { type: 'number', minimum: 1, maximum: 50 }, + similar_limit: { type: 'number', minimum: 1, maximum: 50 }, + }); + expect(def?.annotations).toEqual({ + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + expect(def?.description).toMatch(/report only/i); + expect(def?.description).toMatch(/never executes rule actions/i); + expect(registeredTools.TOOL_SCOPES?.get_triage_context).toBe('read'); + expect(registeredTools.HANDLERS?.get_triage_context) + .toBe(triageTools.handleGetTriageContext); + }); +}); + +describe('get_triage_context handler', () => { + it('returns thread, sender, similar, and report-only rule sections', async () => { + arrangeContextSuccess(); + const rules = [ + { + id: 'rule-subject', + name: 'Planning', + condition_logic: 'AND', + conditions: [{ + field: 'subject', + operator: 'contains', + value: 'planning', + }], + actions: [{ type: 'star' }], + }, + { + id: 'rule-body', + name: 'Body-only rule', + condition_logic: 'AND', + conditions: [{ + field: 'body', + operator: 'contains', + value: 'deadline', + }], + actions: [{ type: 'archive' }], + }, + { + id: 'rule-header', + name: 'Header-only rule', + condition_logic: 'AND', + conditions: [{ + field: 'header', + headerName: 'List-Id', + operator: 'contains', + value: 'newsletter', + }], + actions: [{ type: 'mark_read' }], + }, + ]; + const loaderDeps = contextDeps(rules); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + thread_limit: 12, + similar_limit: 6, + }, scope, loaderDeps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body).toEqual({ + message_id: MESSAGE_ID, + thread: { + available: true, + messages: [{ + ...threadSummary, + sent_at: '2026-07-28T05:00:00Z', + }], + }, + sender_history: { + available: true, + history: senderHistoryRow, + }, + similar: { + available: true, + generation, + messages: [{ + ...similarSummary, + sent_at: '2026-04-01T05:00:00Z', + }], + }, + matched_rules: { + available: true, + rules: [ + { + id: 'rule-subject', + name: 'Planning', + would_match: true, + actions: [{ type: 'star' }], + }, + { + id: 'rule-body', + name: 'Body-only rule', + evaluated: false, + reason: 'body_not_loaded', + actions: [{ type: 'archive' }], + }, + { + id: 'rule-header', + name: 'Header-only rule', + evaluated: false, + reason: 'body_not_loaded', + actions: [{ type: 'mark_read' }], + }, + ], + }, + }); + expect(engineAdapter.getMessage).toHaveBeenCalledWith( + MESSAGE_ID, + scope.accountIds, + ); + expect(engineAdapter.listMessages).toHaveBeenCalledWith({ + accountIds: scope.accountIds, + conversationId: 'thread-1', + limit: 12, + }); + expect(triageAdapter.senderHistory).toHaveBeenCalledWith( + 'alice@example.com', + scope.accountIds, + ); + expect(messageTools.findSimilarSummaries).toHaveBeenCalledWith( + MESSAGE_ID, + { accountIds: scope.accountIds, limit: 6 }, + ); + expect(loaderDeps.loadInboxRules).toHaveBeenCalledWith({ + userId: scope.userId, + accountId: scope.accountIds[0], + accountIds: scope.accountIds, + }); + }); + + it.each([ + ['thread', () => engineAdapter.listMessages.mockRejectedValue(new Error('thread down'))], + ['sender_history', () => triageAdapter.senderHistory.mockRejectedValue(new Error('sender down'))], + ['similar', () => messageTools.findSimilarSummaries.mockRejectedValue(new Error('similar down'))], + ['matched_rules', (_deps) => _deps.loadInboxRules.mockRejectedValue(new Error('rules down'))], + ])('degrades only the %s section and never sets isError', async (failedSection, fail) => { + arrangeContextSuccess(); + const loaderDeps = contextDeps([]); + fail(loaderDeps); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, loaderDeps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body[failedSection]).toEqual({ + available: false, + reason: 'error', + detail: expect.any(String), + }); + for (const section of ['thread', 'sender_history', 'similar', 'matched_rules']) { + if (section !== failedSection) expect(body[section].available).toBe(true); + } + }); + + it('translates VectorUnavailableError inside similar without failing the tool', async () => { + arrangeContextSuccess(); + const error = new Error('no active generation'); + error.name = 'VectorUnavailableError'; + error.reason = 'no_active_generation'; + messageTools.findSimilarSummaries.mockRejectedValue(error); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, contextDeps([])); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.similar).toEqual({ + available: false, + reason: 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + }); + expect(body.thread.available).toBe(true); + expect(body.sender_history.available).toBe(true); + expect(body.matched_rules.available).toBe(true); + }); + + it('degrades only matched_rules when no read-only loader is importable', async () => { + arrangeContextSuccess(); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.matched_rules).toEqual({ + available: false, + reason: 'rules_loader_unavailable', + }); + expect(body.thread.available).toBe(true); + expect(body.sender_history.available).toBe(true); + expect(body.similar.available).toBe(true); + }); + + it('returns a clean input error before any section work when message_id is absent', async () => { + const result = await triageTools.handleGetTriageContext({}, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_id parameter is required'); + expect(engineAdapter.getMessage).not.toHaveBeenCalled(); + }); + + it('returns message not found before starting enrichment sections', async () => { + engineAdapter.getMessage.mockResolvedValue(null); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message not found'); + expect(engineAdapter.listMessages).not.toHaveBeenCalled(); + expect(triageAdapter.senderHistory).not.toHaveBeenCalled(); + expect(messageTools.findSimilarSummaries).not.toHaveBeenCalled(); + }); + + it('degrades all sections without isError when the scoped seed lookup fails', async () => { + engineAdapter.getMessage.mockRejectedValue(new Error('message store down')); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + for (const section of ['thread', 'sender_history', 'similar', 'matched_rules']) { + expect(body[section]).toEqual({ + available: false, + reason: 'seed_lookup_failed', + detail: 'message store down', + }); + } + }); +}); + +describe('mock-drift guard: mocked seams exist on their real modules', () => { + it.each([ + ['triageAdapter', () => triageAdapter, './triageAdapter.js'], + ['engineAdapter', () => engineAdapter, './engineAdapter.js'], + ['messageTools', () => messageTools, './messageTools.js'], + ['inboxRules', () => inboxRules, '../services/inboxRules.js'], + ['hybrid', () => hybrid, '../services/embeddings/hybrid.js'], + ['vectorStore', () => vectorStore, '../services/embeddings/vectorStore.js'], + ['batch', () => batch, '../services/mailbox/batch.js'], + ['triageProbes', () => triageProbes, './triageProbes.js'], + ['mailUtils', () => mailUtils, '../utils/mailUtils.js'], + ['gtdConfig', () => gtdConfig, '../services/gtdConfig.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); diff --git a/backend/src/mcp/writeResult.js b/backend/src/mcp/writeResult.js new file mode 100644 index 00000000..7f95cebe --- /dev/null +++ b/backend/src/mcp/writeResult.js @@ -0,0 +1,72 @@ +import { toRFC3339 } from './envelope.js'; +import { errorResult } from './result.js'; + +export const WRITE_ERROR_CODES = [ + 'account_not_found', + 'alias_not_found', + 'message_not_found', + 'draft_not_found', + 'outbox_not_found', + 'invalid_recipient', + 'too_many_recipients', + 'attachment_too_large', + 'no_drafts_folder', + 'no_sent_folder', + 'smtp_failed', + 'already_sent', + 'invalid_arguments', + 'unsupported', +]; + +const CAMEL_TO_WIRE = { + draftUid: 'draft_uid', + inReplyTo: 'in_reply_to', + messageId: 'message_id', + outboxId: 'outbox_id', + recipientsComputed: 'recipients_computed', + sendAt: 'send_at', + sentCopySaved: 'sent_copy_saved', + undoSeconds: 'undo_seconds', +}; + +function wireTime(value) { + if (value instanceof Date) return toRFC3339(value.toISOString()); + return toRFC3339(value); +} + +function copyResultFields(target, fields) { + for (const [key, value] of Object.entries(fields || {})) { + if (value === undefined) continue; + const wireKey = CAMEL_TO_WIRE[key] || key; + target[wireKey] = wireKey === 'send_at' ? wireTime(value) : value; + } +} + +export function buildWriteReceipt(receipt = {}, resultFields = {}) { + const result = {}; + copyResultFields(result, resultFields); + + for (const key of ['messageId', 'inReplyTo', 'references']) { + if (receipt[key] !== undefined) copyResultFields(result, { [key]: receipt[key] }); + } + + result.from = receipt.from || {}; + result.to = receipt.to || []; + result.cc = receipt.cc || []; + result.bcc = receipt.bcc || []; + result.subject = receipt.subject || ''; + result.attachments = (receipt.attachments || []).map(attachment => ({ + filename: attachment.filename, + size: attachment.size, + ...(attachment.source !== undefined ? { source: attachment.source } : {}), + })); + + for (const key of ['sentCopySaved', 'folder']) { + if (receipt[key] !== undefined) copyResultFields(result, { [key]: receipt[key] }); + } + return result; +} + +export function writeError(code, detail) { + return errorResult(`${code}: ${detail}`); +} diff --git a/backend/src/mcp/writeResult.test.js b/backend/src/mcp/writeResult.test.js new file mode 100644 index 00000000..32dfb3a8 --- /dev/null +++ b/backend/src/mcp/writeResult.test.js @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest'; +import { + WRITE_ERROR_CODES, + buildWriteReceipt, + writeError, +} from './writeResult.js'; + +describe('buildWriteReceipt', () => { + it('matches the documented immediate-send receipt exactly', () => { + expect(buildWriteReceipt({ + from: { name: 'A', email: 'a@b.com' }, + to: [{ name: '', email: 'x@y.com' }], + cc: [], + bcc: [], + subject: 'Quarterly report', + attachments: [{ filename: 'q.pdf', size: 8123 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', + }, { + sent: true, + })).toEqual({ + sent: true, + message_id: '', + from: { name: 'A', email: 'a@b.com' }, + to: [{ name: '', email: 'x@y.com' }], + cc: [], + bcc: [], + subject: 'Quarterly report', + attachments: [{ filename: 'q.pdf', size: 8123 }], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('matches the documented queued-send receipt exactly', () => { + expect(buildWriteReceipt({ + subject: 'Quarterly report', + }, { + queued: true, + outboxId: 'outbox-1', + sendAt: '2026-07-28T10:00:30.000Z', + undoSeconds: 30, + note: 'Cancel with unsend_email before send_at.', + })).toEqual({ + queued: true, + outbox_id: 'outbox-1', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Quarterly report', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); + + it('re-keys result-specific service fields and preserves attachment source metadata', () => { + expect(buildWriteReceipt({ + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Re: Subject', + attachments: [{ filename: 'deck.pdf', size: 2144000, source: 'forwarded' }], + messageId: '', + sentCopySaved: true, + inReplyTo: '', + references: ' ', + }, { + sent: true, + recipientsComputed: { + reply_target: 'sender@example.com', + excluded_self: ['me@example.com'], + }, + })).toEqual({ + sent: true, + message_id: '', + in_reply_to: '', + references: ' ', + recipients_computed: { + reply_target: 'sender@example.com', + excluded_self: ['me@example.com'], + }, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Re: Subject', + attachments: [{ filename: 'deck.pdf', size: 2144000, source: 'forwarded' }], + sent_copy_saved: true, + }); + }); +}); + +describe('writeError', () => { + it.each(WRITE_ERROR_CODES)('%s is emitted as a stable isError prefix', (code) => { + const result = writeError(code, 'detail'); + expect(result).toEqual({ + content: [{ type: 'text', text: `${code}: detail` }], + isError: true, + }); + }); +}); diff --git a/backend/src/routes/accounts.js b/backend/src/routes/accounts.js index d947ead3..6485b7e5 100644 --- a/backend/src/routes/accounts.js +++ b/backend/src/routes/accounts.js @@ -3,64 +3,27 @@ import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; import { imapManager } from '../index.js'; import { encrypt } from '../services/encryption.js'; -import { sanitizeSignature } from '../services/emailSanitizer.js'; +import { hasHeaderInjectionChars, sanitizeSignature } from '../services/emailSanitizer.js'; import { validateHost } from '../services/hostValidation.js'; import { getConnectionPolicy } from '../services/connectionPolicy.js'; import { invalidateGtdConfigCache, sanitizeGtdFoldersDetailed, findGtdFolderCollisions, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js'; import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; -import { createKeyedSerializer } from '../utils/keyedSerializer.js'; import { startAccountBodyBackfill } from '../services/bodyBackfill.js'; import { providerProfile } from '../services/imapManager.js'; import { upsertJob } from '../services/backgroundJobs.js'; - -// Serialize an account's reconnect triggers so a rapid settings change (e.g. a -// gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains — -// connectAccount's in-progress guard would drop the second and leave the GTD sync -// tick armed inconsistently with the final DB value. Queued per account id. -const reconnectQueue = createKeyedSerializer(); - -const ALLOWED_IMAP_PORTS = new Set([143, 993]); -const ALLOWED_SMTP_PORTS = new Set([465, 587]); - -function validatePort(port, allowed) { - const n = Number(port); - if (!Number.isInteger(n) || n < 1 || n > 65535) { - return `Port ${port} is not a valid port number`; - } - // When private/local hosts are explicitly allowed (e.g. Proton Mail Bridge on 1143/1025), - // skip the whitelist — the operator has already opted into unrestricted host access. - if (process.env.ALLOW_PRIVATE_IMAP_HOSTS === 'true') return null; - if (!allowed.has(n)) { - return `Port ${port} is not allowed. Allowed: ${[...allowed].join(', ')}`; - } - return null; -} - -// Reject strings that contain characters that could inject extra email headers. -function hasHeaderInjectionChars(str) { - return typeof str === 'string' && /[\r\n\0]/.test(str); -} +import { safeAccount } from '../services/accountFields.js'; +import { + ALLOWED_IMAP_PORTS, + ALLOWED_SMTP_PORTS, + createAccount, + reconcileConnectionState, + validatePort, +} from '../services/accountService.js'; +import { testConnection } from '../services/connectionTest.js'; const router = Router(); router.use(requireAuth); -// Fields safe to return to the client — matches the GET list, excludes credentials and tokens -const SAFE_FIELDS = [ - 'id', 'name', 'sender_name', 'email_address', 'color', 'protocol', - 'imap_host', 'imap_port', 'imap_skip_tls_verify', - 'smtp_host', 'smtp_port', 'smtp_tls', - 'auth_user', 'oauth_provider', 'enabled', - 'last_sync', 'sync_error', 'sort_order', 'folder_mappings', - 'signature', 'created_at', 'categorization_enabled', - 'gtd_enabled', 'gtd_folders', -]; -function safeAccount(row) { - const obj = Object.fromEntries(SAFE_FIELDS.map(k => [k, row[k]])); - // Sanitize on read so legacy values stored before the write-time sanitizer are safe - if (obj.signature) obj.signature = sanitizeSignature(obj.signature); - return obj; -} - router.get('/', async (req, res) => { const result = await query( `SELECT id, name, sender_name, email_address, color, protocol, imap_host, imap_port, imap_tls, imap_skip_tls_verify, @@ -97,64 +60,9 @@ router.get('/', async (req, res) => { }); router.post('/', async (req, res) => { - const { - name, sender_name = null, email_address, color = '#6366f1', protocol = 'imap', - imap_host, imap_port = 993, imap_skip_tls_verify = false, - smtp_host, smtp_port = 587, smtp_tls = 'STARTTLS', - auth_user, auth_pass, - oauth_provider, oauth_access_token, oauth_refresh_token, - signature = null - } = req.body; - - if (!name || !email_address) return res.status(400).json({ error: 'Name and email required' }); - if (hasHeaderInjectionChars(name) || hasHeaderInjectionChars(email_address)) { - return res.status(400).json({ error: 'Name and email address cannot contain control characters' }); - } - if (sender_name && hasHeaderInjectionChars(sender_name)) { - return res.status(400).json({ error: 'Sender name cannot contain control characters' }); - } - - const policy = await getConnectionPolicy(); - - if (imap_host) { - const err = (await validateHost(imap_host, { allowPrivate: policy.allowPrivateHosts })) - || (!policy.allowNonstandardPorts && validatePort(imap_port, ALLOWED_IMAP_PORTS)); - if (err) return res.status(400).json({ error: `IMAP: ${err}` }); - } - if (smtp_host) { - const err = (await validateHost(smtp_host, { allowPrivate: policy.allowPrivateHosts })) - || (!policy.allowNonstandardPorts && validatePort(smtp_port, ALLOWED_SMTP_PORTS)); - if (err) return res.status(400).json({ error: `SMTP: ${err}` }); - } - - try { - const result = await query(` - INSERT INTO email_accounts ( - user_id, name, sender_name, email_address, color, protocol, - imap_host, imap_port, imap_tls, imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, - auth_user, auth_pass, oauth_provider, oauth_access_token, oauth_refresh_token, - signature - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) - RETURNING * - `, [ - req.session.userId, name, sender_name || null, email_address, color, protocol, - imap_host, imap_port, Number(imap_port) % 1000 === 993, !!imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, - auth_user, encrypt(auth_pass), oauth_provider, encrypt(oauth_access_token), encrypt(oauth_refresh_token), - sanitizeSignature(signature) || null - ]); - - const account = result.rows[0]; - - // Immediately try to connect — needs full credentials from DB row - if (protocol === 'imap') { - imapManager.connectAccount(account).catch(console.error); - } - - res.json(safeAccount(account)); - } catch (err) { - console.error(err); - res.status(500).json({ error: 'Failed to add account' }); - } + const result = await createAccount({ userId: req.session.userId, fields: req.body }); + if (result.error) return res.status(result.status).json({ error: result.error }); + res.json(result.account); }); router.put('/:id', async (req, res) => { @@ -257,39 +165,12 @@ router.put('/:id', async (req, res) => { if ('gtd_enabled' in updates || 'gtd_folders' in updates) invalidateGtdConfigCache(id); // Sync live IMAP state after DB update (fire-and-forget, non-fatal). - // Toggling gtd_enabled reconnects so the GTD sync tick is armed/torn down (it is - // only decided at connectAccount) and the persistent-connection account object the - // INBOX/send transition hooks close over picks up the new flag. Remapping a GTD state - // to a different folder reconnects for the same reason: connectAccount's syncFolders + - // backfillAllFolders is what pulls the newly-designated folder's existing mail into the - // rail (backfillAllFolders backfills every discovered folder except the provider's - // skipFolderPatterns, which a GTD label folder never matches). - const isDisabling = 'enabled' in updates && !updates.enabled; - const needsReconnect = !isDisabling && ( - 'enabled' in updates || - 'auth_user' in updates || - 'auth_pass' in updates || - 'imap_host' in updates || - 'imap_port' in updates || - 'imap_tls' in updates || - 'imap_skip_tls_verify' in updates || - 'gtd_enabled' in updates || - gtdFoldersChanged - ); - - // Both branches queue through the per-account serializer so overlapping settings - // changes (e.g. a rapid gtd_enabled double-toggle) apply their connection-state - // effects in order, never as two overlapping chains. - if (isDisabling) { - reconnectQueue(id, () => imapManager.disconnectAccount(id)) - .catch(err => console.error(`Failed to disconnect account ${id} after disable:`, err.message)); - } else if (needsReconnect && updated.protocol === 'imap' && updated.enabled) { - reconnectQueue(id, () => - imapManager.disconnectAccount(id) - .then(() => query('SELECT * FROM email_accounts WHERE id = $1', [id])) - .then(r => { if (r.rows.length) return imapManager.connectAccount(r.rows[0]); }) - ).catch(err => console.error(`Failed to reconnect account ${id} after update:`, err.message)); - } + reconcileConnectionState({ + id, + updates, + before: { gtdFoldersChanged }, + updated, + }); }); router.delete('/:id', async (req, res) => { @@ -312,6 +193,16 @@ router.delete('/:id', async (req, res) => { } }); +router.post('/:id/test-connection', async (req, res) => { + const result = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + [req.params.id, req.session.userId] + ); + if (!result.rows.length) return res.status(404).json({ error: 'Account not found' }); + + res.json(await testConnection(result.rows[0])); +}); + router.post('/:id/reconnect', async (req, res) => { const { id } = req.params; const result = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [id, req.session.userId]); diff --git a/backend/src/routes/accounts.test.js b/backend/src/routes/accounts.test.js new file mode 100644 index 00000000..530e639a --- /dev/null +++ b/backend/src/routes/accounts.test.js @@ -0,0 +1,123 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { query, testConnection } = vi.hoisted(() => ({ + query: vi.fn(), + testConnection: vi.fn(), +})); + +vi.mock('../services/db.js', () => ({ query })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('../index.js', () => ({ imapManager: {} })); +vi.mock('../services/connectionTest.js', () => ({ testConnection })); +vi.mock('../services/accountService.js', () => ({ + ALLOWED_IMAP_PORTS: new Set([143, 993]), + ALLOWED_SMTP_PORTS: new Set([465, 587]), + createAccount: vi.fn(), + reconcileConnectionState: vi.fn(), + validatePort: vi.fn(), +})); + +import 'express-async-errors'; +import express from 'express'; +import accountRoutes from './accounts.js'; + +const account = { + id: 'account-1', + user_id: 'user-1', + imap_host: 'imap.example.com', + smtp_host: 'smtp.example.com', +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/accounts', accountRoutes); + return app; +} + +let server; +let base; + +beforeAll(async () => { + await new Promise(resolve => { + server = buildApp().listen(0, '127.0.0.1', resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; +}); + +afterAll(async () => { + await new Promise(resolve => server.close(resolve)); +}); + +beforeEach(() => { + query.mockReset(); + testConnection.mockReset(); +}); + +async function postTestConnection(id = account.id) { + const response = await fetch(`${base}/api/accounts/${id}/test-connection`, { + method: 'POST', + }); + return { + status: response.status, + body: await response.json(), + }; +} + +describe('POST /api/accounts/:id/test-connection', () => { + it('404s an absent or unowned account', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + const response = await postTestConnection('foreign-account'); + + expect(response).toEqual({ + status: 404, + body: { error: 'Account not found' }, + }); + expect(query).toHaveBeenCalledWith( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + ['foreign-account', 'user-1'] + ); + expect(testConnection).not.toHaveBeenCalled(); + }); + + it('returns both successful probe legs', async () => { + query.mockResolvedValueOnce({ rows: [account] }); + testConnection.mockResolvedValueOnce({ + imap: { ok: true }, + smtp: { ok: true }, + }); + + const response = await postTestConnection(); + + expect(response).toEqual({ + status: 200, + body: { imap: { ok: true }, smtp: { ok: true } }, + }); + expect(testConnection).toHaveBeenCalledWith(account); + }); + + it.each([ + ['IMAP', { + imap: { ok: false, error: 'Authentication failed.' }, + smtp: { ok: true }, + }], + ['SMTP', { + imap: { ok: true }, + smtp: { ok: false, error: 'Could not connect.' }, + }], + ])('returns the per-leg failure shape when %s fails', async (_leg, probeResult) => { + query.mockResolvedValueOnce({ rows: [account] }); + testConnection.mockResolvedValueOnce(probeResult); + + const response = await postTestConnection(); + + expect(response.status).toBe(200); + expect(response.body).toEqual(probeResult); + }); +}); diff --git a/backend/src/routes/apiTokens.js b/backend/src/routes/apiTokens.js index 598975b9..1bfbced8 100644 --- a/backend/src/routes/apiTokens.js +++ b/backend/src/routes/apiTokens.js @@ -1,7 +1,7 @@ import { Router } from 'express'; import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import { generateToken, hashToken } from '../mcp/auth.js'; +import { ALL_SCOPES, expandScopes, generateToken, hashToken } from '../mcp/auth.js'; const router = Router(); router.use(requireAuth); @@ -9,18 +9,31 @@ router.use(requireAuth); router.post('/', async (req, res) => { const name = (req.body?.name || '').trim(); if (!name) return res.status(400).json({ error: 'name is required' }); + const requestedScopes = Array.isArray(req.body?.scopes) && req.body.scopes.length + ? req.body.scopes + : ['read']; + const unknownScopes = requestedScopes.filter((scope) => !ALL_SCOPES.includes(scope)); + if (unknownScopes.length) { + return res.status(400).json({ error: `unknown scope(s): ${unknownScopes.join(', ')}` }); + } + const scopes = expandScopes(requestedScopes); const token = generateToken(); const { rows } = await query( - 'INSERT INTO api_tokens (user_id, token_hash, name) VALUES ($1, $2, $3) RETURNING id, name', - [req.session.userId, hashToken(token), name], + 'INSERT INTO api_tokens (user_id, token_hash, name, scopes) VALUES ($1, $2, $3, $4) RETURNING id, name, scopes', + [req.session.userId, hashToken(token), name, scopes], ); // Plaintext returned exactly once; only the hash was persisted. - res.status(201).json({ id: rows[0].id, name: rows[0].name, token }); + res.status(201).json({ + id: rows[0].id, + name: rows[0].name, + scopes: rows[0].scopes, + token, + }); }); router.get('/', async (req, res) => { const { rows } = await query( - 'SELECT id, name, created_at, last_used_at FROM api_tokens WHERE user_id = $1 ORDER BY created_at DESC', + 'SELECT id, name, scopes, created_at, last_used_at FROM api_tokens WHERE user_id = $1 ORDER BY created_at DESC', [req.session.userId], ); res.json({ tokens: rows }); diff --git a/backend/src/routes/apiTokens.test.js b/backend/src/routes/apiTokens.test.js index 92309b2e..92b08264 100644 --- a/backend/src/routes/apiTokens.test.js +++ b/backend/src/routes/apiTokens.test.js @@ -30,16 +30,77 @@ async function call(app, method, path, body) { return { status: res.status, body: text ? JSON.parse(text) : null }; } +function mockRes() { + return { + statusCode: 200, + body: null, + status(code) { this.statusCode = code; return this; }, + json(body) { this.body = body; return this; }, + end() { return this; }, + }; +} + +async function callRoute(method, { body = {}, params = {} } = {}) { + const layer = router.stack.find((entry) => entry.route?.path === '/' && entry.route.methods[method]); + const req = { body, params, session: { userId: 'user-1' } }; + const res = mockRes(); + await layer.route.stack[0].handle(req, res); + return { status: res.statusCode, body: res.body }; +} + beforeEach(() => query.mockReset()); describe('POST /api/tokens', () => { + it('defaults a token with no requested scopes to read', async () => { + for (const scopes of [undefined, []]) { + query.mockResolvedValueOnce({ + rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read'] }], + }); + const { status, body } = await callRoute('post', { body: { name: 'laptop', scopes } }); + expect(status).toBe(201); + expect(body.scopes).toEqual(['read']); + expect(query.mock.calls.at(-1)[0]).toMatch(/api_tokens \(user_id, token_hash, name, scopes\)/); + expect(query.mock.calls.at(-1)[1][3]).toEqual(['read']); + } + }); + + it('rejects unknown requested scopes', async () => { + query.mockResolvedValueOnce({ + rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read', 'admin'] }], + }); + const { status, body } = await callRoute('post', { + body: { name: 'laptop', scopes: ['read', 'admin'] }, + }); + expect(status).toBe(400); + expect(body).toEqual({ error: 'unknown scope(s): admin' }); + expect(query).not.toHaveBeenCalled(); + }); + + it('mints a token with expanded requested scopes', async () => { + query.mockResolvedValueOnce({ + rows: [{ id: 'tok-2', name: 'sender', scopes: ['send', 'read'] }], + }); + const { status, body } = await callRoute('post', { + body: { name: 'sender', scopes: ['send'] }, + }); + expect(status).toBe(201); + expect(body).toMatchObject({ + id: 'tok-2', + name: 'sender', + scopes: ['send', 'read'], + }); + expect(body.token).toMatch(/^mcp_/); + expect(query.mock.calls[0][1][3]).toEqual(['send', 'read']); + expect(query.mock.calls[0][1]).not.toContain(body.token); + }); + it('mints a token, returns the plaintext once, and stores only the hash', async () => { - query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop' }] }); + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read'] }] }); const { status, body } = await call(appWith(), 'POST', '/api/tokens', { name: 'laptop' }); expect(status).toBe(201); expect(body.token).toMatch(/^mcp_/); expect(body).toMatchObject({ id: 'tok-1', name: 'laptop' }); - // INSERT bound values: [user_id, token_hash, name] — never the plaintext. + // INSERT bound values: [user_id, token_hash, name, scopes] — never the plaintext. const params = query.mock.calls[0][1]; expect(params[0]).toBe('user-1'); expect(params[1]).toMatch(/^[0-9a-f]{64}$/); @@ -53,11 +114,27 @@ describe('POST /api/tokens', () => { }); describe('GET /api/tokens', () => { + it('selects and returns each token scopes', async () => { + query.mockResolvedValueOnce({ + rows: [{ + id: 'tok-1', + name: 'laptop', + scopes: ['read', 'write'], + created_at: 't', + last_used_at: null, + }], + }); + const { status, body } = await callRoute('get'); + expect(status).toBe(200); + expect(body.tokens[0].scopes).toEqual(['read', 'write']); + expect(query.mock.calls[0][0]).toMatch(/SELECT id, name, scopes, created_at, last_used_at/); + }); + it('lists tokens without hashes or plaintext', async () => { - query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', created_at: 't', last_used_at: null }] }); + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read'], created_at: 't', last_used_at: null }] }); const { status, body } = await call(appWith(), 'GET', '/api/tokens'); expect(status).toBe(200); - expect(body.tokens[0]).toEqual({ id: 'tok-1', name: 'laptop', created_at: 't', last_used_at: null }); + expect(body.tokens[0]).toEqual({ id: 'tok-1', name: 'laptop', scopes: ['read'], created_at: 't', last_used_at: null }); expect(JSON.stringify(body)).not.toContain('token_hash'); }); }); diff --git a/backend/src/routes/auth.js b/backend/src/routes/auth.js index ea84bec2..83e64265 100644 --- a/backend/src/routes/auth.js +++ b/backend/src/routes/auth.js @@ -18,6 +18,7 @@ import { sanitizeGtdPrefs } from '../utils/gtdPrefs.js'; import { sanitizeRightSidebarPrefs } from '../utils/rightSidebarPrefs.js'; import { redisClient } from '../services/redis.js'; import { consume as rlConsume, reset as rlReset } from '../services/rateLimiter.js'; +import { UNDO_CHOICES } from '../services/outboxService.js'; const router = Router(); @@ -759,7 +760,7 @@ router.patch('/preferences', async (req, res) => { expandedAccounts, collapsedFolders, favoriteFolders, recentFolders, fontSize, showAppBadge, showFaviconBadge, replyDefault, sidebarWidth, categorizationEnabled, markReadBehavior, markReadDelay, aiActions, - autoLockMinutes, showMobileAvatars, gravatarAvatars } = req.body; + autoLockMinutes, showMobileAvatars, gravatarAvatars, undoSendSeconds } = req.body; // GTD content and generic right-sidebar layout preferences are independent flat // top-level keys with separate allow-lists. gtdEnabled is intentionally NOT a user // preference — it lives per-account in email_accounts.gtd_enabled. @@ -781,6 +782,7 @@ router.patch('/preferences', async (req, res) => { const markReadBehaviorVal = ['immediate', 'delay', 'manual'].includes(markReadBehavior) ? markReadBehavior : null; const markReadDelayVal = (() => { const n = parseInt(markReadDelay); return (n >= 1 && n <= 10) ? String(n) : null; })(); const autoLockMinutesVal = [0, 1, 5, 15, 30].includes(Number(autoLockMinutes)) ? String(Number(autoLockMinutes)) : null; + const undoSendSecondsVal = UNDO_CHOICES.includes(Number(undoSendSeconds)) ? String(Number(undoSendSeconds)) : null; // User-defined AI actions: bound the array and each field so the JSONB can't grow unbounded. const aiActionsJson = (() => { if (!Array.isArray(aiActions)) return null; @@ -830,6 +832,7 @@ router.patch('/preferences', async (req, res) => { || CASE WHEN $35::text IS NOT NULL THEN jsonb_build_object('autoLockMinutes', $35::text) ELSE '{}'::jsonb END || CASE WHEN $36::boolean IS NOT NULL THEN jsonb_build_object('showMobileAvatars', $36::boolean) ELSE '{}'::jsonb END || CASE WHEN $37::boolean IS NOT NULL THEN jsonb_build_object('gravatarAvatars', $37::boolean) ELSE '{}'::jsonb END + || CASE WHEN $38::int IS NOT NULL THEN jsonb_build_object('undoSendSeconds', $38::int) ELSE '{}'::jsonb END WHERE id = $1 `, [req.session.userId, theme ?? null, font ?? null, layout ?? null, notificationSound ?? null, pageSize ?? null, scrollMode ?? null, syncInterval ?? null, @@ -839,7 +842,7 @@ router.patch('/preferences', async (req, res) => { showAppBadge ?? null, showFaviconBadge ?? null, replyDefaultVal, sidebarWidthVal, categorizationEnabled ?? null, markReadBehaviorVal, markReadDelayVal, aiActionsJson, rightSidebarWidth, rightSidebarHidden, gtdCollapsedSectionsJson, gtdPetSlug, autoLockMinutesVal, - showMobileAvatars ?? null, gravatarAvatars ?? null]); + showMobileAvatars ?? null, gravatarAvatars ?? null, undoSendSecondsVal]); if (syncInterval != null) { const ms = parseInt(syncInterval) * 1000; diff --git a/backend/src/routes/auth.preferences.test.js b/backend/src/routes/auth.preferences.test.js new file mode 100644 index 00000000..59078e2a --- /dev/null +++ b/backend/src/routes/auth.preferences.test.js @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { query, updateSyncIntervalForUser } = vi.hoisted(() => ({ + query: vi.fn(), + updateSyncIntervalForUser: vi.fn(), +})); + +vi.mock('../services/db.js', () => ({ query, pool: {} })); +vi.mock('../index.js', () => ({ + imapManager: { updateSyncIntervalForUser }, +})); +vi.mock('../services/redis.js', () => ({ + redisClient: { get: vi.fn(), set: vi.fn(), del: vi.fn() }, +})); + +import authRoutes from './auth.js'; + +function preferencesHandler() { + const layer = authRoutes.stack.find(item => ( + item.route?.path === '/preferences' && item.route.methods.patch + )); + return layer.route.stack[0].handle; +} + +function responseRecorder() { + return { + statusCode: 200, + body: undefined, + status(code) { + this.statusCode = code; + return this; + }, + json(body) { + this.body = body; + return this; + }, + }; +} + +async function patchPreferences(body) { + const req = { body, session: { userId: 'user-1' } }; + const res = responseRecorder(); + await preferencesHandler()(req, res); + return res; +} + +describe('PATCH /preferences undoSendSeconds', () => { + beforeEach(() => { + query.mockReset(); + query.mockResolvedValue({ rows: [] }); + updateSyncIntervalForUser.mockReset(); + }); + + it('stores an allowed undo window in the next SQL bind', async () => { + const res = await patchPreferences({ undoSendSeconds: 60 }); + + expect(res.statusCode).toBe(200); + const [sql, binds] = query.mock.calls[0]; + expect(sql).toContain( + "CASE WHEN $38::int IS NOT NULL THEN jsonb_build_object('undoSendSeconds', $38::int)", + ); + expect(binds).toHaveLength(38); + expect(binds[37]).toBe('60'); + }); + + it('does not store a value outside the supported undo choices', async () => { + await patchPreferences({ undoSendSeconds: 45 }); + + const [, binds] = query.mock.calls[0]; + expect(binds[37]).toBeNull(); + }); +}); diff --git a/backend/src/routes/draft.js b/backend/src/routes/draft.js index 630fbacd..8d507d7b 100644 --- a/backend/src/routes/draft.js +++ b/backend/src/routes/draft.js @@ -1,188 +1,63 @@ -import nodemailer from 'nodemailer'; -import { randomBytes } from 'crypto'; import { Router } from 'express'; -import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import sanitizeHtml from 'sanitize-html'; -import { sanitizeSignature, sanitizeComposeBody } from '../services/emailSanitizer.js'; -import { embedInlineDataImages } from '../utils/inlineImages.js'; import { imapManager } from '../index.js'; +import { query } from '../services/db.js'; +import { deleteDraft, saveDraft } from '../services/draftService.js'; const router = Router(); router.use(requireAuth); -function sanitizeHeaderValue(value) { - if (typeof value !== 'string') return ''; - return value.replace(/[\r\n\0]/g, '').trim(); -} - -// Extract { name, email } from an RFC 5322 address string ("Name ", -// "", or bare "email") for persisting to_addresses/cc_addresses. -function parseAddress(str) { - if (typeof str !== 'string') return { name: '', email: '' }; - const m = str.match(/^(.+?)\s*<([^>]+)>\s*$/); - if (m) return { name: m[1].trim().replace(/^"|"$/g, '').trim(), email: m[2].trim().toLowerCase() }; - const bare = str.match(/^\s*<([^>]+)>\s*$/); - if (bare) return { name: '', email: bare[1].trim().toLowerCase() }; - return { name: '', email: str.trim().toLowerCase() }; -} -function mapRecipientList(list) { - return (Array.isArray(list) ? list : []).filter(Boolean).map(addr => parseAddress(addr)); -} - -function textToHtml(text) { - return text.split('\n') - .map(l => `

${l.replace(/&/g, '&').replace(//g, '>') || ' '}

`) - .join(''); -} - -async function buildRawDraft({ accountId, aliasId, to, cc, bcc, subject, body, bodyIsHtml, quotedBody, quotedBodyHtml, editedSignature }) { - const acctResult = await query( - 'SELECT * FROM email_accounts WHERE id = $1', - [accountId] - ); - if (!acctResult.rows.length) throw Object.assign(new Error('Account not found'), { status: 404 }); - const account = acctResult.rows[0]; - - let fromName = account.sender_name || account.name; - let fromEmail = account.email_address; - let fromSignature = account.signature; - - if (aliasId) { - const aliasResult = await query( - 'SELECT * FROM account_aliases WHERE id = $1 AND account_id = $2', - [aliasId, accountId] - ); - if (aliasResult.rows.length) { - const alias = aliasResult.rows[0]; - fromName = alias.name; - fromEmail = alias.email; - if (alias.signature !== null) fromSignature = alias.signature; - } - } - - const rawSignature = editedSignature !== undefined ? (editedSignature || null) : fromSignature; - const effectiveSignature = rawSignature ? sanitizeSignature(rawSignature) : null; - - const sigText = effectiveSignature - ? sanitizeHtml(effectiveSignature, { allowedTags: [], allowedAttributes: {} }).trim() - : null; - - const bodyText = bodyIsHtml - ? sanitizeHtml(body || '', { allowedTags: [], allowedAttributes: {} }) - : (body || ''); - - const bodyHtml = bodyIsHtml - ? sanitizeComposeBody(body || '') - : textToHtml(body || ''); - - const rawHtml = bodyHtml + - (effectiveSignature ? `
${effectiveSignature}
` : '') + - (quotedBodyHtml || (quotedBody ? textToHtml(quotedBody) : '')); - const { html: draftHtml, attachments: inlineImageAttachments } = embedInlineDataImages(rawHtml); - - // Stable Message-ID so the appended MIME and the local DB row reference the same - // message (and a later sync reconciles cleanly). - const messageId = `<${randomBytes(16).toString('hex')}@${(fromEmail.split('@')[1] || 'mailflow.local')}>`; - const textBody = sigText ? `${bodyText}\n\n-- \n${sigText}${quotedBody || ''}` : `${bodyText}${quotedBody || ''}`; - - const mailOptions = { - messageId, - from: `${fromName} <${fromEmail}>`, - to: (Array.isArray(to) ? to : [to]).filter(Boolean).join(', ') || undefined, - cc: (Array.isArray(cc) ? cc : []).filter(Boolean).join(', ') || undefined, - bcc: (Array.isArray(bcc) ? bcc : []).filter(Boolean).join(', ') || undefined, - subject: sanitizeHeaderValue(subject || ''), - text: textBody, - html: draftHtml, - ...(inlineImageAttachments.length ? { attachments: inlineImageAttachments } : {}), - }; - - const streamTransport = nodemailer.createTransport({ streamTransport: true, newline: 'unix' }); - const streamInfo = await streamTransport.sendMail(mailOptions); - const chunks = []; - await new Promise((resolve, reject) => { - streamInfo.message.on('data', c => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c))); - streamInfo.message.on('end', resolve); - streamInfo.message.on('error', reject); - }); - // rawHtml (pre inline-image embedding) is what the composer should reopen with — - // inline data: URIs stay editable and getMessageBody serves body_html from the DB. - const snippet = textBody.replace(/\s+/g, ' ').trim().slice(0, 200); - return { - rawMessage: Buffer.concat(chunks), - account, - meta: { messageId, fromName, fromEmail, bodyHtml: rawHtml, bodyText: textBody, snippet }, - }; -} - -async function resolveDraftsFolder(account) { - const mapped = account.folder_mappings?.drafts; - if (mapped) return mapped; - const result = await query( - "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Drafts' LIMIT 1", - [account.id] - ); - return result.rows[0]?.path || null; -} - router.post('/draft', async (req, res) => { - const { accountId, aliasId, to, cc, bcc, subject, body, bodyIsHtml = false, quotedBody, quotedBodyHtml, editedSignature, existingUid, existingFolder } = req.body; + const { + accountId, + aliasId, + to, + cc, + bcc, + subject, + body, + bodyIsHtml = false, + quotedBody, + quotedBodyHtml, + editedSignature, + inReplyTo, + references, + existingUid, + existingFolder, + } = req.body; if (!accountId) return res.status(400).json({ error: 'accountId required' }); - const ownerCheck = await query( - 'SELECT id FROM email_accounts WHERE id = $1 AND user_id = $2', - [accountId, req.session.userId] + const accountResult = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + [accountId, req.session.userId], ); - if (!ownerCheck.rows.length) return res.status(404).json({ error: 'Account not found' }); + if (!accountResult.rows.length) return res.status(404).json({ error: 'Account not found' }); try { - const { rawMessage, account, meta } = await buildRawDraft({ accountId, aliasId, to, cc, bcc, subject, body, bodyIsHtml, quotedBody, quotedBodyHtml, editedSignature }); - - const draftsFolder = await resolveDraftsFolder(account); - if (!draftsFolder) return res.status(422).json({ error: 'No Drafts folder found for this account' }); - - // APPEND the new draft first so we never lose the message - const { uid } = await imapManager.appendToFolder(account, draftsFolder, rawMessage, ['\\Draft', '\\Seen']); - - // Persist a local Drafts row immediately so the composer can reopen this draft - // (recipient/subject/body) even if the folder re-sync is delayed or fails on a - // flaky connection. Non-fatal — the append already stored the message on IMAP. - if (uid != null) { - try { - await imapManager.upsertDraftMessageRecord(account, draftsFolder, uid, { - messageId: meta.messageId, - subject, - fromName: meta.fromName, - fromEmail: meta.fromEmail, - to: mapRecipientList(to), - cc: mapRecipientList(cc), - snippet: meta.snippet, - bodyHtml: meta.bodyHtml, - bodyText: meta.bodyText, - }); - } catch (rowErr) { - console.error(`Draft: failed to persist local row uid=${uid}: ${rowErr.message}`); - } - } - - // Delete the old draft only after the new one is safely stored - if (existingUid && existingFolder) { - try { - await imapManager.permanentDeleteMessage(account, existingUid, existingFolder); - await query( - 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', - [account.id, existingUid, existingFolder] - ); - } catch (delErr) { - console.error(`Draft: failed to delete old uid=${existingUid}: ${delErr.message}`); - } - } - - res.json({ uid, folder: draftsFolder }); + const result = await saveDraft({ + userId: req.session.userId, + account: accountResult.rows[0], + aliasId, + to, + cc, + bcc, + subject, + body, + bodyIsHtml, + quotedBody, + quotedBodyHtml, + editedSignature, + inReplyTo, + references, + existingUid, + existingFolder, + }, { query, imapManager }); + return res.json({ uid: result.uid, folder: result.folder }); } catch (err) { console.error('Save draft failed:', err.message); - res.status(err.status || 500).json({ error: err.message || 'Failed to save draft' }); + const response = { error: err.message || 'Failed to save draft' }; + if (err.code === 'alias_not_found') response.code = err.code; + return res.status(err.status || 500).json(response); } }); @@ -191,25 +66,27 @@ router.delete('/draft/:uid', async (req, res) => { if (!uid || !Number.isFinite(uid)) return res.status(400).json({ error: 'Invalid uid' }); const { accountId, folder } = req.query; - if (!accountId || !folder) return res.status(400).json({ error: 'accountId and folder required' }); + if (!accountId || !folder) { + return res.status(400).json({ error: 'accountId and folder required' }); + } - const ownerCheck = await query( + const accountResult = await query( 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', - [accountId, req.session.userId] + [accountId, req.session.userId], ); - if (!ownerCheck.rows.length) return res.status(404).json({ error: 'Account not found' }); + if (!accountResult.rows.length) return res.status(404).json({ error: 'Account not found' }); try { - const account = ownerCheck.rows[0]; - await imapManager.permanentDeleteMessage(account, uid, folder); - await query( - 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', - [account.id, uid, folder] - ); - res.json({ ok: true }); + const result = await deleteDraft({ + userId: req.session.userId, + account: accountResult.rows[0], + uid, + folder, + }, { query, imapManager }); + return res.json(result); } catch (err) { console.error('Delete draft failed:', err.message); - res.status(500).json({ error: err.message || 'Failed to delete draft' }); + return res.status(500).json({ error: err.message || 'Failed to delete draft' }); } }); diff --git a/backend/src/routes/draft.test.js b/backend/src/routes/draft.test.js index 31b52296..fd1ab774 100644 --- a/backend/src/routes/draft.test.js +++ b/backend/src/routes/draft.test.js @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeAll, afterAll, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; vi.mock('../services/db.js', () => ({ query: vi.fn() })); vi.mock('../middleware/auth.js', () => ({ @@ -11,7 +11,6 @@ const imapManager = vi.hoisted(() => ({ })); vi.mock('../index.js', () => ({ imapManager })); -import express from 'express'; import draftRoutes from './draft.js'; import { query } from '../services/db.js'; @@ -21,27 +20,42 @@ const ACCOUNT_ROW = { sender_name: null, signature: null, folder_mappings: {}, }; -function buildApp() { - const app = express(); - app.use(express.json()); - app.use('/api/mail', draftRoutes); - return app; +function draftHandler() { + const layer = draftRoutes.stack.find(item => ( + item.route?.path === '/draft' && item.route.methods.post + )); + return layer.route.stack[0].handle; +} + +function responseRecorder() { + return { + statusCode: 200, + body: undefined, + status(code) { + this.statusCode = code; + return this; + }, + json(body) { + this.body = body; + return this; + }, + }; +} + +async function postDraft(body) { + const req = { body, session: { userId: 'user-1' } }; + const res = responseRecorder(); + await draftHandler()(req, res); + return res; } describe('POST /api/mail/draft — local row persistence', () => { - let server, base; - beforeAll(async () => { - await new Promise(r => { server = buildApp().listen(0, r); }); - base = `http://127.0.0.1:${server.address().port}`; - }); - afterAll(async () => { await new Promise(r => server.close(r)); }); beforeEach(() => { query.mockReset(); imapManager.appendToFolder.mockReset(); imapManager.upsertDraftMessageRecord.mockReset(); imapManager.permanentDeleteMessage.mockReset(); - // 1) owner check, 2) buildRawDraft account load, 3) resolveDraftsFolder lookup - query.mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID }] }); + // 1) scoped account row, 2) resolveDraftsFolder lookup query.mockResolvedValueOnce({ rows: [ACCOUNT_ROW] }); query.mockResolvedValueOnce({ rows: [{ path: 'Drafts' }] }); imapManager.appendToFolder.mockResolvedValue({ uid: 5, folder: 'Drafts' }); @@ -49,20 +63,18 @@ describe('POST /api/mail/draft — local row persistence', () => { }); it('persists a Drafts row with parsed recipient, subject and body after append', async () => { - const res = await fetch(`${base}/api/mail/draft`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - accountId: ACCOUNT_ID, - to: ['Mike Scanlan '], - cc: [], - subject: 'Re: MailFlow hero', - body: 'hello mike', - bodyIsHtml: false, - }), + const res = await postDraft({ + accountId: ACCOUNT_ID, + to: ['Mike Scanlan '], + cc: [], + subject: 'Re: MailFlow hero', + body: 'hello mike', + bodyIsHtml: false, + inReplyTo: '', + references: ' ', }); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ uid: 5, folder: 'Drafts' }); + expect(res.statusCode).toBe(200); + expect(res.body).toEqual({ uid: 5, folder: 'Drafts' }); expect(imapManager.upsertDraftMessageRecord).toHaveBeenCalledTimes(1); const [acct, folder, uid, meta] = imapManager.upsertDraftMessageRecord.mock.calls[0]; @@ -74,28 +86,22 @@ describe('POST /api/mail/draft — local row persistence', () => { expect(meta.fromEmail).toBe('matthias@mailflow.sh'); expect(meta.bodyHtml).toContain('hello mike'); expect(meta.bodyText).toContain('hello mike'); + expect(meta.inReplyTo).toBe(''); + expect(meta.references).toBe(' '); expect(meta.messageId).toMatch(/^<[0-9a-f]+@mailflow\.sh>$/); }); it('still returns success if the local row persistence throws (append already stored it)', async () => { imapManager.upsertDraftMessageRecord.mockRejectedValueOnce(new Error('db down')); - const res = await fetch(`${base}/api/mail/draft`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }), - }); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ uid: 5, folder: 'Drafts' }); + const res = await postDraft({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }); + expect(res.statusCode).toBe(200); + expect(res.body).toEqual({ uid: 5, folder: 'Drafts' }); }); it('does not persist a row when the append returns no uid (no reliable key)', async () => { imapManager.appendToFolder.mockResolvedValueOnce({ uid: null, folder: 'Drafts' }); - const res = await fetch(`${base}/api/mail/draft`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }), - }); - expect(res.status).toBe(200); + const res = await postDraft({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }); + expect(res.statusCode).toBe(200); expect(imapManager.upsertDraftMessageRecord).not.toHaveBeenCalled(); }); }); diff --git a/backend/src/routes/gtd.js b/backend/src/routes/gtd.js index 39fdedd2..7662ce6f 100644 --- a/backend/src/routes/gtd.js +++ b/backend/src/routes/gtd.js @@ -3,62 +3,27 @@ import { requireAuth } from '../middleware/auth.js'; import { getGtdSections } from '../services/gtdSections.js'; import { queueGistGeneration } from '../services/gtdGist.js'; import { importPet, decodeUploadedSheet, getPetMeta, getPetSheet, parsePetSlug, customPetSlug } from '../services/gtdPet.js'; -import { getGtdConfig, resolveGtdStateFolder, sanitizeGtdFolders, sanitizeGtdFoldersDetailed, DEFAULT_GTD_FOLDERS, planGtdFolderPersist, invalidateGtdConfigCache } from '../services/gtdConfig.js'; -import { fanOutReadToSiblings } from '../utils/mailUtils.js'; -import { archiveInboxCopy } from '../services/archiveInbox.js'; +import { sanitizeGtdFolders, sanitizeGtdFoldersDetailed, DEFAULT_GTD_FOLDERS, planGtdFolderPersist, invalidateGtdConfigCache } from '../services/gtdConfig.js'; import { query } from '../services/db.js'; import { imapManager } from '../index.js'; +import { + classifyTarget, + gtdClassify, + gtdDone, + gtdUnclassify, + resolveDoneFolders, +} from '../services/gtd/actions.js'; const router = Router(); router.use(requireAuth); +export { classifyTarget, resolveDoneFolders }; + // Message/account ids are always UUIDs; pre-validate before any DB lookup so a malformed // id is a clean 400 rather than a parametrized query that just finds nothing (404) or a // driver cast error. Same idiom + regex as mail.js. const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -// Shared classify precondition: an account must have GTD enabled and the request's -// state must resolve to a designated folder. Returns { folder } to proceed, or -// { status, error } to reject. Pure — exported for unit tests. -export function classifyTarget({ enabled, folders, state }) { - if (!enabled) return { status: 400, error: 'GTD is not enabled for this account' }; - const folder = resolveGtdStateFolder(state, folders); - if (!folder) return { status: 400, error: `Unknown GTD state: ${state}` }; - return { folder }; -} - -// The "done" action's precondition, resolving the GTD label folders a done request must -// strip. Two contracts, per the caller: -// • Explicit `states` array (GTD sidebar entry): resolve each state to its designated folder, in -// order, deduped — a merged Waiting row carries both watch and delegated. Unknown state -// rejects. Mirrors classifyTarget but plural. -// • `states === 'all'` (inbox checkmark): strip every designated GTD label the thread -// actually carries. `existing` is the set of folder paths a live copy was found in; we -// intersect it with the account's designated GTD folders (map order, deduped). Absent -// labels are skipped, never an error; a thread with none resolves to { folders: [] } so -// the route degrades to mark-read + archive. -// Returns { folders } to proceed, or { status, error } to reject. Pure — exported for tests. -export function resolveDoneFolders({ enabled, folders, states, existing }) { - if (!enabled) return { status: 400, error: 'GTD is not enabled for this account' }; - if (states === 'all') { - const present = new Set(Array.isArray(existing) ? existing : []); - const resolved = []; - for (const folder of Object.values(folders || {})) { - if (present.has(folder) && !resolved.includes(folder)) resolved.push(folder); - } - return { folders: resolved }; - } - if (!Array.isArray(states) || states.length === 0) { - return { status: 400, error: 'states must be a non-empty array' }; - } - const resolved = []; - for (const state of states) { - const folder = resolveGtdStateFolder(state, folders); - if (!folder) return { status: 400, error: `Unknown GTD state: ${state}` }; - if (!resolved.includes(folder)) resolved.push(folder); - } - return { folders: resolved }; -} // GET /api/gtd/sections — thread heads + counts per GTD state for GTD display surfaces. // accountId absent => unified across the user's gtd_enabled accounts; present => scoped @@ -138,221 +103,53 @@ router.get('/pet/:slug/sheet', async (req, res) => { res.send(sheet.data); }); -// Load a message the caller owns, or send a 404. The email_accounts join is the -// ownership filter (a.user_id = $2); the message row itself carries everything the -// callers need (account_id, uid, folder, message_id), so no account column is selected. -async function loadOwnedMessage(userId, messageId) { - const result = await query( - `SELECT m.* - FROM messages m - JOIN email_accounts a ON a.id = m.account_id - WHERE m.id = $1 AND a.user_id = $2`, - [messageId, userId] - ); - return result.rows[0] || null; -} - -// POST /api/gtd/classify { messageId, state } — apply a GTD label by COPYing the -// message into the state's designated folder (the message stays in its current -// folder; classify never removes it from the inbox). Thin: resolve the folder, -// ensure it exists (callers own folder existence), then delegate to -// imapManager.copyMessage, which also emits gtd_sections_updated. +// Apply a GTD label by copying the message into the state folder. router.post('/classify', async (req, res) => { const { messageId, state } = req.body || {}; if (!messageId || !state) return res.status(400).json({ error: 'messageId and state are required' }); if (!UUID_RE.test(messageId)) return res.status(400).json({ error: 'Invalid message id' }); - const msg = await loadOwnedMessage(req.session.userId, messageId); - if (!msg) return res.status(404).json({ error: 'Message not found' }); - - const { enabled, folders } = await getGtdConfig(msg.account_id); - const target = classifyTarget({ enabled, folders, state }); - if (target.error) return res.status(target.status).json({ error: target.error }); - const toFolder = target.folder; - - // Already labelled with this state — nothing to copy. - if (msg.folder === toFolder) return res.json({ ok: true, folder: toFolder }); - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - const account = accountResult.rows[0]; - - try { - await imapManager.ensureFolder(account, toFolder); - await imapManager.copyMessage(msg.account_id, msg.uid, msg.folder, toFolder); - } catch (err) { - console.error(`GTD classify failed for message ${messageId} -> ${toFolder}:`, err.message); - return res.status(500).json({ error: 'Failed to apply GTD label' }); - } - - res.json({ ok: true, folder: toFolder }); + const result = await gtdClassify(imapManager, { + userId: req.session.userId, + accountIds: null, + messageId, + state, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); -// Resolve the folder-copy uid a message has in `folder` for this account, or null. The -// acted row is used directly when it already lives there; otherwise the shared RFC -// Message-ID (COPY duplicates it verbatim) joins to the sibling copy. Shared by DELETE -// /classify (below) and POST /done's label strip. -async function resolveCopyUid(msg, folder) { - if (msg.folder === folder) return msg.uid; - const sib = await query( - 'SELECT uid FROM messages WHERE account_id = $1 AND folder = $2 AND message_id = $3 AND is_deleted = false LIMIT 1', - [msg.account_id, folder, msg.message_id] - ); - return sib.rows[0]?.uid ?? null; -} - -// DELETE /api/gtd/classify { messageId, state } — remove a GTD label by deleting -// the message's copy that lives in the state folder, leaving all other copies -// (INBOX, other labels) intact. The acted message id identifies the thread member -// by its RFC Message-ID; the copy in the state folder is resolved from that. +// Remove a GTD label copy while leaving all other message copies intact. router.delete('/classify', async (req, res) => { const { messageId, state } = req.body || {}; if (!messageId || !state) return res.status(400).json({ error: 'messageId and state are required' }); if (!UUID_RE.test(messageId)) return res.status(400).json({ error: 'Invalid message id' }); - const msg = await loadOwnedMessage(req.session.userId, messageId); - if (!msg) return res.status(404).json({ error: 'Message not found' }); - - const { enabled, folders } = await getGtdConfig(msg.account_id); - const target = classifyTarget({ enabled, folders, state }); - if (target.error) return res.status(target.status).json({ error: target.error }); - const stateFolder = target.folder; - - // Find the copy that lives in the state folder via resolveCopyUid: the acted row when it - // already lives there, else the shared RFC Message-ID (COPY duplicates it verbatim) joins to - // the sibling. A missing Message-ID only blocks that sibling lookup — the acted-row case - // needs no Message-ID — so guard it there and keep the explicit 400 the client relies on. - if (msg.folder !== stateFolder && !msg.message_id) { - return res.status(400).json({ error: 'Message has no Message-ID — cannot resolve GTD copy' }); - } - const siblingUid = await resolveCopyUid(msg, stateFolder); - if (siblingUid == null) return res.json({ ok: true, removed: false }); - - try { - await imapManager.removeMessageCopy(msg.account_id, siblingUid, stateFolder); - } catch (err) { - console.error(`GTD unclassify failed for message ${messageId} in ${stateFolder}:`, err.message); - return res.status(500).json({ error: 'Failed to remove GTD label' }); - } - - res.json({ ok: true, removed: true, folder: stateFolder }); + const result = await gtdUnclassify(imapManager, { + userId: req.session.userId, + accountIds: null, + messageId, + state, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); -// POST /api/gtd/done { id, states? } — the GTD "done" action. Two callers: the GTD sidebar passes -// an explicit `states` array (strip that section's labels); the inbox checkmark omits states -// (or sends 'all') for "done from anywhere" — strip every GTD label the thread carries. The -// id is its GTD-label-folder copy for the sidebar, or the INBOX copy from the inbox; either way -// the archive step resolves the INBOX copy from the shared Message-ID rather than acting on -// `id` directly. In one round trip this: (a) marks the whole thread read (DB fan-out across -// sibling copies, \Seen on the INBOX copy so it rides the archive move); (b) strips the -// row's own GTD label copies named in `states` — a merged Waiting row passes both -// watch+delegated — leaving any GTD labels in OTHER sections intact; (c) archives the INBOX -// copy if one exists (reusing resolveArchiveFolder + moveMessage, snooze's in-place UPDATE). -// One terminal gtd_sections_updated broadcast makes the row disappear cleanly on refetch. +// Mark a GTD message done, strip labels, and archive its INBOX copy when available. router.post('/done', async (req, res) => { const { id, states } = req.body || {}; if (!id) return res.status(400).json({ error: 'id is required' }); if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - const msg = await loadOwnedMessage(req.session.userId, id); - if (!msg) return res.status(404).json({ error: 'Message not found' }); - if (!msg.message_id) return res.status(400).json({ error: 'Message has no Message-ID — cannot mark done' }); - - const { enabled, folders } = await getGtdConfig(msg.account_id); - - // All-states mode (inbox checkmark) resolves against the label copies that actually exist - // for this thread; the GTD sidebar's explicit-states path is untouched. Only look copies up when - // we'll use them (enabled + all-states); resolveDoneFolders still owns the gtd_enabled gate. - const allStates = states == null || states === 'all'; - let existing; - if (allStates && enabled) { - const copies = await query( - 'SELECT DISTINCT folder FROM messages WHERE account_id = $1 AND message_id = $2 AND is_deleted = false', - [msg.account_id, msg.message_id] - ); - existing = copies.rows.map(r => r.folder); - } - const target = allStates - ? resolveDoneFolders({ enabled, folders, states: 'all', existing }) - : resolveDoneFolders({ enabled, folders, states }); - if (target.error) return res.status(target.status).json({ error: target.error }); - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - const account = accountResult.rows[0]; - - // (a) Mark the whole thread read. The DB fan-out (by Message-ID) covers every sibling - // copy and adjusts each folder's unread count; \Seen is set on the durable INBOX copy - // only (it rides the archive move; Gmail propagates message-wide) — the same per-copy - // asymmetry the ordinary read route accepts. A best-effort flag push is never fatal. - const inbox = await query( - 'SELECT id, uid, is_read FROM messages WHERE account_id = $1 AND folder = $2 AND message_id = $3 AND is_deleted = false LIMIT 1', - [msg.account_id, 'INBOX', msg.message_id] - ); - const inboxCopy = inbox.rows[0] || null; - try { - await fanOutReadToSiblings(msg.account_id, msg.message_id, true); - if (inboxCopy && !inboxCopy.is_read) { - await imapManager.setFlag(account, inboxCopy.uid, 'INBOX', '\\Seen', true); - } - } catch (err) { - console.warn(`GTD done: mark-read for ${id} degraded:`, err.message); - } - - // (b) Strip this row's GTD label copies. Each is a distinct folder copy resolved from - // the shared Message-ID; removeMessageCopy deletes the IMAP + DB copy and adjusts that - // label folder's counts, leaving INBOX and other-section labels untouched. - // - // Strip the acted row's OWN folder (msg.folder) LAST. A merged Waiting done passes both - // watch+delegated, and if an earlier copy's removal throws we 500 — but the same-id retry - // must still resolve the acted message via loadOwnedMessage. Deleting the acted row first - // would 404 that retry and orphan the copies not yet stripped, so keep it alive until every - // other copy is gone. (msg.folder is absent from target.folders in the inbox 'all' case — - // the acted INBOX row is never stripped anyway — so the ordering is a no-op there.) - const stripOrder = [ - ...target.folders.filter(f => f !== msg.folder), - ...target.folders.filter(f => f === msg.folder), - ]; - const removed = []; - try { - for (const folder of stripOrder) { - const uid = await resolveCopyUid(msg, folder); - if (uid == null) continue; // already gone - await imapManager.removeMessageCopy(msg.account_id, uid, folder); - removed.push(folder); - } - } catch (err) { - console.error(`GTD done: label strip for ${id} failed:`, err.message); - return res.status(500).json({ error: 'Failed to mark done' }); - } - - // (c) Archive the INBOX copy if one is present, via the shared per-copy archive primitive - // (resolveArchiveFolder + guarded moveMessage + race-safe DB repoint + count adjust, with - // the Gmail All-Mail DELETE branch). No archive folder configured is a soft outcome, not a - // failure — the GTD labels are already gone, so the thread has left all GTD sections regardless. - let archived = false; - let noArchiveFolder = false; - let archiveFailed = false; - if (inboxCopy) { - try { - const result = await archiveInboxCopy(imapManager, account, inboxCopy); - archived = result.archived; - noArchiveFolder = result.noArchiveFolder; - } catch (err) { - // Step (b) already stripped the labels (or had nothing to strip). A failed archive must - // not 500: that misreports a mostly-successful action, and — with the label row now - // gone — a retry by the same id would 404. Report a partial success (200, archiveFailed - // true, archived false) so the client can surface it and the id stays retryable. - console.error(`GTD done: archive of INBOX copy for ${id} failed:`, err.message); - archiveFailed = true; - } - } - - // One terminal refresh so GTD section data converges to the post-done state (removeMessageCopy - // also emits mid-op, but this covers the archive that follows it). - imapManager.broadcast({ type: 'gtd_sections_updated', accountId: msg.account_id }, account.user_id); - - res.json({ ok: true, removed, archived, noArchiveFolder, archiveFailed }); + const result = await gtdDone(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + states, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // POST /api/gtd/folders/ensure { accountId, folders } — create any of the account's // designated GTD label folders that are missing on the IMAP server, reporting per // folder whether it was created now or already existed. `folders` is the (possibly diff --git a/backend/src/routes/mail.js b/backend/src/routes/mail.js index 15694eab..6385bd75 100644 --- a/backend/src/routes/mail.js +++ b/backend/src/routes/mail.js @@ -7,15 +7,30 @@ import { requireAuth } from '../middleware/auth.js'; import { imapManager } from '../index.js'; import { sanitizeEmail, stripEmailHead, hasRemoteImages, blockRemoteImages, rewriteEbayImageserUrls, rewriteAnchorHrefs } from '../services/emailSanitizer.js'; import { snippetFromBody, decodeMimeWords, parseRawHeaders, buildHeadersFromMessage } from '../services/messageParser.js'; -import { resolveTrashFolder, resolveAllTrashPaths, resolveAllDraftsPaths, resolveArchiveFolder, isAllMailFolder, resolveSpamFolder, resolveAllSpamPaths, getDeleteStrategy, adjustFolderCounts, fanOutReadToSiblings, fanOutStarToSiblings, fanOutBulkReadToSiblings } from '../utils/mailUtils.js'; +import { resolveTrashFolder, resolveAllTrashPaths, resolveAllDraftsPaths, getDeleteStrategy, adjustFolderCounts } from '../utils/mailUtils.js'; import { emitGtdIfRelevant } from '../services/gtdSections.js'; import { listMessages } from '../services/messageService.js'; import { validateHost } from '../services/hostValidation.js'; import { safeFetch } from '../services/safeFetch.js'; +import { bulkSetRead, setRead, setStarred } from '../services/mailbox/flags.js'; +import { bulkMoveToFolder } from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { createFolder, deleteFolder, renameFolder } from '../services/mailbox/folders.js'; +import { setCategory } from '../services/mailbox/category.js'; +import { + gatherSnoozeConversation, + snoozeConversation, + unsnoozeConversation, +} from '../services/mailbox/snooze.js'; +import { markNotSpam, markSpam } from '../services/mailbox/spamLabel.js'; +import { UUID_RE, areValidUUIDs, isValidFolderName } from '../utils/validation.js'; const router = Router(); router.use(requireAuth); +export { gatherSnoozeConversation }; + // Sanitize an attachment filename for use in Content-Disposition. // Strips path separators and control characters; falls back to 'attachment'. function safeFilename(name) { @@ -32,17 +47,6 @@ function safeFilename(name) { return cleaned || 'attachment'; } -// Validate a folder name / path component: no control chars, max 255 chars. -function isValidFolderName(name) { - // eslint-disable-next-line no-control-regex -- intentionally rejecting control characters - return typeof name === 'string' && name.length > 0 && name.length <= 255 && !/[\x00-\x1f\x7f]/.test(name); -} - -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -function areValidUUIDs(ids) { - return ids.every(id => typeof id === 'string' && UUID_RE.test(id)); -} - // Strip NUL bytes from strings before DB writes. PostgreSQL UTF-8 text columns // reject 0x00, and malformed MIME bodies can contain embedded NUL characters. function sanitizeDbText(value) { @@ -50,19 +54,6 @@ function sanitizeDbText(value) { return value.replace(/\0/g, ''); } -// Process IMAP operations in bounded batches so a 500-message bulk action -// does not spawn hundreds of parallel temporary IMAP connections. -async function runInBatches(items, concurrency, fn) { - const results = []; - for (let i = 0; i < items.length; i += concurrency) { - const batch = items.slice(i, i + concurrency); - const batchResults = await Promise.allSettled(batch.map(fn)); - results.push(...batchResults); - } - return results; -} - - // Returns true if a snippet contains content that should never appear in plain-text // preview, indicating it was generated from unclean HTML and needs regeneration: // - &entity; — undecoded HTML entities from before the entity-stripping fix @@ -652,62 +643,14 @@ router.patch('/messages/:id/read', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); const { read } = req.body; - - const result = await query(` - SELECT m.*, a.user_id, - CASE WHEN m.message_id IS NULL THEN 1 - ELSE (SELECT COUNT(*) FROM messages s - WHERE s.account_id = m.account_id AND s.message_id = m.message_id) - END AS sibling_count - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - const message = result.rows[0]; - - // Run DB update and account fetch concurrently — no dependency between them. - // read_changed_at tells the IMAP sync not to overwrite this change for 30 s, - // preventing a race where a concurrent sync fetch sees the old IMAP flag. - const [, accountResult] = await Promise.all([ - query('UPDATE messages SET is_read = $1, read_changed_at = NOW() WHERE id = $2', [read, id]), - query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]), - ]); - - // Keep the cached folder unread_count in sync so pagination totals stay accurate. - if (!!message.is_read !== !!read) { - adjustFolderCounts(message.account_id, message.folder, 0, read ? -1 : 1); - // Notify the user's OTHER sessions so a read/unread on one device reflects on the rest - // in place, without a full folder refetch (the originating device already applied it). - imapManager.broadcast({ type: 'message_flags', accountId: message.account_id, changes: [{ id, is_read: read }] }, req.session.userId); - } - - // GTD: a labeled message owns a sibling row per folder. Fan the read change out to - // those rows (and their folder unread counts) so label views don't go stale. Gated on - // gtd_enabled (so a non-GTD account is byte-identical to pre-GTD behaviour) AND on the - // message actually having siblings — a plain single-folder message keeps the PK-only - // fast path. The IMAP \Seen flag is written to the acted folder only (below): Gmail - // propagates \Seen message-wide server-side, and per-copy writes to N folders would - // multiply round-trips — an asymmetry accepted in the GTD design. - if (accountResult.rows[0]?.gtd_enabled && Number(message.sibling_count) > 1) { - await fanOutReadToSiblings(message.account_id, message.message_id, read); - } - - try { - await imapManager.setFlag(accountResult.rows[0], message.uid, message.folder, '\\Seen', read); - imapManager._resolveFlagPush(message.account_id, id, '\\Seen'); // confirmed — drop any stale queued op - } catch (err) { - console.error('IMAP flag update failed:', err.message); - // Push failed — queue a durable retry so a later flag-sync pull can't silently revert - // the user's change once the 30s local-wins window lapses. - imapManager._enqueueFlagPush(message.account_id, id, '\\Seen', read); - } - - // Refresh GTD section data if this message's thread carries a GTD label (its head shows read state). - emitGtdSectionsRefresh([message], req.session.userId); - - res.json({ ok: true, is_read: read }); + const result = await setRead(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + read, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Star/unstar @@ -715,53 +658,14 @@ router.patch('/messages/:id/star', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); const { starred } = req.body; - - const result = await query(` - SELECT m.*, a.user_id, - CASE WHEN m.message_id IS NULL THEN 1 - ELSE (SELECT COUNT(*) FROM messages s - WHERE s.account_id = m.account_id AND s.message_id = m.message_id) - END AS sibling_count - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - const message = result.rows[0]; - - // Run DB update and account fetch concurrently — no dependency between them. - // star_changed_at tells the IMAP sync not to overwrite this change for 30 s. - const [, accountResult] = await Promise.all([ - query('UPDATE messages SET is_starred = $1, star_changed_at = NOW() WHERE id = $2', [starred, id]), - query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]), - ]); - - // GTD: fan the star change out to the message's sibling label rows (see the read - // handler). Gated on gtd_enabled to keep a non-GTD account byte-identical to pre-GTD. - // Stars don't affect folder unread counts, so no count adjustment. The IMAP \Flagged - // write below stays on the acted folder only. - if (accountResult.rows[0]?.gtd_enabled && Number(message.sibling_count) > 1) { - await fanOutStarToSiblings(message.account_id, message.message_id, starred); - } - - try { - await imapManager.setFlag(accountResult.rows[0], message.uid, message.folder, '\\Flagged', starred); - imapManager._resolveFlagPush(message.account_id, id, '\\Flagged'); // confirmed — drop any stale queued op - } catch (err) { - console.error('IMAP star update failed:', err.message); - // Push failed — queue a durable retry so a later flag-sync pull can't silently revert it. - imapManager._enqueueFlagPush(message.account_id, id, '\\Flagged', starred); - } - - // Refresh GTD section data if this message's thread carries a GTD label (its head shows star state). - emitGtdSectionsRefresh([message], req.session.userId); - // Reflect the star change on the user's other sessions in place (no full refetch). - if (!!message.is_starred !== !!starred) { - imapManager.broadcast({ type: 'message_flags', accountId: message.account_id, changes: [{ id, is_starred: starred }] }, req.session.userId); - } - - res.json({ ok: true, is_starred: starred }); + const result = await setStarred(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + starred, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Manual sync (INBOX) @@ -828,80 +732,49 @@ router.post('/folders', async (req, res) => { if (!accountId || !name?.trim()) return res.status(400).json({ error: 'accountId and name required' }); if (!isValidFolderName(name.trim())) return res.status(400).json({ error: 'Invalid folder name' }); if (parentPath && !isValidFolderName(parentPath)) return res.status(400).json({ error: 'Invalid parent path' }); - const check = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]); - if (!check.rows.length) return res.status(404).json({ error: 'Account not found' }); - - // Build path: if parentPath given, look up the delimiter used by this account's folders - let path = name.trim(); - if (parentPath) { - const delimResult = await query('SELECT delimiter FROM folders WHERE account_id = $1 LIMIT 1', [accountId]); - const delim = delimResult.rows[0]?.delimiter || '/'; - path = `${parentPath}${delim}${name.trim()}`; - } - try { - await imapManager.createFolder(check.rows[0], path); - await query( - `INSERT INTO folders (account_id, path, name) VALUES ($1, $2, $3) - ON CONFLICT (account_id, path) DO NOTHING`, - [accountId, path, name.trim()] - ); - res.json({ ok: true, path }); - } catch (err) { - console.error('Create folder error:', err); - res.status(500).json({ error: 'Failed to create folder' }); - } + const result = await createFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + accountId, + name, + parentPath, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Delete folder router.post('/folders/delete', async (req, res) => { const { accountId, path } = req.body; if (!accountId || !path) return res.status(400).json({ error: 'accountId and path required' }); if (!isValidFolderName(path)) return res.status(400).json({ error: 'Invalid folder path' }); - const check = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]); - if (!check.rows.length) return res.status(404).json({ error: 'Account not found' }); - try { - await imapManager.deleteFolder(check.rows[0], path); - } catch (err) { - console.error(`IMAP deleteFolder failed for ${path}:`, err.message); - return res.status(500).json({ error: 'Failed to delete folder on server' }); - } - await query('DELETE FROM folders WHERE account_id = $1 AND path = $2', [accountId, path]); - await query('DELETE FROM messages WHERE account_id = $1 AND folder = $2', [accountId, path]); - res.json({ ok: true }); + const result = await deleteFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + accountId, + path, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Rename folder router.post('/folders/rename', async (req, res) => { const { accountId, oldPath, newName } = req.body; if (!accountId || !oldPath || !newName?.trim()) return res.status(400).json({ error: 'Missing required fields' }); if (!isValidFolderName(newName.trim())) return res.status(400).json({ error: 'Invalid folder name' }); if (!isValidFolderName(oldPath)) return res.status(400).json({ error: 'Invalid folder path' }); - const check = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]); - if (!check.rows.length) return res.status(404).json({ error: 'Account not found' }); - - // Build the new path by replacing only the last path component - const delimResult = await query('SELECT delimiter FROM folders WHERE account_id = $1 AND path = $2', [accountId, oldPath]); - const delim = delimResult.rows[0]?.delimiter || '/'; - const parts = oldPath.split(delim); - parts[parts.length - 1] = newName.trim(); - const newPath = parts.join(delim); - try { - await imapManager.renameFolder(check.rows[0], oldPath, newPath); - await query( - 'UPDATE folders SET path = $1, name = $2, updated_at = NOW() WHERE account_id = $3 AND path = $4', - [newPath, newName.trim(), accountId, oldPath] - ); - await query('UPDATE messages SET folder = $1 WHERE account_id = $2 AND folder = $3', [newPath, accountId, oldPath]); - res.json({ ok: true, newPath }); - } catch (err) { - console.error('Rename folder error:', err); - res.status(500).json({ error: 'Failed to rename folder' }); - } + const result = await renameFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + accountId, + oldPath, + newName, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Empty folder (delete all messages) router.post('/folders/empty', async (req, res) => { const { accountId, path } = req.body; @@ -941,85 +814,14 @@ router.post('/messages/bulk-read', async (req, res) => { return res.status(400).json({ error: 'read must be a boolean' }); } - try { - const result = await query( - `SELECT m.id, m.uid, m.folder, m.is_read, m.account_id, m.message_id, a.gtd_enabled FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, updated: [] }); - - // Skip messages whose state already matches — avoid spurious DB writes and IMAP round-trips. - const toUpdate = owned.filter(m => !!m.is_read !== !!read); - if (!toUpdate.length) return res.json({ ok: true, updated: [] }); - - await query( - 'UPDATE messages SET is_read = $1, read_changed_at = NOW() WHERE id = ANY($2::uuid[])', - [read, toUpdate.map(m => m.id)] - ); - - // Adjust cached unread counts per account+folder. - const folderDeltas = {}; - for (const msg of toUpdate) { - const key = `${msg.account_id}:${msg.folder}`; - if (!folderDeltas[key]) folderDeltas[key] = { accountId: msg.account_id, folder: msg.folder, delta: 0 }; - folderDeltas[key].delta += read ? -1 : 1; - } - for (const { accountId, folder, delta } of Object.values(folderDeltas)) { - adjustFolderCounts(accountId, folder, 0, delta); - } - - // GTD: fan the read change out to sibling label rows of every updated message that - // belongs to a gtd_enabled account, adjusting each sibling folder's unread count. - // Gating on gtd_enabled keeps a non-GTD account byte-identical to pre-GTD (no extra - // fan-out query); the fan-out itself is also self-limiting for messages without - // siblings. IMAP \Seen is still written per acted row only (below); Gmail propagates - // it message-wide server-side. - // gtdUpdatedIds is scoped to toUpdate (rows whose read-state actually changed), so a - // message already at the target state never triggers sibling fan-out here — unlike the - // single-message handler above, which fans out unconditionally regardless of whether the - // acted message's own state changed. That asymmetry is acceptable: nothing else in this - // path can push a sibling out of sync with its head, and the label-folder tick already - // self-heals any divergence on the next read. - const gtdUpdatedIds = toUpdate.filter(m => m.gtd_enabled).map(m => m.id); - if (gtdUpdatedIds.length) await fanOutBulkReadToSiblings(gtdUpdatedIds, read); - // Reflect the bulk read/unread change on the user's other sessions in place (no full refetch). - imapManager.broadcast({ type: 'message_flags', changes: toUpdate.map(m => ({ id: m.id, is_read: read })) }, req.session.userId); - - // IMAP flag updates — group by account to fetch each account row once. - const byAccount = {}; - for (const msg of toUpdate) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - for (const [accountId, msgs] of Object.entries(byAccount)) { - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - const results = await runInBatches( - msgs, 3, - msg => imapManager.setFlag(account, msg.uid, msg.folder, '\\Seen', read) - ); - results.forEach((r, i) => { - if (r.status === 'rejected') { - console.error(`bulk-read IMAP ${msgs[i].id}:`, r.reason.message); - // Durable retry so a later flag-sync pull can't revert this message to unread. - imapManager._enqueueFlagPush(accountId, msgs[i].id, '\\Seen', read); - } else { - imapManager._resolveFlagPush(accountId, msgs[i].id, '\\Seen'); // confirmed - } - }); - } - - // Refresh GTD section data for any updated thread that carries a GTD label. - emitGtdSectionsRefresh(toUpdate, req.session.userId); - - res.json({ ok: true, updated: toUpdate.map(m => m.id) }); - } catch (err) { - console.error('bulk-read error:', err); - res.status(500).json({ error: 'Failed to update messages' }); - } + const result = await bulkSetRead(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + read, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Bulk delete (move to trash) @@ -1035,202 +837,15 @@ router.post('/messages/bulk-delete', async (req, res) => { return res.status(400).json({ error: 'Invalid message id format' }); } - const moveGuards = []; - try { - const result = await query( - `SELECT m.*, a.user_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, deleted: [] }); - - // Guard source UIDs for the whole operation so reconcileDeletes can't delete a - // trash-move source row between the IMAP move and the re-INSERT CTE (message vanishing - // from both folders). Harmless for the expunge path (those rows are deleted anyway). - // Released in the finally below. - for (const m of owned) { - moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); - imapManager._guardMoveUid(m.account_id, m.folder, m.uid); - } - - const byAccount = {}; - for (const msg of owned) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - - // expungeSucceeded: permanently deleted (already in Trash, or no Trash folder on account). - // trashMoveSucceeded: moved from a non-Trash folder into Trash. - const expungeSucceeded = []; - const trashMoveSucceeded = []; // { msg, trashPath, newUid } - const accountsById = {}; - - for (const [accountId, msgs] of Object.entries(byAccount)) { - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - accountsById[accountId] = account; - const trashPath = await resolveTrashFolder(accountId, msgs[0].folder_mappings); - const allTrashPaths = await resolveAllTrashPaths(accountId, msgs[0].folder_mappings); - const allDraftsPaths = await resolveAllDraftsPaths(accountId, msgs[0].folder_mappings); - - if (!trashPath) { - console.error(`bulk-delete: no Trash folder found for account ${accountId} — skipping ${msgs.length} messages`); - continue; - } - - // Drafts and messages already in Trash are permanently deleted; others move to Trash. - const toExpunge = msgs.filter(m => allTrashPaths.has(m.folder) || allDraftsPaths.has(m.folder)); - const toMove = msgs.filter(m => !allTrashPaths.has(m.folder) && !allDraftsPaths.has(m.folder)); - - // Permanently delete messages already in a trash-like folder (grouped by actual folder). - if (toExpunge.length) { - const byExpungeFolder = {}; - for (const msg of toExpunge) { - (byExpungeFolder[msg.folder] = byExpungeFolder[msg.folder] || []).push(msg); - } - for (const [expungeFolder, folderMsgs] of Object.entries(byExpungeFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { succeeded, failed } = await imapManager.bulkPermanentDelete(account, folderMsgs.map(m => m.uid), expungeFolder); - for (const uid of succeeded) expungeSucceeded.push(uidToMsg.get(String(uid))); - for (const uid of failed) console.error(`bulk-delete IMAP expunge uid ${uid} from ${expungeFolder}: IMAP delete failed`); - } - } - - // Move messages from non-Trash folders into Trash. - if (toMove.length) { - const byFolder = {}; - for (const msg of toMove) { - (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); - } - for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, trashPath); - for (const uid of succeeded) { - trashMoveSucceeded.push({ msg: uidToMsg.get(String(uid)), trashPath, newUid: uidMap.get(Number(uid)) || null }); - } - for (const uid of failed) console.error(`bulk-delete IMAP move uid ${uid}: IMAP move failed`); - } - } - } - - // Permanently deleted: remove DB rows immediately. - if (expungeSucceeded.length) { - await query('DELETE FROM messages WHERE id = ANY($1::uuid[])', [expungeSucceeded.map(m => m.id)]); - } - - // Trash moves: same CTE approach as bulk-move — DELETE source rows and - // immediately re-INSERT at the destination when new UIDs are known. - // Group by trashPath since different accounts may have different Trash folders. - if (trashMoveSucceeded.length) { - const byTrashPath = {}; - for (const u of trashMoveSucceeded) { - (byTrashPath[u.trashPath] = byTrashPath[u.trashPath] || []).push(u); - } - for (const [trashPath, entries] of Object.entries(byTrashPath)) { - const allIds = entries.map(u => u.msg.id); - const withUid = entries.filter(u => u.newUid); - await query(` - WITH deleted AS ( - DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * - ), - uid_map(src_id, new_uid) AS ( - SELECT * FROM unnest($2::uuid[], $3::bigint[]) - ) - INSERT INTO messages ( - account_id, uid, folder, message_id, subject, - from_name, from_email, to_addresses, cc_addresses, - reply_to, in_reply_to, date, snippet, is_read, is_starred, - has_attachments, flags, body_html, body_text, attachments, - thread_references, thread_id, is_bulk, - read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, - spam_verdict, spam_analyzed_at, spam_details, spam_user_override, - category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at - ) - SELECT - d.account_id, u.new_uid, $4, d.message_id, d.subject, - d.from_name, d.from_email, d.to_addresses, d.cc_addresses, - d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, - d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, - d.thread_references, d.thread_id, d.is_bulk, - d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, - d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, - d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at - FROM deleted d - JOIN uid_map u ON d.id = u.src_id - ON CONFLICT (account_id, uid, folder) DO NOTHING - `, [allIds, withUid.map(u => u.msg.id), withUid.map(u => u.newUid), trashPath]); - } - // Non-UIDPLUS trash moves were deleted with no reinsert; pull each affected - // (account, trash folder) now so they reappear promptly instead of via IDLE. - const needResync = new Map(); // accountId -> Set - for (const u of trashMoveSucceeded) { - if (u.newUid) continue; - if (!needResync.has(u.msg.account_id)) needResync.set(u.msg.account_id, new Set()); - needResync.get(u.msg.account_id).add(u.trashPath); - } - for (const [acctId, paths] of needResync) { - const acct = accountsById[acctId]; - if (!acct) continue; - for (const tp of paths) { - imapManager.syncFolderOnDemand(acct, tp) - .catch(err => console.warn('post-trash destination sync failed:', err.message)); - } - } - } - - // Adjust cached folder counts. - // Source folders always lose the message; Trash gains only for non-Trash moves. - const allSucceeded = [ - ...expungeSucceeded.map(m => m.id), - ...trashMoveSucceeded.map(u => u.msg.id), - ]; - if (allSucceeded.length) { - const srcDeltas = {}; - for (const msg of expungeSucceeded) { - const key = `${msg.account_id}:${msg.folder}`; - if (!srcDeltas[key]) srcDeltas[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; - srcDeltas[key].total++; - if (!msg.is_read) srcDeltas[key].unread++; - } - for (const { msg } of trashMoveSucceeded) { - const key = `${msg.account_id}:${msg.folder}`; - if (!srcDeltas[key]) srcDeltas[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; - srcDeltas[key].total++; - if (!msg.is_read) srcDeltas[key].unread++; - } - for (const { accountId, path, total, unread } of Object.values(srcDeltas)) { - adjustFolderCounts(accountId, path, -total, -unread); - } - const dstDeltas = {}; - for (const { msg, trashPath } of trashMoveSucceeded) { - const key = `${msg.account_id}:${trashPath}`; - if (!dstDeltas[key]) dstDeltas[key] = { accountId: msg.account_id, path: trashPath, total: 0, unread: 0 }; - dstDeltas[key].total++; - if (!msg.is_read) dstDeltas[key].unread++; - } - for (const { accountId, path, total, unread } of Object.values(dstDeltas)) { - adjustFolderCounts(accountId, path, total, unread); - } - // Notify clients viewing each Trash folder to refresh silently. - for (const { accountId, path } of Object.values(dstDeltas)) { - imapManager.broadcast({ type: 'folder_updated', folder: path, accountId }, req.session.userId); - } - } - - // Refresh GTD section data for any deleted thread that still carries a GTD label sibling. - emitGtdSectionsRefresh(owned, req.session.userId); - - res.json({ ok: true, deleted: allSucceeded }); - } catch (err) { - console.error('bulk-delete error:', err); - res.status(500).json({ error: 'Failed to delete messages' }); - } finally { - for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); - } + const result = await bulkTrash(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + allowPermanent: true, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Bulk move to folder router.post('/messages/bulk-move', async (req, res) => { const { ids, folder } = req.body; @@ -1247,149 +862,17 @@ router.post('/messages/bulk-move', async (req, res) => { return res.status(400).json({ error: 'Invalid message id format' }); } - const moveGuards = []; - try { - const result = await query( - `SELECT m.*, a.user_id FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, moved: [] }); - - // Guard every source (account, folder, uid) for the whole bulk move. bulkMoveMessages - // removes the UIDs from the server (seconds of wall-clock), and a concurrent - // reconcileDeletes tick would otherwise see the source rows as orphans and delete them - // before the DELETE...RETURNING CTE re-inserts them at the destination — dropping the - // message from BOTH folders. Unguarded in the finally once the CTE has committed. - // Mirrors the single-message move paths. - for (const m of owned) { - moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); - imapManager._guardMoveUid(m.account_id, m.folder, m.uid); - } - - const byAccount = {}; - for (const msg of owned) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - - const movedIds = []; - const uidUpdates = []; - const resyncAccounts = []; // accounts whose moved msgs lacked new UIDs (non-UIDPLUS) - for (const [accountId, msgs] of Object.entries(byAccount)) { - // Verify the destination folder exists for this account - const folderCheck = await query( - 'SELECT 1 FROM folders WHERE account_id = $1 AND path = $2', - [accountId, folder] - ); - if (!folderCheck.rows.length) { - console.warn(`bulk-move: folder "${folder}" not found for account ${accountId}, skipping`); - continue; - } - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - const byFolder = {}; - for (const msg of msgs) { - (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); - } - let accountMissingUid = false; - for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, folder); - for (const uid of succeeded) { - const msg = uidToMsg.get(String(uid)); - movedIds.push(msg.id); - const newUid = uidMap.get(Number(uid)) || null; - if (newUid) uidUpdates.push({ id: msg.id, newUid }); - else accountMissingUid = true; - } - for (const uid of failed) console.error(`bulk-move IMAP uid ${uid}: IMAP move failed`); - } - if (accountMissingUid) resyncAccounts.push(account); - } - - if (movedIds.length > 0) { - // DELETE source rows and, when we have UIDPLUS-provided new UIDs, immediately - // re-INSERT at the destination in one atomic CTE statement. This avoids any - // transient folder/uid state that could collide with existing rows (UIDs are - // per-folder, so the same UID number is valid in two different folders). - // If IMAP IDLE already inserted the destination row, ON CONFLICT DO NOTHING - // keeps it intact. For messages without new UIDs the DELETE-only path relies - // on IMAP IDLE + the message_id pre-check in processMsg to re-insert them. - const uidUpdateMap = new Map(uidUpdates.map(u => [u.id, u.newUid])); - const withNewUid = movedIds.filter(id => uidUpdateMap.has(id)); - await query(` - WITH deleted AS ( - DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * - ), - uid_map(src_id, new_uid) AS ( - SELECT * FROM unnest($2::uuid[], $3::bigint[]) - ) - INSERT INTO messages ( - account_id, uid, folder, message_id, subject, - from_name, from_email, to_addresses, cc_addresses, - reply_to, in_reply_to, date, snippet, is_read, is_starred, - has_attachments, flags, body_html, body_text, attachments, - thread_references, thread_id, is_bulk, - read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, - spam_verdict, spam_analyzed_at, spam_details, spam_user_override, - category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at - ) - SELECT - d.account_id, u.new_uid, $4, d.message_id, d.subject, - d.from_name, d.from_email, d.to_addresses, d.cc_addresses, - d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, - d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, - d.thread_references, d.thread_id, d.is_bulk, - d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, - d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, - d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at - FROM deleted d - JOIN uid_map u ON d.id = u.src_id - ON CONFLICT (account_id, uid, folder) DO NOTHING - `, [movedIds, withNewUid, withNewUid.map(id => uidUpdateMap.get(id)), folder]); - // Messages moved on a non-UIDPLUS server were deleted with no reinsert; pull the - // destination folder now so they reappear promptly instead of waiting for IDLE. - for (const acct of resyncAccounts) { - imapManager.syncFolderOnDemand(acct, folder) - .catch(err => console.warn('post-move destination sync failed:', err.message)); - } - // Adjust cached counts: decrement source folders, increment the destination. - const movedSet = new Set(movedIds); - const srcTotals = {}; - for (const msg of owned) { - if (!movedSet.has(msg.id)) continue; - const key = `${msg.account_id}:${msg.folder}`; - if (!srcTotals[key]) srcTotals[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; - srcTotals[key].total++; - if (!msg.is_read) srcTotals[key].unread++; - } - for (const { accountId, path, total, unread } of Object.values(srcTotals)) { - adjustFolderCounts(accountId, path, -total, -unread); - adjustFolderCounts(accountId, folder, total, unread); - } - - // Notify clients that the destination folder has new content so they - // refresh without sounds or alerts (unlike new_messages). - for (const accountId of Object.keys(srcTotals).map(k => k.split(':')[0])) { - imapManager.broadcast({ type: 'folder_updated', folder, accountId }, req.session.userId); - } - } - - // Refresh GTD section data for any moved thread that still carries a GTD label sibling. - emitGtdSectionsRefresh(owned, req.session.userId); - - res.json({ ok: true, moved: movedIds }); - } catch (err) { - console.error('bulk-move error:', err); - res.status(500).json({ error: 'Failed to move messages' }); - } finally { - for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); - } + const result = await bulkMoveToFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + folder, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + // MCP consumes the richer receipt (movedDetails/skippedAccounts/failed) via the + // service return value directly; REST keeps its original {ok, moved} wire shape. + res.json({ ok: result.ok, moved: result.moved }); }); - // Bulk archive — moves messages to the archive folder for each account router.post('/messages/bulk-archive', async (req, res) => { const { ids } = req.body; @@ -1403,261 +886,17 @@ router.post('/messages/bulk-archive', async (req, res) => { return res.status(400).json({ error: 'Invalid message IDs' }); } - const moveGuards = []; - try { - const result = await query( - `SELECT m.*, a.user_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, archived: [], noArchiveFolder: [] }); - - // Guard source UIDs for the whole operation so reconcileDeletes can't delete a source - // row between the IMAP move and the re-INSERT CTE (message vanishing from both folders). - // Released in the finally below. - for (const m of owned) { - moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); - imapManager._guardMoveUid(m.account_id, m.folder, m.uid); - } - - const byAccount = {}; - for (const msg of owned) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - - const archivedIds = []; - const noArchiveFolder = []; - const accountsById = {}; - // Archive-folder paths that resolved to Gmail's All Mail (special_use '\All'). - // All Mail is excluded from sync/backfill and the relocate guard (imapManager.js), - // so messages archived there get their DB row deleted below instead of re-homed. - const allMailDestFolders = new Set(); - - for (const [accountId, msgs] of Object.entries(byAccount)) { - const archiveFolder = await resolveArchiveFolder(accountId, msgs[0].folder_mappings); - if (!archiveFolder) { - noArchiveFolder.push(accountId); - continue; - } - if (await isAllMailFolder(accountId, archiveFolder)) { - allMailDestFolders.add(archiveFolder); - } - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - accountsById[accountId] = account; - const byFolder = {}; - for (const msg of msgs) { - (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); - } - for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, archiveFolder); - for (const uid of succeeded) { - const msg = uidToMsg.get(String(uid)); - archivedIds.push({ id: msg.id, accountId, folder: archiveFolder, newUid: uidMap.get(Number(uid)) || null }); - } - for (const uid of failed) console.error(`bulk-archive IMAP uid ${uid}: IMAP move failed`); - } - } - - // Update DB: same CTE DELETE+INSERT pattern as bulk-move — except when the - // destination is Gmail's All Mail, where the message just vanishes from our view - // (see allMailDestFolders above), so a plain DELETE with no reinsert is correct. - const byFolder = {}; - for (const { id, folder, newUid } of archivedIds) { - (byFolder[folder] = byFolder[folder] || []).push({ id, newUid }); - } - for (const [archiveFolder, entries] of Object.entries(byFolder)) { - const allIds = entries.map(e => e.id); - if (allMailDestFolders.has(archiveFolder)) { - await query('DELETE FROM messages WHERE id = ANY($1::uuid[])', [allIds]); - continue; - } - const withUid = entries.filter(e => e.newUid != null); - await query(` - WITH deleted AS ( - DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * - ), - uid_map(src_id, new_uid) AS ( - SELECT * FROM unnest($2::uuid[], $3::bigint[]) - ) - INSERT INTO messages ( - account_id, uid, folder, message_id, subject, - from_name, from_email, to_addresses, cc_addresses, - reply_to, in_reply_to, date, snippet, is_read, is_starred, - has_attachments, flags, body_html, body_text, attachments, - thread_references, thread_id, is_bulk, - read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, - spam_verdict, spam_analyzed_at, spam_details, spam_user_override, - category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at - ) - SELECT - d.account_id, u.new_uid, $4, d.message_id, d.subject, - d.from_name, d.from_email, d.to_addresses, d.cc_addresses, - d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, - d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, - d.thread_references, d.thread_id, d.is_bulk, - d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, - d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, - d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at - FROM deleted d - JOIN uid_map u ON d.id = u.src_id - ON CONFLICT (account_id, uid, folder) DO NOTHING - `, [allIds, withUid.map(e => e.id), withUid.map(e => e.newUid), archiveFolder]); - } - - // Non-UIDPLUS archive moves were deleted with no reinsert; pull each affected - // (account, archive folder) now so they reappear promptly instead of via IDLE. - const needResync = new Map(); // accountId -> Set - for (const e of archivedIds) { - if (e.newUid) continue; - if (allMailDestFolders.has(e.folder)) continue; // no DB row there to keep fresh - if (!needResync.has(e.accountId)) needResync.set(e.accountId, new Set()); - needResync.get(e.accountId).add(e.folder); - } - for (const [acctId, paths] of needResync) { - const acct = accountsById[acctId]; - if (!acct) continue; - for (const fp of paths) { - imapManager.syncFolderOnDemand(acct, fp) - .catch(err => console.warn('post-archive destination sync failed:', err.message)); - } - } - - // Adjust cached folder counts: use signed deltas so source and dest share one pass. - if (archivedIds.length > 0) { - const idToArchiveDest = new Map(archivedIds.map(({ id, folder: dest }) => [id, dest])); - const folderDeltas = {}; // key: `${accountId}:${path}` -> { accountId, path, totalDelta, unreadDelta } - for (const msg of owned) { - const dest = idToArchiveDest.get(msg.id); - if (!dest) continue; - const wasUnread = !msg.is_read ? 1 : 0; - const srcKey = `${msg.account_id}:${msg.folder}`; - if (!folderDeltas[srcKey]) folderDeltas[srcKey] = { accountId: msg.account_id, path: msg.folder, totalDelta: 0, unreadDelta: 0 }; - folderDeltas[srcKey].totalDelta--; - folderDeltas[srcKey].unreadDelta -= wasUnread; - if (allMailDestFolders.has(dest)) continue; // All Mail counts aren't tracked - const dstKey = `${msg.account_id}:${dest}`; - if (!folderDeltas[dstKey]) folderDeltas[dstKey] = { accountId: msg.account_id, path: dest, totalDelta: 0, unreadDelta: 0 }; - folderDeltas[dstKey].totalDelta++; - folderDeltas[dstKey].unreadDelta += wasUnread; - } - for (const { accountId, path, totalDelta, unreadDelta } of Object.values(folderDeltas)) { - adjustFolderCounts(accountId, path, totalDelta, unreadDelta); - } - // Notify clients viewing each destination folder to refresh silently. - const destFolders = [...new Set(archivedIds.map(a => a.folder))].filter(f => !allMailDestFolders.has(f)); - for (const dest of destFolders) { - const accountIds = [...new Set(archivedIds.filter(a => a.folder === dest).map(a => { - const msg = owned.find(m => m.id === a.id); - return msg?.account_id; - }).filter(Boolean))]; - for (const accountId of accountIds) { - imapManager.broadcast({ type: 'folder_updated', folder: dest, accountId }, req.session.userId); - } - } - } - - // Refresh GTD section data for any archived thread that still carries a GTD label sibling. - emitGtdSectionsRefresh(owned, req.session.userId); - - res.json({ ok: true, archived: archivedIds.map(a => a.id), noArchiveFolder }); - } catch (err) { - console.error('bulk-archive error:', err); - res.status(500).json({ error: 'Failed to archive messages' }); - } finally { - for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); - } + const result = await bulkArchive(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + // MCP consumes the richer receipt (archivedDetails/failed) via the service return + // value directly; REST keeps its original {ok, archived, noArchiveFolder} wire shape. + res.json({ ok: result.ok, archived: result.archived, noArchiveFolder: result.noArchiveFolder }); }); -// Gather the reply-chain conversation that should be snoozed alongside `msg`. -// -// Snoozing a single message doesn't work on Gmail: Gmail groups the inbox by -// conversation, so moving one message to Snoozed only strips \Inbox from that -// message — its thread siblings keep \Inbox and the whole conversation stays in -// the inbox (#271). MailFlow's own inbox is thread-grouped too. So we snooze the -// entire conversation, but bounded to the RFC 5322 reply chain (Message-ID / -// In-Reply-To / References links) rather than thread_id: thread_id falls back to -// subject grouping and can lump hundreds of unrelated messages together (e.g. -// identical automated-notification emails), which must never be swept into Snoozed. -// -// Returns the messages in `msg`'s source folder reachable from `msg` through -// header links (always including `msg` itself); excludes already-snoozed messages. -export async function gatherSnoozeConversation(msg) { - if (!msg.thread_id) return [msg]; - - // Load the whole thread across ALL folders. thread_id is a superset of the true - // conversation, and the messages that hold a real conversation together — the - // other party's replies, your own Sent messages, the thread root — frequently - // live in Sent / All Mail rather than the inbox. They must be present as graph - // connectors or a genuine thread fragments and only part of it snoozes. The - // reply-chain walk below filters out the subject-only collisions that thread_id - // also collects (e.g. identical automated-notification emails). - const pool = (await query( - `SELECT id, uid, account_id, folder, message_id, in_reply_to, thread_references, is_read - FROM messages - WHERE account_id = $1 AND thread_id = $2 AND message_id IS NOT NULL`, - [msg.account_id, msg.thread_id] - )).rows; - - // Ensure the triggering message is present (the query above could miss it on a - // transient read skew). - if (!pool.some(r => r.message_id === msg.message_id)) pool.push(msg); - - const refsOf = (r) => { - const ids = (r.thread_references || '').match(/<[^>]+>/g) || []; - if (r.in_reply_to) ids.push(r.in_reply_to); - return ids; - }; - - // Undirected reply-chain graph over the whole thread; take the connected - // component containing `msg`. Messages with no header link into that component - // (subject-only collisions) are left out. - const adj = new Map(); - const node = (m) => { let s = adj.get(m); if (!s) { s = new Set(); adj.set(m, s); } return s; }; - for (const r of pool) node(r.message_id); - for (const r of pool) { - for (const ref of refsOf(r)) { - if (adj.has(ref)) { node(r.message_id).add(ref); node(ref).add(r.message_id); } - } - } - const seen = new Set([msg.message_id]); - const queue = [msg.message_id]; - while (queue.length) { - const cur = queue.shift(); - for (const nb of (adj.get(cur) || [])) if (!seen.has(nb)) { seen.add(nb); queue.push(nb); } - } - - // Snooze only the conversation members in the acted-on message's source folder - // (the inbox copies — Sent copies carry no \Inbox and shouldn't move), skipping - // any already snoozed. Already-snoozed messages stay valid graph connectors above. - const already = new Set( - (await query( - 'SELECT message_id_header FROM snoozed_messages WHERE account_id = $1 AND message_id_header = ANY($2)', - [msg.account_id, [...seen]] - )).rows.map(r => r.message_id_header) - ); - // Dedupe by Message-ID so a message that somehow has two rows in the source - // folder isn't moved (and recorded) twice. - const picked = new Map(); - for (const r of pool) { - if (seen.has(r.message_id) && r.folder === msg.folder && !already.has(r.message_id) && !picked.has(r.message_id)) { - picked.set(r.message_id, r); - } - } - // Return the acted-on message first so the caller can treat a failure moving it - // as fatal before any sibling has been touched (no partial snooze on error). - const rest = [...picked.values()].filter(r => r.message_id !== msg.message_id); - // msg always qualifies (it's the acted-on, not-yet-snoozed message in its own - // folder); fall back to it directly if the pool row for it was missed. - const self = picked.get(msg.message_id) || msg; - return [self, ...rest]; -} - // Snooze a message: move it to a Snoozed IMAP folder and record when to restore it router.post('/messages/:id/snooze', async (req, res) => { const { id } = req.params; @@ -1673,84 +912,27 @@ router.post('/messages/:id/snooze', async (req, res) => { maxDate.setDate(maxDate.getDate() + 30); if (untilDate > maxDate) return res.status(400).json({ error: 'until must be within 30 days' }); - // Ownership check - const msgResult = await query( - `SELECT m.*, a.user_id FROM messages m - JOIN email_accounts a ON a.id = m.account_id - WHERE m.id = $1 AND a.user_id = $2`, - [id, req.session.userId] - ); - if (!msgResult.rows.length) return res.status(404).json({ error: 'Message not found' }); - const msg = msgResult.rows[0]; - - if (!msg.message_id) return res.status(400).json({ error: 'Message has no Message-ID header — cannot snooze' }); - - const snoozedFolder = 'Snoozed'; - - if (msg.folder === snoozedFolder) { - return res.status(400).json({ error: 'Message is already in Snoozed folder' }); - } - - // Check if already snoozed - const existing = await query( - 'SELECT id FROM snoozed_messages WHERE account_id = $1 AND message_id_header = $2', - [msg.account_id, msg.message_id] - ); - if (existing.rows.length) return res.status(400).json({ error: 'Message is already snoozed' }); - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - const account = accountResult.rows[0]; - - // Snooze the whole reply-chain conversation, not just this message (see - // gatherSnoozeConversation for why Gmail requires this and why it's bounded - // to the header reply chain rather than thread_id). - const convo = await gatherSnoozeConversation(msg); - - try { - await imapManager.ensureFolder(account, snoozedFolder); - } catch (err) { - console.error(`Snooze ensureFolder failed for message ${id}:`, err.message); - return res.status(500).json({ error: 'Failed to move message to Snoozed folder' }); - } - - for (const tm of convo) { - imapManager._guardMoveUid(tm.account_id, tm.folder, tm.uid); - try { - let snoozedUid; - try { - snoozedUid = await imapManager.moveMessage(account, tm.uid, tm.folder, snoozedFolder); - } catch (err) { - console.error(`Snooze IMAP move failed for message ${tm.id}:`, err.message); - // The message the user acted on must succeed; a failed sibling is logged - // and skipped so the rest of the conversation still snoozes. - if (tm.id === msg.id) return res.status(500).json({ error: 'Failed to move message to Snoozed folder' }); - continue; - } - if (snoozedUid != null) { - await query('UPDATE messages SET folder = $1, uid = $2 WHERE id = $3', [snoozedFolder, snoozedUid, tm.id]); - } else { - imapManager._guardMoveUid(tm.account_id, snoozedFolder, tm.uid); - await query('UPDATE messages SET folder = $1 WHERE id = $2', [snoozedFolder, tm.id]); - setTimeout(() => imapManager._unguardMoveUid(tm.account_id, snoozedFolder, tm.uid), 10_000); - } - - await query( - `INSERT INTO snoozed_messages (user_id, account_id, message_id_header, original_folder, snooze_until, snoozed_folder) - VALUES ($1, $2, $3, $4, $5, $6)`, - [req.session.userId, tm.account_id, tm.message_id, tm.folder, untilDate.toISOString(), snoozedFolder] - ); - - adjustFolderCounts(tm.account_id, tm.folder, -1, tm.is_read ? 0 : -1); - adjustFolderCounts(tm.account_id, snoozedFolder, 1, tm.is_read ? 0 : 1); - } finally { - imapManager._unguardMoveUid(tm.account_id, tm.folder, tm.uid); - } - } + const result = await snoozeConversation(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + until: untilDate, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); +}); - // Refresh GTD section data if the snoozed conversation carries a GTD label (its in_inbox flips). - emitGtdSectionsRefresh(convo, req.session.userId); +router.delete('/messages/:id/snooze', async (req, res) => { + const { id } = req.params; + if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - res.json({ ok: true }); + const result = await unsnoozeConversation(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Delete (move to trash; drafts are permanently deleted) @@ -1851,108 +1033,6 @@ router.delete('/messages/:id', async (req, res) => { // // No automatic classification runs here — that ships in v0.2 (ML) and v0.3 (SA). -// Helper: move a single message to a destination folder, update DB, log to -// training_log, and broadcast folder_updated. Shared between /spam and /ham. -async function moveForSpamLabel(messageId, userId, destinationFolder, label) { - const result = await query(` - SELECT m.*, a.user_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [messageId, userId]); - - if (!result.rows.length) return { ok: false, status: 404, error: 'Message not found' }; - const message = result.rows[0]; - - // No-op: message already in the destination folder. - if (message.folder === destinationFolder) { - // Still record the training label so the user's intent is captured - // (e.g. re-confirming a verdict), but skip the IMAP move. - await query( - `INSERT INTO spam_training_log - (user_id, account_id, message_id_header, message_uid, folder, label) - VALUES ($1, $2, $3, $4, $5, $6)`, - [userId, message.account_id, message.message_id, message.uid, message.folder, label] - ); - await query( - `UPDATE messages SET spam_user_override = $1, spam_verdict = $1, spam_analyzed_at = NOW() WHERE id = $2`, - [label, messageId] - ); - return { ok: true, status: 200, body: { ok: true, alreadyInFolder: true, folder: destinationFolder } }; - } - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]); - const account = accountResult.rows[0]; - - // Guard the source UID before the IMAP move so reconcileDeletes cannot - // delete the DB row if an EXPUNGE arrives while the move is in flight. - imapManager._guardMoveUid(account.id, message.folder, message.uid); - let newUid; - try { - try { - newUid = await imapManager.moveMessage(account, message.uid, message.folder, destinationFolder); - } catch (err) { - console.error(`IMAP move for /${label} failed:`, err.message); - return { ok: false, status: 502, error: `IMAP move failed: ${err.message}` }; - } - if (newUid != null) { - await query('DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3 AND id != $4', - [account.id, newUid, destinationFolder, messageId]); - await query( - `UPDATE messages SET folder = $1, uid = $2, - spam_user_override = $3, spam_verdict = $3, spam_analyzed_at = NOW() - WHERE id = $4`, - [destinationFolder, newUid, label, messageId] - ); - } else { - // Non-UIDPLUS server: DB holds the stale source UID at the destination. - imapManager._guardMoveUid(account.id, destinationFolder, message.uid); - await query( - `UPDATE messages SET folder = $1, - spam_user_override = $2, spam_verdict = $2, spam_analyzed_at = NOW() - WHERE id = $3`, - [destinationFolder, label, messageId] - ); - setTimeout(() => imapManager._unguardMoveUid(account.id, destinationFolder, message.uid), 10_000); - } - } finally { - imapManager._unguardMoveUid(account.id, message.folder, message.uid); - } - - // Adjust cached folder counts. - const wasUnread = !message.is_read ? 1 : 0; - adjustFolderCounts(account.id, message.folder, -1, -wasUnread); - adjustFolderCounts(account.id, destinationFolder, 1, wasUnread); - - // Training log: capture the decision for future model training. - await query( - `INSERT INTO spam_training_log - (user_id, account_id, message_id_header, message_uid, folder, label, source) - VALUES ($1, $2, $3, $4, $5, $6, 'manual')`, - [userId, account.id, message.message_id, message.uid, destinationFolder, label] - ); - - // If folder_mappings.spam is not yet configured, learn from the discovered folder. - if (label === 'spam' && !account.folder_mappings?.spam) { - await query( - `UPDATE email_accounts SET folder_mappings = folder_mappings || jsonb_build_object('spam', $1::text) - WHERE id = $2 AND NOT (folder_mappings ? 'spam')`, - [destinationFolder, account.id] - ).catch(err => console.warn('Failed to auto-persist folder_mappings.spam:', err.message)); - } - - imapManager.broadcast( - { type: 'folder_updated', folder: destinationFolder, accountId: account.id }, - userId - ); - - // Refresh GTD section data if the (un)spammed message's thread carries a GTD label. Covers both - // /spam and /ham, which share this mover. The already-in-folder no-op path above returns - // early without a move, so GTD section data is untouched there. - emitGtdSectionsRefresh([message], userId); - - return { ok: true, status: 200, body: { ok: true, folder: destinationFolder, newUid: newUid || null } }; -} - // POST /api/mail/messages/:id/spam // Moves the message to the account's spam/junk folder and records the user // override as spam. Coexists with the future ML/SA auto-classification: @@ -1961,17 +1041,11 @@ router.post('/messages/:id/spam', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - const lookup = await query(` - SELECT m.account_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!lookup.rows.length) return res.status(404).json({ error: 'Message not found' }); - const spamFolder = await resolveSpamFolder(lookup.rows[0].account_id, lookup.rows[0].folder_mappings); - if (!spamFolder) return res.status(422).json({ error: 'No spam folder configured for this account' }); - - const result = await moveForSpamLabel(id, req.session.userId, spamFolder, 'spam'); + const result = await markSpam(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + }); if (!result.ok) return res.status(result.status).json({ error: result.error }); res.json(result.body); }); @@ -2027,17 +1101,14 @@ router.patch('/messages/:id/category', async (req, res) => { return res.status(400).json({ error: 'Invalid category' }); } - const result = await query( - `UPDATE messages SET category = $1 - FROM email_accounts a - WHERE messages.id = $2 - AND messages.account_id = a.id - AND a.user_id = $3 - RETURNING messages.id`, - [category === 'primary' ? null : category, id, req.session.userId] - ); - if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - res.json({ ok: true, category }); + const result = await setCategory(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + category, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // POST /api/mail/messages/:id/unsubscribe @@ -2119,23 +1190,11 @@ router.post('/messages/:id/ham', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - const lookup = await query(` - SELECT m.account_id, m.folder, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!lookup.rows.length) return res.status(404).json({ error: 'Message not found' }); - const allSpam = await resolveAllSpamPaths(lookup.rows[0].account_id, lookup.rows[0].folder_mappings); - if (!allSpam.has(lookup.rows[0].folder)) { - return res.status(400).json({ error: 'Message is not in the spam folder' }); - } - - // Resolve inbox folder per account — Gmail, Exchange and others may not use - // the literal 'INBOX' (e.g. 'Inbox' on Dovecot, 'Posteingang', etc.). - // Same pattern as folder_mappings.sent / .drafts in send.js and draft.js. - const inboxFolder = lookup.rows[0].folder_mappings?.inbox || 'INBOX'; - const result = await moveForSpamLabel(id, req.session.userId, inboxFolder, 'ham'); + const result = await markNotSpam(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + }); if (!result.ok) return res.status(result.status).json({ error: result.error }); res.json(result.body); }); diff --git a/backend/src/routes/mail.test.js b/backend/src/routes/mail.test.js new file mode 100644 index 00000000..4b44a899 --- /dev/null +++ b/backend/src/routes/mail.test.js @@ -0,0 +1,311 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'u1' }; + next(); + }, +})); +vi.mock('../index.js', () => ({ + imapManager: { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + bulkMoveMessages: vi.fn(), + bulkPermanentDelete: vi.fn(), + syncFolderOnDemand: vi.fn(), + moveMessage: vi.fn(), + moveMessageGetNewUid: vi.fn(), + setFlag: vi.fn(), + broadcast: vi.fn(), + }, +})); +vi.mock('../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + adjustFolderCounts: vi.fn(), + resolveTrashFolder: vi.fn(), + resolveAllTrashPaths: vi.fn(), + resolveAllDraftsPaths: vi.fn(), + resolveArchiveFolder: vi.fn(), + isAllMailFolder: vi.fn(), + resolveSpamFolder: vi.fn(), + resolveAllSpamPaths: vi.fn(), + }; +}); +vi.mock('../services/gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import express from 'express'; +import { query } from '../services/db.js'; +import { imapManager } from '../index.js'; +import { + adjustFolderCounts, + isAllMailFolder, + resolveAllDraftsPaths, + resolveAllSpamPaths, + resolveAllTrashPaths, + resolveArchiveFolder, + resolveSpamFolder, + resolveTrashFolder, +} from '../utils/mailUtils.js'; +import mailRoutes from './mail.js'; + +const MSG_ID = '11111111-1111-4111-8111-111111111111'; +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const account = { id: ACCOUNT_ID, user_id: 'u1', folder_mappings: {} }; +const message = { + id: MSG_ID, + account_id: ACCOUNT_ID, + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, + folder_mappings: {}, +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/mail', mailRoutes); + return app; +} + +const post = (path, body) => fetch(`${base}/api/mail${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), +}); +const del = (path) => fetch(`${base}/api/mail${path}`, { method: 'DELETE' }); + +let server; +let base; + +beforeAll(async () => { + await new Promise((resolve) => { + server = buildApp().listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); +}); + +beforeEach(() => { + query.mockReset(); + Object.values(imapManager).forEach(fn => fn.mockReset()); + [ + adjustFolderCounts, + isAllMailFolder, + resolveAllDraftsPaths, + resolveAllSpamPaths, + resolveAllTrashPaths, + resolveArchiveFolder, + resolveSpamFolder, + resolveTrashFolder, + ].forEach(fn => fn.mockReset()); + imapManager.syncFolderOnDemand.mockResolvedValue(undefined); +}); + +describe('mail mutation route characterization', () => { + it('bulk-move preserves the guarded UIDPLUS move and response contract', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [message] }; + if (sql.includes('SELECT 1 FROM folders')) return { rows: [{ '?column?': 1 }] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.includes('WITH deleted AS')) return { rows: [] }; + return { rows: [] }; + }); + imapManager.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], + }); + + const res = await post('/messages/bulk-move', { ids: [MSG_ID], folder: 'Archive' }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, moved: [MSG_ID] }); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Archive'); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[MSG_ID], [MSG_ID], [110], 'Archive']); + }); + + it('bulk-archive preserves the guarded archive move and receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.includes('WITH deleted AS')) return { rows: [] }; + return { rows: [] }; + }); + resolveArchiveFolder.mockResolvedValue('Archive'); + isAllMailFolder.mockResolvedValue(false); + imapManager.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], + }); + + const res = await post('/messages/bulk-archive', { ids: [MSG_ID] }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, archived: [MSG_ID], noArchiveFolder: [] }); + expect(imapManager.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Archive'); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + }); + + it('bulk-delete preserves the move-to-trash path and deleted-id receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.includes('WITH deleted AS')) return { rows: [] }; + return { rows: [] }; + }); + resolveTrashFolder.mockResolvedValue('Trash'); + resolveAllTrashPaths.mockResolvedValue(new Set(['Trash'])); + resolveAllDraftsPaths.mockResolvedValue(new Set(['Drafts'])); + imapManager.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map([[10, 210]]), + succeeded: [10], + failed: [], + }); + + const res = await post('/messages/bulk-delete', { ids: [MSG_ID] }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, deleted: [MSG_ID] }); + expect(imapManager.bulkPermanentDelete).not.toHaveBeenCalled(); + expect(imapManager.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Trash'); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + }); + + it('/spam resolves the spam folder and preserves the move receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.account_id, a.folder_mappings')) { + return { rows: [{ account_id: ACCOUNT_ID, folder_mappings: {} }] }; + } + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + resolveSpamFolder.mockResolvedValue('Junk'); + imapManager.moveMessage.mockResolvedValue(310); + + const res = await post(`/messages/${MSG_ID}/spam`, {}); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, folder: 'Junk', newUid: 310 }); + expect(imapManager.moveMessage).toHaveBeenCalledWith(account, 10, 'INBOX', 'Junk'); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + }); + + it('/ham requires a spam-like source and preserves the inbox move receipt', async () => { + const spamMessage = { ...message, folder: 'Junk' }; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.account_id, m.folder, a.folder_mappings')) { + return { rows: [{ account_id: ACCOUNT_ID, folder: 'Junk', folder_mappings: { inbox: 'INBOX' } }] }; + } + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [spamMessage] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [{ ...account, folder_mappings: { inbox: 'INBOX' } }] }; + return { rows: [] }; + }); + resolveAllSpamPaths.mockResolvedValue(new Set(['Junk'])); + imapManager.moveMessage.mockResolvedValue(410); + + const res = await post(`/messages/${MSG_ID}/ham`, {}); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, folder: 'INBOX', newUid: 410 }); + expect(imapManager.moveMessage).toHaveBeenCalledWith( + { ...account, folder_mappings: { inbox: 'INBOX' } }, + 10, + 'Junk', + 'INBOX', + ); + }); +}); + +describe('DELETE /api/mail/messages/:id/snooze', () => { + it('returns 404 when the message is not owned', async () => { + query.mockResolvedValue({ rows: [] }); + + const res = await del(`/messages/${MSG_ID}/snooze`); + + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ error: 'Message not found' }); + expect(imapManager.moveMessageGetNewUid).not.toHaveBeenCalled(); + }); + + it('returns 400 when the message is not currently snoozed', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) { + return { rows: [{ ...message, folder: 'Snoozed', thread_id: null }] }; + } + if (sql.includes('FROM snoozed_messages sm')) return { rows: [] }; + return { rows: [] }; + }); + + const res = await del(`/messages/${MSG_ID}/snooze`); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Message is not currently snoozed' }); + }); + + it('restores the whole snoozed reply chain and reports its count', async () => { + const root = { ...message, folder: 'Snoozed', thread_id: 'thread', is_read: true }; + const reply = { + ...root, + id: '22222222-2222-4222-8222-222222222222', + uid: 11, + message_id: '', + in_reply_to: root.message_id, + thread_references: root.message_id, + is_read: false, + }; + const snoozedRows = [ + { + snooze_id: 's1', + user_id: 'u1', + account_id: ACCOUNT_ID, + message_id_header: root.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 10, + is_read: true, + }, + { + snooze_id: 's2', + user_id: 'u1', + account_id: ACCOUNT_ID, + message_id_header: reply.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 11, + is_read: false, + }, + ]; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [root] }; + if (sql.includes('WHERE account_id = $1 AND thread_id = $2')) return { rows: [root, reply] }; + if (sql.includes('FROM snoozed_messages sm')) return { rows: snoozedRows }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + imapManager.moveMessageGetNewUid + .mockResolvedValueOnce(110) + .mockResolvedValueOnce(111); + + const res = await del(`/messages/${MSG_ID}/snooze`); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, restored: 2, folder: 'INBOX' }); + expect(imapManager.moveMessageGetNewUid).toHaveBeenCalledTimes(2); + expect(imapManager.setFlag).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/routes/mcpAccountStages.js b/backend/src/routes/mcpAccountStages.js new file mode 100644 index 00000000..005d6250 --- /dev/null +++ b/backend/src/routes/mcpAccountStages.js @@ -0,0 +1,43 @@ +import { Router } from 'express'; +import { requireAuth } from '../middleware/auth.js'; +import { + completeAccountStage, + discardAccountStage, + listStages, +} from '../services/accountService.js'; + +const router = Router(); +router.use(requireAuth); + +router.get('/', async (req, res) => { + res.json(await listStages(req.session.userId)); +}); + +router.post('/:id/execute', async (req, res) => { + try { + const account = await completeAccountStage({ + stageId: req.params.id, + userId: req.session.userId, + credentials: req.body, + }); + if (!account) return res.status(404).json({ error: 'not found' }); + res.json(account); + } catch (err) { + // completeAccountStage throws when defense-in-depth revalidation (host/port) + // fails on the merged credentials — surface that as a real client error + // rather than falling through to the generic 500 handler. + if (!err.expose) throw err; + res.status(err.status || 400).json({ error: err.message }); + } +}); + +router.delete('/:id', async (req, res) => { + const ok = await discardAccountStage({ + stageId: req.params.id, + userId: req.session.userId, + }); + if (!ok) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/mcpAccountStages.test.js b/backend/src/routes/mcpAccountStages.test.js new file mode 100644 index 00000000..7b054b00 --- /dev/null +++ b/backend/src/routes/mcpAccountStages.test.js @@ -0,0 +1,165 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import express from 'express'; + +const { + listStages, + completeAccountStage, + discardAccountStage, +} = vi.hoisted(() => ({ + listStages: vi.fn(), + completeAccountStage: vi.fn(), + discardAccountStage: vi.fn(), +})); + +vi.mock('../services/accountService.js', () => ({ + listStages, + completeAccountStage, + discardAccountStage, +})); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); + +import router from './mcpAccountStages.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/mcp-account-stages', router); + return app; +} + +async function call(app, method, path, body) { + const { createServer } = await import('node:http'); + const server = createServer(app); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const base = `http://127.0.0.1:${server.address().port}`; + try { + const response = await fetch(base + path, { + method, + headers: body ? { 'Content-Type': 'application/json' } : undefined, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await response.text(); + return { + status: response.status, + body: text ? JSON.parse(text) : null, + }; + } finally { + await new Promise(resolve => server.close(resolve)); + } +} + +beforeEach(() => { + listStages.mockReset(); + completeAccountStage.mockReset(); + discardAccountStage.mockReset(); +}); + +describe('GET /api/mcp-account-stages', () => { + it('lists stages scoped to the session user', async () => { + const rows = [{ id: 'stage-1', status: 'staged', payload: { name: 'Work' } }]; + listStages.mockResolvedValueOnce(rows); + + const response = await call(appWith(), 'GET', '/api/mcp-account-stages'); + + expect(response).toEqual({ status: 200, body: rows }); + expect(listStages).toHaveBeenCalledWith('user-1'); + }); +}); + +describe('POST /api/mcp-account-stages/:id/execute', () => { + it.each(['foreign', 'absent', 'completed'])( + '404s a %s stage', + async suffix => { + completeAccountStage.mockResolvedValueOnce(null); + + const response = await call( + appWith(), + 'POST', + `/api/mcp-account-stages/${suffix}/execute`, + { auth_pass: 'fresh-password' } + ); + + expect(response.status).toBe(404); + } + ); + + it('surfaces a defense-in-depth revalidation failure as a shaped error, not a generic 500', async () => { + completeAccountStage.mockRejectedValueOnce( + Object.assign(new Error('IMAP: Host cannot be a local address'), { status: 400, expose: true }) + ); + + const response = await call( + appWith(), + 'POST', + '/api/mcp-account-stages/stage-1/execute', + { auth_pass: 'fresh-password' } + ); + + expect(response).toEqual({ + status: 400, + body: { error: 'IMAP: Host cannot be a local address' }, + }); + }); + + it('passes fresh credentials to completion and returns the created safe account', async () => { + const credentials = { + auth_pass: 'fresh-password', + oauth_access_token: 'fresh-token', + }; + const account = { + id: 'account-1', + name: 'Work', + email_address: 'work@example.com', + }; + completeAccountStage.mockResolvedValueOnce(account); + + const response = await call( + appWith(), + 'POST', + '/api/mcp-account-stages/stage-1/execute', + credentials + ); + + expect(response).toEqual({ status: 200, body: account }); + expect(completeAccountStage).toHaveBeenCalledWith({ + stageId: 'stage-1', + userId: 'user-1', + credentials, + }); + }); +}); + +describe('DELETE /api/mcp-account-stages/:id', () => { + it('discards a scoped staged account with 204', async () => { + discardAccountStage.mockResolvedValueOnce(true); + + const response = await call( + appWith(), + 'DELETE', + '/api/mcp-account-stages/stage-1' + ); + + expect(response).toEqual({ status: 204, body: null }); + expect(discardAccountStage).toHaveBeenCalledWith({ + stageId: 'stage-1', + userId: 'user-1', + }); + }); + + it('404s an absent, foreign, completed, or discarded stage', async () => { + discardAccountStage.mockResolvedValueOnce(false); + + const response = await call( + appWith(), + 'DELETE', + '/api/mcp-account-stages/not-staged' + ); + + expect(response.status).toBe(404); + }); +}); diff --git a/backend/src/routes/rules.js b/backend/src/routes/rules.js index b0c14a02..2d054c1a 100644 --- a/backend/src/routes/rules.js +++ b/backend/src/routes/rules.js @@ -1,7 +1,7 @@ import { Router } from 'express'; import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import { applyInboxRules, isDangerousRegex } from '../services/inboxRules.js'; +import { applyInboxRules, isDangerousRegex, toRuleMessage } from '../services/inboxRules.js'; const router = Router(); router.use(requireAuth); @@ -123,30 +123,7 @@ router.post('/run', async (req, res) => { lastId = msgResult.rows[msgResult.rows.length - 1].id; - const messages = msgResult.rows.map(row => { - let toArr = []; - try { - const raw = typeof row.to_addresses === 'string' - ? JSON.parse(row.to_addresses) - : row.to_addresses; - if (Array.isArray(raw)) { - toArr = raw.map(a => ({ email: a.address || a.email || '', name: a.name || '' })); - } - } catch { /* malformed to_addresses — leave toArr empty */ } - return { - id: row.id, - uid: row.uid, - folder: row.folder, - fromEmail: row.from_email || '', - fromName: row.from_name || '', - to: toArr, - subject: row.subject || '', - hasAttachments: !!row.has_attachments, - isRead: !!row.is_read, - is_read: !!row.is_read, - parsedHeaders: {}, - }; - }); + const messages = msgResult.rows.map(toRuleMessage); const before = messages.length; const { remaining } = await applyInboxRules(messages, account, imapMgr); diff --git a/backend/src/routes/send.js b/backend/src/routes/send.js index 380a68f9..6fcd1f8e 100644 --- a/backend/src/routes/send.js +++ b/backend/src/routes/send.js @@ -1,573 +1,111 @@ -import nodemailer from 'nodemailer'; -import { randomBytes, createHash, randomUUID } from 'crypto'; import { Router } from 'express'; -import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import { refreshMicrosoftToken } from './oauth.js'; -import { decrypt } from '../services/encryption.js'; -import sanitizeHtml from 'sanitize-html'; -import { sanitizeSignature, sanitizeComposeBody } from '../services/emailSanitizer.js'; -import { embedInlineDataImages } from '../utils/inlineImages.js'; -import { redisClient } from '../services/redis.js'; -import { redactEmail } from '../utils/redact.js'; -import { generateVCard } from '../utils/vcard.js'; -import { resolveForConnection } from '../services/hostValidation.js'; -import { getConnectionPolicy } from '../services/connectionPolicy.js'; import { imapManager } from '../index.js'; -import { runTransitionsForSentMessage } from '../services/gtdTransitions.js'; - -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - -function escapeHtml(str) { - return str.replace(/&/g, '&').replace(//g, '>'); -} - -// Map SMTP/connection errors to user-friendly messages that don't expose server internals. -function sanitizeSmtpError(err) { - const msg = err.message || ''; - if (/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EHOSTUNREACH/i.test(msg)) { - return 'Could not connect to the mail server. Check your SMTP settings.'; - } - if (/535|534|530|invalid.?login|authentication.?fail|bad.*credentials|username.*password|password.*username/i.test(msg)) { - return 'Authentication failed. Check your email account credentials.'; - } - if (/throttl|rate.?limit|too many|4\.2\.|4\.7\.94/i.test(msg)) { - return 'The mail server is rate limiting sends. Please try again shortly.'; - } - if (/550|5\.[13]\.|reject|blacklist|spam|not.?accept/i.test(msg)) { - return 'Message was rejected by the mail server.'; - } - if (/TLS|SSL|certificate|handshake/i.test(msg)) { - return 'Secure connection to the mail server failed. Check your TLS settings.'; - } - return 'Failed to send message. Please try again.'; -} - -// Extract name and email from an RFC 5322 address string. -// Handles "Name ", "Name", bare "", and bare "email" forms. -function parseAddress(str) { - const m = str.match(/^(.+?)\s*<([^>]+)>\s*$/); - if (m) return { name: m[1].trim().replace(/^"|"$/g, '').trim(), email: m[2].trim().toLowerCase() }; - const bare = str.match(/^\s*<([^>]+)>\s*$/); - if (bare) return { name: '', email: bare[1].trim().toLowerCase() }; - return { name: '', email: str.trim().toLowerCase() }; -} - -function mapRecipientList(list) { - return (list || []).map(addr => parseAddress(addr)); -} - -function buildSentSnippet(body, bodyIsHtml) { - return bodyToPlain(body, bodyIsHtml).replace(/\s+/g, ' ').trim().substring(0, 200); -} - -function scheduleSentMetadataUpsert(account, sentFolder, mailOptions, meta) { - if (!sentFolder || !mailOptions.messageId) return; - setImmediate(async () => { - for (const delay of [3000, 10000, 20000]) { - await new Promise(r => setTimeout(r, delay)); - try { - const uid = await imapManager.findUidByMessageId(account, sentFolder, mailOptions.messageId); - if (uid) { - await imapManager.upsertSentMessageRecord(account, sentFolder, uid, meta); - return; - } - } catch (err) { - console.warn('Post-send sent metadata upsert failed:', err.message); - } - } - }); -} - -// Reject any recipient address that contains newlines, null bytes, or looks -// malformed — these are the classic email header-injection vectors. -function normalizeRecipients(list, fieldName) { - if (!Array.isArray(list)) throw Object.assign(new Error(`${fieldName} must be an array`), { status: 400 }); - return list.map((addr, i) => { - if (typeof addr !== 'string' || !addr.trim()) { - throw Object.assign(new Error(`${fieldName}[${i}] is empty or not a string`), { status: 400 }); - } - const trimmed = addr.trim(); - if (/[\r\n\0]/.test(trimmed)) { - throw Object.assign(new Error(`${fieldName}[${i}] contains invalid characters`), { status: 400 }); - } - const at = trimmed.lastIndexOf('@'); - if (at < 1 || at === trimmed.length - 1) { - throw Object.assign(new Error(`${fieldName}[${i}] is not a valid email address`), { status: 400 }); - } - return trimmed; - }); -} - -// Strip header-injection characters from single-line header values. -function sanitizeHeaderValue(value) { - if (typeof value !== 'string') return ''; - return value.replace(/[\r\n\0]/g, '').trim(); -} - -function textToHtml(text) { - return '
' + - text.split('\n').map(l => `

${escapeHtml(l) || ' '}

`).join('') + - '
'; -} - -function sigToPlainText(html) { - return sanitizeHtml(html, { allowedTags: [], allowedAttributes: {} }).trim(); -} - -function bodyToPlain(body, isHtml) { - if (!isHtml) return body; - return sanitizeHtml(body, { allowedTags: [], allowedAttributes: {} }); -} - -function bodyToHtml(body, isHtml) { - if (!isHtml) return textToHtml(body); - return sanitizeComposeBody(body); -} +import { query } from '../services/db.js'; +import { redisClient } from '../services/redis.js'; +import { sendOrEnqueue } from '../services/sendService.js'; +import * as outboxService from '../services/outboxService.js'; +import { normalizeUndoWindow } from '../services/outboxService.js'; +import { sanitizeSmtpError } from '../services/mail/smtp.js'; +import { refreshMicrosoftToken } from './oauth.js'; const router = Router(); router.use(requireAuth); - router.post('/send', async (req, res) => { - const { accountId, aliasId, to, cc = [], bcc = [], subject, body, bodyIsHtml = false, quotedBody, quotedBodyHtml, inReplyTo, references, attachments, editedSignature, forwardedAttachments, priority } = req.body; - const VALID_PRIORITIES = new Set(['high', 'normal', 'low']); - const emailPriority = VALID_PRIORITIES.has(priority) ? priority : 'normal'; - if (!accountId || !to?.length) return res.status(400).json({ error: 'accountId and to required' }); - - // Idempotency guard. The client sends a stable X-Idempotency-Key per logical send: a - // sequential retry after a lost success response returns the cached result, and a - // concurrent same-key submit is blocked by the reservation set just before delivery - // (below). Neither can produce a duplicate email. - const idempotencyKey = typeof req.headers['x-idempotency-key'] === 'string' - ? req.headers['x-idempotency-key'].slice(0, 128) - : null; - const idemKeyRedis = idempotencyKey ? `send_idem:${req.session.userId}:${idempotencyKey}` : null; - if (idemKeyRedis) { - const cached = await redisClient.get(idemKeyRedis).catch(() => null); - if (cached === '__inflight__') return res.status(409).json({ error: 'This message is already being sent.' }); - if (cached) return res.json(JSON.parse(cached)); - } - - if (attachments !== undefined) { - if (!Array.isArray(attachments)) return res.status(400).json({ error: 'attachments must be an array' }); - if (attachments.length > 100) return res.status(400).json({ error: 'Too many attachments (max 100)' }); - const totalBytes = attachments.reduce((sum, a) => sum + (typeof a.content === 'string' ? Math.ceil(a.content.length * 0.75) : 0), 0); - if (totalBytes > 26_214_400) return res.status(400).json({ error: 'Total attachment size exceeds 25 MB' }); - for (const [i, a] of attachments.entries()) { - if (typeof a.filename !== 'string' || !a.filename.trim()) return res.status(400).json({ error: `attachments[${i}].filename is required` }); - if (typeof a.content !== 'string') return res.status(400).json({ error: `attachments[${i}].content must be a base64 string` }); - } + const { accountId, to } = req.body; + if (!accountId || !to?.length) { + return res.status(400).json({ error: 'accountId and to required' }); } - - if (forwardedAttachments !== undefined) { - if (!Array.isArray(forwardedAttachments)) return res.status(400).json({ error: 'forwardedAttachments must be an array' }); - for (const [i, fa] of forwardedAttachments.entries()) { - if (typeof fa.messageId !== 'string' || !UUID_RE.test(fa.messageId)) return res.status(400).json({ error: `forwardedAttachments[${i}].messageId is invalid` }); - if (typeof fa.part !== 'string' || !fa.part.trim()) return res.status(400).json({ error: `forwardedAttachments[${i}].part is required` }); - } + const requestedUndo = req.body.undoSendSeconds; + if ( + requestedUndo !== undefined && + ( + typeof requestedUndo !== 'number' || + !Number.isInteger(requestedUndo) || + requestedUndo < 0 || + requestedUndo > 120 + ) + ) { + return res.status(400).json({ error: 'undoSendSeconds must be an integer from 0 to 120' }); } - let normalizedTo, normalizedCc, normalizedBcc; try { - normalizedTo = normalizeRecipients(to, 'to'); - normalizedCc = normalizeRecipients(cc, 'cc'); - normalizedBcc = normalizeRecipients(bcc, 'bcc'); - } catch (err) { - return res.status(err.status || 400).json({ error: err.message }); - } - const normalizedSubject = sanitizeHeaderValue(subject || ''); - - const [result, prefResult] = await Promise.all([ - query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]), - query('SELECT preferences FROM users WHERE id = $1', [req.session.userId]), - ]); - if (!result.rows.length) return res.status(404).json({ error: 'Account not found' }); - const plaintextEmail = prefResult.rows[0]?.preferences?.plaintextEmail === true; - let account = result.rows[0]; - - // Resolve the From identity — account by default, alias if requested - let fromName = account.sender_name || account.name; - let fromEmail = account.email_address; - let fromSignature = account.signature; - let fromReplyTo = null; - - if (aliasId) { - const aliasResult = await query( - 'SELECT * FROM account_aliases WHERE id = $1 AND account_id = $2', - [aliasId, accountId] + // Ownership remains at the HTTP boundary. Services only receive this already-scoped row. + const [accountResult, prefResult] = await Promise.all([ + query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [ + accountId, + req.session.userId, + ]), + query('SELECT preferences FROM users WHERE id = $1', [req.session.userId]), + ]); + if (!accountResult.rows.length) { + return res.status(404).json({ error: 'Account not found' }); + } + + const preferences = prefResult.rows[0]?.preferences || {}; + const undoSeconds = normalizeUndoWindow( + requestedUndo, + preferences.undoSendSeconds, ); - if (aliasResult.rows.length) { - const alias = aliasResult.rows[0]; - fromName = alias.name; - fromEmail = alias.email; - fromReplyTo = alias.reply_to || null; - // null (DB default) means inherit from account; only override when alias has an explicit signature set - if (alias.signature !== null) fromSignature = alias.signature; - } - } - - // Allow the client to override the signature per-send (editedSignature === undefined means use DB value). - // Sanitize client-supplied HTML to prevent injecting scripts or tracking pixels into sent mail. - const effectiveSignature = editedSignature !== undefined - ? (editedSignature ? sanitizeSignature(editedSignature) : null) - : fromSignature; // fromSignature from DB is already sanitized on write - - // Fetch forwarded attachment content from IMAP before entering the SMTP try-block so that - // attachment errors return descriptive messages rather than being sanitized as SMTP errors. - let resolvedFwdAttachments = []; - if (forwardedAttachments?.length) { - try { - resolvedFwdAttachments = await Promise.all(forwardedAttachments.map(async (fa) => { - const msgResult = await query( - `SELECT m.uid, m.folder, m.attachments, m.account_id FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2`, - [fa.messageId, req.session.userId] - ); - if (!msgResult.rows.length) throw Object.assign(new Error('Forwarded message not found'), { status: 404 }); - const msg = msgResult.rows[0]; - - const storedAtts = typeof msg.attachments === 'string' - ? JSON.parse(msg.attachments || '[]') - : (msg.attachments || []); - const att = storedAtts.find(a => a.part === fa.part); - if (!att) throw Object.assign(new Error('Attachment not found in message'), { status: 404 }); - - const accResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - if (!accResult.rows.length) throw Object.assign(new Error('Account not found'), { status: 404 }); - - const buffer = await imapManager.fetchAttachment(accResult.rows[0], msg.uid, msg.folder, fa.part); - if (!buffer) throw Object.assign(new Error(`Could not fetch attachment: ${att.filename}`), { status: 502 }); + const result = await sendOrEnqueue({ + ...req.body, + userId: req.session.userId, + account: accountResult.rows[0], + plaintextEmail: preferences.plaintextEmail === true, + undoSeconds, + idempotencyKey: typeof req.headers['x-idempotency-key'] === 'string' + ? req.headers['x-idempotency-key'] + : null, + }, { + query, + imapManager, + redisClient, + refreshMicrosoftToken, + outboxService, + }); - return { - filename: sanitizeHeaderValue(att.filename || 'attachment'), - content: buffer, - contentType: att.type || 'application/octet-stream', - }; - })); + if (result.queued) return res.status(202).json(result); - // Combined size check: user uploads + forwarded content - const uploadedBytes = (attachments || []).reduce( - (sum, a) => sum + (typeof a.content === 'string' ? Math.ceil(a.content.length * 0.75) : 0), 0 - ); - const fwdBytes = resolvedFwdAttachments.reduce((sum, a) => sum + (a.content?.length || 0), 0); - if (uploadedBytes + fwdBytes > 26_214_400) { - return res.status(400).json({ error: 'Total attachment size exceeds 25 MB' }); - } - } catch (err) { - return res.status(err.status || 500).json({ error: err.message || 'Failed to fetch forwarded attachments' }); - } + // Preserve the REST response contract while the service exposes a richer receipt to MCP. + const response = { ok: true }; + if (result.sentCopySaved === false) response.sentCopySaved = false; + return res.json(response); + } catch (err) { + console.error('Send failed:', err.message); + const body = { error: err.expose ? err.message : sanitizeSmtpError(err) }; + if (err.code === 'alias_not_found') body.code = err.code; + return res.status(err.status || 500).json(body); } +}); - let delivered = false; // true once transport.sendMail has actually handed off the message +router.post('/outbox/:id/cancel', async (req, res) => { try { - if (account.oauth_provider === 'microsoft') { - // Only refresh when the token is near/at expiry (mirrors imapManager's - // ensureFreshToken). Refreshing on every send needlessly rotates the AAD - // refresh token and can invalidate it under concurrent sends. - const expiryMs = account.oauth_token_expiry ? new Date(account.oauth_token_expiry).getTime() : 0; - if (expiryMs - Date.now() < 5 * 60 * 1000) { - account = await refreshMicrosoftToken(account); - } - } - - let smtpAuth; - if ((account.oauth_provider === 'microsoft' || account.oauth_provider === 'google') - && account.oauth_access_token) { - const accessToken = decrypt(account.oauth_access_token); - if (!accessToken) { - return res.status(502).json({ error: 'OAuth access token is corrupted — please reconnect your account.' }); - } - smtpAuth = { - type: 'OAuth2', - user: account.auth_user || account.email_address, - accessToken, - }; - } else { - const pass = decrypt(account.auth_pass); - if (!pass) { - return res.status(502).json({ error: 'SMTP password is corrupted or missing — please re-enter your account password in Settings.' }); - } - smtpAuth = { user: account.auth_user, pass }; - } - - const policy = await getConnectionPolicy(); - const smtpResolved = await resolveForConnection(account.smtp_host, { allowPrivate: policy.allowPrivateHosts }); - const smtpPlain = account.smtp_tls !== 'STARTTLS' && account.smtp_tls !== 'SSL'; - if (!policy.allowInsecureTls && smtpPlain) { - return res.status(403).json({ error: 'Plain-text SMTP is not allowed: admin must enable "Allow insecure TLS"' }); - } - const smtpTls = { rejectUnauthorized: !(policy.allowInsecureTls && account.imap_skip_tls_verify) }; - if (smtpResolved.servername) smtpTls.servername = smtpResolved.servername; - // For 'SSL': force direct TLS. For 'none': plain with no upgrade. - // For 'STARTTLS' (or any other/legacy value): fall back to port-based detection - // so existing accounts stored with the default 'STARTTLS' on port 465 keep working. - const smtpSecure = account.smtp_tls === 'SSL' || (account.smtp_tls !== 'none' && account.smtp_port === 465); - const transport = nodemailer.createTransport({ - host: smtpResolved.host, - port: account.smtp_port, - secure: smtpSecure, - ...(account.smtp_tls === 'none' ? { ignoreTLS: true } : {}), - auth: smtpAuth, - tls: smtpTls, - }); - - // Use a stable Message-ID so the SMTP copy and any IMAP APPEND reference the same message. - const domain = fromEmail.split('@')[1] || 'mailflow.local'; - const mailOptions = { - messageId: `<${randomBytes(16).toString('hex')}@${domain}>`, - from: `${fromName} <${fromEmail}>`, - ...(fromReplyTo ? { replyTo: fromReplyTo } : {}), - to: normalizedTo.join(', '), - cc: normalizedCc.join(', ') || undefined, - bcc: normalizedBcc.join(', ') || undefined, - subject: normalizedSubject, - ...(emailPriority !== 'normal' ? { priority: emailPriority } : {}), - text: effectiveSignature - ? bodyToPlain(body, bodyIsHtml) + '\n\n-- \n' + sigToPlainText(effectiveSignature) + (quotedBody || '') - : bodyToPlain(body, bodyIsHtml) + (quotedBody || ''), - }; - - let inlineImageAttachments = []; - if (!plaintextEmail) { - const rawHtml = bodyToHtml(body, bodyIsHtml) + - (effectiveSignature - ? '
' + effectiveSignature + '
' - : '') + - (quotedBodyHtml || (quotedBody ? textToHtml(quotedBody) : '')); - const embedded = embedInlineDataImages(rawHtml); - mailOptions.html = embedded.html; - inlineImageAttachments = embedded.attachments; - } - - if (inReplyTo) { - mailOptions.inReplyTo = sanitizeHeaderValue(inReplyTo); - // Use the full prior references chain if available; fall back to just inReplyTo. - mailOptions.references = sanitizeHeaderValue(references || inReplyTo); - } - const allAttachments = [ - ...inlineImageAttachments, - ...(attachments?.length ? attachments.map(a => ({ - filename: sanitizeHeaderValue(a.filename), - content: Buffer.from(a.content, 'base64'), - contentType: typeof a.contentType === 'string' ? a.contentType : 'application/octet-stream', - })) : []), - ...resolvedFwdAttachments, - ]; - if (allAttachments.length) { - mailOptions.attachments = allAttachments; - } - - // OAuth providers (Gmail, Microsoft) save sent mail to IMAP automatically via their - // servers — skip APPEND and sync after a delay. All other accounts use direct IMAP - // APPEND so sent mail reliably appears regardless of what the SMTP server does. - const serverAutoSaves = !!account.oauth_provider; - - // For servers that don't auto-save, generate the raw MIME now so we can APPEND it. - let rawMessage = null; - if (!serverAutoSaves) { - const streamTransport = nodemailer.createTransport({ streamTransport: true, newline: 'unix' }); - const streamInfo = await streamTransport.sendMail(mailOptions); - const chunks = []; - await new Promise((resolve, reject) => { - streamInfo.message.on('data', c => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c))); - streamInfo.message.on('end', resolve); - streamInfo.message.on('error', reject); - }); - rawMessage = Buffer.concat(chunks); - } - - // Reserve the idempotency key atomically right before delivery so a concurrent - // same-key submit cannot also send (the post-send cache alone can't stop concurrent - // duplicates). Overwritten with the result on success; released in the catch only if - // delivery never happened, so a genuine retry after a pre-send failure can proceed. - if (idemKeyRedis) { - // TTL comfortably above the worst-case send (large attachment over a slow SMTP - // server) so the in-flight guard cannot lapse while this request is still running. - const reserved = await redisClient.set(idemKeyRedis, '__inflight__', { NX: true, EX: 300 }).catch(() => 'OK'); - if (reserved === null) return res.status(409).json({ error: 'This message is already being sent.' }); - } - - await transport.sendMail(mailOptions); - delivered = true; - - // Auto-learn sent recipients so they rank above inbound-only senders in autocomplete. - // Fire-and-forget — a DB error here must never affect the send response. - const allRecipients = [...normalizedTo, ...normalizedCc, ...normalizedBcc]; - if (allRecipients.length) { - const userId = req.session.userId; - const now = new Date(); - setImmediate(async () => { - try { - // Ensure the user's default address book exists - const abResult = await query( - `INSERT INTO address_books (user_id, name) VALUES ($1, 'Personal') - ON CONFLICT (user_id, name) DO UPDATE SET updated_at = NOW() - RETURNING id`, - [userId] - ); - const addressBookId = abResult.rows[0].id; - - const results = await Promise.allSettled(allRecipients.map(addr => { - const { name, email } = parseAddress(addr); - if (!email) return Promise.resolve(); - const primaryEmail = email.toLowerCase(); - const displayName = name || primaryEmail; - const uid = randomUUID(); - const emails = [{ value: primaryEmail, type: 'other', primary: true }]; - const vcard = generateVCard({ uid, displayName, emails }); - const etag = createHash('md5').update(vcard).digest('hex'); - // Upsert by (user_id, primary_email) — bump send_count and promote from is_auto. - // On conflict, preserve an existing vcard; only fill it in if the row had none. - return query(` - INSERT INTO contacts ( - address_book_id, user_id, uid, vcard, etag, - display_name, primary_email, emails, is_auto, send_count, last_sent - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, false, 1, $9) - ON CONFLICT (address_book_id, primary_email) WHERE primary_email IS NOT NULL DO UPDATE - SET send_count = contacts.send_count + 1, - last_sent = $9, - is_auto = false, - display_name = CASE WHEN contacts.is_auto THEN $6 ELSE contacts.display_name END, - vcard = COALESCE(contacts.vcard, EXCLUDED.vcard), - etag = COALESCE(contacts.etag, EXCLUDED.etag), - updated_at = NOW() - RETURNING address_book_id - `, [addressBookId, userId, uid, vcard, etag, displayName, primaryEmail, JSON.stringify(emails), now]); - })); - - const failed = results.filter(r => r.status === 'rejected'); - if (failed.length) console.warn('Contact upsert errors:', failed.map(r => r.reason?.message)); - - // Collect distinct address books actually modified (contacts may live in non-default books). - const booksToSync = new Set(); - for (const r of results) { - if (r.status === 'fulfilled' && r.value?.rows?.[0]?.address_book_id) { - booksToSync.add(r.value.rows[0].address_book_id); - } - } - if (!booksToSync.size) booksToSync.add(addressBookId); - - await Promise.all([...booksToSync].map(bookId => - query('UPDATE address_books SET sync_token = gen_random_uuid()::text, updated_at = NOW() WHERE id = $1', [bookId]) - )); - } catch (err) { - console.warn('Contact upsert setup error:', err.message); - } - }); - } - - // Get the Sent folder path (manual mapping takes priority over special_use auto-detect) - let sentFolder = account.folder_mappings?.sent || null; - if (!sentFolder) { - const folderResult = await query( - "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Sent' LIMIT 1", - [accountId] - ); - sentFolder = folderResult.rows[0]?.path || null; - } - console.log(`Post-send: ${redactEmail(account.email_address)} sentFolder=${sentFolder} autoSaves=${serverAutoSaves}`); - - // sentCopySaved: null = not applicable (server auto-saves, or no Sent folder resolved); - // true/false = whether OUR IMAP APPEND landed the Sent copy. Surfaced to the client so - // it can warn when a delivered message could not be saved to Sent. - let sentCopySaved = null; - const sentMeta = sentFolder ? { - messageId: mailOptions.messageId, - subject: normalizedSubject, - fromName, - fromEmail, - to: mapRecipientList(normalizedTo), - cc: mapRecipientList(normalizedCc), - snippet: buildSentSnippet(body, bodyIsHtml), - date: new Date(), - } : null; - - if (sentFolder) { - if (rawMessage) { - // Non-auto-saving account: APPEND the Sent copy ourselves — exactly ONCE. IMAP - // APPEND is NOT idempotent (unlike a \Seen flag), so we must not retry: a retry - // whose first attempt merely timed out (but still lands on the server) would store - // a SECOND copy. Bound the wait so a stalled connection can't hang the response; - // the abandoned append can at worst still save the single copy. On failure, warn - // the user and schedule a fallback sync in case the append landed late. Audit [2]. - sentCopySaved = false; - try { - const { uid } = await Promise.race([ - imapManager.appendToSent(account, sentFolder, rawMessage), - new Promise((_, rej) => setTimeout(() => rej(new Error('Sent APPEND timed out')), 20000)), - ]); - sentCopySaved = true; - if (uid && sentMeta) { - await imapManager.upsertSentMessageRecord(account, sentFolder, uid, sentMeta) - .catch(err => console.warn('Sent metadata upsert failed:', err.message)); - } - setTimeout(() => { - imapManager.syncFolderOnDemand(account, sentFolder) - // Once the Sent copy is in the DB, re-run GTD transitions for its thread: a reply - // to a Todo/Someday thread means the owner acted, so that label should drop. The - // sent message reaches no other GTD hook (Sent isn't INBOX, and the tick watches - // only the state folders), so this is the only trigger. Swallow on failure — the - // next inbound sync / GTD tick self-heals. - .then(() => runTransitionsForSentMessage(imapManager, account, mailOptions.messageId) - .catch(e => console.warn(`Post-append GTD transition failed: ${e.message}`))) - .catch(e => console.error(`Post-append sync failed: ${e.message}`)); - }, 1000); - } catch (appendErr) { - console.error(`IMAP append to Sent failed for ${redactEmail(account.email_address)}/${sentFolder}: ${appendErr.message}`); - // The append may still have landed (or land shortly) — pull the folder so a - // late-completing append self-corrects the DB rather than staying invisible. - setTimeout(() => { - imapManager.syncFolderOnDemand(account, sentFolder) - .catch(e => console.error(`Post-append fallback sync failed: ${e.message}`)); - }, 8000); - } - } else { - // Server auto-saves via SMTP; seed metadata once the Sent copy is searchable. - if (sentMeta) scheduleSentMetadataUpsert(account, sentFolder, mailOptions, sentMeta); - // Server auto-saves via SMTP; just sync after a delay. Two attempts because the - // provider (e.g. Gmail) can be slow to expose the sent message; the 3s pass usually - // catches it, the 15s pass is the safety net. GTD transitions run after each: the 3s - // attempt may miss (Sent copy not yet visible → empty thread set → no-op) and the 15s - // attempt then catches it; if 3s already stripped, 15s is an idempotent no-op. - const syncAttempt = (label) => imapManager.syncFolderOnDemand(account, sentFolder) - .then(() => { - console.log(`Post-send ${label} sync done: ${redactEmail(account.email_address)}/${sentFolder}`); - return runTransitionsForSentMessage(imapManager, account, mailOptions.messageId) - .catch(e => console.warn(`Post-send ${label} GTD transition failed: ${e.message}`)); - }) - .catch(e => console.error(`Post-send ${label} sync failed: ${e.message}`)); - setTimeout(() => syncAttempt('3s'), 3000); - setTimeout(() => syncAttempt('15s'), 15000); - } + const result = await outboxService.cancel( + { id: req.params.id, userId: req.session.userId }, + { query }, + ); + if (result.cancelled || result.reason === 'cancelled') return res.json({ ok: true }); + if (result.reason === 'already_sent') { + return res.status(409).json({ error: 'already_sent' }); } + return res.status(404).json({ error: 'not_found' }); + } catch (err) { + console.error('Outbox cancel failed:', err.message); + return res.status(500).json({ error: 'Internal server error' }); + } +}); - const sendResult = { ok: true }; - // Surface only the problem case so existing success handling is unchanged; the UI warns - // when a delivered message could not be saved to the account's Sent folder. - if (sentCopySaved === false) sendResult.sentCopySaved = false; - // Overwrite the in-flight reservation with the final result so a retry after a lost - // response returns this instead of re-sending. - if (idemKeyRedis) redisClient.set(idemKeyRedis, JSON.stringify(sendResult), { EX: 86400 }).catch(() => {}); - res.json(sendResult); +router.get('/outbox', async (req, res) => { + try { + const pending = await outboxService.listPending( + { userId: req.session.userId }, + { query }, + ); + return res.json({ pending }); } catch (err) { - console.error('Send failed:', err.message); - if (idemKeyRedis) { - if (delivered) { - // The message WAS delivered but a later step threw. Persist a DURABLE success - // result (not just the short-lived reservation) so a retry at ANY time returns it - // instead of re-running transport.sendMail — otherwise the reservation would lapse - // and the same key could deliver a second copy. - redisClient.set(idemKeyRedis, JSON.stringify({ ok: true }), { EX: 86400 }).catch(() => {}); - } else { - // Delivery never happened — release so a genuine retry after a pre-send failure - // can proceed immediately. - redisClient.del(idemKeyRedis).catch(() => {}); - } - } - res.status(500).json({ error: sanitizeSmtpError(err) }); + console.error('Outbox list failed:', err.message); + return res.status(500).json({ error: 'Internal server error' }); } }); diff --git a/backend/src/routes/send.outbox.test.js b/backend/src/routes/send.outbox.test.js new file mode 100644 index 00000000..4738a175 --- /dev/null +++ b/backend/src/routes/send.outbox.test.js @@ -0,0 +1,250 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const smtpSendMail = vi.hoisted(() => vi.fn()); +const query = vi.hoisted(() => vi.fn()); +const redisClient = vi.hoisted(() => ({ + get: vi.fn(), + set: vi.fn(), + del: vi.fn(), +})); +const outbox = vi.hoisted(() => ({ + enqueue: vi.fn(), + cancel: vi.fn(), + listPending: vi.fn(), +})); +const imapManager = vi.hoisted(() => ({ + appendToSent: vi.fn(), + upsertSentMessageRecord: vi.fn(), + syncFolderOnDemand: vi.fn(), + findUidByMessageId: vi.fn(), +})); + +vi.mock('nodemailer', async (importOriginal) => { + const actual = await importOriginal(); + return { + default: { + ...actual.default, + createTransport: vi.fn(options => ( + options?.streamTransport + ? actual.default.createTransport(options) + : { sendMail: smtpSendMail } + )), + }, + }; +}); +vi.mock('../services/db.js', () => ({ query })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('./oauth.js', () => ({ refreshMicrosoftToken: vi.fn() })); +vi.mock('../services/encryption.js', () => ({ decrypt: vi.fn(() => 'smtp-password') })); +vi.mock('../services/redis.js', () => ({ redisClient })); +vi.mock('../services/hostValidation.js', () => ({ + resolveForConnection: vi.fn().mockResolvedValue({ + host: '203.0.113.25', + servername: 'smtp.example.com', + }), +})); +vi.mock('../services/connectionPolicy.js', () => ({ + getConnectionPolicy: vi.fn().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }), +})); +vi.mock('../index.js', () => ({ imapManager })); +vi.mock('../services/gtdTransitions.js', () => ({ + runTransitionsForSentMessage: vi.fn().mockResolvedValue(undefined), +})); +vi.mock('../services/outboxService.js', async (importOriginal) => ({ + ...(await importOriginal()), + ...outbox, +})); + +import express from 'express'; +import sendRoutes from './send.js'; + +const ACCOUNT_ID = '11111111-1111-4111-8111-111111111111'; +const SEND_AT = new Date('2026-07-28T12:00:30.000Z'); +const ACCOUNT_ROW = { + id: ACCOUNT_ID, + user_id: 'user-1', + email_address: 'sender@example.com', + name: 'Sender', + sender_name: null, + signature: null, + auth_user: 'sender@example.com', + auth_pass: 'encrypted', + oauth_provider: null, + smtp_host: 'smtp.example.com', + smtp_port: 587, + smtp_tls: 'STARTTLS', + imap_skip_tls_verify: false, + folder_mappings: { sent: 'Sent' }, +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/mail', sendRoutes); + return app; +} + +function post(base, path, body, headers = {}) { + return fetch(`${base}/api/mail${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body), + }); +} + +function compose(overrides = {}) { + return { + accountId: ACCOUNT_ID, + to: ['Recipient '], + subject: 'Undo send', + body: 'hello', + ...overrides, + }; +} + +describe('undo-send REST routes', () => { + let server; + let base; + + beforeAll(async () => { + await new Promise(resolve => { + server = buildApp().listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; + }); + + afterAll(async () => { + if (server) await new Promise(resolve => server.close(resolve)); + }); + + beforeEach(() => { + query.mockReset(); + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2')) { + return { rows: [ACCOUNT_ROW] }; + } + if (sql.includes('SELECT preferences FROM users')) { + return { rows: [{ preferences: { plaintextEmail: false } }] }; + } + if (sql.includes('INSERT INTO address_books')) return { rows: [{ id: 'book-1' }] }; + if (sql.includes('INSERT INTO contacts')) return { rows: [{ address_book_id: 'book-1' }] }; + return { rows: [] }; + }); + redisClient.get.mockReset().mockResolvedValue(null); + redisClient.set.mockReset().mockResolvedValue('OK'); + redisClient.del.mockReset().mockResolvedValue(1); + smtpSendMail.mockReset().mockResolvedValue({ messageId: '' }); + imapManager.appendToSent.mockReset().mockResolvedValue({ uid: null }); + imapManager.upsertSentMessageRecord.mockReset().mockResolvedValue(undefined); + imapManager.syncFolderOnDemand.mockReset().mockResolvedValue(undefined); + imapManager.findUidByMessageId.mockReset().mockResolvedValue(null); + outbox.enqueue.mockReset().mockResolvedValue({ + outbox_id: 'outbox-1', + send_at: SEND_AT, + undo_seconds: 30, + }); + outbox.cancel.mockReset(); + outbox.listPending.mockReset(); + }); + + it('sends immediately with the unchanged 200 response when undoSendSeconds is 0', async () => { + const response = await post(base, '/send', compose({ undoSendSeconds: 0 })); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + expect(smtpSendMail).toHaveBeenCalledTimes(1); + expect(outbox.enqueue).not.toHaveBeenCalled(); + }); + + it('queues for 30 seconds without calling sendMail', async () => { + const response = await post( + base, + '/send', + compose({ undoSendSeconds: 30 }), + { 'X-Idempotency-Key': 'queued-1' }, + ); + + expect(response.status).toBe(202); + expect(await response.json()).toEqual({ + queued: true, + outboxId: 'outbox-1', + sendAt: SEND_AT.toISOString(), + undoSeconds: 30, + }); + expect(smtpSendMail).not.toHaveBeenCalled(); + expect(outbox.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'queued-1', undoSeconds: 30 }), + expect.any(Object), + ); + }); + + it('sends immediately when the field and user preference are both absent', async () => { + const response = await post(base, '/send', compose()); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + expect(smtpSendMail).toHaveBeenCalledTimes(1); + expect(outbox.enqueue).not.toHaveBeenCalled(); + }); + + it.each([-1, 121, 1.5, '30'])('rejects invalid undoSendSeconds value %j', async (value) => { + const response = await post(base, '/send', compose({ undoSendSeconds: value })); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ error: 'undoSendSeconds must be an integer from 0 to 120' }); + expect(smtpSendMail).not.toHaveBeenCalled(); + expect(outbox.enqueue).not.toHaveBeenCalled(); + }); + + it('maps cancel results to 200, 409, and 404 without changing user scope', async () => { + outbox.cancel + .mockResolvedValueOnce({ cancelled: true }) + .mockResolvedValueOnce({ cancelled: false, reason: 'already_sent' }) + .mockResolvedValueOnce({ cancelled: false, reason: 'not_found' }); + + const cancelled = await post(base, '/outbox/outbox-1/cancel', {}); + expect(cancelled.status).toBe(200); + expect(await cancelled.json()).toEqual({ ok: true }); + + const sent = await post(base, '/outbox/outbox-2/cancel', {}); + expect(sent.status).toBe(409); + expect(await sent.json()).toEqual({ error: 'already_sent' }); + + const missing = await post(base, '/outbox/outbox-3/cancel', {}); + expect(missing.status).toBe(404); + expect(await missing.json()).toEqual({ error: 'not_found' }); + + expect(outbox.cancel.mock.calls.map(([input]) => input)).toEqual([ + { id: 'outbox-1', userId: 'user-1' }, + { id: 'outbox-2', userId: 'user-1' }, + { id: 'outbox-3', userId: 'user-1' }, + ]); + }); + + it('lists pending rows through the session user scope', async () => { + const pending = [{ + id: 'outbox-1', + subject: 'Undo send', + to_preview: ['recipient@example.com'], + send_at: SEND_AT.toISOString(), + }]; + outbox.listPending.mockResolvedValue(pending); + + const response = await fetch(`${base}/api/mail/outbox`); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ pending }); + expect(outbox.listPending).toHaveBeenCalledWith( + { userId: 'user-1' }, + expect.any(Object), + ); + }); +}); diff --git a/backend/src/routes/send.test.js b/backend/src/routes/send.test.js new file mode 100644 index 00000000..40b1a49a --- /dev/null +++ b/backend/src/routes/send.test.js @@ -0,0 +1,185 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const smtpSendMail = vi.hoisted(() => vi.fn()); +const query = vi.hoisted(() => vi.fn()); +const redisClient = vi.hoisted(() => ({ + get: vi.fn(), + set: vi.fn(), + del: vi.fn(), +})); +const imapManager = vi.hoisted(() => ({ + appendToSent: vi.fn(), + upsertSentMessageRecord: vi.fn(), + syncFolderOnDemand: vi.fn(), + findUidByMessageId: vi.fn(), +})); + +vi.mock('nodemailer', async (importOriginal) => { + const actual = await importOriginal(); + return { + default: { + ...actual.default, + createTransport: vi.fn(options => ( + options?.streamTransport + ? actual.default.createTransport(options) + : { sendMail: smtpSendMail } + )), + }, + }; +}); +vi.mock('../services/db.js', () => ({ query })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('./oauth.js', () => ({ refreshMicrosoftToken: vi.fn() })); +vi.mock('../services/encryption.js', () => ({ decrypt: vi.fn(() => 'smtp-password') })); +vi.mock('../services/redis.js', () => ({ redisClient })); +vi.mock('../services/hostValidation.js', () => ({ + resolveForConnection: vi.fn().mockResolvedValue({ + host: '203.0.113.25', + servername: 'smtp.example.com', + }), +})); +vi.mock('../services/connectionPolicy.js', () => ({ + getConnectionPolicy: vi.fn().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }), +})); +vi.mock('../index.js', () => ({ imapManager })); +vi.mock('../services/gtdTransitions.js', () => ({ + runTransitionsForSentMessage: vi.fn().mockResolvedValue(undefined), +})); + +import express from 'express'; +import sendRoutes from './send.js'; + +const ACCOUNT_ID = '11111111-1111-4111-8111-111111111111'; +const ACCOUNT_ROW = { + id: ACCOUNT_ID, + user_id: 'user-1', + email_address: 'sender@example.com', + name: 'Sender', + sender_name: null, + signature: null, + auth_user: 'sender@example.com', + auth_pass: 'encrypted', + oauth_provider: null, + smtp_host: 'smtp.example.com', + smtp_port: 587, + smtp_tls: 'STARTTLS', + imap_skip_tls_verify: false, + folder_mappings: { sent: 'Sent' }, +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/mail', sendRoutes); + return app; +} + +function postSend(base, { headers = {}, body = {} } = {}) { + return fetch(`${base}/api/mail/send`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify({ + accountId: ACCOUNT_ID, + to: ['Recipient '], + subject: 'Golden send', + body: 'hello', + ...body, + }), + }); +} + +describe('POST /api/mail/send — golden parity', () => { + let server; + let base; + + beforeAll(async () => { + await new Promise(resolve => { + server = buildApp().listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; + }); + + afterAll(async () => { + if (server) await new Promise(resolve => server.close(resolve)); + }); + + beforeEach(() => { + query.mockReset(); + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2')) { + return { rows: [ACCOUNT_ROW] }; + } + if (sql.includes('SELECT preferences FROM users')) { + return { rows: [{ preferences: { plaintextEmail: false } }] }; + } + if (sql.includes('INSERT INTO address_books')) return { rows: [{ id: 'book-1' }] }; + if (sql.includes('INSERT INTO contacts')) return { rows: [{ address_book_id: 'book-1' }] }; + return { rows: [] }; + }); + redisClient.get.mockReset().mockResolvedValue(null); + redisClient.set.mockReset().mockResolvedValue('OK'); + redisClient.del.mockReset().mockResolvedValue(1); + smtpSendMail.mockReset().mockResolvedValue({ messageId: '' }); + imapManager.appendToSent.mockReset().mockResolvedValue({ uid: null }); + imapManager.upsertSentMessageRecord.mockReset().mockResolvedValue(undefined); + imapManager.syncFolderOnDemand.mockReset().mockResolvedValue(undefined); + imapManager.findUidByMessageId.mockReset().mockResolvedValue(null); + }); + + it('returns the exact current success response shape', async () => { + const response = await postSend(base); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + }); + + it('passes through sentCopySaved:false when the single Sent APPEND fails', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + imapManager.appendToSent.mockRejectedValueOnce(new Error('append unavailable')); + + const response = await postSend(base); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true, sentCopySaved: false }); + expect(imapManager.appendToSent).toHaveBeenCalledTimes(1); + }); + + it('returns 409 for an in-flight idempotency key without sending', async () => { + redisClient.get.mockResolvedValueOnce('__inflight__'); + + const response = await postSend(base, { + headers: { 'X-Idempotency-Key': 'same-send' }, + }); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ error: 'This message is already being sent.' }); + expect(smtpSendMail).not.toHaveBeenCalled(); + }); + + it('replays a completed cached result without sending', async () => { + redisClient.get.mockResolvedValueOnce(JSON.stringify({ ok: true, sentCopySaved: false })); + + const response = await postSend(base, { + headers: { 'X-Idempotency-Key': 'completed-send' }, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true, sentCopySaved: false }); + expect(smtpSendMail).not.toHaveBeenCalled(); + }); + + it('sanitizes SMTP failures instead of exposing raw server details', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + smtpSendMail.mockRejectedValueOnce(new Error('ECONNREFUSED smtp.secret.internal:2525')); + + const response = await postSend(base); + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: 'Could not connect to the mail server. Check your SMTP settings.', + }); + }); +}); diff --git a/backend/src/services/accountFields.js b/backend/src/services/accountFields.js new file mode 100644 index 00000000..054ef991 --- /dev/null +++ b/backend/src/services/accountFields.js @@ -0,0 +1,19 @@ +import { sanitizeSignature } from './emailSanitizer.js'; + +// Fields safe to return to the client — matches the GET list, excludes credentials and tokens +export const SAFE_FIELDS = [ + 'id', 'name', 'sender_name', 'email_address', 'color', 'protocol', + 'imap_host', 'imap_port', 'imap_skip_tls_verify', + 'smtp_host', 'smtp_port', 'smtp_tls', + 'auth_user', 'oauth_provider', 'enabled', + 'last_sync', 'sync_error', 'sort_order', 'folder_mappings', + 'signature', 'created_at', 'categorization_enabled', + 'gtd_enabled', 'gtd_folders', +]; + +export function safeAccount(row) { + const obj = Object.fromEntries(SAFE_FIELDS.map(k => [k, row[k]])); + // Sanitize on read so legacy values stored before the write-time sanitizer are safe + if (obj.signature) obj.signature = sanitizeSignature(obj.signature); + return obj; +} diff --git a/backend/src/services/accountFields.test.js b/backend/src/services/accountFields.test.js new file mode 100644 index 00000000..925a5af3 --- /dev/null +++ b/backend/src/services/accountFields.test.js @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { SAFE_FIELDS, safeAccount } from './accountFields.js'; + +describe('safeAccount', () => { + it('returns only SAFE_FIELDS and sanitizes the signature on read', () => { + const row = Object.fromEntries(SAFE_FIELDS.map(field => [field, `${field}-value`])); + row.signature = 'Safe'; + row.auth_pass = 'secret'; + row.oauth_access_token = 'access'; + row.oauth_refresh_token = 'refresh'; + row.user_id = 'user-1'; + + const account = safeAccount(row); + + expect(Object.keys(account)).toEqual(SAFE_FIELDS); + expect(account.signature).toBe('Safe'); + expect(account).not.toHaveProperty('auth_pass'); + expect(account).not.toHaveProperty('oauth_access_token'); + expect(account).not.toHaveProperty('oauth_refresh_token'); + expect(account).not.toHaveProperty('user_id'); + }); + + it('keeps every safe field in parity with the REST GET account column list', () => { + const accountsRoute = readFileSync( + fileURLToPath(new URL('../routes/accounts.js', import.meta.url)), + 'utf8' + ); + const select = accountsRoute.match( + /router\.get\('\/'[\s\S]*?`SELECT\s+([\s\S]*?)\s+FROM email_accounts/ + ); + expect(select).not.toBeNull(); + + const selectedFields = select[1] + .split(',') + .map(field => field.trim()) + .filter(Boolean); + + expect(SAFE_FIELDS.every(field => selectedFields.includes(field))).toBe(true); + }); +}); diff --git a/backend/src/services/accountService.js b/backend/src/services/accountService.js new file mode 100644 index 00000000..6647f763 --- /dev/null +++ b/backend/src/services/accountService.js @@ -0,0 +1,225 @@ +import { query } from './db.js'; +import { imapManager } from '../index.js'; +import { encrypt } from './encryption.js'; +import { hasHeaderInjectionChars, sanitizeSignature } from './emailSanitizer.js'; +import { validateHost } from './hostValidation.js'; +import { getConnectionPolicy } from './connectionPolicy.js'; +import { safeAccount } from './accountFields.js'; +import { createKeyedSerializer } from '../utils/keyedSerializer.js'; + +// Serialize an account's reconnect triggers so a rapid settings change (e.g. a +// gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains — +// connectAccount's in-progress guard would drop the second and leave the GTD sync +// tick armed inconsistently with the final DB value. Queued per account id. +const reconnectQueue = createKeyedSerializer(); + +export const ALLOWED_IMAP_PORTS = new Set([143, 993]); +export const ALLOWED_SMTP_PORTS = new Set([465, 587]); + +export function validatePort(port, allowed) { + const n = Number(port); + if (!Number.isInteger(n) || n < 1 || n > 65535) { + return `Port ${port} is not a valid port number`; + } + // When private/local hosts are explicitly allowed (e.g. Proton Mail Bridge on 1143/1025), + // skip the whitelist — the operator has already opted into unrestricted host access. + if (process.env.ALLOW_PRIVATE_IMAP_HOSTS === 'true') return null; + if (!allowed.has(n)) { + return `Port ${port} is not allowed. Allowed: ${[...allowed].join(', ')}`; + } + return null; +} + +async function validateAccountFields(fields) { + const { + name, sender_name = null, email_address, + imap_host, imap_port = 993, + smtp_host, smtp_port = 587, + } = fields; + + if (!name || !email_address) return { error: 'Name and email required', status: 400 }; + if (hasHeaderInjectionChars(name) || hasHeaderInjectionChars(email_address)) { + return { error: 'Name and email address cannot contain control characters', status: 400 }; + } + if (sender_name && hasHeaderInjectionChars(sender_name)) { + return { error: 'Sender name cannot contain control characters', status: 400 }; + } + + const policy = await getConnectionPolicy(); + + if (imap_host) { + const err = (await validateHost(imap_host, { allowPrivate: policy.allowPrivateHosts })) + || (!policy.allowNonstandardPorts && validatePort(imap_port, ALLOWED_IMAP_PORTS)); + if (err) return { error: `IMAP: ${err}`, status: 400 }; + } + if (smtp_host) { + const err = (await validateHost(smtp_host, { allowPrivate: policy.allowPrivateHosts })) + || (!policy.allowNonstandardPorts && validatePort(smtp_port, ALLOWED_SMTP_PORTS)); + if (err) return { error: `SMTP: ${err}`, status: 400 }; + } + return null; +} + +export async function createAccount({ userId, fields }) { + const validation = await validateAccountFields(fields); + if (validation) return validation; + + const { + name, sender_name = null, email_address, color = '#6366f1', protocol = 'imap', + imap_host, imap_port = 993, imap_skip_tls_verify = false, + smtp_host, smtp_port = 587, smtp_tls = 'STARTTLS', + auth_user, auth_pass, + oauth_provider, oauth_access_token, oauth_refresh_token, + signature = null + } = fields; + + try { + const result = await query(` + INSERT INTO email_accounts ( + user_id, name, sender_name, email_address, color, protocol, + imap_host, imap_port, imap_tls, imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, + auth_user, auth_pass, oauth_provider, oauth_access_token, oauth_refresh_token, + signature + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) + RETURNING * + `, [ + userId, name, sender_name || null, email_address, color, protocol, + imap_host, imap_port, Number(imap_port) % 1000 === 993, !!imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, + auth_user, encrypt(auth_pass), oauth_provider, encrypt(oauth_access_token), encrypt(oauth_refresh_token), + sanitizeSignature(signature) || null + ]); + + const account = result.rows[0]; + + // Immediately try to connect — needs full credentials from DB row + if (protocol === 'imap') { + imapManager.connectAccount(account).catch(console.error); + } + + return { account: safeAccount(account) }; + } catch (err) { + console.error(err); + return { error: 'Failed to add account', status: 500 }; + } +} + +export function reconcileConnectionState({ id, updates, before, updated }) { + const gtdFoldersChanged = !!before?.gtdFoldersChanged; + const isDisabling = 'enabled' in updates && !updates.enabled; + const needsReconnect = !isDisabling && ( + 'enabled' in updates || + 'auth_user' in updates || + 'auth_pass' in updates || + 'imap_host' in updates || + 'imap_port' in updates || + 'imap_tls' in updates || + 'imap_skip_tls_verify' in updates || + 'gtd_enabled' in updates || + gtdFoldersChanged + ); + + // Both branches queue through the per-account serializer so overlapping settings + // changes (e.g. a rapid gtd_enabled double-toggle) apply their connection-state + // effects in order, never as two overlapping chains. + if (isDisabling) { + reconnectQueue(id, () => imapManager.disconnectAccount(id)) + .catch(err => console.error(`Failed to disconnect account ${id} after disable:`, err.message)); + } else if (needsReconnect && updated.protocol === 'imap' && updated.enabled) { + reconnectQueue(id, () => + imapManager.disconnectAccount(id) + .then(() => query('SELECT * FROM email_accounts WHERE id = $1', [id])) + .then(r => { if (r.rows.length) return imapManager.connectAccount(r.rows[0]); }) + ).catch(err => console.error(`Failed to reconnect account ${id} after update:`, err.message)); + } +} + +const STAGED_SECRET_FIELDS = [ + 'auth_pass', + 'oauth_access_token', + 'oauth_refresh_token', +]; + +export async function stageAccount({ userId, payload }) { + for (const key of STAGED_SECRET_FIELDS) { + if (Object.prototype.hasOwnProperty.call(payload, key)) { + throw new Error(`Staged account payload cannot contain ${key}`); + } + } + + const validation = await validateAccountFields(payload); + if (validation) return validation; + + const stagedPayload = { + ...payload, + signature: sanitizeSignature(payload.signature) || null, + }; + const result = await query( + `INSERT INTO mcp_account_stages (user_id, payload) + VALUES ($1, $2) + RETURNING *`, + [userId, stagedPayload] + ); + return result.rows[0]; +} + +export async function listStages(userId) { + const result = await query( + `SELECT id, status, payload, created_at, completed_account_id + FROM mcp_account_stages + WHERE user_id = $1 AND status = 'staged' + ORDER BY created_at`, + [userId] + ); + return result.rows; +} + +export async function completeAccountStage({ stageId, userId, credentials }) { + const stageResult = await query( + `SELECT id, payload + FROM mcp_account_stages + WHERE id = $1 AND user_id = $2 AND status = 'staged'`, + [stageId, userId] + ); + if (!stageResult.rows.length) return null; + + const freshCredentials = {}; + for (const key of STAGED_SECRET_FIELDS) { + if (Object.prototype.hasOwnProperty.call(credentials || {}, key)) { + freshCredentials[key] = credentials[key]; + } + } + const fields = { + ...stageResult.rows[0].payload, + ...freshCredentials, + }; + + // createAccount performs the same host/port/header validation again before + // writing, providing defense in depth if a staged payload was tampered with. + const created = await createAccount({ userId, fields }); + if (created.error) { + throw Object.assign(new Error(created.error), { + status: created.status, + expose: true, + }); + } + + await query( + `UPDATE mcp_account_stages + SET status = 'completed', completed_account_id = $1 + WHERE id = $2 AND user_id = $3 AND status = 'staged' + RETURNING id`, + [created.account.id, stageId, userId] + ); + return created.account; +} + +export async function discardAccountStage({ stageId, userId }) { + const result = await query( + `UPDATE mcp_account_stages + SET status = 'discarded' + WHERE id = $1 AND user_id = $2 AND status = 'staged' + RETURNING id`, + [stageId, userId] + ); + return result.rows.length > 0; +} diff --git a/backend/src/services/accountService.test.js b/backend/src/services/accountService.test.js new file mode 100644 index 00000000..904eaaa4 --- /dev/null +++ b/backend/src/services/accountService.test.js @@ -0,0 +1,302 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + query, + encrypt, + validateHost, + getConnectionPolicy, + connectAccount, + disconnectAccount, +} = vi.hoisted(() => ({ + query: vi.fn(), + encrypt: vi.fn(value => value ? `encrypted:${value}` : value), + validateHost: vi.fn(), + getConnectionPolicy: vi.fn(), + connectAccount: vi.fn(), + disconnectAccount: vi.fn(), +})); + +vi.mock('./db.js', () => ({ query })); +vi.mock('./encryption.js', () => ({ encrypt })); +vi.mock('./hostValidation.js', () => ({ validateHost })); +vi.mock('./connectionPolicy.js', () => ({ getConnectionPolicy })); +vi.mock('../index.js', () => ({ + imapManager: { connectAccount, disconnectAccount }, +})); + +import { SAFE_FIELDS } from './accountFields.js'; +import { + completeAccountStage, + createAccount, + discardAccountStage, + listStages, + reconcileConnectionState, + stageAccount, +} from './accountService.js'; + +const validFields = { + name: 'Primary', + sender_name: 'Sender', + email_address: 'sender@example.com', + protocol: 'imap', + imap_host: 'imap.example.com', + imap_port: 993, + smtp_host: 'smtp.example.com', + smtp_port: 587, + auth_user: 'sender@example.com', + auth_pass: 'password', +}; + +beforeEach(() => { + query.mockReset(); + encrypt.mockClear(); + validateHost.mockReset().mockResolvedValue(null); + getConnectionPolicy.mockReset().mockResolvedValue({ + allowPrivateHosts: false, + allowNonstandardPorts: false, + }); + connectAccount.mockReset().mockResolvedValue(undefined); + disconnectAccount.mockReset().mockResolvedValue(undefined); +}); + +describe('createAccount', () => { + it('rejects an invalid host', async () => { + validateHost.mockResolvedValueOnce('Host cannot be a local address'); + + await expect(createAccount({ userId: 'user-1', fields: validFields })).resolves.toEqual({ + error: 'IMAP: Host cannot be a local address', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('rejects an invalid port', async () => { + const result = await createAccount({ + userId: 'user-1', + fields: { ...validFields, imap_port: 25 }, + }); + + expect(result).toEqual({ + error: 'IMAP: Port 25 is not allowed. Allowed: 143, 993', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('encrypts auth_pass, starts a fire-and-forget connection, and returns only safe fields', async () => { + const inserted = { + id: 'account-1', + user_id: 'user-1', + ...validFields, + auth_pass: 'encrypted:password', + oauth_access_token: 'secret-access', + oauth_refresh_token: 'secret-refresh', + signature: 'Sender', + enabled: true, + }; + query.mockResolvedValueOnce({ rows: [inserted] }); + connectAccount.mockReturnValueOnce(new Promise(() => {})); + + const result = await createAccount({ userId: 'user-1', fields: validFields }); + + expect(encrypt).toHaveBeenCalledWith('password'); + expect(query.mock.calls[0][1]).toContain('encrypted:password'); + expect(connectAccount).toHaveBeenCalledWith(inserted); + expect(Object.keys(result.account)).toEqual(SAFE_FIELDS); + expect(result.account.signature).toBe('Sender'); + expect(result.account).not.toHaveProperty('auth_pass'); + }); +}); + +describe('reconcileConnectionState', () => { + it('disconnects only when disabling', async () => { + reconcileConnectionState({ + id: 'disable-1', + updates: { enabled: false }, + before: { gtdFoldersChanged: false }, + updated: { protocol: 'imap', enabled: false }, + }); + + await vi.waitFor(() => expect(disconnectAccount).toHaveBeenCalledWith('disable-1')); + expect(query).not.toHaveBeenCalled(); + expect(connectAccount).not.toHaveBeenCalled(); + }); + + it.each([ + ['enabled', { enabled: true }, false], + ['auth_user', { auth_user: 'new-user' }, false], + ['auth_pass', { auth_pass: 'new-pass' }, false], + ['imap_host', { imap_host: 'new.example.com' }, false], + ['imap_port', { imap_port: 143 }, false], + ['imap_tls', { imap_tls: false }, false], + ['imap_skip_tls_verify', { imap_skip_tls_verify: true }, false], + ['gtd_enabled', { gtd_enabled: true }, false], + ['gtdFoldersChanged', {}, true], + ])('reconnects for the %s trigger', async (name, updates, gtdFoldersChanged) => { + const id = `trigger-${name}`; + const fresh = { id, protocol: 'imap', enabled: true }; + query.mockResolvedValueOnce({ rows: [fresh] }); + + reconcileConnectionState({ + id, + updates, + before: { gtdFoldersChanged }, + updated: fresh, + }); + + await vi.waitFor(() => expect(connectAccount).toHaveBeenCalledWith(fresh)); + expect(disconnectAccount).toHaveBeenCalledWith(id); + }); + + it('does nothing when no reconnect field changed', async () => { + reconcileConnectionState({ + id: 'noop-1', + updates: { color: '#ffffff' }, + before: { gtdFoldersChanged: false }, + updated: { protocol: 'imap', enabled: true }, + }); + await Promise.resolve(); + + expect(disconnectAccount).not.toHaveBeenCalled(); + expect(connectAccount).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); + + it('gives disabling precedence over another reconnect trigger', async () => { + reconcileConnectionState({ + id: 'disable-precedence', + updates: { enabled: false, auth_user: 'new-user' }, + before: { gtdFoldersChanged: false }, + updated: { protocol: 'imap', enabled: false }, + }); + + await vi.waitFor(() => expect(disconnectAccount).toHaveBeenCalledWith('disable-precedence')); + expect(query).not.toHaveBeenCalled(); + expect(connectAccount).not.toHaveBeenCalled(); + }); +}); + +describe('staged accounts', () => { + it.each([ + ['auth_pass', ''], + ['oauth_access_token', null], + ['oauth_refresh_token', false], + ])('hard-rejects payloads containing the secret key %s', async (key, value) => { + const payload = { ...validFields }; + delete payload.auth_pass; + payload[key] = value; + await expect(stageAccount({ + userId: 'user-1', + payload, + })).rejects.toThrow(`Staged account payload cannot contain ${key}`); + expect(query).not.toHaveBeenCalled(); + }); + + it('validates staged hosts before inserting', async () => { + validateHost.mockResolvedValueOnce('Host cannot be a local address'); + const payload = { ...validFields }; + delete payload.auth_pass; + + await expect(stageAccount({ + userId: 'user-1', + payload, + })).resolves.toEqual({ + error: 'IMAP: Host cannot be a local address', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('validates staged ports before inserting', async () => { + const payload = { ...validFields, smtp_port: 25 }; + delete payload.auth_pass; + const result = await stageAccount({ + userId: 'user-1', + payload, + }); + + expect(result).toEqual({ + error: 'SMTP: Port 25 is not allowed. Allowed: 465, 587', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('lists only staged rows scoped to the user', async () => { + const rows = [{ id: 'stage-1', status: 'staged' }]; + query.mockResolvedValueOnce({ rows }); + + await expect(listStages('user-1')).resolves.toEqual(rows); + expect(query).toHaveBeenCalledWith( + expect.stringContaining("user_id = $1 AND status = 'staged'"), + ['user-1'] + ); + }); + + it('completes a scoped stage by merging fresh credentials and marking it completed', async () => { + const payload = { ...validFields }; + delete payload.auth_pass; + const inserted = { + id: 'account-from-stage', + user_id: 'user-1', + ...payload, + auth_pass: 'encrypted:fresh-password', + enabled: true, + }; + query + .mockResolvedValueOnce({ rows: [{ id: 'stage-1', payload }] }) + .mockResolvedValueOnce({ rows: [inserted] }) + .mockResolvedValueOnce({ rows: [{ id: 'stage-1', status: 'completed' }] }); + + const result = await completeAccountStage({ + stageId: 'stage-1', + userId: 'user-1', + credentials: { auth_pass: 'fresh-password' }, + }); + + expect(result.id).toBe('account-from-stage'); + expect(encrypt).toHaveBeenCalledWith('fresh-password'); + expect(query.mock.calls[0]).toEqual([ + expect.stringContaining("user_id = $2 AND status = 'staged'"), + ['stage-1', 'user-1'], + ]); + expect(query.mock.calls[1][0]).toContain('INSERT INTO email_accounts'); + expect(query.mock.calls[2]).toEqual([ + expect.stringContaining("SET status = 'completed', completed_account_id = $1"), + ['account-from-stage', 'stage-1', 'user-1'], + ]); + }); + + it('returns null when completing a missing, foreign, or non-staged row', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(completeAccountStage({ + stageId: 'foreign-stage', + userId: 'user-1', + credentials: { auth_pass: 'fresh-password' }, + })).resolves.toBeNull(); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('discards only a scoped stage that is still staged', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'stage-1' }] }); + + await expect(discardAccountStage({ + stageId: 'stage-1', + userId: 'user-1', + })).resolves.toBe(true); + expect(query).toHaveBeenCalledWith( + expect.stringContaining("status = 'staged'"), + ['stage-1', 'user-1'] + ); + }); + + it('cannot discard a missing, foreign, completed, or discarded stage', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(discardAccountStage({ + stageId: 'not-staged', + userId: 'user-1', + })).resolves.toBe(false); + }); +}); diff --git a/backend/src/services/connectionErrors.js b/backend/src/services/connectionErrors.js new file mode 100644 index 00000000..d624ee55 --- /dev/null +++ b/backend/src/services/connectionErrors.js @@ -0,0 +1,38 @@ +// Map IMAP, SMTP, and network connection errors to user-friendly messages that +// do not expose server internals. SMTP categories and messages intentionally +// match sanitizeSmtpError so the connection probe remains consistent with send. +export function sanitizeConnectionError(err) { + const msg = err?.message || ''; + const code = err?.code || ''; + const responseStatus = err?.responseStatus || ''; + const combined = `${code} ${responseStatus} ${msg}`; + + if (/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EHOSTUNREACH/i.test(combined)) { + return 'Could not connect to the mail server. Check your SMTP settings.'; + } + if ( + err?.authenticationFailed || + /AUTHENTICATIONFAILED|535|534|530|invalid.?login|authentication.?fail|bad.*credentials|username.*password|password.*username|login denied/i.test(combined) + ) { + return 'Authentication failed. Check your email account credentials.'; + } + if (/throttl|rate.?limit|too many|4\.2\.|4\.7\.94/i.test(msg)) { + return 'The mail server is rate limiting sends. Please try again shortly.'; + } + if (/550|5\.[13]\.|reject|blacklist|spam|not.?accept/i.test(msg)) { + return 'Message was rejected by the mail server.'; + } + if (/TLS|SSL|certificate|handshake/i.test(msg)) { + return 'Secure connection to the mail server failed. Check your TLS settings.'; + } + if (/connection test timed out/i.test(msg)) { + return 'Connection test timed out. Check your mail server settings.'; + } + if (/greeting|capabilit|IMAP4rev/i.test(msg)) { + return 'Could not establish an IMAP connection. Check your IMAP settings.'; + } + if (/^(NO|BAD)$/i.test(responseStatus) || /^(NO|BAD)\b/i.test(msg)) { + return 'The IMAP server rejected the connection. Check your IMAP settings.'; + } + return 'Failed to send message. Please try again.'; +} diff --git a/backend/src/services/connectionErrors.test.js b/backend/src/services/connectionErrors.test.js new file mode 100644 index 00000000..bfe73722 --- /dev/null +++ b/backend/src/services/connectionErrors.test.js @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { sanitizeConnectionError } from './connectionErrors.js'; + +describe('sanitizeConnectionError', () => { + const fixtures = [ + { + error: new Error('connect ECONNREFUSED 203.0.113.1:587'), + expectedMessage: 'Could not connect to the mail server. Check your SMTP settings.', + }, + { + error: new Error('535 5.7.8 Authentication failed: invalid login'), + expectedMessage: 'Authentication failed. Check your email account credentials.', + }, + { + error: new Error('454 4.7.94 rate limit exceeded'), + expectedMessage: 'The mail server is rate limiting sends. Please try again shortly.', + }, + { + error: new Error('550 5.1.1 rejected as spam'), + expectedMessage: 'Message was rejected by the mail server.', + }, + { + error: new Error('TLS certificate handshake failed'), + expectedMessage: 'Secure connection to the mail server failed. Check your TLS settings.', + }, + { + error: new Error('unexpected SMTP response'), + expectedMessage: 'Failed to send message. Please try again.', + }, + { + error: { authenticationFailed: true, message: 'Command failed' }, + expectedMessage: 'Authentication failed. Check your email account credentials.', + }, + { + error: { code: 'AUTHENTICATIONFAILED', responseStatus: 'NO', message: 'NO Login denied' }, + expectedMessage: 'Authentication failed. Check your email account credentials.', + }, + { + error: { responseStatus: 'BAD', message: 'BAD invalid command during connection' }, + expectedMessage: 'The IMAP server rejected the connection. Check your IMAP settings.', + }, + { + error: new Error('Failed to receive greeting from server'), + expectedMessage: 'Could not establish an IMAP connection. Check your IMAP settings.', + }, + { + error: new Error('Server did not advertise IMAP4rev1 capability'), + expectedMessage: 'Could not establish an IMAP connection. Check your IMAP settings.', + }, + { + error: new Error('Connection test timed out after 10 seconds'), + expectedMessage: 'Connection test timed out. Check your mail server settings.', + }, + { + error: { responseStatus: 'NO', message: 'NO mailbox unavailable' }, + expectedMessage: 'The IMAP server rejected the connection. Check your IMAP settings.', + }, + ]; + + it.each(fixtures)('maps $error to a safe message', ({ error, expectedMessage }) => { + expect(sanitizeConnectionError(error)).toBe(expectedMessage); + }); +}); diff --git a/backend/src/services/connectionTest.js b/backend/src/services/connectionTest.js new file mode 100644 index 00000000..5fafb699 --- /dev/null +++ b/backend/src/services/connectionTest.js @@ -0,0 +1,74 @@ +import { ImapFlow } from 'imapflow'; +import { makeClientCfg } from './imapManager.js'; +import { resolveForConnection } from './hostValidation.js'; +import { getConnectionPolicy } from './connectionPolicy.js'; +import { buildSmtpTransport } from './mail/smtp.js'; +import { sanitizeConnectionError } from './connectionErrors.js'; + +const CONNECTION_TIMEOUT_MS = 10_000; + +function withTimeout(operation, onTimeout) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + try { + onTimeout?.(); + } finally { + reject(new Error('Connection test timed out after 10 seconds')); + } + }, CONNECTION_TIMEOUT_MS); + }); + return Promise.race([operation, timeout]).finally(() => clearTimeout(timer)); +} + +async function probeImap(account) { + let client; + let connected = false; + try { + const policy = await getConnectionPolicy(); + const resolved = await resolveForConnection(account.imap_host, { + allowPrivate: policy.allowPrivateHosts, + }); + const cfg = makeClientCfg(account, resolved, { + enableIdle: false, + policy, + }); + client = new ImapFlow(cfg); + await withTimeout(client.connect(), () => client.close()); + connected = true; + return { ok: true }; + } catch (err) { + return { ok: false, error: sanitizeConnectionError(err) }; + } finally { + if (connected) { + try { + await client.logout(); + } catch { + client.close(); + } + } else { + client?.close(); + } + } +} + +async function probeSmtp(account) { + let transport; + try { + ({ transport } = await buildSmtpTransport(account)); + await withTimeout(transport.verify(), () => transport.close?.()); + return { ok: true }; + } catch (err) { + return { ok: false, error: sanitizeConnectionError(err) }; + } finally { + transport?.close?.(); + } +} + +export async function testConnection(account) { + const [imap, smtp] = await Promise.all([ + probeImap(account), + probeSmtp(account), + ]); + return { imap, smtp }; +} diff --git a/backend/src/services/connectionTest.test.js b/backend/src/services/connectionTest.test.js new file mode 100644 index 00000000..30645ccf --- /dev/null +++ b/backend/src/services/connectionTest.test.js @@ -0,0 +1,159 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const { + ImapFlow, + makeClientCfg, + buildSmtpTransport, + resolveForConnection, + getConnectionPolicy, + imapConnect, + imapLogout, + imapClose, + smtpVerify, + smtpClose, +} = vi.hoisted(() => { + const imapConnect = vi.fn(); + const imapLogout = vi.fn(); + const imapClose = vi.fn(); + return { + ImapFlow: vi.fn(function MockImapFlow() { + this.connect = imapConnect; + this.logout = imapLogout; + this.close = imapClose; + }), + makeClientCfg: vi.fn(), + buildSmtpTransport: vi.fn(), + resolveForConnection: vi.fn(), + getConnectionPolicy: vi.fn(), + imapConnect, + imapLogout, + imapClose, + smtpVerify: vi.fn(), + smtpClose: vi.fn(), + }; +}); + +vi.mock('imapflow', () => ({ ImapFlow })); +vi.mock('./imapManager.js', () => ({ makeClientCfg })); +vi.mock('./mail/smtp.js', () => ({ buildSmtpTransport })); +vi.mock('./hostValidation.js', () => ({ resolveForConnection })); +vi.mock('./connectionPolicy.js', () => ({ getConnectionPolicy })); + +import { testConnection } from './connectionTest.js'; + +const account = { + imap_host: 'imap.example.com', + smtp_host: 'smtp.example.com', + imap_port: 993, + smtp_port: 587, +}; + +beforeEach(() => { + ImapFlow.mockClear(); + makeClientCfg.mockReset().mockReturnValue({ host: '203.0.113.10' }); + resolveForConnection.mockReset().mockResolvedValue({ + host: '203.0.113.10', + servername: 'imap.example.com', + }); + getConnectionPolicy.mockReset().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }); + imapConnect.mockReset().mockResolvedValue(undefined); + imapLogout.mockReset().mockResolvedValue(undefined); + imapClose.mockReset(); + smtpVerify.mockReset().mockResolvedValue(true); + smtpClose.mockReset(); + buildSmtpTransport.mockReset().mockResolvedValue({ + transport: { verify: smtpVerify, close: smtpClose }, + account, + }); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('testConnection', () => { + it('returns success for both probes', async () => { + await expect(testConnection(account)).resolves.toEqual({ + imap: { ok: true }, + smtp: { ok: true }, + }); + expect(imapLogout).toHaveBeenCalledOnce(); + expect(smtpVerify).toHaveBeenCalledOnce(); + }); + + it('keeps SMTP successful when IMAP fails', async () => { + imapConnect.mockRejectedValueOnce(new Error('authenticationFailed: invalid login')); + + const result = await testConnection(account); + + expect(result.imap).toEqual({ + ok: false, + error: 'Authentication failed. Check your email account credentials.', + }); + expect(result.smtp).toEqual({ ok: true }); + expect(smtpVerify).toHaveBeenCalledOnce(); + }); + + it('keeps IMAP successful when SMTP fails', async () => { + smtpVerify.mockRejectedValueOnce(new Error('connect ECONNREFUSED')); + + const result = await testConnection(account); + + expect(result.imap).toEqual({ ok: true }); + expect(result.smtp).toEqual({ + ok: false, + error: 'Could not connect to the mail server. Check your SMTP settings.', + }); + }); + + it('force-closes a timed-out IMAP probe without hanging SMTP', async () => { + vi.useFakeTimers(); + imapConnect.mockReturnValueOnce(new Promise(() => {})); + + const pending = testConnection(account); + await vi.advanceTimersByTimeAsync(10_000); + const result = await pending; + + expect(imapClose).toHaveBeenCalled(); + expect(result.imap).toEqual({ + ok: false, + error: 'Connection test timed out. Check your mail server settings.', + }); + expect(result.smtp).toEqual({ ok: true }); + }); + + it('force-closes a timed-out SMTP probe without hanging IMAP', async () => { + vi.useFakeTimers(); + smtpVerify.mockReturnValueOnce(new Promise(() => {})); + + const pending = testConnection(account); + await vi.advanceTimersByTimeAsync(10_000); + const result = await pending; + + expect(smtpClose).toHaveBeenCalled(); + expect(result.imap).toEqual({ ok: true }); + expect(result.smtp).toEqual({ + ok: false, + error: 'Connection test timed out. Check your mail server settings.', + }); + }); + + it('imports only the named makeClientCfg helper from imapManager', () => { + const source = readFileSync( + fileURLToPath(new URL('./connectionTest.js', import.meta.url)), + 'utf8' + ); + const imapManagerImport = source.match( + /import\s+([^;]+)\s+from\s+['"]\.\/imapManager\.js['"]/ + ); + + expect(imapManagerImport?.[1]).toMatch(/^\{\s*makeClientCfg\s*\}$/); + expect(source).not.toMatch(/import\s+imapManager\b/); + expect(source.replace(/^import .*$/gm, '')).not.toMatch(/\bimapManager\./); + }); +}); diff --git a/backend/src/services/draftService.js b/backend/src/services/draftService.js new file mode 100644 index 00000000..9c700a3f --- /dev/null +++ b/backend/src/services/draftService.js @@ -0,0 +1,201 @@ +import { randomBytes as defaultRandomBytes } from 'crypto'; +import { sanitizeSignature } from './emailSanitizer.js'; +import { embedInlineDataImages as defaultEmbedInlineDataImages } from '../utils/inlineImages.js'; +import { mapRecipientList, sanitizeHeaderValue } from './mail/addresses.js'; +import { resolveFromIdentity as defaultResolveFromIdentity } from './mail/identity.js'; +import { + bodyToHtml, + bodyToPlain, + buildMailOptions as defaultBuildMailOptions, + renderRaw as defaultRenderRaw, + sigToPlainText, + textToHtml, +} from './mail/mimeBuilder.js'; + +function serviceError(message, status) { + return Object.assign(new Error(message), { status, expose: true }); +} + +async function resolveDraftsFolder(account, deps) { + const mapped = account.folder_mappings?.drafts; + if (mapped) return mapped; + const result = await deps.query( + "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Drafts' LIMIT 1", + [account.id], + ); + return result.rows[0]?.path || null; +} + +export async function buildRawDraft(input, deps) { + if (!input.account) throw serviceError('Account not found', 404); + + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const embedInlineDataImages = deps.embedInlineDataImages || defaultEmbedInlineDataImages; + const buildMailOptions = deps.buildMailOptions || defaultBuildMailOptions; + const renderRaw = deps.renderRaw || defaultRenderRaw; + const makeRandomBytes = deps.randomBytes || defaultRandomBytes; + + const identity = await resolveFromIdentity( + input.account, + { aliasId: input.aliasId, aliasEmail: input.aliasEmail }, + deps, + ); + + const rawSignature = input.editedSignature !== undefined + ? (input.editedSignature || null) + : identity.signature; + const effectiveSignature = rawSignature ? sanitizeSignature(rawSignature) : null; + const signatureText = effectiveSignature ? sigToPlainText(effectiveSignature) : null; + const bodyText = bodyToPlain(input.body || '', input.bodyIsHtml); + const bodyHtml = bodyToHtml(input.body || '', input.bodyIsHtml); + const rawHtml = bodyHtml + + (effectiveSignature + ? `
${effectiveSignature}
` + : '') + + (input.quotedBodyHtml || (input.quotedBody ? textToHtml(input.quotedBody) : '')); + const embedded = embedInlineDataImages(rawHtml); + + const uploadedAttachments = Array.isArray(input.attachments) + ? input.attachments.map(attachment => ({ + filename: sanitizeHeaderValue(attachment.filename || 'attachment'), + content: Buffer.isBuffer(attachment.content) + ? attachment.content + : Buffer.from(attachment.content || '', 'base64'), + contentType: typeof attachment.contentType === 'string' + ? attachment.contentType + : 'application/octet-stream', + })) + : []; + const attachments = [...embedded.attachments, ...uploadedAttachments]; + + // Stable Message-ID so the appended MIME and the local DB row reference the same message. + const domain = identity.fromEmail.split('@')[1] || 'mailflow.local'; + const messageId = `<${makeRandomBytes(16).toString('hex')}@${domain}>`; + const text = signatureText + ? `${bodyText}\n\n-- \n${signatureText}${input.quotedBody || ''}` + : `${bodyText}${input.quotedBody || ''}`; + const replyTo = input.replyTo !== undefined ? input.replyTo : identity.fromReplyTo; + const mailOptions = buildMailOptions({ + messageId, + fromName: identity.fromName, + fromEmail: identity.fromEmail, + replyTo, + to: Array.isArray(input.to) ? input.to : [input.to], + cc: Array.isArray(input.cc) ? input.cc : [], + bcc: Array.isArray(input.bcc) ? input.bcc : [], + subject: input.subject || '', + priority: input.priority, + text, + html: embedded.html, + inReplyTo: input.inReplyTo, + references: input.references, + attachments, + }); + const rawMessage = await renderRaw(mailOptions); + const snippet = text.replace(/\s+/g, ' ').trim().slice(0, 200); + + // rawHtml (before CID embedding) is what the composer should reopen with so data: + // image URIs remain editable. + return { + rawMessage, + account: input.account, + meta: { + messageId, + fromName: identity.fromName, + fromEmail: identity.fromEmail, + bodyHtml: rawHtml, + bodyText: text, + snippet, + inReplyTo: input.inReplyTo || null, + references: input.references || null, + }, + }; +} + +export async function saveDraft(input, deps) { + const { rawMessage, account, meta } = await buildRawDraft(input, deps); + const draftsFolder = await resolveDraftsFolder(account, deps); + if (!draftsFolder) throw serviceError('No Drafts folder found for this account', 422); + + // APPEND the new draft first so a failed update can never lose the message. + const { uid } = await deps.imapManager.appendToFolder( + account, + draftsFolder, + rawMessage, + ['\\Draft', '\\Seen'], + ); + + // Local persistence is non-fatal because IMAP already holds the new draft. + if (uid != null) { + try { + await deps.imapManager.upsertDraftMessageRecord(account, draftsFolder, uid, { + messageId: meta.messageId, + subject: input.subject, + fromName: meta.fromName, + fromEmail: meta.fromEmail, + to: mapRecipientList(input.to), + cc: mapRecipientList(input.cc), + inReplyTo: meta.inReplyTo, + references: meta.references, + snippet: meta.snippet, + bodyHtml: meta.bodyHtml, + bodyText: meta.bodyText, + }); + } catch (rowErr) { + console.error(`Draft: failed to persist local row uid=${uid}: ${rowErr.message}`); + } + } + + // Delete the old draft only after append and local upsert have completed. + if (input.existingUid && input.existingFolder) { + try { + await deps.imapManager.permanentDeleteMessage(account, input.existingUid, input.existingFolder); + await deps.query( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + [account.id, input.existingUid, input.existingFolder], + ); + } catch (delErr) { + console.error(`Draft: failed to delete old uid=${input.existingUid}: ${delErr.message}`); + } + } + + return { uid, folder: draftsFolder, messageId: meta.messageId }; +} + +export async function deleteDraft({ account, uid, folder }, deps) { + await deps.imapManager.permanentDeleteMessage(account, uid, folder); + await deps.query( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + [account.id, uid, folder], + ); + return { ok: true }; +} + +export async function listDrafts({ account, limit = 50, offset = 0 }, deps) { + const folder = await resolveDraftsFolder(account, deps); + if (!folder) return { drafts: [], total: 0 }; + const safeLimit = Math.min(Math.max(parseInt(limit, 10) || 50, 1), 500); + const safeOffset = Math.max(parseInt(offset, 10) || 0, 0); + const countResult = await deps.query( + 'SELECT COUNT(*)::int AS n FROM messages WHERE account_id = $1 AND folder = $2 AND is_deleted = false', + [account.id, folder], + ); + const result = await deps.query( + `SELECT * FROM messages + WHERE account_id = $1 AND folder = $2 AND is_deleted = false + ORDER BY date DESC + LIMIT $3 OFFSET $4`, + [account.id, folder, safeLimit, safeOffset], + ); + return { drafts: result.rows, total: countResult.rows[0]?.n ?? 0 }; +} + +export async function getDraft({ account, uid, folder }, deps) { + const result = await deps.query( + `SELECT * FROM messages + WHERE account_id = $1 AND uid = $2 AND folder = $3 AND is_deleted = false + LIMIT 1`, + [account.id, uid, folder], + ); + return result.rows[0] || null; +} diff --git a/backend/src/services/draftService.test.js b/backend/src/services/draftService.test.js new file mode 100644 index 00000000..bb32bad1 --- /dev/null +++ b/backend/src/services/draftService.test.js @@ -0,0 +1,282 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + buildRawDraft, + deleteDraft, + getDraft, + listDrafts, + saveDraft, +} from './draftService.js'; +import { sendMessage } from './sendService.js'; + +const account = { + id: 'account-1', + email_address: 'sender@example.com', + name: 'Sender', + sender_name: null, + signature: null, + folder_mappings: {}, +}; + +function draftInput(overrides = {}) { + return { + userId: 'user-1', + account, + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Draft subject', + body: 'Draft body', + bodyIsHtml: false, + ...overrides, + }; +} + +function buildDeps(overrides = {}) { + return { + query: vi.fn(), + imapManager: {}, + resolveFromIdentity: vi.fn().mockResolvedValue({ + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: 'reply@example.com', + signature: null, + aliasId: null, + }), + embedInlineDataImages: vi.fn(html => ({ html, attachments: [] })), + buildMailOptions: vi.fn(options => options), + renderRaw: vi.fn().mockResolvedValue(Buffer.from('raw draft')), + randomBytes: vi.fn(() => Buffer.alloc(16, 3)), + ...overrides, + }; +} + +describe('buildRawDraft', () => { + it('uses the already-scoped account and carries reply, attachment, and priority fields into MIME', async () => { + const deps = buildDeps(); + + const result = await buildRawDraft(draftInput({ + aliasId: 'alias-1', + inReplyTo: '', + references: ' ', + replyTo: 'explicit-reply@example.com', + priority: 'high', + attachments: [{ + filename: 'draft.txt', + content: Buffer.from('draft attachment').toString('base64'), + contentType: 'text/plain', + }], + }), deps); + + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasId: 'alias-1', aliasEmail: undefined }, + deps, + ); + expect(deps.query).not.toHaveBeenCalled(); + expect(deps.buildMailOptions).toHaveBeenCalledWith(expect.objectContaining({ + replyTo: 'explicit-reply@example.com', + inReplyTo: '', + references: ' ', + priority: 'high', + attachments: [expect.objectContaining({ + filename: 'draft.txt', + content: Buffer.from('draft attachment'), + })], + })); + expect(result).toMatchObject({ + rawMessage: Buffer.from('raw draft'), + account, + meta: { + fromName: 'Sender', + fromEmail: 'sender@example.com', + bodyHtml: expect.stringContaining('Draft body'), + bodyText: expect.stringContaining('Draft body'), + inReplyTo: '', + references: ' ', + }, + }); + expect(result.meta.messageId).toMatch(/^<03030303.+@example\.com>$/); + }); +}); + +describe('saveDraft', () => { + it('persists reply threading metadata unchanged so a reopened draft can send in the same thread', async () => { + const inReplyTo = ''; + const references = ' '; + let persistedRow; + const imapManager = { + appendToFolder: vi.fn().mockResolvedValue({ uid: 5 }), + upsertDraftMessageRecord: vi.fn(async (_account, _folder, uid, meta) => { + persistedRow = { + uid, + in_reply_to: meta.inReplyTo, + thread_references: meta.references, + }; + }), + permanentDeleteMessage: vi.fn(), + }; + const query = vi.fn(async (sql) => { + if (sql.includes("special_use = '\\Drafts'")) return { rows: [{ path: 'Drafts' }] }; + if (sql.includes('SELECT * FROM messages')) return { rows: [persistedRow] }; + return { rows: [] }; + }); + const deps = buildDeps({ imapManager, query }); + + await saveDraft(draftInput({ inReplyTo, references }), deps); + const reopened = await getDraft({ account, uid: 5, folder: 'Drafts' }, deps); + + expect(reopened).toMatchObject({ + in_reply_to: inReplyTo, + thread_references: references, + }); + + const delivered = []; + const makeSendDeps = () => { + const transport = { + sendMail: vi.fn(async mailOptions => { delivered.push(mailOptions); }), + }; + return { + query: vi.fn(), + imapManager: {}, + resolveFromIdentity: vi.fn().mockResolvedValue({ + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: 'reply@example.com', + signature: null, + aliasId: null, + }), + buildSmtpTransport: vi.fn().mockResolvedValue({ transport, account }), + buildMailOptions: vi.fn(options => options), + renderRaw: vi.fn().mockResolvedValue(Buffer.from('raw mime')), + embedInlineDataImages: vi.fn(html => ({ html, attachments: [] })), + learnSentRecipients: vi.fn(), + resolveSentFolder: vi.fn().mockResolvedValue('Sent'), + persistSentCopy: vi.fn().mockResolvedValue({ sentCopySaved: true }), + randomBytes: vi.fn(() => Buffer.alloc(16, 1)), + }; + }; + + await sendMessage(draftInput({ inReplyTo, references }), makeSendDeps()); + await sendMessage(draftInput({ + inReplyTo: reopened.in_reply_to, + references: reopened.thread_references, + }), makeSendDeps()); + + expect(delivered).toHaveLength(2); + expect({ + inReplyTo: delivered[1].inReplyTo, + references: delivered[1].references, + }).toEqual({ + inReplyTo: delivered[0].inReplyTo, + references: delivered[0].references, + }); + }); + + it('keeps append → upsert → delete-old ordering for crash safety', async () => { + const events = []; + const imapManager = { + appendToFolder: vi.fn(async () => { + events.push('append'); + return { uid: 5 }; + }), + upsertDraftMessageRecord: vi.fn(async () => { events.push('upsert'); }), + permanentDeleteMessage: vi.fn(async () => { events.push('delete-imap'); }), + }; + const query = vi.fn(async (sql) => { + if (sql.includes("special_use = '\\Drafts'")) return { rows: [{ path: 'Drafts' }] }; + if (sql.startsWith('DELETE FROM messages')) events.push('delete-db'); + return { rows: [] }; + }); + const deps = buildDeps({ imapManager, query }); + + const result = await saveDraft(draftInput({ + existingUid: 4, + existingFolder: 'Drafts', + }), deps); + + expect(events).toEqual(['append', 'upsert', 'delete-imap', 'delete-db']); + expect(imapManager.upsertDraftMessageRecord).toHaveBeenCalledWith( + account, + 'Drafts', + 5, + expect.objectContaining({ + subject: 'Draft subject', + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + }), + ); + expect(result).toMatchObject({ uid: 5, folder: 'Drafts' }); + expect(result.messageId).toMatch(/^<03030303.+@example\.com>$/); + }); + + it('keeps local persistence non-fatal after a successful append', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + const imapManager = { + appendToFolder: vi.fn().mockResolvedValue({ uid: 5 }), + upsertDraftMessageRecord: vi.fn().mockRejectedValue(new Error('db down')), + permanentDeleteMessage: vi.fn(), + }; + const deps = buildDeps({ + imapManager, + query: vi.fn().mockResolvedValue({ rows: [{ path: 'Drafts' }] }), + }); + + await expect(saveDraft(draftInput(), deps)).resolves.toMatchObject({ uid: 5, folder: 'Drafts' }); + }); + + it('does not upsert without a reliable uid and errors when no Drafts folder resolves', async () => { + const imapManager = { + appendToFolder: vi.fn().mockResolvedValue({ uid: null }), + upsertDraftMessageRecord: vi.fn(), + permanentDeleteMessage: vi.fn(), + }; + const deps = buildDeps({ + imapManager, + query: vi.fn().mockResolvedValue({ rows: [{ path: 'Drafts' }] }), + }); + await saveDraft(draftInput(), deps); + expect(imapManager.upsertDraftMessageRecord).not.toHaveBeenCalled(); + + const missingDeps = buildDeps({ + imapManager, + query: vi.fn().mockResolvedValue({ rows: [] }), + }); + await expect(saveDraft(draftInput(), missingDeps)).rejects.toMatchObject({ + message: 'No Drafts folder found for this account', + status: 422, + expose: true, + }); + }); +}); + +describe('draft persistence helpers', () => { + it('deletes from IMAP before removing the local row', async () => { + const events = []; + const deps = { + imapManager: { + permanentDeleteMessage: vi.fn(async () => { events.push('imap'); }), + }, + query: vi.fn(async () => { + events.push('db'); + return { rows: [] }; + }), + }; + await expect(deleteDraft({ account, uid: 9, folder: 'Drafts' }, deps)).resolves.toEqual({ ok: true }); + expect(events).toEqual(['imap', 'db']); + }); + + it('lists and gets drafts from the already-scoped account', async () => { + const query = vi.fn() + .mockResolvedValueOnce({ rows: [{ n: 1 }] }) + .mockResolvedValueOnce({ rows: [{ uid: 7, subject: 'Draft' }] }) + .mockResolvedValueOnce({ rows: [{ uid: 7, subject: 'Draft' }] }); + + const mappedAccount = { ...account, folder_mappings: { drafts: 'Drafts' } }; + await expect(listDrafts({ account: mappedAccount, limit: 10, offset: 0 }, { query })) + .resolves.toEqual({ drafts: [{ uid: 7, subject: 'Draft' }], total: 1 }); + await expect(getDraft({ account: mappedAccount, uid: 7, folder: 'Drafts' }, { query })) + .resolves.toEqual({ uid: 7, subject: 'Draft' }); + + expect(query.mock.calls[0][1][0]).toBe('account-1'); + expect(query.mock.calls[2][1]).toEqual(['account-1', 7, 'Drafts']); + }); +}); diff --git a/backend/src/services/emailSanitizer.js b/backend/src/services/emailSanitizer.js index 54b904c9..0b16917f 100644 --- a/backend/src/services/emailSanitizer.js +++ b/backend/src/services/emailSanitizer.js @@ -1,5 +1,10 @@ import sanitizeHtml from 'sanitize-html'; +// Reject strings that contain characters that could inject extra email headers. +export function hasHeaderInjectionChars(str) { + return typeof str === 'string' && /[\r\n\0]/.test(str); +} + // Strip the element from email HTML, preserving any