diff --git a/.gitignore b/.gitignore index 97a8fe0..de9fce7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ Thumbs.db .superpowers/ output/ + +# Search-eval query cache — paraphrases of real mail; the harness regenerates it diff --git a/README.md b/README.md index 0f5c1c0..4188a1b 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,14 @@ docker compose up -d To pin to a specific version instead of `latest`, add `MAILFLOW_VERSION=1.9.0` to your `.env`. +> **Upgrading an existing install across the Postgres image change** (`postgres:16-alpine` → `pgvector/pgvector:pg16`): the new image uses a different C library (musl → glibc), which changes text collation order. Reindex once after the switch or text indexes can silently return wrong results — the backend also prints this warning at startup when it detects the mismatch: +> +> ```bash +> docker compose exec postgres psql -U mailflow -d mailflow \ +> -c 'REINDEX DATABASE "mailflow";' \ +> -c 'ALTER DATABASE "mailflow" REFRESH COLLATION VERSION;' +> ``` + --- ## Option B — Build from source diff --git a/backend/migrations/0035_search_fts.sql b/backend/migrations/0035_search_fts.sql new file mode 100644 index 0000000..519c213 --- /dev/null +++ b/backend/migrations/0035_search_fts.sql @@ -0,0 +1,55 @@ +-- Weighted full-text search column (search_fts) + version stamp, kept fresh by +-- a BEFORE trigger. Fast, metadata-only DDL only (README D1): the nullable +-- columns force no table rewrite on PG16, and the function/trigger are instant. +-- Pre-existing rows are populated by the resumable drainer (ftsBackfill.js); +-- the GIN index is built CONCURRENTLY in the separate no-transaction migration +-- 0037 (a $$-quoted plpgsql body cannot survive the no-transaction ; splitter, +-- so the trigger and the CONCURRENTLY index MUST live in different files). +-- +-- The setweight(...) expression MUST stay identical to +-- lexicalRepo.searchFtsExpr('NEW'); fts_version = 1 matches lexicalRepo.FTS_VERSION. +-- Idempotent (IF NOT EXISTS / CREATE OR REPLACE / DROP IF EXISTS) because a +-- crash before the schema_migrations INSERT retries this migration. + +ALTER TABLE messages ADD COLUMN IF NOT EXISTS search_fts tsvector; +ALTER TABLE messages ADD COLUMN IF NOT EXISTS fts_version int; + +CREATE OR REPLACE FUNCTION messages_search_fts_refresh() RETURNS trigger AS $$ +BEGIN + -- Skip recompute on an UPDATE that changes none of the indexed source + -- columns (read/star flag flips, snippet-only writes), so the trigger does + -- not tax hot sync UPSERTs; this also lets the backfill's explicit SET win. + IF TG_OP = 'UPDATE' + AND NEW.subject IS NOT DISTINCT FROM OLD.subject + AND NEW.from_name IS NOT DISTINCT FROM OLD.from_name + AND NEW.from_email IS NOT DISTINCT FROM OLD.from_email + AND NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses + AND NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses + AND NEW.body_text IS NOT DISTINCT FROM OLD.body_text + THEN + RETURN NEW; + END IF; + + BEGIN + NEW.search_fts := setweight(to_tsvector('english', coalesce(NEW.subject,'')), 'A') || + setweight(to_tsvector('english', coalesce(NEW.from_name,'') || ' ' || coalesce(NEW.from_email,'')), 'B') || + setweight(to_tsvector('english', coalesce(NEW.to_addresses::text,'') || ' ' || coalesce(NEW.cc_addresses::text,'')), 'C') || + setweight(to_tsvector('english', LEFT(coalesce(NEW.body_text,''), 600000)), 'D'); + NEW.fts_version := 1; + EXCEPTION WHEN program_limit_exceeded THEN + -- Even with the 600k LEFT cap, a pathologically dense/multibyte body can + -- exceed Postgres's ~1MB tsvector limit (SQLSTATE 54000). Never fail the + -- row write: leave search_fts NULL so the message still persists and stays + -- findable via the ILIKE fallback; the backfill's row-by-row skip stamps it. + NEW.search_fts := NULL; + NEW.fts_version := NULL; + END; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_messages_search_fts ON messages; +CREATE TRIGGER trg_messages_search_fts + BEFORE INSERT OR UPDATE ON messages + FOR EACH ROW EXECUTE FUNCTION messages_search_fts_refresh(); diff --git a/backend/migrations/0036_background_jobs.sql b/backend/migrations/0036_background_jobs.sql new file mode 100644 index 0000000..d189933 --- /dev/null +++ b/backend/migrations/0036_background_jobs.sql @@ -0,0 +1,19 @@ +-- Generic progress substrate for observable background drainers (first consumer: +-- the FTS backfill). Plain table, fast DDL. One row per (kind, account); global +-- jobs use a NULL account_id, COALESCE'd to '' in the unique index so the upsert +-- has a single conflict target for both cases. + +CREATE TABLE IF NOT EXISTS background_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + kind VARCHAR(64) NOT NULL, + account_id UUID REFERENCES email_accounts(id) ON DELETE CASCADE, + state VARCHAR(20) NOT NULL DEFAULT 'idle', + processed BIGINT NOT NULL DEFAULT 0, + total BIGINT NOT NULL DEFAULT 0, + last_error TEXT, + started_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_background_jobs_kind_account + ON background_jobs (kind, COALESCE(account_id::text, '')); diff --git a/backend/migrations/0037_search_fts_index.sql b/backend/migrations/0037_search_fts_index.sql new file mode 100644 index 0000000..f82b7db --- /dev/null +++ b/backend/migrations/0037_search_fts_index.sql @@ -0,0 +1,24 @@ +-- no-transaction +-- +-- GIN index that serves `search_fts @@ tsquery`, plus a partial btree that lets +-- the backfill drainer (and any "needs backfill" probe) find not-yet-stamped +-- rows without a full seq scan; it self-prunes as fts_version = 1 fills in. +-- CONCURRENTLY must run outside a transaction, so these live apart from 0035's +-- trigger. Idempotent via IF NOT EXISTS (retried if a crash precedes the +-- schema_migrations INSERT). +-- +-- DROP ... IF EXISTS before each CREATE: a cancelled or crashed CREATE INDEX +-- CONCURRENTLY leaves an INVALID index under the target name. On retry, plain +-- IF NOT EXISTS sees that name and silently skips the create, recording the +-- migration as done while the scan stays unindexed forever. This file only re-runs +-- after such a failure — in which case the index is either absent (drop is a no-op) +-- or invalid (drop removes the dead stub) — so dropping first is safe and cheap. + +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_search_fts; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts + ON messages USING GIN (search_fts); + +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_fts_stale_v1; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_fts_stale_v1 + ON messages (date DESC) + WHERE fts_version IS DISTINCT FROM 1; diff --git a/backend/migrations/0038_embed_watermark.sql b/backend/migrations/0038_embed_watermark.sql new file mode 100644 index 0000000..ecf8214 --- /dev/null +++ b/backend/migrations/0038_embed_watermark.sql @@ -0,0 +1,44 @@ +-- Vector-substrate CAS columns + last_modified trigger (slice 04). +-- Extension-independent, fast DDL only. NO vector-typed DDL here — the +-- embeddings/index_generations/embed_watermark/embed_runs tables and the HNSW +-- index are created by ensureVectorSchema() at startup (README invariant). +-- Transactional migration (NOT -- no-transaction): the no-transaction runner +-- splits on ';' and would break the dollar-quoted function below. + +-- last_modified: content-change CAS token the embed worker compares to detect a +-- late-arriving body invalidating a stale subject-only embedding. NOT NULL DEFAULT +-- now() is metadata-only on PG16 (now() is STABLE → one stored missing-value, no +-- table rewrite). +ALTER TABLE messages ADD COLUMN IF NOT EXISTS last_modified TIMESTAMPTZ NOT NULL DEFAULT now(); + +-- embed_gen: the index generation this row is embedded under. NULL = needs embedding. +-- Plain BIGINT soft-stamp (no FK): a generation can be retired/deleted while stamps linger. +ALTER TABLE messages ADD COLUMN IF NOT EXISTS embed_gen BIGINT; + +-- Bump last_modified AND clear embed_gen when an embedding-input column changes. The +-- WHEN clause filters at the C level so unchanged-content UPDATEs (the hot re-sync path) +-- and stamp-only UPDATEs (the worker setting embed_gen) skip the function entirely. +-- Clearing embed_gen is what re-surfaces a late-arriving body: a row embedded +-- subject-only and stamped, whose body later lands (phase-2 drainer / on-open fetch), +-- has its stamp cleared here so the NULL-only embed scan re-finds it and the idempotent +-- upsert replaces the stale chunks. The CAS only guards the read→stamp window; this +-- trigger covers the post-stamp case (Mailflow's late-arriving bodies — msgvault's rows +-- are immutable after ingest, so it never needed this). Together with createGeneration's +-- stamp-reset (which handles generation rebuilds: unchanged content, new fingerprint), +-- the invariant is exact: embed_gen IS NULL ⟺ the row needs embedding. +CREATE OR REPLACE FUNCTION messages_bump_last_modified() RETURNS trigger AS $$ +BEGIN + NEW.last_modified := now(); + NEW.embed_gen := NULL; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_messages_last_modified ON messages; +CREATE TRIGGER trg_messages_last_modified + BEFORE UPDATE ON messages + FOR EACH ROW + WHEN (NEW.subject IS DISTINCT FROM OLD.subject + OR NEW.body_text IS DISTINCT FROM OLD.body_text + OR NEW.body_html IS DISTINCT FROM OLD.body_html) + EXECUTE FUNCTION messages_bump_last_modified(); diff --git a/backend/migrations/0039_embed_pending_index.sql b/backend/migrations/0039_embed_pending_index.sql new file mode 100644 index 0000000..abf9a87 --- /dev/null +++ b/backend/migrations/0039_embed_pending_index.sql @@ -0,0 +1,17 @@ +-- no-transaction +-- Partial index over the embed-scan's steady-state hot predicate. Once a generation +-- reaches full coverage, the only live rows still needing work are newly-arrived +-- messages with embed_gen IS NULL, so the 60s scheduler scan (scanForEmbedding) should +-- touch O(pending), not O(mailbox). Built CONCURRENTLY (hence -- no-transaction) so it +-- never blocks boot on a large messages table; extension-independent and cheap to +-- maintain (only the sparse NULL set is indexed). Idempotent (IF NOT EXISTS) because a +-- crash before the schema_migrations INSERT retries the migration. +-- +-- DROP ... IF EXISTS before the CREATE: a cancelled or crashed CREATE INDEX +-- CONCURRENTLY leaves an INVALID index under this name, and plain IF NOT EXISTS would +-- then silently skip the create on retry — recording the migration as done while the +-- embed scan stays unindexed forever. This file only re-runs after such a failure, so +-- the index is either absent (drop is a no-op) or invalid (drop clears the dead stub). +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending + ON messages (id) WHERE embed_gen IS NULL; diff --git a/backend/migrations/0040_api_tokens.sql b/backend/migrations/0040_api_tokens.sql new file mode 100644 index 0000000..df17efa --- /dev/null +++ b/backend/migrations/0040_api_tokens.sql @@ -0,0 +1,12 @@ +-- MCP API tokens. We store only a SHA-256 hash of the token; the plaintext is +-- shown to the operator exactly once at mint time and is never recoverable. +CREATE TABLE IF NOT EXISTS api_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_used_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_api_tokens_user_id ON api_tokens(user_id); diff --git a/backend/migrations/0041_mcp_deletion_batches.sql b/backend/migrations/0041_mcp_deletion_batches.sql new file mode 100644 index 0000000..cf3b77c --- /dev/null +++ b/backend/migrations/0041_mcp_deletion_batches.sql @@ -0,0 +1,21 @@ +-- Staged (not executed) MCP deletions. stage_deletion records a batch; a separate +-- session-authenticated execute step flips messages.is_deleted (soft delete). No +-- tool ever hard-deletes. Renumbered to 0041 (0040 is api_tokens) per the README +-- migration-numbering rule. +CREATE TABLE IF NOT EXISTS mcp_deletion_batches ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + description TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'staged', + message_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + executed_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS mcp_deletion_batch_messages ( + batch_id UUID NOT NULL REFERENCES mcp_deletion_batches(id) ON DELETE CASCADE, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + PRIMARY KEY (batch_id, message_id) +); + +CREATE INDEX IF NOT EXISTS idx_mcp_deletion_batches_user_id ON mcp_deletion_batches(user_id); diff --git a/backend/package-lock.json b/backend/package-lock.json index e216e59..90e8746 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -1,13 +1,14 @@ { "name": "mailflow-backend", - "version": "2.4.1", + "version": "2.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mailflow-backend", - "version": "2.4.1", + "version": "2.6.0", "dependencies": { + "@modelcontextprotocol/sdk": "^1.29.0", "archiver": "^7.0.1", "bcryptjs": "^2.4.3", "connect-redis": "^7.1.0", @@ -262,6 +263,18 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -431,6 +444,398 @@ "dev": true, "license": "MIT" }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -1116,6 +1521,45 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2269,6 +2713,27 @@ "bare-events": "^2.7.0" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -2334,6 +2799,24 @@ "express": "^4.16.2" } }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/express-session": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.19.0.tgz", @@ -2361,7 +2844,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -2384,6 +2866,22 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-uri": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", + "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-xml-builder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", @@ -2708,6 +3206,15 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.12.30", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", + "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/htmlparser2": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", @@ -2954,6 +3461,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -3028,6 +3541,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -3697,6 +4216,15 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3990,6 +4518,15 @@ "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", "license": "MIT" }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/pngjs": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", @@ -4273,6 +4810,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -4313,6 +4859,55 @@ "@rolldown/binding-win32-x64-msvc": "1.0.3" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/router/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/router/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/router/node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -5163,6 +5758,12 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/ws": { "version": "8.21.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", @@ -5281,6 +5882,24 @@ "engines": { "node": ">= 14" } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/backend/package.json b/backend/package.json index 5864a14..d041a3f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,6 +10,7 @@ "lint": "eslint src --max-warnings 0" }, "dependencies": { + "@modelcontextprotocol/sdk": "^1.29.0", "archiver": "^7.0.1", "bcryptjs": "^2.4.3", "connect-redis": "^7.1.0", diff --git a/backend/scripts/bench-last-modified-trigger.js b/backend/scripts/bench-last-modified-trigger.js new file mode 100644 index 0000000..4593994 --- /dev/null +++ b/backend/scripts/bench-last-modified-trigger.js @@ -0,0 +1,45 @@ +// Benchmark the last_modified trigger overhead on the hot re-sync UPSERT. +// Usage: node scripts/bench-last-modified-trigger.js [rows=5000] [iters=20000] +// Requires DB_* env pointing at a migrated Postgres (trigger present). +import { pool } from '../src/services/db.js'; +import { seedAccount, cleanupAccount } from '../src/services/embeddings/testSupport.js'; + +const ROWS = Number(process.argv[2]) || 5000; +const ITERS = Number(process.argv[3]) || 20000; + +const { accountId: acctId, userId } = await seedAccount(pool, 'bench'); +const ids = []; +for (let i = 0; i < ROWS; i++) { + const r = await pool.query(`INSERT INTO messages (account_id, uid, folder, subject, is_read) + VALUES ($1, $2, 'INBOX', 'bench', false) RETURNING id`, [acctId, 700000 + i]); + ids.push(r.rows[0].id); +} + +// Flag-only churn: toggles is_read (NOT an embedding-input column) so the trigger's +// WHEN clause should skip the function entirely. +async function churn() { + const t0 = process.hrtime.bigint(); + for (let i = 0; i < ITERS; i++) { + const id = ids[i % ids.length]; + await pool.query('UPDATE messages SET is_read = NOT is_read WHERE id = $1', [id]); + } + return Number(process.hrtime.bigint() - t0) / 1e6; // ms +} + +const withTrig = await churn(); +await pool.query('DROP TRIGGER IF EXISTS trg_messages_last_modified ON messages'); +const withoutTrig = await churn(); +// Restore the trigger. +await pool.query(`CREATE TRIGGER trg_messages_last_modified + BEFORE UPDATE ON messages FOR EACH ROW + WHEN (NEW.subject IS DISTINCT FROM OLD.subject OR NEW.body_text IS DISTINCT FROM OLD.body_text OR NEW.body_html IS DISTINCT FROM OLD.body_html) + EXECUTE FUNCTION messages_bump_last_modified()`); + +await cleanupAccount(pool, userId); +await pool.end(); + +const regression = ((withTrig - withoutTrig) / withoutTrig) * 100; +console.log(`with trigger: ${withTrig.toFixed(0)} ms`); +console.log(`without trigger: ${withoutTrig.toFixed(0)} ms`); +console.log(`regression: ${regression.toFixed(1)}% (gate: < 10%)`); +process.exit(regression < 10 ? 0 : 2); diff --git a/backend/scripts/search-eval.mjs b/backend/scripts/search-eval.mjs new file mode 100644 index 0000000..a4ce1cb --- /dev/null +++ b/backend/scripts/search-eval.mjs @@ -0,0 +1,453 @@ +#!/usr/bin/env node +// search-eval.mjs — IR relevance eval harness for Mailflow's lexical/vector/hybrid search. +// +// WHY THIS EXISTS +// A user reported they "can't tell the difference" between search modes on their real +// 18k-message mailbox and suspected hybrid might be *worse* than pure vector. Phase 4's +// rankingQuality.test.js proved hybrid never LOSES a lexical hit on a synthetic 16-doc +// fixture; this harness is the complementary real-corpus measurement: it builds labeled +// query sets FROM the live mailbox, runs them through the deployed REST API in all three +// modes, and reports the standard IR metrics (Recall@1/5/20, MRR@20) plus cross-mode +// result overlap and hybrid explain-score composition — so the "is hybrid working?" +// question is answered with numbers instead of vibes. +// +// DESIGN +// * No new deps. Node >=18 global fetch; psql sampling via `docker exec` (the same +// read-only path the eval brief documents); explain scores via one `docker exec ... +// node` pass against the in-container searchService seam (the deployed REST route does +// not plumb `explain` through, but the seam it calls does). +// * Deterministic. Message sampling is ordered by md5(id || seed); paraphrase queries +// are cached to JSON keyed by {messageId, promptVersion}, so reruns are free & stable. +// * Two query sets, both with a single ground-truth message id: +// KEYWORD — 2 distinctive subject tokens (lexical should win/tie). +// PARAPHRASE — an LLM rewrites the email's topic WITHOUT its distinctive keywords +// (semantic recall test; degrades to hand-written queries if the LLM +// is unavailable). +// +// USAGE +// EVAL_USER='admin@example.com' \ # login username (no default) +// EVAL_PASS='' \ +// OPENAI_API_KEY="$(cat /path/to/key)" \ # or EVAL_KEY_FILE=/path/to/key +// node backend/scripts/search-eval.mjs +// +// Reruns after the first are offline for query generation (cache hit) but still hit the +// live REST API for the actual searches. Set EVAL_SKIP_EXPLAIN=1 to skip the in-container +// diagnostic. All knobs are env vars (see CFG below). No secrets are written to disk. + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const EVAL_DIR = resolve(__dirname, '..', '..', 'specs', 'search-overhaul', 'evals'); + +const PROMPT_VERSION = 'v1'; // bump to invalidate the paraphrase cache + +function readKeyFile() { + const f = process.env.EVAL_KEY_FILE; + if (f && existsSync(f)) return readFileSync(f, 'utf8'); + return ''; +} + +const CFG = { + baseUrl: process.env.EVAL_BASE_URL || 'http://127.0.0.1:8087', + loginUser: process.env.EVAL_USER || '', // required for a fresh login; never defaulted + loginPass: process.env.EVAL_PASS || '', // required for a fresh login; never defaulted + pgContainer: process.env.EVAL_PG_CONTAINER || 'mailflow-postgres', + pgUser: process.env.EVAL_PG_USER || 'mailflow', + pgDb: process.env.EVAL_PG_DB || 'mailflow', + backendContainer: process.env.EVAL_BACKEND_CONTAINER || 'mailflow-backend', + openaiBase: process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1', + openaiModel: process.env.OPENAI_MODEL || 'gpt-4o-mini', + openaiKey: (process.env.OPENAI_API_KEY || readKeyFile()).trim(), + seed: process.env.EVAL_SEED || 'mailflow-eval-2026-07-16', + nKeyword: Number(process.env.EVAL_N_KEYWORD || 15), + nParaphrase: Number(process.env.EVAL_N_PARAPHRASE || 25), + limit: 20, + spacingMs: Number(process.env.EVAL_REQUEST_SPACING_MS || 3300), // stay just under 20/min + llmSpacingMs: Number(process.env.EVAL_LLM_SPACING_MS || 3000), // pace flaky low-tier keys + generateOnly: process.env.EVAL_GENERATE_ONLY === '1', // populate the cache, then exit + skipExplain: process.env.EVAL_SKIP_EXPLAIN === '1', + cacheFile: resolve(EVAL_DIR, 'query-cache.json'), + outFile: process.env.EVAL_OUT || resolve(EVAL_DIR, `results-${new Date().toISOString().slice(0, 10)}.json`), +}; + +const MODES = ['lexical', 'vector', 'hybrid']; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// ── DB sampling (docker exec psql, read-only) ────────────────────────────────────────── +const FSEP = '\x1f'; +function psql(sql) { + const out = execFileSync( + 'docker', + ['exec', CFG.pgContainer, 'psql', '-U', CFG.pgUser, '-d', CFG.pgDb, '-tAF', FSEP, '-c', sql], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + return out.split('\n').filter((l) => l.length > 0).map((l) => l.split(FSEP)); +} + +function resolveScope() { + const rows = psql( + `SELECT u.id, (SELECT string_agg(id::text, ',') FROM email_accounts WHERE user_id=u.id AND enabled=true) + FROM users u WHERE u.username='${CFG.loginUser.replace(/'/g, "''")}'`, + ); + if (!rows.length) throw new Error(`no user ${CFG.loginUser}`); + const [userId, accts] = rows[0]; + return { userId, accountIds: (accts || '').split(',').filter(Boolean) }; +} + +// Sanitize subject/snippet in SQL so newlines/separators never break TSV row parsing. +const CLEAN = (col) => `regexp_replace(coalesce(${col},''), '\\s+', ' ', 'g')`; + +function sampleMessages({ salt, where, n }) { + const acctList = SCOPE.accountIds.map((a) => `'${a}'`).join(','); + return psql( + `SELECT id, ${CLEAN('subject')}, ${CLEAN('snippet')}, coalesce(from_name,'') + FROM messages m + WHERE m.is_deleted=false AND m.account_id IN (${acctList}) AND ${where} + ORDER BY md5(m.id::text || '${salt.replace(/'/g, "''")}') + LIMIT ${n}`, + ).map(([id, subject, snippet, fromName]) => ({ id, subject, snippet, fromName })); +} + +// ── KEYWORD query derivation ─────────────────────────────────────────────────────────── +const STOP = new Set([ + 'the', 'and', 'for', 'you', 'your', 'with', 'from', 'this', 'that', 'have', 'has', 'was', + 'are', 'will', 'not', 'new', 'can', 'all', 'our', 'out', 'get', 'now', 'about', 'been', + 'account', 'email', 'please', 'update', 'updated', 'notification', 're', 'fwd', 'fw', + 'order', 'invoice', 'payment', 'confirm', 'confirmation', 'reminder', 'receipt', 'alert', + 'security', 'verify', 'verification', 'here', 'more', 'just', 'been', 'they', 'them', + 'default', 'routing', 'transaction', 'needs', 'sent', 'may', 'copy', 'left', 'comment', +]); +function keywordTokens(subject) { + const toks = (subject.toLowerCase().match(/[a-z][a-z0-9'-]{3,}/g) || []) + .map((t) => t.replace(/^[-']+|[-']+$/g, '')) + .filter((t) => t.length >= 4 && !STOP.has(t)); + const uniq = [...new Set(toks)].sort((a, b) => b.length - a.length); + return uniq.slice(0, 2); +} + +function buildKeywordSet() { + const cands = sampleMessages({ + salt: `${CFG.seed}:kw`, + where: `length(coalesce(m.subject,'')) BETWEEN 10 AND 160`, + n: CFG.nKeyword * 4, + }); + const set = []; + for (const m of cands) { + if (set.length >= CFG.nKeyword) break; + const toks = keywordTokens(m.subject); + if (toks.length < 2) continue; + set.push({ id: m.id, set: 'keyword', query: toks.join(' '), subject: m.subject }); + } + return set; +} + +// ── PARAPHRASE query generation (LLM, cached) ────────────────────────────────────────── +async function openaiParaphrase(subject, snippet) { + const body = { + model: CFG.openaiModel, + temperature: 0.3, + max_tokens: 40, + messages: [ + { role: 'system', content: 'You write realistic email-search queries the way a busy person would type them months later from memory.' }, + { role: 'user', content: + `Email subject: ${subject}\nEmail preview: ${snippet}\n\n` + + 'Write ONE short natural-language search query (4-9 words) that a person might type to re-find THIS email later. ' + + 'Describe its topic or purpose in everyday words. Do NOT reuse the distinctive names, brands, product names, codes, or rare keywords from the subject — paraphrase them with common synonyms. ' + + 'Output only the query text, no quotes, no punctuation at the end.' }, + ], + }; + // Low-tier keys rate-limit bursts (429), sometimes even reporting it as a quota error; + // back off (3s,6s,12s,24s) and retry before giving up to the hand-written fallback. + const backoff = [3000, 6000, 12000, 24000]; + for (let attempt = 0; attempt <= backoff.length; attempt++) { + const res = await fetch(`${CFG.openaiBase}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${CFG.openaiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (res.status === 429 && attempt < backoff.length) { await sleep(backoff[attempt]); continue; } + if (!res.ok) throw new Error(`openai ${res.status}: ${(await res.text()).slice(0, 120)}`); + const j = await res.json(); + return (j.choices?.[0]?.message?.content || '').trim().replace(/^["']|["']$/g, ''); + } + throw new Error('openai 429: retries exhausted'); +} + +// Hand-written fallbacks, only used if the LLM is unavailable AND nothing is cached. +function fallbackParaphrase(subject) { + const t = keywordTokens(subject); + return t.length ? `email about ${t.join(' and ')}` : 'that email i got recently'; +} + +async function buildParaphraseSet(cache) { + const cands = sampleMessages({ + salt: `${CFG.seed}:para`, + where: `length(coalesce(m.subject,'')) BETWEEN 12 AND 160 AND m.snippet IS NOT NULL AND length(m.snippet) >= 40`, + n: CFG.nParaphrase, + }); + const set = []; + let generated = 0; let usedFallback = 0; + for (const m of cands) { + const key = `${m.id}:${PROMPT_VERSION}`; + let query = cache[key]?.query; + if (!query) { + if (CFG.openaiKey) { + try { + query = await openaiParaphrase(m.subject, m.snippet); + generated++; + await sleep(CFG.llmSpacingMs); // pace LLM calls to avoid burst rate-limits on low-tier keys + } catch (err) { + console.warn(` paraphrase LLM failed for ${m.id.slice(0, 8)}: ${err.message}`); + } + } + if (!query) { query = fallbackParaphrase(m.subject); usedFallback++; } + cache[key] = { query, subject: m.subject, source: generated && query !== fallbackParaphrase(m.subject) ? 'llm' : 'fallback' }; + } + set.push({ id: m.id, set: 'paraphrase', query, subject: m.subject }); + } + // Tally the provenance of every query actually used (cache may hold llm/handwritten/fallback). + const sourceCounts = {}; + for (const m of cands) { + const s = cache[`${m.id}:${PROMPT_VERSION}`]?.source || 'unknown'; + sourceCounts[s] = (sourceCounts[s] || 0) + 1; + } + if (generated || usedFallback) console.log(` paraphrases: ${generated} generated, ${usedFallback} fallback, ${set.length - generated - usedFallback} cached`); + console.log(` paraphrase sources: ${JSON.stringify(sourceCounts)}`); + return { set, usedFallback, sourceCounts }; +} + +// ── Live REST search (session + 429-aware throttling) ────────────────────────────────── +let COOKIE = ''; +async function login() { + if (!CFG.loginUser) throw new Error('EVAL_USER is required for a fresh login (no default; local test creds)'); + if (!CFG.loginPass) throw new Error('EVAL_PASS is required for a fresh login (no default; local test creds)'); + const res = await fetch(`${CFG.baseUrl}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify({ username: CFG.loginUser, password: CFG.loginPass }), + }); + if (!res.ok) throw new Error(`login ${res.status}`); + const set = res.headers.getSetCookie?.() || []; + COOKIE = set.map((c) => c.split(';')[0]).join('; '); + if (!COOKIE) throw new Error('login returned no session cookie'); +} + +async function restSearch(query, mode) { + const url = `${CFG.baseUrl}/api/search/?q=${encodeURIComponent(query)}&mode=${mode}&limit=${CFG.limit}`; + for (let attempt = 0; attempt < 6; attempt++) { + const res = await fetch(url, { headers: { Cookie: COOKIE, 'X-Requested-With': 'XMLHttpRequest' } }); + if (res.status === 429) { + const retry = Number(res.headers.get('retry-after') || 5); + console.log(` 429 rate-limited, sleeping ${retry + 1}s`); + await sleep((retry + 1) * 1000); + continue; + } + if (!res.ok) throw new Error(`search ${res.status} for "${query}" (${mode})`); + return res.json(); + } + throw new Error(`search gave up after retries: "${query}" (${mode})`); +} + +// ── Metrics ──────────────────────────────────────────────────────────────────────────── +function rankOf(ids, targetId) { + const i = ids.indexOf(targetId); + return i < 0 ? null : i + 1; // 1-indexed +} +function jaccard(a, b) { + const sa = new Set(a); const sb = new Set(b); + if (!sa.size && !sb.size) return 1; + let inter = 0; for (const x of sa) if (sb.has(x)) inter++; + return inter / (sa.size + sb.size - inter); +} +function summarize(ranks) { + const n = ranks.length; + const rec = (k) => ranks.filter((r) => r != null && r <= k).length / n; + const mrr = ranks.reduce((s, r) => s + (r != null && r <= 20 ? 1 / r : 0), 0) / n; + const found = ranks.filter((r) => r != null).length; + return { n, recall_at_1: rec(1), recall_at_5: rec(5), recall_at_20: rec(20), mrr_at_20: mrr, found }; +} + +// ── In-container explain diagnostic (single node pass) ───────────────────────────────── +function explainDiagnostic(queries) { + const payload = JSON.stringify(queries.map((q) => ({ id: q.id, q: q.query, set: q.set }))); + const script = ` +import { search } from './src/services/search/searchService.js'; +import { parseQuery } from './src/services/search/queryParser.js'; +const userId = process.env.EVAL_USER_ID; +const queries = JSON.parse(process.env.EVAL_Q); +const out = []; +for (const item of queries) { + const parsed = parseQuery(item.q); + const row = { id: item.id, q: item.q, set: item.set }; + for (const mode of ['hybrid', 'vector']) { + try { + const r = await search({ userId, parsed, mode, limit: 20, explain: true }); + const hits = r.messages || []; + const ids = hits.map((h) => h.id); + const gi = ids.indexOf(item.id); + row[mode] = { + fellBack: !!r.fellBack, mode: r.mode, pool_saturated: !!r.pool_saturated, n: hits.length, + n_bm25: hits.filter((h) => h.score && h.score.bm25 != null).length, + n_vector: hits.filter((h) => h.score && h.score.vector != null).length, + n_both: hits.filter((h) => h.score && h.score.bm25 != null && h.score.vector != null).length, + n_subject_boosted: hits.filter((h) => h.score && h.score.subject_boosted).length, + gt_rank: gi < 0 ? null : gi + 1, + gt_score: gi < 0 ? null : hits[gi].score, + }; + } catch (e) { row[mode] = { error: String(e.message || e) }; } + } + out.push(row); +} +process.stdout.write(JSON.stringify(out)); +`; + const raw = execFileSync( + 'docker', + ['exec', '-e', `EVAL_Q=${payload}`, '-e', `EVAL_USER_ID=${SCOPE.userId}`, + CFG.backendContainer, 'node', '--input-type=module', '-e', script], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + return JSON.parse(raw); +} + +// ── Main ─────────────────────────────────────────────────────────────────────────────── +let SCOPE; +async function main() { + if (!existsSync(EVAL_DIR)) mkdirSync(EVAL_DIR, { recursive: true }); + SCOPE = resolveScope(); + console.log(`user=${SCOPE.userId.slice(0, 8)} accounts=${SCOPE.accountIds.length}`); + + const cache = existsSync(CFG.cacheFile) ? JSON.parse(readFileSync(CFG.cacheFile, 'utf8')) : {}; + + console.log('building keyword set...'); + const keywordSet = buildKeywordSet(); + console.log(`building paraphrase set (LLM=${CFG.openaiKey ? 'on' : 'off'})...`); + const { set: paraphraseSet, usedFallback, sourceCounts } = await buildParaphraseSet(cache); + // cacheFile holds paraphrases of real mail — gitignored, regenerated on demand; never committed. + writeFileSync(CFG.cacheFile, JSON.stringify(cache, null, 2)); + if (CFG.generateOnly) { + console.log(`generate-only: cache written (${paraphraseSet.length} paraphrases, ${usedFallback} still fallback). Exiting.`); + return; + } + + const queries = [...keywordSet, ...paraphraseSet]; + console.log(`total queries: ${queries.length} (${keywordSet.length} keyword, ${paraphraseSet.length} paraphrase)`); + + console.log('logging in...'); + await login(); + + // Run every query in every mode against the live REST API. + const perQuery = []; + let done = 0; + for (const q of queries) { + const rec = { id: q.id, set: q.set, query: q.query, subject: q.subject, byMode: {} }; + for (const mode of MODES) { + const r = await restSearch(q.query, mode); + const ids = (r.messages || []).map((m) => m.id); + rec.byMode[mode] = { ids, rank: rankOf(ids, q.id), fellBack: !!r.fellBack, total: r.total }; + await sleep(CFG.spacingMs); + } + perQuery.push(rec); + done++; + if (done % 5 === 0) console.log(` ${done}/${queries.length} queries searched`); + } + + // Explain diagnostics (in-container seam). + let explain = []; + if (!CFG.skipExplain) { + console.log('collecting hybrid/vector explain scores (in-container)...'); + try { explain = explainDiagnostic(queries); } + catch (e) { console.warn(` explain diagnostic failed: ${e.message}`); } + } + const explainById = Object.fromEntries(explain.map((e) => [`${e.set}:${e.id}`, e])); + + // Aggregate metrics. + const sets = ['keyword', 'paraphrase', 'overall']; + const metrics = {}; + for (const s of sets) { + metrics[s] = {}; + const rows = perQuery.filter((r) => s === 'overall' || r.set === s); + for (const mode of MODES) metrics[s][mode] = summarize(rows.map((r) => r.byMode[mode].rank)); + } + + // Cross-mode overlap (how identical are the result sets?). + const overlap = {}; + for (const s of ['keyword', 'paraphrase', 'overall']) { + const rows = perQuery.filter((r) => s === 'overall' || r.set === s); + const pairMean = (a, b) => rows.reduce((acc, r) => acc + jaccard(r.byMode[a].ids, r.byMode[b].ids), 0) / rows.length; + const identicalTop20 = (a, b) => rows.filter((r) => jaccard(r.byMode[a].ids, r.byMode[b].ids) === 1).length / rows.length; + overlap[s] = { + jaccard_hybrid_vector: pairMean('hybrid', 'vector'), + jaccard_hybrid_lexical: pairMean('hybrid', 'lexical'), + jaccard_vector_lexical: pairMean('vector', 'lexical'), + identical_top20_hybrid_vector: identicalTop20('hybrid', 'vector'), + identical_top20_hybrid_lexical: identicalTop20('hybrid', 'lexical'), + }; + } + + // Explain composition rollup (per set): how often did the BM25 leg actually contribute? + const explainRollup = {}; + for (const s of ['keyword', 'paraphrase']) { + const rows = explain.filter((e) => e.set === s && e.hybrid && !e.hybrid.error); + if (!rows.length) continue; + const avg = (f) => rows.reduce((a, e) => a + f(e), 0) / rows.length; + explainRollup[s] = { + n: rows.length, + queries_with_any_bm25_hit: rows.filter((e) => e.hybrid.n_bm25 > 0).length, + avg_hybrid_bm25_hits: avg((e) => e.hybrid.n_bm25), + avg_hybrid_vector_hits: avg((e) => e.hybrid.n_vector), + avg_hybrid_subject_boosted: avg((e) => e.hybrid.n_subject_boosted), + queries_pool_saturated: rows.filter((e) => e.hybrid.pool_saturated).length, + }; + } + + // ── Print human-readable tables ── + const pct = (x) => (x * 100).toFixed(1).padStart(5); + const num = (x) => x.toFixed(3); + console.log('\n================ RESULTS ================'); + for (const s of sets) { + console.log(`\n[${s.toUpperCase()}] (n=${metrics[s].lexical.n})`); + console.log(' mode R@1 R@5 R@20 MRR@20 found'); + for (const mode of MODES) { + const m = metrics[s][mode]; + console.log(` ${mode.padEnd(7)} ${pct(m.recall_at_1)}% ${pct(m.recall_at_5)}% ${pct(m.recall_at_20)}% ${num(m.mrr_at_20)} ${m.found}/${m.n}`); + } + } + console.log('\n[OVERLAP] mean Jaccard@20 of result sets'); + for (const s of ['keyword', 'paraphrase']) { + const o = overlap[s]; + console.log(` ${s.padEnd(11)} hybrid~vector=${num(o.jaccard_hybrid_vector)} hybrid~lexical=${num(o.jaccard_hybrid_lexical)} vector~lexical=${num(o.jaccard_vector_lexical)} (identical hybrid==vector top20: ${pct(o.identical_top20_hybrid_vector)}%)`); + } + console.log('\n[EXPLAIN] hybrid BM25-leg contribution'); + for (const s of ['keyword', 'paraphrase']) { + const e = explainRollup[s]; if (!e) continue; + console.log(` ${s.padEnd(11)} queries with >=1 BM25 hit: ${e.queries_with_any_bm25_hit}/${e.n} avg BM25 hits=${num(e.avg_hybrid_bm25_hits)} avg vector hits=${num(e.avg_hybrid_vector_hits)} avg subject-boosted=${num(e.avg_hybrid_subject_boosted)}`); + } + + // ── Write results JSON (NO secrets) ── + const results = { + generated_at: new Date().toISOString(), + config: { + baseUrl: CFG.baseUrl, seed: CFG.seed, limit: CFG.limit, + openaiModel: CFG.openaiKey ? CFG.openaiModel : null, + n_keyword: keywordSet.length, n_paraphrase: paraphraseSet.length, + paraphrase_fallback_used: usedFallback, + paraphrase_sources: sourceCounts, + }, + corpus: { messages: Number(psql('SELECT count(*) FROM messages')[0][0]), note: 'bodies mostly NULL; embeddings ~subject-only' }, + metrics, overlap, explainRollup, + perQuery: perQuery.map((r) => ({ + id: r.id, set: r.set, query: r.query, subject: r.subject, + rank: { lexical: r.byMode.lexical.rank, vector: r.byMode.vector.rank, hybrid: r.byMode.hybrid.rank }, + fellBack: { vector: r.byMode.vector.fellBack, hybrid: r.byMode.hybrid.fellBack }, + explain: explainById[`${r.set}:${r.id}`] || null, + })), + }; + writeFileSync(CFG.outFile, JSON.stringify(results, null, 2)); + console.log(`\nwrote ${CFG.outFile}`); + console.log(`wrote ${CFG.cacheFile}`); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/backend/scripts/vector-probe.js b/backend/scripts/vector-probe.js new file mode 100644 index 0000000..763bfd4 --- /dev/null +++ b/backend/scripts/vector-probe.js @@ -0,0 +1,38 @@ +// Dev probe: bring up the vector schema, insert N random D-dim vectors under a throwaway +// generation, then run an ANN query and print the nearest neighbors with scores. +// Usage: node scripts/vector-probe.js [N=100] [D=8] +// Requires DB_* env pointing at a pgvector-enabled Postgres. +import { randomUUID } from 'crypto'; +import { pool } from '../src/services/db.js'; +import { ensureVectorSchema, ensureVectorIndex, upsert, annSearch } from '../src/services/embeddings/vectorStore.js'; +import { createGeneration } from '../src/services/embeddings/generations.js'; +import { seedAccount, cleanupAccount } from '../src/services/embeddings/testSupport.js'; + +const N = Number(process.argv[2]) || 100; +const D = Number(process.argv[3]) || 8; + +function randVec(d) { return Array.from({ length: d }, () => Math.random() * 2 - 1); } + +const { vectorAvailable } = await ensureVectorSchema(); +if (!vectorAvailable) { console.error('vector unavailable — is this a pgvector image?'); process.exit(1); } +await ensureVectorIndex(D); + +const gen = await createGeneration('probe-model', D, `probe:${randomUUID()}`); +// Seed N messages (probe rows) so the ANN liveness EXISTS matches. Uses a throwaway account. +const { accountId: acctId, userId } = await seedAccount(pool, 'probe'); +const chunks = []; +for (let i = 0; i < N; i++) { + const m = await pool.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1,$2,'INBOX',$3) RETURNING id`, + [acctId, 900000 + i, `probe ${i}`]); + chunks.push({ messageId: m.rows[0].id, chunkIndex: 0, vector: randVec(D), sourceCharLen: 8, chunkCharStart: 0, chunkCharEnd: 8, truncated: false }); +} +await upsert(gen, chunks); + +const q = randVec(D); +console.log(`query = [${q.map((x) => x.toFixed(3)).join(', ')}]`); +const hits = await annSearch(gen, q, 5); +for (const h of hits) console.log(` rank ${h.rank} msg ${h.messageId} score ${h.score.toFixed(4)}`); + +await cleanupAccount(pool, userId); +await pool.end(); +process.exit(0); diff --git a/backend/src/index.js b/backend/src/index.js index 483dcfa..98b51b7 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -18,6 +18,7 @@ import accountRoutes from './routes/accounts.js'; import mailRoutes from './routes/mail.js'; import searchRoutes from './routes/search.js'; import adminRoutes from './routes/admin.js'; +import indexingRoutes from './routes/indexing.js'; import totpRoutes from './routes/totp.js'; import oidcApiRouter, { oidcBrowserRouter } from './routes/oidc.js'; import rulesRoutes from './routes/rules.js'; @@ -25,13 +26,20 @@ import blockListRoutes from './routes/blockList.js'; import contactsRoutes from './routes/contacts.js'; import todoistRoutes from './routes/todoist.js'; import aiRoutes from './routes/ai.js'; +import aiEmbeddingsRoutes from './routes/aiEmbeddings.js'; import categoriesRoutes from './routes/categories.js'; import gtdRoutes from './routes/gtd.js'; import carddavRouter from './routes/carddav.js'; import carddavAccountRouter from './routes/carddavAccount.js'; +import { mountMcp } from './mcp/server.js'; +import apiTokensRoutes from './routes/apiTokens.js'; +import mcpDeletionsRoutes from './routes/mcpDeletions.js'; import { startCardavScheduler } from './services/carddavSync.js'; +import { scheduleFtsBackfill } from './services/search/ftsBackfill.js'; import { encryptExistingCredentials, query } from './services/db.js'; -import { runMigrations } from './services/migrations.js'; +import { runMigrations, warnOnCollationMismatch } from './services/migrations.js'; +import { ensureVectorSchema } from './services/embeddings/vectorStore.js'; +import { startEmbeddingScheduler } from './services/embeddings/scheduler.js'; import { parseVCard } from './utils/vcard.js'; import { reloadAuthSettings } from './services/authLimiter.js'; import { setupWebSocket } from './services/websocket.js'; @@ -167,6 +175,7 @@ app.use('/api/mail', sendRoutes); app.use('/api/mail', draftRoutes); app.use('/api/search', searchRoutes); app.use('/api/admin', adminRoutes); +app.use('/api/admin/indexing', indexingRoutes); app.use('/api/totp', totpRoutes); app.use('/api/rules', rulesRoutes); app.use('/api/block-list', blockListRoutes); @@ -174,6 +183,9 @@ app.use('/api/contacts', contactsRoutes); app.use('/api/todoist', todoistRoutes); app.use('/api/carddav', carddavAccountRouter); app.use('/api', aiRoutes); +app.use('/api', aiEmbeddingsRoutes); +app.use('/api/tokens', apiTokensRoutes); +app.use('/api/mcp-deletions', mcpDeletionsRoutes); app.use('/api', categoriesRoutes); // Mounted at the /api/gtd subtree (not bare /api) so gtd.js's router-level // requireAuth cannot intercept the unauthenticated /api/health and /api/version @@ -185,6 +197,10 @@ app.use('/carddav', carddavRouter); // RFC 6764 well-known redirect — handle all methods so PROPFIND probes also redirect app.all('/.well-known/carddav', (req, res) => res.redirect(308, '/carddav/')); +// MCP Streamable-HTTP endpoint. Bearer-authenticated (mcp/auth.js), intentionally +// outside the /api CSRF gate and session middleware — auth is a token, not a cookie. +mountMcp(app); + app.get('/api/health', (req, res) => res.json({ status: 'ok' })); app.get('/api/version', (_req, res) => res.json({ version: APP_VERSION, sha: process.env.BUILD_SHA || 'dev' })); // Server-side update check (#261). Cached in updateCheck.js so repeated hits never @@ -210,6 +226,11 @@ setupWebSocket(wss, sessionMiddleware, imapManager); // Run pending schema migrations then start await runMigrations(); +// Loud, best-effort drift check: a Postgres image swap (e.g. postgres:16-alpine → +// pgvector/pgvector:pg16) changes the libc collation and silently corrupts text-index +// ordering until a REINDEX. Logs the remedy; never blocks the boot. +await warnOnCollationMismatch(); + // One-time backfill: populate photo_data from existing vcard column for contacts // that were synced before CardDAV PUT started persisting photo_data. async function backfillContactPhotos() { @@ -238,12 +259,22 @@ await encryptExistingCredentials(); // Load OAuth integration configs from DB into process.env await loadIntegrationConfigs(); +// Best-effort vector schema bring-up. On stock postgres:16-alpine this logs +// "Vector disabled" and semantic search stays off; lexical is unaffected. +await ensureVectorSchema(); + +// Periodic embedding nudge — drives any building/active generation toward coverage. +startEmbeddingScheduler(); + // Start background snooze watcher — polls every 60 seconds to restore snoozed messages imapManager.startSnoozeWatcher(); // Schedule periodic CardDAV contact sync for any connected accounts. startCardavScheduler(); +// Postgres-only drainer: populate search_fts for pre-existing rows (no IMAP). +scheduleFtsBackfill(); + // Re-connect all enabled IMAP accounts on startup with bounded concurrency so a // large user base doesn't hammer IMAP servers and the DB connection pool at once. try { diff --git a/backend/src/mcp/auth.js b/backend/src/mcp/auth.js new file mode 100644 index 0000000..10addcb --- /dev/null +++ b/backend/src/mcp/auth.js @@ -0,0 +1,45 @@ +import crypto from 'crypto'; +import { query } from '../services/db.js'; + +// Plaintext tokens are shown once at mint time; we persist only the hash. +export function generateToken() { + return 'mcp_' + crypto.randomBytes(32).toString('base64url'); +} + +export function hashToken(plaintext) { + return crypto.createHash('sha256').update(plaintext, 'utf8').digest('hex'); +} + +// The scope object pins multi-user isolation: every MCP tool call is bounded to +// exactly this user's enabled accounts. msgvault is single-archive; Mailflow is not. +export async function resolveScope(userId) { + const { rows } = await query( + 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId], + ); + return { userId, accountIds: rows.map((r) => r.id) }; +} + +export async function mcpBearerAuth(req, res, next) { + try { + const header = req.get('Authorization') || ''; + const m = /^Bearer\s+(.+)$/i.exec(header); + if (!m) return res.status(401).json({ error: 'invalid_token' }); + + const { rows } = await query( + 'SELECT id, user_id FROM api_tokens WHERE token_hash = $1', + [hashToken(m[1].trim())], + ); + if (!rows.length) return res.status(401).json({ error: 'invalid_token' }); + + // Best-effort recency stamp; never block the request on it. + await query('UPDATE api_tokens SET last_used_at = NOW() WHERE id = $1', [rows[0].id]) + .catch(() => {}); + + req.mcpTokenId = rows[0].id; // rate-limit key: per token, not per IP + req.mcpScope = await resolveScope(rows[0].user_id); + next(); + } catch (err) { + next(err); + } +} diff --git a/backend/src/mcp/auth.test.js b/backend/src/mcp/auth.test.js new file mode 100644 index 0000000..9b58274 --- /dev/null +++ b/backend/src/mcp/auth.test.js @@ -0,0 +1,87 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { generateToken, hashToken, resolveScope, mcpBearerAuth } from './auth.js'; + +function mockRes() { + return { statusCode: 200, body: null, status(c){this.statusCode=c;return this;}, json(b){this.body=b;return this;} }; +} + +describe('token helpers', () => { + it('generateToken is prefixed and unique', () => { + const a = generateToken(); const b = generateToken(); + expect(a).toMatch(/^mcp_[A-Za-z0-9_-]{20,}$/); + expect(a).not.toEqual(b); + }); + it('hashToken is deterministic hex SHA-256 and hides the plaintext', () => { + const h = hashToken('mcp_secret'); + expect(h).toMatch(/^[0-9a-f]{64}$/); + expect(h).toEqual(hashToken('mcp_secret')); + expect(h).not.toContain('secret'); + }); +}); + +describe('resolveScope', () => { + it("returns only the token owner's enabled account ids", async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1' }, { id: 'acc-2' }] }); + const scope = await resolveScope('user-1'); + expect(scope).toEqual({ userId: 'user-1', accountIds: ['acc-1', 'acc-2'] }); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/FROM email_accounts WHERE user_id = \$1 AND enabled = true/), + ['user-1'], + ); + }); +}); + +describe('mcpBearerAuth', () => { + beforeEach(() => query.mockReset()); + + it('rejects a request with no Authorization header', async () => { + const req = { get: () => undefined }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(res.body).toEqual({ error: 'invalid_token' }); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects a malformed (non-Bearer) header', async () => { + const req = { get: () => 'Basic abc' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects an unknown / revoked token', async () => { + query.mockResolvedValueOnce({ rows: [] }); // no api_tokens row + const req = { get: () => 'Bearer mcp_revoked' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('accepts a valid token and attaches the user-scoped account ids', async () => { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok-1', user_id: 'user-1' }] }) // token lookup + .mockResolvedValueOnce({ rows: [] }) // last_used_at UPDATE + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); // resolveScope + const req = { get: (h) => (h === 'Authorization' ? 'Bearer mcp_good' : undefined) }; + const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(next).toHaveBeenCalledOnce(); + expect(req.mcpScope).toEqual({ userId: 'user-1', accountIds: ['acc-1'] }); + // token lookup must be by HASH, never the plaintext + expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); + }); + + it('isolates users: the scope carries only the resolved owner', async () => { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok-2', user_id: 'user-2' }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ id: 'acc-9' }] }); + const req = { get: () => 'Bearer mcp_u2' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(req.mcpScope.userId).toBe('user-2'); + expect(req.mcpScope.accountIds).toEqual(['acc-9']); + }); +}); diff --git a/backend/src/mcp/bodyMatch.js b/backend/src/mcp/bodyMatch.js new file mode 100644 index 0000000..a73b4a0 --- /dev/null +++ b/backend/src/mcp/bodyMatch.js @@ -0,0 +1,78 @@ +// All offsets are UTF-8 BYTES (msgvault wire contract). We operate on Buffers, +// never JS string .length (UTF-16 code units). Case-insensitive matching mirrors +// msgvault: find in the lowercased buffer, slice the original at the same offsets +// (ASCII-faithful; identical behavior to strings.ToLower + strings.Index in Go). +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from '../utils/textExcerpt.js'; + +export function contextWindow(bodyLen, pos, termLen, contextChars) { + let start = pos - Math.floor((contextChars - termLen) / 2); + let end = start + contextChars; + if (start < 0) { + start = 0; + end = Math.min(bodyLen, contextChars); + } else if (end > bodyLen) { + end = bodyLen; + start = Math.max(0, end - contextChars); + } + return [start, end]; +} + +export function bodyByteSliceRange(buf, start, end) { + if (start < 0) start = 0; + if (end > buf.length) end = buf.length; + if (start >= buf.length) return { text: '', adjStart: buf.length, adjEnd: buf.length }; + let adjStart = start; + let adjEnd = end <= start ? Math.min(buf.length, start + 1) : end; + while (adjStart < adjEnd && !isRuneStart(buf[adjStart])) adjStart++; + while (adjEnd > adjStart && adjEnd < buf.length && !isRuneStart(buf[adjEnd])) adjEnd--; + return { text: buf.toString('utf8', adjStart, adjEnd), adjStart, adjEnd }; +} + +function bodyByteSlice(buf, start, end) { + return bodyByteSliceRange(buf, start, end).text; +} + +export function findTermMatches(body, term) { + if (!body || !term) return []; + const buf = Buffer.from(body, 'utf8'); + const lower = Buffer.from(body.toLowerCase(), 'utf8'); + const t = Buffer.from(term.toLowerCase(), 'utf8'); + const termLen = t.length; + const matches = []; + let from = 0; + for (;;) { + const idx = lower.indexOf(t, from); + if (idx < 0) break; + from = idx + 1; + const [start, end] = contextWindow(buf.length, idx, termLen, SNIPPET_BYTES); + matches.push({ char_offset: idx, snippet: bodyByteSlice(buf, start, end), line: lineNumberAt(buf, idx) }); + } + return matches; +} + +export function extractContextChar(body, terms, contextChars) { + if (!body || !terms || !terms.length || contextChars <= 0) return null; + const buf = Buffer.from(body, 'utf8'); + const lower = Buffer.from(body.toLowerCase(), 'utf8'); + const spans = []; + for (const term of terms) { + if (!term || term.length < 2) continue; + const t = Buffer.from(term.toLowerCase(), 'utf8'); + let from = 0; + for (;;) { + const idx = lower.indexOf(t, from); + if (idx < 0) break; + from = idx + 1; + spans.push(contextWindow(buf.length, idx, t.length, contextChars)); + } + } + if (!spans.length) return null; + spans.sort((a, b) => (a[0] === b[0] ? a[1] - b[1] : a[0] - b[0])); + const merged = [spans[0].slice()]; + for (const s of spans.slice(1)) { + const last = merged[merged.length - 1]; + if (s[0] <= last[1]) last[1] = Math.max(last[1], s[1]); + else merged.push(s.slice()); + } + return merged.map(([s, e]) => bodyByteSlice(buf, s, e)); +} diff --git a/backend/src/mcp/bodyMatch.test.js b/backend/src/mcp/bodyMatch.test.js new file mode 100644 index 0000000..b494a8b --- /dev/null +++ b/backend/src/mcp/bodyMatch.test.js @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; +import { contextWindow, bodyByteSliceRange, extractContextChar, findTermMatches } from './bodyMatch.js'; + +describe('contextWindow', () => { + it('centers within bounds', () => { + expect(contextWindow(1000, 500, 10, 300)).toEqual([355, 655]); + }); + it('clamps at the start', () => { + expect(contextWindow(1000, 5, 10, 300)).toEqual([0, 300]); + }); + it('clamps at the end', () => { + expect(contextWindow(300, 290, 4, 300)).toEqual([0, 300]); + }); +}); + +describe('byte-offset fidelity on multibyte bodies', () => { + // "café — " then the term. 'é' is 2 bytes (0xC3 0xA9), '—' is 3 bytes. + const body = 'café — meeting notes'; + it('findTermMatches reports the BYTE offset of the term, not the code-unit index', () => { + const m = findTermMatches(body, 'meeting'); + // "café — " = c a f é(2) space —(3) space = 1+1+1+2+1+3+1 = 10 bytes + expect(m).toHaveLength(1); + expect(m[0].char_offset).toBe(10); + expect(m[0].line).toBe(1); + expect(m[0].snippet).toContain('meeting'); + }); + it('bodyByteSliceRange never splits a rune', () => { + const buf = Buffer.from(body, 'utf8'); + // Ask for a window that would cut the 'é' (bytes 3-4) in half at byte 4. + const { text } = bodyByteSliceRange(buf, 0, 4); + expect(Buffer.from(text, 'utf8').every((b) => b !== undefined)).toBe(true); + expect(text).toBe('caf'); // é dropped rather than split + }); +}); + +describe('extractContextChar', () => { + it('merges overlapping windows into snippet strings', () => { + const body = 'alpha beta alpha'; + const snippets = extractContextChar(body, ['alpha'], 300); + expect(snippets).toHaveLength(1); // both hits merge into one window + expect(snippets[0]).toContain('alpha'); + }); + it('ignores terms shorter than 2 chars', () => { + expect(extractContextChar('a a a', ['a'], 300)).toBeNull(); + }); +}); diff --git a/backend/src/mcp/engineAdapter.js b/backend/src/mcp/engineAdapter.js new file mode 100644 index 0000000..034bfc0 --- /dev/null +++ b/backend/src/mcp/engineAdapter.js @@ -0,0 +1,411 @@ +// The query.Engine port: all message/aggregate/deletion SQL lives here, scoped by +// accountIds, so MCP handlers never touch db.js (the no-SQL-in-handlers invariant). +// Row → msgvault-shape mappers pin the wire-casing split: MessageSummary/Detail carry +// snake_case json keys, but Address/AttachmentInfo/AccountInfo/AggregateRow/TotalStats +// have NO Go json tags → Go-default CAPITALIZED keys. D6: ids are UUID strings, +// conversation_id = thread_id. +import { query, withTransaction } from '../services/db.js'; +import { buildOperatorClauses, freeTextTermClause, hasSearchableToken } from '../services/search/lexicalRepo.js'; + +const SUMMARY_COLUMNS = ` + m.id, m.account_id, m.message_id, m.thread_id, m.subject, m.snippet, + m.from_email, m.from_name, m.to_addresses, m.cc_addresses, m.date, + m.has_attachments, m.attachments, m.flags, m.folder`; + +const DETAIL_COLUMNS = SUMMARY_COLUMNS + `, m.body_text, m.body_html`; + +export function mapAddrs(jsonb) { + return (jsonb || []).map((a) => ({ Email: a.email || '', Name: a.name || '' })); +} + +// msgvault labels ≈ Gmail labels; Mailflow's closest analogue is the folder plus IMAP flags. +export function mapLabels(row) { + const flags = Array.isArray(row.flags) ? row.flags : []; + return [row.folder, ...flags].filter(Boolean); +} + +function toISO(d) { + return d instanceof Date ? d.toISOString() : (d || ''); +} + +export function rowToMessageSummary(row) { + const atts = Array.isArray(row.attachments) ? row.attachments : []; + const s = { + id: row.id, + source_id: row.account_id, + source_message_id: row.message_id || '', + conversation_id: row.thread_id || '', + source_conversation_id: row.thread_id || '', + subject: row.subject || '', + snippet: row.snippet || '', + from_email: row.from_email || '', + from_name: row.from_name || '', + sent_at: toISO(row.date), + size_estimate: 0, // divergence: Mailflow stores no byte size + has_attachments: !!row.has_attachments, + attachment_count: atts.length, + labels: mapLabels(row), + message_type: 'email', + }; + const to = mapAddrs(row.to_addresses); + const cc = mapAddrs(row.cc_addresses); + if (to.length) s.to = to; // omitempty parity + if (cc.length) s.cc = cc; + return s; +} + +function mapAttachments(atts) { + return (Array.isArray(atts) ? atts : []).map((a, i) => ({ + ID: i, Filename: a.filename || '', MimeType: a.type || '', Size: a.size || 0, + ContentHash: '', URL: '', StoragePath: '', // content tools are non-goals; synthetic index id + })); +} + +export function rowToMessageDetail(row) { + const base = rowToMessageSummary(row); + return { + ...base, + from: row.from_email ? [{ Email: row.from_email, Name: row.from_name || '' }] : [], + to: mapAddrs(row.to_addresses), + cc: mapAddrs(row.cc_addresses), + bcc: [], + body_text: row.body_text || '', + body_html: row.body_html || '', + attachments: mapAttachments(row.attachments), + }; +} + +// Raw detail row (not yet mapped) for one message, scoped. Handlers map/page it. +export async function getMessage(id, accountIds) { + if (!accountIds || !accountIds.length) return null; + const { rows } = await query( + `SELECT ${DETAIL_COLUMNS} FROM messages m WHERE m.id = $1 AND m.account_id = ANY($2)`, + [id, accountIds], + ); + return rows[0] || null; +} + +export async function getMessageSummariesByIDs(ids, accountIds) { + if (!ids || !ids.length) return []; + const { rows } = await query( + `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE m.id = ANY($1) AND m.account_id = ANY($2)`, + [ids, accountIds], + ); + const byId = new Map(rows.map((r) => [r.id, rowToMessageSummary(r)])); + return ids.map((id) => byId.get(id)).filter(Boolean); // preserve input order +} + +export async function getMessageBodiesByIDs(ids, accountIds) { + const out = new Map(); + if (!ids || !ids.length) return out; + const { rows } = await query( + `SELECT m.id, m.body_text, m.body_html FROM messages m WHERE m.id = ANY($1) AND m.account_id = ANY($2)`, + [ids, accountIds], + ); + for (const r of rows) out.set(r.id, { body_text: r.body_text, body_html: r.body_html }); + return out; +} + +export async function listAccounts(accountIds) { + if (!accountIds || !accountIds.length) return []; + const { rows } = await query( + `SELECT id, protocol, email_address, name FROM email_accounts WHERE id = ANY($1) ORDER BY sort_order`, + [accountIds], + ); + return rows.map((a) => ({ ID: a.id, SourceType: a.protocol || 'imap', Identifier: a.email_address, DisplayName: a.name || '' })); +} + +// Resolve an optional `account` email to a single scoped id (msgvault getAccountID). +// Returns { accountIds } narrowed to the match, or { error } when unknown (msgvault +// errors rather than silently widening). Empty account = no narrowing (all of the +// token's accounts). Shared by the search AND message/aggregate/deletion handlers so +// the `account` argument narrows scope everywhere it is advertised. +export async function resolveAccountScope(account, accountIds) { + if (!account) return { accountIds }; + const accounts = await listAccounts(accountIds); + const match = accounts.find((a) => a.Identifier === account); + if (!match) return { error: `account not found: ${account}` }; + return { accountIds: [match.ID] }; +} + +// Cheap owner-scope membership check for a single message id (find_similar seed). +export async function messageInScope(id, accountIds) { + if (!accountIds || !accountIds.length) return false; + const { rows } = await query('SELECT 1 FROM messages WHERE id = $1 AND account_id = ANY($2) LIMIT 1', [id, accountIds]); + return rows.length > 0; +} + +// Port of msgvault getDateArg (internal/mcp/handlers.go): structured after/before +// tool args are strict YYYY-MM-DD (the format every tool schema advertises). An +// unparseable value throws msgvault's exact error message instead of surfacing a +// raw Postgres timestamptz cast failure; a valid one binds as midnight UTC — the +// same instant lexicalRepo's buildOperatorClauses gives query-string dates — so +// structured args and before:/after: operators sit on one clock. Callers apply +// `before` EXCLUSIVELY (<): msgvault uses < and lexicalRepo already does; the +// list/aggregate/domain paths here had drifted to <=. +function parseDateArg(key, value) { + if (!value) return null; + const d = /^\d{4}-\d{2}-\d{2}$/.test(value) ? new Date(`${value}T00:00:00.000Z`) : null; + // Round-trip guard: Date.parse ROLLS an out-of-range day over ("2025-02-31" + // → Mar 2) instead of rejecting it; msgvault's time.Parse rejects. + if (!d || isNaN(d) || !d.toISOString().startsWith(value)) { + throw new Error(`invalid ${key} date "${value}": expected YYYY-MM-DD`); + } + return d.toISOString(); +} + +// Shared by every structured-arg path below: validate both bounds up front, +// then push the >= / < clauses through the caller's bind. +function pushDateClauses(where, bind, after, before) { + const afterISO = parseDateArg('after', after); + const beforeISO = parseDateArg('before', before); + if (afterISO) where.push(`m.date >= ${bind(afterISO)}`); + if (beforeISO) where.push(`m.date < ${bind(beforeISO)}`); +} + +// Newest-first message list, scoped and filtered. Returns MessageSummary[] (mapped +// here so the handler only pages). `from` filters from_email when it looks like an +// address, else from_name (msgvault list_messages semantics). +export async function listMessages({ accountIds, from, to, label, hasAttachment, after, before, conversationId, limit, offset }) { + if (!accountIds || !accountIds.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + if (from) { + where.push(from.includes('@') + ? `m.from_email ILIKE ${bind('%' + from + '%')}` + : `m.from_name ILIKE ${bind('%' + from + '%')}`); + } + if (to) where.push(`m.to_addresses::text ILIKE ${bind('%' + to + '%')}`); + if (label) where.push(`(m.folder = ${bind(label)} OR m.flags @> ${bind(JSON.stringify([label]))}::jsonb)`); + if (hasAttachment) where.push('m.has_attachments = true'); + pushDateClauses(where, bind, after, before); + if (conversationId) where.push(`m.thread_id = ${bind(conversationId)}`); + + const sql = `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} + ORDER BY m.date DESC NULLS LAST LIMIT ${bind(limit)} OFFSET ${bind(offset)}`; + const { rows } = await query(sql, args); + return rows.map(rowToMessageSummary); +} + +const MAX_STAGE_DELETION = 100000; // msgvault maxStageDeletionResults + +// A free-text term contributes a staging predicate only when it is positive, +// ≥2 chars, and carries a searchable token — the same hygiene the lexical and +// semantic paths apply. Negated terms are deliberately NOT enforced on the +// staging path (unlike lexical search's FTS exclusion), so a negation-only +// query yields zero predicates; countEnforceableQueryPredicates below is what +// lets the handler refuse that rather than stage every live message. +function usablePositiveTerm(t) { + return !t.negate && t.value.length >= 2 && hasSearchableToken(t.value); +} + +// Count the enforceable row predicates a parsed query would contribute to a +// staging WHERE: supported structured filters (buildOperatorClauses; in:/ +// unsupported operators contribute nothing) plus usable positive free-text +// terms. Mirrors resolveStageDeletionIds' predicate construction exactly (same +// builder, same term hygiene) so the two can never disagree about whether a +// query is enforceable. Zero here means the WHERE would carry only account + +// liveness — i.e. the query would soft-delete the whole (capped) mailbox — so +// the stage_deletion handler refuses it. This is the final safety net, reached +// only for queries that carry no unsupported operator yet still yield no +// predicate (e.g. all stopwords or punctuation): the handler already rejects any +// unsupported operator up front via unsupportedSearchOperatorMessage (msgvault +// parity, internal/mcp/handlers.go — so a mixed query like `invoice +// label:promotions` is refused, not silently widened to every "invoice" match). +export function countEnforceableQueryPredicates(parsed) { + if (!parsed) return 0; + let n = buildOperatorClauses(parsed.filters, () => '$0').length; + for (const t of parsed.terms || []) if (usablePositiveTerm(t)) n++; + return n; +} + +// Resolve candidate message ids for staging, scoped to accountIds. Query path builds +// its free-text predicates from lexicalRepo's freeTextTermClause — the EXACT builder +// the search path uses (ranked search_fts match + un-backfilled fallback + stopword +// vacuity) — so the search preview and the staged set can never diverge: `the invoice` +// stages the invoice matches the preview showed, not the zero rows the legacy +// ILIKE/plainto fork produced once "the" normalized to an empty tsquery. Structured +// operators share buildOperatorClauses the same way; the structured-filter path +// builds from the discrete filter args. +async function resolveStageDeletionIds({ accountIds, parsed, from, domain, label, hasAttachment, after, before }) { + const params = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { params.push(v); return `$${params.length}`; }; + + if (parsed) { + for (const cond of buildOperatorClauses(parsed.filters, bind)) where.push(cond); + for (const t of parsed.terms || []) { + if (!usablePositiveTerm(t)) continue; + where.push(freeTextTermClause(t.value, false, bind)); + } + } else { + if (from) { const p = bind('%' + from + '%'); where.push(`(m.from_email ILIKE ${p} OR m.from_name ILIKE ${p})`); } + if (domain) where.push(`m.from_email ILIKE ${bind('%@' + domain)}`); + if (label) where.push(`(m.folder = ${bind(label)} OR m.flags @> ${bind(JSON.stringify([label]))}::jsonb)`); + if (hasAttachment) where.push('m.has_attachments = true'); + pushDateClauses(where, bind, after, before); + } + + const sql = `SELECT m.id FROM messages m WHERE ${where.join(' AND ')} LIMIT ${bind(MAX_STAGE_DELETION)}`; + const { rows } = await query(sql, params); + return rows.map((r) => r.id); +} + +// Record a STAGED batch + its members scoped to the token user. NEVER flips +// is_deleted — execution is a separate, session-authed step. +export async function stageDeletion(opts) { + const ids = await resolveStageDeletionIds(opts); + if (!ids.length) return { batchId: null, messageCount: 0 }; + return withTransaction(async (client) => { + const { rows } = await client.query( + "INSERT INTO mcp_deletion_batches (user_id, description, status, message_count) VALUES ($1, $2, 'staged', $3) RETURNING id", + [opts.userId, opts.description || '', ids.length], + ); + const batchId = rows[0].id; + await client.query( + 'INSERT INTO mcp_deletion_batch_messages (batch_id, message_id) SELECT $1, unnest($2::uuid[])', + [batchId, ids], + ); + return { batchId, messageCount: ids.length }; + }); +} + +// Session-authed soft-delete of a STAGED batch, scoped to the owner's accounts. +// Returns the updated row count, or null when the batch is absent/not owned/not staged. +export async function executeDeletionBatch(batchId, userId) { + return withTransaction(async (client) => { + const { rows } = await client.query( + "SELECT id FROM mcp_deletion_batches WHERE id = $1 AND user_id = $2 AND status = 'staged' FOR UPDATE", + [batchId, userId], + ); + if (!rows.length) return null; + const upd = await client.query( + `UPDATE messages SET is_deleted = true + WHERE id IN (SELECT message_id FROM mcp_deletion_batch_messages WHERE batch_id = $1) + AND account_id IN (SELECT id FROM email_accounts WHERE user_id = $2)`, + [batchId, userId], + ); + await client.query( + "UPDATE mcp_deletion_batches SET status = 'executed', executed_at = NOW() WHERE id = $1", + [batchId], + ); + return upd.rowCount; + }); +} + +// Discard a staged batch (owner-scoped). Cascades member rows; never touches messages. +export async function unstageDeletionBatch(batchId, userId) { + const { rowCount } = await query( + 'DELETE FROM mcp_deletion_batches WHERE id = $1 AND user_id = $2', + [batchId, userId], + ); + return rowCount > 0; +} + +// Archive overview scoped to accountIds. TotalStats has NO Go json tags → CAPITALIZED +// keys. TotalSize is a body-byte proxy and AttachmentSize is 0 (Mailflow stores no +// byte sizes — documented divergence). +export async function getTotalStats(accountIds) { + const empty = { + MessageCount: 0, ActiveMessageCount: 0, SourceDeletedMessageCount: 0, + TotalSize: 0, AttachmentCount: 0, AttachmentSize: 0, LabelCount: 0, + AccountCount: (accountIds && accountIds.length) || 0, + }; + if (!accountIds || !accountIds.length) return empty; + const { rows } = await query( + `SELECT + COUNT(*)::bigint AS message_count, + COUNT(*) FILTER (WHERE is_deleted = false)::bigint AS active_count, + COUNT(*) FILTER (WHERE is_deleted = true)::bigint AS deleted_count, + COALESCE(SUM(octet_length(COALESCE(body_text, ''))), 0)::bigint AS total_size, + COALESCE(SUM(jsonb_array_length(COALESCE(attachments, '[]'::jsonb))), 0)::bigint AS attachment_count, + COUNT(DISTINCT folder)::bigint AS label_count + FROM messages WHERE account_id = ANY($1)`, + [accountIds], + ); + const r = rows[0] || {}; + return { + MessageCount: Number(r.message_count || 0), + ActiveMessageCount: Number(r.active_count || 0), + SourceDeletedMessageCount: Number(r.deleted_count || 0), + TotalSize: Number(r.total_size || 0), + AttachmentCount: Number(r.attachment_count || 0), + AttachmentSize: 0, + LabelCount: Number(r.label_count || 0), + AccountCount: accountIds.length, + }; +} + +// group_by → grouping SQL. `recipient` needs a lateral unnest of to_addresses; +// the rest are scalar expressions over messages. `time` buckets by calendar year. +const AGG_KEY_EXPR = { + sender: 'm.from_email', + domain: "split_part(m.from_email, '@', 2)", + label: 'm.folder', + time: "to_char(m.date, 'YYYY')", +}; + +// Grouped statistics (top senders/recipients/domains/labels, or volume by year). +// Returns AggregateRow[] — capitalized keys (no Go json tags). AttachmentSize is 0 +// (Mailflow stores no per-attachment byte totals — documented divergence). +export async function aggregate(groupBy, { accountIds, after, before, limit }) { + if (!accountIds || !accountIds.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + pushDateClauses(where, bind, after, before); + + const measures = ` + COUNT(*)::bigint AS count, + COALESCE(SUM(octet_length(COALESCE(m.body_text, ''))), 0)::bigint AS total_size, + COALESCE(SUM(jsonb_array_length(COALESCE(m.attachments, '[]'::jsonb))), 0)::bigint AS attachment_count, + COUNT(*) OVER ()::bigint AS total_unique`; + + let sql; + if (groupBy === 'recipient') { + sql = `SELECT (ra->>'email') AS key, ${measures} + FROM messages m, LATERAL jsonb_array_elements(COALESCE(m.to_addresses, '[]'::jsonb)) AS ra + WHERE ${where.join(' AND ')} + GROUP BY key ORDER BY count DESC LIMIT ${bind(limit)}`; + } else { + const expr = AGG_KEY_EXPR[groupBy]; + sql = `SELECT ${expr} AS key, ${measures} + FROM messages m + WHERE ${where.join(' AND ')} + GROUP BY key ORDER BY count DESC LIMIT ${bind(limit)}`; + } + const { rows } = await query(sql, args); + return rows.map((r) => ({ + Key: r.key == null ? '' : String(r.key), + Count: Number(r.count || 0), + TotalSize: Number(r.total_size || 0), + AttachmentSize: 0, + AttachmentCount: Number(r.attachment_count || 0), + TotalUnique: Number(r.total_unique || 0), + })); +} + +// Messages where any participant (from/to/cc) belongs to one of the domains. +// Returns MessageSummary[], newest-first, scoped to accountIds. +export async function searchByDomains(domains, after, before, limit, offset, accountIds) { + if (!accountIds || !accountIds.length || !domains.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + const domainConds = domains.map((d) => { + const from = bind('%@' + d); + const to = bind('%' + d + '%'); + const cc = bind('%' + d + '%'); + return `(m.from_email ILIKE ${from} OR m.to_addresses::text ILIKE ${to} OR m.cc_addresses::text ILIKE ${cc})`; + }); + where.push('(' + domainConds.join(' OR ') + ')'); + pushDateClauses(where, bind, after, before); + + const sql = `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} + ORDER BY m.date DESC NULLS LAST LIMIT ${bind(limit)} OFFSET ${bind(offset)}`; + const { rows } = await query(sql, args); + return rows.map(rowToMessageSummary); +} diff --git a/backend/src/mcp/engineAdapter.test.js b/backend/src/mcp/engineAdapter.test.js new file mode 100644 index 0000000..5f35270 --- /dev/null +++ b/backend/src/mcp/engineAdapter.test.js @@ -0,0 +1,309 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +import { query, withTransaction } from '../services/db.js'; +import { rowToMessageSummary, rowToMessageDetail, mapAddrs, getMessageSummariesByIDs, listAccounts, listMessages, getTotalStats, aggregate, searchByDomains, stageDeletion, resolveAccountScope, messageInScope, countEnforceableQueryPredicates } from './engineAdapter.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { freeTextTermClause, searchLexical } from '../services/search/lexicalRepo.js'; + +const row = { + id: '11111111-1111-1111-1111-111111111111', account_id: 'acc-1', message_id: '', + thread_id: 'tid-9', subject: 'Hi', snippet: 's', from_email: 'a@b.com', from_name: 'A', + to_addresses: [{ name: 'C', email: 'c@d.com' }], cc_addresses: [], + date: '2024-01-01T00:00:00.000Z', has_attachments: true, + attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], + flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello', body_html: '

hello

', +}; + +describe('mapAddrs', () => { + it('emits capitalized Email/Name keys (msgvault no-json-tag quirk)', () => { + expect(mapAddrs([{ name: 'C', email: 'c@d.com' }])).toEqual([{ Email: 'c@d.com', Name: 'C' }]); + expect(mapAddrs(null)).toEqual([]); + }); +}); + +describe('rowToMessageSummary', () => { + it('maps thread_id to conversation_id and keeps the uuid id as a string', () => { + const s = rowToMessageSummary(row); + expect(s.id).toBe('11111111-1111-1111-1111-111111111111'); + expect(s.conversation_id).toBe('tid-9'); + expect(s.source_conversation_id).toBe('tid-9'); + expect(s.source_message_id).toBe(''); + expect(s.source_id).toBe('acc-1'); + expect(s.to).toEqual([{ Email: 'c@d.com', Name: 'C' }]); + expect(s.attachment_count).toBe(1); + expect(s.size_estimate).toBe(0); + expect(s.labels).toContain('INBOX'); + expect(s.labels).toContain('\\Seen'); + expect(s.message_type).toBe('email'); + }); + + it('omits empty to/cc (omitempty parity)', () => { + const s = rowToMessageSummary({ ...row, to_addresses: [], cc_addresses: [] }); + expect(s).not.toHaveProperty('to'); + expect(s).not.toHaveProperty('cc'); + }); +}); + +describe('rowToMessageDetail', () => { + it('adds body + capitalized AttachmentInfo', () => { + const d = rowToMessageDetail(row); + expect(d.body_text).toBe('hello'); + expect(d.body_html).toBe('

hello

'); + expect(d.from).toEqual([{ Email: 'a@b.com', Name: 'A' }]); + expect(d.attachments[0]).toEqual({ ID: 0, Filename: 'f.pdf', MimeType: 'application/pdf', Size: 10, ContentHash: '', URL: '', StoragePath: '' }); + }); +}); + +describe('getMessageSummariesByIDs', () => { + beforeEach(() => query.mockReset()); + it('scopes to accountIds and preserves input order', async () => { + query.mockResolvedValueOnce({ rows: [{ ...row, id: 'b' }, { ...row, id: 'a' }] }); + const out = await getMessageSummariesByIDs(['a', 'b'], ['acc-1']); + expect(out.map((m) => m.id)).toEqual(['a', 'b']); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = ANY/); + expect(params[1]).toContain('acc-1'); // accountIds is the array bound to ANY($2) + }); + it('returns [] for empty ids without hitting the DB', async () => { + expect(await getMessageSummariesByIDs([], ['acc-1'])).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('listAccounts', () => { + beforeEach(() => query.mockReset()); + it('maps to capitalized AccountInfo scoped to accountIds', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'Work' }] }); + const out = await listAccounts(['acc-1']); + expect(out).toEqual([{ ID: 'acc-1', SourceType: 'imap', Identifier: 'a@b.com', DisplayName: 'Work' }]); + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); + }); + it('returns [] for empty scope without querying', async () => { + expect(await listAccounts([])).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('resolveAccountScope', () => { + beforeEach(() => query.mockReset()); + it('no account → full scope, no lookup', async () => { + expect(await resolveAccountScope('', ['acc-1', 'acc-2'])).toEqual({ accountIds: ['acc-1', 'acc-2'] }); + expect(query).not.toHaveBeenCalled(); + }); + it('known account email → narrowed to its id', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-2', protocol: 'imap', email_address: 'c@d.com', name: 'W' }] }); + expect(await resolveAccountScope('c@d.com', ['acc-2'])).toEqual({ accountIds: ['acc-2'] }); + }); + it('unknown account email → error (msgvault getAccountID parity)', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'W' }] }); + expect(await resolveAccountScope('nope@x.com', ['acc-1'])).toEqual({ error: 'account not found: nope@x.com' }); + }); +}); + +describe('messageInScope', () => { + beforeEach(() => query.mockReset()); + it('true when the id belongs to one of the accounts, scoped', async () => { + query.mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }); + expect(await messageInScope('m1', ['acc-1'])).toBe(true); + expect(query.mock.calls[0][1]).toEqual(['m1', ['acc-1']]); + }); + it('false for a foreign/absent id', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await messageInScope('m1', ['acc-1'])).toBe(false); + }); + it('false for empty scope without querying', async () => { + expect(await messageInScope('m1', [])).toBe(false); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('getTotalStats', () => { + beforeEach(() => query.mockReset()); + it('returns capitalized TotalStats keys scoped to accountIds', async () => { + query.mockResolvedValueOnce({ rows: [{ message_count: '10', active_count: '9', deleted_count: '1', total_size: '5000', attachment_count: '3', label_count: '2' }] }); + const s = await getTotalStats(['acc-1', 'acc-2']); + expect(s).toEqual({ + MessageCount: 10, ActiveMessageCount: 9, SourceDeletedMessageCount: 1, + TotalSize: 5000, AttachmentCount: 3, AttachmentSize: 0, LabelCount: 2, AccountCount: 2, + }); + expect(query.mock.calls[0][1]).toEqual([['acc-1', 'acc-2']]); + }); + it('returns a zeroed struct for empty scope without querying', async () => { + const s = await getTotalStats([]); + expect(s.MessageCount).toBe(0); + expect(s.AccountCount).toBe(0); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('aggregate', () => { + beforeEach(() => query.mockReset()); + it('groups sender by from_email and maps capitalized AggregateRow keys, scoped', async () => { + query.mockResolvedValueOnce({ rows: [{ key: 'a@b.com', count: '3', total_size: '10', attachment_count: '1', total_unique: '5' }] }); + const out = await aggregate('sender', { accountIds: ['acc-1'], limit: 50 }); + expect(out).toEqual([{ Key: 'a@b.com', Count: 3, TotalSize: 10, AttachmentSize: 0, AttachmentCount: 1, TotalUnique: 5 }]); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/m\.from_email AS key/); + expect(sql).toMatch(/account_id = ANY\(\$1\)/); + expect(params[0]).toEqual(['acc-1']); + }); + + it("time buckets by calendar year (to_char YYYY)", async () => { + query.mockResolvedValueOnce({ rows: [{ key: '2024', count: '9', total_size: '0', attachment_count: '0', total_unique: '2' }] }); + const out = await aggregate('time', { accountIds: ['acc-1'], limit: 50 }); + expect(out[0].Key).toBe('2024'); + expect(query.mock.calls[0][0]).toMatch(/to_char\(m\.date, 'YYYY'\) AS key/); + }); + + it('recipient unnests to_addresses via a lateral join', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await aggregate('recipient', { accountIds: ['acc-1'], limit: 50 }); + expect(query.mock.calls[0][0]).toMatch(/LATERAL jsonb_array_elements/); + }); +}); + +describe('searchByDomains', () => { + beforeEach(() => query.mockReset()); + it('matches any participant at a domain, scoped, and maps capitalized summaries', async () => { + query.mockResolvedValueOnce({ rows: [{ ...row, id: 'm1' }] }); + const out = await searchByDomains(['gobright.com'], null, null, 100, 0, ['acc-1']); + expect(out[0].id).toBe('m1'); + expect(out[0].to).toEqual([{ Email: 'c@d.com', Name: 'C' }]); // capitalized address keys + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/from_email ILIKE/); + expect(sql).toMatch(/to_addresses::text ILIKE/); + expect(sql).toMatch(/cc_addresses::text ILIKE/); + expect(params[0]).toEqual(['acc-1']); + expect(params).toContain('%@gobright.com'); + }); +}); + +describe('stageDeletion', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + it('resolves ids scoped to accountIds, records a STAGED batch + members, never soft-deletes', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'm1' }, { id: 'm2' }] }); // id resolution + const clientCalls = []; + const client = { query: vi.fn(async (text) => { + clientCalls.push(text); + if (/INSERT INTO mcp_deletion_batches/.test(text)) return { rows: [{ id: 'batch-1' }] }; + return { rows: [] }; + }) }; + withTransaction.mockImplementation(async (fn) => fn(client)); + const out = await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], domain: 'linkedin.com', description: 'filter' }); + expect(out).toEqual({ batchId: 'batch-1', messageCount: 2 }); + expect(query.mock.calls[0][1][0]).toEqual(['acc-1']); // id resolution scoped + expect(clientCalls.some((t) => /INSERT INTO mcp_deletion_batches/.test(t) && /'staged'/.test(t))).toBe(true); + expect(clientCalls.some((t) => /INSERT INTO mcp_deletion_batch_messages/.test(t))).toBe(true); + expect(clientCalls.some((t) => /UPDATE messages SET is_deleted/.test(t))).toBe(false); // never hard/soft deletes here + }); + + it('returns a zero count without opening a transaction when nothing matches', async () => { + query.mockResolvedValueOnce({ rows: [] }); + const out = await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], from: 'x' }); + expect(out).toEqual({ batchId: null, messageCount: 0 }); + expect(withTransaction).not.toHaveBeenCalled(); + }); +}); + +describe('structured after/before args (Wave D Fix 5)', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + it('rejects a malformed after/before with msgvault getDateArg wording, before any SQL runs', async () => { + await expect(listMessages({ accountIds: ['a'], after: 'notadate', limit: 50, offset: 0 })) + .rejects.toThrow('invalid after date "notadate": expected YYYY-MM-DD'); + // Strict YYYY-MM-DD (msgvault time.Parse("2006-01-02")) — no loose forms. + await expect(aggregate('sender', { accountIds: ['a'], before: '2025-1-2', limit: 10 })) + .rejects.toThrow('invalid before date "2025-1-2": expected YYYY-MM-DD'); + await expect(searchByDomains(['x.com'], null, '2025-02-31', 10, 0, ['a'])) + .rejects.toThrow('invalid before date "2025-02-31": expected YYYY-MM-DD'); + await expect(stageDeletion({ userId: 'u', accountIds: ['a'], before: 'garbage' })) + .rejects.toThrow('invalid before date "garbage": expected YYYY-MM-DD'); + expect(query).not.toHaveBeenCalled(); + }); + + it('binds valid dates as midnight-UTC ISO with an EXCLUSIVE before (<), matching lexicalRepo', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await listMessages({ accountIds: ['a'], after: '2025-01-02', before: '2025-02-03', limit: 50, offset: 0 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.date >= $2'); + expect(sql).toContain('m.date < $3'); + expect(sql).not.toContain('m.date <='); + expect(params[1]).toBe('2025-01-02T00:00:00.000Z'); + expect(params[2]).toBe('2025-02-03T00:00:00.000Z'); + }); +}); + +describe('stage_deletion / search predicate parity (Wave D Fix 3)', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + // Rebuild the term clauses through lexicalRepo's shared owner with the same + // ordinals both paths allocate ($1 = accountIds, terms from $2). + function expectedClauses(terms) { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { clauses: terms.map((t) => freeTextTermClause(t, false, bind)), params }; + } + + it('builds the staging WHERE from the ranked/search_fts builder the search path uses, stopword guard included', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], parsed: parseQuery('the invoice') }); + const [sql, params] = query.mock.calls[0]; + const expected = expectedClauses(['the', 'invoice']); + for (const clause of expected.clauses) expect(sql).toContain(clause); + expect(params.slice(1, 5)).toEqual(expected.params); + // Ranked construction, not the legacy ILIKE/plainto fork the staging path + // used to build (preview and staged set must agree): search_fts-first match + // with the stopword vacuity guard, so `the invoice` stages the invoice set + // instead of zero rows ("the" normalizes to an empty tsquery). + expect(sql).toContain('m.search_fts @@'); + expect(sql).toContain('numnode('); + }); + + it('emits byte-identical term predicates to a lexical search of the same query', async () => { + const searchCalls = []; + const client = async (text, params) => { searchCalls.push({ text, params }); return { rows: [] }; }; + await searchLexical(client, { + parsed: parseQuery('the invoice'), accountIds: ['acc-1'], + folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + query.mockResolvedValueOnce({ rows: [] }); + await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], parsed: parseQuery('the invoice') }); + const stagingSql = query.mock.calls[0][0]; + const searchSql = searchCalls[0].text; + const expected = expectedClauses(['the', 'invoice']); + for (const clause of expected.clauses) { + expect(searchSql).toContain(clause); // same ordinals in both WHEREs… + expect(stagingSql).toContain(clause); // …so the clause text is identical + } + // And the bind values line up pairwise (like + fts per term). + expect(searchCalls[0].params.slice(1, 5)).toEqual(query.mock.calls[0][1].slice(1, 5)); + }); +}); + +describe('countEnforceableQueryPredicates', () => { + const n = (q) => countEnforceableQueryPredicates(parseQuery(q)); + + it('counts supported structured filters and usable positive free-text terms', () => { + expect(n('from:amazon invoice')).toBe(2); // from: + invoice + expect(n('is:unread')).toBe(1); // one structured predicate + expect(n('invoice urgent')).toBe(2); // two positive terms + }); + + it('returns 0 when every token is discarded (the stage-everything hazard)', () => { + expect(n('label:promotions')).toBe(0); // unsupported operator only + expect(n('-newsletter')).toBe(0); // negation-only (not enforced on staging) + expect(n('a')).toBe(0); // sub-2-char + expect(n('!!!')).toBe(0); // punctuation-only + expect(n('in:inbox')).toBe(0); // folder scope is not a row predicate + }); + + it('mirrors resolveStageDeletionIds: a real term survives alongside an unsupported operator', () => { + expect(n('invoice label:promotions')).toBe(1); + }); + + it('treats a null parsed query (structured-filter path) as zero', () => { + expect(countEnforceableQueryPredicates(null)).toBe(0); + }); +}); diff --git a/backend/src/mcp/envelope.js b/backend/src/mcp/envelope.js new file mode 100644 index 0000000..d143b7f --- /dev/null +++ b/backend/src/mcp/envelope.js @@ -0,0 +1,30 @@ +export const TOTAL_COUNT_UNKNOWN = -1; + +// msgvault wire timestamps are Go-marshaled RFC3339 with no sub-second part +// (SQLite second-precision sent_at; vector/stats.go formatTime uses +// time.RFC3339 explicitly). engineAdapter's toISO emits Date.toISOString() +// milliseconds — an internal convention REST shares — so the MCP layer +// re-formats at emission instead of touching engineAdapter (Wave D's file). +// TODO(seam): fold into engineAdapter.toISO if it ever becomes wire-final. +export function toRFC3339(value) { + if (typeof value !== 'string' || !value) return value; + return value.replace(/\.\d+(?=(?:Z|[+-]\d\d:\d\d)$)/, ''); +} + +// Re-shape one MessageSummary for the wire: today only sent_at needs +// re-formatting. Non-mutating; tolerates partial summaries from tests. +export function wireSummary(summary) { + if (!summary || typeof summary !== 'object') return summary; + if (typeof summary.sent_at !== 'string' || !summary.sent_at) return summary; + return { ...summary, sent_at: toRFC3339(summary.sent_at) }; +} + +export function newPaginatedResponse(data, total, offset) { + const d = data || []; + return { data: d, total, returned: d.length, offset, has_more: offset + d.length < total }; +} + +export function newPaginatedResponseNoTotal(data, offset, hasMore) { + const d = data || []; + return { data: d, total: TOTAL_COUNT_UNKNOWN, returned: d.length, offset, has_more: hasMore }; +} diff --git a/backend/src/mcp/envelope.test.js b/backend/src/mcp/envelope.test.js new file mode 100644 index 0000000..c4bdb9c --- /dev/null +++ b/backend/src/mcp/envelope.test.js @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, TOTAL_COUNT_UNKNOWN, toRFC3339, wireSummary } from './envelope.js'; + +describe('newPaginatedResponse', () => { + it('computes has_more from offset+returned vs total', () => { + expect(newPaginatedResponse([{ id: 'a' }, { id: 'b' }], 10, 0)).toEqual({ + data: [{ id: 'a' }, { id: 'b' }], total: 10, returned: 2, offset: 0, has_more: true, + }); + expect(newPaginatedResponse([{ id: 'a' }], 1, 0).has_more).toBe(false); + }); + it('coerces null data to an empty array', () => { + expect(newPaginatedResponse(null, 0, 0).data).toEqual([]); + }); +}); + +describe('newPaginatedResponseNoTotal', () => { + it('always reports total -1 and echoes has_more', () => { + const r = newPaginatedResponseNoTotal([{ id: 'a' }], 20, true); + expect(r).toEqual({ data: [{ id: 'a' }], total: TOTAL_COUNT_UNKNOWN, returned: 1, offset: 20, has_more: true }); + expect(TOTAL_COUNT_UNKNOWN).toBe(-1); + }); +}); + +describe('toRFC3339', () => { + it('strips fractional seconds (msgvault wire timestamps are Go RFC3339, no millis)', () => { + expect(toRFC3339('2024-01-01T00:00:00.000Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('2024-01-01T00:00:00.123Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('2024-01-01T00:00:00.123456+02:00')).toBe('2024-01-01T00:00:00+02:00'); + }); + it('passes through already-clean, empty, and non-string values', () => { + expect(toRFC3339('2024-01-01T00:00:00Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('')).toBe(''); + expect(toRFC3339(undefined)).toBe(undefined); + }); +}); + +describe('wireSummary', () => { + it('reformats sent_at to RFC3339 without mutating the input', () => { + const s = { id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }; + expect(wireSummary(s)).toEqual({ id: 'm1', sent_at: '2024-01-01T00:00:00Z' }); + expect(s.sent_at).toBe('2024-01-01T00:00:00.000Z'); + }); + it('leaves summaries without a sent_at string untouched', () => { + const s = { id: 'm1' }; + expect(wireSummary(s)).toBe(s); + expect(wireSummary(null)).toBe(null); + }); +}); diff --git a/backend/src/mcp/goldenParity.test.js b/backend/src/mcp/goldenParity.test.js new file mode 100644 index 0000000..2c503b8 --- /dev/null +++ b/backend/src/mcp/goldenParity.test.js @@ -0,0 +1,319 @@ +// Golden-fixture parity: msgvault's wire shapes (internal/query/models.go, +// internal/mcp/handlers.go, internal/vector/stats.go) transcribed as key+type +// shapes and diffed field-for-field against Mailflow's output. Drives REAL code +// end-to-end with only I/O mocked, so the diff pins the true wire shape. +// +// Divergences the diff intentionally accepts: D6 UUID-string ids (token 'uuid' +// matches any string), ≤1 semantic excerpt, and the capitalized-key split +// (Address/AttachmentInfo/AccountInfo/AggregateRow/TotalStats have no Go json +// tags → Go-default caps; MessageSummary/getMessageResponse are snake_case). +// +// Shapes live inline here (not separate fixtures/*.json files) — a deliberate +// consolidation; the diffing is identical and the reference sits beside the test. +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +vi.mock('../services/search/queryParser.js', () => ({ parseQuery: vi.fn(() => ({ filters: [], terms: [{ value: 'hello', negate: false }], unsupported: [], errors: [] })) })); +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +vi.mock('../services/embeddings/generations.js', () => ({ activeGeneration: vi.fn(), buildingGeneration: vi.fn(), chunkCount: vi.fn() })); +vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); +vi.mock('../services/embeddings/config.js', () => ({ generationFingerprint: vi.fn(() => 'fp'), resolveEmbedConfig: vi.fn(async () => ({ enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 })) })); + +import { query, withTransaction } from '../services/db.js'; +import { search } from '../services/search/searchService.js'; +import * as generations from '../services/embeddings/generations.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { rowToMessageSummary, rowToMessageDetail } from './engineAdapter.js'; +import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; +import { + handleGetMessage, handleListMessages, handleGetStats, handleAggregate, + handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, handleSearchByDomains, +} from './messageTools.js'; +import { HANDLERS } from './tools.js'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +// ---- transcribed reference shapes --------------------------------------------- +const Address = { Email: 'string', Name: 'string' }; +const AttachmentInfo = { ID: 'int', Filename: 'string', MimeType: 'string', Size: 'int', ContentHash: 'string', URL: 'string', StoragePath: 'string' }; +const Generation = { id: 'int', model: 'string', dimension: 'int', fingerprint: 'string', state: 'string' }; +const MessageSummary = { + id: 'uuid', 'source_id?': 'uuid', source_message_id: 'string', conversation_id: 'uuid', + source_conversation_id: 'string', subject: 'string', snippet: 'string', + from_email: 'string', from_name: 'string', 'to?': [Address], 'cc?': [Address], + sent_at: 'string', size_estimate: 'int', has_attachments: 'bool', + attachment_count: 'int', labels: ['string'], message_type: 'string', +}; +const Match = { snippet: 'string', 'char_offset?': 'int', 'line?': 'int', 'score?': 'number' }; +// hybridScoreBreakdown (handlers.go:563-572): every field omitempty — rrf only +// fuses in mode=hybrid, subject_boosted only when true. +const HybridScore = { 'rrf?': 'number', 'bm25?': 'number', 'vector?': 'number', 'subject_boosted?': 'bool' }; +// searchMessageItem (handlers.go:331-342): MessageSummary + matches/ +// matches_truncated/score, all three Go-omitempty. +const SearchMessageItem = { ...MessageSummary, 'matches?': [Match], 'matches_truncated?': 'bool', 'score?': HybridScore }; +// searchMessageBodiesResponse (handlers.go:586-595): paginated (no-total) + +// mode/pool_saturated/generation — shared by keyword and vector/hybrid modes. +const SearchBodiesEnvelope = { + data: [SearchMessageItem], total: 'int', returned: 'int', offset: 'int', has_more: 'bool', + mode: 'string', pool_saturated: 'bool', generation: Generation, +}; +const SHAPES = { + message_summary: MessageSummary, + message_detail: { ...MessageSummary, from: [Address], to: [Address], cc: [Address], bcc: [Address], body_text: 'string', body_html: 'string', attachments: [AttachmentInfo] }, + search_metadata: { data: [MessageSummary], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + list_messages: { data: [MessageSummary], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + get_message: { + id: 'uuid', source_message_id: 'string', conversation_id: 'uuid', source_conversation_id: 'string', + subject: 'string', 'message_type?': 'string', snippet: 'string', sent_at: 'string', + size_estimate: 'int', has_attachments: 'bool', from: [Address], to: [Address], cc: [Address], bcc: [Address], + body_text: 'string', body_html: 'string', 'body_format?': 'string', body_length: 'int', + body_returned: 'int', offset: 'int', has_more: 'bool', labels: ['string'], attachments: [AttachmentInfo], + }, + get_stats: { + stats: { MessageCount: 'int', ActiveMessageCount: 'int', SourceDeletedMessageCount: 'int', TotalSize: 'int', AttachmentCount: 'int', AttachmentSize: 'int', LabelCount: 'int', AccountCount: 'int' }, + accounts: [{ ID: 'uuid', SourceType: 'string', Identifier: 'string', DisplayName: 'string' }], + 'vector_search?': { enabled: 'bool', active_generation: { id: 'int', model: 'string', dimension: 'int', fingerprint: 'string', state: 'string', 'activated_at?': 'string', message_count: 'int' }, 'building_generation?': {}, missing_embeddings_total: 'int' }, + }, + aggregate: [{ Key: 'string', Count: 'int', TotalSize: 'int', AttachmentSize: 'int', AttachmentCount: 'int', TotalUnique: 'int' }], + find_similar: { seed_message_id: 'uuid', returned: 'int', generation: Generation, messages: [MessageSummary] }, + search_in_message: { data: [Match], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + stage_deletion: { batch_id: 'uuid', message_count: 'int', status: 'string', next_step: 'string' }, + search_message_bodies: SearchBodiesEnvelope, + semantic_search_messages: SearchBodiesEnvelope, + search_by_domains: [MessageSummary], // raw array, no envelope (handlers.go:1984-1989) + ping: { pong: 'bool' }, // Mailflow-specific health tool — no msgvault counterpart (documented divergence) +}; + +// ---- diffKeys: [] on parity, else a list of divergence paths ------------------- +function typeOk(val, token) { + const t = token.replace(/\?$/, ''); + if (t.includes('uuid')) return typeof val === 'string'; // D6 id divergence + if (t === 'string') return typeof val === 'string'; + if (t === 'int' || t === 'number' || t === 'float') return typeof val === 'number'; + if (t === 'bool') return typeof val === 'boolean'; + return true; +} +function isOptional(shapeKey, shapeVal) { + return shapeKey.endsWith('?') || (typeof shapeVal === 'string' && shapeVal.endsWith('?')); +} +function diffKeys(actual, shape, path = '$') { + const out = []; + if (typeof shape === 'string') { + if (!typeOk(actual, shape)) out.push(`${path}: type mismatch (want ${shape}, got ${typeof actual})`); + return out; + } + if (Array.isArray(shape)) { + if (!Array.isArray(actual)) { out.push(`${path}: want array, got ${typeof actual}`); return out; } + actual.forEach((a, i) => out.push(...diffKeys(a, shape[0], `${path}[${i}]`))); + return out; + } + if (actual === null || typeof actual !== 'object' || Array.isArray(actual)) { out.push(`${path}: want object, got ${actual === null ? 'null' : typeof actual}`); return out; } + const realKeys = new Set(); + for (const sk of Object.keys(shape)) { + const k = sk.endsWith('?') ? sk.slice(0, -1) : sk; + realKeys.add(k); + if (!(k in actual)) { if (!isOptional(sk, shape[sk])) out.push(`${path}.${k}: missing`); continue; } + out.push(...diffKeys(actual[k], shape[sk], `${path}.${k}`)); + } + for (const k of Object.keys(actual)) if (!realKeys.has(k)) out.push(`${path}.${k}: extra (not in msgvault shape)`); + return out; +} + +const realRow = { + id: '11111111-1111-1111-1111-111111111111', account_id: 'acc-1', message_id: '', thread_id: 'tid-9', + subject: 'Hi', snippet: 's', from_email: 'a@b.com', from_name: 'A', + to_addresses: [{ name: 'C', email: 'c@d.com' }], cc_addresses: [{ name: 'E', email: 'e@f.com' }], + date: new Date('2024-01-01T00:00:00Z'), has_attachments: true, + attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], + flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello world', body_html: '

hello world

', +}; +const jsonOf = (r) => JSON.parse(r.content[0].text); +const scope = { userId: 'u', accountIds: ['acc-1'] }; + +beforeEach(() => { + query.mockReset(); withTransaction.mockReset(); search.mockReset(); matchFromChunk.mockReset(); + generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); + resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); +}); + +// Every function this suite vi.mock()s must actually exist on its real module — +// a renamed/never-implemented seam (e.g. generations.chunkCount) otherwise passes +// here while throwing live. Catches the missing/renamed-export drift class only, +// not value-shape drift. +describe('mock-drift guard: mocked seams exist on their real modules', () => { + // hybrid.js is intentionally omitted: importActual runs its module body, which + // references vectorStore.fusedSearch — not in this suite's vectorStore mock. + it.each([ + ['generations', () => generations, '../services/embeddings/generations.js'], + ['vectorStore', () => ({ loadVector, annSearch }), '../services/embeddings/vectorStore.js'], + ['searchService', () => ({ search }), '../services/search/searchService.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); + +describe('golden parity: structural mappers', () => { + it('rowToMessageSummary matches MessageSummary (snake_case + capitalized Address, id divergence excepted)', () => { + // Round-trip through JSON so absent optional keys behave like the wire. + const s = JSON.parse(JSON.stringify(rowToMessageSummary(realRow))); + expect(diffKeys(s, SHAPES.message_summary)).toEqual([]); + }); + it('rowToMessageDetail matches MessageDetail (capitalized AttachmentInfo)', () => { + const d = JSON.parse(JSON.stringify(rowToMessageDetail(realRow))); + expect(diffKeys(d, SHAPES.message_detail)).toEqual([]); + }); +}); + +describe('golden parity: message tool envelopes', () => { + it('get_message → getMessageResponse', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); + const b = jsonOf(await handleGetMessage({ id: realRow.id }, scope)); + expect(diffKeys(b, SHAPES.get_message)).toEqual([]); + }); + + it('list_messages → paginated MessageSummary envelope', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); // listMessages + const b = jsonOf(await handleListMessages({}, scope)); + expect(diffKeys(b, SHAPES.list_messages)).toEqual([]); + }); + + it('aggregate → capitalized AggregateRow array', async () => { + query.mockResolvedValueOnce({ rows: [{ key: 'a@b.com', count: '3', total_size: '10', attachment_count: '1', total_unique: '5' }] }); + const b = jsonOf(await handleAggregate({ group_by: 'sender' }, scope)); + expect(diffKeys(b, SHAPES.aggregate)).toEqual([]); + }); + + it('get_stats → {stats, accounts, vector_search} with capitalized structs', async () => { + query + .mockResolvedValueOnce({ rows: [{ message_count: '10', active_count: '9', deleted_count: '1', total_size: '5000', attachment_count: '3', label_count: '2' }] }) // getTotalStats + .mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'Work' }] }) // listAccounts + .mockResolvedValueOnce({ rows: [{ n: '4' }] }); // collectStats missingCount + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); // epoch seconds → RFC3339 wire + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(1000); + const b = jsonOf(await handleGetStats({}, scope)); + expect(diffKeys(b, SHAPES.get_stats)).toEqual([]); + // RFC3339 without sub-second digits (vector/stats.go:146-153 formatTime). + expect(b.vector_search.active_generation.activated_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('find_similar_messages → seed/returned/generation/messages', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: realRow.id, score: 0.9, rank: 1 }]); + query + .mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }) // messageInScope(seed) + .mockResolvedValueOnce({ rows: [realRow] }); // getMessageSummariesByIDs + const b = jsonOf(await handleFindSimilarMessages({ message_id: realRow.id }, scope)); + expect(diffKeys(b, SHAPES.find_similar)).toEqual([]); + }); + + it('search_in_message (keyword) → Match envelope with byte char_offset', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); // getMessage + const b = jsonOf(await handleSearchInMessage({ id: realRow.id, query: 'hello' }, scope)); + expect(diffKeys(b, SHAPES.search_in_message)).toEqual([]); + expect(b.data[0].char_offset).toBe(Buffer.from(realRow.body_text, 'utf8').indexOf(Buffer.from('hello', 'utf8'))); + }); + + it('stage_deletion → {batch_id, message_count, status, next_step}', async () => { + query.mockResolvedValueOnce({ rows: [{ id: realRow.id }] }); // id resolution + withTransaction.mockImplementation(async (fn) => fn({ query: vi.fn(async (text) => (/INSERT INTO mcp_deletion_batches/.test(text) ? { rows: [{ id: 'batch-1' }] } : { rows: [] })) })); + const b = jsonOf(await handleStageDeletion({ domain: 'linkedin.com' }, scope)); + expect(diffKeys(b, SHAPES.stage_deletion)).toEqual([]); + expect(b.status).toBe('pending'); // msgvault manifest.StatusPending literal (manifest.go:25) + }); +}); + +describe('golden parity: search tool envelopes', () => { + it('search_metadata → paginated MessageSummary envelope (real hydration)', async () => { + search.mockResolvedValue({ messages: [{ id: realRow.id }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + query.mockResolvedValueOnce({ rows: [realRow] }); // getMessageSummariesByIDs hydration + const b = jsonOf(await handleSearchMetadata({ query: 'x' }, scope)); + expect(diffKeys(b, SHAPES.search_metadata)).toEqual([]); + // Go time.Time wire format: RFC3339 with no sub-second digits. + expect(b.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('search_message_bodies → searchMessageItem envelope; matches/matches_truncated omitted when empty/false (handlers.go:339-341)', async () => { + search.mockResolvedValue({ + messages: [ + { id: realRow.id, body_text: 'hello world, hello again' }, // 1 merged excerpt + { id: 'no-hit-1111-1111-1111-111111111111', body_text: 'nothing relevant' }, // 0 excerpts + ], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + query.mockResolvedValueOnce({ rows: [realRow, { ...realRow, id: 'no-hit-1111-1111-1111-111111111111' }] }); // hydration + const b = jsonOf(await handleSearchMessageBodies({ query: 'hello' }, scope)); + expect(diffKeys(b, SHAPES.search_message_bodies)).toEqual([]); + expect(b.mode).toBe('keyword'); + expect(b.total).toBe(-1); // body search never counts + expect(b.generation).toEqual({ id: 0, model: '', dimension: 0, fingerprint: '', state: '' }); + // diffKeys accepts optional keys whether present or absent, so the + // omitempty contract is asserted explicitly on both sides: + expect(b.data[0]).toHaveProperty('matches'); + expect(b.data[0]).not.toHaveProperty('matches_truncated'); // false → omitted + expect(b.data[1]).not.toHaveProperty('matches'); // empty → omitted + expect(b.data[1]).not.toHaveProperty('matches_truncated'); + }); + + it('semantic_search_messages → mode/pool_saturated/generation envelope; explain score omitempty (handlers.go:563-572)', async () => { + search.mockResolvedValue({ + messages: [{ + message_id: realRow.id, id: realRow.id, + best_chunk: { chunk_index: 0, char_start: 0, char_end: 10, score: 0.9 }, + score: { rrf: 0.03, bm25: 1.2, vector: 0.9, subject_boosted: false }, + }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, + pool_saturated: true, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + matchFromChunk.mockResolvedValue({ char_offset: 5, snippet: 'hello', line: 1, score: 0.9 }); + query.mockResolvedValueOnce({ rows: [realRow] }); // hydration + const b = jsonOf(await handleSemanticSearchMessages({ query: 'hello', mode: 'vector', explain: true }, scope)); + expect(diffKeys(b, SHAPES.semantic_search_messages)).toEqual([]); + expect(b.mode).toBe('vector'); + expect(b.pool_saturated).toBe(true); + expect(b.total).toBe(-1); + expect(b.generation).toEqual({ id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }); + // omitempty asserted explicitly: no rrf in mode=vector (nothing to fuse), + // no subject_boosted when false. + expect(b.data[0].score).toEqual({ bm25: 1.2, vector: 0.9 }); + expect(b.data[0].matches).toHaveLength(1); // ≤1 excerpt — documented divergence from msgvault's ≤5 + }); + + it('search_by_domains → raw MessageSummary array, no envelope (handlers.go:1951-1989)', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); + const b = jsonOf(await handleSearchByDomains({ domains: 'b.com' }, scope)); + expect(diffKeys(b, SHAPES.search_by_domains)).toEqual([]); + expect(b[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('ping → {pong:true} (Mailflow-specific health tool)', async () => { + const b = jsonOf(await HANDLERS.ping({}, scope)); + expect(diffKeys(b, SHAPES.ping)).toEqual([]); + expect(b.pong).toBe(true); + }); +}); + +describe('no SQL in MCP handler modules (one-seam invariant)', () => { + const here = dirname(fileURLToPath(import.meta.url)); + const read = (n) => readFileSync(join(here, n), 'utf8'); + for (const file of ['searchTools.js', 'messageTools.js']) { + it(`${file} contains no raw SQL or db.js import`, () => { + const src = read(file); + expect(src).not.toMatch(/\bSELECT\b|\bINSERT\b|\bUPDATE\b|\bDELETE FROM\b/); + expect(src).not.toMatch(/from '\.\.\/services\/db\.js'/); + expect(src).not.toMatch(/\bpool\b/); + }); + } +}); diff --git a/backend/src/mcp/messageTools.js b/backend/src/mcp/messageTools.js new file mode 100644 index 0000000..c5246f4 --- /dev/null +++ b/backend/src/mcp/messageTools.js @@ -0,0 +1,485 @@ +// The msgvault message/aggregate/deletion tool handlers. Every read goes through +// engineAdapter (all SQL lives there); handlers only shape and page. Byte offsets +// throughout are UTF-8 bytes into the raw body (msgvault wire contract). +import { + getMessage, rowToMessageDetail, listMessages, listAccounts, getTotalStats, aggregate, searchByDomains, + getMessageSummariesByIDs, stageDeletion, resolveAccountScope, messageInScope, countEnforceableQueryPredicates, +} from './engineAdapter.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { bodyByteSliceRange, contextWindow, findTermMatches } from './bodyMatch.js'; +import { jsonResult, errorResult } from './result.js'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, toRFC3339, wireSummary } from './envelope.js'; +import { + searchLimitArg, offsetArg, + queryParseErrorMessage, unsupportedSearchOperatorMessage, HYBRID_RANKING_WINDOW, +} from './searchTools.js'; +import { collectStats } from './vectorStats.js'; +import { translateVectorError } from './vectorErrors.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; // phase-4 public face +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; // phase 3 +import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; // phase-5-owned + +const DEFAULT_BODY_CHARS = 2000; +const MAX_BODY_CHARS = 4000; +const MAX_LIMIT = 1000; + +// msgvault limitArg: absent/non-number → default; negative/NaN → 0; clamp to 1000. +// (Distinct from searchLimitArg's 20/50 search clamp.) +function limitArg(args, key, def) { + const raw = args[key]; + if (typeof raw !== 'number') return def; + if (Number.isNaN(raw) || raw < 0) return 0; + if (raw > MAX_LIMIT) return MAX_LIMIT; + return Math.trunc(raw); +} + +// msgvault getDateArg (handlers.go:264-275): an optional after/before arg is a +// strict YYYY-MM-DD; anything else errors at the handler instead of leaking a +// raw Postgres cast error to the wire. Non-strings/empty are "no filter", and +// JS Date rollover (2024-02-31 → Mar 1) is rejected via the Y/M/D round-trip +// (Go time.Parse errors "day out of range"). Returns { value } or { error }. +function dateArg(args, key) { + const v = args[key]; + if (typeof v !== 'string' || v === '') return { value: undefined }; + const err = { error: `invalid ${key} date "${v}": expected YYYY-MM-DD` }; + const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v); + if (!m) return err; + const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])]; + const t = new Date(Date.UTC(y, mo - 1, d)); + if (t.getUTCFullYear() !== y || t.getUTCMonth() !== mo - 1 || t.getUTCDate() !== d) return err; + return { value: v }; +} + +export const getMessageDef = { + name: 'get_message', + description: + 'Get message details including recipients, labels, attachments, and a slice of the message body. ' + + 'Returns plain text when available; HTML-only messages return a body_html slice with body_format=html. ' + + 'Body paging mirrors search pagination: body_length=total bytes, offset=where this chunk starts, body_returned=bytes in this chunk, has_more=more body follows. ' + + 'To read sequentially: call again with offset += body_returned. ' + + 'To jump to a known match location: use center_at= to center the window on that location. ' + + 'Note: snippet is pre-stored source metadata (may be empty for non-Gmail sources).', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'Message ID' }, + offset: { type: 'number', description: 'Byte offset from the start of the selected body to begin reading (default 0). Ignored when center_at is provided.' }, + center_at: { type: 'number', description: 'Byte offset from the start of the selected body to center the window on. Takes precedence over offset.' }, + max_chars: { type: 'number', description: 'Maximum selected-body bytes to return (default 2000, max 4000). Values above 4000 are clamped to 4000; zero or negative values use the default.' }, + body_format: { type: 'string', enum: ['auto', 'text', 'html'], description: 'Which body representation to page: auto (default, plain text when available, HTML fallback), text, or html.' }, + full_body: { type: 'boolean', description: 'Return the complete selected body in one response, ignoring offset, center_at, and max_chars. Use only when the full content is explicitly needed.' }, + }, + required: ['id'], + }, +}; + +export async function handleGetMessage(args, scope) { + const id = args.id; + if (!id || typeof id !== 'string') return errorResult('id parameter is required'); + const row = await getMessage(id, scope.accountIds); + if (!row) return errorResult('message not found'); + const detail = rowToMessageDetail(row); + + let maxChars = Number(args.max_chars); + if (!Number.isFinite(maxChars) || maxChars <= 0) maxChars = DEFAULT_BODY_CHARS; + else if (maxChars > MAX_BODY_CHARS) maxChars = MAX_BODY_CHARS; + + const requested = args.body_format || 'auto'; + let full = detail.body_text; + let bodyFormat = 'text'; + if (requested === 'auto') { + if (!full && detail.body_html) { full = detail.body_html; bodyFormat = 'html'; } + } else if (requested === 'html') { + full = detail.body_html; bodyFormat = 'html'; + } else if (requested !== 'text') { + return errorResult('body_format must be one of auto, text, html'); + } + + const buf = Buffer.from(full || '', 'utf8'); + const bodyLen = buf.length; + + let start, end; + if (args.full_body === true) { + start = 0; end = bodyLen; + } else if (Number.isFinite(Number(args.center_at)) && Number(args.center_at) >= 0) { + [start, end] = contextWindow(bodyLen, Math.trunc(Number(args.center_at)), 0, maxChars); + } else { + start = Math.min(Number.isFinite(Number(args.offset)) ? Math.trunc(Number(args.offset)) : 0, bodyLen); + end = Math.min(start + maxChars, bodyLen); + } + + const { text: slice, adjStart, adjEnd } = bodyByteSliceRange(buf, start, end); + const returnedBytes = Buffer.byteLength(slice, 'utf8'); + + return jsonResult({ + id: detail.id, + source_message_id: detail.source_message_id, + conversation_id: detail.conversation_id, + source_conversation_id: detail.source_conversation_id, + subject: detail.subject, + message_type: detail.message_type, + snippet: detail.snippet, + sent_at: toRFC3339(detail.sent_at), + size_estimate: detail.size_estimate, + has_attachments: detail.has_attachments, + from: detail.from, + to: detail.to, + cc: detail.cc, + bcc: detail.bcc, + body_text: bodyFormat === 'html' ? '' : slice, + body_html: bodyFormat === 'html' ? slice : '', + body_format: bodyFormat, + body_length: bodyLen, + body_returned: returnedBytes, + offset: adjStart, + has_more: adjEnd < bodyLen, + labels: detail.labels, + attachments: detail.attachments, + }); +} + +export const listMessagesDef = { + name: 'list_messages', + description: + 'List messages with optional filters, newest-first. ' + + "Pass conversation_id to enumerate a thread's messages, then call get_message(id) per message to read bodies — " + + 'there is deliberately no bulk body fetch, to avoid loading huge threads into the context window. ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, total, returned, offset, has_more. ' + + 'total=-1 because the full count is not computed; use has_more for paging.', + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + from: { type: 'string', description: 'Filter by sender email address' }, + to: { type: 'string', description: 'Filter by recipient email address' }, + label: { type: 'string', description: 'Filter by Gmail label' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + conversation_id: { type: 'string', description: 'Filter by conversation/thread ID' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + }, +}; + +export async function handleListMessages(args, scope) { + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Over-fetch one to compute has_more without a count query (msgvault limit+1). + let rows = await listMessages({ + accountIds: acc.accountIds, + from: args.from, to: args.to, label: args.label, + hasAttachment: args.has_attachment === true, + after: after.value, before: before.value, + conversationId: args.conversation_id, + limit: limit + 1, offset, + }); + const hasMore = rows.length > limit; + if (hasMore) rows = rows.slice(0, limit); + return jsonResult(newPaginatedResponseNoTotal(rows.map(wireSummary), offset, hasMore)); +} + +export const getStatsDef = { + name: 'get_stats', + description: 'Get archive overview: total messages, size, attachment count, and accounts.', + inputSchema: { type: 'object', properties: {} }, +}; + +// A broken vector sub-query must never blank the whole stats response (msgvault +// best-effort semantics); on any failure the vector_search field is simply omitted. +async function safeCollectStats(accountIds) { + try { return await collectStats(accountIds); } catch { return null; } +} + +export async function handleGetStats(_args, scope) { + const stats = await getTotalStats(scope.accountIds); + const accounts = await listAccounts(scope.accountIds); + const resp = { stats, accounts }; + const vs = await safeCollectStats(scope.accountIds); + if (vs) resp.vector_search = vs; // omitempty: present only when vector search is enabled + return jsonResult(resp); +} + +const AGGREGATE_GROUP_BY = ['sender', 'recipient', 'domain', 'label', 'time']; + +export const aggregateDef = { + name: 'aggregate', + description: + 'Get grouped statistics (top senders, recipients, domains, labels, or message volume by calendar year). ' + + 'Returns a JSON array of objects with fields Key, Count, TotalSize, AttachmentSize, AttachmentCount, and TotalUnique.', + inputSchema: { + type: 'object', + properties: { + group_by: { type: 'string', enum: ['sender', 'recipient', 'domain', 'label', 'time'], description: 'Dimension to group by. When \'time\', buckets are by calendar year only (Key is a year string like "2024").' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 50)' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + }, + required: ['group_by'], + }, +}; + +export async function handleAggregate(args, scope) { + const groupBy = args.group_by || ''; + if (!groupBy) return errorResult('group_by parameter is required'); + if (!AGGREGATE_GROUP_BY.includes(groupBy)) return errorResult('invalid group_by: ' + groupBy); + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const rows = await aggregate(groupBy, { + accountIds: acc.accountIds, + after: after.value, before: before.value, + limit: limitArg(args, 'limit', 50), + }); + return jsonResult(rows); // raw array, no envelope (msgvault parity) +} + +export const searchByDomainsDef = { + name: 'search_by_domains', + description: 'Find emails where any participant (from, to, or cc) belongs to one of the given domains. Useful for finding all communication with a company regardless of direction.', + inputSchema: { + type: 'object', + properties: { + domains: { type: 'string', description: "Comma-separated domain names (e.g. 'gobright.com,ascentae.com')" }, + limit: { type: 'number', description: 'Maximum results to return (default 100)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + }, + required: ['domains'], + }, +}; + +export async function handleSearchByDomains(args, scope) { + const domainsStr = (args.domains || '').trim(); + if (!domainsStr) return errorResult('domains is required'); + const domains = domainsStr.split(',').map((d) => d.trim()).filter(Boolean); + if (!domains.length) return errorResult('at least one domain is required'); + const limit = limitArg(args, 'limit', 100); + const offset = limitArg(args, 'offset', 0); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const results = await searchByDomains(domains, after.value, before.value, limit, offset, scope.accountIds); + return jsonResult(results.map(wireSummary)); // raw array (msgvault parity) +} + +export const findSimilarMessagesDef = { + name: 'find_similar_messages', + description: 'Find messages whose embeddings are closest to the given message. Requires vector search to be configured and an active index generation.', + inputSchema: { + type: 'object', + properties: { + message_id: { type: 'string', description: 'Seed message ID; its embedding is used as the query vector' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + message_type: { type: 'string', description: 'Restrict results to one message type, such as email, sms, mms, fbmessenger, or calendar_event' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + }, + required: ['message_id'], + }, +}; + +export async function handleFindSimilarMessages(args, scope) { + const seedId = args.message_id; + if (!seedId || typeof seedId !== 'string') return errorResult('message_id parameter is required'); + let limit = Number(args.limit); + if (!Number.isFinite(limit) || limit < 1) limit = 20; + // msgvault clamps to the hybrid page cap (handlers.go:885-888). + if (limit > HYBRID_RANKING_WINDOW) limit = HYBRID_RANKING_WINDOW; + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const messageType = typeof args.message_type === 'string' ? args.message_type.trim().toLowerCase() : ''; + + try { + const { generation: gen } = await resolveActiveGenerationFromConfig(); + + // Owner-scope the seed: loadVector is not account-aware, so a foreign/unknown + // seed id must behave like get_message on a foreign id (contract: every tool + // call is owner-scoped; also closes an embedding-existence oracle). + if (!(await messageInScope(seedId, acc.accountIds))) return errorResult('message not found'); + + let seed; + try { seed = await loadVector(seedId); } + catch (e) { return errorResult(`load seed vector: ${e.message}`); } + + const filter = { accountIds: acc.accountIds }; + if (after.value) filter.after = after.value; + if (before.value) filter.before = before.value; + if (args.has_attachment === true) filter.hasAttachment = true; + // phase-3 annSearch takes the generation ID (not the object) and returns + // [{messageId, score, rank}] rank-ordered. +1 to drop the seed without coming up short. + const hits = await annSearch(gen.id, seed, limit + 1, { filter }); + + const ids = []; + for (const h of hits) { + if (h.messageId === seedId) continue; + if (ids.length >= limit) break; + ids.push(h.messageId); + } + let messages = await getMessageSummariesByIDs(ids, acc.accountIds); + // msgvault applies message_type inside the vector backend filter + // (handlers.go:1042-1044); Mailflow's annSearch filter has no such leg, so + // the advertised filter is applied on the hydrated summaries (every + // Mailflow message is 'email' today — a non-email filter returns zero). + if (messageType) messages = messages.filter((m) => m.message_type === messageType); + return jsonResult({ + seed_message_id: seedId, + returned: messages.length, + generation: { id: gen.id, model: gen.model, dimension: gen.dimension, fingerprint: gen.fingerprint, state: gen.state }, + messages: messages.map(wireSummary), + }); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + throw err; + } +} + +export const searchInMessageDef = { + name: 'search_in_message', + description: + 'Find matches within one message body. Default mode=keyword finds literal term occurrences. ' + + 'mode=vector scores each embedded chunk by semantic similarity to the query (best first, with score on each match). ' + + 'Keyword matches include raw-body char_offset and line. Vector matches always include snippet and score; char_offset and line may be omitted after preprocessing. ' + + 'Use a present char_offset with get_message center_at to read a larger window around the match.', + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'Message ID' }, + query: { type: 'string', description: 'Search query (keyword term, or semantic query when mode=vector)' }, + limit: { type: 'number', description: 'Maximum matches to return (default 10)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + mode: { type: 'string', enum: ['keyword', 'vector'], description: 'Search mode: keyword (default, literal term) or vector (semantic chunk scoring)' }, + min_score: { type: 'number', description: 'Minimum chunk similarity score (0–1) when mode=vector (default 0)' }, + }, + required: ['id', 'query'], + }, +}; + +export async function handleSearchInMessage(args, scope) { + const id = args.id; + if (!id || typeof id !== 'string') return errorResult('id parameter is required'); + const q = (args.query || '').trim(); + if (!q) return errorResult('query parameter is required'); + const limit = limitArg(args, 'limit', 10); + const offset = limitArg(args, 'offset', 0); + const mode = args.mode || 'keyword'; + + if (mode === 'vector') { + try { + const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 0; + // matchesInMessage is scope-aware: it resolves the message under + // scope.accountIds and throws VectorUnavailableError on stock Postgres. + const all = await matchesInMessage(id, q, minScore, { accountIds: scope.accountIds }); + const page = all.slice(offset, offset + limit); + return jsonResult(newPaginatedResponse(page, all.length, offset)); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + throw err; + } + } + if (mode !== 'keyword') return errorResult(`invalid mode "${mode}": must be keyword (default) or vector`); + + const row = await getMessage(id, scope.accountIds); + if (!row) return errorResult('message not found'); + const all = findTermMatches(row.body_text || '', q); // byte-offset keyword matches (real total) + const page = all.slice(offset, offset + limit); + return jsonResult(newPaginatedResponse(page, all.length, offset)); +} + +export const stageDeletionDef = { + name: 'stage_deletion', + description: "Stage messages for deletion. Use EITHER 'query' (Gmail-style search) OR structured filters (from, domain, label, etc.), not both. Does NOT delete immediately - execution is a separate, explicitly-authorized step.", + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + query: { type: 'string', description: "Gmail-style search query (e.g. 'from:linkedin subject:job alert'). Cannot be combined with structured filters." }, + from: { type: 'string', description: 'Filter by sender email address' }, + domain: { type: 'string', description: "Filter by sender domain (e.g. 'linkedin.com')" }, + label: { type: 'string', description: "Filter by Gmail label (e.g. 'CATEGORY_PROMOTIONS')" }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + }, + }, +}; + +// A parsed query that survives validation but produces zero enforceable +// predicates would stage the entire (capped) mailbox — refuse it. Parse errors +// and unsupported operators are already rejected before this runs (msgvault +// ordering), so the only remaining cause is all-discarded terms. +function noEnforceableFiltersMessage() { + return 'query produced no enforceable filters (all query terms were discarded as too short, ' + + 'punctuation-only, or negation-only); refusing to stage deletions'; +} + +export async function handleStageDeletion(args, scope) { + const q = (args.query || '').trim(); + const hasQuery = q !== ''; + const structured = !!(args.from || args.domain || args.label || args.has_attachment || args.after || args.before); + if (hasQuery && structured) return errorResult("use either 'query' or structured filters (from, domain, label, etc.), not both"); + if (!hasQuery && !structured) return errorResult("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + + const parsed = hasQuery ? parseQuery(q) : null; + if (parsed) { + // Parse-value errors and unsupported operators reject the staging query + // outright (msgvault handlers.go:1818-1821). Deletion is where silent + // widening bites hardest: dropping `label:promotions` from + // `invoice label:promotions` would stage a SUPERSET of what was asked. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + // Guard the "stage EVERYTHING" hazard: a query whose tokens are ALL + // discarded (negation-only, sub-2-char/punctuation-only terms) leaves only + // account+liveness in the WHERE. Refuse before staging. + if (countEnforceableQueryPredicates(parsed) === 0) { + return errorResult(noEnforceableFiltersMessage()); + } + } + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + + const { batchId, messageCount } = await stageDeletion({ + userId: scope.userId, accountIds: acc.accountIds, + parsed, + from: args.from, domain: args.domain, label: args.label, + hasAttachment: args.has_attachment === true, after: after.value, before: before.value, + description: hasQuery ? `query: ${q}`.slice(0, 50) : 'filter', + }); + if (!messageCount) return errorResult('no messages match the specified criteria'); + return jsonResult({ + batch_id: batchId, + message_count: messageCount, + // Wire literal is msgvault's manifest.StatusPending ("pending", + // deletion/manifest.go:25, surfaced at handlers.go:1932). The DB row keeps + // Mailflow's internal 'staged' state (engineAdapter.stageDeletion). + status: 'pending', + next_step: `POST /api/mcp-deletions/${batchId}/execute to soft-delete, or DELETE /api/mcp-deletions/${batchId} to cancel`, + }); +} diff --git a/backend/src/mcp/messageTools.test.js b/backend/src/mcp/messageTools.test.js new file mode 100644 index 0000000..a112fca --- /dev/null +++ b/backend/src/mcp/messageTools.test.js @@ -0,0 +1,489 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getMessage: vi.fn(), listMessages: vi.fn(), listAccounts: vi.fn(), getTotalStats: vi.fn(), aggregate: vi.fn(), searchByDomains: vi.fn(), + getMessageSummariesByIDs: vi.fn(), stageDeletion: vi.fn(), + resolveAccountScope: vi.fn(), messageInScope: vi.fn(), + }; +}); +vi.mock('./vectorStats.js', () => ({ collectStats: vi.fn() })); +vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchesInMessage: vi.fn() })); +import * as adapter from './engineAdapter.js'; +import { resolveAccountScope, messageInScope } from './engineAdapter.js'; +import { collectStats } from './vectorStats.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; +import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; + +// The one vector-availability gate returns { cfg, generation }; a VectorUnavailableError +// (name + reason) thrown from it is what every degraded/disabled path surfaces. +class VUE extends Error { constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } } +import { + handleGetMessage, handleListMessages, handleGetStats, handleAggregate, + handleSearchByDomains, handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, +} from './messageTools.js'; + +const scope = { userId: 'u', accountIds: ['acc-1'] }; +const detailRow = { + id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', + from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), + has_attachments: false, attachments: [], flags: [], folder: 'INBOX', + body_text: 'café — meeting notes and a much longer body '.repeat(100), body_html: '', +}; + +beforeEach(() => { + adapter.getMessage.mockReset(); + resolveAccountScope.mockReset(); + messageInScope.mockReset(); + // Defaults: no account narrowing; seed messages are in scope unless a test says otherwise. + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); + messageInScope.mockResolvedValue(true); +}); + +describe('get_message', () => { + it('404s a missing message with a string id', async () => { + adapter.getMessage.mockResolvedValue(null); + const r = await handleGetMessage({ id: 'nope' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + }); + + it('pages by BYTES, never splitting a rune, with correct body_length', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const total = Buffer.byteLength(detailRow.body_text, 'utf8'); + const r = await handleGetMessage({ id: 'm1', max_chars: 20 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_length).toBe(total); + expect(b.offset).toBe(0); + expect(b.body_returned).toBeLessThanOrEqual(20); + expect(b.has_more).toBe(true); + // returned slice is valid UTF-8 (no replacement char from a split é) + expect(b.body_text).not.toContain('�'); + expect(b.body_format).toBe('text'); + expect(b.id).toBe('m1'); // uuid string + // scoped read + expect(adapter.getMessage).toHaveBeenCalledWith('m1', ['acc-1']); + }); + + it('center_at round-trips a byte offset into a window around it', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const r = await handleGetMessage({ id: 'm1', center_at: 200, max_chars: 100 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.offset).toBeLessThanOrEqual(200); + expect(b.offset + b.body_returned).toBeGreaterThanOrEqual(200); + }); + + it('clamps max_chars to 4000', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: 'z'.repeat(9000) }); + const r = await handleGetMessage({ id: 'm1', max_chars: 99999 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_returned).toBe(4000); + }); + + it('full_body returns the whole body ignoring paging', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: 'z'.repeat(5000) }); + const r = await handleGetMessage({ id: 'm1', full_body: true }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_returned).toBe(5000); + expect(b.has_more).toBe(false); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const b = JSON.parse((await handleGetMessage({ id: 'm1' }, scope)).content[0].text); + expect(b.sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('html-only message pages the html body with body_format=html under auto', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: '', body_html: '

hello world

' }); + const r = await handleGetMessage({ id: 'm1' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_format).toBe('html'); + expect(b.body_html).toContain('hello'); + expect(b.body_text).toBe(''); + }); +}); + +describe('list_messages', () => { + beforeEach(() => { adapter.listMessages.mockReset(); adapter.listAccounts.mockReset(); }); + + it('returns a newest-first envelope with total=-1 and pages via has_more (limit+1 over-fetch)', async () => { + // 21 rows for a default limit of 20 -> has_more true, sliced to 20. + adapter.listMessages.mockResolvedValue(Array.from({ length: 21 }, (_, i) => ({ id: `m${i}` }))); + const r = await handleListMessages({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.total).toBe(-1); + expect(b.returned).toBe(20); + expect(b.has_more).toBe(true); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ accountIds: ['acc-1'], limit: 21, offset: 0 })); + }); + + it('threads a string conversation_id filter through to listMessages', async () => { + adapter.listMessages.mockResolvedValue([]); + await handleListMessages({ conversation_id: 'tid-1' }, scope); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ conversationId: 'tid-1' })); + }); + + it('resolves an account email to its id and narrows the scope', async () => { + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + adapter.listMessages.mockResolvedValue([]); + await handleListMessages({ account: 'c@d.com' }, scope); + expect(resolveAccountScope).toHaveBeenCalledWith('c@d.com', ['acc-1']); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ accountIds: ['acc-2'] })); + }); + + it('404-style errors an unknown account (msgvault getAccountID parity)', async () => { + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleListMessages({ account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('account not found: nope@x.com'); + }); + + it('rejects malformed after/before dates before touching the engine (msgvault getDateArg, handlers.go:265-275)', async () => { + for (const [key, bad] of [['after', '13/45/2024'], ['before', '2024-02-31']]) { + const r = await handleListMessages({ [key]: bad }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe(`invalid ${key} date "${bad}": expected YYYY-MM-DD`); + } + expect(adapter.listMessages).not.toHaveBeenCalled(); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + adapter.listMessages.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const r = await handleListMessages({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('get_stats', () => { + beforeEach(() => { adapter.getTotalStats.mockReset(); adapter.listAccounts.mockReset(); collectStats.mockReset(); }); + + const stats = { MessageCount: 10, TotalSize: 500, AttachmentCount: 2, AttachmentSize: 0, LabelCount: 3, AccountCount: 1, ActiveMessageCount: 10, SourceDeletedMessageCount: 0 }; + const accounts = [{ ID: 'acc-1', SourceType: 'imap', Identifier: 'a@b.com', DisplayName: 'Work' }]; + + it('omits vector_search when vector search is disabled (collectStats null)', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + collectStats.mockResolvedValue(null); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.stats).toEqual(stats); + expect(b.accounts).toEqual(accounts); + expect(b).not.toHaveProperty('vector_search'); + expect(adapter.getTotalStats).toHaveBeenCalledWith(['acc-1']); + }); + + it('includes the StatsView when vector search is enabled', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + const vs = { enabled: true, active_generation: null, missing_embeddings_total: 4 }; + collectStats.mockResolvedValue(vs); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.vector_search).toEqual(vs); + }); + + it('survives a broken vector sub-query (omits vector_search, keeps stats)', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + collectStats.mockRejectedValue(new Error('boom')); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.stats).toEqual(stats); + expect(b).not.toHaveProperty('vector_search'); + }); +}); + +describe('aggregate', () => { + beforeEach(() => { adapter.aggregate.mockReset(); adapter.listAccounts.mockReset(); }); + + it('requires group_by', async () => { + const r = await handleAggregate({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('group_by parameter is required'); + }); + + it('rejects an invalid group_by', async () => { + const r = await handleAggregate({ group_by: 'colour' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid group_by: colour'); + }); + + it('returns the raw AggregateRow array (no envelope), default limit 50, scoped', async () => { + const rows = [{ Key: 'a@b.com', Count: 3, TotalSize: 10, AttachmentSize: 0, AttachmentCount: 1, TotalUnique: 5 }]; + adapter.aggregate.mockResolvedValue(rows); + const r = await handleAggregate({ group_by: 'sender' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual(rows); + expect(adapter.aggregate).toHaveBeenCalledWith('sender', expect.objectContaining({ accountIds: ['acc-1'], limit: 50 })); + }); + + it('rejects malformed after/before dates instead of leaking a raw PG error (msgvault getDateArg)', async () => { + const r = await handleAggregate({ group_by: 'sender', after: 'notadate' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid after date "notadate": expected YYYY-MM-DD'); + expect(adapter.aggregate).not.toHaveBeenCalled(); + }); +}); + +describe('search_by_domains', () => { + beforeEach(() => adapter.searchByDomains.mockReset()); + + it('requires domains', async () => { + const r = await handleSearchByDomains({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('domains is required'); + }); + + it('splits/trims the CSV, default limit 100, scoped, returns the raw array', async () => { + const rows = [{ id: 'm1', from_email: 'x@gobright.com' }]; + adapter.searchByDomains.mockResolvedValue(rows); + const r = await handleSearchByDomains({ domains: ' gobright.com , ascentae.com ' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual(rows); + expect(adapter.searchByDomains).toHaveBeenCalledWith(['gobright.com', 'ascentae.com'], undefined, undefined, 100, 0, ['acc-1']); + }); + + it('rejects malformed after/before dates and emits sent_at as RFC3339 (Go wire format)', async () => { + const bad = await handleSearchByDomains({ domains: 'x.com', before: '2024-1-1' }, scope); + expect(bad.isError).toBe(true); + expect(bad.content[0].text).toBe('invalid before date "2024-1-1": expected YYYY-MM-DD'); + expect(adapter.searchByDomains).not.toHaveBeenCalled(); + + adapter.searchByDomains.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const ok = await handleSearchByDomains({ domains: 'x.com' }, scope); + expect(JSON.parse(ok.content[0].text)[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('find_similar_messages', () => { + beforeEach(() => { + resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); + adapter.getMessageSummariesByIDs.mockReset(); adapter.listAccounts.mockReset(); + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + }); + + it('returns vector_not_enabled when embeddings are disabled (config check before the resolver)', async () => { + resolveActiveGenerationFromConfig.mockRejectedValueOnce(new VUE('vector_not_enabled')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('maps a VectorUnavailableError reason from the resolver to its wire string', async () => { + resolveActiveGenerationFromConfig.mockRejectedValue(new VUE('no_active_generation')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^no_active_generation:/); + }); + + it('excludes the seed, hydrates in rank order via annSearch(gen.id, ...), scoped', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([ + { messageId: 'm1', score: 0.9, rank: 1 }, + { messageId: 'seed', score: 1, rank: 0 }, + { messageId: 'm2', score: 0.7, rank: 2 }, + ]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1' }, { id: 'm2' }]); + const r = await handleFindSimilarMessages({ message_id: 'seed', limit: 20 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.seed_message_id).toBe('seed'); + expect(b.returned).toBe(2); + expect(b.generation).toEqual({ id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }); + expect(b.messages).toEqual([{ id: 'm1' }, { id: 'm2' }]); + // real annSearch takes the generation ID and the {filter} with accountIds + expect(annSearch).toHaveBeenCalledWith(3, [0.1, 0.2], 21, { filter: { accountIds: ['acc-1'] } }); + expect(adapter.getMessageSummariesByIDs).toHaveBeenCalledWith(['m1', 'm2'], ['acc-1']); + }); + + it('reports a readable error when the seed has no embedding', async () => { + loadVector.mockRejectedValue(new Error('no embedding for message seed')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('load seed vector: no embedding for message seed'); + }); + + it('rejects a foreign/out-of-scope seed id without loading its vector (owner-scope isolation)', async () => { + messageInScope.mockResolvedValue(false); // seed belongs to another user + const r = await handleFindSimilarMessages({ message_id: 'foreign-seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + expect(loadVector).not.toHaveBeenCalled(); + expect(annSearch).not.toHaveBeenCalled(); + expect(messageInScope).toHaveBeenCalledWith('foreign-seed', ['acc-1']); + }); + + it('applies the advertised message_type filter to hydrated results (msgvault handlers.go:1042-1044)', async () => { + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: 'm1', score: 0.9, rank: 1 }]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1', message_type: 'email' }]); + const none = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed', message_type: 'sms' }, scope)).content[0].text); + expect(none.messages).toEqual([]); + expect(none.returned).toBe(0); + const all = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed', message_type: 'email' }, scope)).content[0].text); + expect(all.messages).toHaveLength(1); + }); + + it('clamps limit to the hybrid page cap (msgvault handlers.go:885-888)', async () => { + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([]); + adapter.getMessageSummariesByIDs.mockResolvedValue([]); + await handleFindSimilarMessages({ message_id: 'seed', limit: 5000 }, scope); + expect(annSearch).toHaveBeenCalledWith(3, [0.1, 0.2], 101, expect.anything()); // 100 + 1 seed over-fetch + }); + + it('rejects malformed after/before dates and emits sent_at as RFC3339 (Go wire format)', async () => { + const bad = await handleFindSimilarMessages({ message_id: 'seed', after: '2024-13-45' }, scope); + expect(bad.isError).toBe(true); + expect(bad.content[0].text).toBe('invalid after date "2024-13-45": expected YYYY-MM-DD'); + expect(annSearch).not.toHaveBeenCalled(); + + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: 'm1', score: 0.9, rank: 1 }]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const ok = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed' }, scope)).content[0].text); + expect(ok.messages[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('search_in_message', () => { + const body = 'café note — the budget is set here'; + const detail = { id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), has_attachments: false, attachments: [], flags: [], folder: 'INBOX', body_text: body, body_html: '' }; + beforeEach(() => { adapter.getMessage.mockReset(); matchesInMessage.mockReset(); }); + + it('keyword mode returns a byte char_offset + real total; the offset round-trips into get_message center_at', async () => { + adapter.getMessage.mockResolvedValue(detail); + const r = await handleSearchInMessage({ id: 'm1', query: 'budget' }, scope); + const b = JSON.parse(r.content[0].text); + const byteOffset = Buffer.from(body, 'utf8').indexOf(Buffer.from('budget', 'utf8')); + expect(b.total).toBe(1); + expect(b.data[0].char_offset).toBe(byteOffset); // BYTE offset (not code-point index) + expect(b.data[0]).not.toHaveProperty('score'); // keyword = no score + const g = JSON.parse((await handleGetMessage({ id: 'm1', center_at: byteOffset, max_chars: 100 }, scope)).content[0].text); + expect(g.offset).toBeLessThanOrEqual(byteOffset); + expect(g.offset + g.body_returned).toBeGreaterThanOrEqual(byteOffset); + }); + + it('404s a missing message in keyword mode', async () => { + adapter.getMessage.mockResolvedValue(null); + const r = await handleSearchInMessage({ id: 'x', query: 'q' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + }); + + it('vector mode maps VectorUnavailableError to vector_not_enabled (stock Postgres)', async () => { + class VUE extends Error { constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } } + matchesInMessage.mockRejectedValue(new VUE('vector_not_enabled')); + const r = await handleSearchInMessage({ id: 'm1', query: 'travel', mode: 'vector' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('vector mode pages scored matches from matchesInMessage (scoped)', async () => { + matchesInMessage.mockResolvedValue([{ snippet: 'a', score: 0.9 }, { snippet: 'b', score: 0.8 }]); + const r = await handleSearchInMessage({ id: 'm1', query: 'travel', mode: 'vector', limit: 1 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.total).toBe(2); + expect(b.returned).toBe(1); + expect(b.has_more).toBe(true); + expect(matchesInMessage).toHaveBeenCalledWith('m1', 'travel', 0, { accountIds: ['acc-1'] }); + }); + + it('rejects an unknown mode', async () => { + const r = await handleSearchInMessage({ id: 'm1', query: 'q', mode: 'fuzzy' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid mode "fuzzy": must be keyword (default) or vector'); + }); +}); + +describe('stage_deletion', () => { + beforeEach(() => adapter.stageDeletion.mockReset()); + + it('rejects query + structured filters together', async () => { + const r = await handleStageDeletion({ query: 'from:x', from: 'y@z.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("use either 'query' or structured filters (from, domain, label, etc.), not both"); + }); + + it('rejects neither query nor filters', async () => { + const r = await handleStageDeletion({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + }); + + it('rejects an all-whitespace query with no structured filters (empty/missing query guard)', async () => { + const r = await handleStageDeletion({ query: ' ' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('rejects a query with an unsupported operator via the taxonomy error — never stages the whole mailbox', async () => { + const r = await handleStageDeletion({ query: 'label:promotions' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('refuses a negation-only / sub-2-char / punctuation-only query (all tokens discarded)', async () => { + for (const query of ['-newsletter', 'a', '!!!']) { + adapter.stageDeletion.mockReset(); + const r = await handleStageDeletion({ query }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('query produced no enforceable filters (all query terms were discarded as too short, punctuation-only, or negation-only); refusing to stage deletions'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + } + }); + + it('rejects a mixed query too — dropping label: would stage a SUPERSET of what was asked (msgvault handlers.go:1818-1821)', async () => { + const r = await handleStageDeletion({ query: 'invoice label:promotions' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim (msgvault q.Err front-door rule)', async () => { + const r = await handleStageDeletion({ query: 'invoice older_than:xyz' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('rejects malformed structured after/before dates (msgvault getDateArg, handlers.go:1793-1800)', async () => { + const r = await handleStageDeletion({ after: '13/45/2024' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid after date "13/45/2024": expected YYYY-MM-DD'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('errors when no messages match', async () => { + adapter.stageDeletion.mockResolvedValue({ batchId: null, messageCount: 0 }); + const r = await handleStageDeletion({ from: 'linkedin.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('no messages match the specified criteria'); + }); + + it('stages a batch with wire status "pending" (msgvault manifest.StatusPending, manifest.go:25) — never soft-deletes', async () => { + adapter.stageDeletion.mockResolvedValue({ batchId: 'batch-9', messageCount: 5 }); + const r = await handleStageDeletion({ domain: 'linkedin.com' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual({ + batch_id: 'batch-9', message_count: 5, status: 'pending', + next_step: 'POST /api/mcp-deletions/batch-9/execute to soft-delete, or DELETE /api/mcp-deletions/batch-9 to cancel', + }); + // batch is scoped to the token user (cross-user isolation) and never flips is_deleted here + expect(adapter.stageDeletion).toHaveBeenCalledWith(expect.objectContaining({ userId: 'u', accountIds: ['acc-1'], domain: 'linkedin.com' })); + }); +}); diff --git a/backend/src/mcp/result.js b/backend/src/mcp/result.js new file mode 100644 index 0000000..f416103 --- /dev/null +++ b/backend/src/mcp/result.js @@ -0,0 +1,9 @@ +// Mirror msgvault's mcp.NewToolResultText / mcp.NewToolResultError: every tool +// returns a single text content block; errors additionally set isError. +export function jsonResult(obj) { + return { content: [{ type: 'text', text: JSON.stringify(obj) }] }; +} + +export function errorResult(msg) { + return { content: [{ type: 'text', text: msg }], isError: true }; +} diff --git a/backend/src/mcp/result.test.js b/backend/src/mcp/result.test.js new file mode 100644 index 0000000..a9e118b --- /dev/null +++ b/backend/src/mcp/result.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { jsonResult, errorResult } from './result.js'; +import { HANDLERS, TOOL_DEFS } from './tools.js'; + +describe('result helpers', () => { + it('jsonResult wraps a JSON string in a text content block', () => { + expect(jsonResult({ ok: true })).toEqual({ + content: [{ type: 'text', text: '{"ok":true}' }], + }); + }); + it('errorResult marks isError and carries the message verbatim', () => { + expect(errorResult('boom')).toEqual({ + content: [{ type: 'text', text: 'boom' }], + isError: true, + }); + }); +}); + +describe('ping tool', () => { + it('is registered with a JSON-schema input', () => { + const def = TOOL_DEFS.find((t) => t.name === 'ping'); + expect(def).toBeTruthy(); + expect(def.inputSchema).toEqual({ type: 'object', properties: {} }); + }); + it('echoes pong and is scope-agnostic', async () => { + const r = await HANDLERS.ping({}, { userId: 'u', accountIds: [] }); + expect(r.content[0].text).toBe('{"pong":true}'); + }); +}); diff --git a/backend/src/mcp/searchTools.js b/backend/src/mcp/searchTools.js new file mode 100644 index 0000000..463a7e6 --- /dev/null +++ b/backend/src/mcp/searchTools.js @@ -0,0 +1,414 @@ +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; +import { jsonResult, errorResult } from './result.js'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, wireSummary } from './envelope.js'; +import { extractContextChar } from './bodyMatch.js'; +import { translateVectorError } from './vectorErrors.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; + +// The search seam returns raw REST-shaped rows (subject/from/date/snippet/… under a +// column set REST froze); MCP must emit msgvault MessageSummary. We hydrate the hit +// ids through engineAdapter (recipients, thread_id, attachments — data the ranked row +// set doesn't carry) so every search tool speaks the same wire shape as the message +// tools. Order is preserved by getMessageSummariesByIDs; ids are already in scope. +// wireSummary re-formats sent_at to Go RFC3339 (no millis) at the wire. +async function hydrateSummaries(messages, accountIds) { + const ids = (messages || []).map((m) => m.id); + const summaries = await getMessageSummariesByIDs(ids, accountIds); + return new Map(summaries.map((s) => [s.id, wireSummary(s)])); +} + +// msgvault front doors reject queries whose known operators carry unparseable +// values (q.Err(), internal/search/parser.go:45-52; returned verbatim at +// handlers.go:373-375) instead of silently dropping the filter and returning +// wider-than-requested results. parsed.errors carries the verbatim messages; +// errors.Join separates with newlines. +export function queryParseErrorMessage(parsed) { + const errs = (parsed && parsed.errors) || []; + return errs.length ? errs.join('\n') : ''; +} + +// Port of unsupportedSearchOperatorMessage (msgvault handlers.go:408-428) with +// per-operator reasons. msgvault's parser-level unsupported set is only +// list:/list-id: (Gmail-only, parser.go:270-273); Mailflow additionally cannot +// serve label:/l:, bcc:, larger:, smaller: — msgvault supports those four, but +// Mailflow's messages schema stores no labels, BCC recipients, or byte sizes +// (documented divergence). The taxonomy prefix is verbatim. +const UNSUPPORTED_OPERATOR_REASONS = { + list: 'is Gmail-only syntax (this server does not index List-ID); use supported operators instead', + 'list-id': 'is Gmail-only syntax (this server does not index List-ID); use supported operators instead', + label: 'is not supported on this server (Mailflow stores no Gmail labels)', + bcc: 'is not supported on this server (Mailflow does not store BCC recipients)', + larger: 'is not supported on this server (Mailflow does not store message sizes)', + smaller: 'is not supported on this server (Mailflow does not store message sizes)', +}; + +// list:/list-id: live in msgvault's parser-level unsupported set; Mailflow's +// queryParser (Wave D's file) currently leaves them as literal free-text +// terms, so recognize them here at the handler seam. +// TODO(seam): consolidate into queryParser's unsupported set with Wave D. +function unsupportedListOperators(parsed) { + const out = []; + for (const t of (parsed && parsed.terms) || []) { + const m = /^(list|list-id):/i.exec(t.value || ''); + if (m) out.push(m[1].toLowerCase()); + } + return out; +} + +export function unsupportedSearchOperatorMessage(parsed) { + const names = []; + const seen = new Set(); + const push = (name) => { if (name && !seen.has(name)) { seen.add(name); names.push(name); } }; + for (const u of (parsed && parsed.unsupported) || []) push(u.key); + for (const n of unsupportedListOperators(parsed)) push(n); + if (!names.length) return ''; + // Group operators sharing a reason, msgvault-style ("name:, name2: "), + // preserving first-appearance order. + const groups = []; + const byReason = new Map(); + for (const n of names) { + const reason = UNSUPPORTED_OPERATOR_REASONS[n] || 'is not supported on this server'; + let g = byReason.get(reason); + if (!g) { g = { reason, ops: [] }; byReason.set(reason, g); groups.push(g); } + g.ops.push(`${n}:`); + } + return 'unsupported_search_operator: ' + + groups.map((g) => `${g.ops.join(', ')} ${g.reason}`).join('; '); +} + +// msgvault clamps hybrid paging to [vector.search].max_page_size_hybrid +// (default 50; vector/config.go:196-205,339-342, enforced at handlers.go:651-668). +// Mailflow has no config knob; its effective ranking window is the fused +// per-signal candidate cap (hybrid.js K_PER_SIGNAL = 100) — offsets past it +// can only return silently-empty pages, so reject them the msgvault way. +// TODO(seam): read this from hybrid.js if Wave D exports the per-signal cap. +export const HYBRID_RANKING_WINDOW = 100; + +// Tool descriptions/schemas follow msgvault internal/mcp/server.go, with the +// Mailflow divergences spelled out in the text: no label:/bcc:/larger:/smaller: +// (schema lacks the data — msgvault supports them), negation IS supported +// (msgvault does not), free-text ordering is relevance-ranked (D5), and +// semantic hits carry at most 1 excerpt. README D6: only id-bearing fields +// diverge to UUID strings — search tools have none. +const SEARCH_METADATA_OPERATOR_DOC = + 'Supported operators: from:, to:, cc:, subject:, has:attachment, ' + + 'before:/after: (YYYY-MM-DD), older_than:/newer_than: (e.g. 7d, 2w, 1m, 1y). ' + + 'Bare domains on from:/to: match any address at that domain. Multiple terms are ANDed. ' + + 'Rejected as unsupported on this server (divergence from msgvault, which supports them): ' + + 'label: (or l:), bcc:, larger:, smaller: — Mailflow stores no labels, BCC recipients, or message sizes; ' + + 'list:/list-id: are Gmail-only and also rejected. ' + + 'Negation with a leading - (e.g. -from:alice, -invoice) IS supported (divergence from msgvault); ' + + 'OR and parentheses grouping are not.'; +const SEARCH_METADATA_FREETEXT_DOC = + 'Free text matches subject, snippet, and sender/recipient metadata only (not bodies). ' + + 'Use search_message_bodies for body keywords or semantic_search_messages for vector/hybrid search.'; +const SEARCH_METADATA_PAGINATION_DOC = + 'Free-text results are relevance-ranked (divergence from msgvault); filter-only queries ' + + 'are ordered newest-first (by sent date). There is no sort parameter — ' + + 'use before:/after: to scope a date range. ' + + 'Paginate with offset/limit (default limit 20, max 50). ' + + 'Response: data, total, returned, offset, has_more.'; + +export const searchMetadataDef = { + name: 'search_metadata', + description: + 'Search message metadata using a subset of Gmail query syntax (not full Gmail compatibility). ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + SEARCH_METADATA_FREETEXT_DOC + ' ' + + SEARCH_METADATA_PAGINATION_DOC + + 'For body keywords use search_message_bodies; for vector/hybrid search use semantic_search_messages.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: "Search query (e.g. 'from:alice subject:meeting after:2024-01-01'). See tool description for supported operators and limitations." }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + required: ['query'], + }, +}; + +const DEFAULT_SEARCH_LIMIT = 20; +const MAX_SEARCH_LIMIT = 50; +export function searchLimitArg(args) { + const v = Number(args.limit); + if (!Number.isFinite(v) || v <= 0) return DEFAULT_SEARCH_LIMIT; + return Math.min(Math.trunc(v), MAX_SEARCH_LIMIT); +} +export function offsetArg(args) { + const v = Number(args.offset); + if (!Number.isFinite(v) || v < 0) return 0; + return Math.trunc(v); +} + +export async function handleSearchMetadata(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:372-378): parse-value errors verbatim, + // then unsupported operators — never silently widen the result set. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Narrow to a single account when `account` is given (msgvault getAccountID) — + // searchService trusts a pre-resolved accountIds as-is, so the narrowing must + // happen here (it never re-reads the `account` email). + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const result = await search({ + mode: 'lexical', scope: 'metadata', + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit, offset, + }); + const byId = await hydrateSummaries(result.messages, acc.accountIds); + const data = (result.messages || []).map((m) => byId.get(m.id)).filter(Boolean); + // total is ALWAYS present on this envelope (msgvault SearchFastCount is a + // real count, handlers.go:400-405). The seam omits it on degenerate queries + // whose terms were all dropped — those return zero rows, so total is 0. + const total = Number.isFinite(result.total) ? result.total : 0; + return jsonResult(newPaginatedResponse(data, total, offset)); +} + +export const searchMessageBodiesDef = { + name: 'search_message_bodies', + description: + 'Keyword full-text search over message bodies. ' + + 'Returns messages whose body text contains the query terms, relevance-ranked, ' + + 'each with matches — up to 5 excerpt snippets centered on matched terms. ' + + 'Backend excerpts may omit char_offset and line when efficient source locations are unavailable; use search_in_message when exact locations are needed. ' + + 'When matches_truncated is true on a hit, more than 5 excerpts matched — use search_in_message or get_message to read the full body. ' + + 'Known Gmail operators (from:, subject:, etc.) apply as metadata filters only and do not satisfy the free-text requirement. ' + + 'Filter-only queries such as from:alice are rejected — use search_metadata for filter-only queries. ' + + 'Unrecognized word:value tokens (e.g. RXD2:V2) are treated as literal body text, not filters. ' + + 'Query syntax: space-separated words are ANDed (each must appear somewhere in the body); ' + + 'a double-quoted phrase is one exact phrase (e.g. "RXD2 V2"); negation with a leading - excludes a term ' + + '(divergence from msgvault); OR is not supported. ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + + 'Results are relevance-ranked, best lexical match first (divergence from msgvault, which orders newest-first). ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more. ' + + 'Body search does not return a total; use has_more to detect more pages.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Body search query with at least one free-text term (bare word or quoted phrase). Gmail operators (from:, subject:, etc.) are metadata filters, not body search — subject:test alone is rejected; combine with body terms (from:alice budget) or use search_metadata for filter-only queries. Unrecognized word:value tokens (RXD2:V2) are literal text. Space-separated words are ANDed; double quotes match an exact phrase; a leading - negates a term; OR unsupported.' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + required: ['query'], + }, +}; + +const EMPTY_GENERATION = { id: 0, model: '', dimension: 0, fingerprint: '', state: '' }; +const MAX_CONTEXT_SNIPPETS = 5; +const SEARCH_CONTEXT_CHARS = 300; + +function freeTextTerms(parsed) { + return (parsed.terms || []).filter((t) => !t.negate).map((t) => t.value); +} + +// hybridScoreBreakdown omitempty parity (msgvault handlers.go:563-572): all +// signal fields are pointer-typed with omitempty — rrf is omitted in +// mode=vector (one signal, nothing to fuse) and subject_boosted when false. +// The seam's explain object always carries rrf + a boolean subject_boosted. +function wireScore(score, mode) { + const out = {}; + if (mode === 'hybrid' && score.rrf != null) out.rrf = score.rrf; + if (score.bm25 != null) out.bm25 = score.bm25; + if (score.vector != null) out.vector = score.vector; + if (score.subject_boosted) out.subject_boosted = true; + return out; +} + +export async function handleSearchMessageBodies(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + // Keyword-only tool: an explicit vector/hybrid mode is rejected, not + // ignored (msgvault handlers.go:442-457 — same two wordings). + const mode = args.mode || 'keyword'; + if (mode === 'vector' || mode === 'hybrid') { + return errorResult( + `invalid mode "${mode}": search_message_bodies is keyword-only; use semantic_search_messages for vector or hybrid search`, + ); + } + if (mode !== 'keyword') { + return errorResult( + `invalid mode "${mode}": search_message_bodies only supports keyword search; use semantic_search_messages for vector or hybrid search`, + ); + } + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:459-465): parse errors, then unsupported + // operators, before the free-text requirement. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const terms = freeTextTerms(parsed); + if (!terms.length) { + return errorResult( + 'search_message_bodies requires at least one free-text term (bare word or quoted phrase); ' + + 'Gmail operators such as from: or subject: are metadata filters and do not count — ' + + 'use search_metadata for filter-only queries', + ); + } + const limit = searchLimitArg(args); + const offset = offsetArg(args); + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + // Over-fetch one to compute has_more without a count query (msgvault limit+1). + const result = await search({ + mode: 'lexical', scope: 'body', + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit: limit + 1, offset, + }); + let hits = result.messages || []; + const hasMore = hits.length > limit; + if (hasMore) hits = hits.slice(0, limit); + + const byId = await hydrateSummaries(hits, acc.accountIds); + const data = hits.map((m) => { + const summary = byId.get(m.id); + if (!summary) return null; + const snippets = extractContextChar(m.body_text || '', terms, SEARCH_CONTEXT_CHARS) || []; + const capped = snippets.slice(0, MAX_CONTEXT_SNIPPETS); + // Go omitempty parity (msgvault handlers.go:339-341): matches is omitted + // when empty and matches_truncated when false — never emitted as []/false. + const item = { ...summary }; + if (capped.length) item.matches = capped.map((snippet) => ({ snippet })); + if (snippets.length > MAX_CONTEXT_SNIPPETS) item.matches_truncated = true; + return item; + }).filter(Boolean); + + return jsonResult({ + ...newPaginatedResponseNoTotal(data, offset, hasMore), + mode: 'keyword', + pool_saturated: false, + generation: EMPTY_GENERATION, + }); +} + +export const semanticSearchMessagesDef = { + name: 'semantic_search_messages', + description: + 'Semantic (embedding) search over each preprocessed message subject and body. ' + + 'Returns messages ranked by similarity to the query — there is no exact total, so page on has_more. ' + + 'Each hit includes matches — at most 1 best-matching embedded subject/body chunk excerpt with a score (divergence from msgvault, which returns up to 5). ' + + 'Vector char_offset and line locations may be omitted because preprocessing usually prevents exact raw-body mapping; use snippet terms with search_in_message keyword mode when navigation is needed. ' + + 'min_score filters chunk excerpts only; it does not remove or reorder ranked messages. ' + + 'Requires at least one free-text term (used to embed); filter-only queries must use search_metadata. ' + + 'Known Gmail operators (from:, subject:, etc.) apply as metadata filters only. ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + + 'mode=vector for pure semantic search or mode=hybrid to fuse BM25 and vector ranking via RRF. ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more, mode, pool_saturated, generation. ' + + 'total is not available; use has_more to page.', + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Free-text query to embed (requires at least one free-text term). Gmail operators are metadata filters, not body search; combine with body terms or use search_metadata for filter-only queries.' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + mode: { type: 'string', enum: ['vector', 'hybrid'], description: 'Search mode: vector (semantic only) or hybrid (BM25 + vector fused via RRF). Defaults to hybrid when omitted.' }, + explain: { type: 'boolean', description: 'Include per-signal scores in the response (for debugging or ranking inspection)' }, + min_score: { type: 'number', description: 'Minimum chunk similarity score for included match excerpts (default 0); does not filter ranked messages' }, + }, + required: ['query'], + }, +}; + +export async function handleSemanticSearchMessages(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + + const mode = args.mode || 'hybrid'; + if (mode !== 'vector' && mode !== 'hybrid') { + return errorResult( + `invalid mode "${mode}": must be vector or hybrid (default hybrid); use search_message_bodies for keyword search`, + ); + } + + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:552-558): parse errors, then unsupported + // operators, before the free-text requirement. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const terms = freeTextTerms(parsed); + if (!terms.length) { + return errorResult( + `missing_free_text: mode=${mode} requires at least one free-text term; use search_metadata for filter-only queries`, + ); + } + + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Offsets past the ranked window cannot be served — reject them instead of + // returning silently-empty pages (msgvault handlers.go:656-663 wording). + if (offset >= HYBRID_RANKING_WINDOW) { + return errorResult( + `pagination_limit: offset ${offset} exceeds hybrid ranking window (max ${HYBRID_RANKING_WINDOW}); ` + + 'use search_metadata or search_message_bodies for deeper pagination', + ); + } + const explain = args.explain === true; + const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 0; + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + + // strictVector: the seam rethrows VectorUnavailableError (msgvault taxonomy) + // instead of the REST silent-lexical fallback. + let result; + try { + result = await search({ + mode, scope: 'body', strictVector: true, + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit, offset, explain, minScore, + }); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + if (err.name === 'MissingFreeTextError') { + // The seam applies stricter term hygiene (sub-2-char / punctuation-only + // tokens never embed) than the raw-terms pre-check above; surface the + // same msgvault wording (handlers.go:631-635) instead of letting it + // escape the tool call as `internal error: missing_free_text`. + return errorResult( + `missing_free_text: mode=${mode} requires at least one free-text term; use search_metadata for filter-only queries`, + ); + } + throw err; + } + + // Phase 4 surfaces best_chunk per hit (chunk_index + code-point char_start/char_end + // into preprocessed text + score). Phase 5 owns the snippet + raw-body byte offsets: + // build the wire matches[] (≤1 excerpt — documented divergence from msgvault's ≤5) + // from best_chunk via chunkmatch.matchFromChunk. + const byId = await hydrateSummaries(result.messages, acc.accountIds); + const data = (await Promise.all((result.messages || []).map(async (m) => { + const summary = byId.get(m.id); + if (!summary) return null; + const item = { ...summary }; + if (m.best_chunk) { + const match = await matchFromChunk(m.id, m.best_chunk, { accountIds: acc.accountIds }); + if (match && match.score >= minScore) item.matches = [match]; + } + if (explain && m.score) item.score = wireScore(m.score, mode); + return item; + }))).filter(Boolean); + + return jsonResult({ + ...newPaginatedResponseNoTotal(data, offset, result.page?.hasMore || false), + mode, + pool_saturated: result.pool_saturated || false, + generation: result.generation || EMPTY_GENERATION, + }); +} diff --git a/backend/src/mcp/searchTools.test.js b/backend/src/mcp/searchTools.test.js new file mode 100644 index 0000000..45f18c9 --- /dev/null +++ b/backend/src/mcp/searchTools.test.js @@ -0,0 +1,389 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/search/queryParser.js', () => ({ parseQuery: vi.fn() })); +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +// The search seam returns raw rows; MCP hydrates hit ids into MessageSummary shape. +vi.mock('./engineAdapter.js', () => ({ getMessageSummariesByIDs: vi.fn(), resolveAccountScope: vi.fn() })); +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; + +class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } +} + +const scope = { userId: 'u1', accountIds: ['acc-1'] }; +function payload(r) { return JSON.parse(r.content[0].text); } +// Hydration echoes each requested id as a minimal summary unless a test overrides it. +function echoSummaries(rows) { + getMessageSummariesByIDs.mockImplementation(async (ids) => ids.map((id) => rows.find((r) => r.id === id)).filter(Boolean)); +} + +beforeEach(() => { + parseQuery.mockReset(); search.mockReset(); matchFromChunk.mockReset(); getMessageSummariesByIDs.mockReset(); + resolveAccountScope.mockReset(); + // Default: no `account` narrowing — pass the token's full scope through. + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); +}); + +describe('search_metadata', () => { + it('requires a query', async () => { + const r = await handleSearchMetadata({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('query parameter is required'); + }); + + it('hydrates hits into the msgvault envelope with a real total, scoped to the token accounts', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + search.mockResolvedValue({ + messages: [{ id: 'm1' }], // raw seam row — only the id is load-bearing for hydration + total: 5, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: true }, + }); + echoSummaries([{ id: 'm1', subject: 'Hi', to: [{ Email: 'c@d.com', Name: 'C' }] }]); + const r = await handleSearchMetadata({ query: 'from:alice' }, scope); + const body = payload(r); + expect(body).toEqual({ data: [{ id: 'm1', subject: 'Hi', to: [{ Email: 'c@d.com', Name: 'C' }] }], total: 5, returned: 1, offset: 0, has_more: true }); + // scope + parsed handed to the seam; MCP never builds SQL; hydration is scoped + expect(search).toHaveBeenCalledWith(expect.objectContaining({ mode: 'lexical', scope: 'metadata', accountIds: ['acc-1'], limit: 20, offset: 0 })); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-1']); + }); + + it('clamps limit to 50', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + search.mockResolvedValue({ messages: [], total: 0, mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + getMessageSummariesByIDs.mockResolvedValue([]); + await handleSearchMetadata({ query: 'x', limit: 999 }, scope); + expect(search.mock.calls[0][0].limit).toBe(50); + }); + + it('narrows scope to a resolved account id (the account arg is no longer a silent no-op)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + search.mockResolvedValue({ messages: [{ id: 'm1' }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', subject: 'Hi' }]); + await handleSearchMetadata({ query: 'x', account: 'work@x.com' }, scope); + expect(resolveAccountScope).toHaveBeenCalledWith('work@x.com', ['acc-1']); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); // narrowed, not the full token scope + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); // hydration narrowed too + }); + + it('rejects an unknown account without querying (msgvault getAccountID parity)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSearchMetadata({ query: 'x', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('account not found: nope@x.com'); + expect(search).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim instead of silently widening (msgvault q.Err, handlers.go:373-375)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [], + errors: ['invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'], + }); + const r = await handleSearchMetadata({ query: 'older_than:xyz' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects schema-unsupported operators with the unsupported_search_operator taxonomy (handlers.go:408-428)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'x', negate: false }], + unsupported: [{ key: 'label', token: 'label:promos' }, { key: 'larger', token: 'larger:5M' }], + errors: [], + }); + const r = await handleSearchMetadata({ query: 'x label:promos larger:5M' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(r.content[0].text).toContain('larger:'); + expect(search).not.toHaveBeenCalled(); + }); + + it('treats literal list:/list-id: terms as unsupported (msgvault parser set, parser.go:270-273, hoisted to the handler)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'list-id:foo', negate: false }], unsupported: [], errors: [] }); + const r = await handleSearchMetadata({ query: 'list-id:foo' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: list-id: is Gmail-only syntax/); + expect(search).not.toHaveBeenCalled(); + }); + + it('always carries a numeric total, even when the seam omits it (degenerate all-terms-dropped query)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'a', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); // no total key + getMessageSummariesByIDs.mockResolvedValue([]); + const body = payload(await handleSearchMetadata({ query: 'a' }, scope)); + expect(body.total).toBe(0); + expect(body.has_more).toBe(false); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ messages: [{ id: 'm1' }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const body = payload(await handleSearchMetadata({ query: 'x' }, scope)); + expect(body.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('search_message_bodies', () => { + it('rejects filter-only queries (no free-text term)', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + const r = await handleSearchMessageBodies({ query: 'from:alice' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toContain('requires at least one free-text term'); + }); + + it('returns keyword envelope: total -1, mode keyword, empty generation, snippet-only matches on a hydrated summary', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + search.mockResolvedValue({ + messages: [{ id: 'm1', body_text: 'the budget is approved. budget notes.' }], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + echoSummaries([{ id: 'm1', subject: 'Q3', from_email: 'a@b.com' }]); + const r = await handleSearchMessageBodies({ query: 'budget' }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.total).toBe(-1); + expect(body.mode).toBe('keyword'); + expect(body.pool_saturated).toBe(false); + expect(body.generation).toEqual({ id: 0, model: '', dimension: 0, fingerprint: '', state: '' }); + expect(body.data[0].subject).toBe('Q3'); // hydrated summary, not the raw row + expect(body.data[0]).not.toHaveProperty('body_text'); // body is not leaked to the wire + expect(body.data[0].matches[0]).toHaveProperty('snippet'); + expect(body.data[0].matches[0]).not.toHaveProperty('char_offset'); // keyword body = snippet only + }); + + it('narrows scope to a resolved account id', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + search.mockResolvedValue({ messages: [{ id: 'm1', body_text: 'budget' }], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', subject: 'Q3' }]); + await handleSearchMessageBodies({ query: 'budget', account: 'work@x.com' }, scope); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); + }); + + it('rejects an unknown account', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSearchMessageBodies({ query: 'budget', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects explicit mode=vector/hybrid — keyword-only tool (msgvault handlers.go:447-452)', async () => { + for (const mode of ['vector', 'hybrid']) { + const r = await handleSearchMessageBodies({ query: 'budget', mode }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + `invalid mode "${mode}": search_message_bodies is keyword-only; use semantic_search_messages for vector or hybrid search`, + ); + } + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects unknown modes with the only-supports wording (msgvault handlers.go:453-457)', async () => { + const r = await handleSearchMessageBodies({ query: 'budget', mode: 'fuzzy' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + 'invalid mode "fuzzy": search_message_bodies only supports keyword search; use semantic_search_messages for vector or hybrid search', + ); + }); + + it('accepts an explicit mode=keyword', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + getMessageSummariesByIDs.mockResolvedValue([]); + const r = await handleSearchMessageBodies({ query: 'budget', mode: 'keyword' }, scope); + expect(r.isError).toBeFalsy(); + }); + + it('returns parser value errors and unsupported operators before the free-text check (msgvault ordering)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [{ key: 'bcc', token: 'bcc:x@y.com' }], errors: [], + }); + const r = await handleSearchMessageBodies({ query: 'bcc:x@y.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: bcc: /); + + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [], + errors: ['invalid value "5X" for smaller: — expected a size like 5M, 100K, or 1G'], + }); + const r2 = await handleSearchMessageBodies({ query: 'smaller:5X' }, scope); + expect(r2.content[0].text).toBe('invalid value "5X" for smaller: — expected a size like 5M, 100K, or 1G'); + expect(search).not.toHaveBeenCalled(); + }); + + it('OMITS matches/matches_truncated when empty/false and emits matches_truncated past 5 excerpts (Go omitempty, handlers.go:339-341)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + // m1: term absent from the body → no snippets; m2: 7 occurrences spaced + // past the 300-byte context window (no merge) → 5 capped + truncated. + const spread = Array.from({ length: 7 }, () => 'budget').join(' filler'.repeat(120)); + search.mockResolvedValue({ + messages: [{ id: 'm1', body_text: 'nothing relevant here' }, { id: 'm2', body_text: spread }], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + echoSummaries([{ id: 'm1', subject: 'A' }, { id: 'm2', subject: 'B' }]); + const body = payload(await handleSearchMessageBodies({ query: 'budget' }, scope)); + expect(body.data[0]).not.toHaveProperty('matches'); + expect(body.data[0]).not.toHaveProperty('matches_truncated'); + expect(body.data[1].matches).toHaveLength(5); + expect(body.data[1].matches_truncated).toBe(true); + }); +}); + +describe('semantic_search_messages', () => { + it('rejects filter-only queries with missing_free_text', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + const r = await handleSemanticSearchMessages({ query: 'from:alice' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^missing_free_text: mode=hybrid/); + }); + + it('rejects mode=keyword', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + const r = await handleSemanticSearchMessages({ query: 'x', mode: 'keyword' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/invalid mode "keyword"/); + }); + + it('returns vector_not_enabled when the seam is strict-unavailable', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + search.mockRejectedValue(new VectorUnavailableError('vector_not_enabled')); + const r = await handleSemanticSearchMessages({ query: 'travel plans' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('builds matches from best_chunk via matchFromChunk on a hydrated summary, with mode/pool_saturated/generation and per-signal scores when explain', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + matchFromChunk.mockResolvedValue({ char_offset: 42, snippet: 'flights', line: 3, score: 0.9 }); + search.mockResolvedValue({ + // Real hybrid/vector seam shape: keyed on message_id, with the additive + // `id` alias searchService now emits (mode-invariant with the lexical path). + messages: [{ message_id: 'm1', id: 'm1', + best_chunk: { chunk_index: 2, char_start: 100, char_end: 180, score: 0.9 }, + score: { rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: true } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, + pool_saturated: true, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const r = await handleSemanticSearchMessages({ query: 'travel plans', explain: true }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.total).toBe(-1); + expect(body.mode).toBe('hybrid'); + expect(body.pool_saturated).toBe(true); + expect(body.generation).toEqual({ id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }); + expect(body.data[0].subject).toBe('Trip'); // hydrated summary + expect(body.data[0].score).toEqual({ rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: true }); + expect(body.data[0].matches).toEqual([{ char_offset: 42, snippet: 'flights', line: 3, score: 0.9 }]); + expect(body.data[0]).not.toHaveProperty('best_chunk'); + expect(matchFromChunk).toHaveBeenCalledWith('m1', { chunk_index: 2, char_start: 100, char_end: 180, score: 0.9 }, { accountIds: scope.accountIds }); + }); + + it('drops the excerpt when its score is below min_score (ranking unaffected)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + matchFromChunk.mockResolvedValue({ snippet: 'weak', score: 0.1 }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', best_chunk: { chunk_index: 0, char_start: 10, char_end: 20, score: 0.1 } }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const r = await handleSemanticSearchMessages({ query: 'travel', mode: 'vector', min_score: 0.5 }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.data[0]).not.toHaveProperty('matches'); // excerpt dropped, message still returned + expect(body.mode).toBe('vector'); + }); + + it('narrows scope to a resolved account id (search + hydration + matchFromChunk)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + matchFromChunk.mockResolvedValue({ snippet: 'x', score: 0.9 }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', best_chunk: { chunk_index: 0, char_start: 0, char_end: 5, score: 0.9 } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + await handleSemanticSearchMessages({ query: 'travel', account: 'work@x.com' }, scope); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); + expect(matchFromChunk).toHaveBeenCalledWith('m1', expect.any(Object), { accountIds: ['acc-2'] }); + }); + + it('rejects an unknown account', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSemanticSearchMessages({ query: 'travel', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(search).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim and unsupported operators as taxonomy errors', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], + errors: ['invalid value "13" for newer_than: — expected a relative age like 7d, 2w, 1m, or 1y'], + }); + const r = await handleSemanticSearchMessages({ query: 'travel newer_than:13' }, scope); + expect(r.content[0].text).toBe('invalid value "13" for newer_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'travel', negate: false }], + unsupported: [{ key: 'smaller', token: 'smaller:1M' }], errors: [], + }); + const r2 = await handleSemanticSearchMessages({ query: 'travel smaller:1M' }, scope); + expect(r2.content[0].text).toMatch(/^unsupported_search_operator: smaller: /); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects offsets past the hybrid ranking window with pagination_limit (msgvault handlers.go:656-663)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [], errors: [] }); + const r = await handleSemanticSearchMessages({ query: 'x', offset: 100 }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + 'pagination_limit: offset 100 exceeds hybrid ranking window (max 100); use search_metadata or search_message_bodies for deeper pagination', + ); + expect(search).not.toHaveBeenCalled(); + }); + + it('maps a seam MissingFreeTextError to the missing_free_text result — never `internal error:` (msgvault handlers.go:631-635)', async () => { + // 'a' survives the handler's raw-terms pre-check, but the seam's stricter + // hygiene (sub-2-char tokens do not embed) throws MissingFreeTextError. + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'a', negate: false }], unsupported: [], errors: [] }); + class MissingFreeTextError extends Error { constructor() { super('missing_free_text'); this.name = 'MissingFreeTextError'; } } + search.mockRejectedValue(new MissingFreeTextError()); + const r = await handleSemanticSearchMessages({ query: 'a' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('missing_free_text: mode=hybrid requires at least one free-text term; use search_metadata for filter-only queries'); + }); + + it('explain omits rrf in mode=vector and subject_boosted when false (Go omitempty, handlers.go:563-572)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', score: { rrf: 0.5, vector: 0.8, subject_boosted: false } }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const body = payload(await handleSemanticSearchMessages({ query: 'travel', mode: 'vector', explain: true }, scope)); + expect(body.data[0].score).toEqual({ vector: 0.8 }); // no rrf (one signal), no false subject_boosted + }); + + it('explain keeps rrf in mode=hybrid but still omits a false subject_boosted', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', score: { rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: false } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const body = payload(await handleSemanticSearchMessages({ query: 'travel', explain: true }, scope)); + expect(body.data[0].score).toEqual({ rrf: 0.5, bm25: 1.2, vector: 0.8 }); + }); +}); diff --git a/backend/src/mcp/semanticSearchIds.regression.test.js b/backend/src/mcp/semanticSearchIds.regression.test.js new file mode 100644 index 0000000..a71a815 --- /dev/null +++ b/backend/src/mcp/semanticSearchIds.regression.test.js @@ -0,0 +1,89 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Regression for the live MCP bug: `semantic_search_messages` returned 0 because +// hybrid/vector seam hits carry `message_id` (fusedSearch DISPLAY_COLS), not `id`, +// so the handler's `byId.get(m.id)` hydration mapped every hit to null. +// +// Unlike searchTools.test.js (which mocks the searchService seam directly), this +// test drives the REAL seam and mocks only the DEEPER dependency (hybridSearch) +// with the REAL fused-row field inventory the running container exposed +// (message_id/uid/folder/…/rrf_score/best_char_*). That exercises the actual +// `id`-alias fix in searchService plus the handler's hydration + excerpt + explain +// assembly end-to-end: it fails (returned:0) without the seam fix and passes with it. +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn(), pool: {} })); +vi.mock('../services/embeddings/hybrid.js', () => ({ + hybridSearch: vi.fn(), + isLexicalFallback: () => false, + MissingFreeTextError: class MissingFreeTextError extends Error {}, + VectorUnavailableError: class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } + }, + resolveActiveGeneration: vi.fn(), +})); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +vi.mock('./engineAdapter.js', () => ({ getMessageSummariesByIDs: vi.fn(), resolveAccountScope: vi.fn() })); + +import { hybridSearch } from '../services/embeddings/hybrid.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { handleSemanticSearchMessages } from './searchTools.js'; + +const scope = { userId: 'u1', accountIds: ['acc-1'] }; +const payload = (r) => JSON.parse(r.content[0].text); + +// The exact key inventory the deployed container returned for a hybrid/vector hit — +// keyed on message_id, with NO `id` (that is the bug the seam fix repairs). +function realFusedHit(overrides = {}) { + return { + message_id: 'uuid-1', uid: 42, folder: 'INBOX', + subject: 'Run failed: CD deploy', from_name: 'GitHub', from_email: 'notifications@github.com', + date: new Date('2026-07-15T00:00:00Z'), snippet: 'deployment failing', is_read: false, + is_starred: false, has_attachments: false, account_id: 'acc-1', + account_name: 'Work', account_email: 'me@work.com', account_color: '#fff', + rrf_score: 0.033, bm25_score: 1.2, vector_score: 0.88, subject_boosted: false, + best_chunk_index: 0, best_char_start: 5, best_char_end: 40, + ...overrides, + }; +} + +beforeEach(() => { + hybridSearch.mockReset(); matchFromChunk.mockReset(); + getMessageSummariesByIDs.mockReset(); resolveAccountScope.mockReset(); + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); + // Hydration echoes each requested id back as a minimal summary. + getMessageSummariesByIDs.mockImplementation(async (ids) => + ids.map((id) => ({ id, subject: 'Run failed: CD deploy', from_email: 'notifications@github.com' }))); +}); + +describe('semantic_search_messages end-to-end over the real seam (message_id → id alias)', () => { + for (const mode of ['hybrid', 'vector']) { + it(`mode=${mode}: hydrates real message_id-keyed seam hits into data (was returned:0)`, async () => { + hybridSearch.mockResolvedValue({ + hits: [realFusedHit()], + poolSaturated: false, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + matchFromChunk.mockResolvedValue({ char_offset: 5, snippet: 'deploy', score: 0.88 }); + + const r = await handleSemanticSearchMessages({ query: 'server deployment failing', mode, explain: true }, scope); + const body = payload(r); + + expect(r.isError).toBeFalsy(); + expect(body.returned).toBe(1); // the bug returned 0 + expect(body.mode).toBe(mode); + expect(body.data[0].id).toBe('uuid-1'); // hydrated via the aliased id + expect(body.data[0].subject).toBe('Run failed: CD deploy'); + // explain surfaces the real per-signal fields with Go omitempty parity + // (msgvault handlers.go:563-572): rrf only fuses in hybrid, and a false + // subject_boosted is omitted from the wire. + expect(body.data[0].score).toEqual( + mode === 'hybrid' ? { rrf: 0.033, bm25: 1.2, vector: 0.88 } : { bm25: 1.2, vector: 0.88 }, + ); + // matches assembled from best_chunk, keyed on the aliased id + expect(body.data[0].matches).toEqual([{ char_offset: 5, snippet: 'deploy', score: 0.88 }]); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['uuid-1'], ['acc-1']); + expect(matchFromChunk).toHaveBeenCalledWith( + 'uuid-1', { chunk_index: 0, char_start: 5, char_end: 40, score: 0.88 }, { accountIds: ['acc-1'] }); + }); + } +}); diff --git a/backend/src/mcp/server.js b/backend/src/mcp/server.js new file mode 100644 index 0000000..9fb25a3 --- /dev/null +++ b/backend/src/mcp/server.js @@ -0,0 +1,149 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import { mcpBearerAuth } from './auth.js'; +import { TOOL_DEFS, HANDLERS } from './tools.js'; +import { errorResult } from './result.js'; + +// --- Origin validation (DNS-rebinding protection) -------------------------------- +// The MCP Streamable HTTP spec REQUIRES servers to validate the Origin header so a +// malicious web page cannot drive a local MCP endpoint from a victim's browser via +// DNS rebinding. Our SDK (@modelcontextprotocol/sdk 1.29) can enforce this in the +// transport (`enableDnsRebindingProtection` + `allowedOrigins` on +// WebStandardStreamableHTTPServerTransport) but ships with it DISABLED by default +// (GHSA-w48q-cv73-mx4w) and compares origins by exact string match. We enforce at +// the Express layer instead: one owner, normalized (URL.origin, lowercased) +// comparison, and rejection happens before bearer auth ever touches the database. +// +// Policy (mirrors websocket.js): a request with NO Origin header passes — MCP +// clients are non-browser processes and send none. A request WITH an Origin must +// resolve to an allowlisted origin: APP_URL, FRONTEND_URL, any localhost / +// 127.0.0.1 / [::1] origin on any port (a DNS-rebinding page always presents the +// attacker's hostname as Origin, never localhost), or an operator-supplied extra +// via MCP_ALLOWED_ORIGINS (comma-separated URLs, e.g. "https://lan-host:8087"). +const LOCAL_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]']); + +export function buildAllowedOrigins(env = process.env) { + const allowed = new Set(); + const add = (raw) => { + if (!raw || !raw.trim()) return; + try { allowed.add(new URL(raw.trim()).origin.toLowerCase()); } + catch { /* malformed allowlist entry — skip it rather than fail the boot */ } + }; + add(env.APP_URL); + add(env.FRONTEND_URL); + for (const entry of (env.MCP_ALLOWED_ORIGINS || '').split(',')) add(entry); + return allowed; +} + +export function mcpOriginGuard(allowed = buildAllowedOrigins()) { + return (req, res, next) => { + const raw = req.get('Origin'); + if (!raw) return next(); // non-browser MCP client + try { + const url = new URL(raw); + if (allowed.has(url.origin.toLowerCase()) || LOCAL_HOSTNAMES.has(url.hostname.toLowerCase())) { + return next(); + } + } catch { /* unparseable Origin (including the literal "null") — reject below */ } + // Same JSON-RPC error envelope the SDK transport uses for its own HTTP-level + // rejections (webStandardStreamableHttp.js createJsonErrorResponse). + res.status(403).json({ + jsonrpc: '2.0', + error: { code: -32000, message: `Origin not allowed: ${raw}` }, + id: null, + }); + }; +} + +// --- Per-token tool-call rate limit ----------------------------------------------- +// REST search is limited per user (routes/search.js); the MCP surface reuses the same +// in-memory bucket pattern, keyed by the api_tokens row id — NOT the client IP, since +// many agents legitimately share one egress. Only tools/call requests count, so the +// initialize / tools-list handshake a stateless client repeats is never throttled. +// Over-limit requests get an HTTP 429 with Retry-After BEFORE the transport, carrying +// the SDK's JSON-RPC error envelope shape. Default 60 tool calls per minute per token, +// overridable via MCP_RATE_LIMIT_PER_MIN. +export function countToolCalls(body) { + if (Array.isArray(body)) return body.filter((m) => m && m.method === 'tools/call').length; + return body && body.method === 'tools/call' ? 1 : 0; +} + +export function createMcpRateLimiter({ limit, windowMs = 60_000, now = Date.now } = {}) { + const envLimit = Number(process.env.MCP_RATE_LIMIT_PER_MIN); + const max = limit ?? (envLimit > 0 ? envLimit : 60); + const buckets = new Map(); + const sweeper = setInterval(() => { + const t = now(); + for (const [k, b] of buckets) if (t > b.resetAt) buckets.delete(k); + }, windowMs); + sweeper.unref?.(); // observability sweeper must never keep the process alive + + return (req, res, next) => { + const calls = countToolCalls(req.body); + if (!calls) return next(); + const t = now(); + let b = buckets.get(req.mcpTokenId); + if (!b || t > b.resetAt) { + b = { count: 0, resetAt: t + windowMs }; + buckets.set(req.mcpTokenId, b); + } + if (b.count + calls > max) { + res.setHeader('Retry-After', Math.ceil((b.resetAt - t) / 1000)); + return res.status(429).json({ + jsonrpc: '2.0', + error: { code: -32000, message: `Rate limit exceeded: at most ${max} tool calls per minute per token — retry shortly` }, + id: Array.isArray(req.body) ? null : (req.body?.id ?? null), + }); + } + b.count += calls; + next(); + }; +} + +// Build a fresh Server bound to one request's scope. Stateless: no session store, +// one Server+transport per HTTP request, matching msgvault's daemon-less posture. +function buildServer(scope) { + const server = new Server( + { name: 'mailflow', version: '1.0.0' }, + { capabilities: { tools: {} } }, + ); + + server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: TOOL_DEFS, + })); + + server.setRequestHandler(CallToolRequestSchema, async (req) => { + const handler = HANDLERS[req.params.name]; + if (!handler) return errorResult(`unknown tool: ${req.params.name}`); + try { + return await handler(req.params.arguments || {}, scope); + } catch (err) { + // Tool-level failures flow as isError results, not JSON-RPC errors, + // so the client sees a readable message (msgvault convention). + return errorResult(`internal error: ${err.message}`); + } + }); + + return server; +} + +export function mountMcp(app) { + const handle = async (req, res) => { + const server = buildServer(req.mcpScope); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + res.on('close', () => { transport.close(); server.close(); }); + await server.connect(transport); + // Global express.json already parsed the body; hand it to the transport. + await transport.handleRequest(req, res, req.body); + }; + + const originGuard = mcpOriginGuard(buildAllowedOrigins()); + const rateLimiter = createMcpRateLimiter(); + + // Tool calls only arrive as POST bodies; GET (SSE open) and DELETE (session + // teardown) carry none, so the rate limiter guards POST alone. + app.post('/mcp', originGuard, mcpBearerAuth, rateLimiter, handle); + app.get('/mcp', originGuard, mcpBearerAuth, handle); + app.delete('/mcp', originGuard, mcpBearerAuth, handle); +} diff --git a/backend/src/mcp/server.test.js b/backend/src/mcp/server.test.js new file mode 100644 index 0000000..3830134 --- /dev/null +++ b/backend/src/mcp/server.test.js @@ -0,0 +1,77 @@ +import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; +import express from 'express'; +import { createServer } from 'http'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { hashToken } from './auth.js'; +import { mountMcp } from './server.js'; + +let server, base; + +beforeAll(async () => { + const app = express(); + app.use(express.json()); + mountMcp(app); + server = createServer(app); + await new Promise((r) => server.listen(0, r)); + base = `http://127.0.0.1:${server.address().port}`; +}); +afterAll(() => new Promise((r) => server.close(r))); + +// Every authed request: token lookup -> last_used_at update -> resolveScope. +function primeAuth(userId = 'user-1', accountIds = ['acc-1']) { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: userId }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: accountIds.map((id) => ({ id })) }); +} + +async function rpc(method, params, { auth = true } = {}) { + query.mockReset(); + if (auth) primeAuth(); + const headers = { 'Content-Type': 'application/json', Accept: 'application/json, text/event-stream' }; + if (auth) headers.Authorization = 'Bearer mcp_good'; + const res = await fetch(`${base}/mcp`, { + method: 'POST', + headers, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), + }); + return res; +} + +describe('/mcp transport', () => { + it('rejects unauthenticated requests with 401', async () => { + const res = await rpc('tools/list', {}, { auth: false }); + expect(res.status).toBe(401); + }); + + it('completes initialize', async () => { + const res = await rpc('initialize', { + protocolVersion: '2025-06-18', + capabilities: {}, + clientInfo: { name: 'test', version: '0' }, + }); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).toContain('"serverInfo"'); + expect(text).toContain('mailflow'); + }); + + it('lists the ping tool', async () => { + const res = await rpc('tools/list', {}); + const text = await res.text(); + expect(text).toContain('"ping"'); + }); + + it('calls ping and returns pong', async () => { + const res = await rpc('tools/call', { name: 'ping', arguments: {} }); + const text = await res.text(); + expect(text).toContain('{\\"pong\\":true}'); + }); + + it('verifies the token by hash, not plaintext', async () => { + await rpc('tools/list', {}); + expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); + }); +}); diff --git a/backend/src/mcp/serverGuards.test.js b/backend/src/mcp/serverGuards.test.js new file mode 100644 index 0000000..7329d81 --- /dev/null +++ b/backend/src/mcp/serverGuards.test.js @@ -0,0 +1,267 @@ +import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; +import express from 'express'; +import { createServer } from 'http'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { + buildAllowedOrigins, mcpOriginGuard, countToolCalls, createMcpRateLimiter, mountMcp, +} from './server.js'; + +function mockReq({ origin, body, tokenId } = {}) { + return { + get: (h) => (h.toLowerCase() === 'origin' ? origin : undefined), + body, + mcpTokenId: tokenId, + }; +} + +function mockRes() { + const res = { statusCode: 200, headers: {}, body: null }; + res.setHeader = (k, v) => { res.headers[k] = v; }; + res.status = (c) => { res.statusCode = c; return res; }; + res.json = (b) => { res.body = b; return res; }; + return res; +} + +describe('buildAllowedOrigins', () => { + it('derives normalized origins from APP_URL, FRONTEND_URL, and MCP_ALLOWED_ORIGINS', () => { + const allowed = buildAllowedOrigins({ + APP_URL: 'https://Mail.Example.com/', // trailing slash + case normalize away + FRONTEND_URL: 'http://localhost:5173', + MCP_ALLOWED_ORIGINS: 'https://lan-host:8087, http://mail.internal', + }); + expect(allowed).toEqual(new Set([ + 'https://mail.example.com', + 'http://localhost:5173', + 'https://lan-host:8087', + 'http://mail.internal', + ])); + }); + + it('skips malformed and empty entries instead of throwing', () => { + const allowed = buildAllowedOrigins({ APP_URL: 'not a url', MCP_ALLOWED_ORIGINS: ' ,, ' }); + expect(allowed.size).toBe(0); + }); +}); + +describe('mcpOriginGuard', () => { + const guard = mcpOriginGuard(buildAllowedOrigins({ APP_URL: 'https://mail.example.com' })); + + it('passes requests with no Origin header (non-browser MCP clients)', () => { + const next = vi.fn(); + guard(mockReq(), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('passes an allowlisted Origin', () => { + const next = vi.fn(); + guard(mockReq({ origin: 'https://mail.example.com' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('normalizes Origin casing before comparing', () => { + const next = vi.fn(); + guard(mockReq({ origin: 'HTTPS://MAIL.EXAMPLE.COM' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('passes localhost variants on any port', () => { + for (const origin of ['http://localhost:8087', 'http://127.0.0.1:3000', 'http://[::1]:8087', 'https://localhost']) { + const next = vi.fn(); + guard(mockReq({ origin }), mockRes(), next); + expect(next, origin).toHaveBeenCalledTimes(1); + } + }); + + it('rejects a non-allowlisted Origin with a 403 JSON-RPC error', () => { + const next = vi.fn(); + const res = mockRes(); + guard(mockReq({ origin: 'http://evil.example' }), res, next); + expect(next).not.toHaveBeenCalled(); + expect(res.statusCode).toBe(403); + expect(res.body).toEqual({ + jsonrpc: '2.0', + error: { code: -32000, message: 'Origin not allowed: http://evil.example' }, + id: null, + }); + }); + + it('rejects an attacker hostname that merely resolves to this host (DNS rebinding)', () => { + const res = mockRes(); + guard(mockReq({ origin: 'http://rebind.attacker.net:8087' }), res, vi.fn()); + expect(res.statusCode).toBe(403); + }); + + it('rejects unparseable Origins, including the literal "null"', () => { + for (const origin of ['null', 'not a url']) { + const res = mockRes(); + guard(mockReq({ origin }), res, vi.fn()); + expect(res.statusCode, origin).toBe(403); + } + }); +}); + +describe('countToolCalls', () => { + it('counts a single tools/call body as 1 and other methods as 0', () => { + expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/call', id: 1 })).toBe(1); + expect(countToolCalls({ jsonrpc: '2.0', method: 'initialize', id: 1 })).toBe(0); + expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/list', id: 1 })).toBe(0); + expect(countToolCalls(undefined)).toBe(0); + }); + + it('counts tools/call entries inside a batch array', () => { + expect(countToolCalls([ + { method: 'tools/call' }, { method: 'notifications/initialized' }, { method: 'tools/call' }, + ])).toBe(2); + }); +}); + +describe('createMcpRateLimiter', () => { + let clock; + const now = () => clock; + const call = (id = 1) => ({ jsonrpc: '2.0', method: 'tools/call', id }); + + beforeEach(() => { clock = 1_000_000; }); + + it('allows up to the limit then 429s with Retry-After and a JSON-RPC error', () => { + const limiter = createMcpRateLimiter({ limit: 2, now }); + for (let i = 0; i < 2; i++) { + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + } + const res = mockRes(); + const next = vi.fn(); + limiter(mockReq({ body: call(42), tokenId: 'tok-1' }), res, next); + expect(next).not.toHaveBeenCalled(); + expect(res.statusCode).toBe(429); + expect(res.headers['Retry-After']).toBe(60); + expect(res.body.jsonrpc).toBe('2.0'); + expect(res.body.error.code).toBe(-32000); + expect(res.body.error.message).toMatch(/rate limit/i); + expect(res.body.id).toBe(42); // correlates with the throttled request + }); + + it('keys buckets per token, not globally', () => { + const limiter = createMcpRateLimiter({ limit: 1, now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-2' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); // a different token has its own budget + }); + + it('never throttles the initialize/tools-list handshake', () => { + const limiter = createMcpRateLimiter({ limit: 1, now }); + for (const method of ['initialize', 'notifications/initialized', 'tools/list', 'tools/list']) { + const next = vi.fn(); + limiter(mockReq({ body: { jsonrpc: '2.0', method, id: 1 }, tokenId: 'tok-1' }), mockRes(), next); + expect(next, method).toHaveBeenCalledTimes(1); + } + }); + + it('resets the budget after the window elapses', () => { + const limiter = createMcpRateLimiter({ limit: 1, now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const blocked = mockRes(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), blocked, vi.fn()); + expect(blocked.statusCode).toBe(429); + clock += 60_001; + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('reads the default limit from MCP_RATE_LIMIT_PER_MIN', () => { + vi.stubEnv('MCP_RATE_LIMIT_PER_MIN', '1'); + try { + const limiter = createMcpRateLimiter({ now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const res = mockRes(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), res, vi.fn()); + expect(res.statusCode).toBe(429); + } finally { + vi.unstubAllEnvs(); + } + }); +}); + +// --- End-to-end through the live Express mount (same harness as server.test.js) --- +describe('mounted /mcp guards', () => { + const servers = []; + afterAll(async () => { + for (const s of servers) await new Promise((r) => s.close(r)); + }); + + async function mountApp(env) { + for (const [k, v] of Object.entries(env)) vi.stubEnv(k, v); + const app = express(); + app.use(express.json()); + mountMcp(app); // captures env-derived allowlist + limit at mount time + vi.unstubAllEnvs(); + const server = createServer(app); + servers.push(server); + await new Promise((r) => server.listen(0, r)); + return `http://127.0.0.1:${server.address().port}`; + } + + // Every authed request: token lookup -> last_used_at update -> resolveScope. + function primeAuth() { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: 'user-1' }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); + } + + async function rpc(base, method, params, { origin } = {}) { + query.mockReset(); + primeAuth(); + const headers = { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + Authorization: 'Bearer mcp_good', + }; + if (origin) headers.Origin = origin; + return fetch(`${base}/mcp`, { + method: 'POST', + headers, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), + }); + } + + it('403s a cross-origin browser request before auth, passes the app origin and no-Origin clients', async () => { + const base = await mountApp({ APP_URL: 'https://mail.example.com' }); + + query.mockReset(); + const evil = await fetch(`${base}/mcp`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Origin: 'http://evil.example' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }), + }); + expect(evil.status).toBe(403); + expect((await evil.json()).error.code).toBe(-32000); + expect(query).not.toHaveBeenCalled(); // rejected before the token ever hits the DB + + const sameOrigin = await rpc(base, 'tools/list', {}, { origin: 'https://mail.example.com' }); + expect(sameOrigin.status).toBe(200); + + const headless = await rpc(base, 'tools/list', {}); + expect(headless.status).toBe(200); + }); + + it('429s tool calls over the per-token budget but leaves tools/list untouched', async () => { + const base = await mountApp({ MCP_RATE_LIMIT_PER_MIN: '1' }); + + const first = await rpc(base, 'tools/call', { name: 'ping', arguments: {} }); + expect(first.status).toBe(200); + + const second = await rpc(base, 'tools/call', { name: 'ping', arguments: {} }); + expect(second.status).toBe(429); + expect(second.headers.get('Retry-After')).toMatch(/^\d+$/); + const body = await second.json(); + expect(body.error.message).toMatch(/rate limit/i); + + const list = await rpc(base, 'tools/list', {}); + expect(list.status).toBe(200); + }); +}); diff --git a/backend/src/mcp/tools.js b/backend/src/mcp/tools.js new file mode 100644 index 0000000..83779aa --- /dev/null +++ b/backend/src/mcp/tools.js @@ -0,0 +1,54 @@ +import { jsonResult } from './result.js'; +import { + searchMetadataDef, handleSearchMetadata, + searchMessageBodiesDef, handleSearchMessageBodies, + semanticSearchMessagesDef, handleSemanticSearchMessages, +} from './searchTools.js'; +import { + getMessageDef, handleGetMessage, + listMessagesDef, handleListMessages, + getStatsDef, handleGetStats, + aggregateDef, handleAggregate, + searchByDomainsDef, handleSearchByDomains, + findSimilarMessagesDef, handleFindSimilarMessages, + searchInMessageDef, handleSearchInMessage, + stageDeletionDef, handleStageDeletion, +} from './messageTools.js'; + +// TOOL_DEFS drives tools/list; HANDLERS drives tools/call. Slices 09 and 10 +// append to both. Keep names, descriptions, and inputSchema field names verbatim +// from internal/mcp/server.go (README D6: id-bearing fields diverge to strings). +export const TOOL_DEFS = [ + { + name: 'ping', + description: 'Health check: returns {"pong":true}. Proves transport + auth round-trip.', + inputSchema: { type: 'object', properties: {} }, + }, + searchMetadataDef, + searchMessageBodiesDef, + semanticSearchMessagesDef, + getMessageDef, + listMessagesDef, + getStatsDef, + aggregateDef, + searchByDomainsDef, + findSimilarMessagesDef, + searchInMessageDef, + stageDeletionDef, +]; + +export const HANDLERS = { + // eslint-disable-next-line no-unused-vars + ping: async (_args, _scope) => jsonResult({ pong: true }), + search_metadata: (a, s) => handleSearchMetadata(a, s), + search_message_bodies: (a, s) => handleSearchMessageBodies(a, s), + semantic_search_messages: (a, s) => handleSemanticSearchMessages(a, s), + get_message: (a, s) => handleGetMessage(a, s), + list_messages: (a, s) => handleListMessages(a, s), + get_stats: (a, s) => handleGetStats(a, s), + aggregate: (a, s) => handleAggregate(a, s), + search_by_domains: (a, s) => handleSearchByDomains(a, s), + find_similar_messages: (a, s) => handleFindSimilarMessages(a, s), + search_in_message: (a, s) => handleSearchInMessage(a, s), + stage_deletion: (a, s) => handleStageDeletion(a, s), +}; diff --git a/backend/src/mcp/vectorErrors.js b/backend/src/mcp/vectorErrors.js new file mode 100644 index 0000000..13b658d --- /dev/null +++ b/backend/src/mcp/vectorErrors.js @@ -0,0 +1,18 @@ +// Code prefixes are verbatim from msgvault (handlers.go translateVectorErr). +// Remediation hints are Mailflow-specific (msgvault references a CLI we don't +// ship) — a documented divergence; golden diffing asserts the prefix only. +const MESSAGES = { + vector_not_enabled: 'vector_not_enabled: vector search is not configured on this server', + index_stale: 'index_stale: the vector index does not match the configured model; reconfigure embeddings to rebuild', + index_building: 'index_building: the initial vector index is still being built', + no_active_generation: 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + // msgvault handlers.go:225-229; remediation names Mailflow's knob (the + // embedding client timeout) instead of msgvault's [vector.embeddings].timeout TOML. + embedding_timeout: 'embedding_timeout: the embedding endpoint did not respond in time; retry, or raise the embedding client timeout in settings', +}; + +export const VECTOR_ERROR_CODES = Object.keys(MESSAGES); + +export function translateVectorError(reason) { + return MESSAGES[reason] || MESSAGES.vector_not_enabled; +} diff --git a/backend/src/mcp/vectorErrors.test.js b/backend/src/mcp/vectorErrors.test.js new file mode 100644 index 0000000..334a07c --- /dev/null +++ b/backend/src/mcp/vectorErrors.test.js @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; +import { translateVectorError } from './vectorErrors.js'; + +describe('translateVectorError', () => { + it('maps each reason to its verbatim code prefix', () => { + expect(translateVectorError('vector_not_enabled')).toMatch(/^vector_not_enabled: /); + expect(translateVectorError('index_stale')).toMatch(/^index_stale: /); + expect(translateVectorError('index_building')).toMatch(/^index_building: /); + expect(translateVectorError('no_active_generation')).toMatch(/^no_active_generation: /); + expect(translateVectorError('embedding_timeout')).toMatch(/^embedding_timeout: /); + }); + it('index_stale matches msgvault prefix-verbatim (handlers.go:206-210 — no inserted "embedding")', () => { + expect(translateVectorError('index_stale')) + .toMatch(/^index_stale: the vector index does not match the configured model; /); + }); + it('embedding_timeout ports msgvault wording (handlers.go:225-229)', () => { + expect(translateVectorError('embedding_timeout')) + .toMatch(/^embedding_timeout: the embedding endpoint did not respond in time; retry, /); + }); + it('falls back to vector_not_enabled for an unknown reason', () => { + expect(translateVectorError('mystery')).toMatch(/^vector_not_enabled: /); + }); +}); diff --git a/backend/src/mcp/vectorStats.js b/backend/src/mcp/vectorStats.js new file mode 100644 index 0000000..dc90f2e --- /dev/null +++ b/backend/src/mcp/vectorStats.js @@ -0,0 +1,75 @@ +import { query } from '../services/db.js'; +import * as generations from '../services/embeddings/generations.js'; // phase 3 + +// Epoch SECONDS (index_generations.activated_at/started_at, bigint — node-pg +// returns it as a string) → RFC3339 UTC string, matching msgvault CollectStats +// (vector/stats.go:146-153: time.Unix(sec,0).UTC().Format(time.RFC3339) — no +// sub-second digits, unlike Date.toISOString()). "" for a missing/zero/ +// unparsable epoch; callers omit the field entirely then (Go omitempty on +// activated_at/started_at, stats.go:47,:56). +function epochToISO(sec) { + const n = Number(sec); + if (!Number.isFinite(n) || n <= 0) return ''; + return new Date(n * 1000).toISOString().replace('.000Z', 'Z'); +} + +// Live-message count still needing embedding, SCOPED to the caller's accounts +// (documented divergence from msgvault's single-archive count — avoids a cross-user +// cardinality leak). Frozen invariant: `embed_gen IS NULL ⟺ needs embedding` — +// createGeneration resets every live row's stamp to NULL on a rebuild, so the count +// is generation-agnostic and the old `OR embed_gen <> gen` arm was dead/double-counting. +async function missingCount(accountIds) { + if (!accountIds || !accountIds.length) return 0; + const { rows } = await query( + `SELECT COUNT(*)::bigint AS n FROM messages + WHERE account_id = ANY($1) AND is_deleted = false AND embed_gen IS NULL`, + [accountIds], + ); + return Number(rows[0].n); +} + +// Port of vector.CollectStats. Returns null when vector search is disabled (the +// generation queries throw on stock Postgres without the vector schema); an +// enabled archive with no active generation yet reports active_generation: null. +// Generation metadata is archive-global; missing_embeddings_total is account-scoped. +export async function collectStats(accountIds) { + let active; + try { + active = await generations.activeGeneration(); // null = none yet; throw = disabled + } catch { + return null; + } + + // Sub-query failures degrade to partial data (msgvault stats.go:69-78: one + // broken sub-query never blanks the whole stats envelope) — every leg below + // catches and falls back rather than throwing the block away. + const out = { enabled: true, active_generation: null, missing_embeddings_total: 0 }; + if (active) { + const messageCount = await generations.chunkCount(active.id).catch(() => 0); + const activatedAt = epochToISO(active.activatedAt); + out.active_generation = { + id: active.id, model: active.model, dimension: active.dimension, + fingerprint: active.fingerprint, state: active.state, + ...(activatedAt ? { activated_at: activatedAt } : {}), // omitempty (stats.go:47) + message_count: messageCount, + }; + } + + const building = await generations.buildingGeneration().catch(() => null); + if (building) { + // A rebuild in flight is the actionable coverage target; active-generation + // top-ups are frozen until activation (msgvault CollectStats semantics). + const done = await generations.chunkCount(building.id).catch(() => 0); + const pending = await missingCount(accountIds).catch(() => 0); + const startedAt = epochToISO(building.startedAt); + out.building_generation = { + id: building.id, model: building.model, dimension: building.dimension, + ...(startedAt ? { started_at: startedAt } : {}), // omitempty (stats.go:56) + progress: { done, total: done + pending }, + }; + out.missing_embeddings_total = pending; + } else if (active) { + out.missing_embeddings_total = await missingCount(accountIds).catch(() => 0); + } + return out; +} diff --git a/backend/src/mcp/vectorStats.test.js b/backend/src/mcp/vectorStats.test.js new file mode 100644 index 0000000..cbc093e --- /dev/null +++ b/backend/src/mcp/vectorStats.test.js @@ -0,0 +1,98 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../services/embeddings/generations.js', () => ({ + activeGeneration: vi.fn(), buildingGeneration: vi.fn(), chunkCount: vi.fn(), +})); +import { query } from '../services/db.js'; +import * as generations from '../services/embeddings/generations.js'; +import { collectStats } from './vectorStats.js'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +beforeEach(() => { query.mockReset(); generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); }); + +describe('mock-drift guard', () => { + it('every mocked generations key exists as a function on the real module', async () => { + // Regression guard: chunkCount was mocked here (and in goldenParity) while the + // real generations.js never implemented it — collectStats threw live. + const real = await vi.importActual('../services/embeddings/generations.js'); + expect(mockSurfaceDrift(generations, real)).toEqual([]); + }); +}); + +describe('collectStats', () => { + it('returns null when vector search is disabled', async () => { + generations.activeGeneration.mockRejectedValue(new Error('disabled')); + expect(await collectStats(['a'])).toBeNull(); + }); + + it('reports the active generation and a scoped missing count', async () => { + // activatedAt is epoch SECONDS (bigint) as generationByState now returns it; + // the wire field is RFC3339 UTC WITHOUT sub-second digits (msgvault + // vector/stats.go:146-153 formatTime uses time.RFC3339, never millis). + // 1704067200 = 2024-01-01T00:00:00Z. + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(1000); + query.mockResolvedValueOnce({ rows: [{ n: '7' }] }); // missing count + const vs = await collectStats(['acc-1']); + expect(vs.enabled).toBe(true); + expect(vs.active_generation).toEqual({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activated_at: '2024-01-01T00:00:00Z', message_count: 1000 }); + expect(vs.missing_embeddings_total).toBe(7); + // chunkCount is keyed by generation id (real signature). The missing count is scoped to + // accountIds and keys off `embed_gen IS NULL` only (frozen invariant — no dead OR arm). + expect(generations.chunkCount).toHaveBeenCalledWith(2); + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); + const sql = query.mock.calls[0][0]; + expect(sql).toMatch(/embed_gen IS NULL/); + expect(sql).not.toMatch(/embed_gen\s*<>/); + }); + + it('enabled with no active generation yet (first build) reports a null active_generation', async () => { + generations.activeGeneration.mockResolvedValue(null); + generations.buildingGeneration.mockResolvedValue(null); + const vs = await collectStats([]); + expect(vs).toEqual({ enabled: true, active_generation: null, missing_embeddings_total: 0 }); + expect(query).not.toHaveBeenCalled(); // empty scope skips the count query + }); + + it('OMITS activated_at when the epoch is absent or zero (msgvault omitempty, stats.go:47)', async () => { + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active' }); // no activatedAt + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(3); + query.mockResolvedValueOnce({ rows: [{ n: '0' }] }); + const vs = await collectStats(['acc-1']); + expect(vs.active_generation).not.toHaveProperty('activated_at'); + }); + + it('reports a building generation with scoped progress and freezes missing on the build target', async () => { + // startedAt is epoch SECONDS → RFC3339 (no millis) wire. 1706745600 = 2024-02-01T00:00:00Z. + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active' }); + generations.buildingGeneration.mockResolvedValue({ id: 3, model: 'm2', dimension: 8, startedAt: 1706745600 }); + generations.chunkCount.mockImplementation(async (id) => (id === 2 ? 100 : 40)); // active done, building done + query.mockResolvedValueOnce({ rows: [{ n: '10' }] }); // missing for building gen 3 + const vs = await collectStats(['acc-1']); + expect(vs.building_generation).toEqual({ id: 3, model: 'm2', dimension: 8, started_at: '2024-02-01T00:00:00Z', progress: { done: 40, total: 50 } }); + expect(vs.missing_embeddings_total).toBe(10); // building coverage is the actionable target + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); // generation-agnostic (embed_gen IS NULL) + }); + + it('OMITS started_at when the building epoch is absent (msgvault omitempty, stats.go:56)', async () => { + generations.activeGeneration.mockResolvedValue(null); + generations.buildingGeneration.mockResolvedValue({ id: 3, model: 'm2', dimension: 8 }); // no startedAt + generations.chunkCount.mockResolvedValue(40); + query.mockResolvedValueOnce({ rows: [{ n: '10' }] }); + const vs = await collectStats(['acc-1']); + expect(vs.building_generation).not.toHaveProperty('started_at'); + }); + + it('degrades to partial data when the missing-count sub-query fails (msgvault stats.go:69-78 best-effort)', async () => { + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(3); + query.mockRejectedValueOnce(new Error('relation vanished')); // missingCount blows up + const vs = await collectStats(['acc-1']); + expect(vs).not.toBeNull(); // one broken sub-query must not blank the whole block + expect(vs.active_generation.id).toBe(2); + expect(vs.missing_embeddings_total).toBe(0); + }); +}); diff --git a/backend/src/routes/accounts.js b/backend/src/routes/accounts.js index 39f2202..7ed1837 100644 --- a/backend/src/routes/accounts.js +++ b/backend/src/routes/accounts.js @@ -8,6 +8,9 @@ import { validateHost } from '../services/hostValidation.js'; import { getConnectionPolicy } from '../services/connectionPolicy.js'; import { invalidateGtdConfigCache, sanitizeGtdFoldersDetailed, findGtdFolderCollisions, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js'; import { createKeyedSerializer } from '../utils/keyedSerializer.js'; +import { startAccountBodyBackfill } from '../services/bodyBackfill.js'; +import { providerProfile } from '../services/imapManager.js'; +import { upsertJob } from '../services/backgroundJobs.js'; // Serialize an account's reconnect triggers so a rapid settings change (e.g. a // gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains — @@ -417,7 +420,18 @@ router.post('/:id/reindex', async (req, res) => { console.error(`Manual reindex error for ${account.email_address}:`, err.message) ); } - res.json({ ok: true, alreadyRunning }); + + // Also kick a body-materialization pass for allowlisted providers. Fire-and-forget: the + // drainer self-guards against concurrent runs and caps each session, and Gmail/PurelyMail/ + // Microsoft are gated off inside startBodyBackfill. Progress lands in background_jobs. + const bodyBackfillEnabled = providerProfile(account).bodyBackfill; + if (bodyBackfillEnabled) { + startAccountBodyBackfill(account, imapManager, upsertJob).catch(err => + console.error(`Body backfill error for ${account.email_address}:`, err.message) + ); + } + + res.json({ ok: true, alreadyRunning, bodyBackfillEnabled }); } catch (err) { console.error('POST /accounts/:id/reindex error:', err.message); res.status(500).json({ error: 'Failed to start reindex' }); diff --git a/backend/src/routes/ai.buildEmbeddingsConfig.test.js b/backend/src/routes/ai.buildEmbeddingsConfig.test.js new file mode 100644 index 0000000..8505fc8 --- /dev/null +++ b/backend/src/routes/ai.buildEmbeddingsConfig.test.js @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../services/encryption.js', () => ({ + encrypt: (v) => `enc:${v}`, + decrypt: (v) => (v ? String(v).replace(/^enc:/, '') : v), +})); +import { buildEmbeddingsConfig } from './ai.js'; + +describe('buildEmbeddingsConfig', () => { + it('encrypts a freshly supplied apiKey', () => { + const out = buildEmbeddingsConfig( + { enabled: true, endpoint: 'http://h/v1', apiKey: 'sk-1', model: 'm', dimension: 768 }, + null, + ); + expect(out.apiKey).toBe('enc:sk-1'); + expect(out.dimension).toBe(768); + expect(out.preprocess.stripHTML).toBe(true); + }); + it('keeps the existing key when the masked sentinel is sent back', () => { + const out = buildEmbeddingsConfig( + { apiKey: '••••••••', model: 'm', dimension: 4 }, + { apiKey: 'enc:old' }, + ); + expect(out.apiKey).toBe('enc:old'); + }); + it('preserves an explicit-false preprocess flag', () => { + const out = buildEmbeddingsConfig( + { model: 'm', dimension: 4, preprocess: { stripQuotes: false } }, + null, + ); + expect(out.preprocess.stripQuotes).toBe(false); + }); +}); diff --git a/backend/src/routes/ai.js b/backend/src/routes/ai.js index 088fc38..8c32ade 100644 --- a/backend/src/routes/ai.js +++ b/backend/src/routes/ai.js @@ -4,9 +4,24 @@ import { requireAuth, requireAdmin } from '../middleware/auth.js'; import { encrypt, decrypt } from '../services/encryption.js'; import { validateHost } from '../services/hostValidation.js'; import { getConnectionPolicy } from '../services/connectionPolicy.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import { applyEmbedDefaults } from '../services/embeddings/config.js'; const router = Router(); +// Merge an embeddings sub-config from a PATCH body with the existing stored block. +// config.js owns every field default (including endpoint trim + trailing-slash strip) +// via applyEmbedDefaults, so the read and write paths normalize identically; here we +// layer only the write-path apiKey concern: a fresh key is encrypted, the masked +// sentinel keeps the stored key. +export function buildEmbeddingsConfig(body = {}, existing = null) { + const resolved = applyEmbedDefaults(body); + resolved.apiKey = body.apiKey && body.apiKey !== '••••••••' + ? encrypt(body.apiKey) + : (existing?.apiKey || null); + return resolved; +} + // ── Admin: AI provider configuration ────────────────────────────────────────── router.get('/admin/ai', requireAdmin, async (req, res) => { @@ -14,7 +29,11 @@ router.get('/admin/ai', requireAdmin, async (req, res) => { if (!result.rows.length) return res.json({ config: null }); try { const cfg = JSON.parse(result.rows[0].value); - res.json({ config: { ...cfg, apiKey: cfg.apiKey ? '••••••••' : '' } }); + const masked = { ...cfg, apiKey: cfg.apiKey ? '••••••••' : '' }; + if (cfg.embeddings) { + masked.embeddings = { ...cfg.embeddings, apiKey: cfg.embeddings.apiKey ? '••••••••' : '' }; + } + res.json({ config: masked }); } catch { res.json({ config: null }); } @@ -49,6 +68,30 @@ router.patch('/admin/ai', requireAdmin, async (req, res) => { } } + let existingEmbeddings = null; + if (existing.rows.length) { + try { existingEmbeddings = JSON.parse(existing.rows[0].value).embeddings || null; } catch { /* keep null */ } + } + const embeddings = req.body.embeddings + ? buildEmbeddingsConfig(req.body.embeddings, existingEmbeddings) + : existingEmbeddings; + + // Host-validate the embeddings endpoint the same way baseUrl is validated. + if (embeddings?.endpoint) { + let embHost; + try { embHost = new URL(embeddings.endpoint).hostname; } catch { + return res.status(400).json({ error: 'Invalid embeddings endpoint URL' }); + } + const policy = await getConnectionPolicy(); + const embErr = await validateHost(embHost, { allowPrivate: policy.allowPrivateHosts }); + if (embErr) { + const hint = embErr.includes('private or reserved') + ? ' To use a local network address, enable "Allow private hosts" in Settings → Security.' + : ''; + return res.status(400).json({ error: `Embeddings endpoint: ${embErr}.${hint}` }); + } + } + const cfg = { enabled: enabled !== false, baseUrl: trimmedBaseUrl, @@ -58,6 +101,7 @@ router.patch('/admin/ai', requireAdmin, async (req, res) => { compose: features?.compose !== false, summarize: features?.summarize !== false, }, + ...(embeddings ? { embeddings } : {}), }; await query( @@ -121,15 +165,17 @@ router.post('/admin/ai/test', requireAdmin, async (req, res) => { router.get('/ai/status', requireAuth, async (req, res) => { const result = await query("SELECT value FROM system_settings WHERE key = 'ai_config'"); - if (!result.rows.length) return res.json({ enabled: false, features: {} }); + if (!result.rows.length) return res.json({ enabled: false, features: {}, vectorAvailable: isVectorAvailable() }); try { const cfg = JSON.parse(result.rows[0].value); res.json({ enabled: cfg.enabled === true && !!cfg.baseUrl && !!cfg.model, features: cfg.features || {}, + vectorAvailable: isVectorAvailable(), + embeddingsEnabled: cfg.embeddings?.enabled === true && !!cfg.embeddings?.endpoint && !!cfg.embeddings?.model, }); } catch { - res.json({ enabled: false, features: {} }); + res.json({ enabled: false, features: {}, vectorAvailable: isVectorAvailable() }); } }); diff --git a/backend/src/routes/aiEmbeddings.js b/backend/src/routes/aiEmbeddings.js new file mode 100644 index 0000000..e41021e --- /dev/null +++ b/backend/src/routes/aiEmbeddings.js @@ -0,0 +1,148 @@ +import { Router } from 'express'; +import { requireAdmin } from '../middleware/auth.js'; +import { resolveEmbedConfig, generationFingerprint } from '../services/embeddings/config.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import * as store from '../services/embeddings/vectorStore.js'; +import * as generations from '../services/embeddings/generations.js'; +const { createGeneration, buildingGeneration, retireGeneration, BuildingInProgressError } = generations; +import { EmbeddingClient } from '../services/embeddings/client.js'; +import { EmbeddingWorker } from '../services/embeddings/worker.js'; +import { tryAcquireEmbedRun, releaseEmbedRun } from '../services/embeddings/embedRunLock.js'; +import { upsertJob } from '../services/backgroundJobs.js'; +import { query } from '../services/db.js'; + +const router = Router(); + +// Returns an error string when the config cannot start a build, else null. Pure helper. +export function validateBuildConfig(cfg) { + if (!isVectorAvailable()) return 'Vector extension unavailable — semantic search is disabled on this database'; + if (!cfg) return 'Embeddings not configured'; + if (!cfg.enabled) return 'Embeddings are disabled'; + if (!cfg.endpoint) return 'Embeddings endpoint is required'; + if (!cfg.model) return 'Embeddings model is required'; + if (!(cfg.dimension > 0)) return 'Embeddings dimension must be a positive integer'; + return null; +} + +// Probe the embedding endpoint with one input and echo the returned dimension. Pure helper. +export async function probeEmbeddings(client) { + const vecs = await client.embed(['mailflow embeddings connectivity probe']); + return { ok: true, dimension: vecs[0].length }; +} + +// The probe client intentionally omits the dimension expectation (dimension: null +// skips the client's per-vector assertion): Test's whole job is to DISCOVER the +// endpoint's real dimension so the UI can reconcile a wrong saved value +// (reconcileDimension auto-fill). With the assertion in place, a mismatch threw +// before the probed dimension ever reached the response, making that UI path +// unreachable. Worker/query paths keep the strict assertion — they construct +// their clients with cfg.dimension. +export function buildProbeClient(cfg) { + return new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: null }); +} + +router.post('/admin/ai/embeddings/test-embeddings', requireAdmin, async (req, res) => { + const cfg = await resolveEmbedConfig(); + if (!cfg || !cfg.endpoint || !cfg.model || !(cfg.dimension > 0)) { + return res.status(400).json({ error: 'Embeddings endpoint, model, and dimension are required' }); + } + try { + res.json(await probeEmbeddings(buildProbeClient(cfg))); + } catch (err) { + res.status(400).json({ error: err.message }); + } +}); + +// Count live messages still needing embedding under generation `gen` (the build's +// initial "total"). Default collaborator for startEmbeddingBuild. +async function countPending(gen) { + const r = await query( + 'SELECT COUNT(*)::int n FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $1) AND is_deleted = false', [gen], + ); + return r.rows[0].n; +} + +// Fire-and-forget worker run toward coverage. Returns worker.runOnce's promise so the +// caller can chain job-state updates and the single-flight release onto its settlement. +function runWorker(gen, total, cfg) { + const client = new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: cfg.dimension }); + const worker = new EmbeddingWorker({ + // `generations` lets the worker activate this building generation once its + // scan drains to full coverage (the shared activation seam in worker.js). + store, client, generations, preprocessCfg: cfg.preprocess, maxInputChars: cfg.maxInputChars, batchSize: cfg.batchSize, + onProgress: (p) => { upsertJob({ kind: 'embeddings', state: 'running', processed: p.done, total }).catch(() => {}); }, + }); + return worker.runOnce(gen); +} + +// Collaborators for startEmbeddingBuild, bound to the real modules by default and +// overridable in tests (the injection pattern used across the embeddings services). +export const BUILD_DEPS = { + tryAcquireEmbedRun, releaseEmbedRun, + createGeneration, buildingGeneration, retireGeneration, generationFingerprint, + countPending, upsertJob, runWorker, log: console.log, +}; + +// Orchestrates an embeddings (re)build. Returns { status, body } for the route to send. +// +// Ordering matters: createGeneration atomically NULLs every live embed_gen stamp. If an +// embed run were mid-flight when that reset lands, it could re-stamp rows with the OLD +// generation id afterward — and the new generation's scan (embed_gen IS NULL only) would +// never see them, so activation's coverage gate blocks forever. So we take the single- +// flight lock BEFORE createGeneration, making the stamp-reset mutually exclusive with any +// embed run. If the lock is busy we return an honest, retryable 409 and never touch the +// stamps. On every non-success exit the lock is released exactly once; on success the +// fire-and-forget worker chain owns the single release when the run settles. +export async function startEmbeddingBuild(cfg, username, deps = {}) { + const d = { ...BUILD_DEPS, ...deps }; + const fingerprint = d.generationFingerprint(cfg); + + if (!d.tryAcquireEmbedRun()) { + return { status: 409, body: { error: 'An embedding run is in progress — retry in a moment' } }; + } + + let gen, total; + try { + try { + gen = await d.createGeneration(cfg.model, cfg.dimension, fingerprint); + } catch (err) { + // A building generation with a DIFFERENT fingerprint blocks this build. A + // new-fingerprint build supersedes an incomplete old-fingerprint one, so retire + // the stale gen (deletes its rows — generations never mix) and retry once. + if (!(err instanceof BuildingInProgressError)) throw err; + const stale = await d.buildingGeneration(); + if (!stale || stale.fingerprint === fingerprint) throw err; + await d.retireGeneration(stale.id); + d.log(`[admin] ${username} retired stale building gen ${stale.id} (fingerprint ${stale.fingerprint}); superseded by ${fingerprint}`); + gen = await d.createGeneration(cfg.model, cfg.dimension, fingerprint); + } + total = await d.countPending(gen); + await d.upsertJob({ kind: 'embeddings', state: 'running', processed: 0, total }); + } catch (err) { + // Any failure before the worker chain is attached below leaves the lock ours to + // free — release it so a failed start never wedges every future build. + d.releaseEmbedRun(); + return { status: 409, body: { error: err.message } }; + } + + // We already hold the single-flight lock; fire the worker and release exactly once + // when it settles, on every outcome. The request returns immediately. + d.runWorker(gen, total, cfg) + .then((r) => d.upsertJob({ kind: 'embeddings', state: 'done', processed: r.succeeded, total })) + .catch((err) => d.upsertJob({ kind: 'embeddings', state: 'error', processed: 0, total, lastError: err.message })) + .finally(() => d.releaseEmbedRun()) + .catch(() => {}); + d.log(`[admin] ${username} started embeddings build gen ${gen} (${total} pending)`); + return { status: 200, body: { ok: true, generationId: gen, total } }; +} + +router.post('/admin/ai/embeddings/build', requireAdmin, async (req, res) => { + const cfg = await resolveEmbedConfig(); + const invalid = validateBuildConfig(cfg); + if (invalid) return res.status(400).json({ error: invalid }); + + const { status, body } = await startEmbeddingBuild(cfg, req.session.username); + res.status(status).json(body); +}); + +export default router; diff --git a/backend/src/routes/aiEmbeddings.test.js b/backend/src/routes/aiEmbeddings.test.js new file mode 100644 index 0000000..7ce05b8 --- /dev/null +++ b/backend/src/routes/aiEmbeddings.test.js @@ -0,0 +1,149 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +vi.mock('../services/embeddings/vectorStore.js', () => ({ isVectorAvailable: vi.fn(() => true) })); +import { validateBuildConfig, probeEmbeddings, buildProbeClient, startEmbeddingBuild } from './aiEmbeddings.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import { BuildingInProgressError } from '../services/embeddings/generations.js'; + +const full = { enabled: true, endpoint: 'http://h/v1', model: 'm', dimension: 768, maxInputChars: 32768, batchSize: 32, preprocess: {} }; + +describe('validateBuildConfig', () => { + it('accepts a complete config when vector is available', () => { + expect(validateBuildConfig(full)).toBeNull(); + }); + it('rejects when vector is unavailable', () => { + isVectorAvailable.mockReturnValueOnce(false); + expect(validateBuildConfig(full)).toMatch(/vector/i); + }); + it('rejects an incomplete config', () => { + expect(validateBuildConfig({ ...full, dimension: 0 })).toMatch(/dimension/i); + expect(validateBuildConfig(null)).toMatch(/not configured/i); + }); +}); + +describe('probeEmbeddings', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('echoes the returned dimension', async () => { + const fakeClient = { embed: vi.fn().mockResolvedValue([[0.1, 0.2, 0.3]]) }; + const out = await probeEmbeddings(fakeClient); + expect(out).toEqual({ ok: true, dimension: 3 }); + }); + it('surfaces an embed error', async () => { + const fakeClient = { embed: vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')) }; + await expect(probeEmbeddings(fakeClient)).rejects.toThrow(/ECONNREFUSED/); + }); + it('returns the endpoint\'s ACTUAL dimension even when the saved config disagrees', async () => { + // Regression pin: with the probe client asserting cfg.dimension, a mismatch threw + // before Test could return the probed value, so the UI's reconcileDimension + // auto-fill was unreachable. The probe client must skip the assertion. + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + status: 200, + headers: { get: () => null }, + json: async () => ({ data: [{ index: 0, embedding: Array(768).fill(0.1) }] }), + text: async () => '', + })); + const out = await probeEmbeddings(buildProbeClient({ ...full, dimension: 1536 })); + expect(out).toEqual({ ok: true, dimension: 768 }); + }); +}); + +// Let the fire-and-forget worker chain (.then/.catch/.finally) settle. +const flush = () => new Promise((r) => setTimeout(r, 0)); + +function makeDeps(overrides = {}) { + return { + tryAcquireEmbedRun: vi.fn(() => true), + releaseEmbedRun: vi.fn(), + createGeneration: vi.fn().mockResolvedValue('gen-1'), + buildingGeneration: vi.fn().mockResolvedValue(null), + retireGeneration: vi.fn().mockResolvedValue(undefined), + generationFingerprint: vi.fn(() => 'fp'), + countPending: vi.fn().mockResolvedValue(5), + upsertJob: vi.fn().mockResolvedValue(undefined), + runWorker: vi.fn().mockResolvedValue({ succeeded: 5 }), + log: vi.fn(), + ...overrides, + }; +} + +describe('startEmbeddingBuild', () => { + it('returns a retryable 409 and never resets stamps when an embed run is active (Fix 1)', async () => { + const deps = makeDeps({ tryAcquireEmbedRun: vi.fn(() => false) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toMatch(/in progress/i); + expect(deps.createGeneration).not.toHaveBeenCalled(); // stamp-reset never runs + expect(deps.releaseEmbedRun).not.toHaveBeenCalled(); // never acquired ⇒ nothing to release + }); + + it('acquires the single-flight lock before createGeneration (Fix 1)', async () => { + const deps = makeDeps(); + await startEmbeddingBuild(full, 'admin', deps); + expect(deps.tryAcquireEmbedRun).toHaveBeenCalled(); + expect(deps.createGeneration).toHaveBeenCalledWith('m', 768, 'fp'); + expect(deps.tryAcquireEmbedRun.mock.invocationCallOrder[0]) + .toBeLessThan(deps.createGeneration.mock.invocationCallOrder[0]); + await flush(); + }); + + it('on success fires the worker and releases the lock exactly once (Fix 1)', async () => { + const deps = makeDeps(); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true, generationId: 'gen-1', total: 5 }); + expect(deps.runWorker).toHaveBeenCalledWith('gen-1', 5, full); + await flush(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('releases the lock and returns 409 when createGeneration fails outright (Fix 1)', async () => { + const deps = makeDeps({ createGeneration: vi.fn().mockRejectedValue(new Error('boom')) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toBe('boom'); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + expect(deps.retireGeneration).not.toHaveBeenCalled(); + }); + + it('releases the lock if the build fails after createGeneration but before the worker starts (Fix 1)', async () => { + // A failing countPending would otherwise leave the lock held forever (it is taken + // before createGeneration now), wedging every future build. + const deps = makeDeps({ countPending: vi.fn().mockRejectedValue(new Error('db down')) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toBe('db down'); + expect(deps.createGeneration).toHaveBeenCalledTimes(1); + expect(deps.runWorker).not.toHaveBeenCalled(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('retires a stale different-fingerprint building gen and retries createGeneration once (Fix 2b)', async () => { + const createGeneration = vi.fn() + .mockRejectedValueOnce(new BuildingInProgressError('building fingerprint=old-fp, requested=fp')) + .mockResolvedValueOnce('gen-2'); + const deps = makeDeps({ + createGeneration, + buildingGeneration: vi.fn().mockResolvedValue({ id: 'stale-1', fingerprint: 'old-fp' }), + }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(deps.retireGeneration).toHaveBeenCalledWith('stale-1'); + expect(createGeneration).toHaveBeenCalledTimes(2); + expect(res.status).toBe(200); + expect(res.body.generationId).toBe('gen-2'); + await flush(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('surfaces a same-fingerprint BuildingInProgressError as 409 without retiring (Fix 2b)', async () => { + const createGeneration = vi.fn().mockRejectedValue(new BuildingInProgressError('already building')); + const deps = makeDeps({ + createGeneration, + buildingGeneration: vi.fn().mockResolvedValue({ id: 'x', fingerprint: 'fp' }), + }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(deps.retireGeneration).not.toHaveBeenCalled(); + expect(createGeneration).toHaveBeenCalledTimes(1); // no infinite retry + expect(res.status).toBe(409); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/routes/apiTokens.js b/backend/src/routes/apiTokens.js new file mode 100644 index 0000000..598975b --- /dev/null +++ b/backend/src/routes/apiTokens.js @@ -0,0 +1,38 @@ +import { Router } from 'express'; +import { query } from '../services/db.js'; +import { requireAuth } from '../middleware/auth.js'; +import { generateToken, hashToken } from '../mcp/auth.js'; + +const router = Router(); +router.use(requireAuth); + +router.post('/', async (req, res) => { + const name = (req.body?.name || '').trim(); + if (!name) return res.status(400).json({ error: 'name is required' }); + const token = generateToken(); + const { rows } = await query( + 'INSERT INTO api_tokens (user_id, token_hash, name) VALUES ($1, $2, $3) RETURNING id, name', + [req.session.userId, hashToken(token), name], + ); + // Plaintext returned exactly once; only the hash was persisted. + res.status(201).json({ id: rows[0].id, name: rows[0].name, token }); +}); + +router.get('/', async (req, res) => { + const { rows } = await query( + 'SELECT id, name, created_at, last_used_at FROM api_tokens WHERE user_id = $1 ORDER BY created_at DESC', + [req.session.userId], + ); + res.json({ tokens: rows }); +}); + +router.delete('/:id', async (req, res) => { + const { rowCount } = await query( + 'DELETE FROM api_tokens WHERE id = $1 AND user_id = $2', + [req.params.id, req.session.userId], + ); + if (!rowCount) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/apiTokens.test.js b/backend/src/routes/apiTokens.test.js new file mode 100644 index 0000000..92309b2 --- /dev/null +++ b/backend/src/routes/apiTokens.test.js @@ -0,0 +1,80 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +// Stub auth to inject a fixed session user. +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { req.session = { userId: 'user-1' }; next(); }, +})); +import { query } from '../services/db.js'; +import router from './apiTokens.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/tokens', router); + return app; +} +async function call(app, method, path, body) { + const { createServer } = await import('http'); + const server = createServer(app); + await new Promise((r) => server.listen(0, r)); + const base = `http://127.0.0.1:${server.address().port}`; + const res = await fetch(base + path, { + method, + headers: body ? { 'Content-Type': 'application/json' } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + server.close(); + return { status: res.status, body: text ? JSON.parse(text) : null }; +} + +beforeEach(() => query.mockReset()); + +describe('POST /api/tokens', () => { + it('mints a token, returns the plaintext once, and stores only the hash', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop' }] }); + const { status, body } = await call(appWith(), 'POST', '/api/tokens', { name: 'laptop' }); + expect(status).toBe(201); + expect(body.token).toMatch(/^mcp_/); + expect(body).toMatchObject({ id: 'tok-1', name: 'laptop' }); + // INSERT bound values: [user_id, token_hash, name] — never the plaintext. + const params = query.mock.calls[0][1]; + expect(params[0]).toBe('user-1'); + expect(params[1]).toMatch(/^[0-9a-f]{64}$/); + expect(params[2]).toBe('laptop'); + expect(params).not.toContain(body.token); + }); + it('rejects a missing name', async () => { + const { status } = await call(appWith(), 'POST', '/api/tokens', {}); + expect(status).toBe(400); + }); +}); + +describe('GET /api/tokens', () => { + it('lists tokens without hashes or plaintext', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', created_at: 't', last_used_at: null }] }); + const { status, body } = await call(appWith(), 'GET', '/api/tokens'); + expect(status).toBe(200); + expect(body.tokens[0]).toEqual({ id: 'tok-1', name: 'laptop', created_at: 't', last_used_at: null }); + expect(JSON.stringify(body)).not.toContain('token_hash'); + }); +}); + +describe('DELETE /api/tokens/:id', () => { + it('revokes only within the session user', async () => { + query.mockResolvedValueOnce({ rowCount: 1, rows: [] }); + const { status } = await call(appWith(), 'DELETE', '/api/tokens/tok-1'); + expect(status).toBe(204); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/DELETE FROM api_tokens WHERE id = \$1 AND user_id = \$2/), + ['tok-1', 'user-1'], + ); + }); + it('404s when the token is absent or not owned', async () => { + query.mockResolvedValueOnce({ rowCount: 0, rows: [] }); + const { status } = await call(appWith(), 'DELETE', '/api/tokens/nope'); + expect(status).toBe(404); + }); +}); diff --git a/backend/src/routes/indexing.js b/backend/src/routes/indexing.js new file mode 100644 index 0000000..d8966d2 --- /dev/null +++ b/backend/src/routes/indexing.js @@ -0,0 +1,13 @@ +import { Router } from 'express'; +import { requireAdmin } from '../middleware/auth.js'; +import { listJobs } from '../services/backgroundJobs.js'; + +const router = Router(); + +// GET /api/admin/indexing/status — admin-gated (same guard as routes/ai.js). +// Surfaces every background drainer's progress, e.g. "FTS backfill: N/M". +router.get('/status', requireAdmin, async (_req, res) => { + res.json({ jobs: await listJobs() }); +}); + +export default router; diff --git a/backend/src/routes/mcpDeletions.js b/backend/src/routes/mcpDeletions.js new file mode 100644 index 0000000..c7cf924 --- /dev/null +++ b/backend/src/routes/mcpDeletions.js @@ -0,0 +1,25 @@ +import { Router } from 'express'; +import { requireAuth } from '../middleware/auth.js'; +import { executeDeletionBatch, unstageDeletionBatch } from '../mcp/engineAdapter.js'; + +// Session-authenticated execute/unstage endpoints for MCP deletion batches. The +// stage_deletion tool only RECORDS a batch; flipping messages.is_deleted (Mailflow's +// soft delete) happens here, behind a browser session, never from the token. Both +// operations are scoped to req.session.userId, so a user cannot execute/cancel +// another user's batch. +const router = Router(); +router.use(requireAuth); + +router.post('/:id/execute', async (req, res) => { + const n = await executeDeletionBatch(req.params.id, req.session.userId); + if (n === null) return res.status(404).json({ error: 'not found' }); + res.json({ batch_id: req.params.id, status: 'executed', deleted: n }); +}); + +router.delete('/:id', async (req, res) => { + const ok = await unstageDeletionBatch(req.params.id, req.session.userId); + if (!ok) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/mcpDeletions.test.js b/backend/src/routes/mcpDeletions.test.js new file mode 100644 index 0000000..1a4eff2 --- /dev/null +++ b/backend/src/routes/mcpDeletions.test.js @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; + +vi.mock('../mcp/engineAdapter.js', () => ({ executeDeletionBatch: vi.fn(), unstageDeletionBatch: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { req.session = { userId: 'user-1' }; next(); }, +})); +import { executeDeletionBatch, unstageDeletionBatch } from '../mcp/engineAdapter.js'; +import router from './mcpDeletions.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/mcp-deletions', router); + return app; +} +async function call(app, method, path) { + const { createServer } = await import('http'); + const server = createServer(app); + await new Promise((r) => server.listen(0, r)); + const base = `http://127.0.0.1:${server.address().port}`; + const res = await fetch(base + path, { method }); + const text = await res.text(); + server.close(); + return { status: res.status, body: text ? JSON.parse(text) : null }; +} + +beforeEach(() => { executeDeletionBatch.mockReset(); unstageDeletionBatch.mockReset(); }); + +describe('POST /api/mcp-deletions/:id/execute', () => { + it('soft-deletes the owner batch and reports the count, scoped to the session user', async () => { + executeDeletionBatch.mockResolvedValue(3); + const { status, body } = await call(appWith(), 'POST', '/api/mcp-deletions/b1/execute'); + expect(status).toBe(200); + expect(body).toEqual({ batch_id: 'b1', status: 'executed', deleted: 3 }); + expect(executeDeletionBatch).toHaveBeenCalledWith('b1', 'user-1'); + }); + + it('404s a batch not owned / not found (cross-user isolation, no update)', async () => { + executeDeletionBatch.mockResolvedValue(null); + const { status } = await call(appWith(), 'POST', '/api/mcp-deletions/other/execute'); + expect(status).toBe(404); + }); +}); + +describe('DELETE /api/mcp-deletions/:id', () => { + it('unstages only the owner batch', async () => { + unstageDeletionBatch.mockResolvedValue(true); + const { status } = await call(appWith(), 'DELETE', '/api/mcp-deletions/b1'); + expect(status).toBe(204); + expect(unstageDeletionBatch).toHaveBeenCalledWith('b1', 'user-1'); + }); + + it('404s when absent or not owned', async () => { + unstageDeletionBatch.mockResolvedValue(false); + const { status } = await call(appWith(), 'DELETE', '/api/mcp-deletions/nope'); + expect(status).toBe(404); + }); +}); diff --git a/backend/src/routes/search.js b/backend/src/routes/search.js index dbd4379..6398064 100644 --- a/backend/src/routes/search.js +++ b/backend/src/routes/search.js @@ -1,10 +1,14 @@ import { Router } from 'express'; import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; const router = Router(); router.use(requireAuth); +const ALLOWED_MODES = new Set(['lexical', 'vector', 'hybrid']); + // Simple in-memory rate limiter: 20 searches per minute per user. const searchBuckets = new Map(); setInterval(() => { @@ -30,230 +34,27 @@ function searchLimiter(req, res, next) { next(); } -// Parses a raw search string into structured operator filters and free-text -// terms. Supports from: to: subject: has: is: after: before: in:, quoted values -// (from:"John Smith"), and a leading '-' that negates either an operator -// (-from:smith) or a bare word (-invoice). Filters are a list (not a map) so -// repeated/negated operators like `from:a -from:b` are all preserved. -export function parseSearchQuery(raw) { - const filters = []; - const terms = []; - - // The leading (-?) captures optional negation. \b sits between an optional '-' - // and the operator name, so both `from:` and `-from:` match. - const opPattern = /(-?)\b(from|to|subject|has|is|after|before|in):("([^"]*)"|([\S]+))/gi; - const remaining = raw.replace(opPattern, (_, neg, key, _v, quoted, unquoted) => { - const k = key.toLowerCase(); - const v = (quoted !== undefined ? quoted : (unquoted || '')).toLowerCase().trim(); - if (v) filters.push({ key: k, value: v, negate: neg === '-' }); - return ' '; - }).trim(); - - for (const word of remaining.split(/\s+/)) { - let w = word.trim(); - if (!w || w === '-') continue; // skip blanks and a lone '-' (nothing to negate) - let negate = false; - if (w[0] === '-' && w.length > 1) { negate = true; w = w.slice(1); } - terms.push({ value: w, negate }); - } - - return { filters, terms }; -} - -// Wraps a positive condition so that when negated it also matches rows where the -// underlying columns are NULL (COALESCE(..., false) treats NULL as "not a match", -// which NOT then flips to a match — the intuitive meaning of exclusion). -function negateCond(sql) { - return `NOT COALESCE((${sql}), false)`; -} - -export function resolveSearchFolderScope(filters, folderParam = '') { - let folderScope; - let folderFuzzy = false; // in: matches loosely; the folder param is exact - - for (const f of filters) { - if (f.key !== 'in') continue; - if (f.value === 'all') { folderScope = null; } - else { folderScope = f.value; folderFuzzy = true; } - } - - if (folderScope === undefined) { - folderScope = (folderParam || '').trim() || null; - folderFuzzy = false; - } - - return { folderScope, folderFuzzy }; -} - -export function shouldExcludeTrashFromSearch(folderScope) { - return folderScope === null; -} - -export function trashFolderExclusionCondition() { - return `NOT EXISTS ( - SELECT 1 - FROM folders f - WHERE f.account_id = m.account_id - AND f.path = m.folder - AND (f.special_use = '\\Trash' - OR lower(f.name) LIKE '%trash%' - OR lower(f.name) LIKE '%deleted%') - )`; -} - -// Postgres refuses to build a tsvector larger than ~1MB of packed lexemes -// (SQLSTATE 54000), so an oversized body would 500 the whole search. Cap the -// text fed to to_tsvector at 600k chars — matching msgvault's maxFTSBodyChars -// (internal/store/dialect_pg.go) — so one huge email can't crash the query. -// Exported because slice 02's search_fts trigger caps the same way. -export const FTS_BODY_CHAR_CAP = 600000; - -// Builds the per-term free-text OR-condition: a term matches if it appears in -// the sender, the subject, the stored search_vector, or the length-capped body. -// Extracted so the body cap is a single, testable source of truth. -export function freeTextTermCondition(likeIdx, ftsIdx) { - return `( - m.from_name ILIKE $${likeIdx} - OR m.from_email ILIKE $${likeIdx} - OR m.subject ILIKE $${likeIdx} - OR m.search_vector @@ plainto_tsquery('english', $${ftsIdx}) - OR to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})) @@ plainto_tsquery('english', $${ftsIdx}) - )`; -} - router.get('/', searchLimiter, async (req, res) => { const { q, accountId, limit = 50, offset = 0 } = req.query; const trimmed = (q || '').trim(); if (!trimmed) return res.json({ messages: [] }); if (trimmed.length > 500) return res.status(400).json({ error: 'Search query too long' }); - const accountsResult = await query( - 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', - [req.session.userId] - ); - const userAccountIds = accountsResult.rows.map(r => r.id); - if (!userAccountIds.length) return res.json({ messages: [] }); - - const targetIds = accountId && userAccountIds.includes(accountId) - ? [accountId] : userAccountIds; - - const cap = Math.max(1, Math.min(parseInt(limit) || 50, 200)); - const { filters, terms } = parseSearchQuery(trimmed); - - const conditions = []; - const params = [targetIds]; - let p = 2; - - // Folder scope. `in:` in the query wins; otherwise the client-supplied `folder` - // param (the folder the user is currently viewing) applies. `undefined` means - // no in: operator was given, so we fall back to the param below. - // folderScope === null → search all folders - // folderScope === string → restrict to that folder - - // ── Operator filters ────────────────────────────────────────────────────── - - for (const f of filters) { - // in: controls scope rather than adding a row condition; negation is - // meaningless here so it's ignored. - if (f.key === 'in') { - continue; - } - - let cond = null; - - if (f.key === 'from') { - params.push(`%${f.value}%`); - cond = `(m.from_email ILIKE $${p} OR m.from_name ILIKE $${p})`; - p++; - } else if (f.key === 'subject') { - params.push(`%${f.value}%`); - cond = `m.subject ILIKE $${p++}`; - } else if (f.key === 'to') { - // to: searches the to/cc address JSON — cast to text covers name and email - params.push(`%${f.value}%`); - cond = `(m.to_addresses::text ILIKE $${p} OR m.cc_addresses::text ILIKE $${p})`; - p++; - } else if (f.key === 'has') { - if (f.value === 'attachment' || f.value === 'attachments') cond = `m.has_attachments = true`; - } else if (f.key === 'is') { - if (f.value === 'unread') cond = `m.is_read = false`; - else if (f.value === 'read') cond = `m.is_read = true`; - else if (f.value === 'starred') cond = `m.is_starred = true`; - } else if (f.key === 'after') { - const d = new Date(f.value); - if (!isNaN(d)) { params.push(d.toISOString()); cond = `m.date >= $${p++}`; } - } else if (f.key === 'before') { - const d = new Date(f.value); - if (!isNaN(d)) { params.push(d.toISOString()); cond = `m.date < $${p++}`; } - } - - if (cond) conditions.push(f.negate ? negateCond(cond) : cond); - } - - // ── Free-text terms ─────────────────────────────────────────────────────── - // Each term must match at least one of: from, subject (ILIKE — good for names - // and partial words), or body content (FTS — good for large text with stemming). - // AND between all terms: every word must appear somewhere in the email. - // A negated term (-word) must appear nowhere. - - for (const term of terms.slice(0, 10)) { - if (term.value.length < 2) continue; // single-char terms are too broad and expensive - params.push(`%${term.value}%`); // ILIKE pattern - const likeIdx = p++; - - params.push(term.value); // raw term for plainto_tsquery - const ftsIdx = p++; - - const cond = freeTextTermCondition(likeIdx, ftsIdx); - conditions.push(term.negate ? negateCond(cond) : cond); - } - - // Require at least one real search condition before applying folder scope, so a - // bare `in:inbox` (or a lone folder param) never dumps an entire folder. - if (!conditions.length) return res.json({ messages: [], query: q }); - - // Resolve folder scope: in: operator wins; otherwise use the param. - const { folderScope, folderFuzzy } = resolveSearchFolderScope(filters, req.query.folder || ''); - if (folderScope) { - if (folderFuzzy) { - // in: — case-insensitive match on a folder named exactly that, or a - // nested folder whose path ends in it (in:sent → "Sent" or "Personal/Sent"). - // A multi-word leaf like "[Gmail]/Sent Mail" needs the quoted form in:"sent mail". - params.push(folderScope); - params.push(`%/${folderScope}`); - conditions.push(`(m.folder ILIKE $${p} OR m.folder ILIKE $${p + 1})`); - p += 2; - } else { - params.push(folderScope); - conditions.push(`m.folder = $${p++}`); - } - } else if (shouldExcludeTrashFromSearch(folderScope)) { - // Deleting moves mail into Trash, where it remains searchable by explicit - // folder queries like in:trash. Keep ordinary all-folder searches from - // resurfacing freshly-deleted messages after the optimistic UI guard expires. - conditions.push(trashFolderExclusionCondition()); - } - - const off = Math.max(0, parseInt(offset) || 0); - params.push(cap); - params.push(off); + const parsed = parseQuery(trimmed); + const mode = ALLOWED_MODES.has(req.query.mode) ? req.query.mode : 'lexical'; try { - const result = await query(` - SELECT - m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, - m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, - a.name as account_name, a.email_address as account_email, a.color as account_color - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.account_id = ANY($1) - AND m.is_deleted = false - AND ${conditions.join('\n AND ')} - ORDER BY m.date DESC - LIMIT $${p} OFFSET $${p + 1} - `, params); - - res.json({ messages: result.rows, query: q }); + const result = await search({ + userId: req.session.userId, + accountId, + parsed, + folderParam: req.query.folder || '', + limit, + offset, + mode, + }); + // Response is a strict superset of the historical { messages, query }. + res.json({ ...result, query: q, unsupported: parsed.unsupported, errors: parsed.errors }); } catch (err) { console.error('Search error:', err); res.status(500).json({ error: 'Search failed' }); diff --git a/backend/src/routes/search.test.js b/backend/src/routes/search.test.js index 3ef54c7..26bcbe6 100644 --- a/backend/src/routes/search.test.js +++ b/backend/src/routes/search.test.js @@ -1,157 +1,96 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; -// search.js opens a DB handle and registers auth middleware at import time; -// neither is exercised by the pure parser under test, so stub them out. +// The route must not hold SQL: it parses, calls the service, and JSON-frames. +// Mock inline + import the mocked binding (avoids the vi.mock hoisting TDZ trap). +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ requireAuth: (req, _res, next) => { req.session = { userId: 'u1' }; next(); } })); vi.mock('../services/db.js', () => ({ query: vi.fn() })); -vi.mock('../middleware/auth.js', () => ({ requireAuth: vi.fn() })); -import { - parseSearchQuery, - resolveSearchFolderScope, - shouldExcludeTrashFromSearch, - trashFolderExclusionCondition, - freeTextTermCondition, - FTS_BODY_CHAR_CAP, -} from './search.js'; - -describe('parseSearchQuery', () => { - it('treats bare words as free-text terms', () => { - const { filters, terms } = parseSearchQuery('hello world'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'hello', negate: false }, - { value: 'world', negate: false }, - ]); - }); - - it('extracts positive operators and lowercases their values', () => { - const { filters, terms } = parseSearchQuery('from:Amazon subject:Invoice hello'); - expect(filters).toEqual([ - { key: 'from', value: 'amazon', negate: false }, - { key: 'subject', value: 'invoice', negate: false }, - ]); - expect(terms).toEqual([{ value: 'hello', negate: false }]); - }); - - it('supports quoted operator values with spaces', () => { - const { filters, terms } = parseSearchQuery('from:"John Smith" report'); - expect(filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); - expect(terms).toEqual([{ value: 'report', negate: false }]); - }); - - it('negates an operator when prefixed with -', () => { - const { filters, terms } = parseSearchQuery('-from:Smith report'); - expect(filters).toEqual([{ key: 'from', value: 'smith', negate: true }]); - expect(terms).toEqual([{ value: 'report', negate: false }]); - }); - - it('negates a free-text term when prefixed with -', () => { - const { filters, terms } = parseSearchQuery('report -invoice'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'report', negate: false }, - { value: 'invoice', negate: true }, - ]); - }); - - it('parses the in: scope operator (in:all and named folders)', () => { - expect(parseSearchQuery('in:all invoice').filters).toEqual([ - { key: 'in', value: 'all', negate: false }, - ]); - expect(parseSearchQuery('in:Sent proposal').filters).toEqual([ - { key: 'in', value: 'sent', negate: false }, - ]); - }); - - it('preserves repeated and mixed positive/negative operators', () => { - const { filters } = parseSearchQuery('from:alice -from:bob is:unread'); - expect(filters).toEqual([ - { key: 'from', value: 'alice', negate: false }, - { key: 'from', value: 'bob', negate: true }, - { key: 'is', value: 'unread', negate: false }, - ]); +import { search } from '../services/search/searchService.js'; +import searchRouter from './search.js'; + +function makeApp() { + const app = express(); + app.use((req, _res, next) => { req.session = { userId: 'u1' }; next(); }); + app.use('/api/search', searchRouter); + return app; +} + +async function get(app, path) { + const { default: request } = await import('node:http'); + return new Promise((resolve) => { + const server = app.listen(0, () => { + const port = server.address().port; + request.get(`http://127.0.0.1:${port}${path}`, (res) => { + let body = ''; + res.on('data', c => (body += c)); + res.on('end', () => { server.close(); resolve({ status: res.statusCode, json: JSON.parse(body || '{}') }); }); + }); + }); }); +} - it('ignores a lone - so it is not treated as a negated empty term', () => { - const { filters, terms } = parseSearchQuery('report - draft'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'report', negate: false }, - { value: 'draft', negate: false }, - ]); - }); +beforeEach(() => search.mockReset()); - it('returns empty structures for blank input', () => { - expect(parseSearchQuery('')).toEqual({ filters: [], terms: [] }); - expect(parseSearchQuery(' ')).toEqual({ filters: [], terms: [] }); +describe('GET /api/search', () => { + it('short-circuits a blank query with { messages: [] } and never calls the service', async () => { + const res = await get(makeApp(), '/api/search?q='); + expect(res.status).toBe(200); + expect(res.json).toEqual({ messages: [] }); + expect(search).not.toHaveBeenCalled(); }); - it('does not treat unknown prefixes as operators', () => { - const { filters, terms } = parseSearchQuery('label:work'); - expect(filters).toEqual([]); - expect(terms).toEqual([{ value: 'label:work', negate: false }]); + it('rejects an over-500-char query with 400', async () => { + const res = await get(makeApp(), `/api/search?q=${'a'.repeat(501)}`); + expect(res.status).toBe(400); }); - it('handles all supported operator keys', () => { - const raw = 'from:a to:b subject:c has:attachment is:starred after:2024-01-01 before:2024-12-31 in:archive'; - const keys = parseSearchQuery(raw).filters.map(f => f.key); - expect(keys).toEqual(['from', 'to', 'subject', 'has', 'is', 'after', 'before', 'in']); - }); - - it('scopes search to the client folder param when no in: operator is present', () => { - const { filters } = parseSearchQuery('subject:newsletter'); - expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ - folderScope: 'INBOX', - folderFuzzy: false, - }); + it('returns a superset response: messages + query + mode + page + unsupported + errors', async () => { + search.mockResolvedValue({ messages: [{ id: 'm1' }], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + const res = await get(makeApp(), '/api/search?q=larger:5M%20invoice'); + expect(res.status).toBe(200); + expect(res.json.messages).toEqual([{ id: 'm1' }]); + expect(res.json.query).toBe('larger:5M invoice'); + expect(res.json.mode).toBe('lexical'); + expect(res.json.page).toEqual({ offset: 0, limit: 50, hasMore: false }); + // larger: is recognized but unserviceable — surfaced, not silently dropped. + expect(res.json.unsupported).toEqual([{ key: 'larger', token: 'larger:5m' }]); + expect(res.json.errors).toEqual([]); }); +}); - it('lets in: override the client folder param', () => { - const { filters } = parseSearchQuery('in:trash subject:newsletter'); - expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ - folderScope: 'trash', - folderFuzzy: true, - }); - }); +describe('GET /api/search mode passthrough (Phase 4 Task 7)', () => { + beforeEach(() => search.mockReset()); - it('excludes trash-like folders from ordinary all-folder searches', () => { - const { filters } = parseSearchQuery('subject:newsletter'); - const { folderScope } = resolveSearchFolderScope(filters); - expect(folderScope).toBeNull(); - expect(shouldExcludeTrashFromSearch(folderScope)).toBe(true); - expect(trashFolderExclusionCondition()).toContain('NOT EXISTS'); - expect(trashFolderExclusionCondition()).toContain('%trash%'); - expect(trashFolderExclusionCondition()).toContain('%deleted%'); + it('defaults mode to lexical when absent', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello'); + expect(search.mock.calls[0][0].mode).toBe('lexical'); }); - it('keeps explicit folder searches eligible to find trash messages', () => { - const { filters } = parseSearchQuery('in:trash subject:newsletter'); - const { folderScope } = resolveSearchFolderScope(filters); - expect(shouldExcludeTrashFromSearch(folderScope)).toBe(false); + it('passes mode=hybrid straight through', async () => { + search.mockResolvedValue({ messages: [], mode: 'hybrid', pool_saturated: false, generation: null, page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=hybrid'); + expect(search.mock.calls[0][0].mode).toBe('hybrid'); }); -}); -describe('freeTextTermCondition (oversized-body crash hotfix)', () => { - it('caps the body tsvector so a multi-megabyte email cannot exceed the 1MB tsvector limit', () => { - const cond = freeTextTermCondition(3, 4); - // The body text is fed to to_tsvector through LEFT(..., FTS_BODY_CHAR_CAP). - // Without this cap a single oversized body raises SQLSTATE 54000 and 500s search. - expect(cond).toContain( - `to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))` - ); - // Guard against a regression back to the uncapped expression. - expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); + it('passes mode=vector straight through', async () => { + search.mockResolvedValue({ messages: [], mode: 'vector', pool_saturated: false, generation: null, page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=vector'); + expect(search.mock.calls[0][0].mode).toBe('vector'); }); - it('pins the cap at 600000 chars (msgvault maxFTSBodyChars parity)', () => { - expect(FTS_BODY_CHAR_CAP).toBe(600000); + it('coerces an unknown mode to lexical', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=bogus'); + expect(search.mock.calls[0][0].mode).toBe('lexical'); }); - it('still matches sender, subject, and the stored search_vector at the given param positions', () => { - const cond = freeTextTermCondition(3, 4); - expect(cond).toContain('m.from_name ILIKE $3'); - expect(cond).toContain('m.from_email ILIKE $3'); - expect(cond).toContain('m.subject ILIKE $3'); - expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + it('returns the service response including fellBack (superset)', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', fellBack: true, page: { offset: 0, limit: 50, hasMore: false } }); + const res = await get(makeApp(), '/api/search?q=hello&mode=hybrid'); + expect(res.json.mode).toBe('lexical'); + expect(res.json.fellBack).toBe(true); }); }); diff --git a/backend/src/services/backgroundJobs.js b/backend/src/services/backgroundJobs.js new file mode 100644 index 0000000..799b05e --- /dev/null +++ b/backend/src/services/backgroundJobs.js @@ -0,0 +1,26 @@ +import { query } from './db.js'; + +// Upsert a drainer's progress row. One row per (kind, account); global jobs +// pass accountId = null. started_at is set once and preserved across updates. +export async function upsertJob({ kind, accountId = null, state, processed = 0, total = 0, lastError = null }) { + await query(` + INSERT INTO background_jobs (kind, account_id, state, processed, total, last_error, started_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW()) + ON CONFLICT (kind, COALESCE(account_id::text, '')) DO UPDATE SET + state = EXCLUDED.state, + processed = EXCLUDED.processed, + total = EXCLUDED.total, + last_error = EXCLUDED.last_error, + started_at = COALESCE(background_jobs.started_at, EXCLUDED.started_at), + updated_at = NOW() + `, [kind, accountId, state, processed, total, lastError]); +} + +export async function listJobs() { + const { rows } = await query( + `SELECT kind, account_id, state, processed, total, last_error, started_at, updated_at + FROM background_jobs + ORDER BY updated_at DESC` + ); + return rows; +} diff --git a/backend/src/services/backgroundJobs.test.js b/backend/src/services/backgroundJobs.test.js new file mode 100644 index 0000000..f3ecfd2 --- /dev/null +++ b/backend/src/services/backgroundJobs.test.js @@ -0,0 +1,25 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('./db.js', () => ({ query: vi.fn() })); +import { query } from './db.js'; +import { upsertJob, listJobs } from './backgroundJobs.js'; + +beforeEach(() => query.mockReset()); + +describe('backgroundJobs', () => { + it('upserts one row per (kind, account) via the COALESCE unique index', async () => { + query.mockResolvedValue({ rows: [] }); + await upsertJob({ kind: 'fts_backfill', state: 'running', processed: 10, total: 100 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('INSERT INTO background_jobs'); + expect(sql).toContain("ON CONFLICT (kind, COALESCE(account_id::text, ''))"); + expect(sql).toContain('started_at = COALESCE(background_jobs.started_at, EXCLUDED.started_at)'); + expect(params).toEqual(['fts_backfill', null, 'running', 10, 100, null]); + }); + + it('lists jobs newest-first', async () => { + query.mockResolvedValue({ rows: [{ kind: 'fts_backfill' }] }); + const jobs = await listJobs(); + expect(jobs).toEqual([{ kind: 'fts_backfill' }]); + expect(query.mock.calls[0][0]).toContain('ORDER BY updated_at DESC'); + }); +}); diff --git a/backend/src/services/bodyBackfill.js b/backend/src/services/bodyBackfill.js new file mode 100644 index 0000000..11e6cbe --- /dev/null +++ b/backend/src/services/bodyBackfill.js @@ -0,0 +1,182 @@ +import { query } from './db.js'; +import { providerProfile } from './imapManager.js'; + +// Body-materialization drainer. Fills messages.body_text/body_html over IMAP so weight-D +// lexical search and embeddings have material to work with, WITHOUT growing the imapManager +// singleton (README invariant) and WITHOUT touching throttle-hostile providers. ALL policy — +// the scan predicate, batching, pacing, provider gating, session cap, quiet-window +// backpressure, and the per-account circuit breaker — lives here. The singleton exposes only +// the narrow fetchBodiesForMessages() entry point this module calls. +// +// Shape is copied from imapManager.startSnippetIndexer, but this module is standalone: its +// per-account run guard and circuit-breaker state are module-level (not instance fields), and +// its IMAP/quiet-window/progress/clock dependencies are injected so it unit-tests against a +// fake fetchBodies with no real IMAP. + +const BATCH_SIZE = 50; // messages fetched per batch +const MIN_BATCH_DELAY_MS = 2000; // floor between batches (provider batchDelay may raise it) +const MAX_BATCHES_PER_RUN = 200; // session cap: 10,000 messages per run, then resume next kick +const QUIET_WINDOW_MS = 8000; // pause extra when the user opened a message this recently +const MAX_CONSECUTIVE_ERRORS = 3; // abort the run after this many failing batches in a row +const BACKOFF_BASE_MS = 10 * 60 * 1000; // first circuit-breaker back-off +const BACKOFF_MAX_MS = 2 * 60 * 60 * 1000; // cap + +// Per-account run guard and circuit breaker. Module-level so a single process runs at most one +// body drainer per account (the snippet indexer uses instance fields for the same purpose). +const running = new Set(); // accountId +const backoff = new Map(); // accountId -> { failures, until } + +// Test seam: reset module state between unit tests. +export function resetBodyBackfillState() { + running.clear(); + backoff.clear(); +} + +const realSleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Core drainer. Dependencies are injected so tests supply a fake fetchBodies: +// fetchBodies(accountId, ids) -> Promise<{ fetched: number }> (writes bodies; throws on a +// connection-level failure so this loop can back off) +// getLastActivityMs(accountId) -> number (ms timestamp of the user's last live body open) +// isSnippetIndexerRunning(accountId) -> boolean (the snippet indexer does its own sustained +// BODY[]-ish fetches over an independent connection; running +// both loops for the same account at once double-books IMAP +// connections against the same provider limit) +// upsertJobProgress({ accountId, kind, processed, total, state }) -> Promise (this is +// backgroundJobs.upsertJob's own shape — callers inject it +// directly; the drainer speaks its vocabulary, no adapter) +// sleep(ms) -> Promise (injectable for fast, deterministic tests) +// now() -> number (injectable clock) +export async function startBodyBackfill(account, deps) { + const { + fetchBodies, + getLastActivityMs = () => 0, + isSnippetIndexerRunning = () => false, + upsertJobProgress = async () => {}, + sleep = realSleep, + now = Date.now, + } = deps; + + const cfg = providerProfile(account); + if (!cfg.bodyBackfill) return; // provider gate (Gmail/PurelyMail/etc.) + if (running.has(account.id)) return; // one drainer per account + const bo = backoff.get(account.id); + if (bo && now() < bo.until) return; // circuit breaker open + running.add(account.id); + + const batchDelay = Math.max(cfg.batchDelay || 0, MIN_BATCH_DELAY_MS); + const kind = 'body_backfill'; + let batchCount = 0; + let failed = false; + + try { + // Denominator + starting coverage for progress. total is the whole eligible mailbox so the + // progress bar reflects real coverage, not just this run. + const countRes = await query( + `SELECT count(*)::int AS total, + count(*) FILTER (WHERE body_text IS NOT NULL OR body_html IS NOT NULL)::int AS have_body + FROM messages WHERE account_id = $1 AND is_deleted = false`, + [account.id] + ); + const total = countRes.rows[0].total; + let haveBody = countRes.rows[0].have_body; + if (total - haveBody <= 0) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'done' }); + return; + } + + // Keyset by id ascending: id is a non-null, unique UUID, so the cursor advances past every + // row exactly once per run — including rows whose body cannot be fetched (they stay NULL but + // are not re-selected this run, so the run always terminates). Recency ordering is + // intentionally not used: body coverage is a background quality lever, and the embedding + // backstop picks up late arrivals regardless (README D4). + let cursorId = null; + let consecutiveErrors = 0; + + while (true) { + // Stop if the account was deleted mid-run. + const alive = await query('SELECT id FROM email_accounts WHERE id = $1', [account.id]); + if (!alive.rows.length) return; + + // Defer to the snippet indexer rather than double-book BODY[] IMAP connections for the + // same account — checked every iteration (not just at kick time) so a session already in + // progress backs off cleanly the moment the indexer starts, instead of racing it. + if (isSnippetIndexerRunning(account.id)) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'deferred' }); + return; + } + + if (batchCount >= MAX_BATCHES_PER_RUN) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'paused' }); + return; + } + + const batchRes = await query( + `SELECT id FROM messages + WHERE account_id = $1 AND body_text IS NULL AND body_html IS NULL AND is_deleted = false + AND ($2::uuid IS NULL OR id > $2) + ORDER BY id ASC + LIMIT $3`, + [account.id, cursorId, BATCH_SIZE] + ); + if (!batchRes.rows.length) break; // drained + const ids = batchRes.rows.map((r) => r.id); + + try { + const { fetched } = await fetchBodies(account.id, ids); + haveBody += fetched; + cursorId = ids[ids.length - 1]; // advance only on success + batchCount++; + consecutiveErrors = 0; + } catch { + consecutiveErrors++; + await sleep(cfg.errorDelay || batchDelay); + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + failed = true; + // Write a terminal 'error' state (ftsBackfill.js convention) so background_jobs + // doesn't keep showing a stale 'running' row for an account that's actually stalled. + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'error' }); + return; // finally trips the circuit breaker + } + continue; // retry the same batch (cursor not advanced); fetchBodies reopens IMAP + } + + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'running' }); + + // Quiet-window backpressure: pause longer when the user is actively opening messages so + // background BODY[] traffic doesn't compete with click-time fetches. + const quietFor = now() - getLastActivityMs(account.id); + const extraDelay = quietFor < QUIET_WINDOW_MS ? QUIET_WINDOW_MS - quietFor : 0; + await sleep(batchDelay + extraDelay); + } + + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'done' }); + } catch { + failed = true; + } finally { + running.delete(account.id); + // Circuit breaker: a run that failed without draining a single batch (e.g. the provider + // refuses the extra connection) backs off exponentially so we stop reopening competing IMAP + // connections. Any progress — or a clean finish — clears the backoff. + if (failed && batchCount === 0) { + const failures = (backoff.get(account.id)?.failures || 0) + 1; + const delay = Math.min(BACKOFF_BASE_MS * 2 ** (failures - 1), BACKOFF_MAX_MS); + backoff.set(account.id, { failures, until: now() + delay }); + } else { + backoff.delete(account.id); + } + } +} + +// Convenience wrapper for the reindex route: builds the injected deps from the imapManager +// singleton and the injected progress sink (backgroundJobs.upsertJob — the drainer already +// emits its shape), then runs the core drainer. Kept here (not in the route) so the wiring is +// unit-testable and bodyBackfill.js never imports backgroundJobs.js directly. +export function startAccountBodyBackfill(account, imapManager, upsertJobProgress) { + return startBodyBackfill(account, { + fetchBodies: (accountId, ids) => imapManager.fetchBodiesForMessages(accountId, ids), + getLastActivityMs: (accountId) => imapManager.lastUserActivity.get(accountId) || 0, + isSnippetIndexerRunning: (accountId) => imapManager.snippetIndexerRunning.has(accountId), + upsertJobProgress, + }); +} diff --git a/backend/src/services/bodyBackfill.test.js b/backend/src/services/bodyBackfill.test.js new file mode 100644 index 0000000..3b3da49 --- /dev/null +++ b/backend/src/services/bodyBackfill.test.js @@ -0,0 +1,290 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('./db.js', () => ({ query: vi.fn() })); +vi.mock('./imapManager.js', () => ({ providerProfile: vi.fn() })); + +import { query } from './db.js'; +import { providerProfile } from './imapManager.js'; +import { startBodyBackfill, startAccountBodyBackfill, resetBodyBackfillState } from './bodyBackfill.js'; + +const account = { id: 'acct-1', imap_host: 'imap.icloud.com' }; +const enabledProfile = { bodyBackfill: true, batchDelay: 2000, errorDelay: 100 }; + +// Instant, assertable deps so the drainer runs with no wall-clock waits. +function baseDeps(overrides = {}) { + return { + fetchBodies: vi.fn().mockResolvedValue({ fetched: 0 }), + getLastActivityMs: () => 0, + isSnippetIndexerRunning: () => false, + upsertJobProgress: vi.fn().mockResolvedValue(), + sleep: vi.fn().mockResolvedValue(), + now: () => 1_000_000, + ...overrides, + }; +} + +// Route the mocked query by SQL shape. `batches` is an array of id-arrays returned in order; +// once exhausted it returns an empty batch (drained). +function mockQuery({ total = 0, haveBody = 0, batches = [], alive = true } = {}) { + let i = 0; + query.mockImplementation((sql) => { + if (/AS total/.test(sql)) return Promise.resolve({ rows: [{ total, have_body: haveBody }] }); + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: alive ? [{ id: 'acct-1' }] : [] }); + if (/body_text IS NULL/.test(sql)) { + const rows = (batches[i++] || []).map((id) => ({ id })); + return Promise.resolve({ rows }); + } + return Promise.resolve({ rows: [] }); + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + resetBodyBackfillState(); +}); + +describe('startBodyBackfill — provider gate', () => { + it('no-ops for a provider with bodyBackfill:false and never queries', async () => { + providerProfile.mockReturnValue({ bodyBackfill: false }); + const deps = baseDeps(); + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('startBodyBackfill — nothing to do', () => { + it('reports complete and fetches nothing when coverage is already full', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 100 }); + const deps = baseDeps(); + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(deps.upsertJobProgress).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'done', processed: 100, total: 100 }) + ); + }); +}); + +describe('startBodyBackfill — batching + keyset + progress', () => { + it('drains 120 messages in batches of 50/50/20 and advances the cursor', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = Array.from({ length: 50 }, (_, k) => `a${k}`); + const b2 = Array.from({ length: 50 }, (_, k) => `b${k}`); + const b3 = Array.from({ length: 20 }, (_, k) => `c${k}`); + mockQuery({ total: 120, haveBody: 0, batches: [b1, b2, b3] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockImplementation((_id, ids) => Promise.resolve({ fetched: ids.length })) }); + + await startBodyBackfill(account, deps); + + expect(deps.fetchBodies).toHaveBeenCalledTimes(3); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(1, 'acct-1', b1); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(2, 'acct-1', b2); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(3, 'acct-1', b3); + // Cursor advances to the last id of the previous batch (keyset param $2). + const scanCalls = query.mock.calls.filter(([sql]) => /body_text IS NULL/.test(sql)); + expect(scanCalls[0][1][1]).toBeNull(); // first scan: cursor null + expect(scanCalls[1][1][1]).toBe('a49'); // second scan: last id of batch 1 + expect(scanCalls[2][1][1]).toBe('b49'); // third scan: last id of batch 2 + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 120, total: 120 }) + ); + }); +}); + +describe('startBodyBackfill — session cap + resume', () => { + it('pauses after MAX_BATCHES_PER_RUN batches, then resumes on the next call', async () => { + providerProfile.mockReturnValue(enabledProfile); + // Never drains: every scan returns 50 fresh ids so the run hits the batch cap. + let n = 0; + query.mockImplementation((sql) => { + if (/AS total/.test(sql)) return Promise.resolve({ rows: [{ total: 100000, have_body: 0 }] }); + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/body_text IS NULL/.test(sql)) return Promise.resolve({ rows: Array.from({ length: 50 }, () => ({ id: `id-${n++}` })) }); + return Promise.resolve({ rows: [] }); + }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 50 }) }); + + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).toHaveBeenCalledTimes(200); // MAX_BATCHES_PER_RUN + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'paused' }) + ); + + // Resume: the run guard was released in finally, so a second call proceeds (not gated). + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).toHaveBeenCalledTimes(400); + }); +}); + +describe('startBodyBackfill — single drainer per account', () => { + it('a second concurrent call for the same account is a no-op', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 50, haveBody: 0, batches: [['x1']] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }) }); + + // Both are invoked synchronously in the same tick: the first adds the account to the + // run guard before its first await, so the second sees it and returns immediately. + const p1 = startBodyBackfill(account, deps); + const p2 = startBodyBackfill(account, deps); + await Promise.all([p1, p2]); + + expect(deps.fetchBodies).toHaveBeenCalledTimes(1); // only p1 did work + }); +}); + +describe('startBodyBackfill — circuit breaker trip + recovery', () => { + it('trips after 3 consecutive batch errors, blocks retries in the window, recovers after it', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 0, batches: [['x1'], ['x1'], ['x1']] }); + let clock = 1_000_000; + const failing = baseDeps({ + fetchBodies: vi.fn().mockRejectedValue(new Error('Command failed')), + now: () => clock, + }); + + // Trip: 3 consecutive rejections abort the run and open the breaker. + await startBodyBackfill(account, failing); + expect(failing.fetchBodies).toHaveBeenCalledTimes(3); + // A terminal 'error' progress row is written so background_jobs doesn't keep showing a + // stale 'running' state for an account that's actually stalled. + expect(failing.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'error' }) + ); + + // Blocked: a call inside the back-off window no-ops (breaker open). + const blocked = baseDeps({ fetchBodies: vi.fn(), now: () => clock }); + await startBodyBackfill(account, blocked); + expect(blocked.fetchBodies).not.toHaveBeenCalled(); + + // Recover: advance past the back-off window; a fresh call runs again. + clock += 3 * 60 * 60 * 1000; // beyond BACKOFF_MAX_MS (2h) + mockQuery({ total: 100, haveBody: 0, batches: [['y1']] }); + const recovered = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }), now: () => clock }); + await startBodyBackfill(account, recovered); + expect(recovered.fetchBodies).toHaveBeenCalledTimes(1); + }); +}); + +describe('startBodyBackfill — quiet-window backpressure', () => { + it('adds the remaining quiet window to the post-batch delay when the user is active', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const NOW = 5_000_000; + const deps = baseDeps({ + fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }), + now: () => NOW, + getLastActivityMs: () => NOW - 2000, // user opened a message 2s ago + }); + + await startBodyBackfill(account, deps); + + // batchDelay = max(2000, 2000) = 2000; quiet window 8000 - 2000 elapsed = 6000 extra. + expect(deps.sleep).toHaveBeenCalledWith(8000); + }); +}); + +describe('startBodyBackfill — defers to the snippet indexer', () => { + it('gates the kick: never fetches when the indexer is already running', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 0, batches: [['x1']] }); + const deps = baseDeps({ isSnippetIndexerRunning: () => true }); + + await startBodyBackfill(account, deps); + + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(deps.upsertJobProgress).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'deferred' }) + ); + }); + + it('re-checks every iteration: defers mid-session once the indexer starts, without tripping the breaker', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = ['a1']; + const b2 = ['b1']; + mockQuery({ total: 100, haveBody: 0, batches: [b1, b2] }); + let indexerRunning = false; + const deps = baseDeps({ + fetchBodies: vi.fn().mockImplementation(() => { + indexerRunning = true; // simulate the snippet indexer kicking in right after batch 1 + return Promise.resolve({ fetched: 1 }); + }), + isSnippetIndexerRunning: () => indexerRunning, + }); + + await startBodyBackfill(account, deps); + + // Batch 1 runs (indexer not yet running); batch 2's scan never happens because the + // re-check at the top of the next iteration sees the indexer is now running and bails. + expect(deps.fetchBodies).toHaveBeenCalledTimes(1); + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'deferred' }) + ); + + // Deferring is not a failure: the circuit breaker must not be armed by it. + const resumed = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }) }); + mockQuery({ total: 100, haveBody: 1, batches: [['c1']] }); + await startBodyBackfill(account, resumed); + expect(resumed.fetchBodies).toHaveBeenCalledTimes(1); + }); +}); + +describe('startBodyBackfill — forward progress', () => { + it('drains to completion even when every batch reports {fetched:0} (resolved, not rejected)', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = Array.from({ length: 50 }, (_, k) => `a${k}`); + const b2 = Array.from({ length: 10 }, (_, k) => `b${k}`); + mockQuery({ total: 60, haveBody: 0, batches: [b1, b2] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 0 }) }); + + await startBodyBackfill(account, deps); + + // The cursor still advances on a resolved (non-throwing) call even when it wrote nothing — + // e.g. every message in the batch legitimately has no fetchable body — so the run still + // reaches completion instead of looping forever or being mistaken for a failure. + expect(deps.fetchBodies).toHaveBeenCalledTimes(2); + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 0, total: 60 }) + ); + }); +}); + +describe('startAccountBodyBackfill — adapter wiring', () => { + it('wires imapManager.fetchBodiesForMessages and lastUserActivity into the core drainer', async () => { + vi.useFakeTimers(); + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const fakeImap = { + fetchBodiesForMessages: vi.fn().mockResolvedValue({ fetched: 1 }), + lastUserActivity: new Map([['acct-1', Date.now()]]), + snippetIndexerRunning: new Set(), + }; + const upsert = vi.fn().mockResolvedValue(); + + const p = startAccountBodyBackfill(account, fakeImap, upsert); + await vi.runAllTimersAsync(); // flush the real setTimeout used by the default sleep + await p; + + expect(fakeImap.fetchBodiesForMessages).toHaveBeenCalledWith('acct-1', ['m1']); + expect(upsert).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', accountId: 'acct-1' }) + ); + vi.useRealTimers(); + }); + + it('wires imapManager.snippetIndexerRunning so an already-running indexer defers the kick', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const fakeImap = { + fetchBodiesForMessages: vi.fn().mockResolvedValue({ fetched: 1 }), + lastUserActivity: new Map(), + snippetIndexerRunning: new Set(['acct-1']), + }; + const upsert = vi.fn().mockResolvedValue(); + + await startAccountBodyBackfill(account, fakeImap, upsert); + + expect(fakeImap.fetchBodiesForMessages).not.toHaveBeenCalled(); + expect(upsert).toHaveBeenCalledWith(expect.objectContaining({ state: 'deferred' })); + }); +}); diff --git a/backend/src/services/embeddings/__testdb__.js b/backend/src/services/embeddings/__testdb__.js new file mode 100644 index 0000000..15ae328 --- /dev/null +++ b/backend/src/services/embeddings/__testdb__.js @@ -0,0 +1,60 @@ +// pgvector test-DB harness for fused search + ranking-quality gates. +// +// Reuses Phase 3's established IT convention (env var VECTOR_IT_DB, the SAME +// var every *.integration.test.js in this directory already gates on) rather +// than introducing a second, parallel gating mechanism. Migrations are applied +// ONCE out-of-band via the real runMigrations() against the scratch DB (see +// implementation.md) — this harness does not run them itself, matching every +// existing IT file here. `ensureVectorSchema()` takes no arguments (reads its +// own module-level pool from db.js), so targeting the scratch DB requires the +// same trick those files use: mutate DB_HOST/DB_PORT/DB_NAME/DB_USER/DB_PASSWORD +// from the DSN, then dynamically `import()` the module AFTER that mutation so +// db.js's pool (constructed once, at import time) picks up the right env. +import pg from 'pg'; + +const DSN = process.env.VECTOR_IT_DB || ''; + +export function hasTestDb() { + return DSN.length > 0; +} + +let _pool = null; +let _ready = null; + +async function ready() { + if (_ready) return _ready; + _ready = (async () => { + const u = new URL(DSN); + Object.assign(process.env, { + DB_HOST: u.hostname, + DB_PORT: u.port, + DB_NAME: u.pathname.slice(1), + DB_USER: u.username, + DB_PASSWORD: u.password, + }); + const { ensureVectorSchema } = await import('./vectorStore.js'); + await ensureVectorSchema(); + _pool = new pg.Pool({ connectionString: DSN, max: 4 }); + return _pool; + })(); + return _ready; +} + +// Runs fn with a clean slate: truncate every table the fused-search / ranking +// tests touch, in one statement (Postgres resolves FK order within a single +// TRUNCATE ... CASCADE regardless of listed order). embed_runs has no ON +// DELETE CASCADE from index_generations by design (Phase 3 finding — the +// production code never hard-deletes a generation), so it must be listed +// explicitly rather than relying on cascade from index_generations alone. +export async function withTestDb(fn) { + const pool = await ready(); + await pool.query( + `TRUNCATE embeddings, embed_runs, embed_watermark, index_generations, + messages, folders, email_accounts, users RESTART IDENTITY CASCADE` + ); + return fn(pool); +} + +export async function closeTestDb() { + if (_pool) { await _pool.end(); _pool = null; _ready = null; } +} diff --git a/backend/src/services/embeddings/activeGeneration.js b/backend/src/services/embeddings/activeGeneration.js new file mode 100644 index 0000000..9eef0e4 --- /dev/null +++ b/backend/src/services/embeddings/activeGeneration.js @@ -0,0 +1,44 @@ +import { VectorUnavailableError } from './vectorErrors.js'; +import * as generations from './generations.js'; +import { resolveEmbedConfig, generationFingerprint } from './config.js'; + +const DEFAULTS = { + activeGeneration: generations.activeGeneration, + buildingGeneration: generations.buildingGeneration, +}; + +// Port of internal/vector/generations.go ResolveActiveForFingerprint. +// Throws VectorUnavailableError('index_stale') on a fingerprint mismatch +// (a rebuild superseded the caller's config), ('index_building') when only +// a build is in progress, ('no_active_generation') when neither exists. +export async function resolveActiveGeneration(fingerprint, overrides = {}) { + const d = { ...DEFAULTS, ...overrides }; + const active = await d.activeGeneration(); + if (active) { + if (fingerprint && active.fingerprint !== fingerprint) throw new VectorUnavailableError('index_stale'); + return { id: active.id, model: active.model, dimension: active.dimension, fingerprint: active.fingerprint, state: active.state }; + } + if (await d.buildingGeneration()) throw new VectorUnavailableError('index_building'); + throw new VectorUnavailableError('no_active_generation'); +} + +const PREAMBLE_DEFAULTS = { + resolveEmbedConfig, + generationFingerprint, + resolveActiveGeneration, +}; + +// The full vector-availability gate shared by every caller that needs an active +// generation from the stored embed config: resolve the config, reject an absent or +// disabled one as vector_not_enabled, then resolve the active generation by +// fingerprint (which throws index_stale/index_building/no_active_generation on a +// degraded index). Returns the resolved cfg alongside the generation — callers need +// both (the embed client + preprocess config, and the generation id/dimension). The +// collaborators are injectable so hybridSearch can thread its own fakes through. +export async function resolveActiveGenerationFromConfig(overrides = {}) { + const d = { ...PREAMBLE_DEFAULTS, ...overrides }; + const cfg = await d.resolveEmbedConfig(); + if (!cfg || cfg.enabled === false) throw new VectorUnavailableError('vector_not_enabled'); + const generation = await d.resolveActiveGeneration(d.generationFingerprint(cfg)); + return { cfg, generation }; +} diff --git a/backend/src/services/embeddings/activeGeneration.test.js b/backend/src/services/embeddings/activeGeneration.test.js new file mode 100644 index 0000000..40a293b --- /dev/null +++ b/backend/src/services/embeddings/activeGeneration.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { resolveActiveGeneration } from './activeGeneration.js'; +import { VectorUnavailableError } from './vectorErrors.js'; + +const gen = { id: 3, model: 'm', dimension: 4, fingerprint: 'm:4:fp', state: 'active', messageCount: 10 }; + +describe('resolveActiveGeneration', () => { + it('returns the rich active generation when the fingerprint matches', async () => { + const g = await resolveActiveGeneration('m:4:fp', + { activeGeneration: async () => gen, buildingGeneration: async () => null }); + expect(g).toEqual({ id: 3, model: 'm', dimension: 4, fingerprint: 'm:4:fp', state: 'active' }); + }); + it('throws index_stale on a fingerprint mismatch', async () => { + const err = await resolveActiveGeneration('OTHER', + { activeGeneration: async () => gen, buildingGeneration: async () => null }).catch(e => e); + expect(err).toBeInstanceOf(VectorUnavailableError); + expect(err.reason).toBe('index_stale'); + }); + it('throws index_building when a build is in progress and none active', async () => { + const err = await resolveActiveGeneration('x', + { activeGeneration: async () => null, buildingGeneration: async () => ({ id: 9 }) }).catch(e => e); + expect(err.reason).toBe('index_building'); + }); + it('throws no_active_generation when nothing exists', async () => { + const err = await resolveActiveGeneration('x', + { activeGeneration: async () => null, buildingGeneration: async () => null }).catch(e => e); + expect(err.reason).toBe('no_active_generation'); + }); +}); diff --git a/backend/src/services/embeddings/chunk.js b/backend/src/services/embeddings/chunk.js new file mode 100644 index 0000000..fb40075 --- /dev/null +++ b/backend/src/services/embeddings/chunk.js @@ -0,0 +1,86 @@ +// Port of internal/vector/embed/chunk.go, in code-point (rune) space. IMPORTANT: any +// change to the window/overlap/soft-break logic MUST bump EMBED_POLICY_VERSION in config.js. + +const SENTENCE_TERMS = [['.', ' '], ['?', ' '], ['!', ' '], ['.', '\n'], ['?', '\n'], ['!', '\n']]; + +// Chunking policy shared by the embed worker (write path) and chunkmatch (read path): +// the per-message span cap and the maxRunes→overlap rule. This module owns them so the +// two paths cannot drift — a mismatch would silently misalign re-chunked read offsets +// from the stored write offsets. Changing either MUST bump EMBED_POLICY_VERSION. +export const MAX_SPANS = 64; // worker.go maxSpansPerMessage + +export function chunkOverlapFor(maxRunes) { + if (maxRunes <= 0) return 0; + if (maxRunes < 200) return 0; + return Math.floor(maxRunes / 30); +} + +// Largest i in [floor, ceil-2] with cps[i]===a && cps[i+1]===b, else -1. +function lastPair(cps, a, b, floor, ceil) { + for (let i = ceil - 2; i >= floor; i--) if (cps[i] === a && cps[i + 1] === b) return i; + return -1; +} +// Largest i in [floor, ceil-1] with cps[i]===ch, else -1. +function lastChar(cps, ch, floor, ceil) { + for (let i = ceil - 1; i >= floor; i--) if (cps[i] === ch) return i; + return -1; +} + +// Return the code-point index of the preferred soft break in [floor, ceil): paragraph +// beats sentence beats word; ceil when none found. +function findSoftBreak(cps, floor, ceil) { + const para = lastPair(cps, '\n', '\n', floor, ceil); + if (para >= 0) return para + 2; + + let sentenceEnd = -1; + let threshold = floor - 1; + for (const [a, b] of SENTENCE_TERMS) { + const start = lastPair(cps, a, b, floor, ceil); + if (start >= 0 && start > threshold) { sentenceEnd = start + 2; threshold = start + 2; } + } + if (sentenceEnd >= 0) return sentenceEnd; + + const sp = lastChar(cps, ' ', floor, ceil); + if (sp > floor) return sp + 1; + + return ceil; +} + +export function chunkText(text, maxRunes, overlapRunes, maxSpans) { + if (text === '') return { spans: [], tailDropped: false }; + if (maxRunes <= 0) return { spans: [{ text, charStart: 0, charEnd: Array.from(text).length }], tailDropped: false }; + + let tailDropped = false; + // Early cap WITHOUT materializing the whole input (guards the 10M-rune fixture). + if (maxSpans > 0) { + const keep = maxSpans * maxRunes; + let count = 0, u16 = 0; + for (const ch of text) { + if (count >= keep) { text = text.slice(0, u16); tailDropped = true; break; } + count++; u16 += ch.length; + } + } + + const cps = Array.from(text); + const total = cps.length; + if (total <= maxRunes) return { spans: [{ text: cps.join(''), charStart: 0, charEnd: total }], tailDropped }; + + let overlap = overlapRunes < 0 ? 0 : overlapRunes; + if (overlap >= maxRunes) overlap = Math.floor(maxRunes / 2); + + const spans = []; + let cursor = 0; + while (cursor < total) { + if (maxSpans > 0 && spans.length >= maxSpans) { tailDropped = true; break; } + const windowEnd = Math.min(cursor + maxRunes, total); + let cut = windowEnd; + if (windowEnd < total) { + const searchFloor = Math.max(cursor + Math.floor((maxRunes * 3) / 4), cursor + 1); + cut = findSoftBreak(cps, searchFloor, windowEnd); + } + spans.push({ text: cps.slice(cursor, cut).join(''), charStart: cursor, charEnd: cut }); + if (cut >= total) break; + cursor += Math.max((cut - cursor) - overlap, 1); + } + return { spans, tailDropped }; +} diff --git a/backend/src/services/embeddings/chunk.test.js b/backend/src/services/embeddings/chunk.test.js new file mode 100644 index 0000000..96032f2 --- /dev/null +++ b/backend/src/services/embeddings/chunk.test.js @@ -0,0 +1,137 @@ +import { describe, it, expect } from 'vitest'; +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; + +const cp = (s) => Array.from(s).length; + +// Pin the shared chunking policy chunk.js owns. worker.js (write path) and chunkmatch.js +// (read path) both import MAX_SPANS + chunkOverlapFor from here, so they cannot drift — +// a mismatch would silently misalign re-chunked read offsets from stored write offsets. +describe('shared chunking policy (one owner)', () => { + it('exports MAX_SPANS = 64 (worker.go maxSpansPerMessage)', () => { + expect(MAX_SPANS).toBe(64); + }); + it('chunkOverlapFor: 0 below 200 runes, floor(maxRunes/30) at or above', () => { + expect(chunkOverlapFor(0)).toBe(0); + expect(chunkOverlapFor(-5)).toBe(0); + expect(chunkOverlapFor(199)).toBe(0); + expect(chunkOverlapFor(200)).toBe(6); // floor(200/30) + expect(chunkOverlapFor(3000)).toBe(100); + }); +}); + +describe('chunkText (msgvault fixture parity)', () => { + it('EmptyInputReturnsNil', () => { expect(chunkText('', 100, 10, 0).spans).toEqual([]); }); + + it('ShortInputReturnsSingleSpan', () => { + const { spans } = chunkText('hello world', 100, 10, 0); + expect(spans).toHaveLength(1); + expect(spans[0].text).toBe('hello world'); + expect(spans[0].charStart).toBe(0); + expect(spans[0].charEnd).toBe(11); + }); + + it('MaxRunesZeroDisablesChunking', () => { + const text = 'x'.repeat(1000); + const { spans } = chunkText(text, 0, 10, 0); + expect(spans).toHaveLength(1); + expect(spans[0].text).toBe(text); + }); + + it('CutsAtParagraphBreakInBackQuarter', () => { + const text = 'a'.repeat(80) + '\n\n' + 'b'.repeat(50); + const { spans } = chunkText(text, 100, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charEnd).toBe(82); + expect(spans[0].text.endsWith('\n\n')).toBe(true); + }); + + it('CutsAtSentenceBoundaryWhenNoParagraph', () => { + const text = 'a'.repeat(80) + '. ' + 'b'.repeat(50); + const { spans } = chunkText(text, 100, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charEnd).toBe(82); + }); + + it('CutsAtWordBoundaryWhenNoSentence', () => { + const text = 'a'.repeat(90) + ' ' + 'b'.repeat(50); + const { spans } = chunkText(text, 100, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charEnd).toBe(91); + }); + + it('HardCutsWhenNoSoftBreakInBackQuarter', () => { + const { spans } = chunkText('a'.repeat(1000), 100, 0, 0); + expect(spans).toHaveLength(10); + for (const s of spans) expect(s.charEnd - s.charStart).toBe(100); + }); + + it('OverlapBetweenConsecutiveChunks', () => { + const { spans } = chunkText('a'.repeat(300), 100, 20, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[1].charStart).toBe(80); + }); + + it('OverlapClampedToHalfWindow', () => { + const { spans } = chunkText('a'.repeat(300), 100, 500, 0); + expect(spans.length).toBeGreaterThan(0); + if (spans.length >= 2) expect(spans[1].charStart).toBe(50); + }); + + it('AllSpansHaveValidUTF8AndCorrectText', () => { + let text = ''; + for (let i = 0; i < 50; i++) text += 'Hello world. ' + 'こんにちは世界。'; + const { spans } = chunkText(text, 80, 10, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + const runes = Array.from(text); + for (const s of spans) { + expect(s.charStart).toBeGreaterThanOrEqual(0); + expect(s.charEnd).toBeLessThanOrEqual(runes.length); + expect(s.charStart).toBeLessThan(s.charEnd); + expect(s.text).toBe(runes.slice(s.charStart, s.charEnd).join('')); + } + }); + + it('MaxSpansCapsInputBytesProcessed', () => { + const text = 'a'.repeat(10_000_000); + const t0 = Date.now(); + const { spans } = chunkText(text, 100, 0, 3); + expect(spans).toHaveLength(3); + for (const s of spans) expect(s.charEnd).toBeLessThanOrEqual(300); + expect(Date.now() - t0).toBeLessThan(500); // O(cap), not O(10M) + }); + + it('MaxSpansCapsOutputAndDropsTail', () => { + const { spans } = chunkText('a'.repeat(1000), 100, 0, 3); + expect(spans).toHaveLength(3); + expect(spans[0].charStart).toBe(0); + expect(spans[spans.length - 1].charEnd).toBeLessThan(1000); + }); + + it('TailDroppedFlagsCapWhenLastChunkLandsOnSoftBreak', () => { + let text = ''; + for (let i = 0; i < 5; i++) text += 'a'.repeat(85) + '. '; + const { spans, tailDropped } = chunkText(text, 90, 0, 2); + expect(spans).toHaveLength(2); + expect(tailDropped).toBe(true); + }); + + it('TailDroppedFalseWhenAllContentEmitted', () => { + const { spans, tailDropped } = chunkText('a'.repeat(150), 100, 0, 10); + expect(spans.length).toBeGreaterThan(0); + expect(tailDropped).toBe(false); + }); + + it('MaxSpansZeroIsUnlimited', () => { expect(chunkText('a'.repeat(1000), 100, 0, 0).spans).toHaveLength(10); }); + it('MaxSpansLargerThanNaturalChunkCountIsNoop', () => { expect(chunkText('a'.repeat(300), 100, 0, 100).spans).toHaveLength(3); }); + + it('ConcatenationCoversInputModuloOverlap', () => { + const text = 'Lorem ipsum dolor sit amet. '.repeat(200); + const { spans } = chunkText(text, 200, 30, 0); + expect(spans.length).toBeGreaterThanOrEqual(2); + expect(spans[0].charStart).toBe(0); + expect(spans[spans.length - 1].charEnd).toBe(cp(text)); + for (let i = 1; i < spans.length; i++) { + expect(spans[i].charStart).toBeLessThanOrEqual(spans[i - 1].charEnd); + } + }); +}); diff --git a/backend/src/services/embeddings/chunkmatch.js b/backend/src/services/embeddings/chunkmatch.js new file mode 100644 index 0000000..33ed982 --- /dev/null +++ b/backend/src/services/embeddings/chunkmatch.js @@ -0,0 +1,147 @@ +// Port of internal/vector/chunkmatch/matches.go. Converts stored vector chunk +// offsets (code points into the PREPROCESSED subject+body) into API-safe match +// excerpts: a snippet plus, only when the preprocessed chunk maps exactly and +// uniquely into the raw body, a raw-body BYTE char_offset and 1-based line. +// +// Two exports, both consumed by the MCP handler layer: +// matchesInMessage — search_in_message mode=vector: re-embeds the query and the +// message's freshly re-chunked text, scores, and builds excerpts. Gates on +// resolveActiveGenerationFromConfig (throws VectorUnavailableError on stock/degraded PG). +// matchFromChunk — semantic_search_messages: phase 4 already ranked and surfaced +// the single best_chunk per hit, so no embedding — just re-derive the snippet +// and map it back to a raw-body byte offset. +import { query } from '../db.js'; +import { resolveActiveGenerationFromConfig } from './hybrid.js'; // phase-4 public face +import { EmbeddingClient } from './client.js'; // phase 3 +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; // phase 3 (shared chunking policy) +import { preprocess } from './preprocess.js'; // phase 3 +import { RAW_BODY_MULT } from './worker.js'; // phase 3 (write-path body cap) +import { resolveEmbedConfig } from './config.js'; // phase 1 +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from '../../utils/textExcerpt.js'; + +// Preprocess options with the derived raw-body cap the worker applies at write time +// (worker.js _embedBatch). Without the same cap, a pathological body preprocesses to +// different text here than what was embedded, silently misaligning chunk offsets. +function derivedPreprocessCfg(cfg) { + const pp = { ...((cfg && cfg.preprocess) || {}) }; + if (!pp.maxBodyRunes && cfg && cfg.maxInputChars > 0) { + pp.maxBodyRunes = cfg.maxInputChars * MAX_SPANS * RAW_BODY_MULT; + } + return pp; +} + +// First maxBytes of a UTF-8 buffer, never splitting a rune (msgvault bytePrefix). +function bytePrefix(buf, maxBytes) { + if (buf.length <= maxBytes) return buf.toString('utf8'); + let end = maxBytes; + while (end > 0 && !isRuneStart(buf[end])) end--; + return buf.toString('utf8', 0, end); +} + +function cosine(a, b) { + let dot = 0, na = 0, nb = 0; + for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; } + return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0; +} + +// Byte offset of chunk in the raw body, only if it occurs exactly once (msgvault uniqueBodyOffset). +function uniqueByteOffset(bodyBuf, chunkBuf) { + const first = bodyBuf.indexOf(chunkBuf); + if (first < 0 || bodyBuf.lastIndexOf(chunkBuf) !== first) return null; + return first; +} + +// The subject-prefix rune count preprocess.js prepends ("Subject: \n\n"), or 0 +// when the subject is empty (preprocess adds no prefix then). A chunk starting at +// or past this offset lies in the body region, so a raw-body offset is meaningful. +function subjectPrefixRunes(subject) { + return subject ? [...('Subject: ' + subject + '\n\n')].length : 0; +} + +// Scoped message load. Body selection matches worker.js (body_text unless blank, +// then body_html) so the preprocessed text — and thus the chunk offsets — align +// with what was embedded. Returns null when the message is out of scope. +async function loadMessage(messageId, accountIds) { + const { rows } = await query( + 'SELECT subject, body_text, body_html FROM messages WHERE id = $1 AND account_id = ANY($2)', + [messageId, accountIds], + ); + if (!rows.length) return null; + const subject = rows[0].subject || ''; + const bodyText = rows[0].body_text; + const rawBody = bodyText && bodyText.trim() !== '' ? bodyText : (rows[0].body_html || ''); + return { subject, rawBody }; +} + +export async function matchesInMessage(messageId, queryText, minScore, { accountIds }) { + const q = (queryText || '').trim(); + if (!q) return []; + + const { cfg } = await resolveActiveGenerationFromConfig(); + + const msg = await loadMessage(messageId, accountIds); + if (!msg) return []; + const { subject, rawBody } = msg; + if (!rawBody) return []; + + const { text: preprocessed } = preprocess(subject, rawBody, 0, derivedPreprocessCfg(cfg)); + const prefixRunes = subjectPrefixRunes(subject); + const window = cfg.maxInputChars || 0; + const { spans } = chunkText(preprocessed, window, chunkOverlapFor(window), MAX_SPANS); + if (!spans.length) return []; + + // Re-embed the query plus every chunk. Embeddings are deterministic for a given + // model+input, so re-embedding the chunk text yields the same vectors the stored + // chunks carry — the equivalent of msgvault's ScoreMessageChunks. The inputs + // (1 query + up to MAX_SPANS chunks) are embedded in slices of cfg.batchSize, + // the same per-call budget the worker's write path honors. + const client = new EmbeddingClient(cfg); + const inputs = [q, ...spans.map((s) => s.text)]; + const batchSize = cfg.batchSize > 0 ? cfg.batchSize : 32; + const vecs = []; + for (let i = 0; i < inputs.length; i += batchSize) { + vecs.push(...await client.embed(inputs.slice(i, i + batchSize))); + } + const queryVec = vecs[0]; + const bodyBuf = Buffer.from(rawBody, 'utf8'); + const min = Number.isFinite(minScore) ? minScore : 0; + + return spans + .map((span, i) => ({ span, score: cosine(queryVec, vecs[i + 1]) })) + .filter((s) => s.score >= min) + .sort((a, b) => b.score - a.score) + .map(({ span, score }) => { + const chunkBuf = Buffer.from(span.text, 'utf8'); + const m = { snippet: bytePrefix(chunkBuf, SNIPPET_BYTES), score }; + if (span.charStart >= prefixRunes) { + const off = uniqueByteOffset(bodyBuf, chunkBuf); + if (off !== null) { m.char_offset = off; m.line = lineNumberAt(bodyBuf, off); } + } + return m; + }); +} + +export async function matchFromChunk(messageId, bestChunk, { accountIds }) { + if (!bestChunk) return null; + const msg = await loadMessage(messageId, accountIds); + if (!msg) return null; + const { subject, rawBody } = msg; + + const cfg = await resolveEmbedConfig(); + const { text: preprocessed } = preprocess(subject, rawBody, 0, derivedPreprocessCfg(cfg)); + const prefixRunes = subjectPrefixRunes(subject); + const cps = [...preprocessed]; // code points — the domain phase-4 char_start/char_end index into + const start = Math.max(0, bestChunk.char_start | 0); + const end = Math.min(cps.length, bestChunk.char_end | 0); + if (end <= start) return null; + + const chunkStr = cps.slice(start, end).join(''); + const chunkBuf = Buffer.from(chunkStr, 'utf8'); + const match = { snippet: bytePrefix(chunkBuf, SNIPPET_BYTES), score: bestChunk.score }; + if (start >= prefixRunes && rawBody) { + const bodyBuf = Buffer.from(rawBody, 'utf8'); + const off = uniqueByteOffset(bodyBuf, chunkBuf); + if (off !== null) { match.char_offset = off; match.line = lineNumberAt(bodyBuf, off); } + } + return match; +} diff --git a/backend/src/services/embeddings/chunkmatch.test.js b/backend/src/services/embeddings/chunkmatch.test.js new file mode 100644 index 0000000..ecec927 --- /dev/null +++ b/backend/src/services/embeddings/chunkmatch.test.js @@ -0,0 +1,171 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Real module shapes (verified against source, not the plan's Consumes sketch): +// preprocess(subject, body, maxChars, cfg) -> { text, truncated } +// chunkText(text, maxRunes, overlap, maxSpans) -> { spans: [{text, charStart, charEnd}] } +// client.js exports the EmbeddingClient class (no bare `embed`) — mirror hybrid.js: +// new EmbeddingClient(cfg).embed([...]) -> number[][] +// resolveEmbedConfig() is async; generationFingerprint(cfg) needs the full config. +const { mockEmbed } = vi.hoisted(() => ({ mockEmbed: vi.fn() })); + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('./hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('./client.js', () => ({ EmbeddingClient: class { embed(...a) { return mockEmbed(...a); } } })); +vi.mock('./chunk.js', async (importOriginal) => ({ ...(await importOriginal()), chunkText: vi.fn() })); +vi.mock('./preprocess.js', () => ({ preprocess: vi.fn() })); +vi.mock('./config.js', () => ({ + generationFingerprint: vi.fn(() => 'fp'), + resolveEmbedConfig: vi.fn(async () => ({ + enabled: true, endpoint: 'http://x', apiKey: null, model: 'm', dimension: 2, + maxInputChars: 100, preprocess: {}, + })), +})); + +import { query } from '../db.js'; +import { resolveActiveGenerationFromConfig } from './hybrid.js'; +import { chunkText } from './chunk.js'; +import { preprocess } from './preprocess.js'; +import { resolveEmbedConfig } from './config.js'; +import { matchesInMessage, matchFromChunk } from './chunkmatch.js'; + +class VectorUnavailableError extends Error { + constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } +} + +const cfg = { + enabled: true, endpoint: 'http://x', apiKey: null, model: 'm', dimension: 2, + maxInputChars: 100, preprocess: {}, +}; + +beforeEach(() => { + query.mockReset(); + resolveActiveGenerationFromConfig.mockReset(); + chunkText.mockReset(); + preprocess.mockReset(); + mockEmbed.mockReset(); + resolveEmbedConfig.mockReset(); + resolveEmbedConfig.mockResolvedValue(cfg); + // matchesInMessage gets its cfg + generation from the one vector-availability gate. + resolveActiveGenerationFromConfig.mockResolvedValue({ cfg, generation: { id: 1 } }); +}); + +describe('matchesInMessage', () => { + it('throws vector_not_enabled when embeddings are disabled', async () => { + resolveActiveGenerationFromConfig.mockRejectedValueOnce(new VectorUnavailableError('vector_not_enabled')); + await expect(matchesInMessage('m1', 'hi', 0, { accountIds: ['a'] })) + .rejects.toHaveProperty('reason', 'vector_not_enabled'); + }); + + it('propagates VectorUnavailableError on stock Postgres', async () => { + resolveActiveGenerationFromConfig.mockRejectedValue(new VectorUnavailableError('no_active_generation')); + await expect(matchesInMessage('m1', 'hi', 0, { accountIds: ['a'] })) + .rejects.toHaveProperty('reason', 'no_active_generation'); + }); + + it('scores body chunks, returns byte char_offset for a unique body chunk, filters by minScore', async () => { + const subject = 'Trip'; + const body = 'café — the flight itinerary is attached and confirmed'; + query.mockResolvedValueOnce({ rows: [{ subject, body_text: body, body_html: '' }] }); + // preprocessed = "Subject: Trip\n\n" + body; prefix rune count decides subject vs body. + preprocess.mockReturnValue({ text: 'Subject: Trip\n\n' + body }); + // one subject chunk, one body chunk that appears verbatim & uniquely in the raw body. + chunkText.mockReturnValue({ spans: [ + { text: 'Subject: Trip', charStart: 0, charEnd: 13 }, + { text: 'the flight itinerary is attached', charStart: 15, charEnd: 47 }, + ] }); + // embed([query, chunk0, chunk1]) -> query aligns with chunk1 (score 1), chunk0 score 0. + mockEmbed.mockResolvedValue([[1, 0], [0, 1], [1, 0]]); + const out = await matchesInMessage('m1', 'flight itinerary', 0.5, { accountIds: ['a'] }); + expect(out).toHaveLength(1); // chunk0 filtered by minScore + const byteOffset = Buffer.from(body, 'utf8').indexOf(Buffer.from('the flight itinerary is attached', 'utf8')); + expect(out[0].char_offset).toBe(byteOffset); // BYTE offset into raw body + expect(out[0].line).toBe(1); + expect(out[0].score).toBeCloseTo(1); + expect(out[0].snippet).toContain('flight itinerary'); + // message load was scoped to accountIds + expect(query.mock.calls[0][1]).toEqual(['m1', ['a']]); + // query embedded first, then each chunk text + expect(mockEmbed).toHaveBeenCalledWith(['flight itinerary', 'Subject: Trip', 'the flight itinerary is attached']); + }); + + it('returns [] when the message is not in scope', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await matchesInMessage('m1', 'x', 0, { accountIds: ['a'] })).toEqual([]); + }); + + it('slices embed calls by cfg.batchSize and keeps query/chunk alignment across slices', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ cfg: { ...cfg, batchSize: 2 }, generation: { id: 1 } }); + query.mockResolvedValueOnce({ rows: [{ subject: '', body_text: 'b', body_html: '' }] }); + preprocess.mockReturnValue({ text: 'b' }); + chunkText.mockReturnValue({ spans: [ + { text: 'c0', charStart: 0, charEnd: 2 }, + { text: 'c1', charStart: 2, charEnd: 4 }, + { text: 'c2', charStart: 4, charEnd: 6 }, + ] }); + // 4 inputs (query + 3 chunks) with batchSize 2 → two calls of 2. + mockEmbed + .mockResolvedValueOnce([[1, 0], [0, 1]]) // [query, c0] + .mockResolvedValueOnce([[1, 0], [0, 1]]); // [c1, c2] — c1 aligns with the query + const out = await matchesInMessage('m1', 'q', 0.5, { accountIds: ['a'] }); + expect(mockEmbed.mock.calls.map((c) => c[0])).toEqual([['q', 'c0'], ['c1', 'c2']]); + expect(out).toHaveLength(1); + expect(out[0].snippet).toBe('c1'); // slice order preserved: vecs[2] is still chunk 1 + }); + + it('preprocesses with the worker\'s derived maxBodyRunes cap so offsets align', async () => { + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'b', body_html: '' }] }); + preprocess.mockReturnValue({ text: '' }); + chunkText.mockReturnValue({ spans: [] }); + await matchesInMessage('m1', 'q', 0, { accountIds: ['a'] }); + // worker.js _embedBatch: maxInputChars(100) * MAX_SPANS(64) * RAW_BODY_MULT(16) + expect(preprocess).toHaveBeenCalledWith('S', 'b', 0, { maxBodyRunes: 102400 }); + }); + + it('never overrides an explicitly configured maxBodyRunes', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { ...cfg, preprocess: { maxBodyRunes: 7 } }, generation: { id: 1 }, + }); + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'b', body_html: '' }] }); + preprocess.mockReturnValue({ text: '' }); + chunkText.mockReturnValue({ spans: [] }); + await matchesInMessage('m1', 'q', 0, { accountIds: ['a'] }); + expect(preprocess).toHaveBeenCalledWith('S', 'b', 0, { maxBodyRunes: 7 }); + }); +}); + +describe('matchFromChunk', () => { + it('re-derives snippet + raw-body byte char_offset from best_chunk, without embedding', async () => { + const subject = 'Trip'; + const body = 'the flight itinerary is attached and confirmed'; + query.mockResolvedValueOnce({ rows: [{ subject, body_text: body, body_html: '' }] }); + preprocess.mockReturnValue({ text: 'Subject: Trip\n\n' + body }); + const prefixLen = [...'Subject: Trip\n\n'].length; // code points before the body + const startCp = prefixLen + body.indexOf('flight itinerary'); + const endCp = startCp + 'flight itinerary'.length; + const m = await matchFromChunk('m1', { chunk_index: 0, char_start: startCp, char_end: endCp, score: 0.87 }, { accountIds: ['acc-1'] }); + expect(m.snippet).toBe('flight itinerary'); + expect(m.score).toBe(0.87); + expect(m.char_offset).toBe(Buffer.from(body, 'utf8').indexOf(Buffer.from('flight itinerary', 'utf8'))); + expect(m.line).toBe(1); + expect(mockEmbed).not.toHaveBeenCalled(); // matchFromChunk never embeds — phase 4 already scored + expect(query.mock.calls[0][1]).toEqual(['m1', ['acc-1']]); // scoped to the token's accounts + }); + + it('returns null for an out-of-scope message', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await matchFromChunk('m1', { char_start: 0, char_end: 5, score: 1 }, { accountIds: ['acc-1'] })).toBeNull(); + }); + + it('returns null when the code-point range is empty', async () => { + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'body', body_html: '' }] }); + preprocess.mockReturnValue({ text: 'Subject: S\n\nbody' }); + expect(await matchFromChunk('m1', { char_start: 5, char_end: 5, score: 1 }, { accountIds: ['acc-1'] })).toBeNull(); + }); + + it('preprocesses with the worker\'s derived maxBodyRunes cap so stored offsets align', async () => { + query.mockResolvedValueOnce({ rows: [{ subject: 'S', body_text: 'body', body_html: '' }] }); + preprocess.mockReturnValue({ text: 'Subject: S\n\nbody' }); + await matchFromChunk('m1', { char_start: 12, char_end: 16, score: 1 }, { accountIds: ['acc-1'] }); + expect(preprocess).toHaveBeenCalledWith('S', 'body', 0, { maxBodyRunes: 102400 }); + }); +}); diff --git a/backend/src/services/embeddings/client.js b/backend/src/services/embeddings/client.js new file mode 100644 index 0000000..44d8408 --- /dev/null +++ b/backend/src/services/embeddings/client.js @@ -0,0 +1,99 @@ +// Port of internal/vector/embed/client.go. + +export class Permanent4xxError extends Error { + constructor(message) { super(message); this.name = 'Permanent4xxError'; this.permanent4xx = true; } +} +export function isPermanent4xx(err) { return err instanceof Permanent4xxError || err?.permanent4xx === true; } + +function parseRetryAfter(v) { + if (!v) return null; + const s = String(v).trim(); + const MAX = 3600 * 1000; + if (/^\d+$/.test(s)) return Math.min(Number(s) * 1000, MAX); + const t = Date.parse(s); + if (!Number.isNaN(t)) { const d = t - Date.now(); return d <= 0 ? 0 : Math.min(d, MAX); } + return null; +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Best-effort drain of a response body we are about to abandon. undici keeps the +// socket checked out until the body is consumed or cancelled; the retry paths +// (429/5xx) throw without decoding a body, so drain here to release the connection +// back to the pool before we back off and retry — otherwise a burst of rate +// limiting across scheduler ticks leaks connections. Errors are swallowed: a failed +// drain must not mask the retryable status we are reporting. +async function drainBody(resp) { + try { await resp.text(); } catch { /* already released/aborted — nothing to free */ } +} + +export class EmbeddingClient { + // `dimension` is the expected vector length; every returned embedding is asserted + // against it. Pass null/undefined to skip the assertion — the admin test-embeddings + // probe does this because its job is to DISCOVER the endpoint's real dimension. + // Worker/query paths always construct with the configured dimension. + constructor({ endpoint, apiKey, model, dimension, timeout = 30000, maxRetries = 3 }) { + this.endpoint = endpoint; + this.apiKey = apiKey || null; + this.model = model; + this.dimension = dimension ?? null; + this.timeout = timeout; + this.maxRetries = maxRetries; + } + + async embed(inputs) { + if (!inputs.length) return []; + const body = JSON.stringify({ input: inputs, model: this.model }); + let lastErr; + for (let attempt = 1; attempt <= this.maxRetries; attempt++) { + try { + return await this._doOnce(body, inputs.length); + } catch (err) { + lastErr = err; + if (!err._retry) throw err; // permanent 4xx or dimension mismatch — no retry + if (attempt === this.maxRetries) break; + let backoff = Math.min(2 ** Math.min(attempt, 8), 256) * 100; + if (err._retryAfterSet) backoff = err._retryAfter; + if (backoff > 0) await sleep(backoff); + } + } + throw new Error(`embed: giving up after ${this.maxRetries} attempts: ${lastErr.message}`); + } + + async _doOnce(body, want) { + const headers = { 'Content-Type': 'application/json' }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + let resp; + try { + resp = await fetch(`${this.endpoint}/embeddings`, { method: 'POST', headers, body, signal: AbortSignal.timeout(this.timeout) }); + } catch (e) { const err = new Error(`http do: ${e.message}`); err._retry = true; throw err; } + + if (resp.status === 429) { + const ra = parseRetryAfter(resp.headers.get('Retry-After')); + await drainBody(resp); // free the socket before backing off — we discard this body + const err = new Error('embed: HTTP 429 (rate limited)'); err._retry = true; + if (ra !== null) { err._retryAfterSet = true; err._retryAfter = ra; } + throw err; + } + if (resp.status >= 500) { + await drainBody(resp); // free the socket before retrying — we discard this body + const err = new Error(`embed: HTTP ${resp.status}`); err._retry = true; throw err; + } + if (resp.status >= 400) { + const txt = (await resp.text().catch(() => '')).slice(0, 4096).trim(); + throw new Permanent4xxError(`embed: HTTP ${resp.status}${txt ? ': ' + txt : ''}`); + } + let data; + try { data = await resp.json(); } catch (e) { const err = new Error(`decode response: ${e.message}`); err._retry = true; throw err; } + + const vecs = new Array(want).fill(null); + for (const d of data.data || []) { + if (d.index < 0 || d.index >= want) throw new Error(`embed: invalid index ${d.index} (len=${want})`); + if (this.dimension !== null && d.embedding.length !== this.dimension) { + throw new Error(`embed: dimension mismatch: got ${d.embedding.length}, configured ${this.dimension}`); + } + vecs[d.index] = d.embedding; + } + for (let i = 0; i < want; i++) if (vecs[i] === null) throw new Error(`embed: missing embedding at index ${i}`); + return vecs; + } +} diff --git a/backend/src/services/embeddings/client.test.js b/backend/src/services/embeddings/client.test.js new file mode 100644 index 0000000..e2fd7ad --- /dev/null +++ b/backend/src/services/embeddings/client.test.js @@ -0,0 +1,119 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { EmbeddingClient, isPermanent4xx } from './client.js'; + +function mockFetchOnce(status, body, headers = {}) { + return { + status, + headers: { get: (k) => headers[k.toLowerCase()] ?? null }, + json: async () => body, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + }; +} +const okBody = (n, dim = 3) => ({ data: Array.from({ length: n }, (_, i) => ({ index: i, embedding: Array(dim).fill(0.1) })), model: 'm' }); + +afterEach(() => vi.unstubAllGlobals()); + +describe('EmbeddingClient.embed', () => { + it('returns one vector per input in order', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(mockFetchOnce(200, okBody(2)))); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3 }); + const out = await c.embed(['a', 'b']); + expect(out).toHaveLength(2); + expect(out[0]).toEqual([0.1, 0.1, 0.1]); + }); + + it('empty input is a no-op (no HTTP call)', async () => { + const f = vi.fn(); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3 }); + expect(await c.embed([])).toEqual([]); + expect(f).not.toHaveBeenCalled(); + }); + + it('retries a 429 (Retry-After: 0) then succeeds', async () => { + const f = vi.fn() + .mockResolvedValueOnce(mockFetchOnce(429, 'slow down', { 'retry-after': '0' })) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + const out = await c.embed(['a']); + expect(out).toHaveLength(1); + expect(f).toHaveBeenCalledTimes(2); + }); + + it('rejects a dimension mismatch without retrying', async () => { + const f = vi.fn().mockResolvedValueOnce(mockFetchOnce(200, okBody(1, 5))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3 }); + await expect(c.embed(['a'])).rejects.toThrow(/dimension mismatch/); + expect(f).toHaveBeenCalledTimes(1); + }); + + it('accepts any vector length when constructed without a dimension expectation (probe mode)', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(mockFetchOnce(200, okBody(1, 5)))); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: null }); + const out = await c.embed(['a']); + expect(out[0]).toHaveLength(5); // discovered, not asserted — the Test probe path + }); + + it('marks a 400 as a permanent 4xx (no retry)', async () => { + const f = vi.fn().mockResolvedValueOnce(mockFetchOnce(400, 'bad request')); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + const err = await c.embed(['a']).catch((e) => e); + expect(isPermanent4xx(err)).toBe(true); + expect(f).toHaveBeenCalledTimes(1); + }); + + it('gives up after maxRetries on a persistent network error', async () => { + const f = vi.fn().mockRejectedValue(new Error('ECONNREFUSED')); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 1 }); + await expect(c.embed(['a'])).rejects.toThrow(/giving up after 1 attempts/); + expect(f).toHaveBeenCalledTimes(1); + }); + + // Undici keeps the socket checked out until the body is read/cancelled; the + // retry paths throw without decoding a body, so they must drain it first or a + // burst of rate limiting across scheduler ticks leaks connections. + it('consumes the response body on a retryable 429 before throwing', async () => { + const textSpy = vi.fn(async () => 'slow down'); + const resp429 = { status: 429, headers: { get: () => null }, json: async () => ({}), text: textSpy }; + const f = vi.fn() + .mockResolvedValueOnce(resp429) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + await c.embed(['a']); + expect(textSpy).toHaveBeenCalledTimes(1); + }); + + it('consumes the response body on a retryable 5xx before throwing', async () => { + const textSpy = vi.fn(async () => 'upstream boom'); + const resp503 = { status: 503, headers: { get: () => null }, json: async () => ({}), text: textSpy }; + const f = vi.fn() + .mockResolvedValueOnce(resp503) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + await c.embed(['a']); + expect(textSpy).toHaveBeenCalledTimes(1); + }); + + it('still retries a 429 even if draining the body throws (best-effort)', async () => { + const resp429 = { + status: 429, + headers: { get: () => null }, + json: async () => ({}), + text: vi.fn().mockRejectedValue(new Error('body already released')), + }; + const f = vi.fn() + .mockResolvedValueOnce(resp429) + .mockResolvedValueOnce(mockFetchOnce(200, okBody(1))); + vi.stubGlobal('fetch', f); + const c = new EmbeddingClient({ endpoint: 'http://h/v1', model: 'm', dimension: 3, maxRetries: 3 }); + const out = await c.embed(['a']); + expect(out).toHaveLength(1); + expect(f).toHaveBeenCalledTimes(2); + }); +}); diff --git a/backend/src/services/embeddings/config.js b/backend/src/services/embeddings/config.js new file mode 100644 index 0000000..4467992 --- /dev/null +++ b/backend/src/services/embeddings/config.js @@ -0,0 +1,58 @@ +import { query } from '../db.js'; +import { decrypt } from '../encryption.js'; + +// Bump when preprocess.js changes its output for the same flags (tighten a regex, +// add a tracking param, add a default-on transform). Folds into the generation +// fingerprint so a policy change forces a new generation. Port of config.go preprocessVersion. +export const PREPROCESS_VERSION = 1; +// Bump when worker.js/chunk.js change the vector layout for the same preprocess +// output (chunk window/overlap/cap). Port of config.go embedPolicyVersion. +export const EMBED_POLICY_VERSION = 1; + +const PREPROCESS_FLAG_ORDER = [ + 'stripQuotes', 'stripSignatures', 'stripHTML', 'stripBase64', 'stripURLTracking', 'collapseWhitespace', +]; + +export function preprocessFingerprint(pp) { + const bits = PREPROCESS_FLAG_ORDER.map((k) => (pp[k] ? '1' : '0')).join(''); + return `p${PREPROCESS_VERSION}-${bits}`; +} + +export function generationFingerprint(cfg) { + return `${cfg.model}:${cfg.dimension}:${preprocessFingerprint(cfg.preprocess)}:c${cfg.maxInputChars}:e${EMBED_POLICY_VERSION}`; +} + +function resolveFlag(v) { return v === undefined || v === null ? true : v === true; } + +export function applyEmbedDefaults(raw) { + const pp = raw.preprocess || {}; + return { + enabled: raw.enabled === true, + endpoint: (raw.endpoint || '').trim().replace(/\/+$/, ''), + apiKey: raw.apiKey || null, + model: (raw.model || '').trim(), + dimension: Number(raw.dimension) || 0, + maxInputChars: Number(raw.maxInputChars) > 0 ? Number(raw.maxInputChars) : 32768, + batchSize: Number(raw.batchSize) > 0 ? Number(raw.batchSize) : 32, + skipExtensionCreate: raw.skipExtensionCreate === true, + preprocess: { + stripQuotes: resolveFlag(pp.stripQuotes), + stripSignatures: resolveFlag(pp.stripSignatures), + stripHTML: resolveFlag(pp.stripHTML), + stripBase64: resolveFlag(pp.stripBase64), + stripURLTracking: resolveFlag(pp.stripURLTracking), + collapseWhitespace: resolveFlag(pp.collapseWhitespace), + }, + }; +} + +export async function resolveEmbedConfig() { + const result = await query("SELECT value FROM system_settings WHERE key = 'ai_config'"); + if (!result.rows.length) return null; + let cfg; + try { cfg = JSON.parse(result.rows[0].value); } catch { return null; } + if (!cfg.embeddings) return null; + const resolved = applyEmbedDefaults(cfg.embeddings); + resolved.apiKey = resolved.apiKey ? decrypt(resolved.apiKey) : null; + return resolved; +} diff --git a/backend/src/services/embeddings/config.test.js b/backend/src/services/embeddings/config.test.js new file mode 100644 index 0000000..cb196de --- /dev/null +++ b/backend/src/services/embeddings/config.test.js @@ -0,0 +1,53 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../encryption.js', () => ({ decrypt: (v) => (v ? v.replace(/^enc:/, '') : v) })); +const { query } = await import('../db.js'); +import { preprocessFingerprint, generationFingerprint, applyEmbedDefaults, resolveEmbedConfig } from './config.js'; + +const allOn = { stripQuotes: true, stripSignatures: true, stripHTML: true, stripBase64: true, stripURLTracking: true, collapseWhitespace: true }; + +describe('preprocessFingerprint', () => { + it('is p1-111111 when all flags on', () => { + expect(preprocessFingerprint(allOn)).toBe('p1-111111'); + }); + it('flips the 3rd bit (strip_html) off in field-declaration order', () => { + expect(preprocessFingerprint({ ...allOn, stripHTML: false })).toBe('p1-110111'); + }); +}); + +describe('generationFingerprint', () => { + it('joins model:dim:preprocess:c:e', () => { + const cfg = { model: 'nomic-embed-text', dimension: 768, maxInputChars: 32768, preprocess: allOn }; + expect(generationFingerprint(cfg)).toBe('nomic-embed-text:768:p1-111111:c32768:e1'); + }); +}); + +describe('applyEmbedDefaults', () => { + it('fills batchSize 32, maxInputChars 32768, all preprocess flags on', () => { + const out = applyEmbedDefaults({ model: 'm', dimension: 4 }); + expect(out.batchSize).toBe(32); + expect(out.maxInputChars).toBe(32768); + expect(out.preprocess).toEqual(allOn); + }); + it('preserves an explicit false preprocess flag', () => { + const out = applyEmbedDefaults({ model: 'm', dimension: 4, preprocess: { stripQuotes: false } }); + expect(out.preprocess.stripQuotes).toBe(false); + expect(out.preprocess.stripHTML).toBe(true); + }); +}); + +describe('resolveEmbedConfig', () => { + it('returns null when ai_config has no embeddings block', async () => { + query.mockResolvedValueOnce({ rows: [{ value: JSON.stringify({ baseUrl: 'x' }) }] }); + expect(await resolveEmbedConfig()).toBeNull(); + }); + it('decrypts the apiKey and applies defaults', async () => { + query.mockResolvedValueOnce({ rows: [{ value: JSON.stringify({ + embeddings: { enabled: true, endpoint: 'http://h:8080/v1', apiKey: 'enc:secret', model: 'm', dimension: 4 }, + }) }] }); + const cfg = await resolveEmbedConfig(); + expect(cfg.apiKey).toBe('secret'); + expect(cfg.batchSize).toBe(32); + expect(cfg.model).toBe('m'); + }); +}); diff --git a/backend/src/services/embeddings/embedRunLock.js b/backend/src/services/embeddings/embedRunLock.js new file mode 100644 index 0000000..f51df0e --- /dev/null +++ b/backend/src/services/embeddings/embedRunLock.js @@ -0,0 +1,18 @@ +// Process-wide single-flight guard for the embed worker. Both the periodic scheduler +// tick and the manual admin build acquire this before running a worker, so at most one +// runs at a time in-process — a scheduler tick skips while a manual build is active, and +// a manual build defers while a scheduler tick is mid-run (C1). It is a plain in-memory +// flag: it does not coordinate across processes (a multi-instance deploy relies on the +// idempotent upsert + CAS + coverage gate for cross-process safety, not this lock). +let _active = false; + +// Returns true and takes the lock when free; returns false when already held. +export function tryAcquireEmbedRun() { + if (_active) return false; + _active = true; + return true; +} + +export function releaseEmbedRun() { _active = false; } + +export function isEmbedRunActive() { return _active; } diff --git a/backend/src/services/embeddings/embedRunLock.test.js b/backend/src/services/embeddings/embedRunLock.test.js new file mode 100644 index 0000000..e65eb84 --- /dev/null +++ b/backend/src/services/embeddings/embedRunLock.test.js @@ -0,0 +1,20 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { tryAcquireEmbedRun, releaseEmbedRun, isEmbedRunActive } from './embedRunLock.js'; + +beforeEach(() => releaseEmbedRun()); + +describe('embedRunLock (single-flight)', () => { + it('grants the first acquire and refuses a second while held', () => { + expect(isEmbedRunActive()).toBe(false); + expect(tryAcquireEmbedRun()).toBe(true); + expect(isEmbedRunActive()).toBe(true); + expect(tryAcquireEmbedRun()).toBe(false); // second caller (scheduler vs manual build) is refused + }); + + it('re-acquires after release', () => { + expect(tryAcquireEmbedRun()).toBe(true); + releaseEmbedRun(); + expect(isEmbedRunActive()).toBe(false); + expect(tryAcquireEmbedRun()).toBe(true); + }); +}); diff --git a/backend/src/services/embeddings/embedScan.integration.test.js b/backend/src/services/embeddings/embedScan.integration.test.js new file mode 100644 index 0000000..f37353f --- /dev/null +++ b/backend/src/services/embeddings/embedScan.integration.test.js @@ -0,0 +1,63 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// S1: the steady-state embed scan must be O(pending), not O(mailbox). The partial +// index idx_messages_embed_pending (migration 0039) only helps if the scan predicate +// is `embed_gen IS NULL` (an OR with `embed_gen <> target` forces a seq scan), which +// in turn is only correct if createGeneration resets stale stamps to NULL so a +// generation rebuild still finds prior-generation rows via the NULL scan. +d('embed scan O(pending) + reset-on-create', () => { + let store, gens, client, acctId, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + gens = await import('./generations.js'); + await store.ensureVectorSchema(); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'scan')); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('scanForEmbedding returns only embed_gen-IS-NULL rows (steady state)', async () => { + // 2000 stamped under an active-like gen id, 3 new (NULL). + await client.query("INSERT INTO messages (account_id, uid, folder, subject, embed_gen) SELECT $1, 500000+g, 'INBOX', 'x', 424242 FROM generate_series(0,1999) g", [acctId]); + const nulls = await client.query("INSERT INTO messages (account_id, uid, folder, subject, embed_gen) SELECT $1, 520000+g, 'INBOX', 'x', NULL FROM generate_series(0,2) g RETURNING id", [acctId]); + const nullIds = nulls.rows.map((r) => r.id).sort(); + const got = (await store.scanForEmbedding('424242', store.ZERO_UUID, 100)).sort(); + expect(got).toEqual(nullIds); + }); + + it('the scan plan uses the partial pending index (no Seq Scan)', async () => { + await client.query('ANALYZE messages'); + const r = await client.query( + `EXPLAIN SELECT id FROM messages WHERE embed_gen IS NULL AND is_deleted = false AND id > $1 ORDER BY id LIMIT $2`, + [store.ZERO_UUID, 32], + ); + const plan = r.rows.map((x) => x['QUERY PLAN']).join('\n'); + expect(plan).toMatch(/idx_messages_embed_pending/); + expect(plan).not.toMatch(/Seq Scan/); + }); + + it('createGeneration resets stale non-null stamps to NULL (so a rebuild re-finds them)', async () => { + // All current rows are stamped 424242 (a prior gen). A new-fingerprint generation + // must reset them to NULL so scanForEmbedding finds them for re-embedding. + const before = await client.query('SELECT COUNT(*)::int n FROM messages WHERE embed_gen IS NULL AND account_id = $1', [acctId]); + expect(before.rows[0].n).toBe(3); // only the 3 new rows are NULL pre-create + const g = await gens.createGeneration('m', 4, 'fp-reset'); + const after = await client.query('SELECT COUNT(*)::int nulls, COUNT(*) FILTER (WHERE embed_gen IS NOT NULL)::int stamped FROM messages WHERE account_id = $1', [acctId]); + expect(after.rows[0].stamped).toBe(0); // every live stamp reset + expect(after.rows[0].nulls).toBe(2003); // all rows now pending for the new gen + // and the scan now surfaces them (bounded here by limit) + const pending = await store.scanForEmbedding(g, store.ZERO_UUID, 100); + expect(pending.length).toBe(100); + // cleanup the building gen so later IT files start clean + await gens.retireGeneration(g, true); + }); +}); diff --git a/backend/src/services/embeddings/generations.chunkCount.test.js b/backend/src/services/embeddings/generations.chunkCount.test.js new file mode 100644 index 0000000..e318108 --- /dev/null +++ b/backend/src/services/embeddings/generations.chunkCount.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Unit-pin generations.chunkCount, the seam get_stats' collectStats calls for +// active_generation.message_count and the building generation's progress.done. +// It was listed in the frozen cross-phase contract but never implemented in the +// real module (only ever a vi.fn() in test mocks) — so live collectStats threw +// "generations.chunkCount is not a function" and the vector_search block vanished. +vi.mock('../db.js', () => ({ pool: { query: vi.fn() }, withTransaction: vi.fn() })); +import { pool } from '../db.js'; +import * as generations from './generations.js'; + +beforeEach(() => { pool.query.mockReset(); }); + +describe('generations.chunkCount (msgvault Stats.EmbeddingCount semantics)', () => { + it('counts DISTINCT message_id (not chunk rows) scoped to the generation, returned as a number', async () => { + // msgvault: "EmbeddingCount is distinct messages, not chunk rows". Live parity: + // 18,193 chunk rows / 18,192 distinct messages → message_count 18,192. + pool.query.mockResolvedValueOnce({ rows: [{ n: '18192' }] }); + const n = await generations.chunkCount(7); + expect(n).toBe(18192); + expect(typeof n).toBe('number'); + const [sql, params] = pool.query.mock.calls[0]; + expect(sql).toMatch(/COUNT\(\s*DISTINCT\s+message_id\s*\)/i); + expect(sql).not.toMatch(/COUNT\(\s*\*\s*\)/); // raw chunk rows would inflate message_count + expect(sql).toMatch(/FROM\s+embeddings/i); + expect(sql).toMatch(/generation_id\s*=\s*\$1/); + expect(params).toEqual([7]); + }); +}); diff --git a/backend/src/services/embeddings/generations.integration.test.js b/backend/src/services/embeddings/generations.integration.test.js new file mode 100644 index 0000000..b1d4a5e --- /dev/null +++ b/backend/src/services/embeddings/generations.integration.test.js @@ -0,0 +1,60 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('generations lifecycle', () => { + let gens, store, client, acctId, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + gens = await import('./generations.js'); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // Clean slate for generations (integration DB is shared across tasks). embed_runs + // FKs index_generations without ON DELETE CASCADE, so clear it before the parent. + await client.query('DELETE FROM embeddings'); + await client.query('DELETE FROM embed_runs'); + await client.query('DELETE FROM index_generations'); + ({ accountId: acctId, userId } = await seedAccount(client, 'gen')); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('creates a building generation, then a same-fingerprint create resumes it', async () => { + const g1 = await gens.createGeneration('m', 4, 'fp-A'); + const g2 = await gens.createGeneration('m', 4, 'fp-A'); + expect(g2).toBe(g1); + const building = await gens.buildingGeneration(); + expect(building.id).toBe(g1); + }); + + it('rejects a mismatched-fingerprint create while building', async () => { + await expect(gens.createGeneration('m', 4, 'fp-B')).rejects.toBeInstanceOf(gens.BuildingInProgressError); + }); + + it('blocks activation while a live message still needs embedding', async () => { + const g = (await gens.buildingGeneration()).id; + await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 82001, 'INBOX', 'needs work')`, [acctId]); + await expect(gens.activateGeneration(g)).rejects.toThrow(/needing embedding/); + }); + + it('activates once coverage is complete and deletes retired rows on the next activate', async () => { + const g = (await gens.buildingGeneration()).id; + await client.query('UPDATE messages SET embed_gen = $1 WHERE account_id = $2', [g, acctId]); + await gens.activateGeneration(g); + expect((await gens.activeGeneration()).id).toBe(g); + + // New generation covering the same corpus; activating it must delete g's rows. + await client.query(`INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, dimension, embedding) + SELECT $1, id, 0, 0, 1, 4, '[1,0,0,0]' FROM messages WHERE account_id = $2`, [g, acctId]); + const g3 = await gens.createGeneration('m', 4, 'fp-C'); + await client.query('UPDATE messages SET embed_gen = $1 WHERE account_id = $2', [g3, acctId]); + await gens.activateGeneration(g3); + const left = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1', [g]); + expect(left.rows[0].n).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/generations.js b/backend/src/services/embeddings/generations.js new file mode 100644 index 0000000..79d03e9 --- /dev/null +++ b/backend/src/services/embeddings/generations.js @@ -0,0 +1,145 @@ +import { pool, withTransaction } from '../db.js'; +import { ensureVectorIndex } from './vectorStore.js'; + +export class BuildingInProgressError extends Error {} +export class NoActiveGenerationError extends Error {} + +const LIVE = 'is_deleted = false'; + +// Coverage gate predicate: a generation is fully covered when NO live message still +// needs embedding for it. Port of backend.go missingForGenExistsClause. +export async function coverageMissing(gen) { + const r = await pool.query( + `SELECT EXISTS (SELECT 1 FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $1) AND ${LIVE}) missing`, + [gen], + ); + return r.rows[0].missing; +} + +// activated_at/started_at are epoch SECONDS (bigint); get_stats' generation +// summaries surface them (active → activated_at, building → started_at), so both +// must be selected here or vectorStats only ever sees undefined and the wire +// fields collapse to "". node-pg returns bigint as a string; the wire formatter +// in vectorStats coerces + renders RFC3339. +async function generationByState(state) { + const r = await pool.query( + `SELECT id, model, dimension, fingerprint, state, message_count AS "messageCount", + activated_at AS "activatedAt", started_at AS "startedAt" + FROM index_generations WHERE state = $1`, + [state], + ); + return r.rows[0] || null; +} + +export async function activeGeneration() { return generationByState('active'); } +export async function buildingGeneration() { return generationByState('building'); } + +// Distinct embedded-message count for a generation — a direct port of msgvault's +// Backend.Stats EmbeddingCount (internal/vector/pgvector/backend.go): "distinct +// messages, not chunk rows — a long message occupies multiple rows but counts as +// one embedded message". get_stats' collectStats consumes it for both +// active_generation.message_count and the building generation's progress.done, +// so COUNT(DISTINCT message_id) is required — COUNT(*) would inflate the count by +// every extra chunk a long message carries (live: 18,193 chunk rows vs 18,192 +// distinct messages → message_count 18,192, matching the index_generations +// .message_count column upsert maintains). Scoped to one generation_id (msgvault's +// gen != 0 path); the aggregate gen == 0 path is unused here. Kept named +// `chunkCount` per the frozen cross-phase contract (README vectorStore bullet) +// even though the value is a message count, not a chunk count. +export async function chunkCount(gen) { + const r = await pool.query( + 'SELECT COUNT(DISTINCT message_id)::bigint AS n FROM embeddings WHERE generation_id = $1', + [gen], + ); + return Number(r.rows[0].n); +} + +// Claim-or-insert a building generation. A same-fingerprint building row is resumed +// (returns its id); a mismatched building fingerprint → BuildingInProgressError. Ensures +// the per-dimension HNSW index exists first (port of backend.go CreateGeneration). +export async function createGeneration(model, dim, fingerprint) { + await ensureVectorIndex(dim); + const fp = fingerprint || `${model}:${dim}`; + const now = Math.floor(Date.now() / 1000); + const existing = await buildingGeneration(); + if (existing) { + if (existing.fingerprint !== fp) { + throw new BuildingInProgressError(`building fingerprint=${existing.fingerprint}, requested=${fp} — activate or retire it first`); + } + return existing.id; + } + try { + return await withTransaction(async (client) => { + await client.query('SET LOCAL statement_timeout = 0'); // corpus-size reset may exceed 30s + const r = await client.query( + `INSERT INTO index_generations (model, dimension, fingerprint, started_at, seeded_at, state) + VALUES ($1,$2,$3,$4,$4,'building') RETURNING id`, + [model, dim, fp, now], + ); + // Reset every live row's stale stamp so scanForEmbedding (embed_gen IS NULL) + // finds each row that must be (re)embedded under the new generation — the + // invariant that lets the scan use the partial pending index. Atomic with the + // insert so a crash never strands prior-generation rows the NULL-only scan can't + // see. Skips already-NULL rows; only fires the last_modified trigger's WHEN + // clause on subject/body columns, so a pure embed_gen reset never bumps it. + await client.query('UPDATE messages SET embed_gen = NULL WHERE embed_gen IS NOT NULL AND is_deleted = false'); + return r.rows[0].id; + }); + } catch (err) { + if (err.code === '23505') { // unique_violation on idx_generations_building — a race + const raced = await buildingGeneration(); + if (raced && raced.fingerprint === fp) return raced.id; + throw new BuildingInProgressError(`building generation exists with a different fingerprint`); + } + throw err; + } +} + +// Retire the current active (if any) and promote gen. The auto-retire DELETEs the +// demoted generation's rows in the same tx (shared per-dimension HNSW graph, README +// invariant). Coverage gate folded into the promote UPDATE unless force. +export async function activateGeneration(gen, force = false) { + const now = Math.floor(Date.now() / 1000); + await withTransaction(async (client) => { + await client.query('SET LOCAL statement_timeout = 0'); // corpus-size DELETE may exceed 30s + const demoted = await client.query( + `UPDATE index_generations SET state = 'retired', completed_at = COALESCE(completed_at, $1) + WHERE state = 'active' RETURNING id`, + [now], + ); + if (demoted.rows.length) { + await client.query('DELETE FROM embeddings WHERE generation_id = $1', [demoted.rows[0].id]); + } + const res = await client.query( + `UPDATE index_generations + SET state = 'active', activated_at = $1, completed_at = COALESCE(completed_at, $2) + WHERE id = $3 AND state = 'building' + AND ($4 OR NOT EXISTS (SELECT 1 FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $3) AND ${LIVE}))`, + [now, now, gen, force], + ); + if (res.rowCount === 0) { + const st = await client.query('SELECT state FROM index_generations WHERE id = $1', [gen]); + if (!st.rows.length) throw new Error(`unknown generation ${gen}`); + if (st.rows[0].state !== 'building') throw new Error(`generation ${gen} not in 'building' state`); + throw new Error(`generation ${gen} still has messages needing embedding; pass force to override`); + } + }); +} + +// Mark gen retired and DELETE its rows. Refuses to retire an active generation unless force. +export async function retireGeneration(gen, force = false) { + await withTransaction(async (client) => { + await client.query('SET LOCAL statement_timeout = 0'); + const res = await client.query( + `UPDATE index_generations SET state = 'retired' WHERE id = $1 AND ($2 OR state != 'active')`, + [gen, force], + ); + if (res.rowCount === 0) { + const st = await client.query('SELECT state FROM index_generations WHERE id = $1', [gen]); + if (!st.rows.length) throw new Error(`unknown generation ${gen}`); + if (st.rows[0].state === 'active' && !force) throw new Error(`refusing to retire active generation ${gen} without force`); + throw new Error(`retire generation ${gen}: no rows affected (state=${st.rows[0].state})`); + } + await client.query('DELETE FROM embeddings WHERE generation_id = $1', [gen]); + }); +} diff --git a/backend/src/services/embeddings/generations.summaryFields.test.js b/backend/src/services/embeddings/generations.summaryFields.test.js new file mode 100644 index 0000000..1e5b9f6 --- /dev/null +++ b/backend/src/services/embeddings/generations.summaryFields.test.js @@ -0,0 +1,38 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// generationByState feeds get_stats' active/building generation summaries. It must +// SELECT the epoch-seconds timestamp columns those summaries emit — activated_at +// (active) and started_at (building) — or vectorStats only ever sees undefined and +// the wire fields collapse to "". (Same family as the chunkCount gap.) +vi.mock('../db.js', () => ({ pool: { query: vi.fn() }, withTransaction: vi.fn() })); +import { pool } from '../db.js'; +import { activeGeneration, buildingGeneration } from './generations.js'; + +beforeEach(() => { pool.query.mockReset(); }); + +describe('generationByState surfaces the timestamp columns get_stats emits', () => { + it('activeGeneration SELECTs activated_at + started_at (camelCase) and returns them', async () => { + pool.query.mockResolvedValueOnce({ rows: [{ + id: 1, model: 'm', dimension: 4, fingerprint: 'f', state: 'active', + messageCount: 5, activatedAt: 1784235357, startedAt: 1784200000, + }] }); + const g = await activeGeneration(); + const [sql, params] = pool.query.mock.calls[0]; + expect(sql).toMatch(/activated_at AS "activatedAt"/); + expect(sql).toMatch(/started_at AS "startedAt"/); + expect(params).toEqual(['active']); + expect(g.activatedAt).toBe(1784235357); + expect(g.startedAt).toBe(1784200000); + }); + + it('buildingGeneration uses the same SELECT so started_at is surfaced', async () => { + pool.query.mockResolvedValueOnce({ rows: [{ + id: 2, model: 'm', dimension: 4, fingerprint: 'f', state: 'building', + messageCount: 0, startedAt: 1784200000, + }] }); + const g = await buildingGeneration(); + expect(pool.query.mock.calls[0][0]).toMatch(/started_at AS "startedAt"/); + expect(pool.query.mock.calls[0][1]).toEqual(['building']); + expect(g.startedAt).toBe(1784200000); + }); +}); diff --git a/backend/src/services/embeddings/hybrid.js b/backend/src/services/embeddings/hybrid.js new file mode 100644 index 0000000..8d1cfc6 --- /dev/null +++ b/backend/src/services/embeddings/hybrid.js @@ -0,0 +1,135 @@ +import { resolveEmbedConfig, generationFingerprint } from './config.js'; +import { resolveActiveGeneration, resolveActiveGenerationFromConfig } from './activeGeneration.js'; +import { VectorUnavailableError } from './vectorErrors.js'; +import { EmbeddingClient } from './client.js'; +import { fusedSearch } from './vectorStore.js'; + +const RRF_K = 60; +const K_PER_SIGNAL = 100; +const SUBJECT_BOOST = 2.0; + +export class MissingFreeTextError extends Error { + constructor() { super('missing_free_text'); this.name = 'MissingFreeTextError'; this.code = 'MISSING_FREE_TEXT'; } +} + +const ORCHESTRATOR_DEFAULTS = { + resolveEmbedConfig, + generationFingerprint, // sync (cfg) => string + resolveActiveGeneration, // throws VectorUnavailableError + resolveActiveGenerationFromConfig, + makeClient: (cfg) => new EmbeddingClient(cfg), + fusedSearch, +}; + +// Port of msgvault's internal/vector/hybrid/{engine,rrf}.go orchestration: +// embed the free text once, resolve the active generation by fingerprint, +// dispatch to fusedSearch (mode:'vector' skips the BM25 leg), then rerank in +// JS. Every degradation (disabled/absent embed config, stale/building/no +// generation, a transient embed failure) collapses to VectorUnavailableError +// so searchService has one predicate (isLexicalFallback) to decide whether +// to fall back silently (REST) or raise (MCP strictVector). +export async function hybridSearch(req, overrides = {}) { + const d = { ...ORCHESTRATOR_DEFAULTS, ...overrides }; + const { mode, accountIds, buildFilters, limit } = req; + const freeText = (req.freeText || '').trim(); + if (!freeText) throw new MissingFreeTextError(); + + // One owner (activeGeneration.js) resolves the embed config, rejects a disabled/absent + // one as VectorUnavailableError('vector_not_enabled'), and resolves the active generation + // by fingerprint (throws index_stale | index_building | no_active_generation). Fakes thread + // through `d`. + const { cfg, generation } = await d.resolveActiveGenerationFromConfig(d); + + let queryVec; + try { + const vecs = await d.makeClient(cfg).embed([freeText]); + if (!Array.isArray(vecs) || vecs.length !== 1) { + throw new Error(`embedder returned ${vecs && vecs.length} vectors, want 1`); + } + queryVec = vecs[0]; + } catch (err) { + if (err instanceof VectorUnavailableError) throw err; + // Embed-step failure (network/timeout/bad upstream response). The config + // already resolved as ENABLED above, so this is transient unavailability, + // not "vector search is not configured" — msgvault's engine.go wraps it + // as ErrEmbeddingTimeout the same way. The reason string is a frozen + // contract with the MCP error mapper (mcp/vectorErrors.js). REST behavior + // is unchanged: isLexicalFallback still matches, so the route silently + // falls back to lexical with fellBack:true. + throw new VectorUnavailableError('embedding_timeout'); + } + + const subjectTerms = freeText.toLowerCase().split(/\s+/).filter(w => w.length >= 2); + // The subject boost is a lexical-signal nudge — msgvault's vector-only path + // never applies it, and it "has no meaning without a BM25 + // leg to nudge against". `wantBoost` only decides whether to fetch a WIDENED + // pool (so the boost can reorder without the SQL LIMIT pre-cutting + // candidates); whether the boost actually runs is gated below on the BM25 + // leg having contributed a hit. + const wantBoost = mode === 'hybrid' && SUBJECT_BOOST > 1.0 && subjectTerms.length > 0; + const sqlLimit = wantBoost ? Math.max(2 * K_PER_SIGNAL, limit) : limit; + + const { hits, poolSaturated, generation: gen } = await d.fusedSearch({ + ftsQuery: mode === 'vector' ? null : freeText, + queryVec, + generation, + accountIds, + buildFilters, + rrfK: RRF_K, + kPerSignal: K_PER_SIGNAL, + limit: sqlLimit, + }); + + // Gate the boost on the BM25 leg actually contributing (any hit with a + // non-null bm25_score). When the FTS AND-leg is empty — no message contains + // ALL query terms, the common case for paraphrase / natural-language queries + // — the fused ranking IS the pure-ANN ranking, so a subject-substring rerank + // over the widened pool would only EVICT keyword-free-subject hits that + // vector correctly surfaced. Real-corpus eval (2026-07-16) measured hybrid + // losing to pure vector on paraphrase R@5/R@20/MRR for exactly this reason. + // Gating makes hybrid ranking == vector ranking when there is no lexical + // signal, while keyword queries (BM25 leg live) keep the nudge that made + // hybrid win there. Documented divergence from msgvault (boosts uncondition- + // ally); see specs/search-overhaul/README.md "Semantic excerpt seam". + const bm25Contributed = hits.some(h => h.bm25_score != null); + let ranked = wantBoost && bm25Contributed ? applySubjectBoost(hits, subjectTerms, SUBJECT_BOOST) : hits; + if (ranked.length > limit) ranked = ranked.slice(0, limit); + return { hits: ranked, poolSaturated, generation: gen }; +} + +export function isLexicalFallback(err) { + return err instanceof VectorUnavailableError || err instanceof MissingFreeTextError; +} + +// hybrid.js is the module's public face: re-export the vector-unavailability types so +// searchService and the MCP handlers pull them from one place. +// Definition stays in the leaf modules — no vectorStore → hybrid import cycle. +export { VectorUnavailableError } from './vectorErrors.js'; +export { resolveActiveGeneration, resolveActiveGenerationFromConfig } from './activeGeneration.js'; + +// Case-insensitive UUID-ascending tie-break, matching the SQL ORDER BY. +function byScoreThenId(a, b) { + if (b.rrf_score !== a.rrf_score) return b.rrf_score - a.rrf_score; + return a.message_id < b.message_id ? -1 : a.message_id > b.message_id ? 1 : 0; +} + +// JS subject-boost rerank (port of fused.go:429-469 + rrf.go boost logic). +// Multiplies rrf_score for any hit whose subject contains one of the +// lowercased free-text terms as a case-insensitive substring, then re-sorts. +// No-op when boost <= 1 or subjectTerms is empty. +export function applySubjectBoost(hits, subjectTerms, boost) { + // Only "boost disabled" skips the whole pass (including the tie-break + // sort) — a boost-enabled call with no terms still re-sorts deterministically, + // it just boosts nothing. + if (!(boost > 1.0)) return hits; + const terms = (subjectTerms || []).map(t => (t || '').toLowerCase()).filter(Boolean); + for (const hit of hits) { + const subj = (hit.subject || '').toLowerCase(); + if (subj && terms.length && terms.some(t => subj.includes(t))) { + hit.rrf_score *= boost; + hit.subject_boosted = true; + } + } + hits.sort(byScoreThenId); + return hits; +} diff --git a/backend/src/services/embeddings/hybrid.test.js b/backend/src/services/embeddings/hybrid.test.js new file mode 100644 index 0000000..22cb2ae --- /dev/null +++ b/backend/src/services/embeddings/hybrid.test.js @@ -0,0 +1,177 @@ +import { describe, it, expect } from 'vitest'; +import { applySubjectBoost, hybridSearch, isLexicalFallback, MissingFreeTextError } from './hybrid.js'; +import { VectorUnavailableError } from './vectorErrors.js'; + +const h = (id, rrf, subject) => ({ message_id: id, rrf_score: rrf, subject }); + +describe('applySubjectBoost', () => { + it('lifts a subject-term match above an equal-scoring non-match (port TestFuse_SubjectBoost)', () => { + const hits = [h('a', 1 / 61, 'ordinary email'), h('b', 1 / 61, 'Quarterly Review meeting')]; + const out = applySubjectBoost(hits, ['meeting'], 2.0); + expect(out[0].message_id).toBe('b'); + expect(out[0].subject_boosted).toBe(true); + expect(out.find(x => x.message_id === 'a').subject_boosted).toBeUndefined(); + }); + + it('matches case-insensitively (port TestFuse_SubjectBoost_CaseInsensitive)', () => { + const out = applySubjectBoost([h('a', 1 / 61, 'MEETING Minutes')], ['meeting'], 2.0); + expect(out[0].subject_boosted).toBe(true); + }); + + it('does nothing when boost <= 1 (port TestFuse_NoBoostWhenFlagUnset)', () => { + const out = applySubjectBoost([h('a', 1 / 61, 'meeting subject')], ['meeting'], 1.0); + expect(out[0].subject_boosted).toBeUndefined(); + expect(out[0].rrf_score).toBeCloseTo(1 / 61, 12); + }); + + it('breaks ties by message_id ascending, deterministically (port TestFuse_TiedRRFScoresStableByMessageID)', () => { + for (let i = 0; i < 20; i++) { + const out = applySubjectBoost([h('m7', 0.05, 'x'), h('m3', 0.05, 'y')], [], 2.0); + expect(out.map(x => x.message_id)).toEqual(['m3', 'm7']); + } + }); +}); + +function fakes(over = {}) { + const calls = { embed: [], fused: [] }; + const base = { + resolveEmbedConfig: async () => ({ enabled: true, model: 'm', dimension: 4 }), + generationFingerprint: (cfg) => `${cfg.model}:${cfg.dimension}:test`, + resolveActiveGeneration: async () => ({ id: 1, model: 'm', dimension: 4, fingerprint: 'm:4:test', state: 'active' }), + makeClient: () => ({ embed: async (texts) => { calls.embed.push(texts); return [[1, 0, 0, 0]]; } }), + // Hit 'a' also surfaced via the FTS leg (bm25_score set) so the subject + // boost is legitimately live; 'b' is ANN-only. (The boost is gated on the + // BM25 leg contributing — see the empty-leg regression test below.) + fusedSearch: async (r) => { calls.fused.push(r); return { + hits: [{ message_id: 'a', rrf_score: 0.02, subject: 'quarterly meeting', bm25_score: 0.5, vector_score: 0.7 }, + { message_id: 'b', rrf_score: 0.03, subject: 'ordinary', bm25_score: null, vector_score: 0.8 }], + poolSaturated: false, generation: r.generation }; }, // echoes the rich generation it was given + }; + return { deps: { ...base, ...over }, calls }; +} + +describe('hybridSearch', () => { + const req = (m) => ({ mode: m, freeText: 'quarterly meeting', accountIds: ['acc'], buildFilters: () => [], limit: 10 }); + + it('vector mode calls fusedSearch with ftsQuery=null and embeds once', async () => { + const { deps, calls } = fakes(); + await hybridSearch(req('vector'), deps); + expect(calls.embed).toHaveLength(1); + expect(calls.fused[0].ftsQuery).toBeNull(); + expect(calls.fused[0].queryVec).toEqual([1, 0, 0, 0]); + }); + + it('hybrid mode passes the free text as the BM25 leg', async () => { + const { deps, calls } = fakes(); + await hybridSearch(req('hybrid'), deps); + expect(calls.fused[0].ftsQuery).toBe('quarterly meeting'); + }); + + it('applies the subject boost so a subject-term hit outranks a higher raw RRF (hybrid mode)', async () => { + const { deps } = fakes(); + const { hits } = await hybridSearch(req('hybrid'), deps); + expect(hits[0].message_id).toBe('a'); // 0.02*2 = 0.04 > 0.03 + expect(hits[0].subject_boosted).toBe(true); + }); + + it('does NOT apply the subject boost in vector-only mode — msgvault\'s vector path is pure ANN (review MINOR 1)', async () => { + const { deps } = fakes(); + const { hits } = await hybridSearch(req('vector'), deps); + const a = hits.find(h => h.message_id === 'a'); + expect(a.rrf_score).toBeCloseTo(0.02, 12); // unboosted — NOT 0.02*2 + expect(hits.every(h => h.subject_boosted === undefined)).toBe(true); + }); + + // Regression: real-corpus eval (2026-07-16) showed hybrid LOSING to pure vector on + // paraphrase queries because the BM25 AND-leg is empty (no doc has all terms) yet the + // subject boost still reranked the widened ANN pool by loose substring matches and + // EVICTED keyword-free-subject targets. Gate: an empty BM25 leg ⇒ no boost ⇒ hybrid + // ranking == vector ranking. (Documented divergence from msgvault, which boosts + // unconditionally.) + it('does NOT boost when the BM25 leg is empty — hybrid ranking == vector ranking (eval fix)', async () => { + // Pure-ANN pool: every hit's bm25_score is null (FTS matched nothing). The + // lower-ranked hit's subject contains both query terms; it must NOT be lifted. + const annOnly = async (r) => ({ + hits: [ + { message_id: 'top', rrf_score: 1 / 61, subject: 'no relevant words in here', bm25_score: null, vector_score: 0.90 }, + { message_id: 'low', rrf_score: 1 / 62, subject: 'quarterly meeting notes', bm25_score: null, vector_score: 0.80 }, + ], + poolSaturated: false, generation: r.generation, + }); + const { deps } = fakes({ fusedSearch: annOnly }); + const { hits } = await hybridSearch(req('hybrid'), deps); + expect(hits.map(h => h.message_id)).toEqual(['top', 'low']); // ANN order preserved, no eviction + expect(hits.every(h => !h.subject_boosted)).toBe(true); // nothing boosted + }); + + it('still boosts when the BM25 leg contributed at least one hit (keyword behavior preserved)', async () => { + // Mixed pool: 'low' also surfaced via the FTS leg (bm25_score set), so the lexical + // signal is live and the subject nudge legitimately applies. + const mixed = async (r) => ({ + hits: [ + { message_id: 'top', rrf_score: 1 / 61, subject: 'no relevant words in here', bm25_score: null, vector_score: 0.90 }, + { message_id: 'low', rrf_score: 1 / 62, subject: 'quarterly meeting notes', bm25_score: 0.5, vector_score: 0.80 }, + ], + poolSaturated: false, generation: r.generation, + }); + const { deps } = fakes({ fusedSearch: mixed }); + const { hits } = await hybridSearch(req('hybrid'), deps); + expect(hits[0].message_id).toBe('low'); // (1/62)*2 ≈ 0.0323 > 1/61 ≈ 0.0164 + expect(hits[0].subject_boosted).toBe(true); + }); + + it('rejects a filter-only (no free text) query with MissingFreeTextError', async () => { + const { deps } = fakes(); + const err = await hybridSearch({ ...req('hybrid'), freeText: ' ' }, deps).catch(e => e); + expect(err).toBeInstanceOf(MissingFreeTextError); + expect(isLexicalFallback(err)).toBe(true); + }); + + it('treats a disabled embed config as VectorUnavailableError(vector_not_enabled)', async () => { + const { deps } = fakes({ resolveEmbedConfig: async () => ({ enabled: false }) }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err).toBeInstanceOf(VectorUnavailableError); + expect(err.reason).toBe('vector_not_enabled'); + expect(isLexicalFallback(err)).toBe(true); + }); + + it('treats a missing (null) embed config as VectorUnavailableError(vector_not_enabled)', async () => { + const { deps } = fakes({ resolveEmbedConfig: async () => null }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err.reason).toBe('vector_not_enabled'); + }); + + it('wraps a transient embed failure as VectorUnavailableError(embedding_timeout) — NOT vector_not_enabled (Wave D Fix 6)', async () => { + // The config was already resolved as enabled by this point, so a failed + // embed call is an upstream/transient event (msgvault engine.go wraps it + // as ErrEmbeddingTimeout), not a "vector search is not configured" one. + const { deps } = fakes({ makeClient: () => ({ embed: async () => { throw new Error('econnrefused'); } }) }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err).toBeInstanceOf(VectorUnavailableError); + expect(err.reason).toBe('embedding_timeout'); + expect(isLexicalFallback(err)).toBe(true); // REST still falls back to lexical silently + }); + + it('wraps a malformed embedder response as embedding_timeout too (embed-step failure)', async () => { + const { deps } = fakes({ makeClient: () => ({ embed: async () => [] }) }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err.reason).toBe('embedding_timeout'); + }); + + it('propagates a stale-index VectorUnavailableError from the resolver', async () => { + const stale = new VectorUnavailableError('index_stale'); + const { deps } = fakes({ resolveActiveGeneration: async () => { throw stale; } }); + const err = await hybridSearch(req('hybrid'), deps).catch(e => e); + expect(err).toBe(stale); + expect(isLexicalFallback(err)).toBe(true); + }); +}); + +describe('hybrid.js public re-exports', () => { + it('exposes VectorUnavailableError + resolveActiveGeneration from ./hybrid.js (phase-5 imports them here)', async () => { + const mod = await import('./hybrid.js'); + expect(typeof mod.resolveActiveGeneration).toBe('function'); + expect(typeof mod.VectorUnavailableError).toBe('function'); + expect(new mod.VectorUnavailableError('index_stale').reason).toBe('index_stale'); + }); +}); diff --git a/backend/src/services/embeddings/migration0038.integration.test.js b/backend/src/services/embeddings/migration0038.integration.test.js new file mode 100644 index 0000000..a83de0c --- /dev/null +++ b/backend/src/services/embeddings/migration0038.integration.test.js @@ -0,0 +1,81 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('0038 last_modified trigger', () => { + let client; + let acctId, userId; + beforeAll(async () => { + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + ({ accountId: acctId, userId } = await seedAccount(client, 'mig')); + }); + afterAll(async () => { + await cleanupAccount(client, userId); + await client.end(); + }); + + async function insertMsg(uid) { + const r = await client.query( + `INSERT INTO messages (account_id, uid, folder, subject, body_text) + VALUES ($1, $2, 'INBOX', 'orig subject', NULL) RETURNING id, last_modified`, + [acctId, uid] + ); + return r.rows[0]; + } + + it('does NOT bump last_modified on a flag-only UPDATE', async () => { + const m = await insertMsg(90001); + await client.query('UPDATE messages SET is_read = true WHERE id = $1', [m.id]); + const after = await client.query('SELECT last_modified FROM messages WHERE id = $1', [m.id]); + expect(after.rows[0].last_modified.getTime()).toBe(m.last_modified.getTime()); + }); + + it('bumps last_modified when body_text changes', async () => { + const m = await insertMsg(90002); + await new Promise(r => setTimeout(r, 5)); + await client.query("UPDATE messages SET body_text = 'a late body arrived' WHERE id = $1", [m.id]); + const after = await client.query('SELECT last_modified FROM messages WHERE id = $1', [m.id]); + expect(after.rows[0].last_modified.getTime()).toBeGreaterThan(m.last_modified.getTime()); + }); + + it('embed_gen defaults to NULL', async () => { + const m = await insertMsg(90003); + const r = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [m.id]); + expect(r.rows[0].embed_gen).toBeNull(); + }); + + it('clears embed_gen (re-surfaces for embedding) when body_text changes post-stamp', async () => { + const m = await insertMsg(90004); + await client.query('UPDATE messages SET embed_gen = 12345 WHERE id = $1', [m.id]); // embedded + stamped + await client.query("UPDATE messages SET body_text = 'a late body arrived' WHERE id = $1", [m.id]); // late body + const r = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [m.id]); + expect(r.rows[0].embed_gen).toBeNull(); // the stale subject-only embedding must be re-done + }); + + it('does NOT clear embed_gen on a stamp-only UPDATE (worker stamp) — trigger does not fire', async () => { + const m = await insertMsg(90005); + const before = await client.query('SELECT last_modified FROM messages WHERE id = $1', [m.id]); + await client.query('UPDATE messages SET embed_gen = 777 WHERE id = $1', [m.id]); + const after = await client.query('SELECT embed_gen, last_modified FROM messages WHERE id = $1', [m.id]); + expect(String(after.rows[0].embed_gen)).toBe('777'); // stamp persists + expect(after.rows[0].last_modified.getTime()).toBe(before.rows[0].last_modified.getTime()); // trigger didn't fire + }); + + it('does NOT clear embed_gen on an identical-content UPSERT (unchanged re-sync)', async () => { + const m = await insertMsg(90006); + await client.query('UPDATE messages SET embed_gen = 888 WHERE id = $1', [m.id]); + await client.query( + `INSERT INTO messages (account_id, uid, folder, subject, body_text) + VALUES ($1, 90006, 'INBOX', 'orig subject', NULL) + ON CONFLICT (account_id, uid, folder) + DO UPDATE SET subject = EXCLUDED.subject, body_text = EXCLUDED.body_text`, + [acctId] + ); + const r = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [m.id]); + expect(String(r.rows[0].embed_gen)).toBe('888'); // unchanged content → trigger no-op → stamp survives + }); +}); diff --git a/backend/src/services/embeddings/postStampReembed.integration.test.js b/backend/src/services/embeddings/postStampReembed.integration.test.js new file mode 100644 index 0000000..ebd216f --- /dev/null +++ b/backend/src/services/embeddings/postStampReembed.integration.test.js @@ -0,0 +1,62 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// Re-verification catch (slice 05 contract): a row embedded subject-only and stamped, +// whose body arrives LATER (phase-2 drainer / on-open fetch — the primary product flow), +// must re-surface for embedding. The 0038 trigger clears embed_gen on a content change, +// so the NULL-only scan finds it and the idempotent upsert replaces the stale chunks. +d('post-stamp late-body re-embed (end-to-end)', () => { + let store, generations, worker, client, acctId, userId, gen, msgId; + const DIM = 4; + // Vector encodes the source length so a re-embed with a longer (subject+body) input is + // observably different from the subject-only embedding. + const fakeClient = { async embed(inputs) { return inputs.map((t) => [t.length, 0, 0, 0]); } }; + + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + generations = await import('./generations.js'); + const { EmbeddingWorker } = await import('./worker.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(DIM); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'restale')); + const m = await client.query(`INSERT INTO messages (account_id, uid, folder, subject, body_text) VALUES ($1, 84001, 'INBOX', 'meeting notes', NULL) RETURNING id`, [acctId]); + msgId = m.rows[0].id; + gen = await generations.createGeneration('fake', DIM, 'fp-restale'); + worker = new EmbeddingWorker({ store, client: fakeClient, generations, preprocessCfg: {}, maxInputChars: 32768, batchSize: 8 }); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('re-embeds a stamped row after a late body arrives (trigger clears embed_gen)', async () => { + // 1. Subject-only embed + stamp. + await worker.runOnce(gen); + const stamped = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [msgId]); + expect(String(stamped.rows[0].embed_gen)).toBe(String(gen)); + const first = await client.query('SELECT source_char_len FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, msgId]); + expect(first.rows.length).toBe(1); + const subjectOnlyLen = first.rows[0].source_char_len; + + // 2. Late body arrives — the trigger must clear embed_gen so the scan re-finds it. + await client.query("UPDATE messages SET body_text = 'a much longer body that arrived after the subject-only embedding was already stamped' WHERE id = $1", [msgId]); + const cleared = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [msgId]); + expect(cleared.rows[0].embed_gen).toBeNull(); + const pending = await store.scanForEmbedding(gen, store.ZERO_UUID, 10); + expect(pending).toContain(msgId); + + // 3. Re-embed replaces the stale chunk (idempotent upsert) with the longer source text. + await worker.runOnce(gen); + const restamped = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [msgId]); + expect(String(restamped.rows[0].embed_gen)).toBe(String(gen)); + const after = await client.query('SELECT source_char_len FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, msgId]); + expect(after.rows.length).toBe(1); // one chunk — replaced, not duplicated + expect(after.rows[0].source_char_len).toBeGreaterThan(subjectOnlyLen); // re-embedded WITH the body + }); +}); diff --git a/backend/src/services/embeddings/preprocess.js b/backend/src/services/embeddings/preprocess.js new file mode 100644 index 0000000..5393464 --- /dev/null +++ b/backend/src/services/embeddings/preprocess.js @@ -0,0 +1,105 @@ +// Port of internal/vector/embed/preprocess.go. IMPORTANT: any change here that shifts +// output for unchanged flags MUST bump PREPROCESS_VERSION in config.js (folds into the +// generation fingerprint). + +const reReplyPreamble = /^On [^\n]+wrote:\s*\n(?:>+[ \t]?.*\n?)+/gm; +const reSigDelim = /\n--\s*\n[\s\S]*$/; +const reQuoteLine = /^>+[ \t]?.*\n?/gm; +const reStyleBlock = /]*>.*?<\/style>/gis; +const reScriptBlock = /]*>.*?<\/script>/gis; +const reHTMLTag = /<\/?[a-z][a-zA-Z0-9-]*(?:\s+[a-zA-Z_:][a-zA-Z0-9_:.-]*\s*=\s*(?:"[^"]{0,400}"|'[^']{0,400}'|[^\s>"']{1,400}))*\s*\/?>/g; +const reDataURI = /data:[a-zA-Z0-9./+-]{0,128};base64,[A-Za-z0-9+/]+={0,2}/gi; +const reBase64Blob = /[A-Za-z0-9+]{200,}={0,2}/g; +const reBase64BlobWithSlash = /[A-Za-z0-9+/]{300,}={0,2}/g; +const reURL = /https?:\/\/[^\s"'<>)]+/g; +const reTrailingHWS = /[ \t]+$/gm; +const reMultiNewline = /\n{3,}/g; +const reHorizontalRun = /[ \t]{2,}/g; + +const TRACKING_PARAMS = new Set([ + 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id', + 'utm_name', 'utm_brand', 'utm_social', 'fbclid', 'gclid', 'dclid', 'gbraid', + 'wbraid', 'msclkid', 'yclid', 'twclid', 'mc_cid', 'mc_eid', 'ml_subscriber', + '_hsenc', '_hsmi', 'hsctatracking', 'vero_conv', 'vero_id', 'ck_subscriber_id', + '_branch_match_id', 'ref', 'ref_src', 's_cid', 'icid', 'spm', +]); + +// Compact analogue of html.UnescapeString: the named entities seen in body_text prose +// plus numeric (decimal/hex) references. PREPROCESS_VERSION pins this behavior. +const NAMED_ENTITIES = { + amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", '#39': "'", nbsp: ' ', + copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', + ndash: '–', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', +}; +function decodeEntities(s) { + return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (m, ent) => { + if (ent[0] === '#') { + const cp = ent[1] === 'x' || ent[1] === 'X' ? parseInt(ent.slice(2), 16) : parseInt(ent.slice(1), 10); + if (Number.isFinite(cp) && cp > 0 && cp <= 0x10ffff) { try { return String.fromCodePoint(cp); } catch { return m; } } + return m; + } + const v = NAMED_ENTITIES[ent]; + return v !== undefined ? v : m; + }); +} + +function stripTrackingParams(s) { + return s.replace(reURL, (raw) => { + let trailing = ''; + while (raw.length && '.,;:!?)]'.includes(raw[raw.length - 1])) { + trailing = raw[raw.length - 1] + trailing; + raw = raw.slice(0, -1); + } + let u; + try { u = new URL(raw); } catch { return raw + trailing; } + if (!u.host) return raw + trailing; + let dropped = false; + for (const k of [...u.searchParams.keys()]) { + if (TRACKING_PARAMS.has(k.toLowerCase())) { u.searchParams.delete(k); dropped = true; } + } + if (!dropped) return raw + trailing; + return u.toString() + trailing; + }); +} + +function capToRunes(s, maxRunes) { + // Cap s to maxRunes code points without materializing a full array for huge inputs. + let count = 0, u16 = 0; + for (const ch of s) { + if (count >= maxRunes) return { text: s.slice(0, u16), truncated: true }; + count++; u16 += ch.length; + } + return { text: s, truncated: false }; +} + +export function preprocess(subject, body, maxChars, cfg = {}) { + let s = String(body).replace(/\r\n/g, '\n'); + let bodyTruncated = false; + + if (cfg.stripBase64) { + s = s.replace(reDataURI, ' ').replace(reBase64Blob, ' ').replace(reBase64BlobWithSlash, ' '); + } + if (cfg.maxBodyRunes > 0) { + const capped = capToRunes(s, cfg.maxBodyRunes); + s = capped.text; bodyTruncated = capped.truncated; + } + if (cfg.stripHTML) { + s = s.replace(reStyleBlock, ' ').replace(reScriptBlock, ' ').replace(reHTMLTag, ' '); + s = decodeEntities(s); + } + if (cfg.stripURLTracking) s = stripTrackingParams(s); + if (cfg.stripQuotes) s = s.replace(reReplyPreamble, '').replace(reQuoteLine, ''); + if (cfg.stripSignatures) s = s.replace(reSigDelim, ''); + if (cfg.collapseWhitespace) { + s = s.replace(reTrailingHWS, '').replace(reHorizontalRun, ' ').replace(reMultiNewline, '\n\n'); + } + s = s.trim(); + + const prefix = subject ? `Subject: ${subject}\n\n` : ''; + const combined = prefix + s; + + if (maxChars <= 0) return { text: combined, truncated: bodyTruncated }; + const cps = Array.from(combined); + if (cps.length <= maxChars) return { text: combined, truncated: bodyTruncated }; + return { text: cps.slice(0, maxChars).join(''), truncated: true }; +} diff --git a/backend/src/services/embeddings/preprocess.test.js b/backend/src/services/embeddings/preprocess.test.js new file mode 100644 index 0000000..76fbafc --- /dev/null +++ b/backend/src/services/embeddings/preprocess.test.js @@ -0,0 +1,87 @@ +import { describe, it, expect } from 'vitest'; +import { preprocess } from './preprocess.js'; + +const cases = [ + { name: 'PlainBody', subject: 'Hello', body: "Hi there,\n\nLet's chat tomorrow.", maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: true }, want: "Subject: Hello\n\nHi there,\n\nLet's chat tomorrow.", wantTrunc: false }, + { name: 'StripsQuotedPreamble', subject: 'Re: plan', body: 'My reply.\n\nOn 2026-01-01, alice wrote:\n> previous message\n> more quote', maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: true }, want: 'Subject: Re: plan\n\nMy reply.', wantTrunc: false }, + { name: 'StripsStandaloneQuoteLines', subject: '', body: '> nested quote 1\n> nested quote 2\nActual content.', maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: false }, want: 'Actual content.', wantTrunc: false }, + { name: 'StripsSignature', subject: 'Hi', body: 'Body here.\n-- \nBob\nPhone: ...', maxChars: 1000, + cfg: { stripQuotes: true, stripSignatures: true }, want: 'Subject: Hi\n\nBody here.', wantTrunc: false }, + { name: 'SignatureWithoutTrailingSpace', subject: 'Hi', body: 'Body.\n--\nBob', maxChars: 1000, + cfg: { stripQuotes: false, stripSignatures: true }, want: 'Subject: Hi\n\nBody.', wantTrunc: false }, + { name: 'Truncates', subject: 'S', body: 'x'.repeat(2000), maxChars: 100, cfg: {}, lenLE: 100, wantTrunc: true }, + { name: 'TruncateAtRuneBoundary', subject: '', body: '☃'.repeat(50), maxChars: 10, cfg: {}, lenLE: 30, wantTrunc: true }, + { name: 'EmptySubjectAndBody', subject: '', body: '', maxChars: 1000, cfg: {}, want: '', wantTrunc: false }, + { name: 'NoConfigPreservesEverything', subject: 'Hi', body: 'Hello\n> not stripped\n-- \nsig kept', maxChars: 1000, + cfg: { stripQuotes: false, stripSignatures: false }, want: 'Subject: Hi\n\nHello\n> not stripped\n-- \nsig kept', wantTrunc: false }, + { name: 'MaxCharsZeroIsUnlimited', subject: 'S', body: 'x'.repeat(100), maxChars: 0, cfg: {}, want: 'Subject: S\n\n' + 'x'.repeat(100), wantTrunc: false }, + { name: 'MultiByteRunesUnderCap', subject: '', body: '☃'.repeat(33), maxChars: 100, cfg: {}, lenLE: 100, wantTrunc: false }, + { name: 'StripsNestedQuotes', subject: '', body: '>> deep quote\n>>> deeper quote\nReal content.', maxChars: 1000, + cfg: { stripQuotes: true }, want: 'Real content.', wantTrunc: false }, + { name: 'StripsQuoteWithoutSpace', subject: '', body: '>no space after caret\n>another\nKept content.', maxChars: 1000, + cfg: { stripQuotes: true }, want: 'Kept content.', wantTrunc: false }, + { name: 'StripsPreambleWithNestedQuotes', subject: 'Re: topic', body: 'My reply.\n\nOn 2026-04-18, bob wrote:\n>> prior reply\n> and response\n>> more', maxChars: 1000, + cfg: { stripQuotes: true }, want: 'Subject: Re: topic\n\nMy reply.', wantTrunc: false }, + { name: 'StripHTMLDropsTagsAndDecodesEntities', subject: '', body: '

Hello & goodbye


End.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Hello & goodbye End.', wantTrunc: false }, + { name: 'StripHTMLPreservesAngleBracketEmailAddress', subject: '', body: 'Please CC John on the next reply.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Please CC John on the next reply.', wantTrunc: false }, + { name: 'StripHTMLPreservesAngleBracketURL', subject: '', body: 'See .', maxChars: 1000, + cfg: { stripHTML: true }, want: 'See .', wantTrunc: false }, + { name: 'StripHTMLPreservesMathPunctuation', subject: '', body: 'Show me rows where x < 3 and y > 4 but z != 0.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Show me rows where x < 3 and y > 4 but z != 0.', wantTrunc: false }, + { name: 'StripHTMLPreservesAngleBracketDatePlaceholder', subject: '', body: 'Schedule for still good?', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Schedule for still good?', wantTrunc: false }, + { name: 'StripHTMLPreservesMailMergePlaceholder', subject: '', body: 'Hi , welcome to .', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Hi , welcome to .', wantTrunc: false }, + { name: 'StripHTMLDropsSingleQuotedAttrValue', subject: '', body: "Click here now.", maxChars: 1000, + cfg: { stripHTML: true }, want: 'Click here now.', wantTrunc: false }, + { name: 'StripHTMLDropsUnquotedAttrValues', subject: '', body: 'Pixel end.', maxChars: 1000, + cfg: { stripHTML: true }, want: 'Pixel end.', wantTrunc: false }, + { name: 'StripHTMLDropsStyleBlock', subject: '', body: 'Hello\n\nWorld', maxChars: 1000, + cfg: { stripHTML: true, collapseWhitespace: true }, want: 'Hello\n\nWorld', wantTrunc: false }, + { name: 'StripHTMLDropsScriptBlock', subject: '', body: 'Pre\n\nPost', maxChars: 1000, + cfg: { stripHTML: true, collapseWhitespace: true }, want: 'Pre\n\nPost', wantTrunc: false }, + { name: 'StripBase64DropsDataURI', subject: '', body: 'Before image data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== After image', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Before image After image', wantTrunc: false }, + { name: 'StripBase64DropsBareBlob', subject: '', body: 'Hello ' + 'A'.repeat(250) + ' world', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Hello world', wantTrunc: false }, + { name: 'StripBase64DropsBareBlobWithSlashes', subject: '', body: 'Before ' + 'abcdefghij/'.repeat(30) + ' After', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Before After', wantTrunc: false }, + { name: 'StripBase64KeepsShortAlphanumeric', subject: '', body: 'Token=abc123XYZ check this hash 0123456789abcdef0123456789abcdef.', maxChars: 1000, + cfg: { stripBase64: true }, want: 'Token=abc123XYZ check this hash 0123456789abcdef0123456789abcdef.', wantTrunc: false }, + { name: 'StripURLTrackingDropsKnownParams', subject: '', body: 'Visit https://example.com/page?utm_source=newsletter&utm_medium=email&fbclid=xyz&id=42 today', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'Visit https://example.com/page?id=42 today', wantTrunc: false }, + { name: 'StripURLTrackingHandlesHubSpotMixedCase', subject: '', body: 'Click https://hub.example.com/cta?hsCtaTracking=abc&id=7 now', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'Click https://hub.example.com/cta?id=7 now', wantTrunc: false }, + { name: 'StripURLTrackingPreservesTrailingPunctuation', subject: '', body: 'See https://example.com/x?utm_source=foo&keep=1.', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'See https://example.com/x?keep=1.', wantTrunc: false }, + { name: 'StripURLTrackingLeavesCleanURLAlone', subject: '', body: 'See https://example.com/page?id=42 and ftp://elsewhere/path', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'See https://example.com/page?id=42 and ftp://elsewhere/path', wantTrunc: false }, + { name: 'CollapseWhitespaceShrinksRuns', subject: '', body: 'Line one\n\n\n\nLine two with big gaps', maxChars: 1000, + cfg: { collapseWhitespace: true }, want: 'Line one\n\nLine two with big gaps', wantTrunc: false }, + { name: 'StripBase64KeepsLongURLPath', subject: '', body: 'https://example.com/' + 'a/b'.repeat(80) + '/end', maxChars: 2000, + cfg: { stripBase64: true }, want: 'https://example.com/' + 'a/b'.repeat(80) + '/end', wantTrunc: false }, + { name: 'StripPipelineSweepsOversizedImgTag', subject: '', body: 'Before ' + 'x' + ' After', maxChars: 5000, + cfg: { stripHTML: true, stripBase64: true, collapseWhitespace: true }, want: 'Before After', wantTrunc: false }, + { name: 'CollapseWhitespaceNormalizesCRLF', subject: '', body: 'Line one \r\n\r\n\r\n\r\nLine two', maxChars: 1000, + cfg: { collapseWhitespace: true }, want: 'Line one\n\nLine two', wantTrunc: false }, + { name: 'PipelineRemovesPollutionEndToEnd', subject: 'Newsletter', + body: '

Hello

\n\n\n' + 'data:image/gif;base64,R0lGODlhAQABAAAAACw= ' + '\n\n\nClick ' + 'https://example.com/?utm_source=x&keep=y' + '\n\n-- \nSig', + maxChars: 1000, cfg: { stripQuotes: true, stripSignatures: true, stripHTML: true, stripBase64: true, stripURLTracking: true, collapseWhitespace: true }, + want: 'Subject: Newsletter\n\nHello\n\nClick https://example.com/?keep=y', wantTrunc: false }, +]; + +describe('preprocess (msgvault fixture parity)', () => { + for (const tt of cases) { + it(tt.name, () => { + const { text, truncated } = preprocess(tt.subject, tt.body, tt.maxChars, tt.cfg); + if (tt.want !== undefined) expect(text).toBe(tt.want); + if (tt.lenLE) expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(tt.lenLE); + expect(truncated).toBe(tt.wantTrunc); + }); + } +}); diff --git a/backend/src/services/embeddings/rankingQuality.test.js b/backend/src/services/embeddings/rankingQuality.test.js new file mode 100644 index 0000000..5bc06aa --- /dev/null +++ b/backend/src/services/embeddings/rankingQuality.test.js @@ -0,0 +1,83 @@ +import { describe, it, expect, afterAll } from 'vitest'; +import { hasTestDb, withTestDb, closeTestDb } from './__testdb__.js'; +import { fusedSearch } from './vectorStore.js'; + +afterAll(async () => { await closeTestDb(); }); + +const DIM = 8; +function vec(axis, jitter = 0) { const v = new Array(DIM).fill(0); v[axis] = 1; if (jitter) v[(axis + 1) % DIM] = jitter; return v; } + +// 8 concepts. Each has a keyword doc and a synonym-only doc that shares the +// vector neighbourhood but NOT the keyword. Queries ask by keyword; the gold +// hit is the synonym-only doc that lexical alone would miss. +const CONCEPTS = [ + { kw: 'invoice', syn: 'billing statement', axis: 0 }, + { kw: 'flight', syn: 'boarding pass itinerary', axis: 1 }, + { kw: 'invoice2', syn: 'amount due remittance', axis: 2 }, + { kw: 'meeting', syn: 'calendar sync standup', axis: 3 }, + { kw: 'refund', syn: 'money back reversal', axis: 4 }, + { kw: 'password', syn: 'credential reset link', axis: 5 }, + { kw: 'shipment', syn: 'parcel tracking dispatch', axis: 6 }, + { kw: 'contract', syn: 'signed agreement terms', axis: 7 }, +]; +// 20 labeled queries (concepts reused with different jitter for the extra 12). +const QUERIES = []; +for (let i = 0; i < 20; i++) { + const c = CONCEPTS[i % CONCEPTS.length]; + QUERIES.push({ text: c.kw, qvec: vec(c.axis, (i >= 8 ? 0.15 : 0)), goldAxis: c.axis }); +} + +describe.skipIf(!hasTestDb())('ranking quality: hybrid must not lose to lexical', () => { + it('hybrid recall of semantic-only gold hits >= lexical, and never lower', async () => { + await withTestDb(async (pool) => { + const u = (await pool.query(`INSERT INTO users (username,password_hash) VALUES ('r','x') RETURNING id`)).rows[0].id; + const acc = (await pool.query( + `INSERT INTO email_accounts (user_id,name,email_address,enabled) VALUES ($1,'A','a@x.test',true) RETURNING id`, [u])).rows[0].id; + const gen = (await pool.query( + `INSERT INTO index_generations (model,dimension,fingerprint,started_at,state) VALUES ('m',$1,'m:8:test',$2,'active') RETURNING id`, + [DIM, Math.floor(Date.now() / 1000)])).rows[0].id; + + const goldByAxis = {}; + let uid = 0; + for (const c of CONCEPTS) { + // keyword doc (axis vector + keyword in subject) + await insertDoc(pool, acc, gen, ++uid, `${c.kw} notice`, vec(c.axis), DIM); + // synonym-only doc: gold semantic hit, NO keyword token + const goldId = await insertDoc(pool, acc, gen, ++uid, c.syn, vec(c.axis, 0.05), DIM); + goldByAxis[c.axis] = goldId; + } + + let hybridWins = 0, lexicalWins = 0; + for (const q of QUERIES) { + const goldId = goldByAxis[q.goldAxis]; + const lex = await fusedSearch({ ftsQuery: q.text, queryVec: null, generation: { id: gen, dimension: DIM }, + accountIds: [acc], buildFilters: () => [], rrfK: 60, kPerSignal: 100, limit: 10 }, { client: pool }); + const hyb = await fusedSearch({ ftsQuery: q.text, queryVec: q.qvec, generation: { id: gen, dimension: DIM }, + accountIds: [acc], buildFilters: () => [], rrfK: 60, kPerSignal: 100, limit: 10 }, { client: pool }); + const lexHit = lex.hits.some(h => h.message_id === goldId); + const hybHit = hyb.hits.some(h => h.message_id === goldId); + if (hybHit && !lexHit) hybridWins++; + if (lexHit && !hybHit) lexicalWins++; + // Never lose: whatever lexical surfaced, hybrid still surfaces. + for (const h of lex.hits) { + expect(hyb.hits.some(x => x.message_id === h.message_id)).toBe(true); + } + } + // Semantic-only gold hits: hybrid recovers them, lexical cannot. + expect(hybridWins).toBeGreaterThanOrEqual(QUERIES.length); + expect(lexicalWins).toBe(0); + console.log(`[ranking-quality] hybridWins=${hybridWins} lexicalWins=${lexicalWins} of ${QUERIES.length}`); + }); + }); +}); + +async function insertDoc(pool, acc, gen, uid, subject, embedding, dim) { + const id = (await pool.query( + `INSERT INTO messages (account_id,uid,folder,subject,from_name,from_email,date,snippet,body_text,is_deleted) + VALUES ($1,$2,'INBOX',$3,'S','s@x.test',now(),$3,$3,false) RETURNING id`, [acc, uid, subject])).rows[0].id; + await pool.query( + `INSERT INTO embeddings (generation_id,message_id,chunk_index,embedded_at,source_char_len,embedding,dimension) + VALUES ($1,$2,0,$3,$4,$5::vector,$6)`, + [gen, id, Math.floor(Date.now() / 1000), subject.length, `[${embedding.join(',')}]`, dim]); + return id; +} diff --git a/backend/src/services/embeddings/scheduler.js b/backend/src/services/embeddings/scheduler.js new file mode 100644 index 0000000..9afe7c2 --- /dev/null +++ b/backend/src/services/embeddings/scheduler.js @@ -0,0 +1,91 @@ +import { resolveEmbedConfig, generationFingerprint } from './config.js'; +import { isVectorAvailable } from './vectorStore.js'; +import * as store from './vectorStore.js'; +import * as generations from './generations.js'; +const { buildingGeneration, activeGeneration, retireGeneration } = generations; +import { EmbeddingClient } from './client.js'; +import { EmbeddingWorker } from './worker.js'; +import { tryAcquireEmbedRun, releaseEmbedRun } from './embedRunLock.js'; + +let _timer = null; +let _running = false; +let _lastBackstop = 0; +let _backstopIntervalMs = 86_400_000; +// Latch for the once-per-fingerprint "paused" log below — holds the config +// fingerprint we last warned about so a mismatched active generation does not +// spam the log every tick. Cleared whenever a matching run proceeds. +let _pausedForFingerprint = null; + +// One scheduler pass: drive an existing building-or-active generation. No-ops when +// vector is unavailable, embeddings are disabled, or no generation exists. Exported +// for unit testing. +export async function runSchedulerTick(nowMs) { + if (!isVectorAvailable()) return; + let cfg; + try { cfg = await resolveEmbedConfig(); } catch { return; } + if (!cfg || !cfg.enabled || !cfg.endpoint || !cfg.model || !(cfg.dimension > 0)) return; + + const building = await buildingGeneration(); + const gen = building || await activeGeneration(); + if (!gen) return; + + // Single-flight: skip this tick if a manual build — or another run — already + // holds the shared lock, so the scheduler never double-drives a generation. + if (!tryAcquireEmbedRun()) return; + try { + // Fingerprint guard: the worker embeds with a client built from the *current* + // config, so it must only drive a generation whose fingerprint matches that + // config. If the admin changed model/dimension/preprocess mid-build, the + // resolved generation is stale — driving it would embed with the new client + // into the old generation and fail the dimension check forever. + const cfgFingerprint = generationFingerprint(cfg); + if (gen.fingerprint !== cfgFingerprint) { + if (building) { + // A building gen is superseded by the config change — retire it (deletes its + // rows; generations never mix) so a fresh, correctly-fingerprinted build can + // start. Done under the single-flight lock so it can't race an embed run. + await retireGeneration(gen.id); + console.log(`[embed-scheduler] retired building generation ${gen.id}: fingerprint '${gen.fingerprint}' superseded by config '${cfgFingerprint}'`); + } else if (_pausedForFingerprint !== cfgFingerprint) { + // An active gen can't be retired from under live search — pause incremental + // embedding until a rebuild lands. Log once per config fingerprint, not every tick. + _pausedForFingerprint = cfgFingerprint; + console.log(`[embed-scheduler] active generation ${gen.id} fingerprint '${gen.fingerprint}' != config '${cfgFingerprint}' — pausing incremental embedding until a rebuild`); + } + return; + } + _pausedForFingerprint = null; // a matching run clears the paused-log latch + + const client = new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: cfg.dimension }); + // Pass `generations` so the worker can promote a fully-covered building + // generation to active at its shared run-completion seam (worker.js). + const worker = new EmbeddingWorker({ store, client, generations, preprocessCfg: cfg.preprocess, maxInputChars: cfg.maxInputChars, batchSize: cfg.batchSize }); + + if (nowMs - _lastBackstop >= _backstopIntervalMs) { + _lastBackstop = nowMs; + await worker.runBackstop(gen.id); + } else { + await worker.runOnce(gen.id); + } + } finally { + releaseEmbedRun(); + } +} + +export function startEmbeddingScheduler({ intervalMs = 60000, backstopIntervalMs = 86_400_000 } = {}) { + if (_timer) return; + _backstopIntervalMs = backstopIntervalMs; + _lastBackstop = Date.now(); // first backstop one interval out, not on boot + _timer = setInterval(async () => { + if (_running) return; // never overlap ticks + _running = true; + try { await runSchedulerTick(Date.now()); } + catch (err) { console.error(`Embedding scheduler tick error: ${err.message}`); } + finally { _running = false; } + }, intervalMs); + _timer.unref?.(); +} + +export function stopEmbeddingScheduler() { + if (_timer) { clearInterval(_timer); _timer = null; } +} diff --git a/backend/src/services/embeddings/scheduler.test.js b/backend/src/services/embeddings/scheduler.test.js new file mode 100644 index 0000000..0ca9f7b --- /dev/null +++ b/backend/src/services/embeddings/scheduler.test.js @@ -0,0 +1,118 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('./config.js', () => ({ resolveEmbedConfig: vi.fn(), generationFingerprint: vi.fn(() => 'fp') })); +vi.mock('./vectorStore.js', () => ({ isVectorAvailable: vi.fn(() => true) })); +vi.mock('./generations.js', () => ({ buildingGeneration: vi.fn(), activeGeneration: vi.fn(), retireGeneration: vi.fn() })); +const runOnce = vi.fn().mockResolvedValue({ claimed: 0 }); +const runBackstop = vi.fn().mockResolvedValue({ claimed: 0 }); +// Regular (constructable) function impls — arrow-function impls throw +// "is not a constructor" when the module under test does `new EmbeddingWorker(...)`. +vi.mock('./worker.js', () => ({ EmbeddingWorker: vi.fn(function () { return { runOnce, runBackstop }; }) })); +vi.mock('./client.js', () => ({ EmbeddingClient: vi.fn(function () {}) })); + +const { resolveEmbedConfig } = await import('./config.js'); +const { isVectorAvailable } = await import('./vectorStore.js'); +const { buildingGeneration, activeGeneration, retireGeneration } = await import('./generations.js'); +const { runSchedulerTick } = await import('./scheduler.js'); +// Real single-flight lock (not mocked) — the scheduler tick must respect it. +const { tryAcquireEmbedRun, releaseEmbedRun, isEmbedRunActive } = await import('./embedRunLock.js'); + +const enabledCfg = { enabled: true, endpoint: 'http://h/v1', model: 'm', dimension: 4, maxInputChars: 32768, batchSize: 32, preprocess: {} }; + +beforeEach(() => { + releaseEmbedRun(); // ensure the shared single-flight lock starts free each test + // Reset the once-queues so an early-returning test can't leave an unconsumed + // mockResolvedValueOnce that poisons a later test's expectations. + resolveEmbedConfig.mockReset(); + buildingGeneration.mockReset(); + activeGeneration.mockReset(); + retireGeneration.mockReset(); + isVectorAvailable.mockReset().mockReturnValue(true); + runOnce.mockClear(); + runBackstop.mockClear(); +}); + +describe('runSchedulerTick', () => { + it('no-ops when vector is unavailable', async () => { + isVectorAvailable.mockReturnValueOnce(false); + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('no-ops when embeddings config is disabled', async () => { + resolveEmbedConfig.mockResolvedValueOnce({ ...enabledCfg, enabled: false }); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('no-ops when there is no building or active generation', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce(null); + activeGeneration.mockResolvedValueOnce(null); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('runs the worker against the building generation', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + expect(runOnce).toHaveBeenCalledWith('9'); + }); + + it('skips when a run is already active (manual build in flight) — C1', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + expect(tryAcquireEmbedRun()).toBe(true); // simulate a manual build holding the lock + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + releaseEmbedRun(); + }); + + it('releases the single-flight lock after running (does not starve later runs)', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + expect(runOnce).toHaveBeenCalledWith('9'); + expect(isEmbedRunActive()).toBe(false); + }); + + // Fingerprint guard: the config's fingerprint must match the generation the worker + // would drive, or a model/dimension change mid-build embeds with the new client into + // the old generation and wedges forever. + it('retires a building generation whose fingerprint no longer matches the config', async () => { + resolveEmbedConfig.mockResolvedValue(enabledCfg); + buildingGeneration.mockResolvedValue({ id: '9', dimension: 4, fingerprint: 'old-fp' }); + await runSchedulerTick(1000); + expect(retireGeneration).toHaveBeenCalledWith('9'); + expect(runOnce).not.toHaveBeenCalled(); + expect(isEmbedRunActive()).toBe(false); // lock released on the retire-and-return path + }); + + it('skips a mismatched active generation and logs once per fingerprint (no retire)', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + resolveEmbedConfig.mockResolvedValue(enabledCfg); + buildingGeneration.mockResolvedValue(null); + // A matching active run first clears the once-per-fingerprint latch so this test + // is independent of whatever ran before it. + activeGeneration.mockResolvedValueOnce({ id: '7', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + runOnce.mockClear(); + + // Two mismatched ticks: skip both, never retire an active gen, log exactly once. + activeGeneration.mockResolvedValue({ id: '7', dimension: 4, fingerprint: 'old-fp' }); + await runSchedulerTick(2000); + await runSchedulerTick(3000); + + expect(runOnce).not.toHaveBeenCalled(); + expect(retireGeneration).not.toHaveBeenCalled(); + expect(isEmbedRunActive()).toBe(false); + const pauseLogs = logSpy.mock.calls.filter(([m]) => /pausing incremental/i.test(String(m))); + expect(pauseLogs).toHaveLength(1); + } finally { + logSpy.mockRestore(); + } + }); +}); diff --git a/backend/src/services/embeddings/stampSkipped.integration.test.js b/backend/src/services/embeddings/stampSkipped.integration.test.js new file mode 100644 index 0000000..41da221 --- /dev/null +++ b/backend/src/services/embeddings/stampSkipped.integration.test.js @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// L2: stamp (embed_gen) + stale-vector prune must be ONE transaction. store.stampSkipped +// CAS-stamps rows with a last_modified token, unconditionally stamps rows without one, +// and deletes embeddings only for ids whose stamp actually landed — all atomically. +d('store.stampSkipped (single-tx stamp + prune)', () => { + let store, client, acctId, userId, gen; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'stamp')); + const gi = await client.query(`INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) VALUES ('m',4,'fp',0,'building') RETURNING id`); + gen = gi.rows[0].id; + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + async function seedMsgWithVector(uid) { + const m = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1,$2,'INBOX','x') RETURNING id, last_modified::text lm`, [acctId, uid]); + const id = m.rows[0].id; + await client.query(`INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, dimension, embedding) VALUES ($1,$2,0,0,1,4,'[1,0,0,0]')`, [gen, id]); + // Keep message_count honest for the seeded vector so the decrement + // assertions below observe real deltas (upsert normally maintains this). + await client.query('UPDATE index_generations SET message_count = message_count + 1 WHERE id = $1', [gen]); + return { id, lm: m.rows[0].lm }; + } + + const messageCount = async () => + Number((await client.query('SELECT message_count FROM index_generations WHERE id = $1', [gen])).rows[0].message_count); + + it('CAS-stamps and prunes the vector when last_modified matches', async () => { + const { id, lm } = await seedMsgWithVector(70001); + const before = await messageCount(); + const missed = await store.stampSkipped(gen, [{ id, lastModified: lm }], []); + expect(missed).toEqual([]); + const stamp = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [id]); + expect(String(stamp.rows[0].embed_gen)).toBe(String(gen)); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(0); // pruned in the same tx + expect(await messageCount()).toBe(before - 1); // Fix 7: the prune decrements the generation + }); + + it('a CAS miss decrements nothing (the vector stays)', async () => { + const { id } = await seedMsgWithVector(70004); + const before = await messageCount(); + await store.stampSkipped(gen, [{ id, lastModified: '1999-01-01 00:00:00+00' }], []); + expect(await messageCount()).toBe(before); + }); + + it('on a CAS miss leaves BOTH the stamp and the vector untouched', async () => { + const { id } = await seedMsgWithVector(70002); + const missed = await store.stampSkipped(gen, [{ id, lastModified: '1999-01-01 00:00:00+00' }], []); + expect(missed).toEqual([id]); + const stamp = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [id]); + expect(stamp.rows[0].embed_gen).toBeNull(); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(1); // not pruned — the row still needs work + }); + + it('unconditionally stamps + prunes a plain (missing-row) id', async () => { + const { id } = await seedMsgWithVector(70003); + const missed = await store.stampSkipped(gen, [], [id]); + expect(missed).toEqual([]); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/testSupport.js b/backend/src/services/embeddings/testSupport.js new file mode 100644 index 0000000..88b0afc --- /dev/null +++ b/backend/src/services/embeddings/testSupport.js @@ -0,0 +1,19 @@ +// Shared IT/dev-script fixture: seed and tear down a throwaway user + email account +// against a real DB. `db` is anything with .query(text, params) — a pg.Client, a +// pooled client, or the app pool. Deleting the user cascades to email_accounts → +// messages (ON DELETE CASCADE), so cleanupAccount removes everything in one step. +import { randomUUID } from 'crypto'; + +export async function seedAccount(db, label = 'it') { + const u = await db.query('INSERT INTO users (username) VALUES ($1) RETURNING id', [`${label}-${randomUUID()}`]); + const userId = u.rows[0].id; + const a = await db.query( + 'INSERT INTO email_accounts (user_id, name, email_address) VALUES ($1, $2, $3) RETURNING id', + [userId, label, `${label}-${randomUUID()}@example.com`], + ); + return { userId, accountId: a.rows[0].id }; +} + +export async function cleanupAccount(db, userId) { + if (userId) await db.query('DELETE FROM users WHERE id = $1', [userId]); +} diff --git a/backend/src/services/embeddings/vectorErrors.js b/backend/src/services/embeddings/vectorErrors.js new file mode 100644 index 0000000..e6516b8 --- /dev/null +++ b/backend/src/services/embeddings/vectorErrors.js @@ -0,0 +1,5 @@ +// Leaf module: imports nothing from vectorStore/hybrid so Phase 3's loadVector +// and Phase 5's MCP handlers can import this without a cycle. +export class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } +} diff --git a/backend/src/services/embeddings/vectorSchema.integration.test.js b/backend/src/services/embeddings/vectorSchema.integration.test.js new file mode 100644 index 0000000..f3501a2 --- /dev/null +++ b/backend/src/services/embeddings/vectorSchema.integration.test.js @@ -0,0 +1,45 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('ensureVectorSchema (pgvector image)', () => { + let store, client; + beforeAll(async () => { + // Point db.js's pool at the test DB via env before importing the module graph. + const u = new URL(DSN); + process.env.DB_HOST = u.hostname; + process.env.DB_PORT = u.port; + process.env.DB_NAME = u.pathname.slice(1); + process.env.DB_USER = u.username; + process.env.DB_PASSWORD = u.password; + store = await import('./vectorStore.js'); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + }); + afterAll(async () => { await client.end(); }); + + it('creates the vector schema and reports available', async () => { + const { vectorAvailable } = await store.ensureVectorSchema(); + expect(vectorAvailable).toBe(true); + expect(store.isVectorAvailable()).toBe(true); + const t = await client.query( + "SELECT to_regclass('embeddings') e, to_regclass('index_generations') g, to_regclass('embed_watermark') w, to_regclass('embed_runs') r" + ); + expect(t.rows[0].e).not.toBeNull(); + expect(t.rows[0].g).not.toBeNull(); + expect(t.rows[0].w).not.toBeNull(); + expect(t.rows[0].r).not.toBeNull(); + }); + + it('is idempotent (second call does not throw)', async () => { + await expect(store.ensureVectorSchema()).resolves.toMatchObject({ vectorAvailable: true }); + }); + + it('builds a partial HNSW index for a dimension on the empty table', async () => { + await store.ensureVectorIndex(4); + const idx = await client.query("SELECT indexname FROM pg_indexes WHERE tablename='embeddings' AND indexname='idx_embeddings_hnsw_d4'"); + expect(idx.rows.length).toBe(1); + }); +}); diff --git a/backend/src/services/embeddings/vectorSchema.unit.test.js b/backend/src/services/embeddings/vectorSchema.unit.test.js new file mode 100644 index 0000000..a1376c6 --- /dev/null +++ b/backend/src/services/embeddings/vectorSchema.unit.test.js @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest'; + +// Mock the module graph so ensureVectorSchema runs without a real DB. The dedicated +// client (withDedicatedClient → new pg.Client) and the pool both resolve; the only +// variable is whether ensureVectorIndex(dim) throws (a non-integer dimension throws +// synchronously), which is exactly the "flag vs return" disagreement path. +const clientMock = { connect: vi.fn().mockResolvedValue(), query: vi.fn().mockResolvedValue({}), end: vi.fn().mockResolvedValue() }; +// Regular (constructable) function impl — `new pg.Client(...)` needs a constructor. +vi.mock('pg', () => ({ default: { Client: vi.fn(function () { return clientMock; }) } })); +vi.mock('../db.js', () => ({ pool: { query: vi.fn().mockResolvedValue({}) }, withTransaction: vi.fn() })); +vi.mock('./config.js', () => ({ resolveEmbedConfig: vi.fn() })); + +const { resolveEmbedConfig } = await import('./config.js'); +const { ensureVectorSchema, isVectorAvailable } = await import('./vectorStore.js'); + +describe('ensureVectorSchema flag/return agreement', () => { + it('reports available and sets the flag when the whole bring-up succeeds', async () => { + resolveEmbedConfig.mockResolvedValueOnce({ dimension: 4, skipExtensionCreate: false }); + const r = await ensureVectorSchema(); + expect(r.vectorAvailable).toBe(true); + expect(isVectorAvailable()).toBe(true); + }); + + it('leaves isVectorAvailable() false when ensureVectorIndex throws after the schema builds', async () => { + // dimension 2.5 passes the `> 0` guard but ensureVectorIndex throws on the + // non-integer, AFTER the extension + schema succeed — the disagreement window. + resolveEmbedConfig.mockResolvedValueOnce({ dimension: 2.5, skipExtensionCreate: false }); + const r = await ensureVectorSchema(); + expect(r.vectorAvailable).toBe(false); + expect(isVectorAvailable()).toBe(false); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.fused.test.js b/backend/src/services/embeddings/vectorStore.fused.test.js new file mode 100644 index 0000000..1f8cf48 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.fused.test.js @@ -0,0 +1,296 @@ +import { describe, it, expect, afterAll } from 'vitest'; +import { hasTestDb, withTestDb, closeTestDb } from './__testdb__.js'; +import { fusedSearch } from './vectorStore.js'; +import { searchLexical } from '../search/lexicalRepo.js'; + +afterAll(async () => { await closeTestDb(); }); + +describe.skipIf(!hasTestDb())('pgvector test DB', () => { + it('boots ensureVectorSchema and exposes the vector extension', async () => { + await withTestDb(async (pool) => { + const { rows } = await pool.query(`SELECT extname FROM pg_extension WHERE extname = 'vector'`); + expect(rows).toHaveLength(1); + }); + }); +}); + +function unitVec(dim, axis) { + const v = new Array(dim).fill(0); + v[axis] = 1; + return v; +} + +// Seeds one user + account + N messages (trigger fills search_fts) and an active +// generation with one chunk per message. Returns { accountId, generation, ids }. +async function seedThree(pool) { + const u = (await pool.query( + `INSERT INTO users (username, password_hash) VALUES ('t','x') RETURNING id`)).rows[0].id; + const acc = (await pool.query( + `INSERT INTO email_accounts (user_id, name, email_address, enabled) + VALUES ($1,'A','a@x.test',true) RETURNING id`, [u])).rows[0].id; + const base = Date.UTC(2025, 0, 15, 12, 0, 0); + const rows = [ + ['alpha quantum project update', 'discussing the quantum roadmap', 0, false], + ['beta vector indexing notes', 'notes about hybrid search and ranking', 1, true], + ['gamma project retrospective', 'retro covering the quantum milestone', 2, false], + ]; + const ids = []; + for (let i = 0; i < rows.length; i++) { + const [subject, body, , hasAtt] = rows[i]; + const id = (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, has_attachments, is_deleted) + VALUES ($1,$2,'INBOX',$3,'Sender','s@x.test',$4,$5,$6,$7,false) RETURNING id`, + [acc, 100 + i, subject, new Date(base + i * 86400000).toISOString(), body, body, hasAtt] + )).rows[0].id; + ids.push(id); + } + const gen = (await pool.query( + `INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) + VALUES ('m', 4, 'm:4:test', $1, 'active') RETURNING id`, + [Math.floor(Date.now() / 1000)])).rows[0].id; + for (let i = 0; i < ids.length; i++) { + await pool.query( + `INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, embedding, dimension) + VALUES ($1,$2,0,$3,4,$4::vector,4)`, + [gen, ids[i], Math.floor(Date.now() / 1000), `[${unitVec(4, i).join(',')}]`]); + } + return { accountId: acc, generation: { id: gen, dimension: 4 }, ids }; +} + +const base = { rrfK: 60, kPerSignal: 10, limit: 10, buildFilters: () => [] }; + +describe.skipIf(!hasTestDb())('fusedSearch', () => { + it('FTS-only: returns the two quantum messages, bm25 set / vector null, RRF descending', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(false); + expect(hits).toHaveLength(2); + const set = new Set(hits.map(h => h.message_id)); + expect(set.has(s.ids[0]) && set.has(s.ids[2])).toBe(true); + for (const h of hits) { + expect(h.vector_score).toBeNull(); + expect(h.bm25_score).not.toBeNull(); + expect(h.rrf_score).toBeGreaterThan(0); + } + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].rrf_score).toBeGreaterThanOrEqual(hits[i].rrf_score); + } + }); + }); + + it('prefix consistency: the fused FTS leg matches the same hit set as the lexical path for a prefix term (Opus review fix-round)', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + // A message whose only match for "invo" is a prefix of "invoice" — none + // of seedThree's quantum-related docs share that prefix, so a correct + // prefix match (and only a correct prefix match) finds exactly this one. + const invoiceId = (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, is_deleted) + VALUES ($1,103,'INBOX','Invoice reminder','Sender','s@x.test',now(),'inv','an invoice is attached',false) + RETURNING id`, + [s.accountId])).rows[0].id; + + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'invo', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const lex = await searchLexical((text, params) => pool.query(text, params), { + parsed: { filters: [], terms: [{ value: 'invo', negate: false }] }, + accountIds: [s.accountId], folderScope: null, folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + + const fusedIds = new Set(hits.map(h => h.message_id)); + const lexIds = new Set(lex.rows.map(r => r.id)); + expect(fusedIds.has(invoiceId)).toBe(true); + expect(fusedIds).toEqual(lexIds); + }); + }); + + it('ANN-only: top hit is the on-axis message, bm25 null / vector set', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, ftsQuery: null, queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(false); + expect(hits[0].message_id).toBe(s.ids[0]); + for (const h of hits) { + expect(h.bm25_score).toBeNull(); + expect(h.vector_score).not.toBeNull(); + } + }); + }); + + it('hybrid: union of the FTS pair and the ANN-only third message (3 hits), RRF descending', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: unitVec(4, 1), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(hits).toHaveLength(3); + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].rrf_score).toBeGreaterThanOrEqual(hits[i].rrf_score); + } + }); + }); + + it('saturation: kPerSignal below the pool size flips poolSaturated and trims', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, kPerSignal: 1, ftsQuery: 'quantum', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(true); + expect(hits).toHaveLength(1); + }); + }); + + it('tenant scope: a different account never leaks into the pool', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: null, generation: s.generation, + accountIds: ['00000000-0000-0000-0000-000000000000'] }, + { client: pool }); + expect(hits).toHaveLength(0); + }); + }); + + it('multi-chunk dedup: a message with a close and a far chunk appears once at its MIN distance', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + // Give the winning (chunk_index=0, on-axis) chunk distinctive offsets so + // Task 5b's best_char_start assertion can tell it apart from the far one. + await pool.query( + `UPDATE embeddings SET chunk_char_start = 6, chunk_char_end = 40 + WHERE generation_id = $1 AND message_id = $2 AND chunk_index = 0`, + [s.generation.id, s.ids[0]]); + // give ids[0] a second, far chunk on axis 2, with DIFFERENT offsets + await pool.query( + `INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, chunk_char_start, chunk_char_end, embedding, dimension) + VALUES ($1,$2,1,$3,4,100,140,$4::vector,4)`, + [s.generation.id, s.ids[0], Math.floor(Date.now() / 1000), `[${unitVec(4, 2).join(',')}]`]); + const { hits } = await fusedSearch( + { ...base, ftsQuery: null, queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const counts = {}; + for (const h of hits) counts[h.message_id] = (counts[h.message_id] || 0) + 1; + expect(counts[s.ids[0]]).toBe(1); + const top = hits.find(h => h.message_id === s.ids[0]); + expect(top.vector_score).toBeCloseTo(1.0, 6); // 1 - MIN(distance)=0 + // The winning (close) chunk's offsets ride through, not the far chunk's. + expect(top.best_chunk_index).toBe(0); + expect(top.best_char_start).toBe(6); + expect(top.best_char_end).toBe(40); + }); + }); + + it('rejects an empty request', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + await expect(fusedSearch( + { ...base, ftsQuery: null, queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool })).rejects.toThrow(); + }); + }); +}); + +// Wave D Fix 1, verified against the live pgvector container: english stopwords +// normalize to an EMPTY tsquery under 'english' (numnode = 0) and `@@ ''` is +// FALSE — pre-fix, ONE stopword in the AND'd term chain zeroed every backfilled +// result ("waiting for invoice" → 0 hits because of "for"). +describe.skipIf(!hasTestDb())('stopword terms never zero a search (Fix 1)', () => { + async function seedWaiting(pool, s) { + return (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, is_deleted) + VALUES ($1,104,'INBOX','Waiting for invoice','Sender','s@x.test',now(),'w', + 'we are waiting for the invoice payment',false) RETURNING id`, + [s.accountId])).rows[0].id; + } + const lex = (pool, terms, s, ordering = 'relevance') => + searchLexical((text, params) => pool.query(text, params), { + parsed: { filters: [], terms }, + accountIds: [s.accountId], folderScope: null, folderFuzzy: false, ordering, limit: 50, offset: 0, + }); + + it('lexical: "waiting for invoice" finds the message despite the stopword', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const res = await lex(pool, [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ], s); + expect(res.rows.map((r) => r.id)).toContain(id); + }); + }); + + it('lexical: a stopword-ONLY query degrades to filter-only/date-order — never zero-by-stopword', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const res = await lex(pool, [{ value: 'the', negate: false }], s); + expect(res.hasCondition).toBe(true); + expect(res.rows).toHaveLength(3); // the whole (trash-excluded) scope + const dates = res.rows.map((r) => new Date(r.date).getTime()); + expect(dates).toEqual([...dates].sort((a, b) => b - a)); // rank 0 ⇒ date tiebreak + }); + }); + + it('lexical: a NEGATED stopword contributes nothing instead of excluding everything', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const res = await lex(pool, [ + { value: 'invoice', negate: false }, + { value: 'the', negate: true }, // body contains "the" — must NOT exclude + ], s); + expect(res.rows.map((r) => r.id)).toContain(id); + }); + }); + + it('fused BM25 leg: "waiting for invoice" matches the same set the lexical path finds', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'waiting for invoice', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const res = await lex(pool, [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ], s); + expect(hits.map((h) => h.message_id)).toContain(id); + expect(new Set(hits.map((h) => h.message_id))).toEqual(new Set(res.rows.map((r) => r.id))); + }); + }); + + it('fused: an ALL-stopword ftsQuery leaves the BM25 leg empty — pure-ANN ranking, silence not noise', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'the for you', queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(hits.length).toBeGreaterThan(0); // ANN still answers + expect(hits.every((h) => h.bm25_score === null)).toBe(true); // FTS contributed nothing + }); + }); + +}); + +describe.skipIf(!hasTestDb())('fusedSearch input validation', () => { + it('rejects a dimension mismatch', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + await expect(fusedSearch( + { ...base, ftsQuery: null, queryVec: [1, 2, 3], generation: s.generation, accountIds: [s.accountId] }, + { client: pool })).rejects.toThrow(); + }); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.fusedSql.test.js b/backend/src/services/embeddings/vectorStore.fusedSql.test.js new file mode 100644 index 0000000..7788219 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.fusedSql.test.js @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest'; +import { fusedSearch, FUSED_ANN_CHUNKS_PER_MESSAGE, HNSW_EF_SEARCH_MAX } from './vectorStore.js'; + +// SQL-text pins for fusedSearch, driven through an injected fake client (a +// plain object — NOT a pg.Pool — so withEfSearch uses it verbatim and every +// statement it issues is recorded in order). +// +// `annPools` scripts the ann_pool_size the fused SELECT reports on each +// successive attempt, so the widening loop's exits are testable. +function fakeDb({ chunkCount = 5000, filteredMessages = 3000, annPools = [999], ftsPool = 5 } = {}) { + const calls = []; + let attempt = -1; + return { + calls, + fusedRuns() { return calls.filter((c) => /^WITH /.test(c.text) && /FROM fused f/.test(c.text)); }, + query: async (text, params) => { + calls.push({ text, params }); + if (/count\(\*\)::int AS n FROM embeddings/.test(text)) return { rows: [{ n: chunkCount }] }; + if (/count\(DISTINCT e\.message_id\)/.test(text)) return { rows: [{ n: filteredMessages }] }; + if (/FROM fused f/.test(text)) { + attempt = Math.min(attempt + 1, annPools.length - 1); + return { + rows: [{ + message_id: 'm1', subject: 's', + fts_pool_size: ftsPool, ann_pool_size: annPools[attempt], + }], + }; + } + return { rows: [] }; + }, + }; +} + +const gen = { id: 7, dimension: 4 }; +const base = { + generation: gen, accountIds: ['a1'], rrfK: 60, kPerSignal: 10, limit: 10, + buildFilters: () => [], +}; + +describe('fusedSearch BM25 leg — stopword-safe combined tsquery (Fix 1)', () => { + it('matches AND ranks with ONE `&&`-combined tsquery so an empty (stopword) operand drops out', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'waiting for invoice', queryVec: [1, 0, 0, 0] }, { client: db }); + const sql = db.fusedRuns()[0].text; + const combined = + "(to_tsquery('english', quote_literal($2) || ':*') && " + + "to_tsquery('english', quote_literal($3) || ':*') && " + + "to_tsquery('english', quote_literal($4) || ':*'))"; + // Single @@ against the combined query — `&&` drops an empty-normalizing + // operand ("for"), so a stopword can no longer zero the whole leg the way + // the old per-term `@@ ... AND @@ ...` chain did. + expect(sql).toContain(`m.search_fts @@ ${combined}`); + // The rank arg is the SAME combined construction (match and rank can't diverge). + expect(sql).toContain(`ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, ${combined}, 32)`); + // No per-term AND chain remains. + expect(sql).not.toMatch(/@@ to_tsquery\('english', quote_literal\(\$\d+\) \|\| ':\*'\) AND/); + const params = db.fusedRuns()[0].params; + expect(params.slice(1, 4)).toEqual(['waiting', 'for', 'invoice']); + }); +}); + +describe('fusedSearch ANN leg — hnsw.ef_search per attempt (Fix 2)', () => { + it('runs each ANN attempt in a transaction that SET LOCALs ef_search to the inner LIMIT', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + const texts = db.calls.map((c) => c.text); + const inner = (base.kPerSignal + 1) * FUSED_ANN_CHUNKS_PER_MESSAGE; // 88 + const begin = texts.indexOf('BEGIN'); + const guc = texts.indexOf(`SET LOCAL hnsw.ef_search = ${inner}`); + const run = texts.findIndex((t) => /FROM fused f/.test(t)); + const commit = texts.indexOf('COMMIT'); + // BEGIN → SET LOCAL → fused SELECT → COMMIT, in that order, on one client. + expect(begin).toBeGreaterThanOrEqual(0); + expect(guc).toBeGreaterThan(begin); + expect(run).toBeGreaterThan(guc); + expect(commit).toBeGreaterThan(run); + }); + + it(`caps the GUC at HNSW_EF_SEARCH_MAX (${HNSW_EF_SEARCH_MAX}) — pgvector rejects larger values`, async () => { + const db = fakeDb(); + // kPerSignal=200 → inner LIMIT (201*8=1608) exceeds the pgvector cap. + await fusedSearch({ ...base, kPerSignal: 200, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + expect(db.calls.some((c) => c.text === `SET LOCAL hnsw.ef_search = ${HNSW_EF_SEARCH_MAX}`)).toBe(true); + expect(db.calls.some((c) => /SET LOCAL hnsw\.ef_search = 1608/.test(c.text))).toBe(false); + }); + + it('re-issues a LARGER ef_search when the widening loop grows the inner LIMIT', async () => { + // First attempt dedups to 5 (< kPerSignal+1 = 11, < filteredCeiling), + // second grows to 11 and exits. + const db = fakeDb({ annPools: [5, 11] }); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + const gucs = db.calls.filter((c) => /^SET LOCAL hnsw\.ef_search = /.test(c.text)).map((c) => c.text); + expect(gucs).toEqual(['SET LOCAL hnsw.ef_search = 88', 'SET LOCAL hnsw.ef_search = 176']); + expect(db.fusedRuns()).toHaveLength(2); + }); + + it('breaks the widening loop when the ann pool stops growing between attempts', async () => { + // The pool sticks at 5 forever; without the no-growth break the loop + // would double 88 → … → 5000 (the chunk ceiling) re-running for nothing. + const db = fakeDb({ annPools: [5, 5] }); + const { poolSaturated } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + expect(db.fusedRuns()).toHaveLength(2); + expect(poolSaturated).toBe(false); + }); + + it('an FTS-only request issues no transaction and no GUC (nothing to tune)', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: null }, { client: db }); + expect(db.calls.some((c) => /hnsw\.ef_search|^BEGIN$/.test(c.text))).toBe(false); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.integration.test.js b/backend/src/services/embeddings/vectorStore.integration.test.js new file mode 100644 index 0000000..fef87f5 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.integration.test.js @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('vectorStore ANN', () => { + let store, client, gen, mA, mB, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(4); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // Re-runnable: clear generation state before seeding. embed_runs FKs + // index_generations without ON DELETE CASCADE (generations are never hard-deleted + // in production), so clear it before the parent to keep the reset re-runnable. + await client.query('DELETE FROM embeddings'); + await client.query('DELETE FROM embed_runs'); + await client.query('DELETE FROM index_generations'); + let acctId; + ({ accountId: acctId, userId } = await seedAccount(client, 'ann')); + const a = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81001, 'INBOX', 'A') RETURNING id`, [acctId]); + const b = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81002, 'INBOX', 'B') RETURNING id`, [acctId]); + mA = a.rows[0].id; mB = b.rows[0].id; + const now = Math.floor(Date.now() / 1000); + const gi = await client.query(`INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) VALUES ('m',4,'fp',$1,'building') RETURNING id`, [now]); + gen = gi.rows[0].id; + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('returns the nearest message first', async () => { + await store.upsert(gen, [ + { messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mB, chunkIndex: 0, vector: [0, 1, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + ]); + const hits = await store.annSearch(gen, [1, 0, 0, 0], 2); + expect(hits[0].messageId).toBe(mA); + expect(hits[0].rank).toBe(1); + expect(hits[0].score).toBeGreaterThan(hits[1].score); + }); + + it('re-upsert is idempotent (PK replace, no duplicate chunks)', async () => { + await store.upsert(gen, [{ messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }]); + const c = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, mA]); + expect(c.rows[0].n).toBe(1); + }); + + it('dedups multi-chunk messages by best (MIN) distance', async () => { + // mB gets a far chunk (0,1,0,0) and a near chunk (1,0,0,0); querying [1,0,0,0] + // should rank mB by its BEST chunk, ahead of a mid message. + await store.upsert(gen, [ + { messageId: mB, chunkIndex: 0, vector: [0, 1, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mB, chunkIndex: 1, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 4, chunkCharEnd: 8, truncated: false }, + ]); + const hits = await store.annSearch(gen, [1, 0, 0, 0], 5); + const mbHit = hits.find((h) => h.messageId === mB); + expect(mbHit.score).toBeGreaterThan(0.9); // best chunk is an exact match + // one hit per message + expect(new Set(hits.map((h) => h.messageId)).size).toBe(hits.length); + }); + + it('filter.accountIds scopes ANN to one account (find_similar_messages needs this)', async () => { + // A second account whose message is an EXACT match for the query — without the + // filter it would surface; with the account filter it must be excluded, and the + // widening still returns the in-scope match. + const other = await seedAccount(client, 'ann-other'); + const oc = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81003, 'INBOX', 'C') RETURNING id`, [other.accountId]); + const mC = oc.rows[0].id; + await store.upsert(gen, [ + { messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mC, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + ]); + const firstAcct = (await client.query('SELECT account_id FROM messages WHERE id = $1', [mA])).rows[0].account_id; + const scoped = await store.annSearch(gen, [1, 0, 0, 0], 5, { filter: { accountIds: [firstAcct] } }); + expect(scoped.some((h) => h.messageId === mC)).toBe(false); + expect(scoped.some((h) => h.messageId === mA)).toBe(true); + const unscoped = await store.annSearch(gen, [1, 0, 0, 0], 5); + expect(unscoped.some((h) => h.messageId === mC)).toBe(true); + await cleanupAccount(client, other.userId); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.js b/backend/src/services/embeddings/vectorStore.js new file mode 100644 index 0000000..a19fe50 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.js @@ -0,0 +1,754 @@ +import pg from 'pg'; +import { pool, withTransaction } from '../db.js'; +import { resolveEmbedConfig } from './config.js'; +import { LEXICAL_RANK_SQL, ftsTermQueryArg, hasSearchableToken } from '../search/lexicalRepo.js'; + +export const ZERO_UUID = '00000000-0000-0000-0000-000000000000'; + +let _vectorAvailable = false; +export function isVectorAvailable() { return _vectorAvailable; } + +// Build a short-lived connection WITHOUT the pool's `-c statement_timeout=30000` +// startup option, then explicitly disable statement_timeout, so a slow DDL build +// (HNSW over a repopulated table after the alpine→Debian image swap) is not killed +// at 30s. Caller closes nothing — this helper owns the connect/end lifecycle. +async function withDedicatedClient(fn) { + const client = new pg.Client({ + host: process.env.DB_HOST || 'postgres', + port: Number(process.env.DB_PORT) || 5432, + database: process.env.DB_NAME || 'mailflow', + user: process.env.DB_USER || 'mailflow', + password: process.env.DB_PASSWORD, + }); + await client.connect(); + try { + await client.query('SET statement_timeout = 0'); + return await fn(client); + } finally { + await client.end().catch(() => {}); + } +} + +const SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS index_generations ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + model TEXT NOT NULL, + dimension INTEGER NOT NULL, + fingerprint TEXT NOT NULL, + started_at BIGINT NOT NULL, + seeded_at BIGINT, + completed_at BIGINT, + activated_at BIGINT, + state TEXT NOT NULL, + message_count BIGINT NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_generations_active ON index_generations(state) WHERE state = 'active'; +CREATE UNIQUE INDEX IF NOT EXISTS idx_generations_building ON index_generations(state) WHERE state = 'building'; + +CREATE TABLE IF NOT EXISTS embeddings ( + generation_id BIGINT NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE, + message_id UUID NOT NULL, + chunk_index INTEGER NOT NULL DEFAULT 0, + embedded_at BIGINT NOT NULL, + source_char_len INTEGER NOT NULL, + chunk_char_start INTEGER NOT NULL DEFAULT 0, + chunk_char_end INTEGER NOT NULL DEFAULT 0, + truncated BOOLEAN NOT NULL DEFAULT FALSE, + dimension INTEGER NOT NULL, + embedding vector NOT NULL, + PRIMARY KEY (generation_id, message_id, chunk_index) +); +CREATE INDEX IF NOT EXISTS idx_embeddings_msg ON embeddings(message_id); +CREATE INDEX IF NOT EXISTS idx_embeddings_dim ON embeddings(dimension); + +CREATE TABLE IF NOT EXISTS embed_runs ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + generation_id BIGINT NOT NULL REFERENCES index_generations(id), + started_at BIGINT NOT NULL, + ended_at BIGINT, + claimed INTEGER NOT NULL DEFAULT 0, + succeeded INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + truncated INTEGER NOT NULL DEFAULT 0, + error TEXT +); + +CREATE TABLE IF NOT EXISTS embed_watermark ( + generation_id BIGINT PRIMARY KEY, + watermark_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +); +`; + +// Best-effort startup routine (pattern: encryptExistingCredentials in db.js). Never +// throws into boot: on any failure it logs, sets vector_available=false, and returns. +export async function ensureVectorSchema() { + _vectorAvailable = false; + let cfg = null; + try { cfg = await resolveEmbedConfig(); } catch { /* ai_config unreadable — treat as unset */ } + const skipExtension = cfg?.skipExtensionCreate === true; + try { + if (!skipExtension) { + await pool.query('CREATE EXTENSION IF NOT EXISTS vector'); + } + // Apply the schema on a dedicated no-timeout connection: on a legacy populated DB + // the idx_embeddings_* builds can exceed the pool's 30s cap (migrate.go). + await withDedicatedClient((client) => client.query(SCHEMA_SQL)); + if (cfg?.dimension > 0) { + await ensureVectorIndex(cfg.dimension); + } + // Set the flag only after the WHOLE bring-up (extension + schema + per-dimension + // HNSW) succeeds, so isVectorAvailable() never disagrees with the return value — + // e.g. an ensureVectorIndex failure must leave the flag false. + _vectorAvailable = true; + console.log('Vector schema ready — semantic search available'); + return { vectorAvailable: true }; + } catch (err) { + _vectorAvailable = false; + console.warn(`Vector disabled: ${err.message} — lexical search unaffected`); + return { vectorAvailable: false }; + } +} + +// Partial per-dimension HNSW cosine index, created while the table is empty so it is +// maintained incrementally (README invariant; port of migrate.go EnsureVectorIndex). +// The `WHERE dimension = N` guard lets generations of different dims coexist. +export async function ensureVectorIndex(dim) { + if (!Number.isInteger(dim) || dim <= 0) throw new Error(`invalid dimension ${dim}`); + const stmt = `CREATE INDEX IF NOT EXISTS idx_embeddings_hnsw_d${dim} + ON embeddings USING hnsw ((embedding::vector(${dim})) vector_cosine_ops) + WHERE dimension = ${dim}`; + await withDedicatedClient((client) => client.query(stmt)); +} + +const LIVE_MESSAGES_WHERE = 'is_deleted = false'; // Mailflow live-message predicate +const ANN_OVERFETCH = 4; // backend.go annOverFetchFactor + +// pgvector's hnsw.ef_search GUC defaults to 40 and hard-caps at 1000: the HNSW +// scan visits at most ef_search candidates, so any inner ANN `ORDER BY <=> +// LIMIT` above it is silently truncated to ~ef_search rows and widening the +// LIMIT re-runs an identical plan. msgvault sizes a per-connection GUC to 1000 +// (store.go HNSWEfSearch): >= the worst-case fused inner LIMIT, +// (kPerSignal+1)*FUSED_ANN_CHUNKS_PER_MESSAGE ≈ 808 at the default +// kPerSignal=100, with headroom, while keeping per-query latency bounded. +// Mailflow's pool sets no per-connection GUCs, so every ANN-issuing statement +// runs in a transaction that `SET LOCAL`s the GUC to its own inner LIMIT, +// capped here (pgvector REJECTS values above 1000; beyond the cap, recall is +// best-effort — the same trade msgvault documents). +export const HNSW_EF_SEARCH_MAX = 1000; + +// Run fn(client) in a transaction whose hnsw.ef_search covers `efSearch` +// candidates (capped at HNSW_EF_SEARCH_MAX). A pg.Pool is pinned to one +// client first — BEGIN / SET LOCAL / queries must share a session; an +// injected single client (tests) is used as-is. +async function withEfSearch(db, efSearch, fn) { + const pinned = db instanceof pg.Pool ? await db.connect() : db; + try { + await pinned.query('BEGIN'); + await pinned.query(`SET LOCAL hnsw.ef_search = ${Math.min(Math.max(1, Math.floor(efSearch)), HNSW_EF_SEARCH_MAX)}`); + const out = await fn(pinned); + await pinned.query('COMMIT'); + return out; + } catch (err) { + await pinned.query('ROLLBACK').catch(() => {}); + throw err; + } finally { + if (pinned !== db) pinned.release(); + } +} + +export function vectorLiteral(vec) { + return `[${vec.map((f) => Number(f).toString()).join(',')}]`; +} + +// Upsert chunks for one generation. Idempotent per message: clears the message's +// prior chunks (chunk count is not stable across re-embeds) then inserts the new set, +// all in one tx. Maintains index_generations.message_count by distinct-message delta. +export async function upsert(gen, chunks) { + if (!chunks.length) return; + await withTransaction(async (client) => { + // Row-lock the generation (serializes against activate/retire) and read its dim. + const g = await client.query('SELECT dimension, state FROM index_generations WHERE id = $1 FOR UPDATE', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + if (g.rows[0].state === 'retired') { const e = new Error(`generation retired ${gen}`); e.code = 'GEN_RETIRED'; throw e; } + const dim = g.rows[0].dimension; + for (const c of chunks) { + if (c.vector.length !== dim) throw new Error(`dimension mismatch: chunk for msg ${c.messageId} has ${c.vector.length}, gen has ${dim}`); + } + const ids = [...new Set(chunks.map((c) => c.messageId))]; + const pre = await client.query( + 'SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', + [gen, ids], + ); + await client.query('DELETE FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', [gen, ids]); + const now = Math.floor(Date.now() / 1000); + for (const c of chunks) { + await client.query( + `INSERT INTO embeddings + (generation_id, message_id, chunk_index, embedded_at, source_char_len, + chunk_char_start, chunk_char_end, truncated, dimension, embedding) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::vector)`, + [gen, c.messageId, c.chunkIndex, now, c.sourceCharLen, c.chunkCharStart, c.chunkCharEnd, c.truncated, dim, vectorLiteral(c.vector)], + ); + } + const delta = ids.length - pre.rows[0].n; + if (delta !== 0) { + await client.query('UPDATE index_generations SET message_count = message_count + $1 WHERE id = $2', [delta, gen]); + } + }); +} + +export async function chunkCount(gen) { + const r = await pool.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1', [gen]); + return r.rows[0].n; +} + +// ANN search over one generation. Inner ORDER BY <=> LIMIT uses the partial HNSW index +// (dimension embedded as a literal); outer GROUP BY collapses multi-chunk messages to +// their best (MIN) distance. Widens the inner LIMIT until k distinct messages survive the +// dedup. Score = 1 - cosine_distance. +// +// An optional structured `filter` narrows the candidate set by pushing predicates INTO +// the inner liveness EXISTS (a Postgres-native join on messages — the equivalent of +// msgvault's json_each resolved-id set, backend.go filtered path). This is REQUIRED for +// account-scoped callers (e.g. plan-phase5 find_similar_messages in a multi-user DB): +// filtering after k-NN can return zero in-scope rows, whereas widening WITH the filter in +// SQL keeps pulling candidates until k in-scope messages survive (or the generation is +// exhausted). filter fields (all optional, AND-combined): +// accountIds: string[] (UUIDs) → m.account_id = ANY($n::uuid[]) +// after / before: timestamptz-comparable (ISO string or Date) → m.date >= / < $n +// hasAttachment: boolean → m.has_attachments = true/false +// With no filter the SQL + params are byte-identical to the fast path (filter clauses and +// their binds simply don't appear). Note: with a filter present the planner may fall back +// from the HNSW index to a scan within the filtered set — the same trade msgvault accepts. +// Append the structured filter's predicates to `args` (relative to messages alias `m`) +// and return the joined WHERE string. Built with a fresh `args` per query so each +// statement's $N ordinals resolve independently (msgvault's bind-closure pattern). +function buildAnnFilter(filter, args) { + const where = [LIVE_MESSAGES_WHERE]; + if (filter) { + if (filter.accountIds?.length) { args.push(filter.accountIds); where.push(`m.account_id = ANY($${args.length}::uuid[])`); } + if (filter.after) { args.push(filter.after); where.push(`m.date >= $${args.length}`); } + // before is EXCLUSIVE (<) — msgvault filter.go parity, and the same + // bound lexicalRepo's before: operator applies (one convention everywhere). + if (filter.before) { args.push(filter.before); where.push(`m.date < $${args.length}`); } + if (filter.hasAttachment === true) where.push('m.has_attachments = true'); + else if (filter.hasAttachment === false) where.push('m.has_attachments = false'); + } + return where.join(' AND '); +} + +// Doubling-widen the inner ANN LIMIT until at least k distinct messages survive the +// outer dedup, the distinct-message early exit is reached, or the candidate ceiling is +// hit. `ceiling` counts CHUNKS (bounds the inner LIMIT); `distinctEarlyExit` counts the +// distinct MESSAGES that can possibly appear (equals k on the empty-filter path, a +// no-op; equals the filtered distinct-message count on the filtered path, so a +// selective filter stops as soon as every in-scope message is surfaced instead of +// widening up to the whole generation's chunk count — msgvault searchWiden parity). +// Port of backend.go searchWiden. Exported for unit testing. +export async function searchWiden(k, ceiling, distinctEarlyExit, run) { + let innerLimit = Math.max(k * ANN_OVERFETCH, k); + for (;;) { + if (innerLimit > ceiling) innerLimit = ceiling; + const hits = await run(innerLimit); + if (hits.length >= k || hits.length >= distinctEarlyExit || innerLimit >= ceiling) { + return hits.slice(0, k).map((h, i) => ({ ...h, rank: i + 1 })); + } + innerLimit *= 2; + } +} + +export async function annSearch(gen, queryVec, k, { efSearch = 100, filter = null } = {}) { + if (!queryVec.length) throw new Error('annSearch: empty query vector'); + const g = await pool.query('SELECT dimension FROM index_generations WHERE id = $1', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + const dim = g.rows[0].dimension; + if (queryVec.length !== dim) throw new Error(`dimension mismatch: query ${queryVec.length}, gen ${dim}`); + const lit = vectorLiteral(queryVec); + + // Widening bounds. Filtered: count only the in-scope candidate set (chunks = ceiling, + // distinct messages = early exit) with the SAME EXISTS predicate, so the loop stops + // once every in-scope message is surfaced. Unfiltered: whole-generation chunk count as + // the ceiling and k as the (no-op) early exit — byte-identical to the original fast path. + let ceiling, distinctEarlyExit; + if (filter) { + const cArgs = [gen]; + const cWhere = buildAnnFilter(filter, cArgs); + const cnt = await pool.query( + `SELECT COUNT(*)::int chunks, COUNT(DISTINCT e.message_id)::int messages + FROM embeddings e + WHERE e.generation_id = $1 + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${cWhere})`, + cArgs, + ); + ceiling = cnt.rows[0].chunks; + distinctEarlyExit = cnt.rows[0].messages; + } else { + ceiling = await chunkCount(gen); + distinctEarlyExit = k; + } + if (ceiling === 0) return []; + + // $1 = query vector, $2 = generation; filter binds (if any) take $3.. ; the two LIMITs + // are appended per widening run as the trailing two params. + const args = [lit, gen]; + const where = buildAnnFilter(filter, args); + const innerArg = args.length + 1; + const outerArg = args.length + 2; + const sql = ` + SELECT ann.message_id, MIN(ann.distance) AS distance + FROM ( + SELECT e.message_id, (e.embedding::vector(${dim})) <=> $1::vector AS distance + FROM embeddings e + WHERE e.generation_id = $2 AND e.dimension = ${dim} + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${where}) + ORDER BY e.embedding::vector(${dim}) <=> $1::vector + LIMIT $${innerArg} + ) ann + GROUP BY ann.message_id + ORDER BY distance, ann.message_id + LIMIT $${outerArg}`; + const client = await pool.connect(); + try { + await client.query('BEGIN'); + const hits = await searchWiden(k, ceiling, distinctEarlyExit, async (innerLimit) => { + // Re-issue the GUC per widening attempt: ef_search must cover the inner + // LIMIT or the HNSW scan truncates it and the widening loop is a no-op + // (see HNSW_EF_SEARCH_MAX). `efSearch` stays the floor for small limits. + await client.query(`SET LOCAL hnsw.ef_search = ${Math.min(Math.max(Number(efSearch), innerLimit), HNSW_EF_SEARCH_MAX)}`); + const r = await client.query(sql, [...args, innerLimit, k]); + return r.rows.map((row, i) => ({ messageId: row.message_id, score: 1 - Number(row.distance), rank: i + 1 })); + }); + await client.query('COMMIT'); + return hits; + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + throw err; + } finally { + client.release(); + } +} + +// Return the chunk_index=0 vector for messageId in the active generation. +export async function loadVector(messageId) { + const active = await pool.query("SELECT id, dimension FROM index_generations WHERE state = 'active'"); + if (!active.rows.length) throw new Error('no active generation'); + const r = await pool.query( + 'SELECT embedding::text lit FROM embeddings WHERE generation_id = $1 AND message_id = $2 AND chunk_index = 0', + [active.rows[0].id, messageId], + ); + if (!r.rows.length) throw new Error(`no embedding for message ${messageId}`); + return r.rows[0].lit.slice(1, -1).split(',').map(Number); +} + +// Forward scan by UUID for live messages needing work, resuming above afterId. The +// predicate is `embed_gen IS NULL` (NOT the OR with `embed_gen <> target`) so the +// partial index idx_messages_embed_pending (migration 0039) drives an O(pending) scan +// even on a huge, fully-covered mailbox — an OR forces a full seq scan. This is +// correct because createGeneration resets every live row's stamp to NULL when a NEW +// generation is created, so a rebuild's prior-generation rows surface as NULL here too: +// no live row ever carries a non-null stamp for a generation OTHER than the current +// target. `target` is kept in the signature (callers pass it) but is not needed in the +// predicate given that invariant. +export async function scanForEmbedding(target, afterId, limit) { + const r = await pool.query( + `SELECT id FROM messages + WHERE embed_gen IS NULL AND ${LIVE_MESSAGES_WHERE} AND id > $1 + ORDER BY id LIMIT $2`, + [afterId, limit], + ); + return r.rows.map((row) => row.id); +} + +export async function setEmbedGen(ids, target) { + if (!ids.length) return; + await pool.query('UPDATE messages SET embed_gen = $1 WHERE id = ANY($2::uuid[])', [target, ids]); +} + +// Optimistic CAS stamp: only stamp rows whose last_modified text token is unchanged +// since the worker read it. Returns the ids that MISSED (last_modified moved) — not +// stamped; the backstop recovers them. Bind the token back as ::timestamptz for +// exact-equality (JS Date would lose the microseconds pg stores). +export async function setEmbedGenIfUnchanged(items, target) { + const missed = []; + await withTransaction(async (client) => { + for (const it of items) { + const res = await client.query( + 'UPDATE messages SET embed_gen = $1 WHERE id = $2 AND last_modified = $3::timestamptz', + [target, it.id, it.lastModified], + ); + if (res.rowCount === 0) missed.push(it.id); + } + }); + return missed; +} + +// Skip-mark rows (empty/missing) and prune their now-stale vectors in ONE transaction +// (parity with msgvault worker.go stampSkipped). CAS-stamps `casItems` (rows with a +// last_modified token) and unconditionally stamps `plainIds` (missing rows with no row +// to guard), then deletes embeddings for every id whose stamp actually landed — a +// CAS-missed id keeps both its NULL stamp and its vector so it is re-found later. +// index_generations.message_count is decremented by the DISTINCT messages the delete +// actually removes (msgvault backend.go:1118-1167 counts, deletes, and applies the +// delta under the generation row lock; this used to delete without decrementing, so +// the count drifted upward on every skipped re-embed). Returns the CAS-missed ids. +// Bind the token as ::timestamptz for exact equality. +export async function stampSkipped(gen, casItems, plainIds) { + const missed = []; + if (!casItems.length && !plainIds.length) return missed; + await withTransaction(async (client) => { + // Lock the generation row FIRST — the same order upsert() takes it (and + // msgvault's Delete, which locks before touching embeddings precisely to + // avoid an ABBA asymmetry with those writers) — so the decrement below is + // serialized against concurrent upsert/activate/retire. + const g = await client.query('SELECT id FROM index_generations WHERE id = $1 FOR UPDATE', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + for (const it of casItems) { + const res = await client.query( + 'UPDATE messages SET embed_gen = $1 WHERE id = $2 AND last_modified = $3::timestamptz', + [gen, it.id, it.lastModified], + ); + if (res.rowCount === 0) missed.push(it.id); + } + if (plainIds.length) { + await client.query('UPDATE messages SET embed_gen = $1 WHERE id = ANY($2::uuid[])', [gen, plainIds]); + } + const missedSet = new Set(missed); + const stamped = [...casItems.map((c) => c.id), ...plainIds].filter((id) => !missedSet.has(id)); + if (stamped.length) { + const pre = await client.query( + 'SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', + [gen, stamped], + ); + await client.query('DELETE FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', [gen, stamped]); + if (pre.rows[0].n > 0) { + await client.query('UPDATE index_generations SET message_count = message_count - $1 WHERE id = $2', [pre.rows[0].n, gen]); + } + } + }); + return missed; +} + +export async function getWatermark(gen) { + const r = await pool.query('SELECT watermark_id FROM embed_watermark WHERE generation_id = $1', [gen]); + return r.rows.length ? r.rows[0].watermark_id : ZERO_UUID; +} + +export async function setWatermark(gen, id) { + await pool.query( + `INSERT INTO embed_watermark (generation_id, watermark_id) VALUES ($1, $2) + ON CONFLICT (generation_id) DO UPDATE SET watermark_id = EXCLUDED.watermark_id`, + [gen, id], + ); +} + +export async function resetWatermark(gen) { + await setWatermark(gen, ZERO_UUID); +} + +// Fetch subject + inline bodies + the last_modified CAS token (as text) for a +// batch of message ids. Bodies are inline in messages: no message_bodies table. +export async function fetchForEmbedding(ids) { + if (!ids.length) return []; + const r = await pool.query( + `SELECT id, + COALESCE(subject, '') AS subject, + COALESCE(body_text, '') AS "bodyText", + COALESCE(body_html, '') AS "bodyHtml", + last_modified::text AS "lastModified" + FROM messages WHERE id = ANY($1::uuid[])`, + [ids], + ); + return r.rows; +} + +export async function startEmbedRun(gen) { + const now = Math.floor(Date.now() / 1000); + const r = await pool.query('INSERT INTO embed_runs (generation_id, started_at) VALUES ($1, $2) RETURNING id', [gen, now]); + return r.rows[0].id; +} + +export async function finalizeEmbedRun(runId, res, err) { + if (!runId) return; + const now = Math.floor(Date.now() / 1000); + await pool.query( + `UPDATE embed_runs SET ended_at = $1, claimed = $2, succeeded = $3, failed = $4, truncated = $5, error = $6 WHERE id = $7`, + [now, res.claimed, res.succeeded, res.failed, res.truncated, err ? String(err.message || err) : null, runId], + ).catch(() => {}); +} + +// ── Fused RRF search — port of internal/vector/pgvector/fused.go ── + +export const FUSED_ANN_CHUNKS_PER_MESSAGE = 8; + +const DISPLAY_COLS = ` + m.id AS message_id, m.uid, m.folder, m.subject, m.from_name, m.from_email, + m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, + a.name AS account_name, a.email_address AS account_email, a.color AS account_color`; + +async function fusedChunkCount(db, generation) { + const { rows } = await db.query( + `SELECT count(*)::int AS n FROM embeddings WHERE generation_id = $1 AND dimension = $2`, + [generation.id, generation.dimension]); + return rows[0].n; +} + +async function filteredChunkMessageCount(db, { generation, accountIds, buildFilters }) { + const args = [generation.id, generation.dimension, accountIds]; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + const filters = buildFilters(bind).map(c => ` AND ${c}`).join(''); + const { rows } = await db.query( + `SELECT count(DISTINCT e.message_id)::int AS n + FROM embeddings e + JOIN messages m ON m.id = e.message_id + WHERE e.generation_id = $1 AND e.dimension = $2 + AND m.account_id = ANY($3) AND m.is_deleted = false${filters}`, + args); + return rows[0].n; +} + +// Single-query hybrid RRF fusion: BM25 leg (weighted ts_rank_cd over +// search_fts, reusing LEXICAL_RANK_SQL) + ANN leg (cosine distance +// over one generation's embeddings), FULL OUTER JOIN'd and combined via +// reciprocal-rank fusion. Either leg can be omitted (ftsQuery/queryVec null) +// for lexical-only or vector-only pools. `buildFilters(bind) → string[]` +// applies the SAME structured operator predicates to both legs (README "one +// search seam" — lexicalRepo remains the single owner of those predicates). +// Each hit also carries best_chunk_index/best_char_start/best_char_end — +// the ANN leg's winning (min-distance) chunk's offsets, null on an +// FTS-only hit. These are CODE POINTS into the PREPROCESSED text (README +// Unicode contract), not raw body_text byte offsets — chunkmatch.js owns +// turning them into a raw-body byte snippet. +export async function fusedSearch(req, { client } = {}) { + const db = client || pool; + const { generation, accountIds, rrfK, kPerSignal, limit } = req; + const buildFilters = req.buildFilters || (() => []); + // Tokenize once (terms don't change across the widening loop below) and + // apply the SAME hygiene the lexical path applies (searchLexical): + // drop sub-2-char and punctuation-only tokens rather than handing Postgres + // a term that would normalize to zero lexemes. This is also what makes + // `useFTS` mean "the FTS leg actually has something to match on", not just + // "a non-empty string was passed" — an all-punctuation ftsQuery degrades to + // ANN-only (or throws below, same as if ftsQuery were absent, if ANN is + // also unavailable). + const ftsTerms = typeof req.ftsQuery === 'string' + ? req.ftsQuery.trim().split(/\s+/).filter(t => t.length >= 2 && hasSearchableToken(t)) + : []; + const useFTS = ftsTerms.length > 0; + const useANN = Array.isArray(req.queryVec) && req.queryVec.length > 0; + if (!useFTS && !useANN) throw new Error('fusedSearch: neither ftsQuery nor queryVec provided'); + if (useANN && req.queryVec.length !== generation.dimension) { + throw new Error(`fusedSearch: dimension mismatch (query ${req.queryVec.length}, generation ${generation.dimension})`); + } + const dim = generation.dimension; + const kPlus1 = kPerSignal + 1; + + let chunkCeiling = 0; + let filteredCeiling = 0; + if (useANN) { + chunkCeiling = await fusedChunkCount(db, generation); + filteredCeiling = await filteredChunkMessageCount(db, { generation, accountIds, buildFilters }); + } + + async function runFused(exec, innerChunks) { + const args = []; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + const accArg = bind(accountIds); + const live = `m.account_id = ANY(${accArg}) AND m.is_deleted = false`; + const filterAnd = buildFilters(bind).map(c => ` AND ${c}`).join(''); + + const ctes = []; + if (useFTS) { + // One bind per term, `&&`-combined into a SINGLE tsquery used for BOTH + // the @@ match and the ts_rank_cd rank — the same per-term prefix-or- + // phrase construction lexicalRepo.js's lexical path uses + // (ftsTermQueryArg), so "invo" matches "invoice" here exactly as it + // does via searchLexical, and msgvault's fused.go shape (one combined + // BuildFTSTerm arg for match and rank alike). `@@ (a && b)` is + // equivalent to `@@ a AND @@ b`, and `&&` DROPS an empty operand + // (verified against pgvector/pg16) — which is what keeps an english + // stopword ("waiting for invoice") from zeroing the whole BM25 leg: + // the stopword's tsquery normalizes empty and simply vanishes from the + // combined query. An ALL-stopword ftsQuery combines to an empty tsquery + // that matches nothing, so the leg contributes silence (not noise) and + // the fused ranking degrades to pure ANN. ftsTermQueryArg wants the raw + // placeholder NUMBER (it prepends its own `$`, matching lexicalRepo.js's + // own internal convention) — push directly onto `args` rather than + // through `bind`, which returns an already-`$`-prefixed string. + const termArgs = ftsTerms.map((term) => { args.push(term); return args.length; }); + const tsquery = `(${ftsTerms.map((term, i) => ftsTermQueryArg(termArgs[i], term)).join(' && ')})`; + const matchWhere = `m.search_fts @@ ${tsquery}`; + const kp1 = bind(kPlus1); + const k = bind(kPerSignal); + const rank = LEXICAL_RANK_SQL('m.search_fts', tsquery); + ctes.push(`fts_pool AS ( + SELECT m.id AS message_id, ${rank} AS bm25 + FROM messages m + WHERE ${matchWhere} + AND ${live}${filterAnd} + ORDER BY bm25 DESC + LIMIT ${kp1} +)`); + ctes.push(`fts_ranked AS ( + SELECT message_id, bm25, + ROW_NUMBER() OVER (ORDER BY bm25 DESC, message_id ASC) AS rnk + FROM fts_pool + ORDER BY bm25 DESC, message_id ASC + LIMIT ${k} +)`); + } + if (useANN) { + const vecArg = bind(vectorLiteral(req.queryVec)); + const genArg = bind(generation.id); + const innerArg = bind(innerChunks); + const kp1 = bind(kPlus1); + const k = bind(kPerSignal); + // DISTINCT ON (message_id), ordered by distance, picks the SAME min-distance + // chunk per message as the MIN(distance)/GROUP BY form — but also + // keeps that winning chunk's offsets, which the excerpt seam + // needs. Offsets are code points into the PREPROCESSED text, not raw + // body_text (README Unicode contract) — phase 5's chunkmatch.js owns + // turning them into a raw-body byte snippet. + ctes.push(`ann_pool AS ( + SELECT d.message_id, d.distance, d.chunk_index, d.chunk_char_start, d.chunk_char_end + FROM ( + SELECT DISTINCT ON (ann.message_id) + ann.message_id, ann.distance, ann.chunk_index, ann.chunk_char_start, ann.chunk_char_end + FROM ( + SELECT e.message_id, e.chunk_index, e.chunk_char_start, e.chunk_char_end, + (e.embedding::vector(${dim})) <=> ${vecArg}::vector AS distance + FROM embeddings e + WHERE e.generation_id = ${genArg} AND e.dimension = ${dim} + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${live}${filterAnd}) + ORDER BY e.embedding::vector(${dim}) <=> ${vecArg}::vector + LIMIT ${innerArg} + ) ann + ORDER BY ann.message_id, ann.distance + ) d + ORDER BY d.distance + LIMIT ${kp1} +)`); + ctes.push(`ann_ranked AS ( + SELECT message_id, distance, chunk_index, chunk_char_start, chunk_char_end, + ROW_NUMBER() OVER (ORDER BY distance ASC, message_id ASC) AS rnk + FROM ann_pool + ORDER BY distance ASC, message_id ASC + LIMIT ${k} +)`); + } + + const poolArgsLen = args.length; // rrfk/limit are appended after this + const poolCTEs = ctes.slice(); + const rrfkArg = bind(rrfK); + const limitArg = bind(limit); + + // 1.0 is a `numeric` literal in Postgres; numeric / bigint (rnk) stays + // numeric, and node-postgres returns numeric columns as strings (to avoid + // silent precision loss). Cast to double precision so rrf_score/bm25_score + // come back as JS numbers, matching msgvault's float64 RRF score. + let fused; + if (useFTS && useANN) { + fused = `fused AS ( + SELECT COALESCE(b.message_id, v.message_id) AS message_id, + COALESCE(1.0::float8 / (${rrfkArg} + b.rnk), 0.0) + COALESCE(1.0::float8 / (${rrfkArg} + v.rnk), 0.0) AS rrf_score, + b.bm25::float8 AS bm25_score, + CASE WHEN v.distance IS NULL THEN NULL ELSE 1.0::float8 - v.distance END AS vector_score, + v.chunk_index AS best_chunk_index, v.chunk_char_start AS best_char_start, v.chunk_char_end AS best_char_end + FROM fts_ranked b + FULL OUTER JOIN ann_ranked v USING (message_id) +)`; + } else if (useFTS) { + fused = `fused AS ( + SELECT b.message_id, 1.0::float8 / (${rrfkArg} + b.rnk) AS rrf_score, + b.bm25::float8 AS bm25_score, CAST(NULL AS double precision) AS vector_score, + CAST(NULL AS int) AS best_chunk_index, CAST(NULL AS int) AS best_char_start, CAST(NULL AS int) AS best_char_end + FROM fts_ranked b +)`; + } else { + fused = `fused AS ( + SELECT v.message_id, 1.0::float8 / (${rrfkArg} + v.rnk) AS rrf_score, + CAST(NULL AS double precision) AS bm25_score, 1.0::float8 - v.distance AS vector_score, + v.chunk_index AS best_chunk_index, v.chunk_char_start AS best_char_start, v.chunk_char_end AS best_char_end + FROM ann_ranked v +)`; + } + ctes.push(fused); + + const ftsPoolExpr = useFTS ? '(SELECT count(*) FROM fts_pool)' : '0'; + const annPoolExpr = useANN ? '(SELECT count(*) FROM ann_pool)' : '0'; + + const sql = `WITH ${ctes.join(',\n')} +SELECT ${DISPLAY_COLS}, + f.rrf_score, f.bm25_score, f.vector_score, + f.best_chunk_index, f.best_char_start, f.best_char_end, + ${ftsPoolExpr} AS fts_pool_size, + ${annPoolExpr} AS ann_pool_size + FROM fused f + JOIN messages m ON m.id = f.message_id + JOIN email_accounts a ON a.id = m.account_id + ORDER BY f.rrf_score DESC, f.message_id ASC + LIMIT ${limitArg}`; + + const { rows } = await exec.query(sql, args); + let ftsPoolSize = 0; + let annPoolSize = 0; + if (rows.length > 0) { + ftsPoolSize = rows[0].fts_pool_size; + annPoolSize = rows[0].ann_pool_size; + } else { + // Empty result: the pool-size subqueries never fired (they ride the row + // stream). Re-run a prefix-only count over just the pool CTEs and their + // args (drop the trailing rrfk/limit args). Port of fused.go:322-342. + const prefix = `WITH ${poolCTEs.join(',\n')}\n`; + const prefixArgs = args.slice(0, poolArgsLen); + if (useFTS) { + ftsPoolSize = (await exec.query(prefix + 'SELECT count(*)::int AS n FROM fts_pool', prefixArgs)).rows[0].n; + } + if (useANN) { + annPoolSize = (await exec.query(prefix + 'SELECT count(*)::int AS n FROM ann_pool', prefixArgs)).rows[0].n; + } + } + const hits = rows.map((row) => { + const h = { ...row }; + delete h.fts_pool_size; + delete h.ann_pool_size; + return h; + }); + return { hits, ftsPoolSize, annPoolSize }; + } + + // Candidate-widening loop (port of fused.go:346-380). Start wide enough that + // the common single-chunk case is one query; grow innerChunks (doubling, + // capped by chunkCeiling) only while the ANN dedup collapses the pool below + // kPerSignal+1 and more chunks remain. FTS never collapses, so only ANN drives it. + let innerChunks = kPlus1 * FUSED_ANN_CHUNKS_PER_MESSAGE; + let result; + let prevAnnPool = -1; + for (;;) { + if (useANN && chunkCeiling > 0 && innerChunks > chunkCeiling) innerChunks = chunkCeiling; + // Each ANN attempt sets hnsw.ef_search to its OWN inner LIMIT (capped at + // HNSW_EF_SEARCH_MAX): at the pgvector default of 40 the HNSW scan + // truncated every attempt to ~40 chunks and this loop re-ran an identical + // plan up to the ceiling. FTS-only requests skip the transaction — no ANN + // scan, nothing to tune. + result = useANN + ? await withEfSearch(db, innerChunks, (tx) => runFused(tx, innerChunks)) + : await runFused(db, innerChunks); + if (!useANN || + result.annPoolSize >= kPlus1 || + result.annPoolSize >= filteredCeiling || + innerChunks >= chunkCeiling) break; + // A widened re-run that failed to GROW the pool will never grow it (the + // graph/ef_search budget is saturated) — stop instead of doubling toward + // the ceiling for identical results. + if (result.annPoolSize <= prevAnnPool) break; + prevAnnPool = result.annPoolSize; + let next = innerChunks * 2; + if (chunkCeiling > 0 && next > chunkCeiling) next = chunkCeiling; + if (next === innerChunks) break; + innerChunks = next; + } + + const poolSaturated = result.ftsPoolSize > kPerSignal || result.annPoolSize > kPerSignal; + return { hits: result.hits, poolSaturated, generation }; +} diff --git a/backend/src/services/embeddings/vectorStore.unit.test.js b/backend/src/services/embeddings/vectorStore.unit.test.js new file mode 100644 index 0000000..2fdd838 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.unit.test.js @@ -0,0 +1,92 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../db.js', () => ({ pool: {}, withTransaction: vi.fn() })); +import { withTransaction } from '../db.js'; +import { vectorLiteral, searchWiden, stampSkipped } from './vectorStore.js'; + +describe('vectorLiteral', () => { + it('formats a float vector as pgvector text', () => { + expect(vectorLiteral([1, 2.5, -3.25])).toBe('[1,2.5,-3.25]'); + }); + it('emits an empty-brackets literal for an empty vector', () => { + expect(vectorLiteral([])).toBe('[]'); + }); +}); + +describe('searchWiden (A1 filtered early-exit)', () => { + it('early-exits once distinctEarlyExit is reached (filtered path, one run)', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return [{ messageId: 'a' }, { messageId: 'b' }]; }; + const hits = await searchWiden(5, 1000, 2, run); // k=5, ceiling=1000, earlyExit=2 + expect(runs).toEqual([20]); // did NOT widen up to the 1000-chunk ceiling + expect(hits.map((h) => h.rank)).toEqual([1, 2]); + }); + + it('widens by doubling until k distinct survive (no-filter: earlyExit=k)', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return Array.from({ length: Math.min(Math.floor(n / 10), 5) }, (_, i) => ({ messageId: 'm' + i })); }; + const hits = await searchWiden(5, 100, 5, run); + expect(runs).toEqual([20, 40, 80]); // 2 < 5, 4 < 5, then 8 → sliced to 5 + expect(hits).toHaveLength(5); + }); + + it('stops at the ceiling even when k is never reached', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return [{ messageId: 'only' }]; }; + const hits = await searchWiden(5, 30, 5, run); + expect(runs).toEqual([20, 30]); // clamps the doubled 40 to the 30 ceiling, then stops + expect(hits).toHaveLength(1); + }); +}); + +describe('stampSkipped (Wave D Fix 7 — message_count decrement)', () => { + beforeEach(() => { withTransaction.mockReset(); }); + + function scriptedClient({ deletedDistinct = 2 } = {}) { + const calls = []; + const client = { + calls, + query: vi.fn(async (text, params) => { + calls.push({ text, params }); + if (/FROM index_generations WHERE id = \$1 FOR UPDATE/.test(text)) return { rows: [{ id: params[0] }] }; + if (/^UPDATE messages SET embed_gen = \$1 WHERE id = \$2 AND/.test(text)) { + return { rowCount: params[1] === 'cas-missed' ? 0 : 1 }; // CAS stamp + } + if (/COUNT\(DISTINCT message_id\)/.test(text)) return { rows: [{ n: deletedDistinct }] }; + return { rows: [], rowCount: 1 }; + }), + }; + withTransaction.mockImplementation(async (fn) => fn(client)); + return client; + } + + it('decrements message_count by the DISTINCT messages actually deleted, in the same tx, under the generation row lock', async () => { + const client = scriptedClient({ deletedDistinct: 2 }); + const missed = await stampSkipped(9, + [{ id: 'cas-ok', lastModified: 't1' }, { id: 'cas-missed', lastModified: 't2' }], + ['plain-1']); + expect(missed).toEqual(['cas-missed']); + const texts = client.calls.map((c) => c.text); + // Generation row locked FIRST — same lock order as upsert()/msgvault Delete. + expect(texts[0]).toMatch(/SELECT id FROM index_generations WHERE id = \$1 FOR UPDATE/); + // Vectors deleted only for ids whose stamp landed (CAS-missed keeps its vector)… + const del = client.calls.find((c) => /DELETE FROM embeddings/.test(c.text)); + expect(del.params).toEqual([9, ['cas-ok', 'plain-1']]); + // …counted BEFORE the delete, and the count decrements the generation. + expect(texts.findIndex((t) => /COUNT\(DISTINCT message_id\)/.test(t))) + .toBeLessThan(texts.findIndex((t) => /DELETE FROM embeddings/.test(t))); + const upd = client.calls.find((c) => /UPDATE index_generations SET message_count = message_count - \$1/.test(c.text)); + expect(upd.params).toEqual([2, 9]); + }); + + it('skips the decrement when the stamped ids had no vectors to delete', async () => { + const client = scriptedClient({ deletedDistinct: 0 }); + await stampSkipped(9, [{ id: 'cas-ok', lastModified: 't1' }], []); + expect(client.calls.some((c) => /UPDATE index_generations SET message_count/.test(c.text))).toBe(false); + expect(client.calls.some((c) => /DELETE FROM embeddings/.test(c.text))).toBe(true); + }); + + it('is a no-op (no transaction) for empty inputs', async () => { + expect(await stampSkipped(9, [], [])).toEqual([]); + expect(withTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/embeddings/worker.integration.test.js b/backend/src/services/embeddings/worker.integration.test.js new file mode 100644 index 0000000..353d257 --- /dev/null +++ b/backend/src/services/embeddings/worker.integration.test.js @@ -0,0 +1,58 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('worker end-to-end (pgvector)', () => { + let store, generations, worker, client, acctId, gen, userId; + const DIM = 4; + const fakeClient = { async embed(inputs) { return inputs.map((_, i) => [0.1, 0.2, 0.3, 0.4 + i * 1e-6]); } }; + + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + generations = await import('./generations.js'); + const { EmbeddingWorker } = await import('./worker.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(DIM); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // embed_runs FKs index_generations without ON DELETE CASCADE, so clear it first. + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); + // runOnce scans EVERY live embed_gen-IS-NULL message, so stray rows left by + // other IT files (withTestDb truncates before each test, not after the last) + // would inflate the coverage counts — clear messages like the tables above. + await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'wrk')); + for (let i = 0; i < 40; i++) { + await client.query(`INSERT INTO messages (account_id, uid, folder, subject, body_text) VALUES ($1,$2,'INBOX',$3,$4)`, + [acctId, 83000 + i, `subject ${i}`, `body content number ${i}`]); + } + gen = await generations.createGeneration('fake', DIM, 'fp-e2e'); + worker = new EmbeddingWorker({ store, client: fakeClient, generations, preprocessCfg: { stripHTML: true, collapseWhitespace: true }, maxInputChars: 32768, batchSize: 8 }); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('drives the corpus to full coverage and auto-activates the building generation', async () => { + // The worker was constructed with `generations` in its deps (above), so + // draining the scan for the building generation promotes it to active at the + // worker seam — NO manual activateGeneration call here. This is the regression + // guard for the wiring that was missing in production. + const res = await worker.runOnce(gen); + expect(res.succeeded).toBe(40); + const pending = await client.query('SELECT COUNT(*)::int n FROM messages WHERE account_id = $1 AND embed_gen IS DISTINCT FROM $2', [acctId, gen]); + expect(pending.rows[0].n).toBe(0); + const emb = await client.query('SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1', [gen]); + expect(emb.rows[0].n).toBe(40); + expect((await generations.activeGeneration()).id).toBe(gen); + expect(await generations.buildingGeneration()).toBeNull(); // promoted, no longer building + }); + + it('re-run is a no-op (idempotent coverage)', async () => { + const res = await worker.runOnce(gen); + expect(res.claimed).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/worker.js b/backend/src/services/embeddings/worker.js new file mode 100644 index 0000000..8ea2caf --- /dev/null +++ b/backend/src/services/embeddings/worker.js @@ -0,0 +1,354 @@ +// Port of internal/vector/embed/worker.go RunOnce/RunBackstop (scan-and-fill). +import { preprocess } from './preprocess.js'; +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; +import { isPermanent4xx } from './client.js'; + +// worker.go rawBodyMultiplier. Exported for chunkmatch.js, whose read-path re-preprocess +// must derive the identical maxBodyRunes cap or chunk offsets misalign on huge bodies. +export const RAW_BODY_MULT = 16; + +export class EmbeddingWorker { + constructor(deps) { + this.deps = { batchSize: 32, maxConsecutiveFailures: 5, log: console, ...deps }; + } + + runOnce(gen) { return this._run(gen, false); } + runBackstop(gen) { return this._run(gen, true); } + + async _run(gen, backstop) { + const { store } = this.deps; + const res = { claimed: 0, succeeded: 0, failed: 0, truncated: 0 }; + let runId; + let runErr = null; + try { runId = (await store.startEmbedRun?.(gen)) || 0; } catch { runId = 0; } + try { + let consecutiveFailures = 0; + let afterId = backstop ? store.ZERO_UUID : await store.getWatermark(gen); + for (;;) { + const ids = await store.scanForEmbedding(gen, afterId, this.deps.batchSize); + if (!ids.length) { + if (!backstop) await store.resetWatermark(gen); // re-scan from start next tick + // Coverage reached: the shared driver seam (build route + scheduler) that + // promotes a fully-embedded 'building' generation to 'active'. + await this._activateIfBuildingCovered(gen); + return res; + } + res.claimed += ids.length; + const batchMax = ids[ids.length - 1]; + + let eb; + try { + eb = await this._embedBatch(gen, ids); + } catch (err) { + consecutiveFailures++; + this.deps.log.warn?.(`embed batch failed (gen ${gen}): ${err.message}`); + + if (isPermanent4xx(err)) { + // Downshift to size=1 and drain (worker.go RunOnce ErrPermanent4xx branch): + // embed what succeeds, stamp-drop confirmed per-message 4xx offenders, and + // leave rows unstamped only on an all-drop where the endpoint embedded nothing. + this.deps.log.info?.(`embed: downshifting to size=1 to drain failing batch (gen ${gen}, ${ids.length})`); + const dr = await this._downshiftDrain(gen, ids, res); + res.succeeded += dr.embedded; + // Reset the cap on embeddedOK (endpoint embedded+upserted something) — a + // CAS-missed stamp still proves the endpoint is healthy. + if (dr.embeddedOK > 0) consecutiveFailures = 0; + // Advance ONLY past the contiguously-stamped prefix so an unstamped + // straggler is never skipped (safeAdvanceID == batchMax on a clean drain). + if (dr.safeAdvanceID && dr.safeAdvanceID > afterId) { + afterId = dr.safeAdvanceID; + if (!backstop) await store.setWatermark(gen, dr.safeAdvanceID); + } + if (dr.drainErr) { + if (dr.drainErr.code === 'GEN_RETIRED') { this.deps.log.info?.(`generation ${gen} retired mid-drain; stopping`); return res; } + // A transient (non-4xx) error during the drain is a hard abort (an earlier + // singleton may have stamped, so the watermark stopped at the contiguous + // prefix and the next run re-finds the rest). + if (!isPermanent4xx(dr.drainErr)) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${dr.drainErr.message}`, { cause: err }); + } + // All-drop 4xx: the rows stay unstamped; let the failure cap trip. + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${dr.drainErr.message}`, { cause: err }); + } + } + continue; + } + + // Non-4xx (transient) error: leave the batch unstamped, do not advance the + // cursor, so the failure cap short-circuits a persistent fault. + res.failed += ids.length; + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${err.message}`, { cause: err }); + } + continue; // do not advance cursor; next scan re-finds the batch + } + res.truncated += eb.truncated; + const skipIds = [...eb.missing, ...eb.empty]; + + if (!eb.chunks.length) { + if (skipIds.length) await this._stampSkipped(gen, skipIds, eb.lastModified); + consecutiveFailures = 0; + afterId = batchMax; + if (!backstop) await store.setWatermark(gen, batchMax); + continue; + } + + // Step 1: upsert embeddings FIRST (idempotent; crash-safe ordering). + try { + await store.upsert(gen, eb.chunks); + } catch (err) { + if (err.code === 'GEN_RETIRED') { this.deps.log.info?.(`generation ${gen} retired mid-run; stopping`); return res; } + consecutiveFailures++; + res.failed += eb.embeddedIds.length; + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${err.message}`, { cause: err }); + } + continue; + } + + // Step 2: skip-mark empty/missing, then CAS-stamp embedded rows. + if (skipIds.length) await this._stampSkipped(gen, skipIds, eb.lastModified); + const missed = await store.setEmbedGenIfUnchanged( + eb.embeddedIds.map((id) => ({ id, lastModified: eb.lastModified.get(id) })), gen, + ); + if (missed.length) this.deps.log.info?.(`embed_gen CAS misses (concurrent edit): ${missed.length} — backstop recovers`); + res.succeeded += eb.embeddedIds.length - missed.length; + consecutiveFailures = 0; + // Advance the watermark even on a whole-batch CAS miss (backstop recovers misses). + afterId = batchMax; + if (!backstop) await store.setWatermark(gen, batchMax); + this.deps.onProgress?.({ done: res.succeeded, claimed: res.claimed, truncated: res.truncated }); + } + } catch (err) { + runErr = err; + throw err; + } finally { + try { await store.finalizeEmbedRun?.(runId, res, runErr); } catch { /* observability only */ } + } + } + + // Activation-on-coverage seam (worker.go parity — see divergence note below). + // A run reaches here only after scanForEmbedding drains, i.e. no live message + // still needs embedding for `gen` (poison messages count as covered — they are + // stamped, not skipped). When `gen` is the CURRENT building generation, promote + // it to 'active'. This is the sole production caller of activateGeneration: + // without it a completed build stays 'building' forever and hybrid search never + // leaves its index_building fallback. + // + // No-ops when: + // - no generations collaborator is wired (a bare worker with nothing to promote), or + // - `gen` is not the building generation — an active generation's incremental + // or backstop run must never re-activate (README: generations never mix). + // + // Tolerates the two lifecycle races activateGeneration encodes, so a completed + // build is never lost and activation never fails the embed run: + // - "still has messages needing embedding": a late content edit re-NULLed a row + // between the drain and the activate (the coverage gate re-asserted inside the + // activate tx caught it) — leave it building; the next scheduler tick re-drains + // and retries. + // - "not in 'building' state": a concurrent activate/retire already moved it — no-op. + // Any other error is logged, not thrown: activation is a post-coverage promotion, + // not part of the run result. + // + // Divergence from msgvault worker.go: there activation lives in the DRIVERS + // (scheduler embed_job.go + the CLI embed_vector.go each re-check coverage and + // call ActivateGeneration). We fold it into this shared worker seam instead so + // BOTH Mailflow drivers (the fire-and-forget build route and the scheduler tick) + // get it from one place and cannot drift out of sync — the exact failure that + // left the live build wedged in 'building'. The coverage precondition (scan + // drained) and the no-force, gate-re-asserted activate call match worker.go. + async _activateIfBuildingCovered(gen) { + const gens = this.deps.generations; + if (!gens?.activateGeneration || !gens?.buildingGeneration) return; + let building; + try { + building = await gens.buildingGeneration(); + } catch (err) { + this.deps.log.warn?.(`embed: could not read building generation to activate ${gen}: ${err.message}`); + return; + } + if (!building || building.id !== gen) return; // active/incremental run — nothing to promote + try { + await gens.activateGeneration(gen); // no force — activateGeneration re-asserts the coverage gate atomically + this.deps.log.info?.(`embed: generation ${gen} fully covered — activated`); + } catch (err) { + this.deps.log.warn?.(`embed: generation ${gen} not activated (${err.message}); scheduler will retry next tick if still building`); + } + } + + async _embedBatch(gen, ids) { + const { store } = this.deps; + const rows = await store.fetchForEmbedding(ids); + const fetched = new Set(); + const lastModified = new Map(); + const msgs = []; + const empty = []; + for (const r of rows) { + fetched.add(r.id); + lastModified.set(r.id, r.lastModified); + const body = r.bodyText && r.bodyText.trim() !== '' ? r.bodyText : (r.bodyHtml || ''); + const ppCfg = { ...this.deps.preprocessCfg }; + if (!ppCfg.maxBodyRunes && this.deps.maxInputChars > 0) { + ppCfg.maxBodyRunes = this.deps.maxInputChars * MAX_SPANS * RAW_BODY_MULT; + } + const { text, truncated } = preprocess(r.subject || '', body, 0, ppCfg); + if (text.trim() === '') { empty.push(r.id); continue; } + msgs.push({ id: r.id, text, bodyTruncated: truncated }); + } + const missing = ids.filter((id) => !fetched.has(id)); + if (!msgs.length) return { chunks: [], embeddedIds: [], missing, empty, truncated: 0, lastModified }; + + const window = this.deps.maxInputChars; + const overlap = chunkOverlapFor(window); + const pieces = []; + const inputs = []; + const truncatedMsg = new Set(); + for (const m of msgs) { + const { spans, tailDropped } = chunkText(m.text, window, overlap, MAX_SPANS); + const msgTrunc = m.bodyTruncated || tailDropped; + spans.forEach((sp, j) => { + const hardCut = window > 0 && (sp.charEnd - sp.charStart) === window && j < spans.length - 1; + const trunc = msgTrunc || hardCut; + if (trunc) truncatedMsg.add(m.id); + pieces.push({ id: m.id, chunkIndex: j, text: sp.text, chars: sp.charEnd - sp.charStart, charStart: sp.charStart, charEnd: sp.charEnd, trunc }); + inputs.push(sp.text); + }); + } + + const vecs = []; + try { + for (let i = 0; i < inputs.length; i += this.deps.batchSize) { + const got = await this.deps.client.embed(inputs.slice(i, i + this.deps.batchSize)); + vecs.push(...got); + } + } catch (err) { + // Attach the CAS tokens fetched before the embed call so a downshift drain can + // CAS-drop a per-message 4xx offender with the last_modified read at fetch time. + err.lastModified = lastModified; + throw err; + } + if (vecs.length !== pieces.length) throw new Error(`embedder returned ${vecs.length} vectors for ${pieces.length} chunk inputs`); + + const chunks = []; + const embeddedIds = []; + const seen = new Set(); + for (let i = 0; i < pieces.length; i++) { + const p = pieces[i]; + chunks.push({ messageId: p.id, chunkIndex: p.chunkIndex, vector: vecs[i], sourceCharLen: p.chars, chunkCharStart: p.charStart, chunkCharEnd: p.charEnd, truncated: p.trunc }); + if (!seen.has(p.id)) { seen.add(p.id); embeddedIds.push(p.id); } + } + return { chunks, embeddedIds, missing, empty, truncated: truncatedMsg.size, lastModified }; + } + + // Port of worker.go downshiftDrain: walk a 4xx-failing batch one message at a time. + // Embed + upsert + CAS-stamp what succeeds; skip-mark empty/missing; and DEFER + // per-message 4xx offenders — stamp-dropping them only when some sibling embedded + // (proving the endpoint is healthy, so the 4xx is message-specific). On an all-drop + // (endpoint embedded nothing) the deferred ids are left UNSTAMPED and a permanent-4xx + // error is returned so a misconfigured endpoint never silently loses work. Returns + // { embedded, embeddedOK, stamped, safeAdvanceID, drainErr }; safeAdvanceID is the + // highest CONTIGUOUSLY-stamped id (batchMax on a clean drain) so the caller never + // advances the watermark past an unstamped straggler. + async _downshiftDrain(gen, ids, res) { + const { store } = this.deps; + let embedded = 0; + let embeddedOK = 0; + let stamped = 0; + let contiguousStampedID = null; + let brokeContiguity = false; + const deferredDrops = []; + let lastDeferredErr = null; + const lm = new Map(); + const advance = (id, didStamp) => { + if (didStamp) { if (!brokeContiguity) contiguousStampedID = id; } + else brokeContiguity = true; + }; + + for (const id of ids) { + let eb; + try { + eb = await this._embedBatch(gen, [id]); + } catch (e) { + if (isPermanent4xx(e)) { + if (e.lastModified) for (const [k, v] of e.lastModified) lm.set(k, v); + deferredDrops.push(id); + lastDeferredErr = e; + brokeContiguity = true; // a deferred id breaks the stamped-from-start prefix + continue; + } + // Transient error: leave this id unstamped and abort the drain; the watermark + // stays at the contiguous stamped prefix so the next run re-finds it. + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: e }; + } + for (const [k, v] of eb.lastModified) lm.set(k, v); + + if (!eb.chunks.length) { + // Missing/empty singleton — skip-mark it. + const skip = [...eb.missing, ...eb.empty]; + let didStamp = true; + if (skip.length) { + const missed = await this._stampSkipped(gen, skip, eb.lastModified); + const stampedSkip = skip.length - missed.length; + stamped += stampedSkip; + didStamp = stampedSkip > 0; // a CAS-missed skip leaves the row unstamped + } + advance(id, didStamp); + continue; + } + + try { + await store.upsert(gen, eb.chunks); + } catch (uerr) { + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: uerr }; + } + embeddedOK++; // endpoint demonstrably embedded + upserted this singleton + const missed = await store.setEmbedGenIfUnchanged( + eb.embeddedIds.map((mid) => ({ id: mid, lastModified: eb.lastModified.get(mid) })), gen, + ); + const stampedHere = eb.embeddedIds.length - missed.length; + res.truncated += eb.truncated; + embedded += stampedHere; + stamped += stampedHere; + advance(id, stampedHere > 0); + } + + // Clean drain: every id is resolved (stamped, skip-marked, or a deferred 4xx below). + const safeAdvanceID = ids[ids.length - 1]; + + if (!deferredDrops.length) return { embedded, embeddedOK, stamped, safeAdvanceID, drainErr: null }; + + if (embeddedOK > 0) { + // The endpoint embedded something, so the 4xxs are message-specific — stamp-drop them. + for (const id of deferredDrops) { + this.deps.log.warn?.(`stamping (dropping) message after singleton 4xx (gen ${gen}, id ${id}): ${lastDeferredErr?.message}`); + } + const missed = await this._stampSkipped(gen, deferredDrops, lm); + stamped += deferredDrops.length - missed.length; + return { embedded, embeddedOK, stamped, safeAdvanceID, drainErr: null }; + } + + // embeddedOK === 0: endpoint embedded nothing — can't distinguish an endpoint-wide + // failure from a batch where every message is unembeddable. Leave the deferred ids + // UNSTAMPED and surface the 4xx (marked permanent so the caller's cap, not a hard + // abort, governs) so a misconfigured endpoint does not silently drop work. + const err = new Error(`downshift all-drop: every singleton returned non-retryable 4xx (left ${deferredDrops.length} row(s) unstamped): ${lastDeferredErr?.message}`); + err.permanent4xx = true; + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: err }; + } + + // Skip-mark empty/missing rows (drops them from the next scan) and remove any stale + // vectors for the rows that were actually stamped — the stamp + prune happen in ONE + // transaction inside store.stampSkipped (msgvault worker.go parity). CAS-protected + // for rows with a last_modified token; unconditional for missing rows. + async _stampSkipped(gen, ids, lastModified) { + const { store } = this.deps; + const cas = []; + const plain = []; + for (const id of ids) { + if (lastModified.has(id)) cas.push({ id, lastModified: lastModified.get(id) }); + else plain.push(id); + } + return store.stampSkipped(gen, cas, plain); + } +} diff --git a/backend/src/services/embeddings/worker.test.js b/backend/src/services/embeddings/worker.test.js new file mode 100644 index 0000000..e61692d --- /dev/null +++ b/backend/src/services/embeddings/worker.test.js @@ -0,0 +1,250 @@ +import { describe, it, expect, vi } from 'vitest'; +import { EmbeddingWorker } from './worker.js'; + +const ZERO = '00000000-0000-0000-0000-000000000000'; +const uid = (n) => `00000000-0000-0000-0000-${String(n).padStart(12, '0')}`; + +function makeStore(messages, opts = {}) { + const watermark = new Map(); + const state = { upsertCalls: 0, resetCalls: 0 }; + return { + ZERO_UUID: ZERO, + _messages: messages, + _state: state, + async scanForEmbedding(target, afterId, limit) { + return messages + .filter((m) => (m.embedGen == null || m.embedGen !== target) && !m.isDeleted && m.id > afterId) + .sort((a, b) => (a.id < b.id ? -1 : 1)) + .slice(0, limit) + .map((m) => m.id); + }, + async fetchForEmbedding(ids) { + return messages.filter((m) => ids.includes(m.id)).map((m) => ({ + id: m.id, subject: m.subject || '', bodyText: m.bodyText || '', bodyHtml: m.bodyHtml || '', lastModified: m.lastModified, + })); + }, + async upsert() { state.upsertCalls++; }, + async setEmbedGen(ids, target) { for (const id of ids) { const m = messages.find((x) => x.id === id); if (m) m.embedGen = target; } }, + async setEmbedGenIfUnchanged(items, target) { + const missed = []; + for (const it of items) { + const m = messages.find((x) => x.id === it.id); + if (opts.casMiss?.has(it.id)) { missed.push(it.id); continue; } + if (m && m.lastModified === it.lastModified) m.embedGen = target; else missed.push(it.id); + } + return missed; + }, + async stampSkipped(target, casItems, plainIds) { + const missed = []; + for (const it of casItems) { + const m = messages.find((x) => x.id === it.id); + if (opts.casMiss?.has(it.id)) { missed.push(it.id); continue; } + if (m && m.lastModified === it.lastModified) m.embedGen = target; else missed.push(it.id); + } + for (const id of plainIds) { const m = messages.find((x) => x.id === id); if (m) m.embedGen = target; } + return missed; + }, + async getWatermark(gen) { state.getWatermarkCalled = true; return watermark.get(gen) || ZERO; }, + async setWatermark(gen, id) { watermark.set(gen, id); }, + async resetWatermark(gen) { state.resetCalls++; watermark.set(gen, ZERO); }, + }; +} +const fakeClient = { async embed(inputs) { return inputs.map(() => [0.1, 0.2, 0.3, 0.4]); } }; +const deps = (store, over = {}) => ({ store, client: fakeClient, preprocessCfg: {}, maxInputChars: 32768, batchSize: 32, ...over }); + +// A generations collaborator stub for the activation seam. `building` is what +// buildingGeneration() returns (null ⇒ the run's gen is NOT building — an active +// or incremental run); `activate` is the activateGeneration spy. +function makeGenerations({ building = null, activate } = {}) { + return { + buildingGeneration: vi.fn(async () => building), + activateGeneration: activate || vi.fn(async () => {}), + }; +} + +describe('EmbeddingWorker.runOnce', () => { + it('embeds pending messages and stamps embed_gen', async () => { + const msgs = [ + { id: uid(1), subject: 'hello', bodyText: 'world', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'foo', bodyText: 'bar', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs); + const w = new EmbeddingWorker(deps(store)); + const res = await w.runOnce('7'); + expect(res.claimed).toBe(2); + expect(res.succeeded).toBe(2); + expect(msgs.every((m) => m.embedGen === '7')).toBe(true); + }); + + it('excludes a CAS miss from succeeded but still advances the watermark', async () => { + const msgs = [ + { id: uid(1), subject: 's1', bodyText: 'b1', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 's2', bodyText: 'b2', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs, { casMiss: new Set([uid(2)]) }); + const w = new EmbeddingWorker(deps(store)); + const res = await w.runOnce('7'); + expect(res.succeeded).toBe(1); + const m2 = msgs.find((m) => m.id === uid(2)); + expect(m2.embedGen).toBeNull(); // recovered by backstop later + }); + + it('is idempotent — a second run finds nothing', async () => { + const msgs = [{ id: uid(1), subject: 's', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + const w = new EmbeddingWorker(deps(store)); + await w.runOnce('7'); + const res2 = await w.runOnce('7'); + expect(res2.claimed).toBe(0); + }); + + it('resets the watermark on scan exhaustion (P2)', async () => { + const store = makeStore([]); + const w = new EmbeddingWorker(deps(store)); + await w.runOnce('7'); + expect(store._state.resetCalls).toBeGreaterThan(0); + }); + + it('aborts after maxConsecutiveFailures on a persistent embed failure (does not loop)', async () => { + const msgs = Array.from({ length: 10 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const scanSpy = vi.spyOn(store, 'scanForEmbedding'); + const failingClient = { async embed() { throw new Error('unreachable'); } }; + const w = new EmbeddingWorker(deps(store, { client: failingClient, maxConsecutiveFailures: 3 })); + await expect(w.runOnce('7')).rejects.toThrow(/aborting after 3 consecutive failures/); + expect(scanSpy.mock.calls.length).toBe(3); // bounded, not infinite + }); + + it('downshifts on a permanent 4xx: isolates the poison message and embeds the rest (W1)', async () => { + const msgs = [ + { id: uid(1), subject: 'good one', bodyText: 'hello', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'poison here', bodyText: 'x', lastModified: 't2', embedGen: null }, + { id: uid(3), subject: 'good two', bodyText: 'world', lastModified: 't3', embedGen: null }, + ]; + const store = makeStore(msgs); + // 4xx whenever the batch contains the poison text; succeeds otherwise. + const poisonClient = { + async embed(inputs) { + if (inputs.some((t) => t.includes('poison'))) { const e = new Error('unembeddable input'); e.permanent4xx = true; throw e; } + return inputs.map(() => [0.1, 0.2, 0.3, 0.4]); + }, + }; + const w = new EmbeddingWorker(deps(store, { client: poisonClient, maxConsecutiveFailures: 3 })); + const res = await w.runOnce('7'); + expect(msgs.find((m) => m.id === uid(1)).embedGen).toBe('7'); // embedded + expect(msgs.find((m) => m.id === uid(3)).embedGen).toBe('7'); // embedded + expect(msgs.find((m) => m.id === uid(2)).embedGen).toBe('7'); // stamp-dropped (leaves the scan) + expect(res.succeeded).toBe(2); // only the two good ones count as embedded + }); + + it('all-4xx batch leaves rows unstamped and aborts after maxConsecutiveFailures (no silent drop) (W1)', async () => { + const msgs = Array.from({ length: 4 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const scanSpy = vi.spyOn(store, 'scanForEmbedding'); + const allBadClient = { async embed() { const e = new Error('always bad'); e.permanent4xx = true; throw e; } }; + const w = new EmbeddingWorker(deps(store, { client: allBadClient, maxConsecutiveFailures: 3 })); + await expect(w.runOnce('7')).rejects.toThrow(/aborting after 3 consecutive failures/); + expect(msgs.every((m) => m.embedGen === null)).toBe(true); // nothing silently dropped + expect(scanSpy.mock.calls.length).toBe(3); // bounded + }); +}); + +// The activation seam: the SHARED worker-run completion point that both drivers +// (the manual build route and the scheduler) reach. When a run drains the scan +// for the generation that is currently 'building', the worker promotes it to +// 'active' — the production wiring that was missing (activateGeneration had no +// non-test caller, so completed builds stayed 'building' forever). +describe('EmbeddingWorker activation seam', () => { + it('activates the building generation once, without force, after the scan drains', async () => { + const gen = '7'; + const msgs = [ + { id: uid(1), subject: 'a', bodyText: 'b', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'c', bodyText: 'd', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + const res = await w.runOnce(gen); + expect(res.succeeded).toBe(2); + expect(generations.buildingGeneration).toHaveBeenCalled(); + expect(activateSpy).toHaveBeenCalledTimes(1); + expect(activateSpy.mock.calls[0]).toEqual([gen]); // called with the gen only — no force + }); + + it('does not activate when the run aborts before draining the scan', async () => { + const gen = '7'; + const msgs = Array.from({ length: 6 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const failingClient = { async embed() { throw new Error('unreachable'); } }; + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { client: failingClient, maxConsecutiveFailures: 2, generations })); + await expect(w.runOnce(gen)).rejects.toThrow(/aborting after/); + expect(activateSpy).not.toHaveBeenCalled(); // the scan never drained → no coverage → no activation + }); + + it('never activates an active generation on an incremental run (no building generation)', async () => { + const gen = '7'; + const store = makeStore([]); // nothing pending — scan drains immediately + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: null, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runOnce(gen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('never activates an active generation on a backstop run', async () => { + const gen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: null, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runBackstop(gen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('does not activate the running generation when a DIFFERENT generation is building (rebuild in progress)', async () => { + const activeGen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: '8' }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runOnce(activeGen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('swallows the "still has messages" activation race and leaves the run result intact', async () => { + const gen = '7'; + const msgs = [{ id: uid(1), subject: 'a', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + const activateSpy = vi.fn(async () => { throw new Error(`generation ${gen} still has messages needing embedding; pass force to override`); }); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + const res = await w.runOnce(gen); // must NOT reject — activation is a post-coverage promotion + expect(res.succeeded).toBe(1); + expect(activateSpy).toHaveBeenCalledTimes(1); + }); + + it('swallows the "not in building state" activation race (already active/retired)', async () => { + const gen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => { throw new Error(`generation ${gen} not in 'building' state`); }); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await expect(w.runOnce(gen)).resolves.toBeTruthy(); + expect(activateSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('EmbeddingWorker.runBackstop', () => { + it('ignores the watermark and finds a sub-watermark straggler', async () => { + const msgs = [{ id: uid(1), subject: 's', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + await store.setWatermark('7', uid(999)); // high watermark that would hide id 1 + const w = new EmbeddingWorker(deps(store)); + const res = await w.runBackstop('7'); + expect(res.succeeded).toBe(1); + expect(msgs[0].embedGen).toBe('7'); + }); +}); diff --git a/backend/src/services/imapManager.js b/backend/src/services/imapManager.js index fb7b544..ade5240 100644 --- a/backend/src/services/imapManager.js +++ b/backend/src/services/imapManager.js @@ -400,6 +400,10 @@ function safeDate(d) { // flagPollEveryTicks: for non-push flag providers, poll flags every N successful sync ticks. // snippetIndex: run the background snippet indexer after backfill. // Disabled for providers that throttle body fetches too aggressively. +// bodyBackfill: run the background body-materialization drainer +// (services/bodyBackfill.js) for this provider. Enabled only for +// providers that tolerate sustained background BODY[] fetches; +// Gmail/PurelyMail/Microsoft stay off in v1 (unproven / throttle-hostile). // skipFolderPatterns: folder path substrings to skip during backfill (label-view dedup). // skipFolderNames: exact folder paths to skip (non-selectable namespace containers). // batchSize/Delay/errorDelay/batchesPerConn: backfill rate-limit tuning. @@ -414,6 +418,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: false, snippetIndex: false, + bodyBackfill: false, speculativeFetch: false, skipFolderPatterns: ['all mail', '[gmail]/starred', '[gmail]/important'], // [Gmail] is a namespace container — not a selectable mailbox. It must be @@ -425,6 +430,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: false, skipFolderPatterns: [], skipFolderNames: [], @@ -435,6 +441,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -444,6 +451,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: false, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -471,6 +479,7 @@ const PROVIDERS = { usesIdle: false, pushesFlags: false, snippetIndex: false, + bodyBackfill: false, speculativeFetch: false, preferFreshBodyFetch: true, freshInboxSync: true, @@ -489,6 +498,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -3637,6 +3647,30 @@ export class ImapManager { // Called in the background (via setImmediate) so it doesn't block the sync path. // By the time the user clicks the email (typically 2–10s later), the body is already // in the DB and the click returns instantly without a live IMAP round-trip. + // Shared fetch+sanitize+snippet+UPDATE step behind every body-materialization path + // (opportunistic new-mail prefetch, on-view folder prefetch, and the body-backfill + // drainer). Kept as one helper so the UPDATE shape that fires the search_fts/last_modified + // triggers never drifts between callers. Returns true if a body was actually written. + // Private (underscore-prefixed, matching this file's convention for internal helpers like + // _enqueueFlagPush) — not new public surface, so it doesn't count against the + // one-narrow-method invariant that governs fetchBodiesForMessages. + async _fetchAndStoreBody(account, msg) { + const { html, text, attachments } = await this.fetchMessageBody( + account, msg.uid, msg.folder || 'INBOX' + ); + const safeHtml = html ? sanitizeEmail(html) : null; + if (!safeHtml && !text) return false; + const snip = snippetFromBody(text, safeHtml || html); + await query( + `UPDATE messages + SET body_html = $1, body_text = $2, attachments = $3, + snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END + WHERE id = $4`, + [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] + ); + return true; + } + async prefetchNewMessageBodies(account, messages) { for (const msg of messages) { try { @@ -3647,26 +3681,58 @@ export class ImapManager { ); if (existing.rows.length) continue; - const { html, text, attachments } = await this.fetchMessageBody( - account, msg.uid, msg.folder || 'INBOX' - ); - const safeHtml = html ? sanitizeEmail(html) : null; - if (safeHtml || text) { - const snip = snippetFromBody(text, safeHtml || html); - await query( - `UPDATE messages - SET body_html = $1, body_text = $2, attachments = $3, - snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END - WHERE id = $4`, - [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] - ); - } + await this._fetchAndStoreBody(account, msg); } catch (err) { console.warn(`Body prefetch failed for uid ${msg.uid}:`, err.message); } } } + // Narrow entry point for the body-materialization drainer (services/bodyBackfill.js). + // Given a set of message IDs, fetch each body over IMAP and persist it with the same + // UPDATE the prefetch path uses. This is the ONLY method the drainer calls into the + // singleton — all batching, pacing, provider gating, and progress live in bodyBackfill.js + // (imapManager invariant, specs/search-overhaul/README.md). Returns the number of bodies + // actually written. + // + // A single poison message (deleted/renamed folder, a per-message server-side rejection) + // must not wedge the whole batch — bodyBackfill.js's keyset cursor only advances past ids + // this method actually processed, and its circuit breaker only trips on a batch that made + // zero progress. So each message's fetch+store is individually caught and skipped here; + // only when the WHOLE batch fails (zero successes and at least one error — almost always a + // dead connection, not a bad message) do we rethrow, so the drainer's breaker can still back + // a genuinely broken account off. + async fetchBodiesForMessages(accountId, messageIds) { + if (!messageIds.length) return { fetched: 0 }; + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + if (!accountResult.rows.length) return { fetched: 0 }; + const account = accountResult.rows[0]; + + // Only touch rows that still lack a body — a concurrent open/prefetch may have filled + // some since the drainer selected them. + const rowsResult = await query( + `SELECT id, uid, folder FROM messages + WHERE id = ANY($1::uuid[]) AND body_html IS NULL AND body_text IS NULL`, + [messageIds] + ); + + let fetched = 0; + let errors = 0; + let lastError = null; + for (const msg of rowsResult.rows) { + try { + if (await this._fetchAndStoreBody(account, msg)) fetched++; + } catch (err) { + errors++; + lastError = err; + console.warn(`Body backfill: skipped message ${msg.id} (uid ${msg.uid}):`, err.message); + } + } + if (fetched === 0 && errors > 0) throw lastError; + return { fetched }; + } + // Background body prefetch for messages currently visible in a folder. // Called after GET /messages responds so the user gets a fast first impression // without waiting for this work. Respects the quiet window — pauses between @@ -3700,18 +3766,7 @@ export class ImapManager { ); if (existing.rows.length) continue; - const { html, text, attachments } = await this.fetchMessageBody(account, msg.uid, msg.folder); - const safeHtml = html ? sanitizeEmail(html) : null; - if (safeHtml || text) { - const snip = snippetFromBody(text, safeHtml || html); - await query( - `UPDATE messages - SET body_html = $1, body_text = $2, attachments = $3, - snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END - WHERE id = $4`, - [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] - ); - } + await this._fetchAndStoreBody(account, msg); } catch (err) { console.warn(`Folder body prefetch failed for uid ${msg.uid}:`, err.message); } diff --git a/backend/src/services/imapManager.test.js b/backend/src/services/imapManager.test.js index 7ae8abd..337f607 100644 --- a/backend/src/services/imapManager.test.js +++ b/backend/src/services/imapManager.test.js @@ -11,10 +11,12 @@ vi.mock('../utils/redact.js', () => ({ redactEmail: vi.fn() })); vi.mock('./hostValidation.js', () => ({ resolveForConnection: vi.fn() })); vi.mock('./gtdTransitions.js', () => ({ runGtdTransitions: vi.fn(), threadKeysForMessageIds: vi.fn(), threadKeysInFolders: vi.fn() })); -import { providerProfile, makeClientCfg, gtdRelocateGuard, insertCopiedSibling, deleteMessageCopyRow, emitAfterDeferredCopySync, emitGtdSectionsRefreshOnDelete, emitGtdSectionsRefreshIfEnabled, selectGtdReevalIds, ensureMailbox, runGtdSyncTick, createKeyedSemaphore, isConnectionRefusal, connectCooldownMs, effectiveSyncIntervalMs, planModseqSync, connectStaggerFor } from './imapManager.js'; +import { providerProfile, makeClientCfg, gtdRelocateGuard, insertCopiedSibling, deleteMessageCopyRow, emitAfterDeferredCopySync, emitGtdSectionsRefreshOnDelete, emitGtdSectionsRefreshIfEnabled, selectGtdReevalIds, ensureMailbox, runGtdSyncTick, createKeyedSemaphore, isConnectionRefusal, connectCooldownMs, effectiveSyncIntervalMs, planModseqSync, connectStaggerFor, ImapManager } from './imapManager.js'; import { query } from './db.js'; import { invalidateGtdConfigCache } from './gtdConfig.js'; import { runGtdTransitions, threadKeysInFolders } from './gtdTransitions.js'; +import { sanitizeEmail } from './emailSanitizer.js'; +import { snippetFromBody } from './messageParser.js'; const account = (imap_host, oauth_provider = null) => ({ imap_host, oauth_provider }); @@ -103,6 +105,22 @@ describe('providerProfile — host detection', () => { }); }); +// ── providerProfile — bodyBackfill allowlist ───────────────────────────────── + +describe('providerProfile — bodyBackfill allowlist', () => { + it('enables body backfill for well-behaved providers (apple, yahoo, generic)', () => { + expect(providerProfile(account('imap.icloud.com')).bodyBackfill).toBe(true); + expect(providerProfile(account('imap.mail.yahoo.com')).bodyBackfill).toBe(true); + expect(providerProfile(account('imap.fastmail.com')).bodyBackfill).toBe(true); + }); + + it('excludes throttle-hostile / unproven providers (google, purelymail, microsoft)', () => { + expect(providerProfile(account('imap.gmail.com')).bodyBackfill).toBe(false); + expect(providerProfile(account('imap.purelymail.com')).bodyBackfill).toBe(false); + expect(providerProfile(account('outlook.office365.com')).bodyBackfill).toBe(false); + }); +}); + // ── providerProfile — oauth_provider detection ──────────────────────────────── describe('providerProfile — oauth_provider fallback', () => { @@ -977,3 +995,117 @@ describe('planModseqSync', () => { expect(planModseqSync({ storedModseq: b, serverModseq: b, uidValidityChanged: false })).toBe('unchanged'); }); }); + +describe('ImapManager.fetchBodiesForMessages', () => { + // The constructor starts four setInterval timers; clear them so tests leave no open handles. + function makeManager() { + const mgr = new ImapManager({}); + clearInterval(mgr._healthCheckTimer); + clearInterval(mgr._snippetSchedulerTimer); + clearInterval(mgr._stalenessCheckTimer); + clearInterval(mgr._flagPushReconcilerTimer); + return mgr; + } + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns {fetched:0} without touching IMAP when given no ids', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn(); + const result = await mgr.fetchBodiesForMessages('acct-1', []); + expect(result).toEqual({ fetched: 0 }); + expect(mgr.fetchMessageBody).not.toHaveBeenCalled(); + }); + + it('fetches each still-empty message and writes its body via UPDATE', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockResolvedValue({ html: '

hi

', text: 'hi', attachments: [] }); + sanitizeEmail.mockReturnValue('

hi

'); + snippetFromBody.mockReturnValue('hi'); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1', email_address: 'x@y' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }, { id: 'm2', uid: 6, folder: 'Sent' }] }); + } + return Promise.resolve({ rows: [] }); // UPDATE + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2']); + + expect(result).toEqual({ fetched: 2 }); + expect(mgr.fetchMessageBody).toHaveBeenCalledWith(expect.objectContaining({ id: 'acct-1' }), 5, 'INBOX'); + expect(mgr.fetchMessageBody).toHaveBeenCalledWith(expect.objectContaining({ id: 'acct-1' }), 6, 'Sent'); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(2); + }); + + it('skips a message whose body comes back empty (no html and no text)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockResolvedValue({ html: null, text: null, attachments: [] }); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }] }); + return Promise.resolve({ rows: [] }); + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1']); + + expect(result).toEqual({ fetched: 0 }); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(0); + }); + + it('propagates a fetchMessageBody failure so the drainer can back off', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockRejectedValue(new Error('Command failed')); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }] }); + return Promise.resolve({ rows: [] }); + }); + + await expect(mgr.fetchBodiesForMessages('acct-1', ['m1'])).rejects.toThrow('Command failed'); + }); + + it('skips a poison message that throws mid-batch and still fetches the rest (no wedge)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn() + .mockRejectedValueOnce(new Error("Mailbox doesn't exist")) + .mockResolvedValueOnce({ html: '

hi

', text: 'hi', attachments: [] }); + sanitizeEmail.mockReturnValue('

hi

'); + snippetFromBody.mockReturnValue('hi'); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'Stale' }, { id: 'm2', uid: 6, folder: 'INBOX' }] }); + } + return Promise.resolve({ rows: [] }); + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2']); + + // The poison message (m1) is skipped, not thrown — the loop keeps going so a batch + // containing it still makes forward progress instead of wedging the account forever. + expect(result).toEqual({ fetched: 1 }); + expect(mgr.fetchMessageBody).toHaveBeenCalledTimes(2); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(1); + }); + + it('rethrows when every message in the batch fails (genuine outage, not one bad message)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockRejectedValue(new Error('Connection closed')); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }, { id: 'm2', uid: 6, folder: 'INBOX' }] }); + } + return Promise.resolve({ rows: [] }); + }); + + await expect(mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2'])).rejects.toThrow('Connection closed'); + expect(mgr.fetchMessageBody).toHaveBeenCalledTimes(2); // both attempted, neither skipped silently + }); +}); diff --git a/backend/src/services/migrations.collation.test.js b/backend/src/services/migrations.collation.test.js new file mode 100644 index 0000000..d069054 --- /dev/null +++ b/backend/src/services/migrations.collation.test.js @@ -0,0 +1,58 @@ +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('./db.js', () => ({ pool: { query: vi.fn(), connect: vi.fn() } })); +import { warnOnCollationMismatch } from './migrations.js'; + +const row = (over = {}) => ({ + rows: [{ db: 'mailflow', recorded: '2.36', actual: '2.36', ...over }], +}); + +describe('warnOnCollationMismatch', () => { + it('warns loudly, naming REINDEX DATABASE, when versions diverge', async () => { + const warn = vi.fn(); + const query = vi.fn().mockResolvedValue(row({ recorded: '1.2.38', actual: '2.36' })); + const hit = await warnOnCollationMismatch({ query, warn }); + expect(hit).toBe(true); + expect(warn).toHaveBeenCalledTimes(1); + const msg = warn.mock.calls[0][0]; + expect(msg).toContain('collation version mismatch'); + expect(msg).toContain('1.2.38'); + expect(msg).toContain('2.36'); + expect(msg).toContain('REINDEX DATABASE "mailflow";'); + expect(msg).toContain('REFRESH COLLATION VERSION'); + }); + + it('stays silent when the recorded and actual versions match', async () => { + const warn = vi.fn(); + const hit = await warnOnCollationMismatch({ query: vi.fn().mockResolvedValue(row()), warn }); + expect(hit).toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); + + it('warns on the alpine→glibc signature: NULL recorded version, versioned current libc', async () => { + // Field-verified: musl (postgres:16-alpine) records NO datcollversion, so after + // the swap to pgvector/pgvector:pg16 the row reads recorded=NULL, actual=2.36 — + // and Postgres itself stays silent. This is the primary case to catch. + const warn = vi.fn(); + const query = vi.fn().mockResolvedValue(row({ recorded: null, actual: '2.36' })); + expect(await warnOnCollationMismatch({ query, warn })).toBe(true); + const msg = warn.mock.calls[0][0]; + expect(msg).toContain('reported no collation'); + expect(msg).toContain('REINDEX DATABASE "mailflow";'); + }); + + it('stays silent for versionless locales (NULL actual, e.g. C/POSIX or still on musl)', async () => { + const warn = vi.fn(); + for (const over of [{ actual: null }, { recorded: null, actual: null }]) { + expect(await warnOnCollationMismatch({ query: vi.fn().mockResolvedValue(row(over)), warn })).toBe(false); + } + expect(warn).not.toHaveBeenCalled(); + }); + + it('never throws — a failing query (e.g. PG < 15) only skips the check', async () => { + const warn = vi.fn(); + const query = vi.fn().mockRejectedValue(new Error('column "datcollversion" does not exist')); + await expect(warnOnCollationMismatch({ query, warn })).resolves.toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/migrations.js b/backend/src/services/migrations.js index 38fc9a2..6345fc9 100644 --- a/backend/src/services/migrations.js +++ b/backend/src/services/migrations.js @@ -94,3 +94,70 @@ export async function runMigrations() { client.release(); } } + +// --- Collation-version drift check (best-effort, never aborts boot) --------------- +// The compose move from postgres:16-alpine (musl libc) to pgvector/pgvector:pg16 +// (Debian, glibc) silently changes how the OS collates text. Postgres records the +// collation version a database was created under (pg_database.datcollversion, PG15+); +// pg_database_collation_actual_version(oid) reports what the running server's libc/ICU +// provides NOW. Two shapes of drift matter here (field-verified on both images): +// - recorded != actual: a versioned libc changed underneath (e.g. a glibc bump). +// Postgres emits its own per-connection WARNING for this, easily lost in logs. +// - recorded IS NULL while actual is not: the database was created under a libc +// that reports NO collation version — exactly what musl/alpine does — and now +// runs under glibc. This is the actual alpine → pgvector upgrade signature, and +// Postgres itself stays completely SILENT about it. +// (actual is NULL for versionless locales like C/POSIX — byte-order collation is +// immune to libc swaps, so there is nothing to warn about.) +// Either way, text indexes built under the old ordering may silently return wrong or +// missing rows until reindexed, so surface it loudly at boot with the remedy. The +// REFRESH COLLATION VERSION step records the current version and silences this +// warning on subsequent boots. Returns whether a mismatch was reported +// (observability only; never throws). +export async function warnOnCollationMismatch(deps = {}) { + const q = deps.query || ((text) => pool.query(text)); + const warn = deps.warn || console.warn; + try { + const { rows } = await q(` + SELECT current_database() AS db, + datcollversion AS recorded, + pg_database_collation_actual_version(oid) AS actual + FROM pg_database + WHERE datname = current_database() + `); + const r = rows[0]; + if (!r || !r.actual || r.recorded === r.actual) return false; + const origin = r.recorded + ? [ + ` Database "${r.db}" was created under collation version ${r.recorded}, but`, + ` the operating system now provides ${r.actual}.`, + ] + : [ + ` Database "${r.db}" was created under a C library that reported no collation`, + ` version (e.g. postgres:16-alpine/musl); the current one provides ${r.actual}.`, + ]; + warn([ + '='.repeat(76), + 'WARNING: database collation version mismatch detected', + '', + ...origin, + ' This happens when the Postgres image\'s C library changes — e.g. the', + ' docker-compose switch from postgres:16-alpine (musl) to', + ' pgvector/pgvector:pg16 (glibc).', + '', + ' Text indexes built under the old collation can silently return wrong or', + ' missing rows. Reindex once, then record the new version:', + '', + ` docker compose exec postgres psql -U mailflow -d ${r.db} \\`, + ` -c 'REINDEX DATABASE "${r.db}";' \\`, + ` -c 'ALTER DATABASE "${r.db}" REFRESH COLLATION VERSION;'`, + '='.repeat(76), + ].join('\n')); + return true; + } catch (err) { + // Postgres < 15 has no datcollversion; a restricted role may not read pg_database. + // The check is observability only — never block or fail the boot on it. + console.log(`Collation version check skipped: ${err.message}`); + return false; + } +} diff --git a/backend/src/services/search/ftsBackfill.js b/backend/src/services/search/ftsBackfill.js new file mode 100644 index 0000000..bacef25 --- /dev/null +++ b/backend/src/services/search/ftsBackfill.js @@ -0,0 +1,108 @@ +import { query } from '../db.js'; +import { searchFtsExpr, FTS_VERSION } from './lexicalRepo.js'; +import { upsertJob } from '../backgroundJobs.js'; + +const KIND = 'fts_backfill'; +const BATCH = 3000; // 2–5k rows/batch +const IDLE_MS = 5 * 60 * 1000; +const defaultSleep = () => new Promise((r) => setTimeout(r, 250)); + +let running = false; + +// FTS_VERSION is inlined as a literal (not a bind param) so the predicate +// matches the partial index idx_messages_fts_stale_v1 exactly and the batch +// scan is index-served rather than a growing seq scan as the tail drains. +const STALE_PRED = `fts_version IS DISTINCT FROM ${FTS_VERSION}`; + +const batchSql = ` + WITH batch AS ( + SELECT id FROM messages + WHERE ${STALE_PRED} + ORDER BY date DESC NULLS LAST + LIMIT $1 + ) + UPDATE messages m + SET search_fts = ${searchFtsExpr('m')}, + fts_version = ${FTS_VERSION} + FROM batch + WHERE m.id = batch.id +`; + +async function processBatch() { + try { + const res = await query(batchSql, [BATCH]); + return res.rowCount; + } catch (err) { + if (err.code !== '54000') throw err; // program_limit_exceeded + return processBatchRowByRow(); + } +} + +// A single pathological body tripped SQLSTATE 54000 for the whole batch. +// Re-process row by row so one bad row can't wedge the drainer. +async function processBatchRowByRow() { + const { rows } = await query( + `SELECT id FROM messages WHERE ${STALE_PRED} ORDER BY date DESC NULLS LAST LIMIT $1`, + [BATCH] + ); + let done = 0; + for (const { id } of rows) { + try { + await query( + `UPDATE messages m SET search_fts = ${searchFtsExpr('m')}, fts_version = ${FTS_VERSION} WHERE m.id = $1`, + [id] + ); + } catch (err) { + if (err.code !== '54000') throw err; + // Stamp the version so it is never retried; leave search_fts NULL (still + // served by the ILIKE fallback in lexicalRepo). + await query(`UPDATE messages SET fts_version = ${FTS_VERSION} WHERE id = $1 AND search_fts IS NULL`, [id]); + console.warn(`FTS backfill: skipped oversized message ${id} (tsvector too large)`); + } + done++; + } + return done; +} + +export async function runFtsBackfill({ sleep = defaultSleep } = {}) { + if (running) return; // single-flight + running = true; + try { + const { rows: [{ remaining }] } = await query( + `SELECT count(*)::int AS remaining FROM messages WHERE ${STALE_PRED}` + ); + if (remaining === 0) { + await upsertJob({ kind: KIND, state: 'done', processed: 0, total: 0 }); + return; + } + const total = remaining; + let processed = 0; + await upsertJob({ kind: KIND, state: 'running', processed, total }); + + while (true) { + let n; + try { + n = await processBatch(); + } catch (err) { + await upsertJob({ kind: KIND, state: 'error', processed, total, lastError: err.message }); + throw err; + } + if (n === 0) break; + processed += n; + await upsertJob({ kind: KIND, state: 'running', processed: Math.min(processed, total), total }); + await sleep(); + } + await upsertJob({ kind: KIND, state: 'done', processed: total, total }); + } finally { + running = false; + } +} + +export function scheduleFtsBackfill() { + runFtsBackfill().catch((err) => console.error('FTS backfill error:', err.message)); + const timer = setInterval(() => { + runFtsBackfill().catch((err) => console.error('FTS backfill error:', err.message)); + }, IDLE_MS); + timer.unref?.(); + return timer; +} diff --git a/backend/src/services/search/ftsBackfill.test.js b/backend/src/services/search/ftsBackfill.test.js new file mode 100644 index 0000000..3539739 --- /dev/null +++ b/backend/src/services/search/ftsBackfill.test.js @@ -0,0 +1,45 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +// Mock inline + import the mocked bindings (avoids the vi.mock hoisting TDZ trap). +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../backgroundJobs.js', () => ({ upsertJob: vi.fn() })); +import { query } from '../db.js'; +import { upsertJob } from '../backgroundJobs.js'; +import { runFtsBackfill } from './ftsBackfill.js'; + +const noSleep = () => Promise.resolve(); +beforeEach(() => { query.mockReset(); upsertJob.mockReset(); }); + +describe('runFtsBackfill', () => { + it('marks the job done and does no work when nothing is stale', async () => { + query.mockResolvedValueOnce({ rows: [{ remaining: 0 }] }); // count + await runFtsBackfill({ sleep: noSleep }); + expect(query).toHaveBeenCalledTimes(1); // count only, no batch UPDATE + expect(upsertJob).toHaveBeenCalledWith(expect.objectContaining({ kind: 'fts_backfill', state: 'done' })); + }); + + it('drains in batches until an UPDATE affects zero rows, reporting progress', async () => { + query + .mockResolvedValueOnce({ rows: [{ remaining: 5000 }] }) // count + .mockResolvedValueOnce({ rowCount: 3000 }) // batch 1 + .mockResolvedValueOnce({ rowCount: 2000 }) // batch 2 + .mockResolvedValueOnce({ rowCount: 0 }); // batch 3 → stop + await runFtsBackfill({ sleep: noSleep }); + const batchSql = query.mock.calls[1][0]; + expect(batchSql).toContain('fts_version IS DISTINCT FROM 1'); + expect(batchSql).toContain("setweight(to_tsvector('english', coalesce(m.subject,'')), 'A')"); + expect(upsertJob).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 5000, total: 5000 }) + ); + }); + + it('is single-flight: a second concurrent call returns immediately', async () => { + let release; + const gate = new Promise((r) => (release = r)); + query.mockImplementationOnce(async () => { await gate; return { rows: [{ remaining: 0 }] }; }); + const first = runFtsBackfill({ sleep: noSleep }); + await runFtsBackfill({ sleep: noSleep }); // returns immediately (guarded) + release(); + await first; + expect(query).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/services/search/lexicalRepo.js b/backend/src/services/search/lexicalRepo.js new file mode 100644 index 0000000..967f474 --- /dev/null +++ b/backend/src/services/search/lexicalRepo.js @@ -0,0 +1,341 @@ +import { shouldExcludeTrashFromSearch } from './queryParser.js'; + +// Postgres refuses to build a tsvector larger than ~1MB of packed lexemes +// (SQLSTATE 54000). Cap the text fed to to_tsvector at 600k chars — matching +// msgvault's maxFTSBodyChars (internal/store/dialect_pg.go). Reused by the +// slice-02 search_fts trigger and backfill via searchFtsExpr(). +export const FTS_BODY_CHAR_CAP = 600000; + +// Single source of truth for the stored-tsvector layout version. The trigger +// and backfill both stamp fts_version = FTS_VERSION; a layout/dictionary +// change bumps this and re-runs the backfill. +export const FTS_VERSION = 1; + +// Wraps a positive condition so that when negated it also matches rows where +// the underlying columns are NULL. +export function negateCond(sql) { + return `NOT COALESCE((${sql}), false)`; +} + +export function trashFolderExclusionCondition() { + return `NOT EXISTS ( + SELECT 1 + FROM folders f + WHERE f.account_id = m.account_id + AND f.path = m.folder + AND (f.special_use = '\\Trash' + OR lower(f.name) LIKE '%trash%' + OR lower(f.name) LIKE '%deleted%') + )`; +} + +// One free-text term matches if it appears in the sender, subject, the stored +// search_vector, or the length-capped body. Extracted so the body cap is a +// single, testable source of truth (the ranked, search_fts-first variant takes +// over once rows are backfilled). +export function freeTextTermCondition(likeIdx, ftsIdx) { + return `( + m.from_name ILIKE $${likeIdx} + OR m.from_email ILIKE $${likeIdx} + OR m.subject ILIKE $${likeIdx} + OR m.search_vector @@ plainto_tsquery('english', $${ftsIdx}) + OR to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})) @@ plainto_tsquery('english', $${ftsIdx}) + )`; +} + +// BM25-style lexical rank (ts_rank_cd; class weights D,C,B,A → 10:4:1 +// subject:from:rest; length normalization 32). Port of pgvector/fused.go:31-36. +// Exported so the fused query reuses it as the lexical leg. +export const LEXICAL_RANK_SQL = (vectorExpr, queryExpr) => + `ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], ${vectorExpr}, ${queryExpr}, 32)`; + +// A term counts as searchable text only if it has at least one letter or +// digit; msgvault's hasFTSToken drops punctuation-only tokens ("!!!", "***") +// the same way, since they'd normalize to zero lexemes and can't usefully +// match or rank anything. Exported so every caller building a free-text +// query (searchLexical, searchService's semantic branch, vectorStore's fused +// BM25 leg) applies the identical hygiene rather than re-deriving it. +export function hasSearchableToken(term) { + return /[\p{L}\p{N}]/u.test(term); +} + +// A bare free-text word is a single search token; a term containing whitespace +// only arises from a quoted multi-word phrase (queryParser doesn't emit these +// today, but the builders below stay correct if it ever does). Phrases keep +// ordinary non-prefix matching — prefix-expanding a multi-word phrase isn't a +// single lexeme operation and would change its meaning. +function isPhraseTerm(term) { + return /\s/.test(term); +} + +// msgvault's BuildFTSArg prefix-matches every bare word (typing "amaz" still +// finds "amazon") — plainto_tsquery only matches whole stemmed words, which +// regressed recall vs the old ILIKE substring behavior once a row is +// backfilled onto search_fts. quote_literal($N) is a plain SQL bind param +// reference (no JS-side string building), so the raw term travels through as +// an ordinary parameter; quote_literal() at query time safely wraps it as a +// single tsquery lexeme literal — neutralizing any &, |, !, (, ), ', or : +// the term might contain — before ':*' marks it for prefix matching. +// +// Exported (as the raw tsquery expression, not the whole `vectorExpr @@ ...` +// predicate) so `vectorStore.fusedSearch`'s BM25 leg builds its `@@` match +// AND its ts_rank_cd query-arg from this SAME per-term construction, rather +// than forking a second, non-prefix `plainto_tsquery` builder — a review +// caught exactly that fork (msgvault's fused.go reuses BuildFTSTerm for the +// identical reason: one construction, every caller). +export function ftsTermQueryArg(ftsIdx, term) { + return isPhraseTerm(term) + ? `plainto_tsquery('english', $${ftsIdx})` + : `to_tsquery('english', quote_literal($${ftsIdx}) || ':*')`; +} + +function ftsMatchExpr(vectorExpr, ftsIdx, term) { + return `${vectorExpr} @@ ${ftsTermQueryArg(ftsIdx, term)}`; +} + +// Ranked variant used once rows carry search_fts: backfilled rows match via the +// GIN-indexed tsvector; rows not yet backfilled (search_fts IS NULL) fall back +// to the legacy ILIKE/search_vector/body branch so recall never regresses +// mid-backfill. The fallback matches nothing once search_fts is populated, +// letting the planner use the GIN index exclusively. +export function freeTextTermConditionRanked(likeIdx, ftsIdx, term) { + return `( + ${ftsMatchExpr('m.search_fts', ftsIdx, term)} + OR (m.search_fts IS NULL AND ${freeTextTermCondition(likeIdx, ftsIdx)}) + )`; +} + +// English stopwords ("the", "for", "you", …) normalize to an EMPTY tsquery +// (numnode = 0), and `tsvector @@ ` is FALSE for every row — so one +// stopword in an AND'd term chain silently nuked ALL results once rows were +// backfilled onto search_fts ("waiting for invoice" → 0 hits, because of +// "for"; the ILIKE fallback arm is dead once search_fts is populated). +// A term whose tsquery normalizes empty must contribute NOTHING instead: +// this wraps the term's already-polarity-applied condition so it is +// vacuously TRUE — for positive AND negated terms alike — whenever the +// term's tsquery is empty. The numnode() probe runs on the SAME bind (and +// the SAME prefix-or-phrase construction, via ftsTermQueryArg) the match +// uses, so guard and match can never disagree; a query of ONLY stopwords +// degrades to a filter-only, date-ordered search. The relevance rank needs +// no guard: `&&` drops an empty tsquery operand and ts_rank_cd over a fully +// empty tsquery is 0 — both verified against pgvector/pg16 (2026-07-17). +export function stopwordSafeCondition(ftsIdx, term, condition) { + return `(numnode(${ftsTermQueryArg(ftsIdx, term)}) = 0 OR ${condition})`; +} + +// The one owner of a free-text term's FULL metadata-scope predicate — ranked +// FTS match, un-backfilled ILIKE fallback, polarity, stopword vacuity — shared +// by searchLexical, the fused query's NOT-conditions (negatedFreeTextClause), +// and MCP stage_deletion (engineAdapter), so a search preview and a staged +// deletion set can never disagree on what a term matches. `bind(value) → '$n'` +// pushes onto the caller's params; the raw ordinals freeTextTermConditionRanked +// expects are recovered from the placeholders so callers don't juggle them. +export function freeTextTermClause(term, negate, bind) { + const likeIdx = Number(bind(`%${term}%`).slice(1)); + const ftsIdx = Number(bind(term).slice(1)); + const cond = freeTextTermConditionRanked(likeIdx, ftsIdx, term); + return stopwordSafeCondition(ftsIdx, term, negate ? negateCond(cond) : cond); +} + +// Negated free-text NOT-condition for the fused (vector/hybrid) path, built from +// the SAME per-term construction (prefix/phrase match + the un-backfilled ILIKE +// fallback + stopword vacuity) that searchLexical applies, so `invoice -draft` +// excludes drafts identically in every mode. Prefix-vs-phrase semantics follow +// the term, via ftsTermQueryArg, exactly as the positive lexical predicate does. +export function negatedFreeTextClause(term, bind) { + return freeTextTermClause(term, true, bind); +} + +// Body-only free-text match for scope:'body' (MCP search_message_bodies leg): +// matches ONLY the message body, length-capped at the SAME FTS_BODY_CHAR_CAP as +// the stored FTS body cap and the body_text return cap (frozen contract — never +// a second cap, so every FTS body match is locatable in the returned text). +// Query-time (not GIN-served); bodies are sparse and this pool is bounded (D3). +export function bodyTermCondition(ftsIdx, term) { + return ftsMatchExpr(`to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))`, ftsIdx, term); +} + +// The weighted tsvector written into messages.search_fts. `ref` is the row +// reference: 'm' for the backfill UPDATE, 'NEW' for the BEFORE trigger. The +// class weights map to ts_rank_cd's D,C,B,A array (10:4:1 subject:from:rest). +// MUST stay identical to migration 0035's trigger body. +export function searchFtsExpr(ref) { + return `setweight(to_tsvector('english', coalesce(${ref}.subject,'')), 'A') || + setweight(to_tsvector('english', coalesce(${ref}.from_name,'') || ' ' || coalesce(${ref}.from_email,'')), 'B') || + setweight(to_tsvector('english', coalesce(${ref}.to_addresses::text,'') || ' ' || coalesce(${ref}.cc_addresses::text,'')), 'C') || + setweight(to_tsvector('english', LEFT(coalesce(${ref}.body_text,''), ${FTS_BODY_CHAR_CAP})), 'D')`; +} + +// Structured operator predicates (from/to/cc/subject/has/is/after/before), +// negation-aware. `bind(value) → '$n'` pushes value onto the caller's param +// list and returns its placeholder — the caller owns `params`/the running +// index, so this stays reusable for both searchLexical (one shared params +// array) and the fused query (its own per-leg bind closure, README "one +// search seam" — lexicalRepo remains the single owner of these predicates). +// Excludes free-text terms and folder scope (`in:`), which are not +// structured row predicates. Extracted from searchLexical byte-identically — +// same branches, same ILIKE-arm reuse, same skip rules. +export function buildOperatorClauses(filters, bind) { + const conditions = []; + for (const f of filters) { + if (f.key === 'in') continue; // controls scope, not a row condition + let cond = null; + + if (f.key === 'from') { + const idx = bind(`%${f.value}%`); + cond = `(m.from_email ILIKE ${idx} OR m.from_name ILIKE ${idx})`; + } else if (f.key === 'subject') { + cond = `m.subject ILIKE ${bind(`%${f.value}%`)}`; + } else if (f.key === 'to') { + const idx = bind(`%${f.value}%`); + cond = `(m.to_addresses::text ILIKE ${idx} OR m.cc_addresses::text ILIKE ${idx})`; + } else if (f.key === 'cc') { + cond = `m.cc_addresses::text ILIKE ${bind(`%${f.value}%`)}`; + } else if (f.key === 'has') { + if (f.value === 'attachment' || f.value === 'attachments') cond = `m.has_attachments = true`; + } else if (f.key === 'is') { + if (f.value === 'unread') cond = `m.is_read = false`; + else if (f.value === 'read') cond = `m.is_read = true`; + else if (f.value === 'starred') cond = `m.is_starred = true`; + } else if (f.key === 'after') { + const d = new Date(f.value); + if (!isNaN(d)) cond = `m.date >= ${bind(d.toISOString())}`; + } else if (f.key === 'before') { + const d = new Date(f.value); + if (!isNaN(d)) cond = `m.date < ${bind(d.toISOString())}`; + } + + if (cond) conditions.push(f.negate ? negateCond(cond) : cond); + } + return conditions; +} + +// Folder-scope predicate(s) for the messages table, shared by the lexical path +// and the fused (vector/hybrid) path so every mode scopes to the SAME folder — +// semantic search must not leak Sent/Archive/Trash into an Inbox search, and an +// explicit in:sent must apply. `folderScope`/`folderFuzzy` come from +// resolveSearchFolderScope: a truthy fuzzy scope (in:) matches the bare +// name OR any .../ path; an exact scope (REST ?folder=) matches the full +// path; a null scope carries no explicit folder and excludes trash-like folders +// (the default search scope). `bind(value) → '$n'` — the caller owns params. +export function buildFolderScopeClauses(folderScope, folderFuzzy, bind) { + const conditions = []; + if (folderScope) { + if (folderFuzzy) { + const name = bind(folderScope); + const path = bind(`%/${folderScope}`); + conditions.push(`(m.folder ILIKE ${name} OR m.folder ILIKE ${path})`); + } else { + conditions.push(`m.folder = ${bind(folderScope)}`); + } + } else if (shouldExcludeTrashFromSearch(folderScope)) { + conditions.push(trashFolderExclusionCondition()); + } + return conditions; +} + +export async function searchLexical(client, { parsed, accountIds, folderScope, folderFuzzy, ordering, scope = 'metadata', limit, offset }) { + const { filters, terms } = parsed; + const params = [accountIds]; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const conditions = buildOperatorClauses(filters, bind); + + for (const term of terms.slice(0, 10)) { + // Single-char terms are too broad/expensive; a punctuation-only term + // (e.g. "!!!") has no letters/digits to search on — msgvault's + // hasFTSToken drops it the same way rather than handing Postgres a + // token that would normalize to zero lexemes. + if (term.value.length < 2 || !hasSearchableToken(term.value)) continue; + if (scope === 'body') { + params.push(term.value); + const ftsIdx = p++; + const cond = bodyTermCondition(ftsIdx, term.value); + conditions.push(stopwordSafeCondition(ftsIdx, term.value, term.negate ? negateCond(cond) : cond)); + } else { + conditions.push(freeTextTermClause(term.value, term.negate, bind)); + } + } + + // A bare in:inbox (or lone folder param) must never dump a whole folder. (No total + // here: MCP search_metadata pre-checks free text, so a no-condition search — a + // filter-only/empty query — only reaches this path via REST, which ignores total.) + if (!conditions.length) return { rows: [], hasCondition: false }; + + for (const cond of buildFolderScopeClauses(folderScope, folderFuzzy, bind)) conditions.push(cond); + + // Snapshot the predicate binds (accountIds + operator/term/folder) BEFORE the + // rank/LIMIT/OFFSET binds are appended, so the metadata COUNT reuses the exact same + // WHERE with the exact same param ordinals (MCP search_metadata needs a real total). + const countParams = params.slice(); + + // D5: rank a free-text search by ts_rank_cd over the combined positive terms + // (date/id tiebreak); a filter-only search stays date-ordered. The rank arg + // must be bound before LIMIT/OFFSET. + let orderBy = 'ORDER BY m.date DESC'; + const positiveTerms = terms.slice(0, 10) + .filter(t => !t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => t.value); + if (ordering === 'relevance' && positiveTerms.length) { + // Rank with the SAME prefix-aware tsquery the MATCH predicate uses + // (freeTextTermConditionRanked → ftsTermQueryArg): one bind per term, + // combined with && (ts_rank_cd takes a single tsquery), mirroring the fused + // query's BM25 leg. Reusing the one term→tsquery-arg builder is what makes + // predicate and rank impossible to diverge — a prefix-only hit ("invo" + // matching "invoice") then ranks by ts_rank_cd instead of getting rank 0 + // (which COALESCE(...,0) would collapse to date order). Phrases keep + // non-prefix matching via ftsTermQueryArg, exactly as the predicate does. + const rankArgs = positiveTerms.map((term) => { params.push(term); return p++; }); + const rankQuery = positiveTerms.map((term, i) => ftsTermQueryArg(rankArgs[i], term)).join(' && '); + const rankExpr = LEXICAL_RANK_SQL('m.search_fts', rankQuery); + // ts_rank_cd(..., NULL::tsvector, ...) returns NULL for un-backfilled rows + // (search_fts IS NULL), and Postgres sorts NULLs FIRST in DESC order — so + // without COALESCE every un-backfilled row would outrank every properly + // ranked hit during the backfill window. COALESCE(...,0) keeps them + // sorting by the date/id tiebreak instead, below any real rank score. + orderBy = `ORDER BY COALESCE(${rankExpr}, 0) DESC, m.date DESC, m.id DESC`; + } + + // scope:'body' returns the body so MCP can compute keyword excerpts without a + // second query. The return cap is FTS_BODY_CHAR_CAP — the SAME constant as the + // FTS body cap (frozen contract: never a second cap), so every FTS body match + // is locatable in the returned text. Metadata scope keeps the historical + // column list (REST response byte-identical). + const bodyCol = scope === 'body' + ? `,\n LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}) AS body_text` + : ''; + + params.push(limit); + params.push(offset); + + const result = await client(` + SELECT + m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, + m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, + a.name as account_name, a.email_address as account_email, a.color as account_color${bodyCol} + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND ${conditions.join('\n AND ')} + ${orderBy} + LIMIT $${p} OFFSET $${p + 1} + `, params); + + // Bounded metadata total: a COUNT(*) over the SAME predicate (no LIMIT/OFFSET), for + // scope:'metadata' only. body/semantic stay total:-1 upstream. + let total; + if (scope === 'metadata') { + const countResult = await client(` + SELECT COUNT(*) AS total + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND ${conditions.join('\n AND ')} + `, countParams); + total = Number(countResult.rows[0]?.total ?? 0); // COUNT(*) always returns one row in prod + } + + return { rows: result.rows, hasCondition: true, ...(total !== undefined ? { total } : {}) }; +} diff --git a/backend/src/services/search/lexicalRepo.test.js b/backend/src/services/search/lexicalRepo.test.js new file mode 100644 index 0000000..8db21d7 --- /dev/null +++ b/backend/src/services/search/lexicalRepo.test.js @@ -0,0 +1,439 @@ +import { describe, it, expect, vi } from 'vitest'; +import { + FTS_BODY_CHAR_CAP, + FTS_VERSION, + negateCond, + trashFolderExclusionCondition, + freeTextTermCondition, + bodyTermCondition, + searchFtsExpr, + searchLexical, + LEXICAL_RANK_SQL, + freeTextTermConditionRanked, + freeTextTermClause, + negatedFreeTextClause, + stopwordSafeCondition, + buildOperatorClauses, + buildFolderScopeClauses, +} from './lexicalRepo.js'; + +describe('SQL fragment builders', () => { + it('pins the body cap at 600000 and FTS version at 1 (msgvault parity)', () => { + expect(FTS_BODY_CHAR_CAP).toBe(600000); + expect(FTS_VERSION).toBe(1); + }); + + it('caps the body tsvector inside the free-text condition', () => { + const cond = freeTextTermCondition(3, 4); + expect(cond).toContain( + `to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))` + ); + expect(cond).toContain('m.from_name ILIKE $3'); + expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); + }); + + it('bodyTermCondition prefix-matches a single-word term (msgvault BuildFTSArg parity)', () => { + const cond = bodyTermCondition(4, 'invoice'); + expect(cond).toBe( + "to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ to_tsquery('english', quote_literal($4) || ':*')" + ); + expect(cond).not.toContain('ILIKE'); + expect(cond).not.toContain('search_vector'); + expect(cond).not.toContain('plainto_tsquery'); + }); + + it('bodyTermCondition keeps non-prefix phrase matching for a quoted multi-word term', () => { + const cond = bodyTermCondition(4, 'weekly report'); + expect(cond).toBe( + "to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ plainto_tsquery('english', $4)" + ); + expect(cond).not.toContain('quote_literal'); + }); + + it('negateCond wraps a positive condition to also match NULL columns', () => { + expect(negateCond('m.is_read = true')).toBe('NOT COALESCE((m.is_read = true), false)'); + }); + + it('trashFolderExclusionCondition targets trash-like folders', () => { + const sql = trashFolderExclusionCondition(); + expect(sql).toContain('NOT EXISTS'); + expect(sql).toContain('%trash%'); + expect(sql).toContain('%deleted%'); + }); + + it('searchFtsExpr builds a weighted A/B/C/D tsvector for the given row alias', () => { + const expr = searchFtsExpr('m'); + expect(expr).toContain("setweight(to_tsvector('english', coalesce(m.subject,'')), 'A')"); + expect(expr).toContain("coalesce(m.from_name,'') || ' ' || coalesce(m.from_email,'')"); + expect(expr).toContain("LEFT(coalesce(m.body_text,''), 600000)), 'D')"); + // NEW.-form for the trigger must be produced by the same builder. + expect(searchFtsExpr('NEW')).toContain("coalesce(NEW.subject,'')"); + }); +}); + +describe('searchLexical', () => { + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [{ id: 'x' }] }; }); + return { fn, calls }; + } + + it('returns hasCondition:false and never queries when there is no real condition', async () => { + const { fn } = mockClient(); + const res = await searchLexical(fn, { + parsed: { filters: [{ key: 'in', value: 'inbox', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'inbox', folderFuzzy: true, ordering: 'date', limit: 50, offset: 0, + }); + expect(res).toEqual({ rows: [], hasCondition: false }); + expect(fn).not.toHaveBeenCalled(); + }); + + it('emits the pre-existing SQL shape and bind params for a from:+term all-folder query', async () => { + const { fn, calls } = mockClient(); + const res = await searchLexical(fn, { + parsed: { + filters: [{ key: 'from', value: 'amazon', negate: false }], + terms: [{ value: 'invoice', negate: false }], + }, + accountIds: ['a1', 'a2'], folderScope: null, folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(res.hasCondition).toBe(true); + expect(res.rows).toEqual([{ id: 'x' }]); + const { text, params } = calls[0]; + // Params: [accountIds, from-like, term-like, term-fts, limit, offset] + expect(params).toEqual([['a1', 'a2'], '%amazon%', '%invoice%', 'invoice', 50, 0]); + // Load-bearing SQL shape (byte-identical gate: the human-runnable old-vs-new diff). + expect(text).toContain('WHERE m.account_id = ANY($1)'); + expect(text).toContain('AND m.is_deleted = false'); + expect(text).toContain('(m.from_email ILIKE $2 OR m.from_name ILIKE $2)'); + expect(text).toContain('NOT EXISTS'); // trash exclusion on all-folder search + expect(text).toContain('ORDER BY m.date DESC'); + expect(text).toContain('LIMIT $5 OFFSET $6'); + }); + + it('adds a cc_addresses predicate for the new cc: operator', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [{ key: 'cc', value: 'boss', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain('m.cc_addresses::text ILIKE $2'); + expect(calls[0].params).toEqual([['a1'], '%boss%', 'INBOX', 50, 0]); + }); + + it('scope:body matches only the capped body branch and returns body_text at the SAME cap', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', scope: 'body', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // Body-only free-text leg (no sender/subject ILIKE, no search_vector), + // prefix-matched (msgvault BuildFTSArg parity — "invoic" still finds "invoice"). + expect(text).toContain("to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ to_tsquery('english', quote_literal($2) || ':*')"); + expect(text).not.toContain('ILIKE $2'); + // body_text returned at FTS_BODY_CHAR_CAP (600000) — same cap as the FTS body match (frozen contract). + expect(text).toContain("LEFT(coalesce(m.body_text,''), 600000) AS body_text"); + // Body scope pushes only the fts param per term (no %like%): [accountIds, term, folder, limit, offset]. + expect(params).toEqual([['a1'], 'invoice', 'INBOX', 50, 0]); + }); +}); + +describe('ranked lexical query (slice 02)', () => { + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [] }; }); + return { fn, calls }; + } + + it('LEXICAL_RANK_SQL emits ts_rank_cd with the D,C,B,A weight array and normalization 32', () => { + const expr = LEXICAL_RANK_SQL('m.search_fts', "plainto_tsquery('english', $7)"); + expect(expr).toBe("ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, plainto_tsquery('english', $7), 32)"); + }); + + it('freeTextTermConditionRanked prefix-matches a single-word term against search_fts, falling back only when it IS NULL', () => { + const cond = freeTextTermConditionRanked(3, 4, 'invoice'); + expect(cond).toContain("m.search_fts @@ to_tsquery('english', quote_literal($4) || ':*')"); + expect(cond).not.toContain("m.search_fts @@ plainto_tsquery"); + expect(cond).toContain('m.search_fts IS NULL AND'); + // The IS-NULL fallback branch is untouched legacy SQL (freeTextTermCondition): + // still plainto_tsquery + ILIKE, since it only serves un-backfilled rows. + expect(cond).toContain(`LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})`); + expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + }); + + it('freeTextTermConditionRanked keeps non-prefix phrase matching for a quoted multi-word term', () => { + const cond = freeTextTermConditionRanked(3, 4, 'weekly report'); + expect(cond).toContain("m.search_fts @@ plainto_tsquery('english', $4)"); + expect(cond).not.toContain("m.search_fts @@ to_tsquery"); + }); + + it('prefix-matching passes the raw term through as an ordinary bind param — quoting/escaping happens in SQL via quote_literal, never in JS', async () => { + const { fn, calls } = mockClient(); + const weird = `d'Angelo:*(evil)`; + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: weird, negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // The term is bound VERBATIM (no JS-side escaping) — quote_literal() at + // query time is what neutralizes '/:/* /()/etc. so it can't act as tsquery + // syntax or break out of the quoted lexeme. + expect(params).toContain(weird); + expect(text).toContain("to_tsquery('english', quote_literal($3) || ':*')"); + }); + + it('drops a punctuation-only term (msgvault hasFTSToken parity) instead of emitting an invalid tsquery', async () => { + const { fn } = mockClient(); + const res = await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: '!!!', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + // A punctuation-only term contributes no condition at all — same + // treatment as an under-length term — so a bare "!!!" search never + // dumps the whole folder and never reaches Postgres with a term that + // would normalize to zero lexemes. + expect(res).toEqual({ rows: [], hasCondition: false }); + expect(fn).not.toHaveBeenCalled(); + }); + + it('drops a punctuation-only term while keeping a real term alongside it', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: '***', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + // Only the real term's params were pushed: [accountIds, like, fts, folder, limit, offset]. + expect(calls[0].params).toEqual([['a1'], '%invoice%', 'invoice', 'INBOX', 50, 0]); + }); + + it('orders by ts_rank_cd then date/id for a relevance query, binding the rank args before LIMIT/OFFSET', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'urgent', negate: false }] }, + accountIds: ['a1'], folderScope: null, folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // NULL-safe: ts_rank_cd(..., NULL::tsvector, ...) returns NULL, and Postgres + // sorts NULLs FIRST in DESC order — without COALESCE, every un-backfilled + // (search_fts IS NULL) row would outrank every properly ranked hit during + // the backfill window. COALESCE(..., 0) keeps un-backfilled rows sorting + // by date/id like today, below any real (non-negative) rank score. + expect(text).toContain('ORDER BY COALESCE(ts_rank_cd'); + expect(text).toContain('32), 0) DESC, m.date DESC, m.id DESC'); + // Fix 5: rank reuses the SAME prefix-aware per-term tsquery as the MATCH + // predicate (ftsTermQueryArg), one bind per term combined with && — NOT a + // single plainto_tsquery over the joined string — so a prefix-only hit ranks + // by ts_rank_cd instead of collapsing to COALESCE(0) date order. + expect(text).toContain( + "ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, to_tsquery('english', quote_literal($6) || ':*') && to_tsquery('english', quote_literal($7) || ':*'), 32)" + ); + // Params: [accountIds, t1-like, t1-fts, t2-like, t2-fts, t1-rank, t2-rank, limit, offset] + expect(params).toEqual([['a1'], '%invoice%', 'invoice', '%urgent%', 'urgent', 'invoice', 'urgent', 50, 0]); + }); + + it('ranks a prefix-only term with the SAME builder output as the MATCH predicate (Fix 5 — never rank 0)', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invo', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // The MATCH predicate prefix-matches via ftsTermQueryArg; the rank arg is the + // SAME per-term construction (different bind ordinal only), so predicate and + // rank can never diverge on prefix-vs-plainto. + expect(text).toContain("m.search_fts @@ to_tsquery('english', quote_literal($3) || ':*')"); // predicate + expect(text).toContain( + "ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, to_tsquery('english', quote_literal($5) || ':*'), 32)" // rank + ); + // Params: [accountIds, term-like, term-fts(predicate), folder, term-fts(rank), limit, offset] + expect(params).toEqual([['a1'], '%invo%', 'invo', 'INBOX', 'invo', 50, 0]); + }); + + it('keeps non-prefix phrase matching in the rank for a quoted multi-word positive term (mirrors the predicate)', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'weekly report', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text } = calls[0]; + // A phrase term ranks (and matches) via plainto_tsquery — no prefix ':*'. + expect(text).toContain("ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, plainto_tsquery('english', $5), 32)"); + expect(text).not.toContain("quote_literal($5) || ':*'"); + }); + + it('keeps date ordering (no ts_rank_cd) for a filter-only query', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain('ORDER BY m.date DESC'); + expect(calls[0].text).not.toContain('ts_rank_cd'); + }); +}); + +describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { + // Root cause (verified against pgvector/pg16, 2026-07-17): + // to_tsquery('english', quote_literal('for') || ':*') normalizes to an + // EMPTY tsquery (numnode = 0), and `tsvector @@ ` is FALSE for + // every row — so once rows were backfilled onto search_fts, one english + // stopword in an AND'd term chain ("waiting for invoice") nuked ALL + // results. The guard makes such a term vacuously TRUE instead. + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [] }; }); + return { fn, calls }; + } + + it('stopwordSafeCondition wraps a clause with a numnode()=0 escape on the SAME bind as the match', () => { + expect(stopwordSafeCondition(4, 'invoice', 'X')).toBe( + "(numnode(to_tsquery('english', quote_literal($4) || ':*')) = 0 OR X)" + ); + // Phrase terms probe emptiness through the SAME plainto construction the + // match uses ("the on" normalizes empty exactly like a stopword word). + expect(stopwordSafeCondition(4, 'weekly report', 'X')).toBe( + "(numnode(plainto_tsquery('english', $4)) = 0 OR X)" + ); + }); + + it('wraps every positive free-text term condition, binding NO extra params', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // One guard per term, reusing that term's fts bind ordinal ($3, $5, $7). + for (const ftsIdx of [3, 5, 7]) { + expect(text).toContain(`(numnode(to_tsquery('english', quote_literal($${ftsIdx}) || ':*')) = 0 OR (`); + } + // Guard adds no binds: [accountIds, like1, fts1, like2, fts2, like3, fts3, folder, limit, offset] + expect(params).toEqual([['a1'], '%waiting%', 'waiting', '%for%', 'for', '%invoice%', 'invoice', 'INBOX', 50, 0]); + }); + + it('applies the guard OUTSIDE the negation so a negated stopword also contributes nothing', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'the', negate: true }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text } = calls[0]; + // (empty OR NOT COALESCE(match)) — vacuously TRUE for a stopword in BOTH + // polarities; a plain NOT-wrap of the guarded condition would instead be + // FALSE and exclude everything. + expect(text).toContain("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE("); + }); + + it('guards the body-scope term condition with the same construction', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', scope: 'body', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain( + "(numnode(to_tsquery('english', quote_literal($2) || ':*')) = 0 OR to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@" + ); + }); + + it('freeTextTermClause is the one owner searchLexical and the staging path share', () => { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const clause = freeTextTermClause('invoice', false, bind); + expect(params).toEqual(['%invoice%', 'invoice']); + expect(clause.startsWith("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR (")).toBe(true); + expect(clause).toContain("m.search_fts @@ to_tsquery('english', quote_literal($3) || ':*')"); + expect(clause).toContain('m.search_fts IS NULL AND'); + }); + + it('negatedFreeTextClause carries the guard outside the NOT (fused NOT-conditions)', () => { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const clause = negatedFreeTextClause('the', bind); + expect(clause.startsWith("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE(")).toBe(true); + }); +}); + +describe('buildOperatorClauses (Phase 4 Task 2a — extracted from searchLexical)', () => { + function bindHarness(start = 2) { + const params = []; + let p = start; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { bind, params }; + } + + it('builds a from: predicate reusing one bind for both ILIKE arms', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'from', value: 'amazon', negate: false }], bind); + expect(conds).toEqual(['(m.from_email ILIKE $2 OR m.from_name ILIKE $2)']); + expect(params).toEqual(['%amazon%']); + }); + + it('builds a cc: predicate', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'cc', value: 'boss', negate: false }], bind); + expect(conds).toEqual(['m.cc_addresses::text ILIKE $2']); + expect(params).toEqual(['%boss%']); + }); + + it('negates a structured operator via negateCond', () => { + const { bind } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'subject', value: 'invoice', negate: true }], bind); + expect(conds).toEqual(['NOT COALESCE((m.subject ILIKE $2), false)']); + }); + + it('skips in: (scope control, not a row condition) and malformed after/before', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([ + { key: 'in', value: 'inbox', negate: false }, + { key: 'after', value: 'not-a-date', negate: false }, + ], bind); + expect(conds).toEqual([]); + expect(params).toEqual([]); + }); + + it('does not bind free-text terms or folder scope (structured operators only)', () => { + const { bind } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'is', value: 'unread', negate: false }], bind); + expect(conds).toEqual(['m.is_read = false']); + }); +}); + +describe('buildFolderScopeClauses (folder scope — shared by lexical + fused)', () => { + function bindHarness(start = 2) { + const params = []; + let p = start; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { bind, params }; + } + + it('fuzzy in: matches the bare name or any .../ path', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses('sent', true, bind); + expect(conds).toEqual(['(m.folder ILIKE $2 OR m.folder ILIKE $3)']); + expect(params).toEqual(['sent', '%/sent']); + }); + + it('an exact folderScope (REST ?folder=) matches the full path', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses('INBOX', false, bind); + expect(conds).toEqual(['m.folder = $2']); + expect(params).toEqual(['INBOX']); + }); + + it('a null folderScope excludes trash-like folders (default search scope), binding nothing', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses(null, false, bind); + expect(conds).toHaveLength(1); + expect(conds[0]).toContain('NOT EXISTS'); + expect(conds[0]).toContain('%trash%'); + expect(params).toEqual([]); + }); +}); diff --git a/backend/src/services/search/lexicalRepo.total.test.js b/backend/src/services/search/lexicalRepo.total.test.js new file mode 100644 index 0000000..065201a --- /dev/null +++ b/backend/src/services/search/lexicalRepo.total.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../db.js', () => ({ query: vi.fn() })); +import { searchLexical } from './lexicalRepo.js'; + +// NOTE: the real searchLexical takes `client` as a FUNCTION (client(sql, params)), +// not an object with a .query method (the plan's Consumes was written earlier — +// real code wins). We adapt the fake accordingly. +const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; + +describe('searchLexical metadata total', () => { + it('returns a real total via a bounded COUNT over the same predicate (no LIMIT/OFFSET)', async () => { + const client = vi.fn() + .mockResolvedValueOnce({ rows: [{ id: 'm1' }] }) // page query + .mockResolvedValueOnce({ rows: [{ total: '42' }] }); // count query + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], scope: 'metadata', limit: 20, offset: 0 }); + expect(out.total).toBe(42); + const countSql = client.mock.calls[1][0]; + expect(countSql).toMatch(/COUNT\(\*\)/i); + expect(countSql).not.toMatch(/\bLIMIT\b/i); + expect(countSql).not.toMatch(/\bOFFSET\b/i); + }); + + it("does not COUNT for scope:'body' (no total)", async () => { + const client = vi.fn().mockResolvedValueOnce({ rows: [] }); // page query only + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], scope: 'body', limit: 20, offset: 0 }); + expect(out.total).toBeUndefined(); + expect(client.mock.calls.every((c) => !/COUNT\(\*\)/i.test(c[0]))).toBe(true); + }); +}); diff --git a/backend/src/services/search/migrations.test.js b/backend/src/services/search/migrations.test.js new file mode 100644 index 0000000..feb0f32 --- /dev/null +++ b/backend/src/services/search/migrations.test.js @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { searchFtsExpr, FTS_VERSION } from './lexicalRepo.js'; + +const MIGRATIONS = join(dirname(fileURLToPath(import.meta.url)), '../../../migrations'); +const read = (f) => readFileSync(join(MIGRATIONS, f), 'utf8'); +const norm = (s) => s.replace(/\s+/g, ' ').trim(); + +describe('0035_search_fts.sql', () => { + const sql = read('0035_search_fts.sql'); + + it('is transactional (no no-transaction header — the $$ body cannot survive the ; splitter)', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(false); + }); + + it('adds nullable search_fts + fts_version with IF NOT EXISTS (fast metadata DDL, D1)', () => { + expect(norm(sql)).toContain('ADD COLUMN IF NOT EXISTS search_fts tsvector'); + expect(norm(sql)).toContain('ADD COLUMN IF NOT EXISTS fts_version int'); + expect(sql).not.toContain('GENERATED ALWAYS AS'); // D1: not a generated column + }); + + it('installs a BEFORE INSERT OR UPDATE trigger whose body equals searchFtsExpr(NEW)', () => { + expect(norm(sql)).toContain('BEFORE INSERT OR UPDATE ON messages'); + expect(norm(sql)).toContain(norm(searchFtsExpr('NEW'))); + expect(norm(sql)).toContain(`NEW.fts_version := ${FTS_VERSION}`); + }); + + it('handles the oversized-tsvector case gracefully (never fails the row write)', () => { + expect(sql).toContain('EXCEPTION WHEN program_limit_exceeded'); + }); + + it('skips recompute only on columns that actually feed searchFtsExpr — snippet is not one of them', () => { + // snippet isn't part of the weighted A/B/C/D expression (subject, from, + // to/cc, body_text), so guarding on it made a snippet-only UPDATE (e.g. + // read/star-adjacent metadata writes that also touch snippet) recompute + // an identical search_fts for nothing. + expect(sql).toContain('NEW.subject IS NOT DISTINCT FROM OLD.subject'); + expect(sql).toContain('NEW.from_name IS NOT DISTINCT FROM OLD.from_name'); + expect(sql).toContain('NEW.from_email IS NOT DISTINCT FROM OLD.from_email'); + expect(sql).toContain('NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses'); + expect(sql).toContain('NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses'); + expect(sql).toContain('NEW.body_text IS NOT DISTINCT FROM OLD.body_text'); + expect(sql).not.toContain('OLD.snippet'); + expect(sql).not.toContain('NEW.snippet'); + }); +}); + +describe('0037_search_fts_index.sql', () => { + const sql = read('0037_search_fts_index.sql'); + + it('runs outside a transaction and builds both indexes CONCURRENTLY', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(true); + expect(sql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts'); + expect(sql).toContain('USING GIN (search_fts)'); + expect(sql).toContain('idx_messages_fts_stale_v1'); + expect(sql).toContain(`WHERE fts_version IS DISTINCT FROM ${FTS_VERSION}`); + expect(sql).not.toContain('$$'); // no function bodies — safe for the ; splitter + }); + + // A cancelled/crashed CREATE INDEX CONCURRENTLY leaves an INVALID index under the + // target name; retrying with IF NOT EXISTS silently skips it, so the scan stays + // unindexed forever. Drop-if-exists before each create makes the retry crash-idempotent. + it('drops each index CONCURRENTLY before creating it (invalid-index retry hazard)', () => { + for (const name of ['idx_messages_search_fts', 'idx_messages_fts_stale_v1']) { + expect(sql).toContain(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`); + expect(sql.indexOf(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)) + .toBeLessThan(sql.indexOf(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${name}`)); + } + }); +}); + +describe('0039_embed_pending_index.sql', () => { + const sql = read('0039_embed_pending_index.sql'); + + it('runs outside a transaction and builds the partial index CONCURRENTLY', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(true); + expect(sql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending'); + expect(sql).toContain('WHERE embed_gen IS NULL'); + expect(sql).not.toContain('$$'); // no function bodies — safe for the ; splitter + }); + + it('drops the index CONCURRENTLY before creating it (invalid-index retry hazard)', () => { + expect(sql).toContain('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending'); + expect(sql.indexOf('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending')) + .toBeLessThan(sql.indexOf('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending')); + }); +}); diff --git a/backend/src/services/search/queryParser.js b/backend/src/services/search/queryParser.js new file mode 100644 index 0000000..bdba1e3 --- /dev/null +++ b/backend/src/services/search/queryParser.js @@ -0,0 +1,185 @@ +// Pure query grammar for lexical search. Ports msgvault's operator set +// (internal/search/parser.go); operators Mailflow's schema cannot serve +// (larger:/smaller: — no size column, bcc: — not stored, label:/l: — no +// labels) are RECORDED as unsupported, never silently widened into a match. + +// Multi-char keys precede single-char `l` so the alternation resolves +// `label:` before `l:`. The leading (-?) captures optional negation; \b sits +// between an optional '-' and the key so both `from:` and `-from:` match. +const OP_KEYS = 'from|to|cc|bcc|subject|has|is|after|before|in|older_than|newer_than|larger|smaller|label|l'; +const OP_PATTERN = new RegExp(`(-?)\\b(${OP_KEYS}):("([^"]*)"|([\\S]+))`, 'gi'); + +// Bare quoted phrases (msgvault tokenize parity, parser.go:395-464): a double- +// OR single-quoted span that STARTS a token (preceded by start/whitespace, +// optionally negated with '-') becomes ONE phrase term. A quote glued to the +// tail of a token — from:"John Smith", d'Angelo — never starts a phrase: +// operator values belong to OP_PATTERN and mid-word apostrophes are text. +// Backslash escapes keep a quote char from terminating the span. +const PHRASE_PATTERN = /(^|\s)(-?)(["'])((?:\\.|(?!\3)[^\\])*)\3/g; + +// Port of msgvault's unescapeQuotedValue: `\\` and an escaped quote collapse +// to the bare char; any other `\x` keeps its backslash literally. +function unescapePhrase(s) { + let out = ''; + let escaped = false; + for (const ch of s) { + if (escaped) { + out += (ch === '\\' || ch === '"' || ch === "'") ? ch : '\\' + ch; + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else { + out += ch; + } + } + if (escaped) out += '\\'; + return out; +} + +// Sizes: 5M / 100K / 1G → bytes. Longer suffixes checked first. Returns null +// on anything unparseable (port of parser.go parseSize). +function parseSize(value) { + const v = value.trim().toUpperCase(); + const mult = { KB: 1024, MB: 1048576, GB: 1073741824, K: 1024, M: 1048576, G: 1073741824 }; + for (const suffix of ['KB', 'MB', 'GB', 'K', 'M', 'G']) { + if (v.endsWith(suffix)) { + const num = parseFloat(v.slice(0, -suffix.length)); + if (Number.isNaN(num)) return null; + return Math.floor(num * mult[suffix]); + } + } + return /^\d+$/.test(v) ? parseInt(v, 10) : null; +} + +// Relative ages: 7d / 2w / 1m / 1y → an absolute ISO timestamp relative to +// `now` (port of parser.go parseRelativeDate). Returns null if unparseable. +function relativeAgeToISO(value, now) { + const m = /^(\d+)([dwmy])$/.exec(value.trim().toLowerCase()); + if (!m) return null; + const n = parseInt(m[1], 10); + const d = new Date(now.getTime()); + switch (m[2]) { + case 'd': d.setUTCDate(d.getUTCDate() - n); break; + case 'w': d.setUTCDate(d.getUTCDate() - n * 7); break; + case 'm': d.setUTCMonth(d.getUTCMonth() - n); break; + case 'y': d.setUTCFullYear(d.getUTCFullYear() - n); break; + default: return null; + } + return d.toISOString(); +} + +function applyOperator(key, value, negate, ctx) { + const { filters, unsupported, errors, now } = ctx; + switch (key) { + case 'from': case 'to': case 'cc': + case 'subject': case 'has': case 'is': + case 'in': + if (value) filters.push({ key, value, negate }); + return; + case 'after': + case 'before': { + if (!value) return; + // A bad date must be RECORDED, not silently dropped downstream + // (buildOperatorClauses skips unparseable dates via isNaN, which would + // silently WIDEN the results). Same uniform message parser.go's + // operatorValueError emits for a bad before:/after: value. + if (isNaN(new Date(value))) { + errors.push(`invalid value "${value}" for ${key}: — expected a date like YYYY-MM-DD`); + return; + } + filters.push({ key, value, negate }); + return; + } + case 'newer_than': + case 'older_than': { + const iso = relativeAgeToISO(value, now); + if (!iso) { + errors.push(`invalid value "${value}" for ${key}: — expected a relative age like 7d, 2w, 1m, or 1y`); + return; + } + filters.push({ key: key === 'newer_than' ? 'after' : 'before', value: iso, negate }); + return; + } + case 'larger': + case 'smaller': { + if (parseSize(value) === null) { + errors.push(`invalid value "${value}" for ${key}: — expected a size like 5M, 100K, or 1G`); + return; + } + // Recognized and well-formed, but messages has no size column. + unsupported.push({ key, token: `${key}:${value}` }); + return; + } + case 'bcc': + case 'label': + case 'l': + // No bcc column / no labels concept in Mailflow. + if (value) unsupported.push({ key: key === 'l' ? 'label' : key, token: `${key}:${value}` }); + return; + default: + return; + } +} + +export function parseQuery(raw, { now = new Date() } = {}) { + const filters = []; + const terms = []; + const unsupported = []; + const errors = []; + const ctx = { filters, unsupported, errors, now }; + + // Phase 1: lift bare quoted phrases out BEFORE the operator grammar runs, so + // a colon inside a phrase ("subject:not an operator") can't be parsed as an + // operator. Each phrase leaves a U+E000-delimited placeholder in the string + // (a Private Use Area char) — restored in term order below — so phrases and + // bare words keep their relative positions. U+E000 is stripped from the raw + // input first, so user text can never forge a placeholder. + const phrases = []; + const withPhrases = (raw || '').replace(/\uE000/g, '').replace(PHRASE_PATTERN, (_, pre, neg, _q, body) => { + const value = unescapePhrase(body); + if (!value.trim()) return pre; // empty phrase ("") contributes nothing + phrases.push({ value, negate: neg === '-' }); + return `${pre}\uE000${phrases.length - 1}\uE000`; + }); + + const remaining = withPhrases.replace(OP_PATTERN, (_, neg, key, _v, quoted, unquoted) => { + const k = key.toLowerCase(); + const value = (quoted !== undefined ? quoted : (unquoted || '')).toLowerCase().trim(); + applyOperator(k, value, neg === '-', ctx); + return ' '; + }).trim(); + + for (const word of remaining.split(/\s+/)) { + let w = word.trim(); + if (!w || w === '-') continue; // skip blanks and a lone '-' (nothing to negate) + const ph = /^\uE000(\d+)\uE000$/.exec(w); + if (ph) { terms.push(phrases[Number(ph[1])]); continue; } + let negate = false; + if (w[0] === '-' && w.length > 1) { negate = true; w = w.slice(1); } + terms.push({ value: w, negate }); + } + + return { filters, terms, unsupported, errors }; +} + +export function resolveSearchFolderScope(filters, folderParam = '') { + let folderScope; + let folderFuzzy = false; // in: matches loosely; the folder param is exact + + for (const f of filters) { + if (f.key !== 'in') continue; + if (f.value === 'all') { folderScope = null; } + else { folderScope = f.value; folderFuzzy = true; } + } + + if (folderScope === undefined) { + folderScope = (folderParam || '').trim() || null; + folderFuzzy = false; + } + + return { folderScope, folderFuzzy }; +} + +export function shouldExcludeTrashFromSearch(folderScope) { + return folderScope === null; +} diff --git a/backend/src/services/search/queryParser.test.js b/backend/src/services/search/queryParser.test.js new file mode 100644 index 0000000..58485aa --- /dev/null +++ b/backend/src/services/search/queryParser.test.js @@ -0,0 +1,225 @@ +import { describe, it, expect } from 'vitest'; +import { + parseQuery, + resolveSearchFolderScope, + shouldExcludeTrashFromSearch, +} from './queryParser.js'; + +const NOW = new Date('2026-07-16T00:00:00.000Z'); + +describe('parseQuery — preserved grammar', () => { + it('treats bare words as free-text terms', () => { + const { filters, terms } = parseQuery('hello world'); + expect(filters).toEqual([]); + expect(terms).toEqual([ + { value: 'hello', negate: false }, + { value: 'world', negate: false }, + ]); + }); + + it('extracts positive operators and lowercases their values', () => { + const { filters, terms } = parseQuery('from:Amazon subject:Invoice hello'); + expect(filters).toEqual([ + { key: 'from', value: 'amazon', negate: false }, + { key: 'subject', value: 'invoice', negate: false }, + ]); + expect(terms).toEqual([{ value: 'hello', negate: false }]); + }); + + it('supports quoted operator values with spaces', () => { + const { filters } = parseQuery('from:"John Smith" report'); + expect(filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); + }); + + it('negates an operator when prefixed with -', () => { + const { filters } = parseQuery('-from:Smith report'); + expect(filters).toEqual([{ key: 'from', value: 'smith', negate: true }]); + }); + + it('negates a free-text term when prefixed with -', () => { + const { terms } = parseQuery('report -invoice'); + expect(terms).toEqual([ + { value: 'report', negate: false }, + { value: 'invoice', negate: true }, + ]); + }); + + it('preserves repeated and mixed positive/negative operators', () => { + const { filters } = parseQuery('from:alice -from:bob is:unread'); + expect(filters).toEqual([ + { key: 'from', value: 'alice', negate: false }, + { key: 'from', value: 'bob', negate: true }, + { key: 'is', value: 'unread', negate: false }, + ]); + }); + + it('ignores a lone - so it is not treated as a negated empty term', () => { + const { terms } = parseQuery('report - draft'); + expect(terms).toEqual([ + { value: 'report', negate: false }, + { value: 'draft', negate: false }, + ]); + }); + + it('returns empty structures for blank input', () => { + expect(parseQuery('')).toEqual({ filters: [], terms: [], unsupported: [], errors: [] }); + expect(parseQuery(' ')).toEqual({ filters: [], terms: [], unsupported: [], errors: [] }); + }); + + it('parses in:all and named folders', () => { + expect(parseQuery('in:all invoice').filters).toEqual([ + { key: 'in', value: 'all', negate: false }, + ]); + expect(parseQuery('in:Sent proposal').filters).toEqual([ + { key: 'in', value: 'sent', negate: false }, + ]); + }); +}); + +describe('parseQuery — new msgvault operators', () => { + it('adds cc: as a real filter', () => { + const { filters } = parseQuery('cc:boss@corp.com report'); + expect(filters).toEqual([{ key: 'cc', value: 'boss@corp.com', negate: false }]); + }); + + it('maps newer_than: to an after: date and older_than: to a before: date', () => { + const newer = parseQuery('newer_than:2w', { now: NOW }).filters; + expect(newer[0].key).toBe('after'); + expect(newer[0].value.startsWith('2026-07-02')).toBe(true); + + const older = parseQuery('older_than:7d', { now: NOW }).filters; + expect(older[0].key).toBe('before'); + expect(older[0].value.startsWith('2026-07-09')).toBe(true); + }); + + it('records larger:/smaller:/bcc:/label:/l: as recognized-but-unsupported (never widened)', () => { + const { filters, unsupported } = parseQuery('larger:5M bcc:x@y.com label:work l:home smaller:100K'); + expect(filters).toEqual([]); // none of these silently become predicates + expect(unsupported).toEqual([ + { key: 'larger', token: 'larger:5m' }, + { key: 'bcc', token: 'bcc:x@y.com' }, + { key: 'label', token: 'label:work' }, + { key: 'label', token: 'l:home' }, + { key: 'smaller', token: 'smaller:100k' }, + ]); + }); + + it('records a malformed typed value as an error, not a filter', () => { + const { errors, unsupported } = parseQuery('larger:5X older_than:3q'); + expect(unsupported).toEqual([]); + expect(errors).toHaveLength(2); + expect(errors[0]).toContain('larger'); + expect(errors[1]).toContain('older_than'); + }); +}); + +describe('parseQuery — bare quoted phrases (Wave D Fix 4, msgvault tokenize parity)', () => { + it('treats a double-quoted phrase as ONE term without the quote chars (port "quoted phrase")', () => { + expect(parseQuery('"hello world"').terms).toEqual([{ value: 'hello world', negate: false }]); + }); + + it('mixes phrases with operators and bare words in order (port "mixed operators and text")', () => { + const { filters, terms } = parseQuery('from:alice@example.com "meeting notes" urgent'); + expect(filters).toEqual([{ key: 'from', value: 'alice@example.com', negate: false }]); + expect(terms).toEqual([ + { value: 'meeting notes', negate: false }, + { value: 'urgent', negate: false }, + ]); + }); + + it('keeps colons inside a phrase from becoming operators (port QuotedPhrasesWithColons)', () => { + expect(parseQuery('"foo:bar"').terms).toEqual([{ value: 'foo:bar', negate: false }]); + expect(parseQuery('"meeting at 10:30"').terms).toEqual([{ value: 'meeting at 10:30', negate: false }]); + expect(parseQuery('"check http://example.com"').terms).toEqual([{ value: 'check http://example.com', negate: false }]); + expect(parseQuery('"a:b:c:d"').terms).toEqual([{ value: 'a:b:c:d', negate: false }]); + }); + + it('parses a colon phrase alongside a real operator (port "quoted colon phrase mixed with real operator")', () => { + const { filters, terms } = parseQuery('from:alice@example.com "subject:not an operator"'); + expect(filters).toEqual([{ key: 'from', value: 'alice@example.com', negate: false }]); + expect(terms).toEqual([{ value: 'subject:not an operator', negate: false }]); + }); + + it('parses a leading phrase before an operator (port "operator followed by quoted colon phrase")', () => { + const { filters, terms } = parseQuery('"re: meeting notes" from:bob@example.com'); + expect(filters).toEqual([{ key: 'from', value: 'bob@example.com', negate: false }]); + expect(terms).toEqual([{ value: 're: meeting notes', negate: false }]); + }); + + it('accepts single-quoted phrases (msgvault tokenize takes both quote chars)', () => { + expect(parseQuery("'hello world'").terms).toEqual([{ value: 'hello world', negate: false }]); + }); + + it('never starts a phrase at an apostrophe INSIDE a word', () => { + expect(parseQuery("d'Angelo report").terms).toEqual([ + { value: "d'Angelo", negate: false }, + { value: 'report', negate: false }, + ]); + }); + + it('unescapes backslash-escaped quotes inside a phrase (msgvault unescapeQuotedValue)', () => { + expect(parseQuery('"say \\"hi\\" now"').terms).toEqual([{ value: 'say "hi" now', negate: false }]); + expect(parseQuery('"a\\\\b"').terms).toEqual([{ value: 'a\\b', negate: false }]); + }); + + it('negates a phrase with a leading -', () => { + expect(parseQuery('report -"weekly digest"').terms).toEqual([ + { value: 'report', negate: false }, + { value: 'weekly digest', negate: true }, + ]); + }); + + it('drops an empty phrase and leaves op:"value" quoting to the operator grammar', () => { + expect(parseQuery('""').terms).toEqual([]); + expect(parseQuery('from:"John Smith"').filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); + expect(parseQuery('from:"John Smith"').terms).toEqual([]); + }); +}); + +describe('parseQuery — before:/after: validation (Wave D Fix 5)', () => { + it('records an invalid before:/after: value as an error, never a filter (no silent widening)', () => { + const { filters, errors } = parseQuery('before:notadate after:2025-99-99 invoice'); + expect(filters).toEqual([]); + expect(errors).toEqual([ + 'invalid value "notadate" for before: — expected a date like YYYY-MM-DD', + 'invalid value "2025-99-99" for after: — expected a date like YYYY-MM-DD', + ]); + }); + + it('keeps valid dates as filters (msgvault parseDate accepts several forms)', () => { + const { filters, errors } = parseQuery('after:2025-01-02 before:2025/03/04'); + expect(errors).toEqual([]); + expect(filters.map((f) => f.key)).toEqual(['after', 'before']); + }); +}); + +describe('resolveSearchFolderScope / shouldExcludeTrashFromSearch', () => { + it('scopes to the client folder param when no in: operator is present', () => { + const { filters } = parseQuery('subject:newsletter'); + expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ + folderScope: 'INBOX', + folderFuzzy: false, + }); + }); + + it('lets in: override the client folder param', () => { + const { filters } = parseQuery('in:trash subject:newsletter'); + expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ + folderScope: 'trash', + folderFuzzy: true, + }); + }); + + it('flags all-folder searches for trash exclusion', () => { + const { filters } = parseQuery('subject:newsletter'); + const { folderScope } = resolveSearchFolderScope(filters); + expect(folderScope).toBeNull(); + expect(shouldExcludeTrashFromSearch(folderScope)).toBe(true); + }); + + it('keeps explicit folder searches eligible to find trash', () => { + const { filters } = parseQuery('in:trash subject:newsletter'); + const { folderScope } = resolveSearchFolderScope(filters); + expect(shouldExcludeTrashFromSearch(folderScope)).toBe(false); + }); +}); diff --git a/backend/src/services/search/searchService.js b/backend/src/services/search/searchService.js new file mode 100644 index 0000000..9ed9a70 --- /dev/null +++ b/backend/src/services/search/searchService.js @@ -0,0 +1,175 @@ +import { query } from '../db.js'; +import { searchLexical, buildOperatorClauses, hasSearchableToken, buildFolderScopeClauses, negatedFreeTextClause } from './lexicalRepo.js'; +import { resolveSearchFolderScope } from './queryParser.js'; +import { hybridSearch, isLexicalFallback, MissingFreeTextError } from '../embeddings/hybrid.js'; + +function clampLimit(limit) { + return Math.max(1, Math.min(parseInt(limit) || 50, 200)); +} + +// MCP-facing envelope helpers, applied only when the caller opts in +// (REST never sets `explain`/`scope:'body'`, so its response is unaffected). +function withExplainScores(hits) { + for (const h of hits) { + h.score = { + rrf: h.rrf_score, + ...(h.bm25_score != null ? { bm25: h.bm25_score } : {}), + ...(h.vector_score != null ? { vector: h.vector_score } : {}), + subject_boosted: !!h.subject_boosted, + }; + } +} +function attachBestChunk(hits) { + for (const h of hits) { + h.best_chunk = h.best_char_start == null ? null + : { chunk_index: h.best_chunk_index, char_start: h.best_char_start, char_end: h.best_char_end, score: h.vector_score }; + } +} + +// The vector/hybrid SQL projects the message id as `message_id` +// (vectorStore.fusedSearch DISPLAY_COLS), but every hit consumer — REST +// serialization and MCP hydration alike — keys on `id`. Additively alias `id` +// onto each vector/hybrid hit so the shape is mode-invariant with lexical, leaving +// `message_id` untouched (the REST response is a superset — additive, never +// renamed). searchService.search() is the one seam both modes flow through, so +// this is the single place the alias is applied. +function aliasIdFromMessageId(hits) { + for (const h of hits) h.id = h.message_id; +} + +// A caller that already resolved its scope (e.g. the MCP handler, from the +// bearer-token owner's enabled accounts) passes accountIds directly and it is +// trusted as-is; otherwise derive it from the session userId. The REST route +// never forwards a raw accountIds, so a browser client cannot widen its scope. +// Shared by both the lexical and semantic branches. +async function resolveAccountIds(request) { + const { userId, accountIds: providedScope, accountId } = request; + let scopeIds; + if (providedScope) { + scopeIds = providedScope; + } else { + const accountsResult = await query( + 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId] + ); + scopeIds = accountsResult.rows.map(r => r.id); + } + if (!scopeIds.length) return []; + // Optional single-account narrowing (REST ?accountId=), only within scope. + return accountId && scopeIds.includes(accountId) ? [accountId] : scopeIds; +} + +// Phase 1's original lexical search, extracted verbatim so the mode dispatch +// below can reuse it both as the default path and as the fallback target when +// a semantic search degrades. Returns the pre-Phase-4 shape (no `mode` field — +// the caller adds that uniformly). +async function runLexical(request) { + const { parsed, folderParam = '', limit = 50, offset = 0, scope } = request; + + // Frozen contract: scope ∈ 'metadata' | 'body' (default metadata). Anything + // else coerces to metadata (today's behavior); phase 5's body tool passes 'body'. + const searchScope = scope === 'body' ? 'body' : 'metadata'; + + const cap = clampLimit(limit); + const off = Math.max(0, parseInt(offset) || 0); + const emptyPage = { offset: off, limit: cap, hasMore: false }; + + const accountIds = await resolveAccountIds(request); + if (!accountIds.length) return { messages: [], page: emptyPage }; + + const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, folderParam); + + // D5: a free-text search (≥1 positive, non-trivial term) ranks by relevance; + // a filter-only search stays date-ordered. + const hasPositiveText = parsed.terms.some(t => !t.negate && t.value.length >= 2); + const ordering = hasPositiveText ? 'relevance' : 'date'; + + const { rows, total, hasCondition } = await searchLexical(query, { + parsed, accountIds, folderScope, folderFuzzy, ordering, scope: searchScope, limit: cap, offset: off, + }); + if (!hasCondition) return { messages: [], page: emptyPage }; + + return { + messages: rows, + ...(total !== undefined ? { total } : {}), + page: { offset: off, limit: cap, hasMore: rows.length === cap }, + }; +} + +// The single search entry point shared by REST and MCP. Owns account scoping, +// mode dispatch (lexical/vector/hybrid), the D5 ordering decision (lexical), +// and result shaping. No HTTP framing here. +export async function search(request) { + const mode = request.mode === 'vector' || request.mode === 'hybrid' ? request.mode : 'lexical'; + if (mode === 'lexical') { + return { ...(await runLexical(request)), mode: 'lexical' }; + } + + const limit = clampLimit(request.limit); + const offset = Math.max(0, parseInt(request.offset) || 0); + // Resolve once, up front, and thread it into a lexical fallback below — a + // fallback must not re-resolve accounts via a second DB round-trip. + const accountIds = await resolveAccountIds(request); + const scopedRequest = { ...request, accountIds }; + try { + if (accountIds.length === 0) { + return { messages: [], mode, pool_saturated: false, generation: null, + page: { offset, limit, hasMore: false } }; + } + const { parsed } = request; // already parsed by the caller — never re-parse + const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, request.folderParam || ''); + // One buildFilters owner threads the SAME predicates the lexical path applies + // into BOTH fused legs (fusedSearch applies it to the FTS pool and the ANN + // EXISTS alike, on the joined messages table): structured operators, the + // folder scope (so semantic search can't leak Sent/Archive/Trash into an + // Inbox search and an explicit in:sent applies), and negated free-text terms + // as NOT-conditions (so `invoice -draft` excludes drafts in semantic mode + // just as FTS exclusion does in lexical mode). Same term hygiene throughout: + // drop sub-2-char / punctuation-only tokens. + const buildFilters = (bind) => [ + ...buildOperatorClauses(parsed.filters, bind), + ...buildFolderScopeClauses(folderScope, folderFuzzy, bind), + ...parsed.terms + .filter(t => t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => negatedFreeTextClause(t.value, bind)), + ]; + // Only non-negated terms drive the embedding + BM25 leg; negated terms are + // enforced via buildFilters above, never fed to the embedder. + const freeText = parsed.terms + .filter(t => !t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => t.value).join(' '); + + // Ranked pools are bounded (D3); fetch one past the page window so a full + // page can observe whether more hits exist, then slice the page. + const window = offset + limit; + const { hits, poolSaturated, generation } = await hybridSearch({ + mode, freeText, accountIds, buildFilters, limit: window + 1, + }); + const page = hits.slice(offset, offset + limit); + aliasIdFromMessageId(page); // mode-invariant `id` alias (seam contract) + if (request.explain) withExplainScores(page); + if (request.scope === 'body') attachBestChunk(page); + return { + messages: page, + mode, + pool_saturated: poolSaturated, + generation: generation || null, // rich {id,model,dimension,fingerprint,state} object + page: { offset, limit, hasMore: hits.length > window }, + }; + } catch (err) { + if (isLexicalFallback(err)) { + if (request.strictVector) { + if (err instanceof MissingFreeTextError) throw err; // invalid input, not unavailability + throw err; // VectorUnavailableError already carries .reason + } + const lexical = { ...(await runLexical(scopedRequest)), mode: 'lexical' }; + // A filter-only query in semantic mode (MissingFreeTextError) is not a + // degradation — there is nothing to embed and the lexical result IS the + // answer, so no fellBack (the UI keys its amber "index building" hint on + // it). Real unavailability (unconfigured/building/stale/ + // embedding_timeout) keeps the flag. + return err instanceof MissingFreeTextError ? lexical : { ...lexical, fellBack: true }; + } + throw err; + } +} diff --git a/backend/src/services/search/searchService.test.js b/backend/src/services/search/searchService.test.js new file mode 100644 index 0000000..eaacb0c --- /dev/null +++ b/backend/src/services/search/searchService.test.js @@ -0,0 +1,364 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock inline and import the mocked bindings — referencing an outer `const fn` +// from a vi.mock factory hits Vitest's hoisting temporal-dead-zone error. +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('./lexicalRepo.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, searchLexical: vi.fn() }; +}); +vi.mock('../embeddings/hybrid.js', () => ({ + hybridSearch: vi.fn(), + isLexicalFallback: vi.fn(() => true), + MissingFreeTextError: class extends Error {}, +})); + +import { query } from '../db.js'; +import { searchLexical } from './lexicalRepo.js'; +import * as hybrid from '../embeddings/hybrid.js'; +import { search } from './searchService.js'; +import { VectorUnavailableError } from '../embeddings/vectorErrors.js'; + +beforeEach(() => { + query.mockReset(); searchLexical.mockReset(); + hybrid.hybridSearch.mockReset(); + hybrid.isLexicalFallback.mockReset(); + hybrid.isLexicalFallback.mockReturnValue(true); +}); + +function withAccounts(ids) { + query.mockResolvedValueOnce({ rows: ids.map(id => ({ id })) }); +} + +describe('searchService.search', () => { + it('returns an empty shaped result when the user has no enabled accounts', async () => { + withAccounts([]); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(res).toEqual({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + expect(searchLexical).not.toHaveBeenCalled(); + }); + + it('scopes to a single account only when it belongs to the user', async () => { + withAccounts(['a1', 'a2']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', accountId: 'a2', parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a2']); + }); + + it('trusts a caller-provided accountIds scope and skips the DB account lookup (MCP path)', async () => { + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ accountIds: ['a1', 'a2'], parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(query).not.toHaveBeenCalled(); // no user_id → email_accounts lookup + expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a1', 'a2']); + }); + + it('passes through an optional total when searchLexical returns one', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'm' }], total: 137, hasCondition: true }); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(res.total).toBe(137); + }); + + it('passes scope through to searchLexical, defaulting to metadata and coercing unknown values', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('metadata'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', scope: 'body', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('body'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', scope: 'nonsense', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('metadata'); + }); + + it('chooses relevance ordering for free text and date ordering for filter-only queries', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].ordering).toBe('relevance'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] } }); + expect(searchLexical.mock.calls[0][1].ordering).toBe('date'); + }); + + it('shapes rows into messages + page, clamping the limit to 200 and flagging hasMore on a full page', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: new Array(200).fill({ id: 'm' }), hasCondition: true }); + const res = await search({ userId: 'u1', limit: 9999, offset: 40, parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(res.messages).toHaveLength(200); + expect(res.mode).toBe('lexical'); + expect(res.page).toEqual({ offset: 40, limit: 200, hasMore: true }); + expect(searchLexical.mock.calls[0][1].limit).toBe(200); + }); + + it('returns empty when there is no real search condition', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: false }); + const res = await search({ userId: 'u1', parsed: { filters: [{ key: 'in', value: 'inbox', negate: false }], terms: [] } }); + expect(res.messages).toEqual([]); + expect(res.page.hasMore).toBe(false); + }); +}); + +describe('searchService mode dispatch (Phase 4 Task 5)', () => { + const hybridReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'quarterly', negate: false }, { value: 'revenue', negate: false }] }, + ...extra, + }); + + it('defaults to lexical and marks the mode', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'hello', negate: false }] } }); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBeUndefined(); + expect(hybrid.hybridSearch).not.toHaveBeenCalled(); + }); + + it('hybrid success returns score-ordered messages with pool_saturated + the rich generation object, no total', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, subject: 's' }], + poolSaturated: true, generation: { id: 7 }, + }); + const res = await search(hybridReq()); + expect(res.mode).toBe('hybrid'); + expect(res.pool_saturated).toBe(true); + // Task 5b supersedes Task 5's bare generation.id with the rich object + // fusedSearch/hybridSearch echoes back. + expect(res.generation).toEqual({ id: 7 }); + expect(res.total).toBeUndefined(); + expect(res.messages[0].message_id).toBe('a'); + }); + + it('additively aliases id onto message_id for BOTH vector and hybrid hits (mode-invariant seam contract), leaving message_id intact', async () => { + // The vector/hybrid SQL (fusedSearch DISPLAY_COLS) projects the message id + // as `message_id`, NOT `id` — the real seam shape the live MCP round-trip + // exposed. The lexical path and every hit consumer (REST serialization, MCP + // hydration via getMessageSummariesByIDs) key on `id`, so the seam must add + // it for both modes. Regression guard for `semantic_search_messages` + // returning 0. + for (const mode of ['vector', 'hybrid']) { + query.mockReset(); hybrid.hybridSearch.mockReset(); + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'real-uuid', uid: 7, folder: 'INBOX', subject: 'Run failed: CD deploy', rrf_score: 0.04 }], + poolSaturated: false, generation: { id: 1 }, + }); + const res = await search({ + userId: 'u1', mode, limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'deploy', negate: false }] }, + }); + expect(res.messages[0].message_id).toBe('real-uuid'); // unchanged — REST reads it + expect(res.messages[0].id).toBe('real-uuid'); // additive alias — MCP hydration + lexical parity + } + }); + + it('drops sub-2-char and punctuation-only terms from the semantic freeText, matching the lexical path\'s hygiene (review MINOR 3)', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + await search(hybridReq({ + parsed: { + filters: [], + terms: [ + { value: 'quarterly', negate: false }, + { value: 'a', negate: false }, // 1-char — dropped + { value: '!!!', negate: false }, // punctuation-only — dropped + { value: 'revenue', negate: false }, + ], + }, + })); + expect(hybrid.hybridSearch.mock.calls[0][0].freeText).toBe('quarterly revenue'); + }); + + it('falls back to lexical (fellBack:true) when hybrid degrades', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'lex1', subject: 'kw' }], hasCondition: true }); + hybrid.hybridSearch.mockRejectedValue(Object.assign(new Error('building'), { code: 'INDEX_BUILDING' })); + const res = await search(hybridReq()); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBe(true); + expect(res.messages[0].id).toBe('lex1'); + }); + + it('serves a filter-only semantic query lexically WITHOUT fellBack — not an "index building" event (Wave D Fix 6)', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'lex1' }], hasCondition: true }); + hybrid.hybridSearch.mockRejectedValue(new hybrid.MissingFreeTextError()); + const res = await search(hybridReq({ parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] } })); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBeUndefined(); // the UI keys an amber degradation hint on fellBack + expect(res.messages[0].id).toBe('lex1'); + }); + + it('propagates an unexpected (non-degradation) error instead of falling back', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockRejectedValue(new Error('boom')); + hybrid.isLexicalFallback.mockReturnValue(false); + await expect(search(hybridReq())).rejects.toThrow('boom'); + }); +}); + +describe('searchService fused folder scope + negation (Fix 2 + Fix 4)', () => { + const semReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + ...extra, + }); + // Run the buildFilters closure the semantic branch hands hybridSearch, so we + // can inspect the SQL predicates + params it applies to BOTH fused legs. + function runBuildFilters() { + const buildFilters = hybrid.hybridSearch.mock.calls[0][0].buildFilters; + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { clauses: buildFilters(bind), params }; + } + + beforeEach(() => { + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + }); + + it('scopes the fused query to an in: for BOTH vector and hybrid (never leaks other folders)', async () => { + for (const mode of ['vector', 'hybrid']) { + query.mockReset(); hybrid.hybridSearch.mockReset(); + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + withAccounts(['a1']); + await search(semReq({ mode, parsed: { filters: [{ key: 'in', value: 'sent', negate: false }], terms: [{ value: 'invoice', negate: false }] } })); + const { clauses, params } = runBuildFilters(); + expect(clauses.some((c) => /m\.folder ILIKE .+ OR m\.folder ILIKE/.test(c))).toBe(true); + expect(params).toContain('sent'); + expect(params).toContain('%/sent'); + } + }); + + it('honors an exact REST folderParam on the fused query', async () => { + withAccounts(['a1']); + await search(semReq({ folderParam: 'INBOX' })); + const { clauses, params } = runBuildFilters(); + expect(clauses.some((c) => c === 'm.folder = $2')).toBe(true); + expect(params).toContain('INBOX'); + }); + + it('defaults the fused query to the trash-excluding scope when no folder is specified', async () => { + withAccounts(['a1']); + await search(semReq()); + const { clauses } = runBuildFilters(); + expect(clauses.some((c) => /NOT EXISTS/.test(c) && /%trash%/.test(c))).toBe(true); + }); + + it('enforces a negated free-text term as a NOT-condition on the fused query (invoice -draft excludes drafts)', async () => { + withAccounts(['a1']); + await search(semReq({ parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'draft', negate: true }] } })); + const { clauses, params } = runBuildFilters(); + // Positive term drives freeText (embedded + BM25 leg); the negated term becomes + // a stopword-guarded NOT COALESCE(...) filter using the SAME prefix-aware + // FTS builder as lexical (guard OUTSIDE the NOT, so a negated stopword + // contributes nothing instead of excluding everything — Fix 1). + expect(hybrid.hybridSearch.mock.calls[0][0].freeText).toBe('invoice'); + const notClause = clauses.find((c) => /OR NOT COALESCE/.test(c)); + expect(notClause).toBeDefined(); + expect(notClause).toMatch(/^\(numnode\(to_tsquery\('english', quote_literal\(\$\d+\) \|\| ':\*'\)\) = 0 OR NOT COALESCE/); + expect(notClause).toContain("m.search_fts @@ to_tsquery('english', quote_literal($"); + expect(params).toContain('draft'); + }); + + it('drops a sub-2-char / punctuation-only negated term from the fused NOT-conditions (lexical hygiene parity)', async () => { + withAccounts(['a1']); + await search(semReq({ parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'x', negate: true }, { value: '!!!', negate: true }] } })); + const { clauses } = runBuildFilters(); + expect(clauses.some((c) => /NOT COALESCE/.test(c))).toBe(false); + }); +}); + +describe('searchService semantic pagination hasMore (Fix 3)', () => { + const semReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 2, offset: 0, + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + ...extra, + }); + + it('fetches window+1 and flags hasMore when a sentinel hit is present', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 3 }, { message_id: 'b', rrf_score: 2 }, { message_id: 'c', rrf_score: 1 }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq()); + expect(hybrid.hybridSearch.mock.calls[0][0].limit).toBe(3); // window(2) + 1 + expect(res.messages).toHaveLength(2); // page sliced to limit + expect(res.page.hasMore).toBe(true); + }); + + it('flags hasMore false when exactly the window is returned', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 2 }, { message_id: 'b', rrf_score: 1 }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq()); + expect(res.messages).toHaveLength(2); + expect(res.page.hasMore).toBe(false); + }); + + it('respects offset: window+1 = offset+limit+1 and slices the page', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a' }, { message_id: 'b' }, { message_id: 'c' }, { message_id: 'd' }, { message_id: 'e' }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq({ offset: 2, limit: 2 })); + expect(hybrid.hybridSearch.mock.calls[0][0].limit).toBe(5); // offset(2)+limit(2)+1 + expect(res.messages.map((m) => m.message_id)).toEqual(['c', 'd']); + expect(res.page.hasMore).toBe(true); // 5 hits > window(4) + }); +}); + +describe('searchService strictVector + envelope (Task 5b)', () => { + const hybridReq2 = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'q', negate: false }] }, + ...extra, + }); + + it('rethrows the VectorUnavailableError (no fallback) under strictVector', async () => { + withAccounts(['a1']); + const unavail = new VectorUnavailableError('index_building'); + hybrid.hybridSearch.mockRejectedValue(unavail); + const err = await search(hybridReq2({ strictVector: true })).catch(e => e); + expect(err).toBe(unavail); + expect(err.reason).toBe('index_building'); + }); + + it('returns the rich generation object and per-hit score under explain', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, bm25_score: 0.01, vector_score: 0.9, subject_boosted: true, best_char_start: null }], + poolSaturated: false, generation: { id: 7, model: 'm', dimension: 4, fingerprint: 'm:4:x', state: 'active' }, + }); + const res = await search(hybridReq2({ explain: true })); + expect(res.generation).toEqual({ id: 7, model: 'm', dimension: 4, fingerprint: 'm:4:x', state: 'active' }); + expect(res.messages[0].score).toEqual({ rrf: 0.04, bm25: 0.01, vector: 0.9, subject_boosted: true }); + }); + + it('attaches best_chunk metadata under scope=body', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, vector_score: 0.9, best_chunk_index: 2, best_char_start: 6, best_char_end: 40 }], + poolSaturated: false, generation: { id: 1, model: 'm', dimension: 4, fingerprint: 'f', state: 'active' }, + }); + const res = await search(hybridReq2({ scope: 'body' })); + expect(res.messages[0].best_chunk).toEqual({ chunk_index: 2, char_start: 6, char_end: 40, score: 0.9 }); + }); +}); diff --git a/backend/src/services/search/searchService.total.test.js b/backend/src/services/search/searchService.total.test.js new file mode 100644 index 0000000..f7c5bf8 --- /dev/null +++ b/backend/src/services/search/searchService.total.test.js @@ -0,0 +1,12 @@ +import { it, expect, vi } from 'vitest'; +vi.mock('./lexicalRepo.js', () => ({ searchLexical: vi.fn(), buildOperatorClauses: vi.fn(), hasSearchableToken: vi.fn() })); +import { searchLexical } from './lexicalRepo.js'; +import { search } from './searchService.js'; + +it('passes the metadata total through to the service result', async () => { + searchLexical.mockResolvedValue({ rows: [{ id: 'm1' }], total: 42, hasCondition: true }); + const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; + const r = await search({ mode: 'lexical', scope: 'metadata', parsed, accountIds: ['acc-1'], limit: 20, offset: 0 }); + expect(r.total).toBe(42); + expect(searchLexical.mock.calls[0][1]).toMatchObject({ scope: 'metadata', accountIds: ['acc-1'] }); +}); diff --git a/backend/src/testSupport/mockSurface.js b/backend/src/testSupport/mockSurface.js new file mode 100644 index 0000000..aa3fc44 --- /dev/null +++ b/backend/src/testSupport/mockSurface.js @@ -0,0 +1,24 @@ +import { vi } from 'vitest'; + +// Mock-drift guard. Given a mocked module namespace (`import * as ns` from a +// vi.mock'd module, or a hand-built `{ fnA, fnB }` of its named imports) and the +// real module (`await vi.importActual(path)`), return the names that are mocked +// as functions but do NOT exist as functions on the real module. A non-empty +// result means a suite invented a seam the production module never implemented — +// e.g. `generations.chunkCount`, mocked as a vi.fn() in two suites while the real +// module never exported it, which made live `collectStats` throw and silently drop +// get_stats' vector_search block. +// +// Scope note: this catches the missing/renamed EXPORT drift class only, not +// value-shape drift (a mock whose fn returns the wrong row shape still passes — +// the earlier message_id/id bug would not have been caught here). Pair it with +// real-shape fixtures where the return shape is load-bearing. +export function mockSurfaceDrift(mockedNs, realNs) { + const drift = []; + for (const [name, value] of Object.entries(mockedNs)) { + if (vi.isMockFunction(value) && typeof realNs[name] !== 'function') { + drift.push(name); + } + } + return drift; +} diff --git a/backend/src/utils/textExcerpt.js b/backend/src/utils/textExcerpt.js new file mode 100644 index 0000000..7c9f92c --- /dev/null +++ b/backend/src/utils/textExcerpt.js @@ -0,0 +1,21 @@ +// Low-level UTF-8 / line primitives shared by the excerpt builders on both search +// paths: mcp/bodyMatch.js (keyword matches) and services/embeddings/chunkmatch.js +// (vector chunk matches). All offsets are UTF-8 BYTES (msgvault wire contract). The +// higher-level keyword-vs-chunk assembly stays in each owner; only these primitives +// live here so they cannot drift. + +// Default snippet width in bytes for a match excerpt. +export const SNIPPET_BYTES = 300; + +// True when `byte` is a UTF-8 leading byte (not a 10xxxxxx continuation byte), so a +// slice boundary at it does not split a multi-byte rune. +export const isRuneStart = (byte) => (byte & 0xc0) !== 0x80; + +// 1-based line number at a byte offset: one plus the count of '\n' bytes before it. +export function lineNumberAt(buf, byteOffset) { + if (byteOffset <= 0) return 1; + const o = Math.min(byteOffset, buf.length); + let n = 1; + for (let i = 0; i < o; i++) if (buf[i] === 0x0a) n++; + return n; +} diff --git a/backend/src/utils/textExcerpt.test.js b/backend/src/utils/textExcerpt.test.js new file mode 100644 index 0000000..036046b --- /dev/null +++ b/backend/src/utils/textExcerpt.test.js @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from './textExcerpt.js'; + +describe('SNIPPET_BYTES', () => { + it('is the shared 300-byte excerpt width', () => { + expect(SNIPPET_BYTES).toBe(300); + }); +}); + +describe('isRuneStart', () => { + it('flags UTF-8 leading bytes, not continuation bytes', () => { + const buf = Buffer.from('é', 'utf8'); // 0xC3 0xA9 + expect(isRuneStart(buf[0])).toBe(true); // 0xC3 lead byte + expect(isRuneStart(buf[1])).toBe(false); // 0xA9 continuation byte + expect(isRuneStart(0x61)).toBe(true); // ASCII 'a' + }); +}); + +describe('lineNumberAt', () => { + it('counts newlines before the byte offset (1-based)', () => { + const buf = Buffer.from('a\nb\nc', 'utf8'); + expect(lineNumberAt(buf, 0)).toBe(1); + expect(lineNumberAt(buf, 2)).toBe(2); + expect(lineNumberAt(buf, 4)).toBe(3); + }); + it('clamps a negative or over-long offset', () => { + const buf = Buffer.from('a\nb', 'utf8'); + expect(lineNumberAt(buf, -1)).toBe(1); + expect(lineNumberAt(buf, 999)).toBe(2); + }); +}); diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 3c51af4..aaeb02e 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -71,7 +71,7 @@ services: # ── PostgreSQL ─────────────────────────────────────────────────────────────── postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 container_name: mailflow-postgres restart: unless-stopped environment: diff --git a/docker-compose.yml b/docker-compose.yml index 160326f..23d768e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: # ── PostgreSQL ─────────────────────────────────────────────────────────────── postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 container_name: mailflow-postgres restart: unless-stopped environment: diff --git a/frontend/nginx.conf b/frontend/nginx.conf index c4ca0c8..5d48640 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -87,6 +87,19 @@ server { proxy_read_timeout 60s; } + # MCP endpoint (Streamable HTTP — long-lived SSE responses, no buffering) + location /mcp { + proxy_pass http://backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_cache off; + } + # WebSocket proxy location /ws { proxy_pass http://backend:3000; @@ -215,6 +228,19 @@ server { proxy_read_timeout 60s; } + # MCP endpoint (Streamable HTTP — long-lived SSE responses, no buffering) + location /mcp { + proxy_pass http://backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_cache off; + } + # WebSocket proxy location /ws { proxy_pass http://backend:3000; diff --git a/frontend/src/components/AdminPanel.jsx b/frontend/src/components/AdminPanel.jsx index a3141dd..dd53446 100644 --- a/frontend/src/components/AdminPanel.jsx +++ b/frontend/src/components/AdminPanel.jsx @@ -13,6 +13,7 @@ import SignatureEditor from './SignatureEditor.jsx'; import GtdZeroPet from './GtdZeroPet.jsx'; import { getEffectiveShortcuts, getGroupedActions, ACTION_DEFS, SPECIAL_KEY_LABELS, parseModKey, modLabel } from '../utils/defaultShortcuts.js'; import { DEFAULT_GTD_FOLDERS, GTD_STATES, resolveAccountGtdFolders, diffGtdFolders, findGtdFolderCollisions } from '../utils/gtd.js'; +import { emptyEmbeddingsForm, embeddingsFormFromConfig, buildEmbeddingsPayload, embeddingsDirty, isSameAsChatProvider, reconcileDimension, embeddingsJob, canSaveAiConfig, EMBEDDING_MODEL_HINTS } from '../utils/embeddingsSettings.js'; // ─── Shared field component ─────────────────────────────────────────────────── function Field({ label, required, children }) { @@ -3350,28 +3351,54 @@ function AISection() { const [config, setConfig] = useState(null); const [loading, setLoading] = useState(true); const [form, setForm] = useState({ enabled: true, baseUrl: '', apiKey: '', model: '', features: { compose: true, summarize: true } }); + const [emb, setEmb] = useState(emptyEmbeddingsForm); const [saving, setSaving] = useState(false); const [testing, setTesting] = useState(false); const [msg, setMsg] = useState(null); + // Embeddings block: a background_jobs 'embeddings' row (or null), whether the + // pgvector extension is present, and the two async action flags. + const [job, setJob] = useState(null); + const [vectorAvailable, setVectorAvailable] = useState(true); + const [testingEmb, setTestingEmb] = useState(false); + const [building, setBuilding] = useState(false); + + const loadConfig = async () => { + const { config: cfg } = await api.ai.getConfig(); + setConfig(cfg || null); + if (cfg) { + setForm({ enabled: cfg.enabled !== false, baseUrl: cfg.baseUrl || '', apiKey: cfg.apiKey || '', model: cfg.model || '', features: { compose: cfg.features?.compose !== false, summarize: cfg.features?.summarize !== false } }); + setEmb(embeddingsFormFromConfig(cfg)); + } else { + setEmb(emptyEmbeddingsForm()); + } + }; + + const refreshJob = async () => { + try { + const { jobs } = await api.ai.indexingStatus(); + setJob(embeddingsJob(jobs)); + } catch { /* status is best-effort; leave the last known job in place */ } + }; useEffect(() => { - api.ai.getConfig() - .then(({ config: cfg }) => { - if (cfg) { - setConfig(cfg); - setForm({ enabled: cfg.enabled !== false, baseUrl: cfg.baseUrl || '', apiKey: cfg.apiKey || '', model: cfg.model || '', features: { compose: cfg.features?.compose !== false, summarize: cfg.features?.summarize !== false } }); - } - }) - .catch(console.error) - .finally(() => setLoading(false)); + loadConfig().catch(console.error).finally(() => setLoading(false)); + refreshJob(); + api.ai.status().then(s => setVectorAvailable(s?.vectorAvailable !== false)).catch(() => {}); }, []); + // Poll the indexing status while a build is running so progress ticks live. + useEffect(() => { + if (!job?.active) return; + const id = setInterval(refreshJob, 2000); + return () => clearInterval(id); + }, [job?.active]); + const handleSave = async (e) => { e.preventDefault(); setSaving(true); setMsg(null); try { - await api.ai.saveConfig(form); - setConfig({ ...form }); + await api.ai.saveConfig({ ...form, embeddings: buildEmbeddingsPayload(emb) }); + await loadConfig(); // re-mask keys + reset the dirty baseline setMsg({ type: 'ok', text: t('admin.ai.saved') }); } catch (err) { setMsg({ type: 'error', text: err.message }); @@ -3392,9 +3419,35 @@ function AISection() { await api.ai.deleteConfig(); setConfig(null); setForm({ enabled: true, baseUrl: '', apiKey: '', model: '', features: { compose: true, summarize: true } }); + setEmb(emptyEmbeddingsForm()); setMsg({ type: 'ok', text: t('admin.ai.removed') }); }; + const handleTestEmbeddings = async () => { + setTestingEmb(true); setMsg(null); + try { + const { dimension } = await api.ai.testEmbeddings(); + const { dimension: next, changed } = reconcileDimension(emb.dimension, dimension); + if (changed) setEmb(e => ({ ...e, dimension: String(next) })); + setMsg({ type: 'ok', text: t('admin.ai.emb.testOk', { dimension }) }); + } catch (err) { + setMsg({ type: 'error', text: `${t('admin.ai.testFail')}: ${err.message}` }); + } finally { setTestingEmb(false); } + }; + + const handleBuild = async () => { + setBuilding(true); setMsg(null); + try { + await api.ai.buildEmbeddings(); + setMsg({ type: 'ok', text: t('admin.ai.emb.buildStarted') }); + await refreshJob(); + } catch (err) { + // 409 = a build is already running; still surface progress rather than a dead error. + setMsg({ type: 'error', text: err.message }); + await refreshJob(); + } finally { setBuilding(false); } + }; + const field = (label, key, type = 'text', placeholder = '') => (
@@ -3409,6 +3462,22 @@ function AISection() {
); + // Embeddings-form counterpart of `field` — reads/writes the `emb` state. + const embField = (label, key, type = 'text', placeholder = '', hint = '') => ( +
+ + setEmb(f => ({ ...f, [key]: e.target.value }))} + placeholder={placeholder} + autoComplete={type === 'password' ? 'new-password' : 'off'} + style={{ width: '100%', background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 6, padding: '7px 10px', color: 'var(--text-primary)', fontSize: 13 }} + /> + {hint &&
{hint}
} +
+ ); + const toggle = (label, checked, onChange) => (