feat(search): weighted FTS ranking + searchable message bodies

backend/migrations/0035_search_fts.sql

@@ -0,0 +1,55 @@
+-- Weighted full-text search column (search_fts) + version stamp, kept fresh by
+-- a BEFORE trigger. Fast, metadata-only DDL only (README D1): the nullable
+-- columns force no table rewrite on PG16, and the function/trigger are instant.
+-- Pre-existing rows are populated by the resumable drainer (ftsBackfill.js);
+-- the GIN index is built CONCURRENTLY in the separate no-transaction migration
+-- 0037 (a $$-quoted plpgsql body cannot survive the no-transaction ; splitter,
+-- so the trigger and the CONCURRENTLY index MUST live in different files).
+--
+-- The setweight(...) expression MUST stay identical to
+-- lexicalRepo.searchFtsExpr('NEW'); fts_version = 1 matches lexicalRepo.FTS_VERSION.
+-- Idempotent (IF NOT EXISTS / CREATE OR REPLACE / DROP IF EXISTS) because a
+-- crash before the schema_migrations INSERT retries this migration.
+
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS search_fts tsvector;
+ALTER TABLE messages ADD COLUMN IF NOT EXISTS fts_version int;
+
+CREATE OR REPLACE FUNCTION messages_search_fts_refresh() RETURNS trigger AS $$
+BEGIN
+  -- Skip recompute on an UPDATE that changes none of the indexed source
+  -- columns (read/star flag flips, snippet-only writes), so the trigger does
+  -- not tax hot sync UPSERTs; this also lets the backfill's explicit SET win.
+  IF TG_OP = 'UPDATE'
+     AND NEW.subject      IS NOT DISTINCT FROM OLD.subject
+     AND NEW.from_name    IS NOT DISTINCT FROM OLD.from_name
+     AND NEW.from_email   IS NOT DISTINCT FROM OLD.from_email
+     AND NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses
+     AND NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses
+     AND NEW.body_text    IS NOT DISTINCT FROM OLD.body_text
+  THEN
+     RETURN NEW;
+  END IF;
+
+  BEGIN
+    NEW.search_fts := setweight(to_tsvector('english', coalesce(NEW.subject,'')), 'A') ||
+    setweight(to_tsvector('english', coalesce(NEW.from_name,'') || ' ' || coalesce(NEW.from_email,'')), 'B') ||
+    setweight(to_tsvector('english', coalesce(NEW.to_addresses::text,'') || ' ' || coalesce(NEW.cc_addresses::text,'')), 'C') ||
+    setweight(to_tsvector('english', LEFT(coalesce(NEW.body_text,''), 600000)), 'D');
+    NEW.fts_version := 1;
+  EXCEPTION WHEN program_limit_exceeded THEN
+    -- Even with the 600k LEFT cap, a pathologically dense/multibyte body can
+    -- exceed Postgres's ~1MB tsvector limit (SQLSTATE 54000). Never fail the
+    -- row write: leave search_fts NULL so the message still persists and stays
+    -- findable via the ILIKE fallback; the backfill's row-by-row skip stamps it.
+    NEW.search_fts := NULL;
+    NEW.fts_version := NULL;
+  END;
+
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_messages_search_fts ON messages;
+CREATE TRIGGER trg_messages_search_fts
+  BEFORE INSERT OR UPDATE ON messages
+  FOR EACH ROW EXECUTE FUNCTION messages_search_fts_refresh();

backend/migrations/0036_background_jobs.sql

@@ -0,0 +1,19 @@
+-- Generic progress substrate for observable background drainers (first consumer:
+-- the FTS backfill). Plain table, fast DDL. One row per (kind, account); global
+-- jobs use a NULL account_id, COALESCE'd to '' in the unique index so the upsert
+-- has a single conflict target for both cases.
+
+CREATE TABLE IF NOT EXISTS background_jobs (
+  id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  kind        VARCHAR(64) NOT NULL,
+  account_id  UUID REFERENCES email_accounts(id) ON DELETE CASCADE,
+  state       VARCHAR(20) NOT NULL DEFAULT 'idle',
+  processed   BIGINT NOT NULL DEFAULT 0,
+  total       BIGINT NOT NULL DEFAULT 0,
+  last_error  TEXT,
+  started_at  TIMESTAMPTZ,
+  updated_at  TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS ux_background_jobs_kind_account
+  ON background_jobs (kind, COALESCE(account_id::text, ''));

backend/migrations/0037_search_fts_index.sql

@@ -0,0 +1,24 @@
+-- no-transaction
+--
+-- GIN index that serves `search_fts @@ tsquery`, plus a partial btree that lets
+-- the backfill drainer (and any "needs backfill" probe) find not-yet-stamped
+-- rows without a full seq scan; it self-prunes as fts_version = 1 fills in.
+-- CONCURRENTLY must run outside a transaction, so these live apart from 0035's
+-- trigger. Idempotent via IF NOT EXISTS (retried if a crash precedes the
+-- schema_migrations INSERT).
+--
+-- DROP ... IF EXISTS before each CREATE: a cancelled or crashed CREATE INDEX
+-- CONCURRENTLY leaves an INVALID index under the target name. On retry, plain
+-- IF NOT EXISTS sees that name and silently skips the create, recording the
+-- migration as done while the scan stays unindexed forever. This file only re-runs
+-- after such a failure — in which case the index is either absent (drop is a no-op)
+-- or invalid (drop removes the dead stub) — so dropping first is safe and cheap.
+
+DROP INDEX CONCURRENTLY IF EXISTS idx_messages_search_fts;
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts
+  ON messages USING GIN (search_fts);
+
+DROP INDEX CONCURRENTLY IF EXISTS idx_messages_fts_stale_v1;
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_fts_stale_v1
+  ON messages (date DESC)
+  WHERE fts_version IS DISTINCT FROM 1;

backend/src/index.js

@@ -18,6 +18,7 @@ import accountRoutes from './routes/accounts.js';
 import mailRoutes from './routes/mail.js';
 import searchRoutes from './routes/search.js';
 import adminRoutes from './routes/admin.js';
+import indexingRoutes from './routes/indexing.js';
 import totpRoutes from './routes/totp.js';
 import oidcApiRouter, { oidcBrowserRouter } from './routes/oidc.js';
 import rulesRoutes from './routes/rules.js';
@@ -30,6 +31,7 @@ import gtdRoutes from './routes/gtd.js';
 import carddavRouter from './routes/carddav.js';
 import carddavAccountRouter from './routes/carddavAccount.js';
 import { startCardavScheduler } from './services/carddavSync.js';
+import { scheduleFtsBackfill } from './services/search/ftsBackfill.js';
 import { encryptExistingCredentials, query } from './services/db.js';
 import { runMigrations } from './services/migrations.js';
 import { parseVCard } from './utils/vcard.js';
@@ -167,6 +169,7 @@ app.use('/api/mail', sendRoutes);
 app.use('/api/mail', draftRoutes);
 app.use('/api/search', searchRoutes);
 app.use('/api/admin', adminRoutes);
+app.use('/api/admin/indexing', indexingRoutes);
 app.use('/api/totp', totpRoutes);
 app.use('/api/rules', rulesRoutes);
 app.use('/api/block-list', blockListRoutes);
@@ -244,6 +247,9 @@ imapManager.startSnoozeWatcher();
 // Schedule periodic CardDAV contact sync for any connected accounts.
 startCardavScheduler();
 
+// Postgres-only drainer: populate search_fts for pre-existing rows (no IMAP).
+scheduleFtsBackfill();
+
 // Re-connect all enabled IMAP accounts on startup with bounded concurrency so a
 // large user base doesn't hammer IMAP servers and the DB connection pool at once.
 try {

backend/src/routes/accounts.js

@@ -9,6 +9,9 @@ import { getConnectionPolicy } from '../services/connectionPolicy.js';
 import { invalidateGtdConfigCache, sanitizeGtdFoldersDetailed, findGtdFolderCollisions, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js';
 import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js';
 import { createKeyedSerializer } from '../utils/keyedSerializer.js';
+import { startAccountBodyBackfill } from '../services/bodyBackfill.js';
+import { providerProfile } from '../services/imapManager.js';
+import { upsertJob } from '../services/backgroundJobs.js';
 
 // Serialize an account's reconnect triggers so a rapid settings change (e.g. a
 // gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains —
@@ -421,7 +424,18 @@ router.post('/:id/reindex', async (req, res) => {
         console.error(`Manual reindex error for ${account.email_address}:`, err.message)
       );
     }
-    res.json({ ok: true, alreadyRunning });
+
+    // Also kick a body-materialization pass for allowlisted providers. Fire-and-forget: the
+    // drainer self-guards against concurrent runs and caps each session, and Gmail/PurelyMail/
+    // Microsoft are gated off inside startBodyBackfill. Progress lands in background_jobs.
+    const bodyBackfillEnabled = providerProfile(account).bodyBackfill;
+    if (bodyBackfillEnabled) {
+      startAccountBodyBackfill(account, imapManager, upsertJob).catch(err =>
+        console.error(`Body backfill error for ${account.email_address}:`, err.message)
+      );
+    }
+
+    res.json({ ok: true, alreadyRunning, bodyBackfillEnabled });
   } catch (err) {
     console.error('POST /accounts/:id/reindex error:', err.message);
     res.status(500).json({ error: 'Failed to start reindex' });

backend/src/routes/indexing.js

@@ -0,0 +1,13 @@
+import { Router } from 'express';
+import { requireAdmin } from '../middleware/auth.js';
+import { listJobs } from '../services/backgroundJobs.js';
+
+const router = Router();
+
+// GET /api/admin/indexing/status — admin-gated (same guard as routes/ai.js).
+// Surfaces every background drainer's progress, e.g. "FTS backfill: N/M".
+router.get('/status', requireAdmin, async (_req, res) => {
+  res.json({ jobs: await listJobs() });
+});
+
+export default router;