feat: per-book roles for CardDAV multi-address-book sync

.github/workflows/ci.yml

@@ -11,6 +11,24 @@ jobs:
     name: Backend
     runs-on: ubuntu-latest
 
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+
+    env:
+      TEST_DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres
+
     steps:
       - uses: actions/checkout@v6
 

backend/migrations/0034_carddav_incremental_sync.sql

@@ -0,0 +1,76 @@
+-- Remote CardDAV collection state is separate from the sync token and ETags
+-- Mailflow serves to its own CardDAV clients.
+DO $$
+BEGIN
+  IF EXISTS (
+    WITH ranked AS (
+      SELECT id,
+             row_number() OVER (
+               PARTITION BY user_id, external_url
+               ORDER BY created_at, id
+             ) AS position
+      FROM address_books
+      WHERE source = 'carddav' AND external_url IS NOT NULL
+    )
+    SELECT 1
+    FROM ranked
+    JOIN contacts contact ON contact.address_book_id = ranked.id
+    WHERE ranked.position > 1
+  ) THEN
+    RAISE EXCEPTION 'cannot consolidate populated duplicate CardDAV address books';
+  END IF;
+END $$;
+
+ALTER TABLE address_books
+  ADD COLUMN IF NOT EXISTS remote_sync_token TEXT,
+  ADD COLUMN IF NOT EXISTS remote_sync_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_sync_capability IN ('unknown', 'sync-collection', 'snapshot')),
+  ADD COLUMN IF NOT EXISTS remote_sync_revision BIGINT NOT NULL DEFAULT 0,
+  ADD COLUMN IF NOT EXISTS remote_projection_fingerprint TEXT;
+
+CREATE TABLE IF NOT EXISTS carddav_remote_objects (
+  address_book_id UUID NOT NULL REFERENCES address_books(id) ON DELETE CASCADE,
+  href TEXT NOT NULL,
+  remote_etag TEXT,
+  vcard TEXT NOT NULL,
+  primary_email TEXT,
+  disposition TEXT NOT NULL CHECK (disposition IN ('separate', 'merge', 'skip')),
+  local_contact_id UUID REFERENCES contacts(id) ON DELETE SET NULL,
+  merge_before JSONB,
+  merge_applied JSONB,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  PRIMARY KEY (address_book_id, href)
+);
+
+UPDATE user_integrations
+SET config = config || jsonb_build_object('connectionGeneration', gen_random_uuid()::text)
+WHERE provider = 'carddav'
+  AND NOT (config ? 'connectionGeneration');
+
+WITH ranked AS (
+  SELECT id,
+         row_number() OVER (
+           PARTITION BY user_id, external_url
+           ORDER BY created_at, id
+         ) AS position
+  FROM address_books
+  WHERE source = 'carddav' AND external_url IS NOT NULL
+)
+DELETE FROM address_books
+WHERE id IN (SELECT id FROM ranked WHERE position > 1);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_remote_book_idx
+  ON address_books (user_id, external_url)
+  WHERE source = 'carddav' AND external_url IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS carddav_remote_object_contact_idx
+  ON carddav_remote_objects (local_contact_id)
+  WHERE local_contact_id IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS carddav_remote_object_email_idx
+  ON carddav_remote_objects (address_book_id, primary_email);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_merge_source_per_contact_idx
+  ON carddav_remote_objects (local_contact_id)
+  WHERE disposition = 'merge' AND local_contact_id IS NOT NULL;

backend/migrations/0035_carddav_bidirectional_sync.sql

@@ -0,0 +1,76 @@
+ALTER TABLE address_books
+  ADD COLUMN IF NOT EXISTS remote_create_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_create_capability IN ('unknown', 'allowed', 'denied')),
+  ADD COLUMN IF NOT EXISTS remote_update_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_update_capability IN ('unknown', 'allowed', 'denied')),
+  ADD COLUMN IF NOT EXISTS remote_delete_capability TEXT NOT NULL DEFAULT 'unknown'
+    CHECK (remote_delete_capability IN ('unknown', 'allowed', 'denied'));
+
+ALTER TABLE contacts
+  ADD COLUMN IF NOT EXISTS additional_fields JSONB NOT NULL DEFAULT '[]'::jsonb;
+
+ALTER TABLE carddav_remote_objects
+  ALTER COLUMN disposition SET DEFAULT 'separate',
+  ADD COLUMN IF NOT EXISTS mapping_status TEXT NOT NULL DEFAULT 'pending_materialization'
+    CHECK (mapping_status IN ('pending_materialization', 'synced', 'pending_push', 'conflict')),
+  ADD COLUMN IF NOT EXISTS vcard_version TEXT
+    CHECK (vcard_version IS NULL OR vcard_version IN ('3.0', '4.0')),
+  ADD COLUMN IF NOT EXISTS remote_semantic_hash TEXT,
+  ADD COLUMN IF NOT EXISTS local_contact_hash TEXT,
+  ADD COLUMN IF NOT EXISTS mapping_revision BIGINT NOT NULL DEFAULT 0
+    CHECK (mapping_revision >= 0),
+  ADD COLUMN IF NOT EXISTS last_synced_at TIMESTAMPTZ,
+  ADD COLUMN IF NOT EXISTS last_push_error_code TEXT,
+  ADD COLUMN IF NOT EXISTS last_push_error_at TIMESTAMPTZ,
+  ADD COLUMN IF NOT EXISTS legacy_projection JSONB;
+
+UPDATE carddav_remote_objects
+SET legacy_projection = jsonb_build_object(
+  'disposition', disposition,
+  'merge_before', merge_before,
+  'merge_applied', merge_applied
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_active_mapping_per_contact_idx
+  ON carddav_remote_objects (local_contact_id)
+  WHERE local_contact_id IS NOT NULL
+    AND mapping_status <> 'pending_materialization';
+
+CREATE TABLE IF NOT EXISTS carddav_conflicts (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  address_book_id UUID NOT NULL,
+  href TEXT NOT NULL,
+  user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  base_local_hash TEXT,
+  remote_etag TEXT,
+  local_vcard TEXT,
+  remote_vcard TEXT,
+  local_tombstone BOOLEAN NOT NULL DEFAULT false,
+  remote_tombstone BOOLEAN NOT NULL DEFAULT false,
+  status TEXT NOT NULL DEFAULT 'unresolved'
+    CHECK (status IN ('unresolved', 'resolved')),
+  resolution TEXT
+    CHECK (resolution IS NULL OR resolution IN ('keep-mailflow', 'keep-carddav')),
+  resolved_by UUID REFERENCES users(id) ON DELETE SET NULL,
+  resolved_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT carddav_conflicts_remote_object_fkey FOREIGN KEY (address_book_id, href)
+    REFERENCES carddav_remote_objects(address_book_id, href) ON DELETE CASCADE,
+  CHECK (local_tombstone OR local_vcard IS NOT NULL),
+  CHECK (remote_tombstone OR remote_vcard IS NOT NULL),
+  CHECK (
+    (status = 'unresolved'
+      AND resolution IS NULL
+      AND resolved_by IS NULL
+      AND resolved_at IS NULL)
+    OR
+    (status = 'resolved'
+      AND resolution IS NOT NULL
+      AND resolved_at IS NOT NULL)
+  )
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_unresolved_conflict_per_mapping_idx
+  ON carddav_conflicts (address_book_id, href)
+  WHERE status = 'unresolved';

backend/migrations/0036_carddav_bidirectional_cleanup.sql

@@ -0,0 +1,42 @@
+UPDATE user_integrations
+SET config = config - 'dupMode'
+WHERE provider = 'carddav' AND config ? 'dupMode';
+
+DROP INDEX IF EXISTS carddav_one_merge_source_per_contact_idx;
+
+ALTER TABLE carddav_remote_objects
+  DROP COLUMN disposition,
+  DROP COLUMN merge_before,
+  DROP COLUMN merge_applied,
+  DROP COLUMN legacy_projection,
+  ADD COLUMN IF NOT EXISTS pending_operation TEXT
+    CHECK (pending_operation IS NULL OR pending_operation IN ('update', 'delete')),
+  ADD COLUMN IF NOT EXISTS pending_vcard TEXT,
+  ADD COLUMN IF NOT EXISTS pending_local_hash TEXT,
+  ADD COLUMN IF NOT EXISTS pending_remote_semantic_hash TEXT,
+  ADD COLUMN IF NOT EXISTS pending_started_at TIMESTAMPTZ,
+  ADD CHECK (
+    (
+      pending_operation IS NULL
+      AND pending_vcard IS NULL
+      AND pending_local_hash IS NULL
+      AND pending_remote_semantic_hash IS NULL
+      AND pending_started_at IS NULL
+    )
+    OR
+    (
+      pending_operation = 'update'
+      AND pending_vcard IS NOT NULL
+      AND pending_local_hash IS NOT NULL
+      AND pending_remote_semantic_hash IS NOT NULL
+      AND pending_started_at IS NOT NULL
+    )
+    OR
+    (
+      pending_operation = 'delete'
+      AND pending_vcard IS NULL
+      AND pending_local_hash IS NOT NULL
+      AND pending_remote_semantic_hash IS NULL
+      AND pending_started_at IS NOT NULL
+    )
+  );

backend/migrations/0037_carddav_conflict_retention.sql

@@ -0,0 +1,6 @@
+ALTER TABLE carddav_conflicts
+  DROP CONSTRAINT carddav_conflicts_remote_object_fkey;
+
+ALTER TABLE carddav_conflicts
+  ADD CONSTRAINT carddav_conflicts_address_book_id_fkey
+  FOREIGN KEY (address_book_id) REFERENCES address_books(id) ON DELETE CASCADE;

backend/migrations/0038_carddav_multi_book.sql

@@ -0,0 +1,63 @@
+-- Multi-address-book CardDAV: surface per-book roles (write-target /
+-- subscribed / lookup-only) so a connected server exposing several address
+-- books can be treated by role rather than symmetrically. This migration is
+-- schema-only — it does not change sync
+-- behavior. Backfill preserves today's behavior exactly: every existing
+-- carddav book stays subscribed + lookup, and the first create-capable book
+-- per user becomes the write-target (mirroring the current selectedCreateBook
+-- rule), so no existing user's write destination changes.
+
+ALTER TABLE address_books
+  ADD COLUMN IF NOT EXISTS is_write_target  BOOLEAN NOT NULL DEFAULT false,
+  ADD COLUMN IF NOT EXISTS is_subscribed    BOOLEAN NOT NULL DEFAULT false,
+  ADD COLUMN IF NOT EXISTS is_lookup_source BOOLEAN NOT NULL DEFAULT true;
+
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_constraint
+    WHERE conname = 'address_books_write_target_subscribed'
+      AND conrelid = 'address_books'::regclass
+  ) THEN
+    ALTER TABLE address_books
+      ADD CONSTRAINT address_books_write_target_subscribed
+      CHECK (NOT is_write_target OR is_subscribed);
+  END IF;
+END $$;
+
+-- At most one write-target per user among carddav books.
+CREATE UNIQUE INDEX IF NOT EXISTS carddav_one_write_target_idx
+  ON address_books (user_id)
+  WHERE source = 'carddav' AND is_write_target;
+
+-- Backfill: preserve today's behavior for existing single/multi-book users.
+UPDATE address_books
+  SET is_subscribed = true, is_lookup_source = true
+  WHERE source = 'carddav';
+
+-- First create-capable book per user becomes the write-target (mirrors the
+-- current selectedCreateBook rule; tie-break by insertion order).
+WITH ranked AS (
+  SELECT id, row_number() OVER (
+    PARTITION BY user_id
+    ORDER BY (remote_create_capability = 'allowed') DESC, created_at, id
+  ) AS position
+  FROM address_books
+  WHERE source = 'carddav'
+    AND remote_create_capability IN ('allowed', 'unknown')
+)
+UPDATE address_books SET is_write_target = true
+  WHERE id IN (SELECT id FROM ranked WHERE position = 1);
+
+-- Ledger gains a terminal 'lookup' status and a projected display name.
+ALTER TABLE carddav_remote_objects
+  DROP CONSTRAINT IF EXISTS carddav_remote_objects_mapping_status_check,
+  ADD  CONSTRAINT carddav_remote_objects_mapping_status_check
+    CHECK (mapping_status IN
+      ('pending_materialization','synced','pending_push','conflict','lookup')),
+  ADD COLUMN IF NOT EXISTS lookup_display_name TEXT;
+
+-- Fast inbound lookup by email across a user's lookup books.
+CREATE INDEX IF NOT EXISTS carddav_lookup_email_idx
+  ON carddav_remote_objects (primary_email)
+  WHERE mapping_status = 'lookup';

backend/migrations/0039_carddav_write_target_alias.sql

@@ -0,0 +1,12 @@
+-- Multi-address-book CardDAV follow-up: write-target routing
+-- resolves the stored is_write_target book against a fresh CardDAV discovery
+-- snapshot by URL. Some servers advertise a stable "alias" href in PROPFIND
+-- discovery that 3xx-redirects (REPORT/PUT/DELETE) to a different canonical
+-- collection URL; carddavSync.js already detects and reconciles this by
+-- rewriting address_books.external_url to the canonical URL, but discovery
+-- keeps returning the alias forever afterward. Record the alias alongside the
+-- canonical URL so write-target resolution (and the export sweep) can match
+-- either, instead of a fresh discovery snapshot falsely reporting the
+-- write-target missing. Additive only — no existing column or row changes.
+ALTER TABLE address_books
+  ADD COLUMN IF NOT EXISTS discovery_alias_url TEXT;

backend/migrations/0040_carddav_publish_intent.sql

@@ -0,0 +1,20 @@
+-- Multi-address-book CardDAV: separate "the user deliberately wants
+-- this contact in their address book" from is_auto.
+--
+-- routes/send.js sets is_auto = false on any contact the user sends mail to, and
+-- routes/search.js ranks autocomplete on exactly that flag ("contacts the user
+-- explicitly sent to or manually created"). is_auto therefore cannot also mean
+-- "publish this to CardDAV" — gating publication on it means replying to a
+-- harvested sender silently pushes them into the user's shared address book.
+-- Publication is gated on this column instead; is_auto keeps its upstream meaning
+-- and send.js is untouched, so autocomplete ranking is unaffected.
+--
+-- Backfill = NOT is_auto: every contact that is explicit on the old schema is
+-- being exported by the sweep today, so it keeps exporting. The false default
+-- applies only to rows created from here on, where a contact that became explicit
+-- merely by being emailed is not published until the user deliberately promotes
+-- it (or turns on the publishEmailedContacts setting).
+ALTER TABLE contacts
+  ADD COLUMN IF NOT EXISTS carddav_publish_intent BOOLEAN NOT NULL DEFAULT false;
+
+UPDATE contacts SET carddav_publish_intent = true WHERE is_auto = false;

backend/package.json

@@ -5,7 +5,9 @@
   "scripts": {
     "start": "node src/index.js",
     "dev": "node --watch src/index.js",
-    "test": "vitest run",
+    "test": "npm run test:unit && npm run test:db",
+    "test:unit": "vitest run --exclude='**/*.db.test.js' --exclude='**/*.integration.test.js'",
+    "test:db": "vitest run src/**/*.db.test.js src/**/*.integration.test.js",
     "test:watch": "vitest",
     "lint": "eslint src --max-warnings 0"
   },

backend/src/index.js

@@ -29,6 +29,7 @@ import categoriesRoutes from './routes/categories.js';
 import gtdRoutes from './routes/gtd.js';
 import carddavRouter from './routes/carddav.js';
 import carddavAccountRouter from './routes/carddavAccount.js';
+import carddavConflictsRouter from './routes/carddavConflicts.js';
 import { startCardavScheduler } from './services/carddavSync.js';
 import { encryptExistingCredentials, query } from './services/db.js';
 import { runMigrations } from './services/migrations.js';
@@ -159,6 +160,7 @@ app.use('/api/rules', rulesRoutes);
 app.use('/api/block-list', blockListRoutes);
 app.use('/api/contacts', contactsRoutes);
 app.use('/api/todoist', todoistRoutes);
+app.use('/api/carddav/conflicts', carddavConflictsRouter);
 app.use('/api/carddav', carddavAccountRouter);
 app.use('/api', aiRoutes);
 app.use('/api', categoriesRoutes);