fix(vault): make a large OpenCreds import possible at all

apps/extension/__tests__/vault-api-paging.test.js

@@ -0,0 +1,72 @@
+/**
+ * Paging in the vault items fetch.
+ *
+ * The server has always capped one response at MAX_PAGE_SIZE (1000) and the
+ * client only ever made one request, so a vault larger than a page was listed
+ * short with no error anywhere: the items were stored and simply never shown.
+ * An OpenCreds import of a few thousand keys hits this immediately.
+ * @module __tests__/vault-api-paging.test
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+const { fetchRef } = vi.hoisted(() => ({ fetchRef: { calls: [], pages: [], paged: true } }));
+
+vi.mock('../src/lib/api.js', () => ({
+  apiRequest: vi.fn(async (path) => {
+    fetchRef.calls.push(path);
+    const url = new URL(path, 'https://example.test');
+    const offset = Number(url.searchParams.get('offset') || 0);
+    const limit = Number(url.searchParams.get('limit') || 1000);
+    const items = fetchRef.pages.slice(offset, offset + limit);
+    return {
+      ok: true,
+      status: 200,
+      json: async () => (fetchRef.paged ? { items, paged: true } : { items }),
+    };
+  }),
+}));
+
+const load = async () => import('../src/lib/vault-api.js');
+
+beforeEach(() => {
+  fetchRef.calls = [];
+  fetchRef.paged = true;
+  vi.resetModules();
+});
+
+describe('fetchVaultItems paging', () => {
+  it('returns every item of a vault larger than one page', async () => {
+    fetchRef.pages = Array.from({ length: 3122 }, (_, i) => ({ id: `id-${i}` }));
+    const { fetchVaultItems } = await load();
+
+    const items = await fetchVaultItems();
+
+    expect(items).toHaveLength(3122);
+    // No duplicates and nothing dropped.
+    expect(new Set(items.map((i) => i.id)).size).toBe(3122);
+    expect(fetchRef.calls).toHaveLength(4);
+  });
+
+  it('makes exactly one request when everything fits in a page', async () => {
+    fetchRef.pages = Array.from({ length: 12 }, (_, i) => ({ id: `id-${i}` }));
+    const { fetchVaultItems } = await load();
+
+    expect(await fetchVaultItems()).toHaveLength(12);
+    expect(fetchRef.calls).toHaveLength(1);
+  });
+
+  it('stops after one page against a server that does not understand offset', async () => {
+    // An older deployment ignores `offset` and returns the same full page every
+    // time. Without the `paged` flag to tell them apart, the client would ask
+    // forever and accumulate the same 1000 rows.
+    fetchRef.paged = false;
+    fetchRef.pages = Array.from({ length: 3122 }, (_, i) => ({ id: `id-${i}` }));
+    const { fetchVaultItems } = await load();
+
+    const items = await fetchVaultItems();
+
+    expect(items).toHaveLength(1000);
+    expect(fetchRef.calls).toHaveLength(1);
+  });
+});

apps/extension/__tests__/vault-session.test.js

@@ -67,6 +67,10 @@ vi.mock('../src/lib/vault-api.js', () => ({
     serverRef.items.filter((row) => Boolean(row.deleted_at) === Boolean(trash))
   ),
   createVaultItem: vi.fn(async (row) => {
+    // Mirrors the real route: the id is chosen by the client, so a second
+    // create of the same id is a 409 rather than a duplicate row. Import
+    // resume depends on that being reported as a conflict, not an error.
+    if (serverRef.items.some((r) => r.id === row.id)) return { conflict: true };
     const stored = { ...row, revision: 1, deleted_at: null };
     serverRef.items.push(stored);
     return stored;
@@ -321,6 +325,95 @@ describe('items', () => {
     expect(res.imported).toBe(2);
     expect((await mod.listItems()).items).toHaveLength(2);
   }, 30_000);
+
+  /**
+   * A database of a few thousand keys is a few thousand POSTs. Blocking the
+   * caller for that long is what made a large import impossible: the popup that
+   * was awaiting it had already been dismissed, so the result went nowhere. A
+   * big import is scheduled and reported through progress instead.
+   */
+  it('schedules a large import and finishes it in the background', async () => {
+    const mod = await loadModule();
+    await mod.setupVault(PASSWORD);
+
+    const batch = Array.from({ length: 120 }, (_, i) =>
+      mod.buildItem('login', { name: `Item ${i}` })
+    );
+    const res = await mod.importItems(batch);
+
+    // Returns immediately, before the writes are done.
+    expect(res.success).toBe(true);
+    expect(res.started).toBe(true);
+    expect(res.total).toBe(120);
+
+    await vi.waitFor(
+      () => {
+        const { job } = mod.getImportProgress();
+        expect(job.running).toBe(false);
+      },
+      { timeout: 30_000, interval: 50 }
+    );
+
+    const { job } = mod.getImportProgress();
+    expect(job.imported).toBe(120);
+    expect(job.failed).toBe(0);
+    expect(job.done).toBe(120);
+    expect((await mod.listItems()).items).toHaveLength(120);
+  }, 60_000);
+
+  /**
+   * The service worker can be killed mid-import and the job is lost with it, so
+   * the recovery is to import the same file again. That is only safe if an item
+   * already stored counts as done rather than as a failure.
+   */
+  it('counts already-stored items as present, so re-running resumes', async () => {
+    const mod = await loadModule();
+    await mod.setupVault(PASSWORD);
+
+    const batch = Array.from({ length: 120 }, (_, i) =>
+      mod.buildItem('login', { name: `Item ${i}` })
+    );
+
+    await mod.importItems(batch);
+    await vi.waitFor(() => expect(mod.getImportProgress().job.running).toBe(false), {
+      timeout: 30_000,
+      interval: 50,
+    });
+
+    // The same file again: every id is already on the server.
+    await mod.importItems(batch);
+    await vi.waitFor(() => expect(mod.getImportProgress().job.running).toBe(false), {
+      timeout: 30_000,
+      interval: 50,
+    });
+
+    const { job } = mod.getImportProgress();
+    expect(job.already).toBe(120);
+    expect(job.imported).toBe(0);
+    expect(job.failed).toBe(0);
+    // Re-running must not duplicate anything.
+    expect((await mod.listItems()).items).toHaveLength(120);
+  }, 60_000);
+
+  it('refuses to start a second import while one is running', async () => {
+    const mod = await loadModule();
+    await mod.setupVault(PASSWORD);
+
+    const batch = Array.from({ length: 120 }, (_, i) =>
+      mod.buildItem('login', { name: `Item ${i}` })
+    );
+    const first = await mod.importItems(batch);
+    expect(first.started).toBe(true);
+
+    const second = await mod.importItems(batch);
+    expect(second.success).toBe(false);
+    expect(second.error).toMatch(/already running/i);
+
+    await vi.waitFor(() => expect(mod.getImportProgress().job.running).toBe(false), {
+      timeout: 30_000,
+      interval: 50,
+    });
+  }, 60_000);
 });
 
 describe('buildItem', () => {

apps/extension/src/background/index.js

@@ -27,6 +27,7 @@ import {
   restoreItem as restoreVaultItem,
   destroyItem as destroyVaultItem,
   importItems as importVaultItems,
+  getImportProgress as getVaultImportProgress,
 } from './vault-session.js';
 import {
   initAdblock,
@@ -3715,6 +3716,9 @@ browser.runtime.onMessage.addListener((message, sender) => {
     case 'VAULT_IMPORT':
       return importVaultItems(message.payload?.items || []);
 
+    case 'VAULT_IMPORT_PROGRESS':
+      return Promise.resolve(getVaultImportProgress());
+
     case 'VAULT_GET_PREFS':
       return getVaultPrefs().then((prefs) => ({ success: true, ...prefs }));
 

apps/extension/src/background/vault-session.js

@@ -378,35 +378,124 @@ export async function destroyItem(id) {
   }
 }
 
+/**
+ * How many item writes are in flight at once.
+ *
+ * Every item is a separate POST -- the API creates one item per request -- so a
+ * database of a few thousand keys is a few thousand round trips. Sequentially
+ * that is minutes of wall clock; the popup that started it is long gone and the
+ * user is looking at an empty vault. Eight is enough to make the trip
+ * network-bound rather than latency-bound without looking like a flood.
+ */
+const IMPORT_CONCURRENCY = 8;
+
+/**
+ * Progress for the running (or last) import.
+ *
+ * Deliberately module-level and deliberately free of item data. The service
+ * worker can be killed mid-import, and the one thing that must never happen is
+ * plaintext items being written somewhere to survive that. Losing the job on a
+ * worker restart is fine because re-running the same file resumes: every item
+ * carries its own id, so an item already stored comes back 409 and is counted
+ * as done rather than as an error.
+ *
+ * @type {{total: number, done: number, imported: number, already: number,
+ *   failed: number, running: boolean, failures: Array, error: string|null}|null}
+ */
+let importJob = null;
+
+/** Progress for the popup/options page to poll. */
+export function getImportProgress() {
+  if (!importJob) return { success: true, job: null };
+  return { success: true, job: { ...importJob, failures: importJob.failures.slice(0, 20) } };
+}
+
 /**
  * Import parsed items, encrypting each one.
  *
+ * Returns as soon as the work is scheduled rather than when it finishes: the
+ * caller is a popup or an options tab, and neither is guaranteed to still be
+ * open in three minutes. Progress is polled through {@link getImportProgress},
+ * and closing the page no longer abandons the import.
+ *
  * Reports per-item failures instead of stopping, so one bad row out of five
  * hundred does not abandon the other four hundred and ninety-nine.
+ *
  * @param {Object[]} items plaintext items from parseImport
  */
 export async function importItems(items) {
+  if (importJob?.running) {
+    return { success: false, error: 'An import is already running' };
+  }
+
+  let userKey;
   try {
-    const userKey = await requireKey();
-    let imported = 0;
-    const failures = [];
+    userKey = await requireKey();
+  } catch (err) {
+    if (err.message === 'LOCKED') return { success: false, locked: true, error: 'Vault is locked' };
+    return { success: false, error: err.message };
+  }
 
-    for (const item of items) {
+  const queue = Array.isArray(items) ? items.slice() : [];
+  importJob = {
+    total: queue.length,
+    done: 0,
+    imported: 0,
+    already: 0,
+    failed: 0,
+    running: true,
+    failures: [],
+    error: null,
+  };
+  const job = importJob;
+
+  let next = 0;
+  const worker = async () => {
+    for (;;) {
+      const index = next++;
+      if (index >= queue.length) return;
+      const item = queue[index];
       try {
         const row = await encryptItem(userKey, item);
         const created = await createVaultItem(row);
-        if (created) imported += 1;
-        else failures.push({ name: item.name, reason: 'Server rejected the item' });
+        if (created?.conflict) job.already += 1;
+        else if (created) job.imported += 1;
+        else {
+          job.failed += 1;
+          job.failures.push({ name: item.name, reason: 'Server rejected the item' });
+        }
       } catch (err) {
-        failures.push({ name: item.name, reason: err.message });
+        job.failed += 1;
+        job.failures.push({ name: item.name, reason: err.message });
       }
+      job.done += 1;
     }
+  };
 
-    return { success: true, imported, failures };
-  } catch (err) {
-    if (err.message === 'LOCKED') return { success: false, locked: true, error: 'Vault is locked' };
-    return { success: false, error: err.message };
+  const run = Promise.all(
+    Array.from({ length: Math.min(IMPORT_CONCURRENCY, queue.length) }, worker)
+  )
+    .catch((err) => {
+      job.error = err.message;
+    })
+    .finally(() => {
+      job.running = false;
+    });
+
+  // Awaited only so a caller that wants the old blocking behaviour -- the tests,
+  // and a small CSV where waiting is nicer than polling -- still gets a result.
+  if (queue.length <= IMPORT_CONCURRENCY * 4) {
+    await run;
+    return {
+      success: true,
+      imported: job.imported,
+      already: job.already,
+      failures: job.failures,
+      finished: true,
+    };
   }
+
+  return { success: true, started: true, total: job.total };
 }
 
 /**