Preserve completed searches within BookOrbit’s deadline

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bookorbit-shadowlib-plugin",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "BookOrbit ShadowLib indexer with Open Slum mirror discovery and encrypted credentials",
   "scripts": {
     "build": "node scripts/build.mjs",

src/library.ts

@@ -1,3 +1,4 @@
+import * as Clock from "effect/Clock";
 import * as Effect from "effect/Effect";
 import * as Result from "effect/Result";
 import * as Schema from "effect/Schema";
@@ -84,12 +85,18 @@ const searchAll = Effect.fnUntraced(function* (
   cfg: PluginIndexerConfig,
   prepared: Prepared,
   warming = false,
+  budgetMs = warming ? 14000 : 8000,
 ) {
   const { host, browser } = yield* Network;
   const { accounts, mirrors, enabled } = prepared;
-  const outcomes = yield* Effect.forEach(
+  const outcomes: {
+    family: Family;
+    outcome: Result.Result<Hit[], SourceError>;
+  }[] = [];
+  let expired = false;
+  yield* Effect.forEach(
     enabled,
-    (family) => {
+    (family, index) => {
       const candidates = mirrors
         .filter((m) => m.family === family)
         .sort(byPriority(accounts));
@@ -137,23 +144,24 @@ const searchAll = Effect.fnUntraced(function* (
         return yield* Effect.fail(last);
       });
       return searchFamily.pipe(
-        Effect.timeoutOrElse({
-          duration: warming ? "14 seconds" : "8 seconds",
-          orElse: () =>
-            answered
-              ? Effect.succeed([] as Hit[])
-              : Effect.fail(
-                  failure(
-                    `${family}: mirror search budget expired.`,
-                    "timeout",
-                  ),
-                ),
-        }),
         Effect.result,
-        Effect.map((outcome) => ({ family, outcome })),
+        Effect.map((outcome) => {
+          outcomes[index] = { family, outcome };
+        }),
       );
     },
-    { concurrency: 4 },
+    { concurrency: 5, discard: true },
+  ).pipe(
+    Effect.timeoutOrElse({
+      duration: Math.max(0, budgetMs),
+      orElse: () =>
+        Effect.sync(() => {
+          expired = true;
+          host.logger.warn(
+            "Library search budget expired; keeping completed results.",
+          );
+        }),
+    }),
   );
   const failures = outcomes.flatMap(({ family, outcome }) =>
     Result.isFailure(outcome) ? [`${family}: ${outcome.failure.message}`] : [],
@@ -166,10 +174,11 @@ const searchAll = Effect.fnUntraced(function* (
     const kinds = outcomes.flatMap(({ outcome }) =>
       Result.isFailure(outcome) ? [outcome.failure.kind] : [],
     );
+    if (expired) kinds.push("timeout");
     const kind = kinds[0] ?? "error";
     return yield* Effect.fail(
       failure(
-        `No library completed a search. ${failures.join(" ")}`,
+        `No library completed a search. ${expired ? "Library search budget expired. " : ""}${failures.join(" ")}`,
         kinds.every((value) => value === kind) ? kind : "error",
       ),
     );
@@ -179,7 +188,15 @@ const searchAll = Effect.fnUntraced(function* (
   const count = Math.max(0, ...results.map((r) => r.length));
   for (let i = 0; i < count; i++)
     for (const list of results) if (list[i]) mixed.push(list[i]);
-  return mergeHits(mixed, q.mediaKind, q.language, q.limit);
+  const releases = mergeHits(mixed, q.mediaKind, q.language, q.limit);
+  if (!releases.length && (expired || failures.length))
+    return yield* Effect.fail(
+      failure(
+        "Some libraries could not complete the search. Retry or check source diagnostics in the BookOrbit logs.",
+        expired ? "timeout" : "error",
+      ),
+    );
+  return releases;
 });
 
 const prepare = Effect.fnUntraced(function* (
@@ -233,8 +250,11 @@ export const search = Effect.fnUntraced(function* (
   cfg: PluginIndexerConfig,
 ) {
   const { host } = yield* Network;
+  const started = yield* Clock.currentTimeMillis;
   const prepared = yield* prepare(cfg);
-  return yield* searchAll(q, cfg, prepared).pipe(
+  const elapsed = (yield* Clock.currentTimeMillis) - started;
+  // Include discovery in the query budget so BookOrbit can try another search.
+  return yield* searchAll(q, cfg, prepared, false, 8000 - elapsed).pipe(
     Effect.provideService(Network, { host, browser: prepared.browser }),
   );
 });

test/plugin.test.ts

@@ -2151,7 +2151,7 @@ test("API quota exhaustion does not retry other mirrors of the same library", as
   }
 });
 
-test("a valid empty catalogue answer survives the remaining mirror budget", async () => {
+test("an empty answer does not hide unfinished mirror searches", async () => {
   const config = cfg("libgen");
   const h = host((url) => {
     if (url.origin === config.baseUrl)
@@ -2169,16 +2169,17 @@ test("a valid empty catalogue answer survives the remaining mirror budget", asyn
       return new Response("No results");
     return new Promise<Response>(() => {});
   });
-  const releases = await plugin.search(
-    q,
-    config,
-    h.value,
-    AbortSignal.timeout(10000),
+  await assert.rejects(
+    plugin.search(q, config, h.value, AbortSignal.timeout(10000)),
+    (error: Error & { kind?: string }) => {
+      assert.equal(error.kind, "timeout");
+      assert.match(error.message, /budget expired/);
+      return true;
+    },
   );
-  assert.deepEqual(releases, []);
   assert(
     h.calls.some(
-      ({ url }) => new URL(url).hostname === "empty-followup-3.example",
+      ({ url }) => new URL(url).hostname === "empty-followup-2.example",
     ),
   );
 });
@@ -2424,3 +2425,95 @@ test("narrator metadata identifies audio archives without classifying ebook file
     0,
   );
 });
+
+test("search keeps completed libraries within a budget that includes discovery", async () => {
+  const config = cfg("libgen,memory");
+  const h = host(async (url) => {
+    if (url.origin === config.baseUrl) {
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+      return new Response(
+        directory({
+          libgen: Array.from(
+            { length: 4 },
+            (_, i) => `https://budget-${i}.example`,
+          ),
+          other: ["https://library.memoryoftheworld-budget.example"],
+        }),
+      );
+    }
+    if (url.hostname === "library.memoryoftheworld-budget.example")
+      return Response.json(memoryBody());
+    return new Promise<Response>(() => {});
+  });
+  const started = performance.now();
+  const releases = await plugin.search(
+    q,
+    config,
+    h.value,
+    AbortSignal.timeout(9500),
+  );
+  const elapsed = performance.now() - started;
+  assert.equal(releases.length, 1);
+  assert.equal(JSON.parse(releases[0].guid).locations[0].family, "memory");
+  assert(elapsed >= 7500 && elapsed < 9000, `Search took ${elapsed}ms`);
+  assert(
+    h.logs.some((message) => message.includes("keeping completed results")),
+  );
+});
+
+test("an empty completed library cannot turn a timed-out search into no matches", async () => {
+  const config = cfg("libgen,memory");
+  const h = host((url) => {
+    if (url.origin === config.baseUrl)
+      return new Response(
+        directory({
+          libgen: Array.from(
+            { length: 4 },
+            (_, i) => `https://empty-budget-${i}.example`,
+          ),
+          other: ["https://library.memoryoftheworld-empty-budget.example"],
+        }),
+      );
+    if (url.hostname === "library.memoryoftheworld-empty-budget.example")
+      return Response.json({ _items: [] });
+    return new Promise<Response>(() => {});
+  });
+  await assert.rejects(
+    plugin.search(q, config, h.value, AbortSignal.timeout(9500)),
+    (error: Error & { kind?: string }) => {
+      assert.equal(error.kind, "timeout");
+      assert.match(error.message, /could not complete/);
+      return true;
+    },
+  );
+});
+
+test("caller cancellation rejects even when another library has completed", async () => {
+  const config = cfg("libgen,memory");
+  const controller = new AbortController();
+  const h = host((url) => {
+    if (url.origin === config.baseUrl)
+      return new Response(
+        directory({
+          libgen: ["https://caller-budget.example"],
+          other: ["https://library.memoryoftheworld-caller-budget.example"],
+        }),
+      );
+    if (url.hostname === "library.memoryoftheworld-caller-budget.example") {
+      setTimeout(() => controller.abort(), 50);
+      return Response.json(memoryBody());
+    }
+    return new Promise<Response>(() => {});
+  });
+  await assert.rejects(
+    plugin.search(q, config, h.value, controller.signal),
+    /cancelled|timed out/,
+  );
+  assert(h.logs.some((message) => message.startsWith("memory: 1 releases")));
+  assert.equal(
+    h.calls.filter(
+      ({ url }) => new URL(url).hostname === "caller-budget.example",
+    ).length,
+    1,
+  );
+});