diff --git a/package.json b/package.json index 5ca49af..f7eebb9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bookorbit-shadowlib-plugin", - "version": "0.1.20", + "version": "0.1.21", "description": "BookOrbit ShadowLib indexer with Open Slum mirror discovery and encrypted credentials", "scripts": { "build": "node scripts/build.mjs", diff --git a/src/library.ts b/src/library.ts index 30d5a29..0fe7b6e 100644 --- a/src/library.ts +++ b/src/library.ts @@ -1,3 +1,4 @@ +import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; @@ -84,12 +85,18 @@ const searchAll = Effect.fnUntraced(function* ( cfg: PluginIndexerConfig, prepared: Prepared, warming = false, + budgetMs = warming ? 14000 : 8000, ) { const { host, browser } = yield* Network; const { accounts, mirrors, enabled } = prepared; - const outcomes = yield* Effect.forEach( + const outcomes: { + family: Family; + outcome: Result.Result; + }[] = []; + let expired = false; + yield* Effect.forEach( enabled, - (family) => { + (family, index) => { const candidates = mirrors .filter((m) => m.family === family) .sort(byPriority(accounts)); @@ -137,23 +144,24 @@ const searchAll = Effect.fnUntraced(function* ( return yield* Effect.fail(last); }); return searchFamily.pipe( - Effect.timeoutOrElse({ - duration: warming ? "14 seconds" : "8 seconds", - orElse: () => - answered - ? Effect.succeed([] as Hit[]) - : Effect.fail( - failure( - `${family}: mirror search budget expired.`, - "timeout", - ), - ), - }), Effect.result, - Effect.map((outcome) => ({ family, outcome })), + Effect.map((outcome) => { + outcomes[index] = { family, outcome }; + }), ); }, - { concurrency: 4 }, + { concurrency: 5, discard: true }, + ).pipe( + Effect.timeoutOrElse({ + duration: Math.max(0, budgetMs), + orElse: () => + Effect.sync(() => { + expired = true; + host.logger.warn( + "Library search budget expired; keeping completed results.", + ); + }), + }), ); const failures = outcomes.flatMap(({ family, outcome }) => Result.isFailure(outcome) ? [`${family}: ${outcome.failure.message}`] : [], @@ -166,10 +174,11 @@ const searchAll = Effect.fnUntraced(function* ( const kinds = outcomes.flatMap(({ outcome }) => Result.isFailure(outcome) ? [outcome.failure.kind] : [], ); + if (expired) kinds.push("timeout"); const kind = kinds[0] ?? "error"; return yield* Effect.fail( failure( - `No library completed a search. ${failures.join(" ")}`, + `No library completed a search. ${expired ? "Library search budget expired. " : ""}${failures.join(" ")}`, kinds.every((value) => value === kind) ? kind : "error", ), ); @@ -179,7 +188,15 @@ const searchAll = Effect.fnUntraced(function* ( const count = Math.max(0, ...results.map((r) => r.length)); for (let i = 0; i < count; i++) for (const list of results) if (list[i]) mixed.push(list[i]); - return mergeHits(mixed, q.mediaKind, q.language, q.limit); + const releases = mergeHits(mixed, q.mediaKind, q.language, q.limit); + if (!releases.length && (expired || failures.length)) + return yield* Effect.fail( + failure( + "Some libraries could not complete the search. Retry or check source diagnostics in the BookOrbit logs.", + expired ? "timeout" : "error", + ), + ); + return releases; }); const prepare = Effect.fnUntraced(function* ( @@ -233,8 +250,11 @@ export const search = Effect.fnUntraced(function* ( cfg: PluginIndexerConfig, ) { const { host } = yield* Network; + const started = yield* Clock.currentTimeMillis; const prepared = yield* prepare(cfg); - return yield* searchAll(q, cfg, prepared).pipe( + const elapsed = (yield* Clock.currentTimeMillis) - started; + // Include discovery in the query budget so BookOrbit can try another search. + return yield* searchAll(q, cfg, prepared, false, 8000 - elapsed).pipe( Effect.provideService(Network, { host, browser: prepared.browser }), ); }); diff --git a/test/plugin.test.ts b/test/plugin.test.ts index d3765a0..9f12f5f 100644 --- a/test/plugin.test.ts +++ b/test/plugin.test.ts @@ -2151,7 +2151,7 @@ test("API quota exhaustion does not retry other mirrors of the same library", as } }); -test("a valid empty catalogue answer survives the remaining mirror budget", async () => { +test("an empty answer does not hide unfinished mirror searches", async () => { const config = cfg("libgen"); const h = host((url) => { if (url.origin === config.baseUrl) @@ -2169,16 +2169,17 @@ test("a valid empty catalogue answer survives the remaining mirror budget", asyn return new Response("No results"); return new Promise(() => {}); }); - const releases = await plugin.search( - q, - config, - h.value, - AbortSignal.timeout(10000), + await assert.rejects( + plugin.search(q, config, h.value, AbortSignal.timeout(10000)), + (error: Error & { kind?: string }) => { + assert.equal(error.kind, "timeout"); + assert.match(error.message, /budget expired/); + return true; + }, ); - assert.deepEqual(releases, []); assert( h.calls.some( - ({ url }) => new URL(url).hostname === "empty-followup-3.example", + ({ url }) => new URL(url).hostname === "empty-followup-2.example", ), ); }); @@ -2424,3 +2425,95 @@ test("narrator metadata identifies audio archives without classifying ebook file 0, ); }); + +test("search keeps completed libraries within a budget that includes discovery", async () => { + const config = cfg("libgen,memory"); + const h = host(async (url) => { + if (url.origin === config.baseUrl) { + await new Promise((resolve) => setTimeout(resolve, 2000)); + return new Response( + directory({ + libgen: Array.from( + { length: 4 }, + (_, i) => `https://budget-${i}.example`, + ), + other: ["https://library.memoryoftheworld-budget.example"], + }), + ); + } + if (url.hostname === "library.memoryoftheworld-budget.example") + return Response.json(memoryBody()); + return new Promise(() => {}); + }); + const started = performance.now(); + const releases = await plugin.search( + q, + config, + h.value, + AbortSignal.timeout(9500), + ); + const elapsed = performance.now() - started; + assert.equal(releases.length, 1); + assert.equal(JSON.parse(releases[0].guid).locations[0].family, "memory"); + assert(elapsed >= 7500 && elapsed < 9000, `Search took ${elapsed}ms`); + assert( + h.logs.some((message) => message.includes("keeping completed results")), + ); +}); + +test("an empty completed library cannot turn a timed-out search into no matches", async () => { + const config = cfg("libgen,memory"); + const h = host((url) => { + if (url.origin === config.baseUrl) + return new Response( + directory({ + libgen: Array.from( + { length: 4 }, + (_, i) => `https://empty-budget-${i}.example`, + ), + other: ["https://library.memoryoftheworld-empty-budget.example"], + }), + ); + if (url.hostname === "library.memoryoftheworld-empty-budget.example") + return Response.json({ _items: [] }); + return new Promise(() => {}); + }); + await assert.rejects( + plugin.search(q, config, h.value, AbortSignal.timeout(9500)), + (error: Error & { kind?: string }) => { + assert.equal(error.kind, "timeout"); + assert.match(error.message, /could not complete/); + return true; + }, + ); +}); + +test("caller cancellation rejects even when another library has completed", async () => { + const config = cfg("libgen,memory"); + const controller = new AbortController(); + const h = host((url) => { + if (url.origin === config.baseUrl) + return new Response( + directory({ + libgen: ["https://caller-budget.example"], + other: ["https://library.memoryoftheworld-caller-budget.example"], + }), + ); + if (url.hostname === "library.memoryoftheworld-caller-budget.example") { + setTimeout(() => controller.abort(), 50); + return Response.json(memoryBody()); + } + return new Promise(() => {}); + }); + await assert.rejects( + plugin.search(q, config, h.value, controller.signal), + /cancelled|timed out/, + ); + assert(h.logs.some((message) => message.startsWith("memory: 1 releases"))); + assert.equal( + h.calls.filter( + ({ url }) => new URL(url).hostname === "caller-budget.example", + ).length, + 1, + ); +});