diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2279457cb..e8f3d53fcd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -883,8 +883,15 @@ jobs: if [[ "$RELEASE_PLATFORM" == "mac" && "$RELEASE_ARCH" == "x64" ]]; then for manifest in release-publish/*-mac.yml; do mv "$manifest" "${manifest%.yml}-x64.yml"; done fi - printf '{"platform":"%s","arch":"%s","signed":%s}\n' \ - "$RELEASE_PLATFORM" "$RELEASE_ARCH" "$signed" \ + # A signed Windows build pins this publisher in the installed app's app-update.yml. + publisher="" + if [[ "$RELEASE_PLATFORM" == "win" && "$signed" == "true" ]]; then + publisher="$AZURE_TRUSTED_SIGNING_PUBLISHER_NAME" + fi + jq -cn --arg platform "$RELEASE_PLATFORM" --arg arch "$RELEASE_ARCH" \ + --argjson signed "$signed" --arg publisher "$publisher" \ + '{platform: $platform, arch: $arch, signed: $signed} + + (if $publisher == "" then {} else {publisherName: $publisher} end)' \ > "release-publish/signing-$RELEASE_PLATFORM-$RELEASE_ARCH.json" if [[ "$RELEASE_PLATFORM" == "mac" ]]; then echo "SCIENT_MAC_SIGNING_READY=$signed" >> "$GITHUB_ENV" @@ -1007,9 +1014,37 @@ jobs: run: | $attestation = Get-Content verify/signing.json -Raw | ConvertFrom-Json if ($attestation.platform -ne 'win' -or $attestation.arch -ne 'x64') { throw "Invalid signing attestation target" } + # electron-updater checks each later installer against the publisherName list in the + # installed app's app-update.yml, comparing the signer certificate's CN. + $pinned = @() + $inList = $false + $updateConfig = 'verify/win-unpacked/resources/app-update.yml' + $lines = if (Test-Path $updateConfig) { Get-Content $updateConfig } else { @() } + if ($attestation.signed -and $lines.Count -eq 0) { throw "The signed Windows app has no app-update.yml" } + foreach ($line in $lines) { + if ($line -match '^publisherName:\s*(\S.*)?$') { + $inList = $true + if ($Matches[1]) { $pinned += $Matches[1].Trim().Trim("'").Trim('"') } + continue + } + if ($inList -and $line -match '^\s+-\s+(.+)$') { $pinned += $Matches[1].Trim().Trim("'").Trim('"'); continue } + $inList = $false + } if ($attestation.signed) { + $expected = [string]$attestation.publisherName + if (-not $expected) { throw "The signed Windows attestation names no publisher" } + if ($pinned.Count -ne 1 -or $pinned[0] -cne $expected) { + throw "The installed app would pin publisher '$($pinned -join "', '")', not the configured '$expected'" + } $signature = Get-AuthenticodeSignature (Resolve-Path verify/installer.exe) if ($signature.Status -ne "Valid") { throw "Invalid Windows signature: $($signature.Status)" } + $subject = $signature.SignerCertificate.Subject + if ($subject -notmatch '(?:^|,\s*)CN=("(?:[^"]|"")*"|[^,]+)') { throw "The installer's signer has no CN: $subject" } + $signer = $Matches[1].Trim() + if ($signer.StartsWith('"')) { $signer = $signer.Substring(1, $signer.Length - 2).Replace('""', '"') } + if ($signer -cne $expected) { throw "The installer is signed by '$signer', not the configured publisher '$expected'" } + } elseif ($pinned.Count -gt 0) { + throw "An unsigned Windows build must not pin publisher '$($pinned -join "', '")'" } # Needs every target's resource monitor and SyncTeX runtime, not their installers. @@ -1163,6 +1198,7 @@ jobs: shell: bash env: ALLOW_UNSIGNED_WINDOWS: ${{ needs.preflight.outputs.allow_unsigned_windows }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail for attestation in release-assets/signing-mac-arm64.json release-assets/signing-mac-x64.json; do @@ -1171,16 +1207,16 @@ jobs: exit 1 } done - if ! jq -e '.signed == true' release-assets/signing-win-x64.json >/dev/null; then - if [[ "$ALLOW_UNSIGNED_WINDOWS" != "true" ]]; then - echo "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true." >&2 - exit 1 - fi - echo "WARNING: publishing an unsigned Windows installer by explicit release approval." - fi + node scripts/windows-publication-signing.ts \ + --attestation release-assets/signing-win-x64.json \ + --allow-unsigned "$ALLOW_UNSIGNED_WINDOWS" \ + --stable-repository "$SCIENT_DESKTOP_CANONICAL_REPOSITORY" \ + --beta-repository "$SCIENT_DESKTOP_BETA_REPOSITORY" - name: Remove internal build attestations shell: bash run: | + # The handoff keeps the Windows signing record for the next publication's gate. + cp release-assets/signing-win-x64.json "$RUNNER_TEMP/signing-win-x64.json" rm -f release-assets/signing-*.json rm -f release-assets/builder-debug.yml - name: Verify complete updater and server artifact set @@ -1207,6 +1243,7 @@ jobs: --channel "$RELEASE_CHANNEL" \ --source-sha "$SOURCE_SHA" \ --source-tree "$SOURCE_TREE" \ + --windows-signing "$RUNNER_TEMP/signing-win-x64.json" \ --output "release-assets/scient-release-handoff.json" - id: release_candidate name: Upload immutable release candidate @@ -1258,6 +1295,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 environment: ${{ needs.preflight.outputs.channel == 'beta' && 'beta' || 'production' }} + # Both channels share installed Windows publisher pins; hold through public promotion. + concurrency: + group: scient-desktop-stable-beta-publication + cancel-in-progress: false permissions: # SCIENT-FORK:START — write lets the stable publish dispatch the hosted app's deploy. actions: write @@ -1280,6 +1321,12 @@ jobs: run: | set -euo pipefail [[ -n "$BETA_TOKEN" ]] || { echo "SCIENT_BETA_RELEASE_TOKEN is required for the isolated artifact repository." >&2; exit 1; } + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + ref: ${{ needs.preflight.outputs.source_sha }} + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: package.json - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: scient-release-v${{ needs.preflight.outputs.version }} @@ -1341,6 +1388,7 @@ jobs: shell: bash env: GH_TOKEN: ${{ needs.preflight.outputs.channel == 'beta' && secrets.SCIENT_BETA_RELEASE_TOKEN || github.token }} + PIN_READ_TOKEN: ${{ github.token }} DISTRIBUTION_REPOSITORY: ${{ needs.preflight.outputs.repository }} RELEASE_TAG: ${{ needs.preflight.outputs.tag }} RELEASE_VERSION: ${{ needs.preflight.outputs.version }} @@ -1359,7 +1407,7 @@ jobs: release_args=(--draft --latest=false) if [[ "$RELEASE_CHANNEL" == "beta" ]]; then [[ "$DISTRIBUTION_REPOSITORY" == "$SCIENT_DESKTOP_BETA_REPOSITORY" ]] || exit 1 - stable_latest_before="$(gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" + stable_latest_before="$(GH_TOKEN="$PIN_READ_TOKEN" gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" jq -ne --arg beta "$RELEASE_VERSION" --arg stable "$stable_latest_before" ' ($beta | split("-")[0] | split(".") | map(tonumber)) > ($stable | ltrimstr("v") | split(".") | map(tonumber)) @@ -1382,10 +1430,17 @@ jobs: (cd published-assets && sha256sum -- * | sort) > "$RUNNER_TEMP/published-release-sha256.txt" diff -u "$RUNNER_TEMP/expected-release-sha256.txt" "$RUNNER_TEMP/published-release-sha256.txt" + # Read both public channels with the source token, not Beta's repository-only write token. + GH_TOKEN="$PIN_READ_TOKEN" node scripts/windows-publication-signing.ts \ + --candidate-handoff release-assets/scient-release-handoff.json \ + --allow-unsigned "$ALLOW_UNSIGNED_WINDOWS" \ + --stable-repository "$SCIENT_DESKTOP_CANONICAL_REPOSITORY" \ + --beta-repository "$SCIENT_DESKTOP_BETA_REPOSITORY" + if [[ "$RELEASE_CHANNEL" == "beta" ]]; then gh release edit "$RELEASE_TAG" --repo "$DISTRIBUTION_REPOSITORY" --draft=false --prerelease --latest=false gh release view "$RELEASE_TAG" --repo "$DISTRIBUTION_REPOSITORY" --json isDraft,isPrerelease | jq -e '.isDraft == false and .isPrerelease == true' >/dev/null - stable_latest_after="$(gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" + stable_latest_after="$(GH_TOKEN="$PIN_READ_TOKEN" gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" [[ "$stable_latest_before" == "$stable_latest_after" ]] || { echo "Stable Latest changed during Beta publication; investigate before accepting the release." >&2 exit 1 diff --git a/UPSTREAM.md b/UPSTREAM.md index ad500c9601..ca418a50ea 100644 --- a/UPSTREAM.md +++ b/UPSTREAM.md @@ -218,6 +218,11 @@ call live in `apps/web/src/scient/desktopUpdate/updateReadyNotice.tsx`; callers callers; do not reintroduce the confirmation during alignment. A managed policy's feed stays the update source on every channel. Install-on-quit and early native (Squirrel.Mac) preparation stay off: they need native-readiness tracking and packaged-build tests first. +Update outcomes and summaries (`app.update.*`, see `docs/internals/product-analytics.md`) live in +`apps/desktop/src/scient/updateOutcomes/`; `DesktopUpdates.ts`, `DesktopTelemetryPublisher.ts`, +`DesktopBackendManager.ts`, `main.ts`, the telemetry contract and `DesktopTelemetryReceiver.ts` +carry only marked calls into them. The four-hour poller contains an unexpected failure per +iteration instead of ending; keep that when aligning upstream's poller. The update pill shows only this app's own update. Other machines' servers and apps, provider CLIs and Scient-managed runtimes appear in a Scient-owned circle and card (`apps/web/src/scient/desktopUpdate/SidebarOtherUpdates.tsx`, `useOtherUpdates.ts`), mounted in diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 8b4f154b5d..56f107f487 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -943,7 +943,9 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const program = runBackendProcess({ ...config.value, - desktopTelemetryStream: desktopTelemetryPublisher.encoded, + // SCIENT-FORK: this backend's stream carries the update facts offered to it. + desktopTelemetryStream: + desktopTelemetryPublisher.encodedFor?.(spec.id) ?? desktopTelemetryPublisher.encoded, desktopBrowserStream: desktopBrowserHost.events, onDesktopBrowserCommand: desktopBrowserHost.handleCommandLine, onDesktopTelemetryControl: (message) => diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 4bf7da22f5..a66066de41 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -27,6 +27,9 @@ import * as DesktopVoice from "./app/DesktopVoice.ts"; // SCIENT-FORK:START — keep the computer awake while threads work. import * as DesktopKeepAwake from "./scient/keepAwake/DesktopKeepAwake.ts"; // SCIENT-FORK:END +// SCIENT-FORK:START — the update facts journal the updater writes and the telemetry pipe delivers. +import * as DesktopUpdateFacts from "./scient/updateOutcomes/DesktopUpdateFacts.ts"; +// SCIENT-FORK:END import * as ElectronApp from "./electron/ElectronApp.ts"; import * as ElectronDialog from "./electron/ElectronDialog.ts"; import * as ElectronMenu from "./electron/ElectronMenu.ts"; @@ -198,7 +201,8 @@ const layerDesktopPreview = PreviewManager.layer.pipe( ); const layerDesktopWindow = DesktopWindow.layer.pipe( - Layer.provideMerge(DesktopTelemetryPublisher.layer), + // SCIENT-FORK: the same journal layer as the updater's, so both share one journal. + Layer.provideMerge(DesktopTelemetryPublisher.layer.pipe(Layer.provide(DesktopUpdateFacts.layer))), Layer.provideMerge(DesktopShellEnvironment.layer), Layer.provideMerge(layerDesktopServerExposure), Layer.provideMerge(layerDesktopPreview), @@ -246,6 +250,8 @@ const layerDesktopApplication = Layer.mergeAll( Layer.provideMerge( DesktopUpdates.layer.pipe( Layer.provide(DesktopUpdatePolicy.layer), + // SCIENT-FORK: the same journal layer as the telemetry publisher's. + Layer.provide(DesktopUpdateFacts.layer), // The same layer as the foundation's, so the policy is still read once. Layer.provide(DesktopManagedPolicy.layer), ), diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts new file mode 100644 index 0000000000..1d7d01d59e --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts @@ -0,0 +1,183 @@ +// @effect-diagnostics nodeBuiltinImport:off -- a real temporary journal directory. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import type { DesktopUpdateFact } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; + +import * as DesktopConfig from "../../app/DesktopConfig.ts"; +import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; +import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import { PRIMARY_INSTANCE_ID } from "../../backend/DesktopBackendPool.ts"; +import * as DesktopUpdateFacts from "./DesktopUpdateFacts.ts"; + +const WSL_SOURCE = "wsl:Ubuntu"; + +function journalLayer(home: string) { + const config = DesktopConfig.layerTest({ SCIENT_NEXT_HOME: home }); + const environment = DesktopEnvironment.layer({ + dirname: "/repo/apps/desktop/src", + homeDirectory: home, + platform: "darwin", + processArch: "arm64", + appVersion: "1.2.3", + appPath: "/repo", + isPackaged: true, + resourcesPath: "/missing/resources", + runningUnderArm64Translation: false, + }).pipe(Layer.provide(Layer.mergeAll(NodeServices.layer, config))); + return DesktopUpdateFacts.layer.pipe( + Layer.provide( + Layer.succeed(DesktopManagedPolicy.DesktopManagedPolicy, { + read: Effect.undefined, + }), + ), + Layer.provide(environment), + Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ); +} + +/** One desktop process: its journal service, with analytics on or off for that launch. */ +function inLaunch( + home: string, + analytics: "true" | "false", + program: ( + facts: DesktopUpdateFacts.DesktopUpdateFacts["Service"], + ) => Effect.Effect, +) { + return Effect.scoped( + Effect.gen(function* () { + return yield* program(yield* DesktopUpdateFacts.DesktopUpdateFacts); + }), + ).pipe( + Effect.provide(journalLayer(home)), + Effect.provideService(HostProcess.Environment, { SCIENT_ANALYTICS_ENABLED: analytics }), + ); +} + +/** Collects the next offer each backend receives once `trigger` runs. */ +const nextOffers = Effect.fn("test.nextOffers")(function* ( + facts: DesktopUpdateFacts.DesktopUpdateFacts["Service"], + trigger: Effect.Effect, +) { + const watch = (sourceId: string) => + facts.deliveries(sourceId).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped); + const primary = yield* watch(PRIMARY_INSTANCE_ID); + const wsl = yield* watch(WSL_SOURCE); + yield* Effect.yieldNow; + yield* trigger; + const offered = (yield* Fiber.join(primary)).flat(); + // Two backends have two analytics identities: an offer to both would count each fact twice. + yield* Fiber.interrupt(wsl); + assert.isFalse(Exit.isSuccess(yield* Fiber.await(wsl))); + return offered; +}); + +const ids = (facts: ReadonlyArray) => facts.map((fact) => fact.id); + +function temporaryHome(): string { + return NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-update-facts-")); +} + +function journalFile(home: string): string { + return NodePath.join(home, "runtime", DesktopUpdateFacts.DESKTOP_UPDATE_FACTS_FILE); +} + +describe("DesktopUpdateFacts", () => { + it.effect( + "offers journaled facts to one backend, with the same ids after a restart, until acknowledged", + () => + Effect.gen(function* () { + const home = temporaryHome(); + const recorded = yield* inLaunch(home, "true", (facts) => + Effect.gen(function* () { + yield* facts.sourceReady(WSL_SOURCE); + return yield* nextOffers( + facts, + Effect.all([ + facts.sourceReady(PRIMARY_INSTANCE_ID), + facts.record("app.update.outcome", { + step: "install", + outcome: "handoff-attempted", + }), + ]).pipe(Effect.asVoid), + ); + }), + ); + assert.lengthOf(recorded, 1); + + // The app quit before any backend acknowledged; the next launch offers the same events. + const replayed = yield* inLaunch(home, "true", (facts) => + Effect.gen(function* () { + const offered = yield* nextOffers(facts, facts.sourceReady(PRIMARY_INSTANCE_ID)); + yield* facts.acknowledge(ids(offered)); + return offered; + }), + ); + assert.deepEqual(ids(replayed), ids(recorded)); + const journal = JSON.parse(NodeFS.readFileSync(journalFile(home), "utf8")) as { + readonly facts: ReadonlyArray; + }; + assert.lengthOf(journal.facts, 0); + }), + ); + + it.effect("never keeps or replays facts when analytics is switched off", () => + Effect.gen(function* () { + const home = temporaryHome(); + yield* inLaunch(home, "true", (facts) => + facts.record("app.update.outcome", { step: "apply", outcome: "not-applied" }), + ); + assert.isTrue(NodeFS.existsSync(journalFile(home))); + + yield* inLaunch(home, "false", (facts) => + facts.record("app.update.outcome", { step: "install", outcome: "handoff-attempted" }), + ); + assert.isFalse(NodeFS.existsSync(journalFile(home))); + + const offered = yield* inLaunch(home, "true", (facts) => + nextOffers( + facts, + facts + .sourceReady(PRIMARY_INSTANCE_ID) + .pipe( + Effect.andThen( + facts.record("app.update.summary", { reason: "launch", status: "idle" }), + ), + ), + ), + ); + assert.deepEqual( + offered.map((fact) => fact.name), + ["app.update.summary"], + ); + }), + ); + + it.effect("counts facts dropped for room in the next summary", () => + Effect.gen(function* () { + const home = temporaryHome(); + const journal = yield* inLaunch(home, "true", (facts) => + Effect.gen(function* () { + for (let index = 0; index < 66; index += 1) { + yield* facts.record("app.update.outcome", { step: "check", outcome: "completed" }); + } + yield* facts.record("app.update.summary", { reason: "daily", status: "up-to-date" }); + return yield* nextOffers(facts, facts.sourceReady(PRIMARY_INSTANCE_ID)); + }), + ); + assert.lengthOf(journal, 64); + assert.equal(journal.at(-1)?.properties.droppedFactCount, 2); + }), + ); +}); diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts new file mode 100644 index 0000000000..9a6fcf274c --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts @@ -0,0 +1,219 @@ +/** + * DesktopUpdateFacts - persisted update facts, kept until one backend's analytics outbox holds + * them: installs run with every backend stopped, and apply outcomes precede any backend. + * @module DesktopUpdateFacts + */ +import { + DESKTOP_UPDATE_FACTS_LIMIT, + DesktopUpdateFact, + type DesktopUpdateFact as DesktopUpdateFactValue, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; + +import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; +import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import * as DesktopObservability from "../../app/DesktopObservability.ts"; +// Read only at call time: the backend manager's stream imports this module. +import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; +import { scientAnalyticsMetadata } from "../../backend/scientAnalyticsMetadata.ts"; + +export const DESKTOP_UPDATE_FACTS_FILE = "desktop-update-facts.json"; +const MAX_FACT_AGE_MS = 14 * 86_400_000; + +const Journal = Schema.Struct({ + version: Schema.Literal(1), + /** Facts dropped for room or age since a summary last reported the count. */ + dropped: Schema.Number, + facts: Schema.Array(DesktopUpdateFact), +}); +type Journal = typeof Journal.Type; +const JournalJson = Schema.fromJsonString(Journal); +const decodeJournal = Schema.decodeUnknownEffect(JournalJson); +const encodeJournal = Schema.encodeEffect(JournalJson); + +export type DesktopUpdateFactProperties = Readonly>; + +export class DesktopUpdateFacts extends Context.Service< + DesktopUpdateFacts, + { + /** Journals a fact and offers the journal to the designated backend; off records nothing. */ + readonly record: ( + name: DesktopUpdateFactValue["name"], + properties: DesktopUpdateFactProperties, + ) => Effect.Effect; + /** The facts offered to one backend; each offer carries every pending fact. */ + readonly deliveries: (sourceId: string) => Stream.Stream>; + readonly sourceReady: (sourceId: string) => Effect.Effect; + readonly sourceRemoved: (sourceId: string) => Effect.Effect; + /** The backend's outbox holds these facts, or analytics there must never send them. */ + readonly acknowledge: (ids: ReadonlyArray) => Effect.Effect; + } +>()("@t3tools/desktop/scient/updateOutcomes/DesktopUpdateFacts") {} + +const { logWarning } = DesktopObservability.makeComponentLogger("desktop-update-facts"); + +const emptyJournal: Journal = { version: 1, dropped: 0, facts: [] }; + +/** Keeps the newest facts within the count and age bounds, counting what it drops. */ +function boundJournal(journal: Journal, nowMs: number): Journal { + const fresh = journal.facts.filter((fact) => { + const occurredAt = Date.parse(fact.occurredAt); + return Number.isFinite(occurredAt) && nowMs - occurredAt <= MAX_FACT_AGE_MS; + }); + const kept = fresh.slice(-DESKTOP_UPDATE_FACTS_LIMIT); + return { + version: 1, + dropped: journal.dropped + journal.facts.length - kept.length, + facts: kept, + }; +} + +/** Primary when it is ready; otherwise the earliest ready backend, as in a WSL-only setup. */ +function designatedSource(ready: ReadonlyArray): string | undefined { + const primary = DesktopBackendManager.PRIMARY_INSTANCE_ID; + return ready.includes(primary) ? primary : ready[0]; +} + +const make = Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const random = yield* SecureRandom.SecureRandom; + const env = yield* HostProcess.Environment; + const metadata = scientAnalyticsMetadata(environment, env.SCIENT_ANALYTICS_ENABLED); + // Read once and shared with the updater's policy; a policy that cannot be used counts as off. + const telemetryAllowed = yield* (yield* DesktopManagedPolicy.DesktopManagedPolicy).read.pipe( + Effect.map((found) => found?.document?.limits?.telemetry !== false), + Effect.orElseSucceed(() => false), + ); + // The same rule as the rest of analytics: on unless switched off or denied by policy. + const enabled = metadata.SCIENT_ANALYTICS_ENABLED === "true" && telemetryAllowed; + const buildChannel = + metadata.SCIENT_ANALYTICS_BUILD_CHANNEL as DesktopUpdateFactValue["buildChannel"]; + + const directory = environment.path.join(environment.baseDir, "runtime"); + const journalPath = environment.path.join(directory, DESKTOP_UPDATE_FACTS_FILE); + const temporaryPath = `${journalPath}.tmp`; + const writeLock = yield* Semaphore.make(1); + + const persist = (journal: Journal) => + encodeJournal(journal).pipe( + Effect.flatMap((text) => + fileSystem + .makeDirectory(directory, { recursive: true }) + .pipe( + Effect.andThen(fileSystem.writeFileString(temporaryPath, text)), + Effect.andThen(fileSystem.rename(temporaryPath, journalPath)), + ), + ), + Effect.catch((error) => + logWarning("Could not save the update facts journal.", { errorTag: error._tag }), + ), + ); + + const nowMs = Clock.currentTimeMillis; + const loaded = yield* fileSystem.readFileString(journalPath).pipe( + Effect.flatMap(decodeJournal), + Effect.orElseSucceed(() => emptyJournal), + ); + // Facts are never kept for a session that may not send them, nor sent from one later. + const initial = enabled ? boundJournal(loaded, yield* nowMs) : emptyJournal; + if (!enabled && loaded.facts.length > 0) { + yield* fileSystem.remove(journalPath, { force: true }).pipe(Effect.ignore); + } + const journalRef = yield* Ref.make(initial); + const readyRef = yield* Ref.make>([]); + const offers = yield* PubSub.unbounded<{ + readonly sourceId: string; + readonly facts: ReadonlyArray; + }>(); + + const offer = Effect.gen(function* () { + const target = designatedSource(yield* Ref.get(readyRef)); + const { facts } = yield* Ref.get(journalRef); + if (target === undefined || facts.length === 0) return; + yield* PubSub.publish(offers, { sourceId: target, facts }); + }); + + const update = (f: (journal: Journal, nowMs: number) => Journal) => + writeLock.withPermits(1)( + Effect.gen(function* () { + const next = f(yield* Ref.get(journalRef), yield* nowMs); + yield* Ref.set(journalRef, next); + yield* persist(next); + }), + ); + + const record = Effect.fn("DesktopUpdateFacts.record")(function* ( + name: DesktopUpdateFactValue["name"], + properties: DesktopUpdateFactProperties, + ) { + if (!enabled) return; + const id = yield* random.uuidV4(); + const occurredAt = DateTime.formatIso(yield* DateTime.now); + yield* update((journal, now) => { + // A summary reports the drops since the last one; its own fact then carries the count. + const reportsDrops = name === "app.update.summary"; + const fact: DesktopUpdateFactValue = { + id, + name, + occurredAt, + appVersion: metadata.SCIENT_ANALYTICS_APP_VERSION, + buildChannel, + properties: reportsDrops + ? { ...properties, droppedFactCount: journal.dropped } + : properties, + }; + return boundJournal( + { + version: 1, + dropped: reportsDrops ? 0 : journal.dropped, + facts: [...journal.facts, fact], + }, + now, + ); + }); + yield* offer; + }); + + return DesktopUpdateFacts.of({ + record, + deliveries: (sourceId) => + Stream.fromPubSub(offers).pipe( + Stream.filter((delivery) => delivery.sourceId === sourceId), + Stream.map((delivery) => delivery.facts), + ), + sourceReady: Effect.fn("DesktopUpdateFacts.sourceReady")(function* (sourceId) { + yield* Ref.update(readyRef, (ready) => + ready.includes(sourceId) ? ready : [...ready, sourceId], + ); + yield* offer; + }), + sourceRemoved: Effect.fn("DesktopUpdateFacts.sourceRemoved")(function* (sourceId) { + const designated = yield* Ref.modify(readyRef, (ready) => [ + designatedSource(ready) === sourceId, + ready.filter((id) => id !== sourceId), + ]); + if (designated) yield* offer; + }), + acknowledge: Effect.fn("DesktopUpdateFacts.acknowledge")(function* (ids) { + yield* update((journal) => ({ + ...journal, + facts: journal.facts.filter((fact) => !ids.includes(fact.id)), + })); + }), + }); +}); + +export const layer = Layer.effect(DesktopUpdateFacts, make); diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts new file mode 100644 index 0000000000..27cf3ad45a --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts @@ -0,0 +1,362 @@ +/** + * DesktopUpdateOutcomes - what the desktop updater records about itself: one outcome per step + * attempt (`app.update.outcome`) and summaries of where an update stands (`app.update.summary`). + * Every fact goes through the persisted DesktopUpdateFacts journal. + * + * @module DesktopUpdateOutcomes + */ +import type { DesktopUpdateState } from "@t3tools/contracts"; +import { compareSemverVersions } from "@t3tools/shared/semver"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import * as DesktopUpdateFacts from "./DesktopUpdateFacts.ts"; +import { classifyUpdateFailure, type UpdateFailureClass } from "./updateFailureClass.ts"; + +export type UpdateTrigger = "user" | "background"; +export type UpdatePackageKind = "zip" | "nsis" | "appimage" | "deb"; +export type UpdateSourceKind = "github" | "scient_mirror" | "administrator"; + +/** Who asked for the update work running in this fiber; background loops provide their own. */ +export const CurrentUpdateTrigger = Context.Reference( + "@t3tools/desktop/scient/updateOutcomes/CurrentUpdateTrigger", + { defaultValue: () => "user" }, +); + +const BACKGROUND_CHECK_REASONS = new Set(["startup", "poll", "startup-recovery"]); +const SUMMARY_INTERVAL = "24 hours"; + +function checkTrigger(reason: string): UpdateTrigger { + return BACKGROUND_CHECK_REASONS.has(reason) ? "background" : "user"; +} + +export function updatePackageKind(input: { + readonly platform: NodeJS.Platform; + readonly appImage: boolean; + readonly debPackage: boolean; +}): UpdatePackageKind | undefined { + if (input.platform === "darwin") return "zip"; + if (input.platform === "win32") return "nsis"; + if (input.platform !== "linux") return undefined; + return input.appImage ? "appimage" : input.debPackage ? "deb" : undefined; +} + +/** The updater's feed: an administrator's, else Scient's own (GitHub releases or the mirror). */ +export function updateSourceKind(input: { + readonly administratorFeed: boolean; + readonly defaultProvider: string; +}): UpdateSourceKind { + if (input.administratorFeed) return "administrator"; + return input.defaultProvider === "github" ? "github" : "scient_mirror"; +} + +export interface UpdateInstallRecord { + readonly fromVersion: string; + readonly targetVersion: string; + readonly startedAt?: number | undefined; + readonly attemptRef?: string | undefined; +} + +/** What a launch observes about the install a previous run handed to the installer. */ +export function applyOutcome( + record: UpdateInstallRecord, + runningVersion: string, +): "completed" | "not-applied" | "superseded" | undefined { + if (record.fromVersion === record.targetVersion) return undefined; + if (runningVersion === record.targetVersion) return "completed"; + if (runningVersion === record.fromVersion) return "not-applied"; + return "superseded"; +} + +function disabledClass(state: DesktopUpdateState, policyOff: boolean): string { + if (state.channelBlockedByPolicy) return "channel-policy"; + if (policyOff) return "policy"; + const message = state.message ?? ""; + if (/safety policy/u.test(message)) return "safety"; + if (/no update feed/u.test(message)) return "no-feed"; + if (/packaged production builds/u.test(message)) return "development"; + if (/T3CODE_DISABLE_AUTO_UPDATE/u.test(message)) return "environment"; + if (/on Linux require/u.test(message)) return "linux-package"; + return "other"; +} + +/** Status that a summary reports a change of; a check in progress is not a change. */ +function summaryKey(state: DesktopUpdateState): string | undefined { + if (state.status === "checking") return undefined; + return state.status === "error" ? `error:${state.errorContext ?? ""}` : state.status; +} + +interface Attempt { + readonly ref: string; + readonly startedAt: number; + readonly trigger: UpdateTrigger; + readonly targetVersion: string | null; + readonly finished: boolean; +} + +type AttemptStep = "check" | "download" | "install"; + +export const make = Effect.fn("DesktopUpdateOutcomes.make")(function* (options: { + readonly facts: Option.Option; + readonly packageKind: UpdatePackageKind | undefined; + readonly sourceKind: UpdateSourceKind | undefined; + readonly policyOff: boolean; +}) { + const random = yield* SecureRandom.SecureRandom; + const attempts = yield* Ref.make>>({}); + const lastCheckRef = yield* Ref.make( + Option.none<{ readonly ok: boolean; readonly at: number }>(), + ); + const lastSuccessfulCheckAtRef = yield* Ref.make(Option.none()); + const context = { + ...(options.packageKind === undefined ? {} : { packageKind: options.packageKind }), + ...(options.sourceKind === undefined ? {} : { sourceKind: options.sourceKind }), + }; + + const record = ( + name: "app.update.outcome" | "app.update.summary", + properties: DesktopUpdateFacts.DesktopUpdateFactProperties, + ) => + Option.match(options.facts, { + onNone: () => Effect.void, + onSome: (facts) => facts.record(name, properties), + }); + + const newRef = random.uuidV4().pipe(Effect.map((uuid) => uuid.replaceAll("-", "").slice(0, 16))); + + const begin = Effect.fn("DesktopUpdateOutcomes.begin")(function* ( + step: AttemptStep, + targetVersion: string | null, + ) { + const attempt: Attempt = { + ref: yield* newRef, + startedAt: yield* Clock.currentTimeMillis, + trigger: yield* CurrentUpdateTrigger, + targetVersion, + finished: false, + }; + yield* Ref.update(attempts, (current) => ({ ...current, [step]: attempt })); + return attempt; + }); + + const outcome = ( + step: AttemptStep, + attempt: Attempt, + result: string, + details: { + readonly targetVersion?: string | null; + readonly failureClass?: UpdateFailureClass; + readonly durationMs?: number; + } = {}, + ) => { + const targetVersion = details.targetVersion ?? attempt.targetVersion; + return record("app.update.outcome", { + step, + outcome: result, + trigger: attempt.trigger, + attemptRef: attempt.ref, + ...context, + ...(targetVersion === null ? {} : { targetVersion }), + ...(details.failureClass === undefined ? {} : { failureClass: details.failureClass }), + ...(details.durationMs === undefined ? {} : { durationMs: details.durationMs }), + }); + }; + + /** Records the attempt's terminal outcome once, however many paths report its end. */ + const finish = Effect.fn("DesktopUpdateOutcomes.finish")(function* ( + step: AttemptStep, + result: string, + details: { readonly targetVersion?: string | null; readonly cause?: unknown } = {}, + ) { + const attempt = yield* Ref.modify(attempts, (current) => { + const active = current[step]; + return active === undefined || active.finished + ? [undefined, current] + : [active, { ...current, [step]: { ...active, finished: true } }]; + }); + if (attempt === undefined) return; + const now = yield* Clock.currentTimeMillis; + if (step === "check") { + yield* Ref.set(lastCheckRef, Option.some({ ok: result !== "failed", at: now })); + if (result !== "failed") yield* Ref.set(lastSuccessfulCheckAtRef, Option.some(now)); + } + yield* outcome(step, attempt, result, { + ...(details.targetVersion === undefined ? {} : { targetVersion: details.targetVersion }), + ...(result === "failed" ? { failureClass: classifyFailure(details.cause) } : {}), + ...(step === "install" ? {} : { durationMs: Math.max(0, now - attempt.startedAt) }), + }); + }); + + const summary = Effect.fn("DesktopUpdateOutcomes.summary")(function* ( + reason: "launch" | "change" | "daily", + state: DesktopUpdateState, + times: SummaryTimes, + ) { + const now = yield* Clock.currentTimeMillis; + const lastCheck = yield* Ref.get(lastCheckRef); + const lastSuccessfulCheckAt = yield* Ref.get(lastSuccessfulCheckAtRef); + const targetVersion = state.downloadedVersion ?? state.availableVersion; + yield* record("app.update.summary", { + reason, + status: state.status, + channel: state.channel, + behind: + Option.isSome(lastCheck) && !lastCheck.value.ok + ? "unknown" + : targetVersion !== null + ? compareSemverVersions(targetVersion, state.currentVersion) > 0 + ? "yes" + : "no" + : Option.isSome(lastCheck) + ? "no" + : "unknown", + stageAgeMs: Math.max(0, now - times.statusSince), + ...Option.match(lastSuccessfulCheckAt, { + onNone: () => ({}), + onSome: (at) => ({ lastCheckAgeMs: Math.max(0, now - at) }), + }), + ...(state.status === "error" && state.errorContext !== null + ? { errorContext: state.errorContext } + : {}), + ...(state.status === "downloading" + ? { + downloadPercent: state.downloadPercent ?? 0, + sinceProgressMs: Math.max(0, now - times.progressAt), + } + : {}), + ...(state.status === "downloaded" + ? { readyAgeMs: Math.max(0, now - times.downloadedAt) } + : {}), + ...(state.status === "disabled" + ? { disabledClass: disabledClass(state, options.policyOff) } + : {}), + }); + }); + + return { + checkStarted: (reason: string) => + begin("check", null).pipe( + Effect.provideService(CurrentUpdateTrigger, checkTrigger(reason)), + Effect.flatMap((attempt) => outcome("check", attempt, "started")), + ), + checkFinished: (state: DesktopUpdateState) => + finish("check", "completed", { + targetVersion: state.availableVersion, + }), + checkFailed: (cause: unknown) => finish("check", "failed", { cause }), + downloadStarted: (targetVersion: string | null) => + begin("download", targetVersion).pipe( + Effect.flatMap((attempt) => outcome("download", attempt, "started")), + ), + downloadCompleted: finish("download", "completed"), + downloadFailed: (cause: unknown) => finish("download", "failed", { cause }), + /** Starts an install attempt; its reference goes in the install record for the next launch. */ + installStarted: (targetVersion: string) => + Option.isNone(options.facts) + ? Effect.undefined + : begin("install", targetVersion).pipe(Effect.map((attempt) => attempt.ref)), + /** Before the installer takes over: the app may exit before anything else is recorded. */ + installHandoff: Ref.get(attempts).pipe( + Effect.flatMap((current) => + current.install === undefined + ? Effect.void + : outcome("install", current.install, "handoff-attempted"), + ), + ), + installFailed: (cause: unknown) => finish("install", "failed", { cause }), + applied: Effect.fn("DesktopUpdateOutcomes.applied")(function* ( + install: UpdateInstallRecord, + runningVersion: string, + ) { + const result = applyOutcome(install, runningVersion); + if (result === undefined) return; + const now = yield* Clock.currentTimeMillis; + yield* record("app.update.outcome", { + step: "apply", + outcome: result, + targetVersion: install.targetVersion, + // The prior process's download source is not stored in the install record. + ...(options.packageKind === undefined ? {} : { packageKind: options.packageKind }), + ...(install.attemptRef === undefined ? {} : { attemptRef: install.attemptRef }), + ...(install.startedAt === undefined + ? {} + : { durationMs: Math.max(0, now - install.startedAt) }), + }); + }), + /** Summaries at launch, when the status settles somewhere new, and once a day. */ + summaries: Effect.fn("DesktopUpdateOutcomes.summaries")(function* ( + latest: DesktopUpdateState, + changes: Stream.Stream, + ) { + if (Option.isNone(options.facts)) return; + const startedAt = yield* Clock.currentTimeMillis; + const stateRef = yield* Ref.make(latest); + const timesRef = yield* Ref.make({ + key: summaryKey(latest), + statusSince: startedAt, + progressAt: startedAt, + downloadedAt: startedAt, + }); + yield* summary("launch", latest, yield* Ref.get(timesRef)); + yield* Effect.sleep(SUMMARY_INTERVAL).pipe( + Effect.andThen( + Effect.all([Ref.get(stateRef), Ref.get(timesRef)]).pipe( + Effect.flatMap(([state, times]) => summary("daily", state, times)), + ), + ), + Effect.forever, + Effect.forkScoped, + ); + return yield* Stream.runForEach(changes, (state) => + Effect.gen(function* () { + const previous = yield* Ref.getAndSet(stateRef, state); + const now = yield* Clock.currentTimeMillis; + const key = summaryKey(state); + const [changed, times] = yield* Ref.modify(timesRef, (current) => { + const changed = key !== undefined && key !== current.key; + const next: SummaryTimes = { + key: changed ? key : current.key, + statusSince: changed ? now : current.statusSince, + progressAt: + state.status === "downloading" && + (previous.status !== "downloading" || + previous.downloadPercent !== state.downloadPercent) + ? now + : current.progressAt, + downloadedAt: + state.downloadedVersion !== null && + previous.downloadedVersion !== state.downloadedVersion + ? now + : current.downloadedAt, + }; + return [[changed, next] as const, next]; + }); + if (changed) yield* summary("change", state, times); + }), + ); + }), + }; +}); + +interface SummaryTimes { + readonly key: string | undefined; + readonly statusSince: number; + readonly progressAt: number; + readonly downloadedAt: number; +} + +export type DesktopUpdateOutcomes = Effect.Success>; + +/** A defect in the update code itself is internal, not an updater or transport failure. */ +function classifyFailure(cause: unknown): UpdateFailureClass { + return typeof cause === "object" && + cause !== null && + (cause as { readonly _tag?: unknown })._tag === "DesktopUpdateUnexpectedActionError" + ? "internal" + : classifyUpdateFailure(cause); +} diff --git a/apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts new file mode 100644 index 0000000000..a610832dcd --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts @@ -0,0 +1,80 @@ +import * as NodeModule from "node:module"; + +import { describe, expect, it } from "@effect/vitest"; + +import { classifyUpdateFailure } from "./updateFailureClass.ts"; + +// electron-updater's own error helpers, which apps/desktop does not declare. +const requireFromUpdater = NodeModule.createRequire( + NodeModule.createRequire(import.meta.url).resolve("electron-updater"), +); +const { newError, HttpError, CancellationError } = requireFromUpdater("builder-util-runtime") as { + newError: (message: string, code: string) => Error; + HttpError: new (statusCode: number, message?: string) => Error; + CancellationError: new () => Error; +}; + +const nodeError = (code: string, message: string) => Object.assign(new Error(message), { code }); + +describe("classifyUpdateFailure", () => { + it("classifies the errors electron-updater 6.8 reports", () => { + const cases: ReadonlyArray = [ + // GitHubProvider wraps the transport failure in a feed code; the transport cause wins. + [ + newError( + "Unable to find latest version on GitHub (https://github.com/x/y/releases/latest), please ensure a production release exists: Error: net::ERR_INTERNET_DISCONNECTED", + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + ), + "network", + ], + [ + newError( + `Unable to find latest version on GitHub (https://github.com/x/y/releases/latest), please ensure a production release exists: HttpError: 403 "rate limited"`, + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + ), + "http_4xx", + ], + // A release without its channel file is an incomplete feed, not a transport failure. + [ + newError( + "Cannot find latest-mac.yml in the latest release artifacts (https://github.com/x/y/releases/download/v1/latest-mac.yml): HttpError: 404 ", + "ERR_UPDATER_CHANNEL_FILE_NOT_FOUND", + ), + "feed", + ], + [new HttpError(503), "http_5xx"], + [ + new Error( + 'Cannot download "https://objects.githubusercontent.com/x", status 502: Bad Gateway', + ), + "http_5xx", + ], + [ + newError( + "New version 1.2.4 is not signed by the application owner: publisherNames: Scient, raw info: {}", + "ERR_UPDATER_INVALID_SIGNATURE", + ), + "signature", + ], + [ + new Error( + "Code signature at URL file:///private/var/update.zip did not pass validation: code object is not signed at all", + ), + "signature", + ], + [ + newError("sha512 checksum mismatch, expected a, got b", "ERR_CHECKSUM_MISMATCH"), + "checksum", + ], + [nodeError("ENOSPC", "ENOSPC: no space left on device, write"), "disk_full"], + [nodeError("EACCES", "spawn EACCES"), "permission"], + [new Error("net::ERR_CERT_AUTHORITY_INVALID"), "certificate"], + [nodeError("ETIMEDOUT", "connect ETIMEDOUT 140.82.112.3:443"), "timeout"], + [new CancellationError(), "cancelled"], + [new Error("something else entirely"), "unknown"], + ]; + for (const [error, expected] of cases) { + expect([String(error), classifyUpdateFailure(error)]).toEqual([String(error), expected]); + } + }); +}); diff --git a/apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts new file mode 100644 index 0000000000..48826f5f07 --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts @@ -0,0 +1,135 @@ +import * as Cause from "effect/Cause"; + +/** Why an update step failed, from the updater's own error codes, statuses and Node/Chromium errors. */ +export type UpdateFailureClass = + | "network" + | "timeout" + | "certificate" + | "http_4xx" + | "http_5xx" + | "feed" + | "checksum" + | "signature" + | "disk_full" + | "permission" + | "configuration" + | "cancelled" + | "internal" + | "unknown"; + +const FEED_CODES = new Set([ + "ERR_UPDATER_CHANNEL_FILE_NOT_FOUND", + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + "ERR_UPDATER_NO_PUBLISHED_VERSIONS", + "ERR_UPDATER_RELEASE_NOT_FOUND", + "ERR_UPDATER_ASSET_NOT_FOUND", + "ERR_UPDATER_ZIP_FILE_NOT_FOUND", + "ERR_UPDATER_INVALID_UPDATE_INFO", + "ERR_UPDATER_INVALID_RELEASE_FEED", + "ERR_UPDATER_NO_FILES_PROVIDED", + "ERR_UPDATER_NO_CHECKSUM", + "ERR_UPDATER_INVALID_VERSION", +]); +const CONFIGURATION_CODES = new Set([ + "ERR_UPDATER_INVALID_PROVIDER_CONFIGURATION", + "ERR_UPDATER_UNSUPPORTED_PROVIDER", + "ERR_UPDATER_INVALID_CHANNEL", + "ERR_UPDATER_WEB_INSTALLER_DISABLED", +]); + +const SIGNATURE = /not signed by the application owner|code ?signature|codesign/iu; +const CHECKSUM = /checksum mismatch/iu; +const DISK_FULL = /\bENOSPC\b|no space left/iu; +const PERMISSION = /\b(?:EACCES|EPERM|EROFS)\b|permission denied|operation not permitted/iu; +const CERTIFICATE = + /net::ERR_(?:CERT_|SSL_)|\b(?:CERT_[A-Z_]+|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT(?:_LOCALLY)?|SELF_SIGNED_CERT_IN_CHAIN|DEPTH_ZERO_SELF_SIGNED_CERT|ERR_TLS_CERT_ALTNAME_INVALID)\b/u; +const TIMEOUT = + /\b(?:ETIMEDOUT|ESOCKETTIMEDOUT)\b|net::ERR_(?:CONNECTION_)?TIMED_OUT|\btimed? ?out\b/iu; +const NETWORK = + /\b(?:ENOTFOUND|ECONNRESET|ECONNREFUSED|ECONNABORTED|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ENETDOWN|EPIPE)\b|net::ERR_[A-Z0-9_]+/u; +const HTTP_STATUS = /\b(?:HttpError: |status )([1-5]\d\d)\b/u; + +interface ErrorFacts { + readonly codes: ReadonlyArray; + readonly statuses: ReadonlyArray; + readonly names: ReadonlyArray; + /** Local only: classified here and never stored or sent. */ + readonly text: string; +} + +function collect(cause: unknown): ErrorFacts { + const codes: string[] = []; + const statuses: number[] = []; + const names: string[] = []; + const texts: string[] = []; + const visit = (value: unknown, depth: number): void => { + if (depth > 4 || value === null || value === undefined) return; + if (Cause.isCause(value)) { + for (const reason of value.reasons) { + if (reason._tag === "Fail") visit(reason.error, depth + 1); + else if (reason._tag === "Die") visit(reason.defect, depth + 1); + } + return; + } + if (typeof value === "string") { + texts.push(value); + return; + } + if (typeof value !== "object") return; + const error = value as { + readonly code?: unknown; + readonly statusCode?: unknown; + readonly name?: unknown; + readonly message?: unknown; + readonly cause?: unknown; + }; + if (typeof error.code === "string") codes.push(error.code); + if (typeof error.statusCode === "number") statuses.push(error.statusCode); + if (typeof error.name === "string") names.push(error.name); + // builder-util-runtime's CancellationError keeps the default name. + if (typeof value.constructor?.name === "string") names.push(value.constructor.name); + if (typeof error.message === "string") texts.push(error.message); + visit(error.cause, depth + 1); + }; + visit(cause, 0); + return { codes, statuses, names, text: texts.join("\n") }; +} + +function httpStatus(facts: ErrorFacts): number | undefined { + const status = + facts.statuses.find((value) => value >= 100 && value <= 599) ?? + facts.codes.map((code) => /^HTTP_ERROR_(\d{3})$/u.exec(code)?.[1]).find(Boolean) ?? + HTTP_STATUS.exec(facts.text)?.[1]; + return status === undefined ? undefined : Number(status); +} + +/** Classifies an electron-updater failure; feed errors wrap transport causes, so those win. */ +export function classifyUpdateFailure(cause: unknown): UpdateFailureClass { + try { + const facts = collect(cause); + const has = (code: string) => facts.codes.includes(code); + if (has("ERR_UPDATER_INVALID_SIGNATURE") || SIGNATURE.test(facts.text)) return "signature"; + if (has("ERR_CHECKSUM_MISMATCH") || CHECKSUM.test(facts.text)) return "checksum"; + if (has("ENOSPC") || DISK_FULL.test(facts.text)) return "disk_full"; + if (["EACCES", "EPERM", "EROFS"].some(has) || PERMISSION.test(facts.text)) return "permission"; + if (CERTIFICATE.test(facts.text) || facts.codes.some((code) => CERTIFICATE.test(code))) { + return "certificate"; + } + if (TIMEOUT.test(facts.text) || facts.codes.some((code) => TIMEOUT.test(code))) { + return "timeout"; + } + if (NETWORK.test(facts.text) || facts.codes.some((code) => NETWORK.test(code))) { + return "network"; + } + const feed = facts.codes.some((code) => FEED_CODES.has(code)); + const status = httpStatus(facts); + if (feed && (status === undefined || status === 404)) return "feed"; + if (status !== undefined && status >= 400) return status >= 500 ? "http_5xx" : "http_4xx"; + if (feed) return "feed"; + if (facts.codes.some((code) => CONFIGURATION_CODES.has(code))) return "configuration"; + if (facts.names.includes("CancellationError")) return "cancelled"; + return "unknown"; + } catch { + return "unknown"; + } +} diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts index c6868aec83..28b93af1c1 100644 --- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts +++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts @@ -26,6 +26,9 @@ import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronPowerMonitor from "../electron/ElectronPowerMonitor.ts"; import * as DesktopRendererHistory from "./DesktopRendererHistory.ts"; import * as DesktopShellEnvironment from "../shell/DesktopShellEnvironment.ts"; +// SCIENT-FORK:START — journaled update facts reach one backend's analytics. +import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +// SCIENT-FORK:END const LIVE_SAMPLE_INTERVAL = Duration.seconds(1); const BATTERY_SAMPLE_INTERVAL = Duration.seconds(5); @@ -66,6 +69,10 @@ export class DesktopTelemetryPublisher extends Context.Service< readonly latest: Effect.Effect>; readonly changes: Stream.Stream; readonly encoded: Stream.Stream; + // SCIENT-FORK:START — one backend's stream, which also carries the update facts offered to it; + // without it a backend reads `encoded`. + readonly encodedFor?: (sourceId: string) => Stream.Stream; + // SCIENT-FORK:END readonly handleControlForSource: ( sourceId: string, message: DesktopTelemetryControlMessage, @@ -150,6 +157,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { const powerMonitor = yield* ElectronPowerMonitor.ElectronPowerMonitor; const rendererHistory = yield* DesktopRendererHistory.DesktopRendererHistory; const shellEnvironment = yield* DesktopShellEnvironment.DesktopShellEnvironment; + // SCIENT-FORK: absent in tests and tools that never deliver update facts. + const updateFacts = yield* Effect.serviceOption(DesktopUpdateFacts.DesktopUpdateFacts); yield* electronApp.whenReady; const initialPowerState: PowerState = { @@ -355,8 +364,18 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { return Queue.offer(updateCommitQueue, message).pipe(Effect.asVoid); case "cancelDesktopUpdate": return Queue.offer(updateCancellationQueue, message).pipe(Effect.asVoid); + // SCIENT-FORK:START — a ready backend can take journaled update facts. case "serverReady": - return Effect.void; + return Option.match(updateFacts, { + onNone: () => Effect.void, + onSome: (facts) => facts.sourceReady(sourceId), + }); + case "ackDesktopUpdateFacts": + return Option.match(updateFacts, { + onNone: () => Effect.void, + onSome: (facts) => facts.acknowledge(message.ids), + }); + // SCIENT-FORK:END } }; const removeControlSource: DesktopTelemetryPublisher["Service"]["removeControlSource"] = ( @@ -373,6 +392,13 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { ? Effect.void : Queue.offer(sampleTriggers, undefined).pipe(Effect.asVoid), ), + // SCIENT-FORK: pending update facts move to another ready backend. + Effect.andThen( + Option.match(updateFacts, { + onNone: () => Effect.void, + onSome: (facts) => facts.sourceRemoved(sourceId), + }), + ), ); const snapshots = Stream.unwrap( @@ -401,17 +427,47 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { ); }), ); - const encoded = Stream.concat( - Stream.make({ - version: 1, - type: "desktopTelemetryHello", - electronPid: process.pid, - } as const), - Stream.merge( - Stream.merge(Stream.merge(snapshots, updateReports), shellEnvironment.updates), - Stream.fromPubSub(appHealthChanges), - ), - ).pipe(Stream.map((message) => textEncoder.encode(`${encodeMessage(message)}\n`))); + // SCIENT-FORK:START — a backend's own stream adds the update facts offered to it. + const encodedWith = (extra: Stream.Stream) => + Stream.concat( + Stream.make({ + version: 1, + type: "desktopTelemetryHello", + electronPid: process.pid, + } as const), + Stream.merge( + Stream.merge( + Stream.merge(Stream.merge(snapshots, updateReports), shellEnvironment.updates), + Stream.fromPubSub(appHealthChanges), + ), + extra, + ), + ).pipe(Stream.map((message) => textEncoder.encode(`${encodeMessage(message)}\n`))); + const encoded = encodedWith(Stream.empty); + const encodedFor = (sourceId: string) => + encodedWith( + Option.match(updateFacts, { + onNone: () => Stream.empty, + onSome: (facts) => + facts.deliveries(sourceId).pipe( + Stream.mapEffect((delivered) => + Ref.get(latestUpdateReport).pipe( + Effect.map( + Option.map((report): DesktopUpdateStatusReport => ({ + version: 1, + type: "desktopUpdateStatus", + state: report.state, + facts: delivered, + })), + ), + ), + ), + Stream.filter(Option.isSome), + Stream.map((report) => report.value), + ), + }), + ); + // SCIENT-FORK:END const publishUpdateReport: DesktopTelemetryPublisher["Service"]["publishUpdateReport"] = ( report, @@ -425,6 +481,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { latest: Ref.get(latest), changes: Stream.fromPubSub(changes), encoded, + // SCIENT-FORK: per-backend stream. + encodedFor, handleControlForSource, removeControlSource, publishUpdateReport, diff --git a/apps/desktop/src/updates/DesktopUpdates.scient.test.ts b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts new file mode 100644 index 0000000000..75817f7c19 --- /dev/null +++ b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts @@ -0,0 +1,225 @@ +import * as NodeModule from "node:module"; + +import { assert, describe, it } from "@effect/vitest"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; +import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +import * as DesktopUpdateOutcomes from "../scient/updateOutcomes/DesktopUpdateOutcomes.ts"; +import * as DesktopUpdates from "./DesktopUpdates.ts"; +import { flushCallbacks, makeHarness, type UpdatesHarnessOptions } from "./updatesTestHarness.ts"; +import { createInitialDesktopUpdateState } from "./updateMachine.ts"; + +const requireFromUpdater = NodeModule.createRequire( + NodeModule.createRequire(import.meta.url).resolve("electron-updater"), +); +const { newError } = requireFromUpdater("builder-util-runtime") as { + newError: (message: string, code: string) => Error; +}; + +type Recorded = { readonly name: string; readonly properties: Record }; + +/** The updater with a journal that keeps what it was given. */ +function recordingHarness(options: UpdatesHarnessOptions = {}) { + const recorded: Recorded[] = []; + const harness = makeHarness(options); + const facts = Layer.succeed( + DesktopUpdateFacts.DesktopUpdateFacts, + DesktopUpdateFacts.DesktopUpdateFacts.of({ + record: (name, properties) => Effect.sync(() => void recorded.push({ name, properties })), + deliveries: () => Stream.empty, + sourceReady: () => Effect.void, + sourceRemoved: () => Effect.void, + acknowledge: () => Effect.void, + }), + ); + const outcomes = (step: string) => + recorded + .filter((fact) => fact.name === "app.update.outcome" && fact.properties.step === step) + .map((fact) => fact.properties); + return { + harness, + outcomes, + facts, + recorded, + layer: Layer.merge(TestClock.layer(), harness.layer.pipe(Layer.provide(facts))), + }; +} + +describe("DesktopUpdates outcomes", () => { + it.effect("records one outcome for a check the updater reports twice", () => { + const offline = newError( + "Unable to find latest version on GitHub (https://github.com/x/y/releases/latest), please ensure a production release exists: Error: net::ERR_INTERNET_DISCONNECTED", + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + ); + // electron-updater emits `error` and also rejects the check's promise. + const { harness, outcomes, layer } = recordingHarness({ + checkForUpdates: Effect.suspend(() => { + harness.emit("error", offline); + return Effect.fail( + new ElectronUpdater.ElectronUpdaterCheckForUpdatesError({ + channel: "latest", + cause: offline, + }), + ); + }), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* updates.check("menu"); + yield* flushCallbacks; + const checks = outcomes("check"); + assert.deepEqual( + checks.map((check) => check.outcome), + ["started", "failed"], + ); + assert.include(checks[1], { + failureClass: "network", + trigger: "user", + attemptRef: checks[0]?.attemptRef, + }); + }), + ).pipe(Effect.provide(layer)); + }); + + it.effect("does not label an approved downgrade or an unreachable feed as behind", () => { + const { facts, recorded, layer } = recordingHarness(); + return Effect.gen(function* () { + const outcomes = yield* DesktopUpdateOutcomes.make({ + facts: Option.some(yield* DesktopUpdateFacts.DesktopUpdateFacts), + packageKind: "zip", + sourceKind: "github", + policyOff: false, + }); + const state = { + ...createInitialDesktopUpdateState( + "1.2.4-nightly.20261027.1", + { appArch: "x64", hostArch: "x64", runningUnderArm64Translation: false }, + "latest", + ), + status: "available" as const, + availableVersion: "1.2.3", + }; + yield* outcomes.checkStarted("menu"); + yield* outcomes.checkFinished(state); + yield* outcomes.summaries(state, Stream.empty); + assert.equal(recorded.at(-1)?.properties.behind, "no"); + yield* outcomes.checkStarted("menu"); + yield* outcomes.checkFailed(Object.assign(new Error("offline"), { code: "ENETUNREACH" })); + yield* outcomes.summaries( + { ...state, status: "error", availableVersion: "1.2.5" }, + Stream.empty, + ); + assert.equal(recorded.at(-1)?.properties.behind, "unknown"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(layer, facts, NodeCrypto.layer))); + }); + + it.effect("records one failed install after the handoff, however many paths report it", () => { + const refused = Object.assign(new Error("spawn EACCES"), { code: "EACCES" }); + const { harness, outcomes, layer } = recordingHarness({ + quitAndInstall: Effect.suspend(() => { + harness.emit("error", refused); + return Effect.fail( + new ElectronUpdater.ElectronUpdaterQuitAndInstallError({ + channel: "latest", + isSilent: true, + isForceRunAfter: true, + cause: refused, + }), + ); + }), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + yield* updates.install; + yield* flushCallbacks; + const installs = outcomes("install"); + assert.deepEqual( + installs.map((install) => install.outcome), + ["handoff-attempted", "failed"], + ); + assert.include(installs[1], { + failureClass: "permission", + targetVersion: "1.2.4", + attemptRef: installs[0]?.attemptRef, + }); + }), + ).pipe(Effect.provide(layer)); + }); + + it.effect("reports an install that did not apply once, not at every launch", () => + Effect.gen(function* () { + const record = JSON.stringify({ + fromVersion: "1.2.3", + targetVersion: "1.2.4", + startedAt: 0, + attemptRef: "0123456789abcdef", + }); + const first = recordingHarness({ + installRecord: record, + env: { T3CODE_DESKTOP_MOCK_UPDATES: "false" }, + appUpdateYml: "provider: github\nowner: ScientFactory\nrepo: scient-desktop\n", + }); + yield* Effect.scoped( + DesktopUpdates.DesktopUpdates.pipe(Effect.flatMap((updates) => updates.configure)), + ).pipe(Effect.provide(first.layer)); + assert.deepEqual(first.outcomes("apply"), [ + { + step: "apply", + outcome: "not-applied", + targetVersion: "1.2.4", + packageKind: "zip", + attemptRef: "0123456789abcdef", + durationMs: 0, + }, + ]); + const kept = [...first.harness.installRecords.values()][0]; + assert.isDefined(kept); + + const second = recordingHarness({ + installRecord: kept, + env: { T3CODE_DESKTOP_MOCK_UPDATES: "false" }, + appUpdateYml: "provider: generic\nurl: https://downloads.scientfactory.com/updates/\n", + }); + yield* Effect.scoped( + DesktopUpdates.DesktopUpdates.pipe(Effect.flatMap((updates) => updates.configure)), + ).pipe(Effect.provide(second.layer)); + assert.include(kept, '"outcomeReported":true'); + assert.lengthOf(second.outcomes("apply"), 0); + }), + ); + + it.effect("keeps polling after a poll fails unexpectedly", () => { + let checks = 0; + const { harness, layer } = recordingHarness({ + checkForUpdates: Effect.suspend(() => + ++checks === 2 ? Effect.die(new Error("unexpected poll failure")) : Effect.void, + ), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("4 hours"); + assert.equal(harness.checkCount(), 2); + yield* TestClock.adjust("4 hours"); + assert.equal(harness.checkCount(), 3); + }), + ).pipe(Effect.provide(layer)); + }); +}); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index c9693b2507..f6cda6c961 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -37,6 +37,10 @@ import * as DesktopState from "../app/DesktopState.ts"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as IpcChannels from "../ipc/channels.ts"; +// SCIENT-FORK:START — update outcomes and summaries for product analytics. +import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +import * as DesktopUpdateOutcomes from "../scient/updateOutcomes/DesktopUpdateOutcomes.ts"; +// SCIENT-FORK:END import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopSshEnvironment from "../ssh/DesktopSshEnvironment.ts"; import * as DesktopUpdatePolicy from "./DesktopUpdatePolicy.ts"; @@ -108,8 +112,12 @@ const DesktopUpdateInstallRecord = Schema.fromJsonString( Schema.Struct({ fromVersion: Schema.String, targetVersion: Schema.String, - // SCIENT-FORK: when the install began, so the next launch can report how long it took. + // SCIENT-FORK:START — when the install began, so the next launch can report how long it + // took; which attempt it was, and whether that launch's outcome was already recorded. startedAt: Schema.optionalKey(Schema.Number), + attemptRef: Schema.optionalKey(Schema.String), + outcomeReported: Schema.optionalKey(Schema.Boolean), + // SCIENT-FORK:END }), ); const decodeInstallRecord = Schema.decodeUnknownEffect(DesktopUpdateInstallRecord); @@ -490,6 +498,24 @@ export const make = Effect.gen(function* () { ) : false; + // SCIENT-FORK:START — update outcomes and summaries for product analytics. + const outcomes = yield* DesktopUpdateOutcomes.make({ + facts: yield* Effect.serviceOption(DesktopUpdateFacts.DesktopUpdateFacts), + packageKind: DesktopUpdateOutcomes.updatePackageKind({ + platform: environment.platform, + appImage: Option.isSome(config.appImagePath), + debPackage: isDebPackage, + }), + sourceKind: mockUpdates + ? undefined + : DesktopUpdateOutcomes.updateSourceKind({ + administratorFeed: policy.feedUrl !== undefined, + defaultProvider: scientDesktopUpdateFeed("latest").provider, + }), + policyOff: policy.mode === "off", + }); + // SCIENT-FORK:END + const hasUpdateFeedConfig = Ref.get(appUpdateYmlConfigRef).pipe( Effect.map( (appUpdateYmlConfig) => @@ -632,9 +658,13 @@ export const make = Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnCheckStart(state, yield* currentIsoTimestamp)); } yield* logUpdaterInfo("checking for updates", { reason }); + // SCIENT-FORK: one outcome per check. + yield* outcomes.checkStarted(reason); return yield* runUpdaterCheck(allowDowngrade).pipe( Effect.as(true), + // SCIENT-FORK: the updater's events applied the check's result before it resolved. + Effect.tap(() => Ref.get(updateStateRef).pipe(Effect.flatMap(outcomes.checkFinished))), Effect.catchTags({ ElectronUpdaterCheckForUpdatesError: Effect.fn( "desktop.updates.handleCheckForUpdatesFailure", @@ -645,12 +675,16 @@ export const make = Effect.gen(function* () { reduceDesktopUpdateStateOnNoUpdate(current, checkedAt), ); yield* logUpdaterInfo("no beta releases published yet"); + // SCIENT-FORK: an empty Beta feed is a completed check. + yield* outcomes.checkFinished(yield* Ref.get(updateStateRef)); return true; } const failedAt = yield* currentIsoTimestamp; yield* updateState((current) => reduceDesktopUpdateStateOnCheckFailure(current, error.message, failedAt), ); + // SCIENT-FORK: the rejected check, not the updater's matching error event, is its outcome. + yield* outcomes.checkFailed(error.cause); yield* logUpdaterError(error.message, { errorTag: error._tag, cause: describeUpdateErrorCause(error.cause), @@ -720,11 +754,15 @@ export const make = Effect.gen(function* () { return { accepted: true, completed: false }; } yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); + // SCIENT-FORK: a download that never ends still counts as started. + yield* outcomes.downloadStarted(state.availableVersion); yield* electronUpdater.setDisableDifferentialDownload( isArm64HostRunningIntelBuild(environment.runtimeInfo), ); yield* logUpdaterInfo("downloading update"); yield* electronUpdater.downloadUpdate; + // SCIENT-FORK: one outcome per download. + yield* outcomes.downloadCompleted; return { accepted: true, completed: true }; }).pipe( Effect.catchTags({ @@ -734,6 +772,8 @@ export const make = Effect.gen(function* () { yield* updateState((current) => reduceDesktopUpdateStateOnDownloadFailure(current, `${error.message} ${cause}`), ); + // SCIENT-FORK: one outcome per download. + yield* outcomes.downloadFailed(error.cause); yield* logUpdaterError(error.message, { errorTag: error._tag, cause, @@ -757,6 +797,8 @@ export const make = Effect.gen(function* () { yield* updateState((current) => reduceDesktopUpdateStateOnDownloadFailure(current, error.message), ); + // SCIENT-FORK: one outcome per download. + yield* outcomes.downloadFailed(error); yield* logUpdaterError(error.message, { errorTag: error._tag, cause: describeUpdateErrorCause(error.cause), @@ -805,7 +847,9 @@ export const make = Effect.gen(function* () { continue; } // A check offered this update just now, so the download needs no refresh first. - const result = yield* downloadAvailableUpdate(false); + const result = yield* downloadAvailableUpdate(false).pipe( + Effect.provideService(DesktopUpdateOutcomes.CurrentUpdateTrigger, "background"), + ); if (result.completed) { yield* Ref.set(autoDownloadBackoffRef, Option.none()); return; @@ -915,16 +959,26 @@ export const make = Effect.gen(function* () { updateRestartMarkerDir, DESKTOP_UPDATE_INSTALL_RECORD_FILE, ); - const writeInstallRecord = (targetVersion: string) => - Clock.currentTimeMillis.pipe( - Effect.flatMap((startedAt) => - encodeInstallRecord({ fromVersion: environment.appVersion, targetVersion, startedAt }), - ), + // SCIENT-FORK:START — the record also names the install attempt. + const saveInstallRecord = (record: typeof DesktopUpdateInstallRecord.Type) => + encodeInstallRecord(record).pipe( Effect.flatMap((text) => fileSystem.writeFileString(installRecordPath, text)), Effect.catch((error) => logUpdaterWarning("Could not record the update install.", { errorTag: error._tag }), ), ); + const writeInstallRecord = (targetVersion: string, attemptRef: string | undefined) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((startedAt) => + saveInstallRecord({ + fromVersion: environment.appVersion, + targetVersion, + startedAt, + ...(attemptRef === undefined ? {} : { attemptRef }), + }), + ), + ); + // SCIENT-FORK:END const removeInstallRecord = fileSystem .remove(installRecordPath, { force: true }) .pipe(Effect.ignore); @@ -947,12 +1001,31 @@ export const make = Effect.gen(function* () { ), ) : Effect.void; + // A record that did not apply stays for the next launch's notice, so it is reported once. + const reportInstall = (record: typeof DesktopUpdateInstallRecord.Type) => + logAppliedInstall(record).pipe( + Effect.andThen( + record.outcomeReported === true + ? Effect.void + : outcomes + .applied(record, environment.appVersion) + .pipe( + Effect.andThen( + DesktopUpdateOutcomes.applyOutcome(record, environment.appVersion) === + "not-applied" + ? saveInstallRecord({ ...record, outcomeReported: true }) + : Effect.void, + ), + ), + ), + ); // SCIENT-FORK:END const readNotAppliedInstallVersion = fileSystem.readFileString(installRecordPath).pipe( Effect.flatMap(decodeInstallRecord), Effect.option, - Effect.tap((record) => (Option.isSome(record) ? logAppliedInstall(record.value) : Effect.void)), + // SCIENT-FORK: the outcome of the install a previous run handed over. + Effect.tap((record) => (Option.isSome(record) ? reportInstall(record.value) : Effect.void)), Effect.map((record) => Option.flatMap(record, (value) => Option.fromNullishOr(resolveNotAppliedInstallVersion(value, environment.appVersion)), @@ -1069,7 +1142,13 @@ export const make = Effect.gen(function* () { yield* writeUpdateRestartMarker; yield* writeUpdateInProgressMarker; const { downloadedVersion } = yield* Ref.get(updateStateRef); - if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion); + // SCIENT-FORK: the install attempt starts with its record. + if (downloadedVersion !== null) { + yield* writeInstallRecord( + downloadedVersion, + yield* outcomes.installStarted(downloadedVersion), + ); + } // Stop pooled backends and SSH tunnels before quitAndInstall can outrun their scope finalizers. const instances = yield* pool.list; // stop clears desiredRunning, so retain the recovery set beforehand. @@ -1089,6 +1168,8 @@ export const make = Effect.gen(function* () { ], { concurrency: "unbounded" }, ); + // SCIENT-FORK: the installer may end this process before anything else is recorded. + yield* outcomes.installHandoff; yield* electronUpdater.quitAndInstall({ isSilent: true, isForceRunAfter: true, @@ -1098,6 +1179,8 @@ export const make = Effect.gen(function* () { Effect.catchTags({ ElectronUpdaterQuitAndInstallError: Effect.fn("desktop.updates.handleInstallFailure")( function* (error) { + // SCIENT-FORK: one outcome per install, however many paths report its failure. + yield* outcomes.installFailed(error.cause); yield* recoverFailedInstall(error.message); yield* logUpdaterError(error.message, { errorTag: error._tag, @@ -1117,6 +1200,8 @@ export const make = Effect.gen(function* () { return yield* Effect.failCause(cause); } const error = new DesktopUpdateUnexpectedActionError({ action: "install", cause }); + // SCIENT-FORK: one outcome per install, however many paths report its failure. + yield* outcomes.installFailed(error); yield* recoverFailedInstall(error.message); yield* logUpdaterError(error.message, { errorTag: error._tag, @@ -1204,10 +1289,10 @@ export const make = Effect.gen(function* () { ); yield* Effect.sleep(AUTO_UPDATE_POLL_INTERVAL).pipe( Effect.andThen(checkForUpdates("poll")), - Effect.forever, + // SCIENT-FORK:START — an unexpected failure ends one poll, not polling. Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { - return Effect.void; + return Effect.failCause(cause); } const error = new DesktopUpdatePollerError({ poller: "poll", cause }); return logUpdaterError(error.message, { @@ -1216,6 +1301,8 @@ export const make = Effect.gen(function* () { poller: error.poller, }); }), + Effect.forever, + // SCIENT-FORK:END Effect.forkScoped, ); }).pipe(Effect.withSpan("desktop.updates.startPollers")); @@ -1301,6 +1388,8 @@ export const make = Effect.gen(function* () { cause, }); if (Option.isSome(activeAction) && activeAction.value === "install") { + // SCIENT-FORK: installers report failure here; one outcome per install. + yield* outcomes.installFailed(cause); yield* recoverFailedInstall(error.message); yield* logUpdaterError(error.message, { errorTag: error._tag, @@ -1464,6 +1553,12 @@ export const make = Effect.gen(function* () { Option.orElse(disabledReason, () => channelBlockedReason(settings.updateChannel)), ), ); + // SCIENT-FORK:START — summaries for the app's lifetime, also when updates are off. + const summaryChanges = yield* PubSub.subscribe(stateChanges); + yield* outcomes + .summaries(yield* Ref.get(updateStateRef), Stream.fromSubscription(summaryChanges)) + .pipe(Effect.forkScoped); + // SCIENT-FORK:END if (Option.isSome(disabledReason)) { return; } diff --git a/apps/server/src/provider/ScientProviderInstanceSeams.ts b/apps/server/src/provider/ScientProviderInstanceSeams.ts index 56124b9bd7..fa523994a9 100644 --- a/apps/server/src/provider/ScientProviderInstanceSeams.ts +++ b/apps/server/src/provider/ScientProviderInstanceSeams.ts @@ -330,11 +330,6 @@ export interface ProviderManagedRuntimeProgress { readonly download?: ProviderRuntimeOperation["download"] | undefined; } -export interface ProviderManagedRuntimeRunOptions { - /** Scient started it on its own: the download waits as long as data keeps arriving. */ - readonly background?: boolean | undefined; -} - export interface ProviderManagedRuntimeActions { readonly getSummary: Effect.Effect; readonly plan: ( @@ -350,7 +345,6 @@ export interface ProviderManagedRuntimeActions { catalogRevision: string, report: (progress: ProviderManagedRuntimeProgress) => Effect.Effect, awaitActivationWindow?: Effect.Effect, - options?: ProviderManagedRuntimeRunOptions, ) => Effect.Effect; /** * Whether a fresh check would select a different runtime than the one this diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts index 9acf95990f..de28436563 100644 --- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts +++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts @@ -37,6 +37,10 @@ import { eventVersion as messageVersion } from "./NativeTelemetryClient.ts"; import * as ServerConfig from "../config.ts"; import * as ServerSettings from "../serverSettings.ts"; import { AnalyticsService } from "../telemetry/AnalyticsService.ts"; +// SCIENT-FORK:START — journaled desktop update facts +import { ingestDesktopUpdateFacts } from "../telemetry/DesktopUpdateFacts.ts"; +import { ScientRecordedAnalyticsIngest } from "../telemetry/ScientAnalytics.ts"; +// SCIENT-FORK:END import { subscribeBeforeSnapshotWithoutMutex } from "../utils/subscribeBeforeSnapshot.ts"; const INITIAL_SAMPLE_DEADLINE_MS = 90_000; @@ -406,6 +410,9 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make") const applyShellSnapshot = makeApplyDesktopShellEnvironmentSnapshot(environment, platform); const serverSettings = yield* ServerSettings.ServerSettingsService; const analytics = yield* Effect.serviceOption(AnalyticsService); + // SCIENT-FORK:START — ingest for desktop-journaled update facts + const recordedAnalytics = yield* ScientRecordedAnalyticsIngest; + // SCIENT-FORK:END const latest = yield* Ref.make(Option.none()); const receiverStartedAt = yield* DateTime.now; const lastContactAtMs = yield* Ref.make( @@ -604,6 +611,25 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make") return recordContact.pipe(Effect.andThen(applyShellSnapshot(message))); } + // SCIENT-FORK:START — a report carrying facts delivers them; the desktop forgets them only + // after this backend's outbox holds them, and their status is not a change. + if (message.type === "desktopUpdateStatus" && message.facts !== undefined) { + const facts = message.facts; + return recordContact.pipe( + Effect.andThen( + ingestDesktopUpdateFacts(recordedAnalytics, facts).pipe( + Effect.flatMap((ids) => + ids.length === 0 + ? Effect.void + : sendControlMessage({ version: 1, type: "ackDesktopUpdateFacts", ids }), + ), + Effect.ignoreCause(), + ), + ), + Effect.asVoid, + ); + } + // SCIENT-FORK:END if (message.type === "desktopUpdateStatus") { return recordContact.pipe( Effect.andThen(Ref.set(latestUpdateReport, Option.some(message))), diff --git a/apps/server/src/scient/ScientServerLayers.ts b/apps/server/src/scient/ScientServerLayers.ts index b83ea912c4..682ea975ce 100644 --- a/apps/server/src/scient/ScientServerLayers.ts +++ b/apps/server/src/scient/ScientServerLayers.ts @@ -15,6 +15,8 @@ import * as ProviderConnectionManager from "./providerLifecycle/ProviderConnecti import * as ProviderLifecycleCoordinator from "./providerLifecycle/ProviderLifecycleCoordinator.ts"; import * as ProviderActivity from "./providerLifecycle/ProviderActivity.ts"; import * as ProviderRuntimeManager from "./providerLifecycle/ProviderRuntimeManager.ts"; +import * as ProviderRuntimeRecovery from "./providerLifecycle/ProviderRuntimeRecovery.ts"; +import * as ProviderRuntimeRequests from "./providerLifecycle/ProviderRuntimeRequests.ts"; import * as ManagedRuntimeCatalogReconciler from "./providerLifecycle/ManagedRuntimeCatalogReconciler.ts"; import * as ScientAgentBackgroundRuntime from "./providerLifecycle/ScientAgentBackgroundRuntime.ts"; import * as ToolDiscovery from "./compute/ToolDiscovery.ts"; @@ -91,7 +93,10 @@ export const WorkspaceBindingResolverLayerLive = WorkspaceBindingResolver.layer. ); /** Connection, runtime and catalog managers over one lifecycle coordinator. */ -export const ScientProviderLifecycleLive = ScientAgentBackgroundRuntime.layer.pipe( +export const ScientProviderLifecycleLive = Layer.mergeAll( + ScientAgentBackgroundRuntime.layer, + ProviderRuntimeRecovery.layer, +).pipe( Layer.provideMerge( Layer.mergeAll( ProviderConnectionManager.layer, @@ -99,6 +104,7 @@ export const ScientProviderLifecycleLive = ScientAgentBackgroundRuntime.layer.pi ManagedRuntimeCatalogReconciler.layer, ), ), + Layer.provideMerge(ProviderRuntimeRequests.layer), Layer.provideMerge(ProviderLifecycleCoordinator.layer), Layer.provideMerge(ProviderActivity.layer), ); diff --git a/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts b/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts index 3cfc4d5021..a44c448a1a 100644 --- a/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts +++ b/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts @@ -46,6 +46,7 @@ import { import { detectTargetSafely, isManagedRuntimeDowngrade, + makeManagedRuntimeProgressFilter, managedRuntimeDowngradeMessage, managedRuntimeInstallationFailureMessage, managedRuntimeRetryMessage, @@ -105,6 +106,7 @@ function mapProgress(progress: ManagedCodexRuntimeProgress): ProviderManagedRunt ? {} : { downloadedBytes: progress.downloadedBytes }), ...(progress.totalBytes === undefined ? {} : { totalBytes: progress.totalBytes }), + ...(progress.download ? { download: progress.download } : {}), }; } @@ -773,8 +775,7 @@ export const makeCodexManagedRuntimeResolution = Effect.fn("CodexManagedRuntime. if (!actionArtifact) { return yield* runtimeError("No qualified Codex artifact is available."); } - let lastStatus: ManagedCodexRuntimeProgress["stage"] | undefined; - let lastReportedBytes = 0; + const worthReporting = makeManagedRuntimeProgressFilter(); yield* Effect.tryPromise({ try: (signal) => runtime.install({ @@ -806,16 +807,7 @@ export const makeCodexManagedRuntimeResolution = Effect.fn("CodexManagedRuntime. } }, onProgress: (progress) => { - const stageChanged = progress.stage !== lastStatus; - const downloadedBytes = progress.downloadedBytes ?? 0; - const downloadAdvanced = downloadedBytes - lastReportedBytes >= 1024 * 1024; - const downloadFinished = - progress.totalBytes !== undefined && downloadedBytes === progress.totalBytes; - if (!stageChanged && !downloadAdvanced && !downloadFinished && !progress.retry) - return; - lastStatus = progress.stage; - lastReportedBytes = downloadedBytes; - runFork(report(mapProgress(progress))); + if (worthReporting(progress)) runFork(report(mapProgress(progress))); }, }), catch: (cause) => diff --git a/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts b/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts index 8680bc5919..04ea30ca3d 100644 --- a/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts +++ b/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts @@ -86,6 +86,31 @@ export function managedRuntimeRetryMessage( return `Couldn't download ${providerName} (${retry.reason}). Trying again, ${retry.retry} of ${retry.retries}.`; } +/** + * Passes a progress report on when it says something new: another stage, a retry, download facts, + * or a move of at least 1 MiB either way (a server that ignores a range starts over from zero). + */ +export function makeManagedRuntimeProgressFilter(): ( + progress: ManagedProviderRuntimeProgress, +) => boolean { + let lastStage: ManagedProviderRuntimeProgress["stage"] | undefined; + let lastBytes = 0; + return (progress) => { + const bytes = progress.downloadedBytes ?? 0; + const report = + progress.stage !== lastStage || + Math.abs(bytes - lastBytes) >= 1024 * 1024 || + (progress.totalBytes !== undefined && bytes === progress.totalBytes) || + progress.retry !== undefined || + progress.download !== undefined; + if (report) { + lastStage = progress.stage; + lastBytes = bytes; + } + return report; + }; +} + /** What `binary --version` prints when it exits 0 within five seconds. */ export const readConfiguredRuntimeVersion = Effect.fn( "ManagedProviderRuntimeActions.readConfiguredRuntimeVersion", @@ -626,7 +651,6 @@ export const makeManagedProviderRuntimeResolution = Effect.fn( catalogRevision: Parameters[1], report: Parameters[2], awaitActivationWindow: NonNullable[3]>, - options: Parameters[4], ) { const prepared = yield* prepareAction(action); const planned = prepared.plan; @@ -655,34 +679,18 @@ export const makeManagedProviderRuntimeResolution = Effect.fn( if (!actionArtifact) { return yield* runtimeError(`No qualified ${providerName} artifact is available.`); } - let lastStatus: ManagedProviderRuntimeProgress["stage"] | undefined; - let lastReportedBytes = 0; + const worthReporting = makeManagedRuntimeProgressFilter(); yield* Effect.tryPromise({ try: (signal) => runtime.install({ artifact: actionArtifact, signal, - ...(options?.background ? { downloadTimeLimitMs: null } : {}), ...(input.qualify ? { qualify: input.qualify } : {}), beforeActivate: (activationSignal) => runPromise(awaitActivationWindow, { signal: activationSignal }), onProgress: (progress) => { - const stageChanged = progress.stage !== lastStatus; - const downloadedBytes = progress.downloadedBytes ?? 0; - const downloadAdvanced = downloadedBytes - lastReportedBytes >= 1024 * 1024; - const downloadFinished = - progress.totalBytes !== undefined && downloadedBytes === progress.totalBytes; + if (!worthReporting(progress)) return; const retry = progress.retry; - if ( - !stageChanged && - !downloadAdvanced && - !downloadFinished && - !retry && - !progress.download - ) - return; - lastStatus = progress.stage; - lastReportedBytes = downloadedBytes; runFork( report({ status: progress.stage, @@ -711,8 +719,7 @@ export const makeManagedProviderRuntimeResolution = Effect.fn( catalogRevision, report, awaitActivationWindow = Effect.void, - options, - ) => runEffect(action, catalogRevision, report, awaitActivationWindow, options); + ) => runEffect(action, catalogRevision, report, awaitActivationWindow); const summary = yield* getSummary.pipe( Effect.orElseSucceed(() => ({ diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts index 2da3574de7..5c0285a369 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts @@ -1,4 +1,5 @@ import { assert, describe, it } from "@effect/vitest"; +import * as NodeOS from "node:os"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -16,6 +17,8 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as FileSystem from "effect/FileSystem"; +import * as PlatformError from "effect/PlatformError"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -48,6 +51,13 @@ import { make, ProviderRuntimeManager, } from "./ProviderRuntimeManager.ts"; +import { make as makeRequests, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; +import { make as makeRecovery } from "./ProviderRuntimeRecovery.ts"; +import * as ServerConfig from "../../config.ts"; +import { ProcessRunner } from "../../processRunner.ts"; +import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; +import { make as makeAgentBackground } from "./ScientAgentBackgroundRuntime.ts"; +import * as AgentPolicy from "./ScientAgentRuntimePolicy.ts"; const CODEX = ProviderDriverKind.make("codex"); const INSTANCE = ProviderInstanceId.make("codex"); @@ -134,6 +144,9 @@ function makeHarness( readonly useProductionLayer?: boolean; readonly isBusy?: ProviderActivityShape["isBusy"]; readonly closeInstance?: ProviderSessionManagerV2Shape["closeInstance"]; + readonly requests?: ProviderRuntimeRequests["Service"]; + readonly agentPolicy?: AgentPolicy.ScientAgentRuntimePolicy["Service"]; + readonly observeRegistryChanges?: boolean; } = {}, ) { return Effect.gen(function* () { @@ -204,7 +217,9 @@ function makeHarness( setProviderConnectionOperation: () => SubscriptionRef.get(providersRef), setProviderAuthenticationFailure: () => SubscriptionRef.get(providersRef), setProviderManagedRuntimeSummary: setRuntime, - streamChanges: Stream.empty, + streamChanges: hooks.observeRegistryChanges + ? SubscriptionRef.changes(providersRef) + : Stream.empty, }; const coordinator = yield* makeLifecycleCoordinator; const lifecycleReleaseCountRef = yield* SubscriptionRef.make(0); @@ -224,6 +239,25 @@ function makeHarness( }); const managerScope = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(managerScope, Exit.void)); + const requestLogRef = yield* SubscriptionRef.make>([]); + const requestsLayer = hooks.requests + ? Layer.succeed(ProviderRuntimeRequests, hooks.requests) + : Layer.mock(ProviderRuntimeRequests)({ + assertAvailable: Effect.void, + list: Effect.succeed([]), + changes: Stream.empty, + requested: (input) => + SubscriptionRef.update(requestLogRef, (log) => [...log, `requested:${input.action}`]), + settled: (input) => + SubscriptionRef.update(requestLogRef, (log) => [...log, `settled:${input.outcome}`]), + }); + const policyRef = yield* Ref.make({}); + const agentPolicy = + hooks.agentPolicy ?? + AgentPolicy.ScientAgentRuntimePolicy.of({ + read: Ref.get(policyRef), + update: (change) => Ref.modify(policyRef, (before) => [before, change(before)]), + }); const providerSessionsLayer = Layer.mock(ProviderSessionManagerV2)({ closeInstance: (instanceId) => Ref.update(stopCountRef, (count) => count + 1).pipe( @@ -240,8 +274,12 @@ function makeHarness( Layer.succeed(ProviderLifecycleCoordinator, trackedCoordinator), Layer.succeed(ProviderActivity, activity), providerSessionsLayer, + requestsLayer, NodeServices.layer, NodeCrypto.layer, + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-manager-layer-" }).pipe( + Layer.provide(NodeServices.layer), + ), ), ), ), @@ -250,16 +288,24 @@ function makeHarness( Effect.map((services) => Context.get(services, ProviderRuntimeManager)), ) : yield* make().pipe( + Effect.provideService(AgentPolicy.ScientAgentRuntimePolicy, agentPolicy), Effect.provideService(ProviderRegistry, registry), Effect.provideService(ProviderLifecycleCoordinator, trackedCoordinator), Effect.provideService(ProviderActivity, activity), Effect.provide( - Layer.mergeAll(providerSessionsLayer, NodeServices.layer, NodeCrypto.layer), + Layer.mergeAll( + providerSessionsLayer, + requestsLayer, + NodeServices.layer, + NodeCrypto.layer, + ), ), Scope.provide(managerScope), ); return { manager, + registry, + agentPolicy, providersRef, reloadCountRef, reloadedInstancesRef, @@ -268,6 +314,7 @@ function makeHarness( stopCountRef, coordinator: trackedCoordinator, lifecycleReleaseCountRef, + requestLogRef, closeManager: Scope.close(managerScope, Exit.void), }; }); @@ -285,6 +332,48 @@ function installPlan() { }; } +const AGENT = ProviderDriverKind.make("scient"); +const agentRuntime: ProviderRuntimeSummary = { + ...missingRuntime, + qualifiedVersion: "0.1.6", + availableManagedVersion: "0.1.6", + actions: ["install", "update", "repair"], +}; +const agentProvider: ServerProvider = { + ...provider, + driver: AGENT, + connection: { ...provider.connection!, runtime: agentRuntime }, +}; + +const agentTestLayer = (prefix: string) => + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ); + +const startAgentLoop = Effect.fn("test.startAgentLoop")(function* ( + manager: ProviderRuntimeManager["Service"], + registry: ProviderRegistry["Service"], + policy: AgentPolicy.ScientAgentRuntimePolicy["Service"], +) { + yield* makeAgentBackground().pipe( + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + Effect.provideService(AgentPolicy.ScientAgentRuntimePolicy, policy), + Effect.provide(Layer.mergeAll(Layer.mock(ProcessRunner)({}), Layer.mock(AnalyticsService)({}))), + Effect.provideServiceEffect( + ServerConfig.ServerConfig, + ServerConfig.ServerConfig.pipe( + Effect.map((config) => ({ ...config, mode: "desktop" as const })), + ), + ), + Effect.provideService(HostProcess.Platform, "darwin"), + ); +}); + +const loadAgentPolicy = Layer.build(AgentPolicy.layer).pipe( + Effect.map((services) => Context.get(services, AgentPolicy.ScientAgentRuntimePolicy)), +); + describe("ProviderRuntimeManager", () => { it.effect("publishes one managed install and reloads every instance of the driver", () => Effect.gen(function* () { @@ -988,6 +1077,42 @@ describe("ProviderRuntimeManager", () => { }), ); + // A later try fixes a server that stayed busy, not one that does not have the file. + it.effect.each([ + { + caseTitle: "keeps the user's request after HTTP 429 on every try", + cause: new ManagedRuntimeFileError("HTTP 429 on all 5 tries.", { + failureClass: "http_4xx", + retryable: true, + }), + outcome: "retryable", + }, + { + caseTitle: "drops the user's request after HTTP 404", + cause: new ManagedRuntimeFileError("HTTP 404.", { failureClass: "http_4xx" }), + outcome: "failed", + }, + ])("$caseTitle", ({ cause, outcome }) => + Effect.gen(function* () { + const actions: ProviderManagedRuntimeActions = { + getSummary: Effect.succeed(systemRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => + Effect.fail(new ProviderConnectionActionError({ message: "Install failed.", cause })), + }; + const { manager, requestLogRef } = yield* makeHarness(actions, [systemProvider]); + yield* manager.start({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + }); + assert.deepStrictEqual(yield* awaitState(requestLogRef, (log) => log.length === 2), [ + "requested:install", + `settled:${outcome}`, + ]); + }), + ); + it.effect("does not mutate a shared runtime when active sessions cannot stop", () => Effect.gen(function* () { const runCount = yield* Ref.make(0); @@ -1054,24 +1179,19 @@ describe("ProviderRuntimeManager", () => { }), ); - it.effect("runs a background install without a download limit and says how it ended", () => + it.effect("runs a background install, says how it ended, and records no user request", () => Effect.gen(function* () { - const backgroundRuns = yield* Ref.make>([]); const actions: ProviderManagedRuntimeActions = { getSummary: Effect.succeed(missingRuntime), plan: () => Effect.succeed(installPlan()), - run: (_action, _revision, report, _await, options) => - Ref.update(backgroundRuns, (runs) => [...runs, options?.background]).pipe( - Effect.andThen( - report({ - status: "verifying", - message: "Verifying.", - download: { resumedBytes: 40, source: "fallback", compressed: true }, - }), - ), - ), + run: (_action, _revision, report) => + report({ + status: "verifying", + message: "Verifying.", + download: { resumedBytes: 40, source: "fallback", compressed: true }, + }), }; - const { manager, providersRef } = yield* makeHarness(actions); + const { manager, providersRef, requestLogRef } = yield* makeHarness(actions); const outcome = yield* manager.startInBackground({ instanceId: INSTANCE, action: "install", @@ -1079,7 +1199,7 @@ describe("ProviderRuntimeManager", () => { attempt: 2, }); assert.strictEqual(outcome, "succeeded"); - assert.deepStrictEqual(yield* Ref.get(backgroundRuns), [true]); + assert.deepStrictEqual(yield* SubscriptionRef.get(requestLogRef), ["settled:succeeded"]); const operation = (yield* SubscriptionRef.get(providersRef))[0]?.connection?.runtime ?.operation; assert.strictEqual(operation?.status, "succeeded"); @@ -1098,11 +1218,15 @@ describe("ProviderRuntimeManager", () => { () => Effect.gen(function* () { const never = yield* Deferred.make(); + const runs = yield* Ref.make(0); const actions: ProviderManagedRuntimeActions = { getSummary: Effect.succeed(missingRuntime), plan: () => Effect.succeed(installPlan()), - run: (_action, _revision, _report, _await, options) => - options?.background ? Deferred.await(never) : Effect.void, + // Only the user's change, the second run, finishes; background runs wait. + run: () => + Ref.getAndUpdate(runs, (count) => count + 1).pipe( + Effect.flatMap((count) => (count === 1 ? Effect.void : Deferred.await(never))), + ), }; const other: ServerProvider = { ...provider, @@ -1139,6 +1263,645 @@ describe("ProviderRuntimeManager", () => { }), ); + it.effect.each(["same instance", "another provider"] as const)( + "gives foreground startup priority during background summary discovery: %s", + (choice) => + Effect.gen(function* () { + const summaryStarted = yield* Deferred.make(); + const summaries = yield* Ref.make(0); + const runs = yield* SubscriptionRef.make(0); + const actions: ProviderManagedRuntimeActions = { + getSummary: Ref.getAndUpdate(summaries, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 0 + ? Deferred.succeed(summaryStarted, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.succeed(missingRuntime), + ), + ), + plan: () => Effect.succeed(installPlan()), + run: () => + SubscriptionRef.update(runs, (count) => count + 1).pipe(Effect.andThen(Effect.never)), + }; + const other: ServerProvider = { + ...provider, + instanceId: ProviderInstanceId.make("claude"), + driver: ProviderDriverKind.make("claudeAgent"), + }; + const { manager, providersRef, coordinator, lifecycleReleaseCountRef } = yield* makeHarness( + actions, + [provider, other], + ); + const background = yield* manager + .startInBackground({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + attempt: 1, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(summaryStarted); + const reserved = yield* coordinator.current(INSTANCE); + assert.ok(reserved); + assert.strictEqual( + (yield* SubscriptionRef.get(providersRef))[0]?.connection?.runtime?.operation, + null, + ); + const foregroundInstance = choice === "same instance" ? INSTANCE : other.instanceId; + const foreground = yield* manager.start({ + instanceId: foregroundInstance, + action: "install", + catalogRevision: "reviewed:1", + }); + assert.strictEqual(yield* Fiber.join(background), "yielded"); + const operation = foreground.providers.find( + (candidate) => candidate.instanceId === foregroundInstance, + )?.connection?.runtime?.operation; + assert.ok(operation); + assert.notStrictEqual(operation.operationId, reserved.operationId); + assert.strictEqual( + (yield* coordinator.current(foregroundInstance))?.operationId, + operation.operationId, + ); + yield* awaitState(runs, (count) => count === 1); + assert.strictEqual(yield* SubscriptionRef.get(lifecycleReleaseCountRef), 1); + if (choice === "another provider") + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + yield* manager.cancel({ + instanceId: foregroundInstance, + operationId: operation.operationId, + }); + assert.strictEqual(yield* coordinator.current(foregroundInstance), undefined); + assert.strictEqual(yield* SubscriptionRef.get(lifecycleReleaseCountRef), 2); + }), + ); + + it.effect.each(["waiting", "starting"] as const)( + "exposes and durably cancels restarted pending recovery while %s", + (phase) => + Effect.gen(function* () { + const before = yield* makeRequests(); + const request = { + requestId: "requested-before-restart", + instanceId: INSTANCE, + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* before.requested(request); + yield* before.settled({ ...request, outcome: "retryable" }); + const requests = yield* makeRequests(); + const summaryStarted = yield* Deferred.make(); + const runs = yield* Ref.make(0); + const actions: ProviderManagedRuntimeActions = { + getSummary: Deferred.succeed(summaryStarted, undefined).pipe( + Effect.andThen(Effect.never), + ), + plan: () => Effect.succeed(installPlan()), + run: () => Ref.update(runs, (count) => count + 1), + }; + const { manager, registry, providersRef, coordinator } = yield* makeHarness( + actions, + [provider], + () => Effect.void, + undefined, + Effect.void, + { requests }, + ); + const pending = (yield* SubscriptionRef.get(providersRef))[0]?.connection?.runtime; + assert.strictEqual(pending?.operation, null); + assert.strictEqual(pending?.pendingRecovery?.requestId, request.requestId); + const background = + phase === "starting" + ? yield* manager.startInBackground({ ...request, attempt: 1 }).pipe(Effect.forkChild) + : undefined; + if (background) yield* Deferred.await(summaryStarted); + const cancelled = yield* manager.cancel({ + instanceId: INSTANCE, + requestId: request.requestId, + }); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.pendingRecovery); + assert.strictEqual(cancelled.providers[0]?.connection?.runtime?.operation, null); + if (background) assert.strictEqual(yield* Fiber.join(background), "cancelled"); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.deepStrictEqual(yield* requests.list, []); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + ); + yield* TestClock.adjust("10 minutes"); + assert.strictEqual(yield* Ref.get(runs), 0); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-restarted-recovery-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + + it.effect.each(["install", "update", "repair"] as const)( + "keeps pending Agent %s cancellation across restart and automatic startup", + (action) => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const request = { + requestId: "pending-agent", + instanceId: INSTANCE, + driver: AGENT, + action, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + yield* requests.settled({ + ...request, + outcome: "retryable", + retryAfterMs: 2 * 60 * 60_000, + }); + const restarted = yield* makeRequests(); + const policy = yield* loadAgentPolicy; + const runs = yield* Ref.make(0); + const summary: ProviderRuntimeSummary = { + ...agentRuntime, + source: action === "install" ? "missing" : "scient_managed", + managedVersion: action === "install" ? null : "0.1.5", + }; + const { manager, registry } = yield* makeHarness( + { + getSummary: Effect.succeed(summary), + plan: () => Effect.succeed({ ...installPlan(), action, version: "0.1.6" }), + run: () => Ref.update(runs, (count) => count + 1), + }, + [{ ...agentProvider, connection: { ...agentProvider.connection!, runtime: summary } }], + undefined, + undefined, + undefined, + { requests: restarted, agentPolicy: policy, observeRegistryChanges: true }, + ); + yield* startAgentLoop(manager, registry, policy); + const cancelled = yield* manager.cancel({ + instanceId: INSTANCE, + requestId: request.requestId, + }); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.operation); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.pendingRecovery); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + const persisted = yield* (yield* loadAgentPolicy).read; + assert.deepStrictEqual( + persisted, + action === "update" + ? { skippedVersion: "0.1.6" } + : action === "repair" + ? { userStopped: true, skippedVersion: "0.1.6" } + : { userStopped: true }, + ); + yield* TestClock.adjust("61 seconds"); + assert.strictEqual(yield* Ref.get(runs), 0); + }).pipe(Effect.provide(agentTestLayer("scient-agent-pending-cancel-"))), + ); + + it.effect.each(["reviewed:1", "reviewed:2"])( + "defers automatic Agent starts, including shared instances, until requested recovery revalidates %s", + (reviewedPlan) => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const request = { + requestId: "cooling-agent", + instanceId: INSTANCE, + driver: AGENT, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + yield* requests.settled({ + ...request, + outcome: "retryable", + retryAfterMs: 2 * 60 * 60_000, + }); + const runs = yield* Ref.make(0); + const revision = yield* Ref.make("reviewed:1"); + const policy = yield* loadAgentPolicy; + const { manager, registry, providersRef, lifecycleReleaseCountRef } = yield* makeHarness( + { + getSummary: Effect.succeed(agentRuntime), + plan: () => + Ref.get(revision).pipe( + Effect.map((catalogRevision) => ({ + ...installPlan(), + version: "0.1.6", + catalogRevision, + })), + ), + run: () => Ref.update(runs, (count) => count + 1), + }, + [agentProvider, { ...agentProvider, instanceId: SECOND_INSTANCE }], + undefined, + undefined, + undefined, + { requests, agentPolicy: policy, observeRegistryChanges: true }, + ); + yield* startAgentLoop(manager, registry, policy); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + ); + for (const instanceId of [INSTANCE, SECOND_INSTANCE]) { + const error = yield* manager + .startInBackground({ + instanceId, + action: "install", + catalogRevision: "reviewed:1", + attempt: 1, + }) + .pipe(Effect.flip); + assert.strictEqual(error.reason, "runtime_busy"); + } + yield* TestClock.adjust("61 seconds"); + assert.strictEqual(yield* Ref.get(runs), 0); + yield* Ref.set(revision, reviewedPlan); + yield* TestClock.adjust(2 * 60 * 60_000 - 61_000); + if (reviewedPlan === "reviewed:1") { + yield* awaitState(lifecycleReleaseCountRef, (count) => count === 1); + assert.strictEqual(yield* Ref.get(runs), 1); + const operation = (yield* registry.getProviders)[0]?.connection?.runtime?.operation; + assert.strictEqual(operation?.status, "succeeded"); + assert.isTrue(operation?.background); + } else { + yield* awaitState( + providersRef, + (providers) => providers[0]?.connection?.runtime?.pendingRecovery === null, + ); + assert.strictEqual(yield* Ref.get(runs), 0); + } + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe(Effect.provide(agentTestLayer("scient-agent-cooldown-"))), + ); + + it.effect( + "rechecks Agent stop policy after another instance cancels during automatic planning", + () => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const pending = { + requestId: "cancel-other-instance", + instanceId: SECOND_INSTANCE, + driver: AGENT, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(pending); + const policy = yield* loadAgentPolicy; + const planning = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const runs = yield* Ref.make(0); + const { manager } = yield* makeHarness( + { + getSummary: Effect.succeed(agentRuntime), + plan: () => + Deferred.succeed(planning, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + Effect.as({ ...installPlan(), version: "0.1.6" }), + ), + run: () => Ref.update(runs, (count) => count + 1), + }, + [agentProvider, { ...agentProvider, instanceId: SECOND_INSTANCE }], + undefined, + undefined, + undefined, + { requests, agentPolicy: policy }, + ); + const automatic = yield* manager + .startInBackground({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + attempt: 1, + }) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(planning); + yield* manager.cancel({ instanceId: SECOND_INSTANCE, requestId: pending.requestId }); + assert.deepStrictEqual(yield* policy.read, { userStopped: true }); + assert.deepStrictEqual(yield* requests.list, []); + yield* Deferred.succeed(resume, undefined); + const result = yield* Fiber.join(automatic); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") + assert.strictEqual(result.failure.reason, "runtime_plan_stale"); + assert.strictEqual(yield* Ref.get(runs), 0); + }).pipe(Effect.provide(agentTestLayer("scient-agent-policy-race-"))), + ); + + it.effect("starts the manager with an unavailable journal but rejects runtime mutations", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + readFileString: (path, encoding) => + path.endsWith("requests.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: path, + }), + ) + : fs.readFileString(path, encoding), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const runs = yield* Ref.make(0); + const { manager, registry } = yield* makeHarness( + { + getSummary: Effect.succeed(missingRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => Ref.update(runs, (count) => count + 1), + }, + undefined, + undefined, + undefined, + undefined, + { requests }, + ); + assert.strictEqual( + (yield* manager.plan({ instanceId: INSTANCE, action: "install" })).version, + "0.147.0", + ); + const input = { + instanceId: INSTANCE, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + for (const action of [ + manager.start(input).pipe(Effect.asVoid), + manager.startInBackground({ ...input, attempt: 1 }).pipe(Effect.asVoid), + manager + .cancel({ instanceId: INSTANCE, requestId: "unreadable-request" }) + .pipe(Effect.asVoid), + ]) { + const failure = yield* action.pipe(Effect.flip); + assert.strictEqual(failure.reason, "runtime_operation_failed"); + } + assert.strictEqual(yield* Ref.get(runs), 0); + assert.strictEqual((yield* registry.getProviders)[0]?.connection?.runtime?.operation, null); + }).pipe(Effect.provide(agentTestLayer("scient-manager-unavailable-"))), + ); + + it.effect("keeps the server usable and retains intent when initial pruning cannot persist", () => + Effect.gen(function* () { + const before = yield* makeRequests(); + const orphan = { + requestId: "removed-instance", + instanceId: ProviderInstanceId.make("removed"), + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* before.requested(orphan); + const fs = yield* FileSystem.FileSystem; + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + to.endsWith("requests.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const { manager, registry } = yield* makeHarness( + { + getSummary: Effect.succeed(missingRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => Effect.die("Unexpected runtime mutation"), + }, + undefined, + undefined, + undefined, + undefined, + { requests }, + ); + assert.strictEqual( + (yield* manager.plan({ instanceId: INSTANCE, action: "install" })).catalogRevision, + "reviewed:1", + ); + assert.strictEqual((yield* registry.getProviders)[0]?.instanceId, INSTANCE); + assert.deepInclude((yield* requests.list)[0], orphan); + assert.deepInclude((yield* (yield* makeRequests()).list)[0], orphan); + }).pipe(Effect.provide(agentTestLayer("scient-manager-prune-denied-"))), + ); + + it.effect.each([ + { phase: "pending", file: "requests.json" }, + { phase: "active", file: "requests.json" }, + { phase: "pending", file: "scient-agent.json" }, + { phase: "active", file: "scient-agent.json" }, + ] as const)( + "does not acknowledge $phase Agent cancellation when $file cannot be written", + ({ phase, file }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const denied = yield* Ref.make(false); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + Ref.get(denied).pipe( + Effect.flatMap((fail) => + fail && to.endsWith(file) + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + ), + ), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const request = { + requestId: "cancel-denied", + instanceId: INSTANCE, + driver: AGENT, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + const policy = yield* loadAgentPolicy.pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const running = yield* Deferred.make(); + const { manager, coordinator } = yield* makeHarness( + { + getSummary: Effect.succeed(agentRuntime), + plan: () => Effect.succeed({ ...installPlan(), version: "0.1.6" }), + run: () => Deferred.succeed(running, undefined).pipe(Effect.andThen(Effect.never)), + }, + [agentProvider], + undefined, + undefined, + undefined, + { requests, agentPolicy: policy }, + ); + const background = + phase === "active" + ? yield* manager.startInBackground({ ...request, attempt: 1 }).pipe(Effect.forkChild) + : undefined; + if (background) yield* Deferred.await(running); + const before = yield* requests.list; + yield* Ref.set(denied, true); + const error = yield* manager + .cancel({ instanceId: INSTANCE, requestId: request.requestId }) + .pipe(Effect.flip); + assert.strictEqual(error.reason, "runtime_operation_failed"); + assert.deepStrictEqual(yield* requests.list, before); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, before); + if (background) assert.isDefined(yield* coordinator.current(INSTANCE)); + yield* Ref.set(denied, false); + yield* manager.cancel({ instanceId: INSTANCE, requestId: request.requestId }); + if (background) assert.strictEqual(yield* Fiber.join(background), "cancelled"); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + assert.deepStrictEqual(yield* (yield* loadAgentPolicy).read, { userStopped: true }); + }).pipe(Effect.provide(agentTestLayer("scient-agent-cancel-denied-"))), + ); + + it.effect( + "settles background ownership after a failed persistence write and recovers on a later pass", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const denied = yield* Ref.make(false); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + Ref.get(denied).pipe( + Effect.flatMap((fail) => + fail && to.endsWith("requests.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + ), + ), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const request = { + requestId: "settlement-denied", + instanceId: INSTANCE, + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + const runs = yield* Ref.make(0); + const { manager, registry, coordinator, lifecycleReleaseCountRef } = yield* makeHarness( + { + getSummary: Effect.succeed(missingRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => + Ref.update(runs, (count) => count + 1).pipe(Effect.andThen(Ref.set(denied, true))), + }, + [provider], + undefined, + undefined, + undefined, + { requests }, + ); + assert.strictEqual( + yield* manager.startInBackground({ ...request, attempt: 1 }), + "succeeded", + ); + yield* awaitState(lifecycleReleaseCountRef, (count) => count === 1); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.lengthOf(yield* requests.list, 1); + assert.lengthOf(yield* (yield* makeRequests()).list, 1); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + ); + // A failed lease write must not terminate recovery or start an unleased operation. + yield* TestClock.adjust("7 minutes"); + assert.strictEqual(yield* Ref.get(runs), 1); + yield* Ref.set(denied, false); + yield* TestClock.adjust("1 minute"); + yield* awaitState(lifecycleReleaseCountRef, (count) => count === 2); + assert.strictEqual(yield* Ref.get(runs), 2); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + }).pipe(Effect.provide(agentTestLayer("scient-settlement-denied-"))), + ); + + it.effect("cancels a requested foreground startup before its operation is published", () => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const summaryStarted = yield* Deferred.make(); + const runs = yield* Ref.make(0); + const actions: ProviderManagedRuntimeActions = { + getSummary: Deferred.succeed(summaryStarted, undefined).pipe(Effect.andThen(Effect.never)), + plan: () => Effect.succeed(installPlan()), + run: () => Ref.update(runs, (count) => count + 1), + }; + const { manager, providersRef, coordinator } = yield* makeHarness( + actions, + [provider], + () => Effect.void, + undefined, + Effect.void, + { requests }, + ); + const caller = yield* manager + .start({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(summaryStarted); + const pending = (yield* awaitState( + providersRef, + (providers) => providers[0]?.connection?.runtime?.pendingRecovery != null, + ))[0]!.connection!.runtime!; + assert.strictEqual(pending.operation, null); + const cancelled = yield* manager.cancel({ + instanceId: INSTANCE, + requestId: pending.pendingRecovery!.requestId, + }); + assert.isTrue(Exit.isFailure(yield* Fiber.await(caller))); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.pendingRecovery); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.strictEqual(yield* Ref.get(runs), 0); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-cancel-startup-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + it.effect("holds the lifecycle reservation until cancelled runtime state is published", () => Effect.gen(function* () { const cancellationPublicationStarted = yield* Deferred.make(); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts index b61f492336..60c783c94d 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts @@ -12,7 +12,11 @@ import { type ProviderRuntimeSummary, type ServerProvider, } from "@t3tools/contracts"; -import { managedRuntimeFailureClass } from "@scientfactory/provider-runtime"; +import { + managedRuntimeFailureClass, + managedRuntimeFailureRetryable, + managedRuntimeRetryAfterMs, +} from "@scientfactory/provider-runtime"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -27,6 +31,7 @@ import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; import type { ProviderManagedRuntimeActions, @@ -37,6 +42,13 @@ import { ProviderSessionManagerV2 } from "../../orchestration-v2/ProviderSession import { ProviderActivity } from "./ProviderActivity.ts"; import { ProviderConnectionActionError } from "./ProviderConnectionActions.ts"; import { ProviderLifecycleCoordinator } from "./ProviderLifecycleCoordinator.ts"; +import { + ProviderRuntimeRequests, + sameRuntime, + type RuntimeRequest, +} from "./ProviderRuntimeRequests.ts"; +import * as ScientAgentRuntimePolicy from "./ScientAgentRuntimePolicy.ts"; +import { recordAfterUserChange } from "./ScientAgentRuntimePolicy.ts"; export interface ProviderRuntimeManagerShape { readonly plan: ( @@ -60,11 +72,11 @@ export interface ProviderRuntimeManagerShape { */ readonly reselect: (instanceId: ProviderInstanceId) => Effect.Effect; /** - * Starts a change the user did not ask for and waits for it to end. It is marked - * `background`, its download has no time limit, and any start by the user stops it. + * Waits for a background recovery or Scient Agent provisioning attempt. A foreground + * start preempts it until its runtime change commits. */ readonly startInBackground: ( - input: ProviderRuntimeStartInput & { readonly attempt: number }, + input: ProviderRuntimeStartInput & { readonly attempt: number; readonly requestId?: string }, ) => Effect.Effect; } @@ -97,12 +109,20 @@ interface ActiveRuntimeOperation { readonly committedRef: Ref.Ref; readonly fiberRef: Ref.Ref | undefined>; readonly transitionLock: Semaphore.Semaphore; - readonly background: { readonly attempt: number } | undefined; + readonly background: { readonly attempt: number; readonly requestId?: string } | undefined; + readonly requestId: string | undefined; readonly outcome: Deferred.Deferred; /** Set once the download is complete. */ readonly downloadRef: Ref.Ref; } +interface RuntimeStartup { + readonly instanceId: ProviderInstanceId; + readonly background: ActiveRuntimeOperation["background"]; + readonly interrupt: Effect.Effect; + readonly outcome: Deferred.Deferred; +} + type ActiveRuntimeCleanupMode = "complete" | "interrupt"; const nowIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); @@ -188,10 +208,80 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const providerSessions = yield* ProviderSessionManagerV2; const lifecycleCoordinator = yield* ProviderLifecycleCoordinator; const activity = yield* ProviderActivity; + const requests = yield* ProviderRuntimeRequests; + const agentPolicy = yield* ScientAgentRuntimePolicy.ScientAgentRuntimePolicy; const random = yield* SecureRandom.SecureRandom; const activeRef = yield* Ref.make>( new Map(), ); + const startupsRef = yield* Ref.make>(new Map()); + const publicationLock = yield* Semaphore.make(1); + + const pendingRecovery = ( + runtime: ProviderRuntimeSummary, + request: RuntimeRequest | undefined, + ): NonNullable | null => + request && !ACTIVE_RUNTIME_STATUSES.has(runtime.operation?.status ?? "failed") + ? { + requestId: request.requestId, + action: request.action, + nextRetryAt: DateTime.formatIso(DateTime.makeUnsafe(request.dueAt)), + } + : null; + + const publishRuntime: typeof providerRegistry.setProviderManagedRuntimeSummary = Effect.fn( + "ProviderRuntimeManager.publishRuntime", + )(function* (input) { + return yield* publicationLock.withPermits(1)( + Effect.gen(function* () { + const request = (yield* requests.list).find( + (candidate) => candidate.instanceId === input.instanceId, + ); + return yield* providerRegistry.setProviderManagedRuntimeSummary({ + ...input, + runtime: input.runtime + ? { ...input.runtime, pendingRecovery: pendingRecovery(input.runtime, request) } + : null, + }); + }), + ); + }); + + const syncPendingRecovery = Effect.fn("ProviderRuntimeManager.syncPendingRecovery")(function* () { + yield* publicationLock.withPermits(1)( + Effect.gen(function* () { + yield* requests.assertAvailable; + const providers = yield* providerRegistry.getProviders; + const pending = yield* requests.list; + for (const request of pending) { + const provider = providers.find( + (candidate) => candidate.instanceId === request.instanceId, + ); + if (!provider?.enabled || provider.driver !== request.driver) { + yield* requests.update(request.requestId, () => undefined); + } + } + const current = yield* requests.list; + for (const provider of providers) { + const runtime = provider.connection?.runtime; + if (!runtime) continue; + const next = pendingRecovery( + runtime, + current.find((request) => request.instanceId === provider.instanceId), + ); + if ( + (runtime.pendingRecovery?.requestId ?? null) === (next?.requestId ?? null) && + runtime.pendingRecovery?.nextRetryAt === next?.nextRetryAt + ) + continue; + yield* providerRegistry.setProviderManagedRuntimeSummary({ + instanceId: provider.instanceId, + runtime: { ...runtime, pendingRecovery: next }, + }); + } + }), + ); + }); const readTarget = Effect.fn("ProviderRuntimeManager.readTarget")(function* ( instanceId: ProviderInstanceId, @@ -213,6 +303,35 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { return { actions, snapshot, provider }; }); + const persistCancellation = Effect.fn("ProviderRuntimeManager.persistCancellation")(function* ( + target: RuntimeTarget, + action: ProviderManagedRuntimeAction, + requestId: string | undefined, + ) { + if (target.provider === ProviderDriverKind.make("scient")) { + const runtime = target.snapshot.connection?.runtime; + const change = recordAfterUserChange( + { action, status: "cancelled" }, + runtime?.availableManagedVersion ?? runtime?.qualifiedVersion ?? null, + ); + if (change) yield* agentPolicy.update(change); + } + yield* requests.settled({ + instanceId: target.snapshot.instanceId, + driver: target.provider, + outcome: "cancelled", + ...(requestId ? { requestId } : {}), + }); + }); + + const persistenceActionError = (target: RuntimeTarget, message: string) => + makeError({ + provider: target.provider, + instanceId: target.snapshot.instanceId, + reason: "runtime_operation_failed", + message, + }); + const takeIfCurrent = (instanceId: ProviderInstanceId, operationId: string) => Ref.modify(activeRef, (active) => { const current = active.get(instanceId); @@ -281,7 +400,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { Effect.orElseSucceed(() => active.baseSummary), ); const finishedAt = yield* nowIso; - return yield* providerRegistry.setProviderManagedRuntimeSummary({ + return yield* publishRuntime({ instanceId, runtime: { ...latestSummary, @@ -301,7 +420,14 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const cleanupInterruptedStart = Effect.fn("ProviderRuntimeManager.cleanupInterruptedStart")( function* (instanceId: ProviderInstanceId, operationId: string) { - const active = yield* takeIfCurrent(instanceId, operationId); + const candidate = (yield* Ref.get(activeRef)).get(instanceId); + if (candidate?.operationId !== operationId) return; + const active = yield* candidate.transitionLock.withPermits(1)( + Effect.gen(function* () { + if (yield* Ref.get(candidate.committedRef)) return undefined; + return yield* takeIfCurrent(instanceId, operationId); + }), + ); if (!active) return; yield* settleActive( active, @@ -415,7 +541,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); if (actions) { const summary = yield* actions.getSummary; - yield* providerRegistry.setProviderManagedRuntimeSummary({ + yield* publishRuntime({ instanceId: candidate.instanceId, runtime: { ...summary, @@ -458,8 +584,13 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { input: ProviderRuntimeStartInput, background: ActiveRuntimeOperation["background"], outcome: Deferred.Deferred, + operationId: string, + registeredActiveRef: Ref.Ref, ) { const target = yield* readTarget(input.instanceId); + yield* requests.assertAvailable.pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); const planned = yield* plan({ instanceId: input.instanceId, action: input.action }); if (planned.catalogRevision !== input.catalogRevision) { return yield* makeError({ @@ -469,7 +600,50 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider setup plan changed. Review it again before continuing.", }); } - const operationId = `runtime-${yield* random.uuidV4()}`; + if ( + background?.requestId && + !(yield* requests.list).some((request) => request.requestId === background.requestId) + ) { + yield* Deferred.succeed(outcome, "cancelled"); + return yield* Effect.interrupt; + } + if ( + background && + !background.requestId && + (yield* requests.list).some((request) => + sameRuntime(request, { instanceId: input.instanceId, driver: target.provider }), + ) + ) { + return yield* makeError({ + provider: target.provider, + instanceId: input.instanceId, + reason: "runtime_busy", + message: "A requested runtime recovery owns this runtime.", + }); + } + if ( + background && + !background.requestId && + target.provider === ProviderDriverKind.make("scient") + ) { + const latest = yield* readTarget(input.instanceId); + const record = yield* agentPolicy.read.pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + const eligible = ScientAgentRuntimePolicy.backgroundRuntimeAction(latest.snapshot, record); + if ( + eligible.kind !== "start" || + eligible.action !== input.action || + eligible.version !== planned.version + ) { + return yield* makeError({ + provider: target.provider, + instanceId: input.instanceId, + reason: "runtime_plan_stale", + message: "Automatic Agent setup policy changed. No runtime was started.", + }); + } + } const reserved = yield* lifecycleCoordinator.reserve({ instanceId: input.instanceId, provider: target.provider, @@ -483,10 +657,20 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "Another setup or connection operation is already running for this provider.", }); } + if (background === undefined) { + yield* requests + .requested({ + requestId: operationId, + instanceId: input.instanceId, + driver: target.provider, + action: input.action, + catalogRevision: input.catalogRevision, + }) + .pipe(Effect.mapError((error) => persistenceActionError(target, error.message))); + } const startedAt = yield* nowIso; const baseSummary = yield* target.actions.getSummary.pipe( - // Nothing is published yet, so Cancel cannot reach a stalled read; the - // bound keeps it from holding the reservation indefinitely. + // Bound discovery even when no client cancels the pending request. Effect.timeoutOrElse({ duration: STARTUP_SUMMARY_TIMEOUT, orElse: () => @@ -510,6 +694,27 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { : lifecycleCoordinator.release({ operationId }).pipe(Effect.asVoid), ), ); + if (background?.requestId) { + const current = yield* readTarget(input.instanceId); + const requested = (yield* requests.list).find( + (request) => request.requestId === background.requestId, + ); + if ( + !requested || + !current.snapshot.enabled || + current.provider !== target.provider || + current.actions !== target.actions || + requested.driver !== target.provider || + requested.action !== input.action || + requested.catalogRevision !== input.catalogRevision + ) { + yield* requests + .update(background.requestId, () => undefined) + .pipe(Effect.mapError((error) => persistenceActionError(target, error.message))); + yield* Deferred.succeed(outcome, "cancelled"); + return yield* Effect.interrupt; + } + } const fiberRef = yield* Ref.make | undefined>(undefined); const committedRef = yield* Ref.make(false); const transitionLock = yield* Semaphore.make(1); @@ -525,17 +730,22 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { fiberRef, transitionLock, background, + requestId: background === undefined ? operationId : background.requestId, outcome, downloadRef, }; - const runtimeReserved = yield* Ref.modify(activeRef, (current) => { - if ([...current.values()].some((candidate) => candidate.provider === target.provider)) { - return [false, current] as const; - } - const next = new Map(current); - next.set(input.instanceId, active); - return [true, next] as const; - }); + const runtimeReserved = yield* Effect.uninterruptible( + Ref.modify(activeRef, (current) => { + if ([...current.values()].some((candidate) => candidate.provider === target.provider)) { + return [false, current] as const; + } + const next = new Map(current); + next.set(input.instanceId, active); + return [true, next] as const; + }).pipe( + Effect.tap((reserved) => (reserved ? Ref.set(registeredActiveRef, true) : Effect.void)), + ), + ); if (!runtimeReserved) { yield* lifecycleCoordinator.release({ operationId }); return yield* makeError({ @@ -562,7 +772,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { if (progress.download) yield* Ref.set(downloadRef, progress.download); const current = (yield* Ref.get(activeRef)).get(input.instanceId); if (current?.operationId !== operationId) return; - yield* providerRegistry.setProviderManagedRuntimeSummary({ + yield* publishRuntime({ instanceId: input.instanceId, runtime: { ...baseSummary, @@ -606,26 +816,24 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ), ); - const initialProviders = yield* providerRegistry - .setProviderManagedRuntimeSummary({ - instanceId: input.instanceId, - runtime: { - ...baseSummary, - operation: operation({ - operationId, - action: input.action, - status: "preparing", - startedAt, - message: "Preparing the provider runtime operation.", - background, - }), - }, - }) - .pipe(Effect.onExit(cleanupStartupExit)); + const initialProviders = yield* publishRuntime({ + instanceId: input.instanceId, + runtime: { + ...baseSummary, + operation: operation({ + operationId, + action: input.action, + status: "preparing", + startedAt, + message: "Preparing the provider runtime operation.", + background, + }), + }, + }).pipe(Effect.onExit(cleanupStartupExit)); const publishUnexpectedFailure = Effect.gen(function* () { const finishedAt = yield* nowIso; - yield* providerRegistry.setProviderManagedRuntimeSummary({ + yield* publishRuntime({ instanceId: input.instanceId, runtime: { ...baseSummary, @@ -652,7 +860,6 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { input.catalogRevision, publishProgress, awaitActivationWindow, - { background: background !== undefined }, ), ).pipe(Effect.tap(() => Ref.set(committedRef, true))), ); @@ -686,6 +893,10 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { } const finishedAt = yield* nowIso; const download = yield* Ref.get(downloadRef); + const retryAfterMs = + completion._tag === "Failure" + ? managedRuntimeRetryAfterMs(completion.failure) + : undefined; const latestSummary = result._tag === "Success" ? ((yield* providerRegistry.getProviders).find( @@ -696,43 +907,61 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { Effect.gen(function* () { const current = (yield* Ref.get(activeRef)).get(input.instanceId); if (current?.operationId !== operationId) return; - yield* providerRegistry - .setProviderManagedRuntimeSummary({ - instanceId: input.instanceId, - runtime: { - ...latestSummary, - operation: operation({ - operationId, - action: input.action, - status: completion._tag === "Success" ? "succeeded" : "failed", - startedAt, - finishedAt, - message: + yield* publishRuntime({ + instanceId: input.instanceId, + runtime: { + ...latestSummary, + operation: operation({ + operationId, + action: input.action, + status: completion._tag === "Success" ? "succeeded" : "failed", + startedAt, + finishedAt, + message: + completion._tag === "Success" + ? runtimeSuccessMessage(input.action) + : completion.failure.message, + ...(completion._tag === "Success" + ? {} + : { + failureClass: managedRuntimeFailureClass(completion.failure, lastStage), + }), + background, + download, + }), + }, + }).pipe( + Effect.catchCause(() => publishUnexpectedFailure), + Effect.andThen( + requests + .settled({ + instanceId: input.instanceId, + driver: target.provider, + ...(active.requestId ? { requestId: active.requestId } : {}), + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + outcome: completion._tag === "Success" - ? runtimeSuccessMessage(input.action) - : completion.failure.message, - ...(completion._tag === "Success" - ? {} - : { - failureClass: managedRuntimeFailureClass(completion.failure, lastStage), - }), - background, - download, - }), - }, - }) - .pipe( - Effect.catchCause(() => publishUnexpectedFailure), - Effect.andThen( - Deferred.succeed( - outcome, - completion._tag === "Success" ? "succeeded" : "failed", + ? "succeeded" + : managedRuntimeFailureRetryable(completion.failure, lastStage) + ? "retryable" + : "failed", + }) + .pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning( + "Runtime request settlement could not be persisted; retaining recovery intent", + ), + }), ), - ), - Effect.ensuring( - cleanupIfCurrent(input.instanceId, operationId, "complete").pipe(Effect.asVoid), - ), - ); + ), + Effect.andThen( + Deferred.succeed(outcome, completion._tag === "Success" ? "succeeded" : "failed"), + ), + Effect.ensuring( + cleanupIfCurrent(input.instanceId, operationId, "complete").pipe(Effect.asVoid), + ), + ); }), ); }), @@ -767,64 +996,183 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { return { providers: initialProviders }; }); - // Once asked, a runtime change belongs to the manager: planning can stall for - // a long time on a loaded machine, and a client that reconnects meanwhile only - // detaches from the reply. Cancel and manager shutdown still stop it. - const start: ProviderRuntimeManagerShape["start"] = Effect.fn("ProviderRuntimeManager.start")( - function* (input) { - yield* yieldBackgroundWork(input.instanceId); - const outcome = yield* Deferred.make(); - const fiber = yield* startup(input, undefined, outcome).pipe(Effect.forkIn(managerScope)); - return yield* Fiber.join(fiber); - }, - ); + const startupAdmission = yield* Semaphore.make(1); - // The user's own change comes first: a background change of another provider stops - // and keeps its partial download for its next try. - const yieldBackgroundWork = Effect.fn("ProviderRuntimeManager.yieldBackgroundWork")(function* ( - instanceId: ProviderInstanceId, + const interruptStartup = Effect.fn("ProviderRuntimeManager.interruptStartup")(function* ( + operationId: string, + outcome: ProviderRuntimeOutcome, ) { - const background = [...(yield* Ref.get(activeRef))].filter( - ([candidate, active]) => candidate !== instanceId && active.background !== undefined, - ); - yield* Effect.forEach( - background, - ([candidate, active]) => - active.transitionLock - .withPermits(1)( + const starting = (yield* Ref.get(startupsRef)).get(operationId); + if (!starting) return; + yield* Deferred.succeed(starting.outcome, outcome); + yield* starting.interrupt; + }); + + const beginStartup = Effect.fn("ProviderRuntimeManager.beginStartup")(function* ( + input: ProviderRuntimeStartInput, + background: ActiveRuntimeOperation["background"], + outcome: Deferred.Deferred, + ) { + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const operationId = `runtime-${yield* random.uuidV4()}`; + const registered = yield* Deferred.make(); + const registeredActiveRef = yield* Ref.make(false); + const fiber = yield* restore( + Deferred.await(registered).pipe( + Effect.andThen(startup(input, background, outcome, operationId, registeredActiveRef)), + ), + ).pipe( + Effect.onExit((exit) => Effect.gen(function* () { - if (yield* Ref.get(active.committedRef)) return undefined; - return yield* takeIfCurrent(candidate, active.operationId); + if (Exit.isFailure(exit)) { + yield* cleanupInterruptedStart(input.instanceId, operationId); + // After registration, the active operation's claimant owns release through publication. + if (!(yield* Ref.get(registeredActiveRef))) { + const reservation = yield* lifecycleCoordinator.current(input.instanceId); + if (reservation?.operationId === operationId) { + yield* lifecycleCoordinator.release({ operationId }); + } + } + yield* Deferred.succeed(outcome, "interrupted"); + } + yield* Ref.update(startupsRef, (current) => { + const next = new Map(current); + next.delete(operationId); + return next; + }); }), - ) - .pipe( - Effect.flatMap((claim) => - claim - ? settleActive(claim, "interrupt", publishCancelled(candidate, claim), "yielded") - : Effect.void, - ), - Effect.ignore, ), - { discard: true }, + Effect.forkIn(managerScope), + ); + yield* Ref.update(startupsRef, (current) => + new Map(current).set(operationId, { + instanceId: input.instanceId, + background, + interrupt: Fiber.interrupt(fiber).pipe(Effect.asVoid), + outcome, + }), + ); + yield* Deferred.succeed(registered, undefined); + return fiber; + }), ); }); + const yieldBackgroundWork = Effect.fn("ProviderRuntimeManager.yieldBackgroundWork")(function* () { + const background = [...(yield* Ref.get(activeRef))].filter( + ([, active]) => active.background !== undefined, + ); + for (const [instanceId, active] of background) { + const claim = yield* active.transitionLock.withPermits(1)( + Effect.gen(function* () { + if (yield* Ref.get(active.committedRef)) return undefined; + return yield* takeIfCurrent(instanceId, active.operationId); + }), + ); + if (!claim) continue; + yield* interruptStartup(active.operationId, "yielded"); + yield* settleActive(claim, "interrupt", publishCancelled(instanceId, claim), "yielded").pipe( + Effect.ignore, + ); + } + for (const [operationId, starting] of yield* Ref.get(startupsRef)) { + if (!starting.background) continue; + // Committed changes must finish reconciliation; only unregistered startup is left to yield. + if ((yield* Ref.get(activeRef)).get(starting.instanceId)?.operationId === operationId) + continue; + yield* interruptStartup(operationId, "yielded"); + } + }); + + // A disconnected caller detaches from the reply; the manager still owns the startup. + const start: ProviderRuntimeManagerShape["start"] = Effect.fn("ProviderRuntimeManager.start")( + function* (input) { + const outcome = yield* Deferred.make(); + const fiber = yield* startupAdmission.withPermits(1)( + Effect.gen(function* () { + yield* yieldBackgroundWork(); + return yield* beginStartup(input, undefined, outcome); + }), + ); + return yield* Fiber.join(fiber); + }, + ); + const startInBackground: ProviderRuntimeManagerShape["startInBackground"] = Effect.fn( "ProviderRuntimeManager.startInBackground", )(function* (input) { const outcome = yield* Deferred.make(); - yield* startup(input, { attempt: input.attempt }, outcome).pipe( - Effect.forkIn(managerScope), - Effect.flatMap(Fiber.join), + const fiber = yield* startupAdmission.withPermits(1)( + Effect.gen(function* () { + if ([...(yield* Ref.get(startupsRef)).values()].some((starting) => !starting.background)) { + return yield* makeError({ + provider: ProviderDriverKind.make("unknown"), + instanceId: input.instanceId, + reason: "runtime_busy", + message: "A requested runtime change is starting.", + }); + } + return yield* beginStartup( + input, + { + attempt: input.attempt, + ...(input.requestId ? { requestId: input.requestId } : {}), + }, + outcome, + ); + }), ); + const exit = yield* Fiber.await(fiber); + if (Exit.isFailure(exit) && !Cause.hasInterruptsOnly(exit.cause)) { + return yield* Effect.failCause(exit.cause); + } return yield* Deferred.await(outcome); }); const cancel: ProviderRuntimeManagerShape["cancel"] = Effect.fn("ProviderRuntimeManager.cancel")( function* (input) { const target = yield* readTarget(input.instanceId); - const candidate = (yield* Ref.get(activeRef)).get(input.instanceId); - if (!candidate || candidate.operationId !== input.operationId) { + yield* requests.assertAvailable.pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + let candidate = (yield* Ref.get(activeRef)).get(input.instanceId); + const pending = + "requestId" in input + ? (yield* requests.list).find( + (request) => + request.instanceId === input.instanceId && request.requestId === input.requestId, + ) + : undefined; + let matchesActive = + candidate && + ("operationId" in input + ? candidate.operationId === input.operationId + : pending !== undefined && candidate.requestId === input.requestId); + if (!matchesActive && pending) { + yield* persistCancellation(target, pending.action, pending.requestId).pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + for (const [operationId, starting] of yield* Ref.get(startupsRef)) { + if ( + operationId === pending.requestId || + starting.background?.requestId === pending.requestId + ) { + if ((yield* Ref.get(activeRef)).get(starting.instanceId)?.operationId === operationId) + continue; + yield* interruptStartup(operationId, "cancelled"); + } + } + candidate = (yield* Ref.get(activeRef)).get(input.instanceId); + if (candidate?.requestId !== pending.requestId) { + yield* syncPendingRecovery().pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + return { providers: yield* providerRegistry.getProviders }; + } + matchesActive = true; + } + if (!candidate || !matchesActive) { return yield* makeError({ provider: target.provider, instanceId: input.instanceId, @@ -832,12 +1180,17 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider runtime operation is no longer active.", }); } + const operationId = candidate.operationId; const claim = yield* candidate.transitionLock.withPermits(1)( Effect.gen(function* () { const current = (yield* Ref.get(activeRef)).get(input.instanceId); - if (current?.operationId !== input.operationId) return undefined; + if (current?.operationId !== operationId) return undefined; if (yield* Ref.get(current.committedRef)) return "committed" as const; - return yield* takeIfCurrent(input.instanceId, input.operationId); + yield* persistCancellation(target, current.action, current.requestId).pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + if (yield* Ref.get(current.committedRef)) return "committed" as const; + return yield* takeIfCurrent(input.instanceId, operationId); }), ); if (claim === "committed") { @@ -856,6 +1209,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider runtime operation is no longer active.", }); } + yield* interruptStartup(claim.operationId, "cancelled"); const providers = yield* settleActive( claim, "interrupt", @@ -864,6 +1218,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); return { providers }; }, + Effect.uninterruptible, ); // Instances of one driver share its runtime, so one reselection per driver @@ -954,7 +1309,38 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); }).pipe(Effect.delay(RESELECT_INTERVAL), Effect.forever, Effect.forkScoped); + yield* syncPendingRecovery().pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning("Runtime recovery intent is unavailable; managed changes are disabled"), + }), + ); + yield* requests.changes.pipe( + Stream.runForEach(() => + syncPendingRecovery().pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning("Runtime recovery intent could not be pruned"), + }), + ), + ), + Effect.forkScoped, + ); + yield* providerRegistry.streamChanges.pipe( + Stream.runForEach(() => + syncPendingRecovery().pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning("Runtime recovery intent could not be pruned"), + }), + ), + ), + Effect.forkScoped, + ); + return ProviderRuntimeManager.of({ plan, start, cancel, reselect, startInBackground }); }); -export const layer = Layer.effect(ProviderRuntimeManager, make()); +export const layer = Layer.effect(ProviderRuntimeManager, make()).pipe( + Layer.provide(ScientAgentRuntimePolicy.layer), +); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts new file mode 100644 index 0000000000..924e5effe2 --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts @@ -0,0 +1,331 @@ +import * as NodeOS from "node:os"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { assert, describe, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + type ProviderRuntimeSummary, + type ServerProvider, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as PlatformError from "effect/PlatformError"; +import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerConfig from "../../config.ts"; +import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; +import { ProviderRuntimeManager, type ProviderRuntimeOutcome } from "./ProviderRuntimeManager.ts"; +import { make as makeRecovery } from "./ProviderRuntimeRecovery.ts"; +import { make as makeRequests, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; + +const INSTANCE = ProviderInstanceId.make("codex"); +const CODEX = ProviderDriverKind.make("codex"); +const MINUTE = 60_000; + +const runtime: ProviderRuntimeSummary = { + source: "missing", + supportTier: "fully_assisted", + target: "darwin-arm64", + actions: ["install"], + managedVersion: null, + previousManagedVersion: null, + operation: null, + message: "Codex setup is available.", +}; + +const codex: ServerProvider = { + instanceId: INSTANCE, + driver: CODEX, + enabled: true, + installed: false, + version: null, + status: "error", + auth: { status: "unknown" }, + checkedAt: "2026-11-16T00:00:00.000Z", + models: [], + slashCommands: [], + skills: [], + connection: { methods: [], canDisconnect: false, operation: null, runtime }, +}; + +const plan = (catalogRevision: string) => ({ + instanceId: INSTANCE, + action: "install" as const, + target: "darwin-arm64", + version: "0.147.0", + downloadBytes: 100, + sourceLabel: "Official OpenAI release", + catalogRevision, + message: "Install reviewed Codex.", +}); + +const request = { + requestId: "requested-install", + instanceId: INSTANCE, + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", +}; + +describe("ProviderRuntimeRequests", () => { + it.effect("keeps the user's request across a restart without making it due at once", () => + Effect.gen(function* () { + const before = yield* makeRequests(); + yield* before.requested(request); + const after = yield* makeRequests(); + const [kept] = yield* after.list; + assert.deepInclude(kept, { ...request, retries: 0 }); + // A restart while the user's own download runs waits as long as a failure would. + assert.isAtLeast(kept!.dueAt, (yield* Clock.currentTimeMillis) + 4 * MINUTE); + yield* after.requested({ ...request, action: "remove" }); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + + it.effect( + "retains the visible and persisted request when cancellation cannot be written, then retries", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const denied = yield* Ref.make(false); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + Ref.get(denied).pipe( + Effect.flatMap((fail) => + fail + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + ), + ), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + yield* requests.requested(request); + const before = yield* requests.list; + yield* Ref.set(denied, true); + const failure = yield* requests + .settled({ ...request, outcome: "cancelled" }) + .pipe(Effect.flip); + assert.strictEqual(failure._tag, "ProviderRuntimeRequestsPersistenceError"); + assert.deepStrictEqual(yield* requests.list, before); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, before); + yield* Ref.set(denied, false); + yield* requests.settled({ ...request, outcome: "cancelled" }); + assert.deepStrictEqual(yield* requests.list, []); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-denied-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + + it.effect.each(["malformed", "denied"] as const)( + "preserves an unreadable journal and disables only managed changes (%s)", + (mode) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + const file = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); + const contents = mode === "malformed" ? "{broken" : "[]"; + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(file, contents); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + readFileString: (target, encoding) => + mode === "denied" && target === file + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: target, + }), + ) + : fs.readFileString(target, encoding), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const unavailable = yield* requests.assertAvailable.pipe(Effect.flip); + assert.strictEqual(unavailable._tag, "ProviderRuntimeRequestsPersistenceError"); + for (const mutation of [ + requests.requested(request), + requests.update(request.requestId, () => undefined), + requests.settled({ ...request, outcome: "cancelled" }), + ]) { + assert.strictEqual((yield* mutation.pipe(Effect.flip))._tag, unavailable._tag); + } + assert.strictEqual(yield* fs.readFileString(file), contents); + yield* fs.writeFileString(file, "[]"); + const restarted = yield* makeRequests(); + yield* restarted.assertAvailable; + yield* restarted.requested(request); + assert.deepInclude((yield* (yield* makeRequests()).list)[0], request); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-unavailable-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + + it.effect.each([1_000, 2 * 60 * MINUTE])( + "persists the later of retry jitter and the source cooldown (%s ms)", + (retryAfterMs) => + Effect.gen(function* () { + const requests = yield* makeRequests(); + yield* requests.requested(request); + const now = yield* Clock.currentTimeMillis; + yield* requests.settled({ ...request, outcome: "retryable", retryAfterMs }); + const [kept] = yield* (yield* makeRequests()).list; + assert.deepInclude(kept, request); + assert.isAtLeast(kept!.dueAt, now + Math.max(retryAfterMs, 4 * MINUTE)); + assert.isAtMost(kept!.dueAt, now + Math.max(retryAfterMs, 6 * MINUTE)); + const replacement = { ...request, requestId: "replacement", catalogRevision: "reviewed:2" }; + yield* requests.requested(replacement); + yield* requests.settled({ ...request, outcome: "cancelled" }); + yield* requests.update(request.requestId, () => undefined); + assert.deepInclude((yield* requests.list)[0], replacement); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-source-cooldown-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); +}); + +describe("ProviderRuntimeRecovery", () => { + // The clock owns retry deadlines; starts record completed attempts. + const harness = (catalogRevision: string) => + Effect.gen(function* () { + const starts = yield* SubscriptionRef.make>( + [], + ); + const requests = yield* makeRequests(); + yield* requests.requested(request); + // The user's own try failed in a way a later try can fix. + yield* requests.settled({ instanceId: INSTANCE, driver: CODEX, outcome: "retryable" }); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provide( + Layer.mergeAll( + Layer.mock(ProviderRegistry)({ + getProviders: Effect.succeed([codex]), + streamChanges: Stream.never, + }), + Layer.mock(ProviderRuntimeManager)({ + plan: () => Effect.succeed(plan(catalogRevision)), + // As the manager does: a failure that may pass keeps the request. + startInBackground: (input) => + Effect.gen(function* () { + const at = yield* Clock.currentTimeMillis; + yield* requests + .settled({ + instanceId: input.instanceId, + driver: CODEX, + outcome: "retryable", + }) + .pipe(Effect.orDie); + yield* SubscriptionRef.update(starts, (all) => [ + ...all, + { attempt: input.attempt, at }, + ]); + return "failed" as ProviderRuntimeOutcome; + }), + }), + ), + ), + ); + const runFor = (minutes: number) => + Effect.forEach( + Array.from({ length: minutes }), + () => + Effect.gen(function* () { + const nextMinute = (yield* Clock.currentTimeMillis) + MINUTE; + const due = (yield* requests.list).some((pending) => pending.dueAt <= nextMinute); + const completed = (yield* SubscriptionRef.get(starts)).length; + yield* TestClock.adjust("1 minute"); + if (!due) return; + // Real durable writes are external to TestClock. Await the completed transition + // before advancing again, rather than racing fsync with the simulated clock. + if (catalogRevision === request.catalogRevision) { + yield* SubscriptionRef.changes(starts).pipe( + Stream.filter((entries) => entries.length > completed), + Stream.runHead, + ); + } else { + yield* requests.changes.pipe( + Stream.filter( + (pending) => !pending.some((entry) => entry.requestId === request.requestId), + ), + Stream.runHead, + ); + } + }), + { discard: true }, + ); + return { starts, requests, runFor }; + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-recovery-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ); + + it.effect("tries again about 5 minutes, then 30 minutes, then 2 hours after each failure", () => + Effect.gen(function* () { + const { starts, runFor } = yield* harness("reviewed:1"); + yield* runFor(4 * 60); + const tries = yield* SubscriptionRef.get(starts); + assert.deepStrictEqual( + tries.map((entry) => entry.attempt), + [1, 2, 3], + ); + const [first, second, third] = tries.map((entry) => entry.at / MINUTE); + // Jittered by a fifth either way, then picked up at the next one-minute check. + assert.isTrue(first! >= 4 && first! <= 7, `first try at ${first}`); + assert.isTrue(second! - first! >= 24 && second! - first! <= 37, `second at ${second}`); + assert.isTrue(third! - second! >= 96 && third! - second! <= 145, `third at ${third}`); + }), + ); + + it.effect("drops the request when the plan changed instead of installing another one", () => + Effect.gen(function* () { + const { starts, requests, runFor } = yield* harness("reviewed:2"); + yield* runFor(10); + assert.deepStrictEqual(yield* SubscriptionRef.get(starts), []); + assert.deepStrictEqual(yield* requests.list, []); + }), + ); +}); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts new file mode 100644 index 0000000000..12a5313c5b --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts @@ -0,0 +1,95 @@ +import type { ProviderRuntimeOperation } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; +import { ProviderRuntimeManager } from "./ProviderRuntimeManager.ts"; +import { nextTryAt, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; + +const START_DELAY = "60 seconds"; +const POLL_INTERVAL = "1 minute"; +/** Stopped for the user's own change, or another change was running. */ +const BUSY_DELAY_MS = 5 * 60_000; + +const isRunning = (operation: ProviderRuntimeOperation | null | undefined) => + operation != null && + operation.finishedAt === null && + operation.status !== "failed" && + operation.status !== "cancelled" && + operation.status !== "succeeded"; + +/** + * Tries again, one at a time, the runtime changes the user asked for that a restart or a temporary + * failure stopped. Each try first checks that the instance is still enabled and offers the same + * reviewed plan; otherwise the request is dropped. + */ +export const make = Effect.fn("ProviderRuntimeRecovery.make")(function* () { + const requests = yield* ProviderRuntimeRequests; + const manager = yield* ProviderRuntimeManager; + const registry = yield* ProviderRegistry; + + const tryDue = Effect.fn("ProviderRuntimeRecovery.tryDue")(function* () { + yield* requests.assertAvailable; + const now = yield* Clock.currentTimeMillis; + const request = (yield* requests.list) + .filter((candidate) => candidate.dueAt <= now) + .toSorted((left, right) => left.dueAt - right.dueAt)[0]; + if (!request) return; + const providers = yield* registry.getProviders; + if (providers.some((provider) => isRunning(provider.connection?.runtime?.operation))) return; + const provider = providers.find((candidate) => candidate.instanceId === request.instanceId); + const planned = + provider?.enabled === true && provider.driver === request.driver + ? yield* manager + .plan({ instanceId: request.instanceId, action: request.action }) + .pipe(Effect.option) + : Option.none(); + if (Option.isNone(planned) || planned.value.catalogRevision !== request.catalogRevision) { + return yield* requests.update(request.requestId, () => undefined); + } + const attempt = request.retries + 1; + const lease = yield* nextTryAt(attempt); + yield* requests.update(request.requestId, (current) => ({ + ...current, + retries: attempt, + dueAt: lease, + })); + if (!(yield* requests.list).some((current) => current.requestId === request.requestId)) return; + // The manager settles the request when the change ends; only a try that never ran is undone. + const outcome = yield* manager + .startInBackground({ + instanceId: request.instanceId, + action: request.action, + catalogRevision: request.catalogRevision, + attempt, + requestId: request.requestId, + }) + .pipe(Effect.orElseSucceed(() => "busy" as const)); + if (outcome === "interrupted") return yield* Effect.interrupt; + if (outcome === "yielded" || outcome === "busy") { + const later = (yield* Clock.currentTimeMillis) + BUSY_DELAY_MS; + yield* requests.update(request.requestId, (current) => + current.retries === attempt + ? { ...current, retries: request.retries, dueAt: later } + : current, + ); + } + }); + + yield* tryDue().pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logError("provider runtime recovery failed", { cause: Cause.pretty(cause) }), + ), + Effect.andThen(Effect.sleep(POLL_INTERVAL)), + Effect.forever, + Effect.delay(START_DELAY), + Effect.forkScoped, + ); +}); + +export const layer = Layer.effectDiscard(make()); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts new file mode 100644 index 0000000000..cb9e91ceb9 --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts @@ -0,0 +1,261 @@ +import { + ProviderDriverKind, + ProviderInstanceId, + type ProviderManagedRuntimeAction, +} from "@t3tools/contracts"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Random from "effect/Random"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; + +import { ServerConfig } from "../../config.ts"; + +/** After the user's own try, then after each try Scient makes; the cooldown repeats after that. */ +const RETRY_DELAYS_MS = [5 * 60_000, 30 * 60_000, 2 * 60 * 60_000] as const; +const COOLDOWN_MS = 4 * 60 * 60_000; +const ACP_REGISTRY = ProviderDriverKind.make("acpRegistry"); + +/** An install, update or repair the user asked for that has not succeeded yet. */ +const RuntimeRequest = Schema.Struct({ + requestId: Schema.String, + instanceId: ProviderInstanceId, + driver: ProviderDriverKind, + action: Schema.Literals(["install", "update", "repair"]), + /** The plan the user reviewed; a different plan is never installed in its place. */ + catalogRevision: Schema.String, + /** Tries Scient made on its own since the user's. */ + retries: Schema.Number, + /** No try starts before this (epoch ms). Set when a try starts too, so a crash cannot hurry the next. */ + dueAt: Schema.Number, +}); +export type RuntimeRequest = typeof RuntimeRequest.Type; +const RequestsJson = Schema.fromJsonString(Schema.Array(RuntimeRequest)); +const decodeRequests = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array( + Schema.Struct({ ...RuntimeRequest.fields, requestId: Schema.optionalKey(Schema.String) }), + ), + ), +); +const encodeRequests = Schema.encodeEffect(RequestsJson); + +/** How a change of a runtime ended; `retryable` failed in a way a later try may not. */ +export type RuntimeChangeOutcome = "succeeded" | "cancelled" | "failed" | "retryable"; + +/** When the try after `retries` of Scient's own may start, spread so installs don't retry together. */ +export const nextTryAt = Effect.fnUntraced(function* (retries: number) { + const delay = RETRY_DELAYS_MS[retries] ?? COOLDOWN_MS; + const now = yield* Clock.currentTimeMillis; + return now + Math.round(delay * (0.8 + (yield* Random.next) * 0.4)); +}); + +// Instances of a driver share one managed runtime; each ACP Registry agent has its own. +export const sameRuntime = ( + request: RuntimeRequest, + change: { readonly instanceId: ProviderInstanceId; readonly driver: ProviderDriverKind }, +) => + request.driver === change.driver && + (change.driver !== ACP_REGISTRY || request.instanceId === change.instanceId); + +export class ProviderRuntimeRequestsPersistenceError extends Schema.TaggedError()( + "ProviderRuntimeRequestsPersistenceError", + { path: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return "Scient could not save the requested runtime change. Try again."; + } +} + +/** + * Runtime changes the user asked for, kept across restarts until one succeeds, the user cancels or + * removes it, or chooses again. The runtime manager records them; recovery tries them again. + */ +export class ProviderRuntimeRequests extends Context.Service< + ProviderRuntimeRequests, + { + /** The user started a change: it replaces any request for the same runtime; Remove only clears. */ + readonly requested: (input: { + readonly requestId: string; + readonly instanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + readonly action: ProviderManagedRuntimeAction; + readonly catalogRevision: string; + }) => Effect.Effect; + readonly settled: (input: { + readonly instanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + readonly outcome: RuntimeChangeOutcome; + readonly requestId?: string; + readonly retryAfterMs?: number; + }) => Effect.Effect; + /** Unreadable state disables managed changes, not the rest of the server. */ + readonly assertAvailable: Effect.Effect; + readonly list: Effect.Effect>; + readonly changes: Stream.Stream>; + /** Changes this exact request if it still exists; `undefined` drops it. */ + readonly update: ( + requestId: string, + change: (request: RuntimeRequest) => RuntimeRequest | undefined, + ) => Effect.Effect; + } +>()("t3/scient/providerLifecycle/ProviderRuntimeRequests") {} + +export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { + const config = yield* ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; + const random = yield* SecureRandom.SecureRandom; + // Kept outside the runtime folders, which Remove deletes. + const filePath = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); + const loaded = yield* fs.readFileString(filePath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed("[]"), + ), + Effect.flatMap(decodeRequests), + Effect.mapError( + (cause) => new ProviderRuntimeRequestsPersistenceError({ path: filePath, cause }), + ), + Effect.result, + ); + const stored = loaded._tag === "Success" ? loaded.success : []; + const unavailable = yield* Ref.make( + loaded._tag === "Failure" ? loaded.failure : undefined, + ); + const assertAvailable = Ref.get(unavailable).pipe( + Effect.flatMap((error) => (error === undefined ? Effect.void : Effect.fail(error))), + ); + const requestsRef = yield* SubscriptionRef.make>( + yield* Effect.forEach(stored, (request) => + Effect.gen(function* () { + return { ...request, requestId: request.requestId ?? `request-${yield* random.uuidV4()}` }; + }), + ), + ); + const lock = yield* Semaphore.make(1); + + const change = ( + next: (requests: ReadonlyArray) => Effect.Effect>, + ) => + lock.withPermits(1)( + Effect.uninterruptible( + Effect.gen(function* () { + yield* assertAvailable; + const current = yield* SubscriptionRef.get(requestsRef); + const updated = yield* next(current); + if (updated === current) return; + yield* encodeRequests(updated).pipe( + Effect.flatMap((contents) => + writeFileStringAtomically({ filePath, contents, mode: 0o600, durable: true }), + ), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcess.Platform, platform), + Effect.mapError( + (cause) => new ProviderRuntimeRequestsPersistenceError({ path: filePath, cause }), + ), + ); + yield* SubscriptionRef.set(requestsRef, updated); + }), + ), + ); + + if (stored.some((request) => request.requestId === undefined)) { + yield* change((requests) => Effect.succeed([...requests])).pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: (error) => Ref.set(unavailable, error), + }), + ); + } + + const updateEach = ( + requests: ReadonlyArray, + requestId: string, + next: (request: RuntimeRequest) => RuntimeRequest | undefined, + ) => + requests.flatMap((request) => { + if (request.requestId !== requestId) return [request]; + const updated = next(request); + return updated ? [updated] : []; + }); + + const requested = Effect.fn("ProviderRuntimeRequests.requested")(function* ( + input: Parameters[0], + ) { + const dueAt = yield* nextTryAt(0); + yield* change((requests) => { + const others = requests.filter((request) => !sameRuntime(request, input)); + if (input.action === "remove") return Effect.succeed(others); + const { requestId, instanceId, driver, action, catalogRevision } = input; + return Effect.succeed([ + ...others, + { requestId, instanceId, driver, action, catalogRevision, retries: 0, dueAt }, + ]); + }); + }); + + const settled = Effect.fn("ProviderRuntimeRequests.settled")(function* ( + input: Parameters[0], + ) { + yield* change((requests) => + Effect.gen(function* () { + const request = requests.find((candidate) => + input.requestId === undefined + ? candidate.instanceId === input.instanceId && sameRuntime(candidate, input) + : candidate.requestId === input.requestId, + ); + if (!request) return requests; + switch (input.outcome) { + case "succeeded": + return requests.filter((request) => !sameRuntime(request, input)); + case "cancelled": + case "failed": + return updateEach(requests, request.requestId, () => undefined); + case "retryable": { + const dueAt = Math.max( + yield* nextTryAt(request.retries), + (yield* Clock.currentTimeMillis) + (input.retryAfterMs ?? 0), + ); + return updateEach(requests, request.requestId, (current) => ({ ...current, dueAt })); + } + } + }), + ); + }); + + const update = Effect.fn("ProviderRuntimeRequests.update")(function* ( + requestId: string, + next: (request: RuntimeRequest) => RuntimeRequest | undefined, + ) { + yield* change((requests) => + Effect.succeed( + requests.some((request) => request.requestId === requestId) + ? updateEach(requests, requestId, next) + : requests, + ), + ); + }); + + return ProviderRuntimeRequests.of({ + assertAvailable, + requested, + settled, + list: SubscriptionRef.get(requestsRef), + changes: SubscriptionRef.changes(requestsRef), + update, + }); +}); + +export const layer = Layer.effect(ProviderRuntimeRequests, make()); diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts index 7a9028322a..6c53e113a5 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts @@ -10,9 +10,7 @@ import { } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as PlatformError from "effect/PlatformError"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -23,13 +21,13 @@ import { ProcessRunner } from "../../processRunner.ts"; import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; import { ProviderRuntimeManager, type ProviderRuntimeOutcome } from "./ProviderRuntimeManager.ts"; +import { firstPickState, make, parseMeteredOutput } from "./ScientAgentBackgroundRuntime.ts"; + import { backgroundRuntimeAction, - firstPickState, - make, - parseMeteredOutput, recordAfterUserChange, -} from "./ScientAgentBackgroundRuntime.ts"; + ScientAgentRuntimePolicy, +} from "./ScientAgentRuntimePolicy.ts"; const INSTANCE = ProviderInstanceId.make("scient"); @@ -108,6 +106,19 @@ describe("backgroundRuntimeAction", () => { backgroundRuntimeAction(scientAgent(), { failures: { version: "0.1.5", count: 5 } }).kind, "start", ); + assert.deepStrictEqual( + backgroundRuntimeAction( + scientAgent({ + pendingRecovery: { + requestId: "reviewed", + action: "install", + nextRetryAt: "2026-11-16T02:00:00.000Z", + }, + }), + {}, + ), + { kind: "wait", reason: "requested-recovery" }, + ); }); }); @@ -130,6 +141,12 @@ describe("recordAfterUserChange", () => { skippedVersion: "0.1.6", }), ).toEqual({}); + expect(recordAfterUserChange({ action: "repair", status: "cancelled" }, "0.1.6")?.({})).toEqual( + { + userStopped: true, + skippedVersion: "0.1.6", + }, + ); expect(recordAfterUserChange({ action: "repair", status: "failed" }, "0.1.6")).toBeUndefined(); }); }); @@ -226,23 +243,19 @@ describe("ScientAgentBackgroundRuntime", () => { Layer.mock(AnalyticsService)({ record: (event) => Ref.update(events, (all) => [...all, event]), }), - FileSystem.layerNoop({ - makeDirectory: () => Effect.void, - readFileString: (path) => - SubscriptionRef.get(record).pipe( - Effect.filterOrFail( - (text): text is string => text !== null, - () => - PlatformError.systemError({ - _tag: "NotFound", - module: "FileSystem", - method: "readFileString", - pathOrDescriptor: path, - }), - ), + Layer.succeed( + ScientAgentRuntimePolicy, + ScientAgentRuntimePolicy.of({ + read: SubscriptionRef.get(record).pipe( + Effect.map((text) => JSON.parse(text ?? "{}")), ), - writeFileString: (_path, text) => SubscriptionRef.set(record, text), - }), + update: (change) => + SubscriptionRef.modify(record, (text) => { + const before = JSON.parse(text ?? "{}"); + return [before, JSON.stringify(change(before))]; + }), + }), + ), ), ), Effect.provideServiceEffect( diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts index 04367673a0..eb29c8b616 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts @@ -4,20 +4,14 @@ import * as NodeFSP from "node:fs/promises"; import { ProviderDriverKind, type ProviderInstanceId, - type ProviderManagedRuntimeAction, - type ProviderRuntimeSummary, type ServerProvider, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; -import * as Schema from "effect/Schema"; -import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { ServerConfig } from "../../config.ts"; @@ -25,6 +19,12 @@ import * as ProcessRunner from "../../processRunner.ts"; import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; import { ProviderRuntimeManager } from "./ProviderRuntimeManager.ts"; +import * as ScientAgentRuntimePolicy from "./ScientAgentRuntimePolicy.ts"; +import { + backgroundRuntimeAction, + recordAfterUserChange, + type BackgroundRecord, +} from "./ScientAgentRuntimePolicy.ts"; const SCIENT_AGENT = ProviderDriverKind.make("scient"); const START_DELAY = "60 seconds"; @@ -33,79 +33,9 @@ const RECHECK_INTERVAL = "1 hour"; const FAILURE_BACKOFF = ["5 minutes", "30 minutes", "2 hours"] as const; /** Busy with other work, or stopped for the user's own change. */ const BUSY_RETRY = "5 minutes"; -const MAX_FAILED_ATTEMPTS_PER_RELEASE = 5; /** The download, the binary, the add-on it unpacks on first run, and staging. */ const MIN_FREE_BYTES = 2 * 1024 ** 3; -/** Kept outside the runtime folder, which Remove deletes. */ -const BackgroundRecord = Schema.Struct({ - /** The user removed Scient Agent or cancelled its install: install only on request. */ - userStopped: Schema.optionalKey(Schema.Boolean), - /** An update the user cancelled; a newer release is offered again. */ - skippedVersion: Schema.optionalKey(Schema.String), - failures: Schema.optionalKey(Schema.Struct({ version: Schema.String, count: Schema.Number })), - firstPickRecorded: Schema.optionalKey(Schema.Boolean), -}); -type BackgroundRecord = typeof BackgroundRecord.Type; -const BackgroundRecordJson = Schema.fromJsonString(BackgroundRecord); -const decodeRecord = Schema.decodeEffect(BackgroundRecordJson); -const encodeRecord = Schema.encodeEffect(BackgroundRecordJson); - -export type BackgroundCheck = - | { readonly kind: "start"; readonly action: "install" | "update"; readonly version: string } - | { readonly kind: "wait"; readonly reason: string }; - -/** What, if anything, Scient may get ready on its own for this instance. */ -export function backgroundRuntimeAction( - provider: ServerProvider, - record: BackgroundRecord, -): BackgroundCheck { - const runtime = provider.connection?.runtime; - if (!provider.enabled || !runtime) return { kind: "wait", reason: "unavailable" }; - const action: ProviderManagedRuntimeAction | undefined = - runtime.source === "missing" && runtime.actions.includes("install") - ? "install" - : runtime.source === "scient_managed" && runtime.actions.includes("update") - ? "update" - : undefined; - const version = - action === "update" - ? runtime.availableManagedVersion - : (runtime.availableManagedVersion ?? runtime.qualifiedVersion); - if (action === undefined || !version) return { kind: "wait", reason: "nothing-to-do" }; - if (action === "install" && record.userStopped) return { kind: "wait", reason: "user-stopped" }; - if (action === "update" && record.skippedVersion === version) - return { kind: "wait", reason: "user-stopped" }; - if ( - record.failures?.version === version && - record.failures.count >= MAX_FAILED_ATTEMPTS_PER_RELEASE - ) - return { kind: "wait", reason: "gave-up" }; - return { kind: "start", action, version }; -} - -/** - * How a finished change the user made moves the record: Remove, or Cancel of an install, stops - * background installs; Cancel of an update skips that release; their own install or update - * lets Scient take over again. - */ -export function recordAfterUserChange( - operation: Pick, "action" | "status">, - offeredVersion: string | null, -): ((record: BackgroundRecord) => BackgroundRecord) | undefined { - const { action, status } = operation; - if ( - (action === "remove" && status === "succeeded") || - (action === "install" && status === "cancelled") - ) - return (record) => ({ ...record, userStopped: true }); - if (action === "update" && status === "cancelled" && offeredVersion !== null) - return (record) => ({ ...record, skippedVersion: offeredVersion }); - if (action !== "remove" && status === "succeeded") - return ({ userStopped: _stopped, skippedVersion: _skipped, ...rest }) => rest; - return undefined; -} - /** True when the OS says the connection is metered. Unknown counts as not metered. */ export function parseMeteredOutput(platform: NodeJS.Platform, output: string): boolean { if (platform === "win32") return /^(Fixed|Variable)$/mu.test(output.trim()); @@ -137,33 +67,11 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () const processes = yield* ProcessRunner.ProcessRunner; const analytics = yield* AnalyticsService; const platform = yield* HostProcess.Platform; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const recordPath = path.join( - config.baseDir, - "provider-runtimes", - ".background", - "scient-agent.json", - ); - - const readRecord = fs.readFileString(recordPath).pipe( - Effect.flatMap(decodeRecord), - Effect.orElseSucceed((): BackgroundRecord => ({})), - ); - const recordLock = yield* Semaphore.make(1); - /** Applies a change under the lock and returns the record from before it. */ - const updateRecord = (change: (record: BackgroundRecord) => BackgroundRecord) => - recordLock.withPermits(1)( - Effect.gen(function* () { - const record = yield* readRecord; - const text = yield* encodeRecord(change(record)); - yield* fs.makeDirectory(path.dirname(recordPath), { recursive: true }); - yield* fs.writeFileString(recordPath, text); - return record; - }), - ); + const policy = yield* ScientAgentRuntimePolicy.ScientAgentRuntimePolicy; + const readRecord = policy.read; + const updateRecord = policy.update; const changeRecord = (change: (record: BackgroundRecord) => BackgroundRecord) => - updateRecord(change).pipe(Effect.ignore); + updateRecord(change).pipe(Effect.asVoid); const freeBytes = Effect.promise(() => NodeFSP.statfs(config.baseDir).then( @@ -219,54 +127,56 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () ); type Pass = "ready" | "failed" | "busy" | "waiting"; - const tryOnce = Effect.fn("ScientAgentBackgroundRuntime.tryOnce")( - function* (): Effect.fn.Return { - const provider = yield* scientAgent; - if (!provider) return "waiting"; - const record = yield* readRecord; - const check = backgroundRuntimeAction(provider, record); - if (check.kind === "wait") return "waiting"; - if (yield* anyRuntimeBusy) return "busy"; - if ((yield* freeBytes) < MIN_FREE_BYTES) return "waiting"; - if (yield* metered) return "waiting"; - const planned = yield* manager - .plan({ instanceId: provider.instanceId, action: check.action }) - .pipe(Effect.option); - if (Option.isNone(planned)) return "waiting"; - const failures = record.failures?.version === check.version ? record.failures.count : 0; - const outcome = yield* manager - .startInBackground({ - instanceId: provider.instanceId, - action: check.action, - catalogRevision: planned.value.catalogRevision, - attempt: failures + 1, - }) - .pipe(Effect.orElseSucceed(() => "busy" as const)); - switch (outcome) { - case "succeeded": - yield* changeRecord(({ failures: _failures, ...rest }) => rest); - return "ready"; - case "failed": - yield* changeRecord((current) => ({ - ...current, - failures: { version: check.version, count: failures + 1 }, - })); - return "failed"; - case "cancelled": - yield* changeRecord((current) => - check.action === "install" - ? { ...current, userStopped: true } - : { ...current, skippedVersion: check.version }, - ); - return "waiting"; - case "yielded": - case "busy": - return "busy"; - case "interrupted": - return yield* Effect.interrupt; + const tryOnce = Effect.fn("ScientAgentBackgroundRuntime.tryOnce")(function* (): Effect.fn.Return< + Pass, + ScientAgentRuntimePolicy.ScientAgentRuntimePolicyError + > { + const provider = yield* scientAgent; + if (!provider) return "waiting"; + const record = yield* readRecord; + const check = backgroundRuntimeAction(provider, record); + if (check.kind === "wait") return "waiting"; + if (yield* anyRuntimeBusy) return "busy"; + if ((yield* freeBytes) < MIN_FREE_BYTES) return "waiting"; + if (yield* metered) return "waiting"; + const planned = yield* manager + .plan({ instanceId: provider.instanceId, action: check.action }) + .pipe(Effect.option); + if (Option.isNone(planned)) return "waiting"; + const failures = record.failures?.version === check.version ? record.failures.count : 0; + const outcome = yield* manager + .startInBackground({ + instanceId: provider.instanceId, + action: check.action, + catalogRevision: planned.value.catalogRevision, + attempt: failures + 1, + }) + .pipe(Effect.orElseSucceed(() => "busy" as const)); + switch (outcome) { + case "succeeded": + yield* changeRecord(({ failures: _failures, ...rest }) => rest); + return "ready"; + case "failed": + yield* changeRecord((current) => ({ + ...current, + failures: { version: check.version, count: failures + 1 }, + })); + return "failed"; + case "cancelled": { + const change = recordAfterUserChange( + { action: check.action, status: "cancelled" }, + check.version, + ); + if (change) yield* changeRecord(change); + return "waiting"; } - }, - ); + case "yielded": + case "busy": + return "busy"; + case "interrupted": + return yield* Effect.interrupt; + } + }); // A newer release wakes the loop at once, so no update notice shows first. const offeredVersion = (provider: ServerProvider | undefined) => @@ -287,7 +197,9 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () let failedInARow = 0; while (true) { yield* Ref.set(lastOfferRef, offeredVersion(yield* scientAgent)); - const pass = yield* tryOnce(); + const pass = yield* tryOnce().pipe( + Effect.catchTags({ ScientAgentRuntimePolicyError: () => Effect.succeed("busy" as const) }), + ); if (pass === "failed") { const delay = FAILURE_BACKOFF[failedInARow]; failedInARow += 1; @@ -313,11 +225,16 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () ?.runtime; const operation = runtime?.operation; if (!operation || operation.background || operation.finishedAt === null) return; - if ((yield* Ref.getAndSet(handled, operation.operationId)) === operation.operationId) - return; + if ((yield* Ref.get(handled)) === operation.operationId) return; const change = recordAfterUserChange(operation, runtime.availableManagedVersion ?? null); if (change) yield* changeRecord(change); - }), + yield* Ref.set(handled, operation.operationId); + }).pipe( + Effect.catchTags({ + ScientAgentRuntimePolicyError: () => + Effect.logWarning("Agent background policy persistence failed"), + }), + ), ), ); @@ -368,4 +285,5 @@ export function firstPickState(provider: ServerProvider): { export const layer = Layer.effect(ScientAgentBackgroundRuntime, make()).pipe( Layer.provide(ProcessRunner.layer), + Layer.provide(ScientAgentRuntimePolicy.layer), ); diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts b/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts new file mode 100644 index 0000000000..ff02217e67 --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts @@ -0,0 +1,162 @@ +import type { + ProviderManagedRuntimeAction, + ProviderRuntimeSummary, + ServerProvider, +} from "@t3tools/contracts"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import { ServerConfig } from "../../config.ts"; + +/** Kept outside the runtime folder, which Remove deletes. */ +const BackgroundRecord = Schema.Struct({ + /** The user removed Scient Agent or cancelled install/repair: install only on request. */ + userStopped: Schema.optionalKey(Schema.Boolean), + /** An update the user cancelled; a newer release is offered again. */ + skippedVersion: Schema.optionalKey(Schema.String), + failures: Schema.optionalKey(Schema.Struct({ version: Schema.String, count: Schema.Number })), + firstPickRecorded: Schema.optionalKey(Schema.Boolean), +}); +export type BackgroundRecord = typeof BackgroundRecord.Type; +const BackgroundRecordJson = Schema.fromJsonString(BackgroundRecord); +const decodeRecord = Schema.decodeEffect(BackgroundRecordJson); +const encodeRecord = Schema.encodeEffect(BackgroundRecordJson); +const MAX_FAILED_ATTEMPTS_PER_RELEASE = 5; + +type BackgroundCheck = + | { readonly kind: "start"; readonly action: "install" | "update"; readonly version: string } + | { readonly kind: "wait"; readonly reason: string }; + +/** What, if anything, Scient may get ready on its own for this instance. */ +export function backgroundRuntimeAction( + provider: ServerProvider, + record: BackgroundRecord, +): BackgroundCheck { + const runtime = provider.connection?.runtime; + if (runtime?.pendingRecovery) return { kind: "wait", reason: "requested-recovery" }; + if (!provider.enabled || !runtime) return { kind: "wait", reason: "unavailable" }; + const action: ProviderManagedRuntimeAction | undefined = + runtime.source === "missing" && runtime.actions.includes("install") + ? "install" + : runtime.source === "scient_managed" && runtime.actions.includes("update") + ? "update" + : undefined; + const version = + action === "update" + ? runtime.availableManagedVersion + : (runtime.availableManagedVersion ?? runtime.qualifiedVersion); + if (action === undefined || !version) return { kind: "wait", reason: "nothing-to-do" }; + if (action === "install" && record.userStopped) return { kind: "wait", reason: "user-stopped" }; + if (action === "update" && record.skippedVersion === version) + return { kind: "wait", reason: "user-stopped" }; + if ( + record.failures?.version === version && + record.failures.count >= MAX_FAILED_ATTEMPTS_PER_RELEASE + ) + return { kind: "wait", reason: "gave-up" }; + return { kind: "start", action, version }; +} +/** + * Remove or cancelled install stops automatic installs; cancelled update skips that release. + * Cancelled repair does both. A successful requested change lets automatic setup resume. + */ +export function recordAfterUserChange( + operation: Pick, "action" | "status">, + offeredVersion: string | null, +): ((record: BackgroundRecord) => BackgroundRecord) | undefined { + const { action, status } = operation; + if ( + (action === "remove" && status === "succeeded") || + (action === "install" && status === "cancelled") + ) + return (record) => ({ ...record, userStopped: true }); + if (action === "repair" && status === "cancelled") + return (record) => ({ + ...record, + userStopped: true, + ...(offeredVersion === null ? {} : { skippedVersion: offeredVersion }), + }); + if (action === "update" && status === "cancelled" && offeredVersion !== null) + return (record) => ({ ...record, skippedVersion: offeredVersion }); + if (action !== "remove" && status === "succeeded") + return ({ userStopped: _stopped, skippedVersion: _skipped, ...rest }) => rest; + return undefined; +} + +export class ScientAgentRuntimePolicyError extends Schema.TaggedError()( + "ScientAgentRuntimePolicyError", + { path: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return "Scient could not save the Agent background setup policy. Try again."; + } +} + +export class ScientAgentRuntimePolicy extends Context.Service< + ScientAgentRuntimePolicy, + { + readonly read: Effect.Effect; + /** Returns the previous record only after the change is persisted. */ + readonly update: ( + change: (record: BackgroundRecord) => BackgroundRecord, + ) => Effect.Effect; + } +>()("t3/scient/providerLifecycle/ScientAgentRuntimePolicy") {} + +const make = Effect.gen(function* () { + const config = yield* ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; + const recordPath = path.join( + config.baseDir, + "provider-runtimes", + ".background", + "scient-agent.json", + ); + const read = fs.readFileString(recordPath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed("{}"), + ), + Effect.flatMap(decodeRecord), + Effect.mapError((cause) => new ScientAgentRuntimePolicyError({ path: recordPath, cause })), + ); + const lock = yield* Semaphore.make(1); + const update = Effect.fn("ScientAgentRuntimePolicy.update")(function* ( + change: (record: BackgroundRecord) => BackgroundRecord, + ) { + return yield* lock.withPermits(1)( + Effect.uninterruptible( + Effect.gen(function* () { + const record = yield* read; + const contents = yield* encodeRecord(change(record)).pipe(Effect.orDie); + yield* writeFileStringAtomically({ + filePath: recordPath, + contents, + mode: 0o600, + durable: true, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcess.Platform, platform), + Effect.mapError( + (cause) => new ScientAgentRuntimePolicyError({ path: recordPath, cause }), + ), + ); + return record; + }), + ), + ); + }); + return ScientAgentRuntimePolicy.of({ read, update }); +}); + +export const layer = Layer.effect(ScientAgentRuntimePolicy, make); diff --git a/apps/server/src/server.scient.test.ts b/apps/server/src/server.scient.test.ts index 639203c00f..52fb85ac00 100644 --- a/apps/server/src/server.scient.test.ts +++ b/apps/server/src/server.scient.test.ts @@ -4,6 +4,7 @@ import type { AnalyticsRuntime } from "@scientfactory/analytics"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { assert, it } from "@effect/vitest"; import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as HttpClient from "effect/http/HttpClient"; @@ -14,9 +15,12 @@ import * as ServerConfig from "./config.ts"; import * as ManagedPolicy from "./policy/ManagedPolicy.ts"; import { layerAnalytics } from "./server.ts"; import { AnalyticsService } from "./telemetry/AnalyticsService.ts"; -import { ScientAnalyticsAvailability } from "./telemetry/ScientAnalytics.ts"; +import { + ScientAnalyticsAvailability, + ScientRecordedAnalyticsIngest, +} from "./telemetry/ScientAnalytics.ts"; -const scient = vi.hoisted(() => ({ recorded: [] as string[] })); +const scient = vi.hoisted(() => ({ recorded: [] as string[], ingested: [] as string[] })); vi.mock("@scientfactory/analytics", async (importOriginal) => ({ ...(await importOriginal()), createAnalyticsRuntime: (): AnalyticsRuntime => ({ @@ -25,6 +29,10 @@ vi.mock("@scientfactory/analytics", async (importOriginal) => ({ scient.recorded.push(name); return true; }, + ingest: async (events) => { + scient.ingested.push(...events.map((event) => event.name)); + return true; + }, flush: async () => 0, pendingCount: async () => 0, close: async () => undefined, @@ -36,6 +44,7 @@ it.effect( () => Effect.gen(function* () { scient.recorded.length = 0; + scient.ingested.length = 0; const posthogRequests: string[] = []; const posthog = HttpClient.make((request) => Effect.sync(() => { @@ -48,6 +57,17 @@ it.effect( yield* analytics.record("provider.turn.sent", { provider: "codex" }); yield* analytics.recordOwners([{ source: "openai", ownerId: "a".repeat(64) }], "server"); yield* analytics.flush; + // Facts another process journaled reach the same outbox, not the never-sent default. + yield* (yield* ScientRecordedAnalyticsIngest).ingest([ + { + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24", + name: "app.update.summary", + occurredAt: DateTime.formatIso(yield* DateTime.now), + appVersion: "0.6.29", + buildChannel: "stable", + properties: { reason: "launch", status: "idle" }, + }, + ]); return (yield* ScientAnalyticsAvailability).available; }).pipe( Effect.provide( @@ -83,6 +103,7 @@ it.effect( "install.owner_observed", "app.session.ended", ]); + assert.deepEqual(scient.ingested, ["app.update.summary"]); assert.isTrue(available); }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); diff --git a/apps/server/src/telemetry/AnalyticsServiceControls.test.ts b/apps/server/src/telemetry/AnalyticsServiceControls.test.ts index 5ff6922e4a..ca15556431 100644 --- a/apps/server/src/telemetry/AnalyticsServiceControls.test.ts +++ b/apps/server/src/telemetry/AnalyticsServiceControls.test.ts @@ -32,6 +32,7 @@ function fakeRuntime(enabled = true) { } return alive; }, + ingest: async () => true, flush: async () => 0, pendingCount: async () => 0, close: async () => { diff --git a/apps/server/src/telemetry/DesktopUpdateFacts.ts b/apps/server/src/telemetry/DesktopUpdateFacts.ts new file mode 100644 index 0000000000..b421c1c32d --- /dev/null +++ b/apps/server/src/telemetry/DesktopUpdateFacts.ts @@ -0,0 +1,20 @@ +/** + * Update facts the desktop app journaled (outcomes of check, download, install and apply, and + * update summaries). They keep the desktop's event ids and times, so a fact offered again after a + * lost acknowledgement stays one event in the outbox and at the gateway. + * + * @module DesktopUpdateFacts + */ +import type { DesktopUpdateFact } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import type { RecordedAnalyticsIngest } from "./ScientAnalytics.ts"; + +/** The ids the desktop may forget: held by the outbox, or never to be sent from here. */ +export const ingestDesktopUpdateFacts = Effect.fn("DesktopUpdateFacts.ingest")(function* ( + recorded: RecordedAnalyticsIngest, + facts: ReadonlyArray, +) { + const settled = yield* recorded.ingest(facts); + return settled ? facts.map((fact) => fact.id) : []; +}); diff --git a/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts b/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts index e2e7143370..e6d4c77cf3 100644 --- a/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts +++ b/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts @@ -147,31 +147,51 @@ describe("Provider lifecycle analytics", () => { ]); }); - it("says a background change started on its own and how its download went", () => { + it("says a background change started on its own and how far its download got", () => { const mapper = createProviderLifecycleAnalyticsMapper(); mapper.observe([provider()]); const background = { ...operation("downloading"), background: true, attempt: 2 }; - expect(mapper.observe([provider(background)])).toMatchObject([ - { name: "provider.lifecycle.started", properties: { trigger: "background", attempt: 2 } }, - ]); + const [started] = mapper.observe([provider(background)]); + expect(started).toMatchObject({ + name: "provider.lifecycle.started", + properties: { trigger: "background", attempt: 2 }, + }); + expect(started!.properties).not.toHaveProperty("failureClass"); expect( mapper.observe([ provider({ ...background, status: "failed", finishedAt: "2026-08-31T00:00:12.000Z", - download: { resumedBytes: 41_000_000, source: "fallback", compressed: true }, + failureClass: "timeout", + download: { + resumedBytes: 41_000_000, + source: "fallback", + compressed: true, + sourceKind: "github", + tries: 5, + progressPercent: 37, + bytesPerSecond: 410_000, + networkKind: "stalled", + }, }), ]), ).toMatchObject([ { name: "provider.lifecycle.failed", properties: { + // Older clients decode only `timeout`; the stall is told apart here. + failureClass: "stalled", trigger: "background", attempt: 2, resumedBytes: 41_000_000, downloadSource: "fallback", - compressed: true, + downloadVariant: "compressed", + sourceKind: "github", + downloadAttempts: 5, + progressPercent: 37, + bytesPerSecond: 410_000, + networkKind: "stalled", }, }, ]); diff --git a/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts b/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts index 5eaf28886d..b6fd13c7ef 100644 --- a/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts +++ b/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts @@ -1,4 +1,4 @@ -import type { ServerProvider } from "@t3tools/contracts"; +import type { ProviderRuntimeOperation, ServerProvider } from "@t3tools/contracts"; const MAX_INSTANCES = 1_000; @@ -30,6 +30,37 @@ interface Event { readonly properties: Readonly>; } +/** How the download went: closed values and numbers the contract buckets, never hosts or paths. */ +function downloadDetails( + download: NonNullable, +): Readonly> { + return { + resumedBytes: download.resumedBytes, + downloadSource: download.source, + downloadVariant: download.compressed ? "compressed" : "raw", + ...(download.sourceKind === undefined ? {} : { sourceKind: download.sourceKind }), + ...(download.tries === undefined ? {} : { downloadAttempts: download.tries }), + ...(download.progressPercent === undefined + ? {} + : { progressPercent: download.progressPercent }), + ...(download.bytesPerSecond === undefined ? {} : { bytesPerSecond: download.bytesPerSecond }), + ...(download.networkKind === undefined ? {} : { networkKind: download.networkKind }), + ...(download.compressedFailure === undefined + ? {} + : { compressedFailure: download.compressedFailure }), + }; +} + +/** Older clients cannot decode a new operation failure class, so a stall travels as `timeout`. */ +function runtimeFailureClass( + failureClass: NonNullable, + download: ProviderRuntimeOperation["download"], +): string { + return failureClass === "timeout" && download?.networkKind === "stalled" + ? "stalled" + : failureClass; +} + function snapshot(provider: ServerProvider): Snapshot { const runtime = provider.connection?.runtime; const operation = runtime?.operation; @@ -53,17 +84,13 @@ function snapshot(provider: ServerProvider): Snapshot { : operation.status === "failed" || operation.status === "cancelled" ? operation.status : null, - ...(operation.failureClass === undefined ? {} : { failureClass: operation.failureClass }), + ...(operation.failureClass === undefined + ? {} + : { failureClass: runtimeFailureClass(operation.failureClass, operation.download) }), details: { trigger: operation.background ? "background" : "user", ...(operation.attempt === undefined ? {} : { attempt: operation.attempt }), - ...(operation.download === undefined - ? {} - : { - resumedBytes: operation.download.resumedBytes, - downloadSource: operation.download.source, - compressed: operation.download.compressed, - }), + ...(operation.download === undefined ? {} : downloadDetails(operation.download)), }, } : null, @@ -126,7 +153,7 @@ function lifecycleEvent( durationMs, // The UI message is deliberately not inspected: it can contain paths, // credentials, provider output, or URLs. The owner sets the typed class. - failureClass: next.outcome === "failed" ? (next.failureClass ?? "unknown") : "unknown", + ...(next.outcome === "failed" ? { failureClass: next.failureClass ?? "unknown" } : {}), ...next.details, }, }; diff --git a/apps/server/src/telemetry/ScientAnalytics.ts b/apps/server/src/telemetry/ScientAnalytics.ts index ee6b6242d1..8965fd8fbb 100644 --- a/apps/server/src/telemetry/ScientAnalytics.ts +++ b/apps/server/src/telemetry/ScientAnalytics.ts @@ -5,7 +5,11 @@ * * @module ScientAnalytics */ -import { createAnalyticsRuntime, type AnalyticsRuntime } from "@scientfactory/analytics"; +import { + createAnalyticsRuntime, + type AnalyticsRuntime, + type RecordedAnalyticsEvent, +} from "@scientfactory/analytics"; import * as NodePath from "node:path"; import * as NodeProcess from "node:process"; import * as NodeSea from "node:sea"; @@ -61,9 +65,26 @@ export class ScientAnalyticsAvailability extends Context.Reference) => Effect.Effect; +} + +const neverSent: RecordedAnalyticsIngest = { ingest: () => Effect.succeed(true) }; + +/** Ingest for events recorded in another process; unprovided means they are never sent. */ +export class ScientRecordedAnalyticsIngest extends Context.Reference( + "scient/telemetry/ScientRecordedAnalyticsIngest", + { defaultValue: () => neverSent }, +) {} + export interface ScientAnalytics { readonly analytics: AnalyticsService["Service"]; readonly available: boolean; + readonly recorded: RecordedAnalyticsIngest; } function parseBuildChannel( @@ -104,7 +125,11 @@ export const disabledAnalyticsService: AnalyticsService["Service"] = { flush: Effect.void, }; -const disabled: ScientAnalytics = { analytics: disabledAnalyticsService, available: false }; +const disabled: ScientAnalytics = { + analytics: disabledAnalyticsService, + available: false, + recorded: neverSent, +}; export const make = Effect.gen(function* () { const analyticsConfig = yield* AnalyticsEnvConfig; @@ -196,17 +221,27 @@ export const make = Effect.gen(function* () { Effect.asVoid, ); + const recorded: RecordedAnalyticsIngest = { + ingest: (events) => + Effect.tryPromise({ + try: () => runtime.ingest(events), + catch: () => "analytics-ingest-failed" as const, + }).pipe(Effect.orElseSucceed(() => false)), + }; + return { analytics: AnalyticsService.of({ record, recordOwners, flush }), available: true, + recorded, } satisfies ScientAnalytics; }); /** Provides AnalyticsService and its availability in place of T3's PostHog layer. */ export const layer = Layer.effectContext( - Effect.map(make, ({ analytics, available }) => + Effect.map(make, ({ analytics, available, recorded }) => Context.make(AnalyticsService, analytics).pipe( Context.add(ScientAnalyticsAvailability, { available }), + Context.add(ScientRecordedAnalyticsIngest, recorded), ), ), ); diff --git a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx index cde7c6ef52..afac648c95 100644 --- a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx +++ b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx @@ -1836,6 +1836,7 @@ describe("ProviderRuntimeSection", () => { const cancelledProvider: ServerProvider = { ...provider, + driver: ProviderDriverKind.make("claudeAgent"), installed: true, version: "2.1.170", status: "ready", @@ -1877,6 +1878,51 @@ describe("ProviderRuntimeSection", () => { expect(markup).toContain('aria-label="Use Scient-managed Claude"'); expect(markup).not.toContain("Provider runtime setup cancelled"); expect(markup).not.toContain("previous working runtime"); + + const restartedProvider: ServerProvider = { + ...cancelledProvider, + connection: { + ...cancelledProvider.connection!, + runtime: { + ...cancelledProvider.connection!.runtime!, + operation: null, + pendingRecovery: { + requestId: "requested-before-restart", + action: "install", + nextRetryAt: "2026-11-16T12:05:00.000Z", + }, + }, + }, + }; + const clearedProvider: ServerProvider = { + ...restartedProvider, + connection: { + ...restartedProvider.connection!, + runtime: { ...restartedProvider.connection!.runtime!, pendingRecovery: null }, + }, + }; + commands.cancel.mockResolvedValue({ _tag: "Success", value: { providers: [clearedProvider] } }); + const host = await renderSection(_fixture, { + compact: true, + environmentId, + provider: restartedProvider, + displayName: "Claude", + }); + expect(host.textContent).toContain("Scient will retry this requested install after"); + expect(host.querySelector('[role="progressbar"]')).toBeNull(); + expect(host.querySelector('[aria-label="Cancel Claude install retry"]')).not.toBeNull(); + const cancelRetry = findActionButton(host, "Cancel retry"); + expect(cancelRetry).toBeDefined(); + await act(() => cancelRetry!.click()); + await vi.waitFor(() => { + expect(commands.cancel).toHaveBeenCalledWith({ + environmentId, + input: { instanceId, requestId: "requested-before-restart" }, + }); + expect(host.textContent).not.toContain("Cancel retry"); + }); + expect(host.textContent).toContain("System installation"); + expect(commands.start).not.toHaveBeenCalled(); }); it("shows the current missing-runtime state after removal instead of a stale success row", async () => { diff --git a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx index e166329380..c17158ab3b 100644 --- a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx +++ b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx @@ -75,6 +75,11 @@ type LocalRuntimeFailure = readonly kind: "operation"; readonly operationId: ProviderRuntimeOperation["operationId"]; readonly message: string; + } + | { + readonly kind: "recovery"; + readonly requestId: string; + readonly message: string; }; function runtimeFromResult( @@ -101,6 +106,9 @@ function localRuntimeFailureMessage( ? failure.message : null; } + if (failure.kind === "recovery") { + return runtime.pendingRecovery?.requestId === failure.requestId ? failure.message : null; + } const operation = runtime.operation; return operation?.operationId === failure.operationId && isActiveProviderRuntimeOperation(operation) @@ -114,6 +122,12 @@ export function resolveProviderRuntimeForPresentation( ): ProviderRuntimeSummary | null | undefined { if (!serverRuntime) return localRuntime; if (!localRuntime) return serverRuntime; + if ( + serverRuntime.pendingRecovery && + localRuntime.pendingRecovery === null && + !isActiveProviderRuntimeOperation(localRuntime.operation) + ) + return localRuntime; if (serverRuntime.operation?.operationId === localRuntime.operation?.operationId) { return serverRuntime; } @@ -258,6 +272,7 @@ export function ProviderRuntimeSection(props: { const operation = runtime?.operation ?? null; const runtimeActions = runtime?.actions; const activeOperation = isActiveProviderRuntimeOperation(operation) ? operation : null; + const pendingRecovery = runtime?.pendingRecovery; const plan = !activeOperation && preparedPlan?.action === "remove" && runtimeActions?.includes("remove") ? preparedPlan @@ -444,27 +459,30 @@ export function ProviderRuntimeSection(props: { }; const cancel = async () => { - if (!activeOperation) return; + if (!activeOperation && !pendingRecovery) return; setLocalFailure(null); setPendingAction("cancel"); const result = await cancelRuntime({ environmentId, input: { instanceId: provider.instanceId, - operationId: activeOperation.operationId, + ...(activeOperation + ? { operationId: activeOperation.operationId } + : { requestId: pendingRecovery!.requestId }), }, }); setPendingAction(null); if (result._tag === "Failure") { if (!isAtomCommandInterrupted(result)) { - setLocalFailure({ - kind: "operation", - operationId: activeOperation.operationId, - message: providerLifecycleFailureMessage( - squashAtomCommandFailure(result), - `Scient could not cancel the ${runtimeDisplayName} runtime operation.`, - ), - }); + const message = providerLifecycleFailureMessage( + squashAtomCommandFailure(result), + `Scient could not cancel the ${runtimeDisplayName} runtime operation.`, + ); + setLocalFailure( + activeOperation + ? { kind: "operation", operationId: activeOperation.operationId, message } + : { kind: "recovery", requestId: pendingRecovery!.requestId, message }, + ); } return; } @@ -650,6 +668,28 @@ export function ProviderRuntimeSection(props: { : "space-y-3 rounded-lg border p-3" } > + {pendingRecovery ? ( +
+

+ Scient will retry this requested {pendingRecovery.action} after{" "} + + . +

+ +
+ ) : null}
>> = { + check: ["started", "completed", "failed"], + download: ["started", "completed", "failed"], + install: ["handoff-attempted", "failed"], + apply: ["completed", "not-applied", "superseded"], +}; +const UPDATE_FAILURE_CLASSES = [ + "network", + "timeout", + "certificate", + "http_4xx", + "http_5xx", + "feed", + "checksum", + "signature", + "disk_full", + "permission", + "configuration", + "cancelled", + "internal", + "unknown", +]; +const UPDATE_STATUSES = [ + "disabled", + "idle", + "checking", + "up-to-date", + "available", + "downloading", + "downloaded", + "error", +]; +const UPDATE_DISABLED_CLASSES = [ + "policy", + "channel-policy", + "safety", + "no-feed", + "development", + "environment", + "linux-package", + "other", +]; +const UPDATE_ATTEMPT_REF_PATTERN = /^[0-9a-f]{16}$/u; const OPERATION_KINDS = new Set([ "file-preview", "pdf-open", @@ -842,6 +901,17 @@ function byteBucket(value: unknown): string { return "over-100mb"; } +/** Bytes per second, bucketed in megabits per second as connections are quoted. */ +function throughputBucket(value: unknown): string { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown"; + const megabits = (value * 8) / 1_000_000; + if (megabits < 1) return "under-1mbps"; + if (megabits < 5) return "1-5mbps"; + if (megabits < 20) return "5-20mbps"; + if (megabits < 100) return "20-100mbps"; + return "over-100mbps"; +} + function percentBucket(value: unknown): string { if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown"; if (value < 25) return "0-24"; @@ -861,6 +931,98 @@ export function durationBucket(value: unknown): string { return "over-10m"; } +/** How long an update stage or offer has lasted: minutes while transferring, days while waiting. */ +function updateAgeBucket(value: unknown): string { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown"; + if (value < 60_000) return "under-1m"; + if (value < 600_000) return "1-10m"; + if (value < 3_600_000) return "10-60m"; + if (value < 21_600_000) return "1-6h"; + if (value < 86_400_000) return "6-24h"; + if (value < 3 * 86_400_000) return "1-3d"; + if (value < 7 * 86_400_000) return "3-7d"; + return "over-7d"; +} + +/** Copies a field only when it is one of `allowed`; any other value stays absent. */ +function optionalClosed( + input: Readonly>, + key: string, + allowed: ReadonlyArray, +): Record { + const value = property(input, key); + return typeof value === "string" && allowed.includes(value) ? { [key]: value } : {}; +} + +/** Marks every field read, so an event sends only what its normalizer sets. */ +function closedEvent(input: Readonly>): void { + for (const key of Object.keys(input)) property(input, key); +} + +function updateOutcomeProperties(input: Readonly>) { + closedEvent(input); + const step = closedCategory(property(input, "step"), Object.keys(UPDATE_STEP_OUTCOMES)); + const outcome = closedCategory(property(input, "outcome"), UPDATE_STEP_OUTCOMES[step] ?? []); + const failureClass = closedCategory(property(input, "failureClass"), UPDATE_FAILURE_CLASSES); + const trigger = property(input, "trigger"); + const targetVersion = property(input, "targetVersion"); + const attemptRef = property(input, "attemptRef"); + const durationMs = property(input, "durationMs"); + return { + step, + outcome, + ...(outcome === "failed" ? { failureClass } : {}), + ...(step !== "apply" && (trigger === "user" || trigger === "background") ? { trigger } : {}), + ...(typeof targetVersion === "string" && analyticsAppVersion(targetVersion) !== "unknown" + ? { targetVersion } + : {}), + ...optionalClosed(input, "sourceKind", ["github", "scient_mirror", "administrator"]), + ...optionalClosed(input, "packageKind", ["zip", "nsis", "appimage", "deb"]), + ...(typeof attemptRef === "string" && UPDATE_ATTEMPT_REF_PATTERN.test(attemptRef) + ? { attemptRef } + : {}), + ...(durationMs === undefined ? {} : { durationBucket: updateAgeBucket(durationMs) }), + }; +} + +function updateSummaryProperties(input: Readonly>) { + closedEvent(input); + const status = closedCategory(property(input, "status"), UPDATE_STATUSES); + const lastCheckAgeMs = property(input, "lastCheckAgeMs"); + return { + reason: closedCategory(property(input, "reason"), ["launch", "change", "daily"]), + status, + channel: closedCategory(property(input, "channel"), ["latest", "beta", "nightly"]), + behind: closedCategory(property(input, "behind"), ["yes", "no", "unknown"]), + stageAgeBucket: updateAgeBucket(property(input, "stageAgeMs")), + lastCheckAgeBucket: lastCheckAgeMs === undefined ? "never" : updateAgeBucket(lastCheckAgeMs), + droppedFactsBucket: countBucket(property(input, "droppedFactCount") ?? 0), + ...(status === "error" + ? { + errorContext: closedCategory(property(input, "errorContext"), [ + "check", + "download", + "install", + ]), + } + : {}), + ...(status === "downloading" + ? { + progressBucket: percentBucket(property(input, "downloadPercent")), + sinceProgressBucket: updateAgeBucket(property(input, "sinceProgressMs")), + } + : {}), + ...(status === "downloaded" + ? { readyAgeBucket: updateAgeBucket(property(input, "readyAgeMs")) } + : {}), + ...(status === "disabled" + ? { + disabledClass: closedCategory(property(input, "disabledClass"), UPDATE_DISABLED_CLASSES), + } + : {}), + }; +} + function turnTerminalStatus(value: unknown): string { if (value === "cancelled" || value === "interrupted") return "stopped"; return normalizedEnum(value, new Set(["completed", "failed", "stopped"]), "other"); @@ -1037,29 +1199,43 @@ function normalizeEvent( case "provider.lifecycle.started": case "provider.lifecycle.completed": case "provider.lifecycle.failed": - case "provider.lifecycle.cancelled": + case "provider.lifecycle.cancelled": { + const failed = name === "provider.lifecycle.failed"; + // Read on every lifecycle event so a stray class never passes through on the others. + const failureClass = normalizedEnum(property(input, "failureClass"), FAILURE_CLASSES); return { name, - privacyLevel: name === "provider.lifecycle.failed" ? "essential" : "product", - priority: name === "provider.lifecycle.failed" ? "critical" : "core", + privacyLevel: failed ? "essential" : "product", + priority: failed ? "critical" : "core", properties: { provider, action: normalizedEnum(property(input, "action"), LIFECYCLE_ACTIONS), runtimeSource: normalizedEnum(property(input, "source"), RUNTIME_SOURCES), stage: normalizedEnum(property(input, "stage"), LIFECYCLE_STAGES), - failureClass: normalizedEnum(property(input, "failureClass"), FAILURE_CLASSES), + ...(failed ? { failureClass } : {}), durationBucket: durationBucket(property(input, "durationMs")), trigger: normalizedEnum(property(input, "trigger"), LIFECYCLE_TRIGGERS, "user"), ...optionalProperties(input, { attempt: "count", downloadSource: DOWNLOAD_SOURCES, - compressed: "boolean", + sourceKind: DOWNLOAD_SOURCE_KINDS, + downloadVariant: DOWNLOAD_VARIANTS, + downloadAttempts: "count", + networkKind: DOWNLOAD_NETWORK_KINDS, + compressedFailure: FAILURE_CLASSES, }), ...(property(input, "resumedBytes") === undefined ? {} : { resumedBucket: byteBucket(property(input, "resumedBytes")) }), + ...(property(input, "progressPercent") === undefined + ? {} + : { progressBucket: percentBucket(property(input, "progressPercent")) }), + ...(property(input, "bytesPerSecond") === undefined + ? {} + : { throughputBucket: throughputBucket(property(input, "bytesPerSecond")) }), }, }; + } case "provider.runtime.first_pick": return { name, @@ -1112,6 +1288,23 @@ function normalizeEvent( ...optionalProperties(input, { stoppedCount: "count" }), }, }; + case "app.update.outcome": { + const properties = updateOutcomeProperties(input); + return { + name, + privacyLevel: "essential", + priority: + properties.step === "install" || properties.step === "apply" ? "critical" : "core", + properties, + }; + } + case "app.update.summary": + return { + name, + privacyLevel: "essential", + priority: "summary", + properties: updateSummaryProperties(input), + }; case "app.session.started": return { name, diff --git a/packages/scient-analytics/src/index.ts b/packages/scient-analytics/src/index.ts index 70590c5e11..09fe6701a5 100644 --- a/packages/scient-analytics/src/index.ts +++ b/packages/scient-analytics/src/index.ts @@ -14,5 +14,6 @@ export { createAnalyticsRuntime, type AnalyticsRuntime, type AnalyticsRuntimeOptions, + type RecordedAnalyticsEvent, } from "./runtime.ts"; export type { AnalyticsWorkerInput } from "./workerProtocol.ts"; diff --git a/packages/scient-analytics/src/runtime.test.ts b/packages/scient-analytics/src/runtime.test.ts index cb44280f23..bf020a7dc0 100644 --- a/packages/scient-analytics/src/runtime.test.ts +++ b/packages/scient-analytics/src/runtime.test.ts @@ -280,4 +280,81 @@ describe("Scient analytics background runtime", () => { expect(deadLetters.count).toBe(1); expect(server.bodies).toHaveLength(0); }); + + it("ingests a recorded event once, keeping its id, time and release", async () => { + const outboxPath = fixturePath(); + const runtime = createAnalyticsRuntime( + options({ outboxPath, now: () => new Date("2026-11-16T12:00:00.000Z") }), + ); + const recorded = { + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24", + name: "app.update.outcome", + occurredAt: "2026-11-16T11:00:00.000Z", + appVersion: "0.6.29", + buildChannel: "stable", + properties: { step: "install", outcome: "handoff-attempted", targetVersion: "0.6.30" }, + } as const; + try { + expect(await runtime.ingest([recorded])).toBe(true); + // A replay after a lost acknowledgement finds the same row. + expect(await runtime.ingest([recorded])).toBe(true); + expect(await runtime.pendingCount()).toBe(1); + } finally { + await runtime.close(); + } + const database = new NodeSqlite.DatabaseSync(outboxPath); + const row = database + .prepare("SELECT id, occurred_at, privacy_level, properties_json FROM analytics_outbox") + .get() as { + readonly id: string; + readonly occurred_at: string; + readonly privacy_level: string; + readonly properties_json: string; + }; + database.close(); + expect(row).toMatchObject({ + id: recorded.id, + occurred_at: recorded.occurredAt, + privacy_level: "essential", + }); + expect(JSON.parse(row.properties_json)).toMatchObject({ + appVersion: "0.6.29", + step: "install", + targetVersion: "0.6.30", + }); + }); + + it("settles recorded events it must not send without queueing them", async () => { + const runtime = createAnalyticsRuntime( + options({ now: () => new Date("2026-11-16T12:00:00.000Z") }), + ); + const summary = { + name: "app.update.summary", + appVersion: "0.6.29", + buildChannel: "stable", + properties: { reason: "launch", status: "idle" }, + } as const; + try { + // Too old, a malformed id, and an unknown name are never sent. + expect( + await runtime.ingest([ + { + ...summary, + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a25", + occurredAt: "2026-10-01T00:00:00.000Z", + }, + { ...summary, id: "not-a-uuid", occurredAt: "2026-11-16T11:00:00.000Z" }, + { + ...summary, + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a26", + name: "Not An Event", + occurredAt: "2026-11-16T11:00:00.000Z", + }, + ]), + ).toBe(true); + expect(await runtime.pendingCount()).toBe(0); + } finally { + await runtime.close(); + } + }); }); diff --git a/packages/scient-analytics/src/runtime.ts b/packages/scient-analytics/src/runtime.ts index 93b8e46234..49fb56b91d 100644 --- a/packages/scient-analytics/src/runtime.ts +++ b/packages/scient-analytics/src/runtime.ts @@ -22,6 +22,9 @@ const MAX_IN_FLIGHT_BATCHES = 2; const WRITE_COALESCE_MS = 100; const CONTROL_TIMEOUT_MS = 3_500; const LOCAL_SHUTDOWN_BUDGET_MS = 300; +const RECORDED_EVENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u; +const RECORDED_EVENT_MAX_AGE_MS = 30 * 86_400_000; +const RECORDED_EVENT_MAX_SKEW_MS = 86_400_000; const PRIORITY_RANK: Readonly> = { critical: 0, @@ -40,9 +43,22 @@ export interface AnalyticsRuntimeOptions extends NormalizationContext { readonly controlTimeoutMs?: number; } +/** An event another Scient process recorded; it keeps its own id, time and release. */ +export interface RecordedAnalyticsEvent extends NormalizationContext { + readonly id: string; + readonly name: string; + readonly occurredAt: string; + readonly properties: Readonly>; +} + export interface AnalyticsRuntime { readonly enabled: boolean; readonly record: (name: string, properties?: Readonly>) => boolean; + /** + * True once the outbox holds these events or none of them may be sent (shape, age); + * false when that is unconfirmed, so the caller keeps them to offer again. + */ + readonly ingest: (events: ReadonlyArray) => Promise; readonly flush: () => Promise; readonly pendingCount: () => Promise; readonly close: () => Promise; @@ -52,6 +68,7 @@ function disabledRuntime(): AnalyticsRuntime { return { enabled: false, record: () => false, + ingest: async () => true, flush: async () => 0, pendingCount: async () => 0, close: async () => undefined, @@ -98,6 +115,7 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt let nextRequestId = 1; const queue: AnalyticsEventDraft[] = []; const inFlight = new Map>(); + const ingestions = new Map void>(); const requests = new Map< number, { @@ -127,6 +145,8 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt } queue.length = 0; inFlight.clear(); + for (const settle of ingestions.values()) settle(false); + ingestions.clear(); settleRequests(); }; @@ -203,6 +223,8 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt return; case "persisted": inFlight.delete(response.batchId); + ingestions.get(response.batchId)?.(true); + ingestions.delete(response.batchId); drain(); return; case "result": { @@ -262,11 +284,68 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt return accepted; }; + const recordedDraft = (event: RecordedAnalyticsEvent): AnalyticsEventDraft | null => { + const occurredAt = Date.parse(event.occurredAt); + const nowMs = now().valueOf(); + if ( + !RECORDED_EVENT_ID.test(event.id) || + !Number.isFinite(occurredAt) || + occurredAt > nowMs + RECORDED_EVENT_MAX_SKEW_MS || + occurredAt < nowMs - RECORDED_EVENT_MAX_AGE_MS + ) { + return null; + } + const normalized = normalizeInheritedEvent(event.name, event.properties, { + appVersion: event.appVersion, + buildChannel: event.buildChannel, + }); + if (!normalized) return null; + return { + id: event.id, + name: normalized.name, + session_id: sessionId, + occurred_at: new Date(occurredAt).toISOString(), + privacy_level: normalized.privacyLevel, + // Same constant `record` sends: the gateway still reads a consent level. + consent_level: "diagnostic", + properties: normalized.properties, + priority: normalized.priority, + }; + }; + + const ingest: AnalyticsRuntime["ingest"] = async (events) => { + if (closed || !available) return false; + const drafts = events.flatMap((event) => recordedDraft(event) ?? []); + if (drafts.length === 0) return true; + await waitForCondition(() => ready || !available, Date.now() + controlTimeoutMs); + if (!ready || !available || closed) return false; + const batchId = nextBatchId++; + return new Promise((resolve) => { + const timer = setTimeout(() => { + ingestions.delete(batchId); + resolve(false); + }, controlTimeoutMs); + timer.unref(); + ingestions.set(batchId, (persisted) => { + clearTimeout(timer); + resolve(persisted); + }); + inFlight.set(batchId, drafts); + if (!post({ type: "enqueue", batchId, events: drafts })) { + inFlight.delete(batchId); + ingestions.delete(batchId); + clearTimeout(timer); + resolve(false); + } + }); + }; + return { get enabled() { return available && !closed; }, record, + ingest, flush: async () => { if (closed || !available) return 0; await waitForCondition(() => ready || !available, Date.now() + controlTimeoutMs); diff --git a/packages/scient-provider-runtime/src/managedProviderRuntime.ts b/packages/scient-provider-runtime/src/managedProviderRuntime.ts index b6829a4f89..ef5f3ee95a 100644 --- a/packages/scient-provider-runtime/src/managedProviderRuntime.ts +++ b/packages/scient-provider-runtime/src/managedProviderRuntime.ts @@ -15,6 +15,11 @@ import { downloadManagedRuntime, type ManagedRuntimeDownloadResult, type ManagedRuntimeDownloadRetry, + type ManagedRuntimeFailureClass, + managedRuntimeFailureClass, + managedRuntimeFailureRetryable, + managedRuntimeRetryAfterMs, + ManagedRuntimeFileError, materializeManagedRuntimeArtifact, resolveManagedRuntimeArtifactPath, unpackManagedRuntimeZstd, @@ -45,14 +50,26 @@ export interface ManagedProviderRuntimeProgress { readonly totalBytes?: number | undefined; /** Set while the download waits to try again after a temporary failure. */ readonly retry?: ManagedRuntimeDownloadRetry | undefined; - /** Set once the download is complete, on the `verifying` stage. */ + /** How the download went: after each failed try, and once complete on the `verifying` stage. */ readonly download?: ManagedRuntimeDownloadFacts | undefined; } -/** How a finished download went, for analytics. */ +/** Who served a download: Scient's own copy, GitHub, or the provider's own host. */ +export type ManagedRuntimeSourceKind = "scient_mirror" | "github" | "vendor"; + +/** How a download went, for analytics. */ export interface ManagedRuntimeDownloadFacts extends ManagedRuntimeDownloadResult { /** The zstd copy, not the raw binary, was downloaded. */ readonly compressed: boolean; + /** Who served the latest try. */ + readonly sourceKind: ManagedRuntimeSourceKind; + /** Why the zstd copy failed, when the raw binary was downloaded after it. */ + readonly compressedFailure?: ManagedRuntimeFailureClass | undefined; +} + +function hostSourceKind(url: string): ManagedRuntimeSourceKind { + const host = new URL(url).hostname.toLowerCase(); + return host === "github.com" || host.endsWith(".githubusercontent.com") ? "github" : "vendor"; } export interface ManagedProviderRuntimeStateV1 { @@ -116,8 +133,6 @@ export interface ManagedProviderRuntimeInstallInput { readonly beforeActivate?: (signal: AbortSignal) => Promise; /** Told why the compressed copy failed, before the raw binary is downloaded; a throw stops the install. */ readonly onCompressedFallback?: (cause: unknown) => void; - /** Per download copy; `null` waits as long as data keeps arriving. Ten minutes by default. */ - readonly downloadTimeLimitMs?: number | null | undefined; } interface ManagedProviderRuntimeActivation { @@ -683,11 +698,13 @@ export class ManagedProviderRuntime { readonly signal: AbortSignal; readonly onProgress: ManagedProviderRuntimeInstallInput["onProgress"]; readonly onCompressedFallback: ManagedProviderRuntimeInstallInput["onCompressedFallback"]; - readonly timeLimitMs: ManagedProviderRuntimeInstallInput["downloadTimeLimitMs"]; }): Promise { const { artifact, archivePath, signal, onProgress } = input; const compressed = artifact.compressed; const unpack = this.#dependencies.unpackZstd; + let compressedFailure: ManagedRuntimeFailureClass | undefined; + let compressedCause: unknown; + const sourceCooldowns = new Map(); const fetchVerified = async (copy: { readonly url: string; readonly mirrorUrl?: string | undefined; @@ -696,7 +713,18 @@ export class ManagedProviderRuntime { readonly destination: string; readonly compressed: boolean; }) => { - onProgress?.({ stage: "downloading", downloadedBytes: 0, totalBytes: copy.size }); + const describe = (download: ManagedRuntimeDownloadResult): ManagedRuntimeDownloadFacts => ({ + ...download, + compressed: copy.compressed, + sourceKind: + download.source === "fallback" || !copy.mirrorUrl + ? hostSourceKind(copy.url) + : "scient_mirror", + ...(compressedFailure ? { compressedFailure } : {}), + }); + let downloadedBytes = 0; + // The downloader reports the bytes an earlier try kept as soon as it has checked them. + onProgress?.({ stage: "downloading", totalBytes: copy.size }); const result = await this.#dependencies.download({ ...(copy.mirrorUrl ? { @@ -705,21 +733,27 @@ export class ManagedProviderRuntime { allowedHosts: [...artifact.allowedHosts, new URL(copy.mirrorUrl).hostname], } : { url: copy.url, allowedHosts: artifact.allowedHosts }), + sourceCooldowns, checksum: copy.checksum, destination: copy.destination, partialPath: this.#partialDownloadPath(copy.checksum), - ...(input.timeLimitMs === undefined ? {} : { timeLimitMs: input.timeLimitMs }), expectedSize: copy.size, signal, - onProgress: (downloadedBytes, totalBytes) => - onProgress?.({ stage: "downloading", downloadedBytes, totalBytes }), + onProgress: (bytes, totalBytes) => { + downloadedBytes = bytes; + onProgress?.({ stage: "downloading", downloadedBytes: bytes, totalBytes }); + }, onRetry: (retry) => - onProgress?.({ stage: "downloading", downloadedBytes: 0, totalBytes: copy.size, retry }), - }); - onProgress?.({ - stage: "verifying", - ...(result ? { download: { ...result, compressed: copy.compressed } } : {}), + onProgress?.({ stage: "downloading", downloadedBytes, totalBytes: copy.size, retry }), + onFailedTry: (download) => + onProgress?.({ + stage: "downloading", + downloadedBytes, + totalBytes: copy.size, + download: describe(download), + }), }); + onProgress?.({ stage: "verifying", ...(result ? { download: describe(result) } : {}) }); }; if (compressed && unpack) { const compressedPath = `${archivePath}.zst`; @@ -737,34 +771,55 @@ export class ManagedProviderRuntime { } catch (cause) { if (signal.aborted) throw cause; await NodeFSP.rm(archivePath, { force: true }); + compressedCause = cause; + compressedFailure = managedRuntimeFailureClass(cause); input.onCompressedFallback?.(cause); } finally { await NodeFSP.rm(compressedPath, { force: true }); } } - await fetchVerified({ - url: artifact.url, - mirrorUrl: managedRuntimeMirrorUrl(artifact), - checksum: artifact.checksum, - size: artifact.size, - destination: archivePath, - compressed: false, - }); - await this.#dependencies.verify(archivePath, artifact.checksum); + try { + await fetchVerified({ + url: artifact.url, + mirrorUrl: managedRuntimeMirrorUrl(artifact), + checksum: artifact.checksum, + size: artifact.size, + destination: archivePath, + compressed: false, + }); + await this.#dependencies.verify(archivePath, artifact.checksum); + } catch (cause) { + if (signal.aborted || compressedCause === undefined) throw cause; + const failureClass = managedRuntimeFailureClass(cause); + const sourceFailure = + failureClass === "http_4xx" || + failureClass === "http_5xx" || + failureClass === "network" || + failureClass === "timeout" || + failureClass === "checksum" || + failureClass === "size_mismatch" || + failureClass === "certificate"; + throw new ManagedRuntimeFileError( + cause instanceof Error ? cause.message : "Managed runtime download failed.", + { + cause, + failureClass, + retryable: + managedRuntimeFailureRetryable(cause) || + (sourceFailure && managedRuntimeFailureRetryable(compressedCause)), + retryAfterMs: Math.max( + managedRuntimeRetryAfterMs(cause) ?? 0, + managedRuntimeRetryAfterMs(compressedCause) ?? 0, + ), + }, + ); + } } async #installOwned( input: ManagedProviderRuntimeInstallInput, ): Promise { - const { - artifact, - signal, - onProgress, - qualify, - beforeActivate, - onCompressedFallback, - downloadTimeLimitMs, - } = input; + const { artifact, signal, onProgress, qualify, beforeActivate, onCompressedFallback } = input; await NodeFSP.mkdir(this.#stagingDir, { recursive: true, mode: 0o700 }); // This process launches what it installs. Pruning needs the mutation lock held here. await this.#lease(artifact.version); @@ -789,7 +844,6 @@ export class ManagedProviderRuntime { signal, onProgress, onCompressedFallback, - timeLimitMs: downloadTimeLimitMs, }); onProgress?.({ stage: "installing" }); const stagedExecutable = await this.#dependencies.materialize({ diff --git a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts index 0e8c18aa26..d5a4174320 100644 --- a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts +++ b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts @@ -8,9 +8,11 @@ import * as NodeZlib from "node:zlib"; import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import type { ManagedProviderRuntimeProgress } from "./managedProviderRuntime.ts"; import { hydrateManagedRuntimeArtifact } from "./managedRuntimeArtifact.ts"; import { ManagedScientAgentRuntime } from "./managedScientAgentRuntime.ts"; import { resolveScientAgentArtifactPolicy } from "./scientAgentManifest.ts"; +import { managedRuntimeFailureRetryable, managedRuntimeRetryAfterMs } from "./runtimeFiles.ts"; vi.mock("node:timers/promises", { spy: true }); // The transient-response backoff resolves at once but still honours cancellation. @@ -62,6 +64,7 @@ async function installedFixture() { } afterEach(async () => { + vi.useRealTimers(); vi.unstubAllGlobals(); await Promise.all( roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })), @@ -110,6 +113,41 @@ it("recovers the Scient download and still runs smoke, admission, and qualificat expect(await NodeFSP.readFile(otherState, "utf8")).toBe("keep"); }); +// regression: progress fell back to zero while a resumed download waited or started again. +it("reports progress from the bytes kept by an earlier try", async () => { + const { root } = await installedFixture(); + // Cut off after 5 bytes on Scient's copy and after 10 on GitHub, then whole on Scient's. + const ends = [5, 10, bytes.length]; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: URL, init: RequestInit) => { + const range = (init.headers as Record | undefined)?.range; + const from = Number(/bytes=(\d+)-/u.exec(range ?? "")?.[1] ?? 0); + const chunk = new Uint8Array(bytes.subarray(from, ends.shift())); + return from === 0 + ? new Response(chunk, { headers: { "content-length": String(bytes.length) } }) + : new Response(chunk, { + status: 206, + headers: { "content-range": `bytes ${from}-${bytes.length - 1}/${bytes.length}` }, + }); + }), + ); + const downloaded: number[] = []; + const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); + await runtime.install({ + artifact: artifact("0.2.0"), + signal: new AbortController().signal, + onProgress: (report) => { + if (report.stage === "downloading" && report.downloadedBytes !== undefined) + downloaded.push(report.downloadedBytes); + }, + }); + expect(downloaded.at(-1)).toBe(bytes.length); + expect(downloaded.slice(downloaded.indexOf(5))).toEqual( + downloaded.slice(downloaded.indexOf(5)).toSorted((left, right) => left - right), + ); +}); + it.each(["checksum", "smoke", "admission", "qualification", "cancellation"] as const)( "preserves the installed release when %s refuses an update after a transient response", async (failure) => { @@ -258,28 +296,88 @@ describe("installing from the compressed copy", () => { }); it.each([ - { failure: "a damaged copy on both servers", catalogCopy: packed, served: Buffer.from("junk") }, + { + failure: "a damaged copy on both servers", + catalogCopy: packed, + served: Buffer.from("junk"), + compressedFailure: "size_mismatch", + }, { failure: "a copy that unpacks to other bytes", catalogCopy: NodeZlib.zstdCompressSync(Buffer.from("other bytes")), served: NodeZlib.zstdCompressSync(Buffer.from("other bytes")), + compressedFailure: "other", }, - ])("falls back to the binary after $failure", async ({ catalogCopy, served }) => { - const { root } = await installedFixture(); + ])( + "falls back to the binary after $failure", + async ({ catalogCopy, served, compressedFailure }) => { + const { root } = await installedFixture(); + const urls: string[] = []; + vi.stubGlobal("fetch", serve(served, urls)); + const onCompressedFallback = vi.fn(); + const progress: ManagedProviderRuntimeProgress[] = []; + const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); + await runtime.install({ + artifact: compressedArtifact("0.2.0", catalogCopy), + signal: new AbortController().signal, + onCompressedFallback, + onProgress: (report) => progress.push(report), + }); + expect(onCompressedFallback).toHaveBeenCalledOnce(); + // The binary's download still says why the compressed copy was given up. + expect(progress.findLast((report) => report.stage === "verifying")?.download).toMatchObject({ + compressed: false, + sourceKind: "scient_mirror", + compressedFailure, + }); + expect(urls.at(-1)).toBe( + "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64", + ); + await expectInstalled(runtime, root, "0.2.0"); + }, + ); + + it("carries a compressed source cooldown through raw fallback and the terminal failure", async () => { + const { root, previousState } = await installedFixture(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(0); + using wait = vi.spyOn(NodeTimersPromises, "setTimeout"); + wait.mockClear(); const urls: string[] = []; - vi.stubGlobal("fetch", serve(served, urls)); - const onCompressedFallback = vi.fn(); - const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); - await runtime.install({ - artifact: compressedArtifact("0.2.0", catalogCopy), - signal: new AbortController().signal, - onCompressedFallback, + const fetch = vi.fn(async (url: URL) => { + urls.push(String(url)); + return url.hostname === "downloads.scientfactory.com" + ? new Response("busy", { status: 429, headers: { "retry-after": "3600" } }) + : new Response("missing", { status: 404 }); }); - expect(onCompressedFallback).toHaveBeenCalledOnce(); - expect(urls.at(-1)).toBe( - "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64", - ); - await expectInstalled(runtime, root, "0.2.0"); + vi.stubGlobal("fetch", fetch); + const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); + const compressedFailures: unknown[] = []; + const failure = await runtime + .install({ + artifact: compressedArtifact("0.2.0"), + signal: new AbortController().signal, + onCompressedFallback: (cause) => { + compressedFailures.push(cause); + vi.advanceTimersByTime(10_000); + }, + }) + .catch((cause: unknown) => cause); + expect(compressedFailures).toHaveLength(1); + expect(managedRuntimeRetryAfterMs(compressedFailures[0])).toBe(3_590_000); + expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: failure }))).toBe(3_590_000); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); + expect(String(failure)).toContain("github.com Managed runtime download failed with HTTP 404"); + expect(urls).toEqual([ + "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64.zst", + "https://github.com/ScientFactory/scient-agent/releases/download/v0.2.0/scient-agent-darwin-arm64.zst", + "https://github.com/ScientFactory/scient-agent/releases/download/v0.2.0/scient-agent-darwin-arm64", + ]); + expect(wait).not.toHaveBeenCalled(); + expect(await runtime.readState()).toEqual(previousState); + expect( + await NodeFSP.readdir(NodePath.join(root, "provider-runtimes", "scient-agent", "staging")), + ).toEqual([]); }); it("stops without the binary when told not to fall back, or when cancelled", async () => { diff --git a/packages/scient-provider-runtime/src/runtimeDownload.test.ts b/packages/scient-provider-runtime/src/runtimeDownload.test.ts index 4e59c46c06..6ec7522253 100644 --- a/packages/scient-provider-runtime/src/runtimeDownload.test.ts +++ b/packages/scient-provider-runtime/src/runtimeDownload.test.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off -- Synthetic responses exercise the private download boundary without network or provider credentials. +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off -- Synthetic responses exercise the private download boundary without network or provider credentials; the slow-download case paces its body on the test clock. import * as NodeCrypto from "node:crypto"; import * as NodeFSP from "node:fs/promises"; import * as NodeOS from "node:os"; @@ -11,6 +11,8 @@ import { downloadManagedRuntime, ManagedRuntimeFileError, managedRuntimeFailureClass, + managedRuntimeFailureRetryable, + managedRuntimeRetryAfterMs, verifyManagedRuntimeChecksum, } from "./runtimeFiles.ts"; @@ -18,12 +20,14 @@ vi.mock("node:timers/promises", { spy: true }); // The retry backoff resolves at once but still honours cancellation; the // "cancels the retry delay promptly" test drives the real timer. beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.mocked(NodeTimersPromises.setTimeout).mockImplementation((async ( - _delay: number, + delay: number, value: unknown, options?: { signal?: AbortSignal }, ) => { options?.signal?.throwIfAborted(); + vi.advanceTimersByTime(delay); return value; }) as never); }); @@ -42,6 +46,7 @@ async function input(signal = new AbortController().signal) { } afterEach(async () => { + vi.useRealTimers(); vi.unstubAllGlobals(); await Promise.all( roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })), @@ -109,41 +114,60 @@ describe("transient managed runtime downloads", () => { await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); }); - it("waits as long as the server asks, up to a minute", async () => { + it("honours a short server cooldown inside the local retry budget", async () => { const request = await input(); const fetch = vi .fn() .mockResolvedValueOnce(new Response("busy", { status: 503, headers: { "retry-after": "7" } })) - .mockResolvedValueOnce( - new Response("busy", { status: 429, headers: { "retry-after": "3600" } }), - ) .mockResolvedValueOnce(new Response("ok", { headers: { "content-length": "2" } })); vi.stubGlobal("fetch", fetch); vi.mocked(NodeTimersPromises.setTimeout).mockClear(); await downloadManagedRuntime(request); expect(vi.mocked(NodeTimersPromises.setTimeout).mock.calls.map(([delay]) => delay)).toEqual([ - 7_000, 60_000, + 7_000, ]); + expect(fetch).toHaveBeenCalledTimes(2); }); - it("cancels recovery without sending another request", async () => { - const controller = new AbortController(); - const request = await input(controller.signal); - const fetch = vi.fn().mockImplementation(() => - Promise.resolve( - new Response( - new ReadableStream({ - cancel: () => controller.abort(), - }), - { status: 503 }, + it.each(["3600", "99999999999999999999999999999999999999999999999999999999999999999999999"])( + "leaves long Retry-After %s to durable recovery without waiting locally", + async (header) => { + const request = await input(); + const fetch = vi.fn( + async () => new Response("busy", { status: 429, headers: { "retry-after": header } }), + ); + vi.stubGlobal("fetch", fetch); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ManagedRuntimeFileError); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); + expect(managedRuntimeRetryAfterMs(failure)).toBe(header === "3600" ? 3_600_000 : 14_400_000); + expect(fetch).toHaveBeenCalledOnce(); + expect(NodeTimersPromises.setTimeout).not.toHaveBeenCalled(); + }, + ); + + it.each([undefined, "3600"])( + "cancels recovery without another request (Retry-After %s)", + async (header) => { + const controller = new AbortController(); + const request = await input(controller.signal); + const fetch = vi.fn().mockImplementation(() => + Promise.resolve( + new Response( + new ReadableStream({ + cancel: () => controller.abort(), + }), + { status: 503, ...(header ? { headers: { "retry-after": header } } : {}) }, + ), ), - ), - ); - vi.stubGlobal("fetch", fetch); - await expect(downloadManagedRuntime(request)).rejects.toMatchObject({ name: "AbortError" }); - expect(fetch).toHaveBeenCalledOnce(); - await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); - }); + ); + vi.stubGlobal("fetch", fetch); + await expect(downloadManagedRuntime(request)).rejects.toMatchObject({ name: "AbortError" }); + expect(fetch).toHaveBeenCalledOnce(); + await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); it("retains host admission when a retried response redirects", async () => { const request = await input(); @@ -207,37 +231,46 @@ describe("transient managed runtime downloads", () => { await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); }); - it("cancels the retry delay promptly", async () => { - const controller = new AbortController(); - const request = await input(controller.signal); - const bodyReleased = Promise.withResolvers(); - const fetch = vi.fn().mockImplementation( - async () => - new Response(new ReadableStream({ cancel: () => bodyReleased.resolve() }), { - status: 503, - }), - ); - vi.stubGlobal("fetch", fetch); - const retryStarted = Promise.withResolvers(); - const { setTimeout: waitForRetry } = - await vi.importActual("node:timers/promises"); - vi.mocked(NodeTimersPromises.setTimeout).mockClear(); - using retryTimer = vi.spyOn(NodeTimersPromises, "setTimeout").mockImplementation((...args) => { - const waiting = waitForRetry(...args); - retryStarted.resolve(); - return waiting; - }); - const download = downloadManagedRuntime(request); - await bodyReleased.promise; - await retryStarted.promise; - const [delay] = retryTimer.mock.calls[0]!; - // The first wait is about two seconds, spread by up to a fifth either way. - expect(delay).toBeGreaterThanOrEqual(1_600); - expect(delay).toBeLessThanOrEqual(2_400); - controller.abort(); - await expect(download).rejects.toMatchObject({ name: "AbortError" }); - expect(fetch).toHaveBeenCalledOnce(); - }); + it.each([undefined, "60"])( + "cancels the retry delay promptly (Retry-After %s)", + async (header) => { + const controller = new AbortController(); + const request = await input(controller.signal); + const bodyReleased = Promise.withResolvers(); + const fetch = vi.fn().mockImplementation( + async () => + new Response(new ReadableStream({ cancel: () => bodyReleased.resolve() }), { + status: 503, + ...(header ? { headers: { "retry-after": header } } : {}), + }), + ); + vi.stubGlobal("fetch", fetch); + const retryStarted = Promise.withResolvers(); + const { setTimeout: waitForRetry } = + await vi.importActual("node:timers/promises"); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + using retryTimer = vi + .spyOn(NodeTimersPromises, "setTimeout") + .mockImplementation((...args) => { + const waiting = waitForRetry(...args); + retryStarted.resolve(); + return waiting; + }); + const download = downloadManagedRuntime(request); + await bodyReleased.promise; + await retryStarted.promise; + const [delay] = retryTimer.mock.calls[0]!; + if (header) expect(delay).toBe(60_000); + else { + // The first wait is about two seconds, spread by up to a fifth either way. + expect(delay).toBeGreaterThanOrEqual(1_600); + expect(delay).toBeLessThanOrEqual(2_400); + } + controller.abort(); + await expect(download).rejects.toMatchObject({ name: "AbortError" }); + expect(fetch).toHaveBeenCalledOnce(); + }, + ); it("retries a failed connection and names it when every try fails", async () => { const request = await input(); @@ -348,29 +381,88 @@ describe("downloads with Scient's own copy first", () => { expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl]); }); - it("gives GitHub its own time limit when Scient's copy runs out of time", async () => { + it.each([ + { mirrorFirst: true, fallbackElapsedMs: 0 }, + { mirrorFirst: false, fallbackElapsedMs: 0 }, + { mirrorFirst: true, fallbackElapsedMs: 10_000 }, + ])( + "preserves the mirror cooldown after GitHub fails ($mirrorFirst, $fallbackElapsedMs)", + async ({ mirrorFirst, fallbackElapsedMs }) => { + vi.setSystemTime(0); + using _random = vi.spyOn(Math, "random").mockReturnValue(0.5); + const base = await twoSources(); + const request = mirrorFirst + ? base + : { ...base, url: base.fallbackUrl, fallbackUrl: base.url }; + let fallbackTries = 0; + const fetch = vi.fn(async (url: URL) => { + if (url.hostname === "downloads.scientfactory.com") + return new Response("busy", { status: 429, headers: { "retry-after": "3600" } }); + if (fallbackTries++ === 0) vi.advanceTimersByTime(fallbackElapsedMs); + return new Response("busy", { status: 503 }); + }); + vi.stubGlobal("fetch", fetch); + const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ManagedRuntimeFileError); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); + expect(failure).toMatchObject({ retryAfterMs: 3_548_000 - fallbackElapsedMs }); + expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: failure }))).toBe( + 3_548_000 - fallbackElapsedMs, + ); + expect(urls(fetch).filter((url) => url === mirror)).toHaveLength(1); + expect(fallbackTries).toBe(5); + expect(fetch).toHaveBeenCalledTimes(6); + expect(String(failure)).toContain("downloads.scientfactory.com returned HTTP 429"); + expect(String(failure)).toContain("github.com returned HTTP 503"); + vi.advanceTimersByTime(3_600_000); + expect(managedRuntimeRetryAfterMs(failure)).toBeUndefined(); + }, + ); + + it.each([false, true])( + "exposes the longest outstanding source cooldown (reverse %s)", + async (reverse) => { + vi.setSystemTime(0); + const base = await twoSources(); + const request = reverse ? { ...base, url: base.fallbackUrl, fallbackUrl: base.url } : base; + const fetch = vi.fn( + async (url: URL) => + new Response("busy", { + status: 429, + headers: { + "retry-after": url.hostname === "downloads.scientfactory.com" ? "7200" : "3600", + }, + }), + ); + vi.stubGlobal("fetch", fetch); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause); + expect(managedRuntimeRetryAfterMs(failure)).toBe(7_200_000); + expect(fetch).toHaveBeenCalledTimes(2); + expect(NodeTimersPromises.setTimeout).not.toHaveBeenCalled(); + }, + ); + + it("revisits a cooled source only once its deadline has elapsed", async () => { + vi.setSystemTime(0); + using _random = vi.spyOn(Math, "random").mockReturnValue(0.5); const request = await twoSources(); - const limits: AbortController[] = []; - using _timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation(() => { - const limit = new AbortController(); - limits.push(limit); - return limit.signal; - }); const fetch = vi .fn() - .mockImplementationOnce( - (_url: URL, init: RequestInit) => - new Promise((_resolve, reject) => { - init.signal!.addEventListener("abort", () => reject(init.signal!.reason)); - // Scient's copy is still busy when its own ten minutes end. - limits[0]!.abort(new DOMException("time limit", "TimeoutError")); - }), - ) + .mockResolvedValueOnce(new Response("busy", { status: 429, headers: { "retry-after": "7" } })) + .mockResolvedValueOnce(new Response("busy", { status: 503 })) + .mockResolvedValueOnce(new Response("busy", { status: 503 })) .mockResolvedValueOnce(ok()); vi.stubGlobal("fetch", fetch); - await downloadManagedRuntime(request); - expect(urls(fetch)).toEqual([mirror, request.fallbackUrl]); - expect(limits).toHaveLength(2); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + await expect(downloadManagedRuntime(request)).resolves.toMatchObject({ + tries: 4, + source: "primary", + }); + expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl, mirror]); + expect(vi.mocked(NodeTimersPromises.setTimeout).mock.calls.map(([delay]) => delay)).toEqual([ + 2_000, 5_000, + ]); }); it("fails only when both copies fail, and names both", async () => { @@ -427,9 +519,11 @@ describe("resumable downloads", () => { return partialContent(2, "llo"); }); vi.stubGlobal("fetch", fetch); - await expect(downloadManagedRuntime(request)).resolves.toEqual({ + await expect(downloadManagedRuntime(request)).resolves.toMatchObject({ resumedBytes: 2, source: "primary", + tries: 2, + progressPercent: 100, }); expect(await NodeFSP.readFile(request.destination, "utf8")).toBe(body); expect(await NodeFSP.readdir(NodePath.dirname(request.partialPath))).toEqual([]); @@ -511,36 +605,36 @@ describe("resumable downloads", () => { return partialContent(2, "llo"); }); vi.stubGlobal("fetch", fetch); - await expect(downloadManagedRuntime(request)).resolves.toEqual({ + await expect(downloadManagedRuntime(request)).resolves.toMatchObject({ resumedBytes: 2, source: "fallback", }); }); - it("keeps the bytes when the time limit ends and says trying again continues", async () => { + it("says how far a failed download got, and that a later try may finish it", async () => { const request = await resumable(); - const limit = new AbortController(); - using _timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(limit.signal); - await NodeFSP.mkdir(NodePath.dirname(request.partialPath), { recursive: true }); - await NodeFSP.writeFile(request.partialPath, "he"); - const fetch = vi.fn().mockImplementationOnce( - (_url: URL, init: RequestInit) => - new Promise((_resolve, reject) => { - init.signal!.addEventListener("abort", () => reject(init.signal!.reason)); - limit.abort(new DOMException("time limit", "TimeoutError")); - }), - ); + const reset = Object.assign(new Error("private detail"), { code: "ECONNRESET" }); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response("he")) + .mockRejectedValue(new TypeError("fetch failed", { cause: reset })); vi.stubGlobal("fetch", fetch); - const failure = await downloadManagedRuntime(request).then( - () => expect.unreachable("the download should stop"), + const onFailedTry = vi.fn(); + const failure = await downloadManagedRuntime({ ...request, onFailedTry }).then( + () => expect.unreachable("the download should fail"), (cause: unknown) => cause, ); - expect(failure).toBeInstanceOf(ManagedRuntimeFileError); - expect((failure as Error).message).toBe( - "The download took longer than 10 minutes. Scient kept what it downloaded; trying again continues from there.", - ); - expect(managedRuntimeFailureClass(failure)).toBe("timeout"); - expect(await NodeFSP.readFile(request.partialPath, "utf8")).toBe("he"); + expect(onFailedTry).toHaveBeenCalledTimes(5); + expect(onFailedTry.mock.calls[0]![0]).toMatchObject({ tries: 1, networkKind: "cut_off" }); + expect(onFailedTry.mock.calls.at(-1)![0]).toMatchObject({ + resumedBytes: 2, + source: "primary", + tries: 5, + progressPercent: 40, + networkKind: "reset", + }); + expect(managedRuntimeFailureClass(failure)).toBe("network"); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); }); it("starts the retries afresh while a slow line keeps bringing data", async () => { @@ -571,6 +665,73 @@ describe("resumable downloads", () => { // Nine tries, more than the five a failing server gets. expect(fetch).toHaveBeenCalledTimes(9); }); + + it("bounds retries when a server repeatedly replaces the same retained prefix", async () => { + const request = await resumable(); + const content = Buffer.alloc(2 * 1024 * 1024, 7); + const fetch = vi.fn(async () => new Response(content.subarray(0, 1024 * 1024))); + vi.stubGlobal("fetch", fetch); + await expect( + downloadManagedRuntime({ + ...request, + expectedSize: content.length, + checksum: { + algorithm: "sha256", + digest: NodeCrypto.createHash("sha256").update(content).digest("hex"), + }, + }), + ).rejects.toThrow("cut off on all 5 tries"); + expect(fetch).toHaveBeenCalledTimes(5); + expect((await NodeFSP.stat(request.partialPath)).size).toBe(1024 * 1024); + }); +}); + +describe("downloads without an overall time limit", () => { + // regression: slow downloads failed after ten minutes while data still arrived. + it("keeps a slow download going past ten minutes while data arrives", async () => { + const request = { ...(await input()), expectedSize: 36 * 1024 }; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); + // A limit built on AbortSignal.timeout would otherwise ignore the test clock. + using _timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation((ms) => { + const limit = new AbortController(); + setTimeout(() => limit.abort(new DOMException("time limit", "TimeoutError")), ms); + return limit.signal; + }); + let body!: ReadableStreamDefaultController; + const response = new Response( + new ReadableStream({ start: (controller) => (body = controller) }), + { + headers: { "content-length": String(request.expectedSize) }, + }, + ); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response)); + let awaited: { readonly bytes: number; readonly resolve: () => void } | undefined; + const download = downloadManagedRuntime({ + ...request, + onProgress: (bytes) => { + if (awaited && bytes >= awaited.bytes) awaited.resolve(); + }, + }); + const ended = download.then( + () => undefined, + () => undefined, + ); + // Opening the file takes real time, so the clock moves on only once each KiB has arrived. + const written = (bytes: number) => + Promise.race([new Promise((resolve) => (awaited = { bytes, resolve })), ended]); + // 1 KiB every 20 seconds, inside the 30-second stall bound: twelve minutes in all. + for (let chunk = 1; chunk <= 36; chunk += 1) { + await vi.advanceTimersByTimeAsync(20_000); + body.enqueue(new Uint8Array(1024).fill(7)); + await written(chunk * 1024); + } + body.close(); + await expect(download).resolves.toMatchObject({ + tries: 1, + progressPercent: 100, + bytesPerSecond: Math.round(request.expectedSize / 720), + }); + }); }); describe("managed runtime failure classes", () => { @@ -607,6 +768,35 @@ describe("managed runtime failure classes", () => { ); }); + it.each([ + ["HTTP 429 on every try", () => new Response("slow down", { status: 429 }), true], + ["HTTP 501", () => new Response("never", { status: 501 }), false], + ["a missing file", () => new Response("missing", { status: 404 }), false], + [ + "an intercepted certificate", + () => Promise.reject(fetchFailure("SELF_SIGNED_CERT_IN_CHAIN")), + false, + ], + ])("says whether a later try may fix %s", async (_case, response, retryable) => { + vi.stubGlobal("fetch", vi.fn().mockImplementation(response)); + const failure = await failureOf(downloadManagedRuntime(await input())); + expect(managedRuntimeFailureRetryable(failure)).toBe(retryable); + }); + + it("reads the remaining bounded cooldown through wrappers without hiding a longer inner one", () => { + vi.setSystemTime(0); + const inner = new ManagedRuntimeFileError("busy", { retryAfterMs: 99_999_999 }); + const wrapped = new ManagedRuntimeFileError("outer", { cause: inner, retryAfterMs: 1_000 }); + expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: wrapped }))).toBe(14_400_000); + vi.advanceTimersByTime(60_000); + expect(inner.retryAfterMs).toBe(14_340_000); + expect(wrapped.retryAfterMs).toBeUndefined(); + expect(managedRuntimeRetryAfterMs(wrapped)).toBe(14_340_000); + vi.advanceTimersByTime(14_340_000); + expect(managedRuntimeRetryAfterMs(wrapped)).toBeUndefined(); + expect(managedRuntimeRetryAfterMs(new Error("Retry-After: 3600"))).toBeUndefined(); + }); + it("classifies a damaged download, cancellation and local disk errors without reading messages", async () => { const request = await input(); await NodeFSP.writeFile(request.destination, "ko"); @@ -634,6 +824,8 @@ describe("managed runtime failure classes", () => { new Error("Scient could not install it.", { cause: Object.assign(new Error("x"), { code }) }); expect(managedRuntimeFailureClass(wrapped("ENOSPC"))).toBe("disk_full"); expect(managedRuntimeFailureClass(wrapped("EACCES"))).toBe("permission"); + for (const local of [damaged, wrapped("ENOSPC"), wrapped("EACCES")]) + expect(managedRuntimeFailureRetryable(local)).toBe(false); expect( managedRuntimeFailureClass(new ManagedRuntimeFileError("Unsafe archive path"), "installing"), ).toBe("extract"); diff --git a/packages/scient-provider-runtime/src/runtimeFiles.ts b/packages/scient-provider-runtime/src/runtimeFiles.ts index 432622ce19..bfb3a7018a 100644 --- a/packages/scient-provider-runtime/src/runtimeFiles.ts +++ b/packages/scient-provider-runtime/src/runtimeFiles.ts @@ -27,7 +27,9 @@ function isTransientDownloadStatus(status: number): boolean { * minute in all, spread so many installs don't retry at the same moment. */ const RETRY_DELAYS_MS = [2_000, 5_000, 15_000, 30_000] as const; -const MAX_RETRY_AFTER_MS = 60_000; +const MAX_RETRY_AFTER_MS = 4 * 60 * 60 * 1_000; +// Covers all four jittered backoffs; longer server waits belong to durable recovery. +const LOCAL_RETRY_WAIT_BUDGET_MS = 65_000; /** Largest artifact the installer downloads; release discovery rejects larger ones. */ export const MANAGED_RUNTIME_MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024; const MAX_DOWNLOAD_BYTES = MANAGED_RUNTIME_MAX_DOWNLOAD_BYTES; @@ -38,7 +40,6 @@ const DEFAULT_EXTRACTION_LIMITS: ManagedRuntimeExtractionLimits = { maxExpandedBytes: 512 * 1024 * 1024, maxEntries: 32, }; -const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1_000; const DOWNLOAD_IDLE_TIMEOUT_MS = 30 * 1_000; /** A failed try that still brought this much of a resumable download starts the retries afresh. */ const RESUME_PROGRESS_BYTES = 1024 * 1024; @@ -62,16 +63,60 @@ export type ManagedRuntimeFailureClass = // @effect-diagnostics-next-line extendsNativeError:off - @scientfactory/provider-runtime has no Effect runtime; callers catch it as a native error. export class ManagedRuntimeFileError extends Error { readonly failureClass: ManagedRuntimeFailureClass | undefined; + /** Whether trying the same change again later may succeed, when the class alone can't say. */ + readonly retryable: boolean | undefined; + readonly #retryNotBefore: number | undefined; + /** Remaining server cooldown, bounded to four hours; absent once it has elapsed. */ + get retryAfterMs(): number | undefined { + if (this.#retryNotBefore === undefined) return undefined; + const remaining = Math.min(MAX_RETRY_AFTER_MS, this.#retryNotBefore - Date.now()); + return remaining > 0 ? remaining : undefined; + } constructor( message: string, - options?: ErrorOptions & { readonly failureClass?: ManagedRuntimeFailureClass }, + options?: ErrorOptions & { + readonly failureClass?: ManagedRuntimeFailureClass; + readonly retryable?: boolean; + readonly retryAfterMs?: number; + }, ) { super(message, options); this.name = "ManagedRuntimeFileError"; this.failureClass = options?.failureClass; + this.retryable = options?.retryable; + const delay = options?.retryAfterMs; + this.#retryNotBefore = + delay !== undefined && Number.isFinite(delay) && delay > 0 + ? Date.now() + Math.min(delay, MAX_RETRY_AFTER_MS) + : undefined; } } +/** How a try's connection failed, from its error code; never its message. */ +export type ManagedRuntimeNetworkKind = + | "dns" + | "refused" + | "reset" + | "unreachable" + | "connect_timeout" + | "stalled" + | "cut_off" + | "other"; + +const NETWORK_KINDS: Readonly> = { + ENOTFOUND: "dns", + EAI_AGAIN: "dns", + ECONNREFUSED: "refused", + ECONNRESET: "reset", + ECONNABORTED: "reset", + EPIPE: "reset", + UND_ERR_SOCKET: "reset", + ENETUNREACH: "unreachable", + EHOSTUNREACH: "unreachable", + ETIMEDOUT: "connect_timeout", + UND_ERR_CONNECT_TIMEOUT: "connect_timeout", +}; + function assertAllowedHttpsUrl(url: URL, allowedHosts: ReadonlySet): void { if (url.protocol !== "https:") { throw new ManagedRuntimeFileError("Managed runtime downloads must use HTTPS."); @@ -99,11 +144,12 @@ class TransientDownloadFailure extends Error { readonly reason: string; readonly retryAfterMs: number | null; readonly status: number | null; + readonly networkKind: ManagedRuntimeNetworkKind | undefined; constructor( kind: TransientDownloadFailure["kind"], reason: string, retryAfterMs: number | null = null, - options?: ErrorOptions & { readonly status?: number }, + options?: ErrorOptions & { readonly status?: number; readonly code?: string | undefined }, ) { super(reason, options); this.name = "TransientDownloadFailure"; @@ -111,6 +157,14 @@ class TransientDownloadFailure extends Error { this.reason = reason; this.retryAfterMs = retryAfterMs; this.status = options?.status ?? null; + this.networkKind = + kind === "stalled" + ? "stalled" + : kind === "cut-off" + ? "cut_off" + : kind === "connection" + ? ((options?.code === undefined ? undefined : NETWORK_KINDS[options.code]) ?? "other") + : undefined; } } @@ -160,12 +214,7 @@ export function managedRuntimeFailureClass( cause: unknown, stage?: string, ): ManagedRuntimeFailureClass { - const chain: unknown[] = []; - for (let current = cause; current !== undefined && chain.length < 8;) { - chain.push(current); - current = current instanceof Error ? current.cause : undefined; - } - const errors = chain.filter((entry): entry is Error => entry instanceof Error); + const errors = causeChain(cause); for (const error of errors) { if (error instanceof ManagedRuntimeFileError && error.failureClass !== undefined) return error.failureClass; @@ -184,10 +233,49 @@ export function managedRuntimeFailureClass( return "other"; } +function causeChain(cause: unknown): ReadonlyArray { + const errors: Error[] = []; + for (let current = cause; current instanceof Error && errors.length < 8; current = current.cause) + errors.push(current); + return errors; +} + +const RETRYABLE_FAILURE_CLASSES = new Set([ + "network", + "timeout", + "busy", +]); + +/** + * Whether the same change may succeed if tried again later, from error types: a server that stayed + * busy (HTTP 408, 429, most 5xx) or a lost connection may; a 501, certificate, checksum, disk or + * permission failure will not. + */ +export function managedRuntimeFailureRetryable(cause: unknown, stage?: string): boolean { + for (const error of causeChain(cause)) { + if (error instanceof ManagedRuntimeFileError && error.retryable !== undefined) + return error.retryable; + if (error instanceof TransientDownloadFailure) return true; + } + return RETRYABLE_FAILURE_CLASSES.has(managedRuntimeFailureClass(cause, stage)); +} + +/** Remaining server cooldown through wrapped failures; contains no transport details. */ +export function managedRuntimeRetryAfterMs(cause: unknown): number | undefined { + let remaining = 0; + for (const error of causeChain(cause)) { + if (error instanceof ManagedRuntimeFileError) + remaining = Math.max(remaining, error.retryAfterMs ?? 0); + } + return remaining > 0 ? remaining : undefined; +} + function retryAfterMs(header: string | null): number | null { if (!header) return null; const seconds = Number(header); - const ms = Number.isFinite(seconds) ? seconds * 1_000 : Date.parse(header) - Date.now(); + const ms = Number.isFinite(seconds) + ? Math.min(seconds, MAX_RETRY_AFTER_MS / 1_000) * 1_000 + : Date.parse(header) - Date.now(); return Number.isFinite(ms) ? Math.min(Math.max(ms, 0), MAX_RETRY_AFTER_MS) : null; } @@ -209,7 +297,7 @@ function connectionFailure(cause: unknown): TransientDownloadFailure | ManagedRu "connection", code ? `connection failed, ${code}` : "connection failed", null, - { cause }, + { cause, code }, ); } @@ -283,6 +371,8 @@ interface ManagedRuntimeDownloadInput { * before going back to a source that already failed. */ readonly fallbackUrl?: string | undefined; + /** Absolute not-before times by HTTPS origin, shared across compressed and raw copies. */ + readonly sourceCooldowns?: Map | undefined; /** Checked after every try, so a damaged copy counts as that source failing. */ readonly checksum?: ManagedRuntimeChecksum | undefined; readonly destination: string; @@ -291,21 +381,38 @@ interface ManagedRuntimeDownloadInput { * Name it by the release digest: both copies serve the same bytes. Needs `checksum`. */ readonly partialPath?: string | undefined; - /** Per copy; `null` waits as long as data keeps arriving. Ten minutes by default. */ - readonly timeLimitMs?: number | null | undefined; readonly allowedHosts: ReadonlyArray; readonly expectedSize: number; readonly signal: AbortSignal; + /** From the bytes kept by an earlier try, when a try starts, then as bytes arrive. */ readonly onProgress?: (downloaded: number, total: number) => void; readonly onRetry?: (retry: ManagedRuntimeDownloadRetry) => void; + /** After each failed try, before Scient waits or gives up, so a failed download says how far it got. */ + readonly onFailedTry?: (download: ManagedRuntimeDownloadResult) => void; } -/** What a finished download continued from, for analytics. */ +/** How a download went up to its latest try, for analytics. */ export interface ManagedRuntimeDownloadResult { - /** Bytes already on disk when the successful try started; 0 for a fresh download. */ + /** Bytes already on disk when the latest try started; 0 for a fresh download. */ readonly resumedBytes: number; - /** `fallback` when the second copy served the file. */ + /** The copy the latest try used; `fallback` is the second. */ readonly source: "primary" | "fallback"; + /** Tries made, the first included. */ + readonly tries: number; + /** How much of the file the latest try had when it ended, 0 to 100. */ + readonly progressPercent: number; + /** Received bytes per second of request time, kept bytes and waits excluded; absent with none. */ + readonly bytesPerSecond?: number; + /** How the latest try's connection failed. */ + readonly networkKind?: ManagedRuntimeNetworkKind; +} + +/** Shared by the tries of one download. */ +interface DownloadTransfer { + resumedBytes: number; + downloadedBytes: number; + receivedBytes: number; + activeMs: number; } function describeSourceFailure(failure: Error): string { @@ -323,10 +430,10 @@ function describeSourceFailure(failure: Error): string { } /** - * Downloads one reviewed artifact, retrying temporary failures for about a - * minute. With a fallback copy, any failure of one copy moves to the other. - * Cancellation, the time limit and local disk errors always stay final. - * With `partialPath`, a failed try keeps its bytes and the next try asks only for the rest. + * Downloads one reviewed artifact, retrying temporary failures (30 seconds without data among them) + * for about a minute; a fallback copy takes over from a failing one. No overall time limit: a slow + * download goes on while data arrives. Cancellation and local disk errors stay final. With + * `partialPath`, a failed try keeps its bytes and the next asks only for the rest. */ export async function downloadManagedRuntime( input: ManagedRuntimeDownloadInput, @@ -335,116 +442,128 @@ export async function downloadManagedRuntime( const allowedHosts = new Set(input.allowedHosts.map((host) => host.toLowerCase())); const expected = input.checksum ? expectedDigest(input.checksum) : undefined; const partialPath = input.checksum ? input.partialPath : undefined; - const timeLimitMs = input.timeLimitMs === undefined ? DOWNLOAD_TIMEOUT_MS : input.timeLimitMs; - // Each copy gets the whole time limit, so a slow copy never uses up the other's. - const deadlines = new Map(); - const deadline = (source: string): AbortSignal => { - const existing = deadlines.get(source); - if (existing) return existing; - const created = - timeLimitMs === null ? new AbortController().signal : AbortSignal.timeout(timeLimitMs); - deadlines.set(source, created); - return created; + const transfer: DownloadTransfer = { + resumedBytes: 0, + downloadedBytes: 0, + receivedBytes: 0, + activeMs: 0, }; - const waitSignal = - sources.length === 1 ? AbortSignal.any([input.signal, deadline(input.url)]) : input.signal; const lastFailure = new Map(); let usable = [...sources]; - let round = [...usable]; + const sourceCooldowns = input.sourceCooldowns ?? new Map(); + const notBefore = (source: string) => sourceCooldowns.get(new URL(source).origin) ?? 0; + const remainingCooldown = () => + Math.max(0, ...sources.map((source) => notBefore(source) - Date.now())); + let waitBudgetMs = LOCAL_RETRY_WAIT_BUDGET_MS; + let retryProgressBytes = 0; + let cause: unknown; let tries = 0; + const facts = (source: string, cause?: unknown): ManagedRuntimeDownloadResult => { + const networkKind = cause instanceof TransientDownloadFailure ? cause.networkKind : undefined; + return { + resumedBytes: transfer.resumedBytes, + source: source === input.url ? "primary" : "fallback", + tries, + progressPercent: Math.min( + 100, + Math.floor((transfer.downloadedBytes / input.expectedSize) * 100), + ), + ...(transfer.receivedBytes > 0 && transfer.activeMs > 0 + ? { bytesPerSecond: Math.round((transfer.receivedBytes * 1_000) / transfer.activeMs) } + : {}), + ...(networkKind ? { networkKind } : {}), + }; + }; + const exhausted = async (): Promise => { + input.signal.throwIfAborted(); + const kept = + partialPath && (await partialDownloadSize(partialPath)) > 0 + ? " Scient kept what it downloaded; trying again continues from there." + : ""; + const retryAfterMs = remainingCooldown(); + const retryable = + retryAfterMs > 0 || + [...lastFailure.values()].some((failure) => failure instanceof TransientDownloadFailure); + if (sources.length === 1 && cause instanceof TransientDownloadFailure) { + throw new ManagedRuntimeFileError(`${exhaustedMessage(cause, tries)}${kept}`, { + cause, + failureClass: transientFailureClass(cause), + retryable, + retryAfterMs, + }); + } + const details = sources + .flatMap((candidate) => { + const failure = lastFailure.get(candidate); + return failure ? [`${new URL(candidate).hostname} ${describeSourceFailure(failure)}`] : []; + }) + .join(" "); + throw new ManagedRuntimeFileError( + `Scient couldn't download it from ${sources.length > 1 ? "either source" : "the source"} after ${tries} tries. ${details}${kept}`, + { + cause, + failureClass: managedRuntimeFailureClass(cause), + retryable, + retryAfterMs, + }, + ); + }; for (let retry = 0; ;) { - const source = round.shift()!; - tries += 1; - const received = { bytes: 0 }; - let cause: unknown; - try { - const attempt = await downloadOnce( - input, - source, - allowedHosts, - deadline(source), - partialPath, - received, - ); - let resumedBytes = attempt.resumedBytes; - if (expected !== undefined && attempt.digest !== expected) { - await removePartialDownload(partialPath ?? input.destination); - // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once. - if (resumedBytes === 0) throw checksumMismatch(); - const fresh = await downloadOnce( - input, - source, - allowedHosts, - deadline(source), - partialPath, - received, - ); - if (fresh.digest !== expected) { + for (const source of usable) { + input.signal.throwIfAborted(); + if (notBefore(source) > Date.now()) continue; + tries += 1; + try { + const attempt = await downloadOnce(input, source, allowedHosts, partialPath, transfer); + if (expected !== undefined && attempt.digest !== expected) { await removePartialDownload(partialPath ?? input.destination); - throw checksumMismatch(); + // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once. + if (attempt.resumedBytes === 0) throw checksumMismatch(); + const fresh = await downloadOnce(input, source, allowedHosts, partialPath, transfer); + if (fresh.digest !== expected) { + await removePartialDownload(partialPath ?? input.destination); + throw checksumMismatch(); + } } - resumedBytes = 0; - } - if (partialPath) { - await NodeFSP.rename(partialPath, input.destination); - await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch(() => undefined); + if (partialPath) { + await NodeFSP.rename(partialPath, input.destination); + await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch( + () => undefined, + ); + } + return facts(source); + } catch (caught) { + cause = caught; } - return { resumedBytes, source: source === input.url ? "primary" : "fallback" }; - } catch (caught) { - cause = caught; - } - // With a second copy, a problem with this one, running out of time - // included, only rules this copy out. Cancellation still ends everything. - const outOfTime = deadline(source).aborted && !input.signal.aborted; - const ruledOut = sources.length > 1 && (cause instanceof ManagedRuntimeFileError || outOfTime); - if (!(cause instanceof TransientDownloadFailure) && !ruledOut) { - if (outOfTime && partialPath && (await partialDownloadSize(partialPath)) > 0) { - throw new ManagedRuntimeFileError( - `The download took longer than ${describeTimeLimit(timeLimitMs)}. Scient kept what it downloaded; trying again continues from there.`, - { cause, failureClass: "timeout" }, + input.signal.throwIfAborted(); + if (cause instanceof TransientDownloadFailure && cause.retryAfterMs !== null) { + sourceCooldowns.set( + new URL(source).origin, + Math.max(notBefore(source), Date.now() + cause.retryAfterMs), ); } - throw cause; + input.onFailedTry?.(facts(source, cause)); + // With a second copy, a problem with this one only rules this copy out. + const ruledOut = sources.length > 1 && cause instanceof ManagedRuntimeFileError; + if (!(cause instanceof TransientDownloadFailure) && !ruledOut) throw cause; + lastFailure.set(source, cause as Error); + if (ruledOut) usable = usable.filter((candidate) => candidate !== source); } - lastFailure.set( - source, - outOfTime - ? new Error(`took longer than ${describeTimeLimit(timeLimitMs)}.`) - : (cause as Error), - ); - if (ruledOut) { - usable = usable.filter((candidate) => candidate !== source); - round = round.filter((candidate) => candidate !== source); + // Only a growing retained prefix replenishes retries; a server may ignore Range. + if (partialPath && transfer.downloadedBytes - retryProgressBytes >= RESUME_PROGRESS_BYTES) { + retry = 0; + retryProgressBytes = transfer.downloadedBytes; + waitBudgetMs = LOCAL_RETRY_WAIT_BUDGET_MS; } - if (usable.length > 0 && round.length > 0) continue; - // Steady progress on a slow line is not a failing server; keep going while it lasts. - if (received.bytes >= RESUME_PROGRESS_BYTES) retry = 0; const baseDelay = RETRY_DELAYS_MS[retry]; - if (usable.length === 0 || baseDelay === undefined) { - const kept = - partialPath && (await partialDownloadSize(partialPath)) > 0 - ? " Scient kept what it downloaded; trying again continues from there." - : ""; - if (sources.length === 1) { - throw new ManagedRuntimeFileError( - `${exhaustedMessage(cause as TransientDownloadFailure, tries)}${kept}`, - { cause, failureClass: transientFailureClass(cause as TransientDownloadFailure) }, - ); - } - const details = sources - .map( - (candidate) => - `${new URL(candidate).hostname} ${describeSourceFailure(lastFailure.get(candidate)!)}`, - ) - .join(" "); - throw new ManagedRuntimeFileError( - `Scient couldn't download it from either source after ${tries} tries. ${details}${kept}`, - { cause, failureClass: outOfTime ? "timeout" : managedRuntimeFailureClass(cause) }, - ); - } - const delayMs = - (cause instanceof TransientDownloadFailure ? cause.retryAfterMs : null) ?? - Math.round(baseDelay * (0.8 + Math.random() * 0.4)); + if (usable.length === 0 || baseDelay === undefined) return exhausted(); + const delayMs = Math.max( + Math.round(baseDelay * (0.8 + Math.random() * 0.4)), + Math.min(...usable.map((source) => Math.max(0, notBefore(source) - Date.now()))), + ); + // Long server cooldowns belong to durable recovery, not an open install dialog. + if (delayMs > waitBudgetMs) return exhausted(); + waitBudgetMs -= delayMs; retry += 1; input.onRetry?.({ retry, @@ -452,16 +571,10 @@ export async function downloadManagedRuntime( reason: cause instanceof TransientDownloadFailure ? cause.reason : "download failed", delayMs, }); - await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: waitSignal }); - round = [...usable]; + await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: input.signal }); } } -function describeTimeLimit(timeLimitMs: number | null): string { - const minutes = Math.round((timeLimitMs ?? 0) / 60_000); - return minutes === 1 ? "1 minute" : `${minutes} minutes`; -} - function checksumMismatch(): ManagedRuntimeFileError { return new ManagedRuntimeFileError("Managed runtime verification failed: checksum mismatch.", { failureClass: "checksum", @@ -535,9 +648,8 @@ async function downloadOnce( input: ManagedRuntimeDownloadInput, url: string, allowedHosts: ReadonlySet, - timeoutSignal: AbortSignal, partialPath: string | undefined, - received: { bytes: number }, + transfer: DownloadTransfer, ): Promise<{ readonly resumedBytes: number; readonly digest: string | undefined }> { const target = partialPath ?? input.destination; const newHash = () => @@ -549,10 +661,10 @@ async function downloadOnce( have = 0; } if (have > 0 && hash) await hashPrefix(target, have, hash); - if (have === input.expectedSize) { - input.onProgress?.(have, input.expectedSize); - return { resumedBytes: have, digest: hash?.digest("hex") }; - } + transfer.resumedBytes = have; + transfer.downloadedBytes = have; + input.onProgress?.(have, input.expectedSize); + if (have === input.expectedSize) return { resumedBytes: have, digest: hash?.digest("hex") }; const idleController = new AbortController(); let idleTimer: ReturnType | undefined; @@ -560,12 +672,12 @@ async function downloadOnce( if (idleTimer) clearTimeout(idleTimer); idleTimer = setTimeout(() => idleController.abort(), DOWNLOAD_IDLE_TIMEOUT_MS); }; - const signal = AbortSignal.any([input.signal, timeoutSignal, idleController.signal]); - // A stall is worth another try; cancellation and the overall limit are not. + const signal = AbortSignal.any([input.signal, idleController.signal]); + // A stall is worth another try; cancellation is not. const classify = (cause: unknown, fromNetwork: boolean): unknown => { if (cause instanceof ManagedRuntimeFileError || cause instanceof TransientDownloadFailure) return cause; - if (input.signal.aborted || timeoutSignal.aborted) return cause; + if (input.signal.aborted) return cause; if (idleController.signal.aborted) return new TransientDownloadFailure("stalled", "download stalled", null, { cause }); return fromNetwork ? connectionFailure(cause) : cause; @@ -585,6 +697,7 @@ async function downloadOnce( }); }; resetIdleTimer(); + const startedAt = Date.now(); try { let response = await request(have); @@ -595,6 +708,8 @@ async function downloadOnce( response = await request(0); } have = 0; + transfer.resumedBytes = 0; + transfer.downloadedBytes = 0; hash = newHash(); } if (!response.ok || !response.body) { @@ -628,7 +743,8 @@ async function downloadOnce( const source = NodeStream.Readable.fromWeb(response.body as never); source.on("data", (chunk: Buffer) => { downloaded += chunk.byteLength; - received.bytes += chunk.byteLength; + transfer.downloadedBytes = downloaded; + transfer.receivedBytes += chunk.byteLength; hash?.update(chunk); resetIdleTimer(); if (downloaded > MAX_DOWNLOAD_BYTES || downloaded > input.expectedSize) { @@ -665,6 +781,7 @@ async function downloadOnce( return { resumedBytes: have, digest: hash?.digest("hex") }; } finally { if (idleTimer) clearTimeout(idleTimer); + transfer.activeMs += Date.now() - startedAt; } } diff --git a/scripts/create-scient-release-handoff.ts b/scripts/create-scient-release-handoff.ts index db009b7822..28e6e1aaad 100644 --- a/scripts/create-scient-release-handoff.ts +++ b/scripts/create-scient-release-handoff.ts @@ -10,6 +10,8 @@ import { scientReleaseRepository, } from "@t3tools/shared/scientRelease"; +import { readWindowsSigningAttestation } from "./windows-publication-signing.ts"; + interface ReleaseAsset { readonly name: string; readonly size: number; @@ -45,6 +47,12 @@ export function createScientReleaseHandoff(args: ReadonlyArray): string const sourceSha = value(args, "--source-sha"); const sourceTree = value(args, "--source-tree"); const output = NodePath.resolve(value(args, "--output")); + // Later publications read this to learn which publisher installed Windows clients pin. + const windows = args.includes("--windows-signing") + ? readWindowsSigningAttestation( + JSON.parse(NodeFS.readFileSync(value(args, "--windows-signing"), "utf8")) as unknown, + ) + : null; const excluded = new Set([NodePath.basename(output), "SHA256SUMS.txt"]); const assets: ReleaseAsset[] = NodeFS.readdirSync(assetsDir) .filter((name) => !excluded.has(name)) @@ -70,6 +78,13 @@ export function createScientReleaseHandoff(args: ReadonlyArray): string version, tag: `v${version}`, distribution: { repository: distributionRepository, channel }, + ...(windows === null + ? {} + : { + windows: windows.signed + ? { signed: true, publisherName: windows.publisherName } + : { signed: false }, + }), source: { repository: sourceRepository, commit: sourceSha, diff --git a/scripts/scient-beta-publication.test.ts b/scripts/scient-beta-publication.test.ts index f4a1e0270c..bb2cabb076 100644 --- a/scripts/scient-beta-publication.test.ts +++ b/scripts/scient-beta-publication.test.ts @@ -99,6 +99,7 @@ else if (args.includes('repos/ScientFactory/scient-desktop-beta/releases?per_pag { channel: "beta", failure: "none" }, { channel: "beta", failure: "corrupt-upload" }, { channel: "beta", failure: "stable-advanced" }, + { channel: "beta", failure: "publisher-pinned" }, ])( "publishes $channel only into its owning repository, failure=$failure", ({ channel, failure }) => { @@ -110,6 +111,21 @@ else if (args.includes('repos/ScientFactory/scient-desktop-beta/releases?per_pag try { NodeFS.mkdirSync(bin); NodeFS.mkdirSync(NodePath.join(root, "release-assets")); + NodeFS.mkdirSync(NodePath.join(root, "scripts")); + NodeFS.copyFileSync( + NodePath.join(import.meta.dirname, "windows-publication-signing.ts"), + NodePath.join(root, "scripts/windows-publication-signing.ts"), + ); + NodeFS.writeFileSync( + NodePath.join(root, "release-assets/scient-release-handoff.json"), + JSON.stringify({ + version, + windows: + channel === "beta" + ? { signed: false } + : { signed: true, publisherName: "ScientFactory Ltd" }, + }), + ); NodeFS.writeFileSync( NodePath.join(root, "release-assets/fixture.txt"), "immutable fixture", @@ -122,17 +138,32 @@ const path = require('node:path'); const args = process.argv.slice(2); fs.appendFileSync(process.env.CALLS, JSON.stringify(args) + '\\n'); if (args[0] === 'api') { - if (args[1].endsWith('/releases/latest')) process.stdout.write(process.env.RELEASE_CHANNEL === 'beta' ? (process.env.FAILURE === 'stable-advanced' ? 'v0.6.23' : 'v0.6.22') : process.env.RELEASE_TAG); - else if (args[1] === 'repos/ScientFactory/scient-desktop-beta') process.stdout.write('main'); + const endpoint = args.find((arg) => arg.startsWith('repos/')); + if (endpoint?.endsWith('/releases/latest')) { + const tag = process.env.RELEASE_CHANNEL === 'beta' + ? (process.env.FAILURE === 'stable-advanced' ? 'v0.6.23' : 'v0.6.22') + : process.env.RELEASE_TAG; + process.stdout.write(args.includes('--jq') ? tag : JSON.stringify({ tag_name: 'v0.6.22' })); + } else if (endpoint === 'repos/ScientFactory/scient-desktop-beta/releases?per_page=100') { + process.stdout.write(JSON.stringify([[]])); + } else if (endpoint === 'repos/ScientFactory/scient-desktop-beta') process.stdout.write('main'); else process.exit(2); } else if (args[0] === 'release' && ['create', 'edit'].includes(args[1])) { if (args[1] === 'create') fs.writeFileSync(process.env.NOTES_RECEIPT, fs.readFileSync(args[args.indexOf('--notes-file') + 1])); } else if (args[0] === 'release' && args[1] === 'view') { process.stdout.write(JSON.stringify({ isDraft: false, isPrerelease: true })); } else if (args[0] === 'release' && args[1] === 'download') { - const destination = args[args.indexOf('--dir') + 1]; - for (const name of fs.readdirSync('release-assets')) fs.copyFileSync(path.join('release-assets', name), path.join(destination, name)); - if (process.env.FAILURE === 'corrupt-upload') fs.writeFileSync(path.join(destination, 'fixture.txt'), 'modified upload'); + if (args.includes('--output')) { + process.stdout.write(JSON.stringify({ + windows: process.env.FAILURE === 'publisher-pinned' + ? { signed: true, publisherName: 'ScientFactory Ltd' } + : { signed: false }, + })); + } else { + const destination = args[args.indexOf('--dir') + 1]; + for (const name of fs.readdirSync('release-assets')) fs.copyFileSync(path.join('release-assets', name), path.join(destination, name)); + if (process.env.FAILURE === 'corrupt-upload') fs.writeFileSync(path.join(destination, 'fixture.txt'), 'modified upload'); + } } else process.exit(2); `, { mode: 0o700 }, @@ -155,6 +186,7 @@ process.stdout.write(process.env.SOURCE_SHA + '\\trefs/tags/' + process.env.RELE ...process.env, PATH: `${bin}:${process.env.PATH}`, GH_TOKEN: "synthetic-unused-token", + PIN_READ_TOKEN: "synthetic-read-token", CALLS: calls, FAILURE: failure, NOTES_RECEIPT: NodePath.join(root, "published-notes.md"), @@ -184,11 +216,11 @@ process.stdout.write(process.env.SOURCE_SHA + '\\trefs/tags/' + process.env.RELE ); if (failure !== "none") { expect(result.status).not.toBe(0); - expect(mutations).toHaveLength(failure === "corrupt-upload" ? 1 : 0); + expect(mutations).toHaveLength(failure === "stable-advanced" ? 0 : 1); expect(mutations.some((args) => args[1] === "edit")).toBe(false); return; } - expect(result.status).toBe(0); + expect(result.status, result.stderr.toString()).toBe(0); expect(mutations).toHaveLength(2); for (const args of mutations) expect(args[args.indexOf("--repo") + 1]).toBe(repository); if (channel === "beta") { diff --git a/scripts/scient-release-machinery.test.ts b/scripts/scient-release-machinery.test.ts index 2ddea191ab..11dde615c1 100644 --- a/scripts/scient-release-machinery.test.ts +++ b/scripts/scient-release-machinery.test.ts @@ -143,23 +143,6 @@ describe("Scient release machinery", () => { } }); - it("keeps unsigned Windows publication an explicit exception", () => { - const workflow = NodeFS.readFileSync( - NodePath.join(import.meta.dirname, "../.github/workflows/release.yml"), - "utf8", - ); - assert.include(workflow, "allow_unsigned_windows:"); - assert.include( - workflow, - "allow_unsigned_windows is only valid for an explicitly publishing run.", - ); - assert.include( - workflow, - "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true.", - ); - assert.include(workflow, "Publication requires signed macOS artifacts:"); - }); - // regression: 20117b09c1 fix(release): use Bash for version alignment on all platforms (#44) it("pins Bash for every package-version alignment step", () => { const workflow = NodeFS.readFileSync( diff --git a/scripts/windows-publication-signing.test.ts b/scripts/windows-publication-signing.test.ts new file mode 100644 index 0000000000..88a63f269e --- /dev/null +++ b/scripts/windows-publication-signing.test.ts @@ -0,0 +1,349 @@ +// @effect-diagnostics nodeBuiltinImport:off - Rehearse the publication gate with temporary files and a local gh fixture. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { assert, describe, it } from "@effect/vitest"; + +import { createScientReleaseHandoff } from "./create-scient-release-handoff.ts"; +import { + decideWindowsPublication, + pinnedWindowsPublishers, + readWindowsSigningAttestation, +} from "./windows-publication-signing.ts"; + +const PUBLISHER = "ScientFactory Ltd"; +const signed = { signed: true, publisherName: PUBLISHER } as const; +const unsigned = { signed: false, publisherName: null } as const; + +describe("Windows publication signing gate", () => { + it("keeps today's unsigned exception while no installed client pins a publisher", () => { + // Handoffs published before the signing record pin nothing. + const pinned = pinnedWindowsPublishers([{ version: "0.6.29" }, { windows: { signed: false } }]); + assert.deepStrictEqual(pinned, []); + assert.throws( + () => decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: false }), + /allow_unsigned_windows=true/, + ); + assert.match( + decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: true }).warning ?? "", + /unsigned Windows installer/, + ); + assert.isNull( + decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning, + ); + }); + + it("refuses an unsigned or differently signed installer once any channel pins a publisher", () => { + const pinned = pinnedWindowsPublishers([ + { windows: { signed: false } }, + { windows: { signed: true, publisherName: PUBLISHER } }, + ]); + assert.deepStrictEqual(pinned, [PUBLISHER]); + assert.throws( + () => decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: true }), + /cannot override/, + ); + assert.throws( + () => + decideWindowsPublication({ + candidate: { signed: true, publisherName: "Someone Else" }, + pinned, + allowUnsigned: true, + }), + /signed by 'Someone Else'/, + ); + assert.isNull( + decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning, + ); + const conflictingPins = pinnedWindowsPublishers([ + { windows: { signed: true, publisherName: PUBLISHER } }, + { windows: { signed: true, publisherName: "Different Channel Publisher" } }, + ]); + for (const publisherName of conflictingPins) { + assert.throws( + () => + decideWindowsPublication({ + candidate: { signed: true, publisherName }, + pinned: conflictingPins, + allowUnsigned: true, + }), + /would fail their update signature check/, + ); + } + }); + + it("fails closed on attestations and records it cannot read", () => { + assert.throws( + () => readWindowsSigningAttestation({ platform: "win", arch: "x64", signed: true }), + /must name its publisher/, + ); + assert.throws( + () => + readWindowsSigningAttestation({ + platform: "win", + arch: "x64", + signed: false, + publisherName: PUBLISHER, + }), + /cannot name a publisher/, + ); + assert.throws(() => pinnedWindowsPublishers([{ windows: { signed: "yes" } }])); + }); + + it("records the signing attestation in the handoff the next publication reads", () => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-handoff-")); + try { + const assets = NodePath.join(root, "assets"); + NodeFS.mkdirSync(assets); + NodeFS.writeFileSync(NodePath.join(assets, "Scient-0.6.30-x64.exe"), "installer"); + const attestation = NodePath.join(root, "signing-win-x64.json"); + const output = NodePath.join(assets, "scient-release-handoff.json"); + const create = () => + createScientReleaseHandoff([ + "--assets-dir", + assets, + "--version", + "0.6.30", + "--channel", + "stable", + "--repository", + "ScientFactory/scient-desktop", + "--source-sha", + "a".repeat(40), + "--source-tree", + "b".repeat(40), + "--windows-signing", + attestation, + "--output", + output, + ]); + const pinnedAfter = (record: object) => { + NodeFS.writeFileSync( + attestation, + JSON.stringify({ platform: "win", arch: "x64", ...record }), + ); + create(); + return pinnedWindowsPublishers([JSON.parse(NodeFS.readFileSync(output, "utf8"))]); + }; + + assert.deepStrictEqual(pinnedAfter({ signed: true, publisherName: PUBLISHER }), [PUBLISHER]); + assert.deepStrictEqual(pinnedAfter({ signed: false }), []); + } finally { + NodeFS.rmSync(root, { recursive: true, force: true }); + } + }); +}); + +const stableRepository = "ScientFactory/scient-desktop"; +const betaRepository = "ScientFactory/scient-desktop-beta"; +const stableTag = "v0.6.30"; +const betaTag = "v0.6.31-beta.20261116.1"; +const betaRelease = { tag_name: betaTag, draft: false }; + +function publicationFixture() { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-gate-")); + const bin = NodePath.join(root, "bin"); + const assets = NodePath.join(root, "release-assets"); + const attestation = NodePath.join(assets, "signing-win-x64.json"); + const candidateHandoff = NodePath.join(assets, "scient-release-handoff.json"); + const statePath = NodePath.join(root, "published.json"); + NodeFS.mkdirSync(bin); + NodeFS.mkdirSync(assets); + NodeFS.writeFileSync(NodePath.join(assets, "Scient-0.6.31-x64.exe"), "fixture installer"); + const setCandidate = ( + windows: { signed: boolean; publisherName?: string }, + channel: "stable" | "beta" = "stable", + ) => { + NodeFS.writeFileSync(attestation, JSON.stringify({ platform: "win", arch: "x64", ...windows })); + createScientReleaseHandoff([ + "--assets-dir", + assets, + "--version", + channel === "beta" ? betaTag.slice(1) : "0.6.31", + "--channel", + channel, + "--repository", + stableRepository, + "--source-sha", + "a".repeat(40), + "--source-tree", + "b".repeat(40), + "--windows-signing", + attestation, + "--output", + candidateHandoff, + ]); + }; + const setPublished = (state: { + betaPages?: unknown; + stableHandoff?: unknown; + betaHandoff?: unknown; + failure?: { key: string; status: number }; + }) => NodeFS.writeFileSync(statePath, JSON.stringify(state)); + NodeFS.writeFileSync( + NodePath.join(bin, "gh"), + `#!/usr/bin/env node +const fs = require('node:fs'); +const args = process.argv.slice(2); +const state = JSON.parse(fs.readFileSync(process.env.PUBLISHED_STATE, 'utf8')); +const endpoint = args.find(arg => arg.startsWith('repos/')); +let key; +let result; +if (args[0] === 'api' && endpoint === 'repos/${stableRepository}/releases/latest') { + key = 'stable'; + result = { tag_name: '${stableTag}' }; +} else if (args[0] === 'api' && args.includes('--paginate') && args.includes('--slurp') && endpoint === 'repos/${betaRepository}/releases?per_page=100') { + key = 'beta'; + result = state.betaPages ?? [[]]; +} else if (args[0] === 'release' && args[1] === 'download' && args[args.indexOf('--output') + 1] === '-') { + const repo = args[args.indexOf('--repo') + 1]; + if (repo === '${stableRepository}' && args[2] === '${stableTag}') { + key = 'stable-handoff'; + result = state.stableHandoff ?? { version: '0.6.30' }; + } else if (repo === '${betaRepository}' && args[2] === '${betaTag}') { + key = 'beta-handoff'; + result = state.betaHandoff ?? { windows: { signed: true, publisherName: '${PUBLISHER}' } }; + } +} +if (!key) process.exit(2); +if (state.failure?.key === key) { + process.stderr.write('HTTP ' + state.failure.status + '\\n'); + process.exit(1); +} +process.stdout.write(JSON.stringify(result)); +`, + { mode: 0o700 }, + ); + setCandidate({ signed: false }); + setPublished({}); + return { + setCandidate, + setPublished, + candidateHandoff, + attestation, + run(phase: "assembly" | "publication", allowUnsigned = true) { + return NodeChildProcess.spawnSync( + process.execPath, + [ + NodePath.join(import.meta.dirname, "windows-publication-signing.ts"), + phase === "assembly" ? "--attestation" : "--candidate-handoff", + phase === "assembly" ? attestation : candidateHandoff, + "--allow-unsigned", + String(allowUnsigned), + "--stable-repository", + stableRepository, + "--beta-repository", + betaRepository, + ], + { + cwd: root, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + GH_TOKEN: "local-fixture", + PUBLISHED_STATE: statePath, + }, + encoding: "utf8", + }, + ); + }, + [Symbol.dispose]() { + NodeFS.rmSync(root, { recursive: true, force: true }); + }, + }; +} + +// oxlint-disable-next-line t3code/no-global-process-runtime -- The gh fixture uses a POSIX shebang, like the Linux publication job. +describe.skipIf(process.platform === "win32")("Windows publication gate CLI", () => { + it("reads the newest Stable and Beta handoffs before deciding", () => { + using fixture = publicationFixture(); + // A full first page of drafts must not hide the newest published Beta on the next page. + fixture.setPublished({ + betaPages: [ + Array.from({ length: 100 }, () => ({ ...betaRelease, draft: true })), + [betaRelease], + ], + }); + const pinned = fixture.run("assembly"); + assert.notStrictEqual(pinned.status, 0); + assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/); + fixture.setPublished({}); + const none = fixture.run("assembly"); + assert.strictEqual(none.status, 0, none.stderr); + assert.match(none.stdout, /WARNING: Publishing an unsigned Windows installer/); + const unapproved = fixture.run("assembly", false); + assert.notStrictEqual(unapproved.status, 0); + assert.match(unapproved.stderr, /allow_unsigned_windows=true/); + }); + + it("rejects the same unsigned candidate when Beta gains a pin while Stable awaits approval", () => { + using fixture = publicationFixture(); + const candidate = NodeFS.readFileSync(fixture.candidateHandoff, "utf8"); + const assembled = fixture.run("assembly"); + assert.strictEqual(assembled.status, 0, assembled.stderr); + NodeFS.unlinkSync(fixture.attestation); + const beforePin = fixture.run("publication"); + assert.strictEqual(beforePin.status, 0, beforePin.stderr); + + fixture.setPublished({ betaPages: [[betaRelease]] }); + const afterPin = fixture.run("publication"); + assert.notStrictEqual(afterPin.status, 0); + assert.match(afterPin.stderr, /cannot override/); + assert.strictEqual(NodeFS.readFileSync(fixture.candidateHandoff, "utf8"), candidate); + + fixture.setCandidate({ signed: true, publisherName: PUBLISHER }); + const compatible = fixture.run("publication", false); + assert.strictEqual(compatible.status, 0, compatible.stderr); + fixture.setCandidate({ signed: true, publisherName: "Someone Else" }); + const incompatible = fixture.run("publication"); + assert.notStrictEqual(incompatible.status, 0); + assert.match(incompatible.stderr, /signed by 'Someone Else'/); + }); + + it("also enforces a Stable pin when publishing Beta and refuses missing candidate signing", () => { + using fixture = publicationFixture(); + fixture.setCandidate({ signed: false }, "beta"); + fixture.setPublished({ stableHandoff: { windows: signed } }); + const pinned = fixture.run("publication"); + assert.notStrictEqual(pinned.status, 0); + assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/); + + fixture.setPublished({}); + const handoff = JSON.parse(NodeFS.readFileSync(fixture.candidateHandoff, "utf8")); + delete handoff.windows; + NodeFS.writeFileSync(fixture.candidateHandoff, JSON.stringify(handoff)); + const missing = fixture.run("publication"); + assert.notStrictEqual(missing.status, 0); + assert.match(missing.stderr, /candidate handoff must record Windows signing/); + }); + + it("fails closed on public lookup and download errors, including 404 and 5xx", () => { + using fixture = publicationFixture(); + for (const key of ["stable", "beta", "stable-handoff", "beta-handoff"]) { + for (const status of [404, 503]) { + fixture.setPublished({ betaPages: [[betaRelease]], failure: { key, status } }); + const failed = fixture.run("publication"); + assert.notStrictEqual(failed.status, 0, `${key}: HTTP ${status}`); + assert.match(failed.stderr, new RegExp(`HTTP ${status}`)); + } + } + }); + + it("fails closed on malformed public inventories and Windows signing records", () => { + using fixture = publicationFixture(); + for (const state of [ + { betaPages: {} }, + { betaPages: [[{ tag_name: betaTag }]] }, + { stableHandoff: [] }, + { stableHandoff: { windows: null } }, + { betaPages: [[betaRelease]], betaHandoff: { windows: { signed: true } } }, + ]) { + fixture.setPublished(state); + const failed = fixture.run("publication"); + assert.notStrictEqual(failed.status, 0, JSON.stringify(state)); + assert.match(failed.stderr, /Cannot read|Invalid|must name its publisher/); + } + }); +}); diff --git a/scripts/windows-publication-signing.ts b/scripts/windows-publication-signing.ts new file mode 100644 index 0000000000..2fb165ed18 --- /dev/null +++ b/scripts/windows-publication-signing.ts @@ -0,0 +1,200 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off - Standalone release gate reads build attestations and published handoffs before publication. + +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeURL from "node:url"; + +/** What a release records about its Windows installer's Authenticode signature. */ +export interface WindowsSigningRecord { + readonly signed: boolean; + /** The certificate CN the installed app pins in app-update.yml; null when unsigned. */ + readonly publisherName: string | null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** The Windows build's signing attestation, as the release workflow writes it. */ +export function readWindowsSigningAttestation(value: unknown): WindowsSigningRecord { + if ( + !isRecord(value) || + value.platform !== "win" || + value.arch !== "x64" || + typeof value.signed !== "boolean" + ) { + throw new Error("Invalid Windows signing attestation."); + } + const publisherName = value.publisherName; + if (value.signed) { + if (typeof publisherName !== "string" || publisherName.trim() === "") { + throw new Error("A signed Windows attestation must name its publisher."); + } + return { signed: true, publisherName }; + } + if (publisherName !== undefined) { + throw new Error("An unsigned Windows attestation cannot name a publisher."); + } + return { signed: false, publisherName: null }; +} + +/** + * Publishers that installed Windows clients already pin, from each channel's newest published + * handoff. Once a signed build ships, every later build is signed by the same publisher, so the + * newest release of each channel carries the pin. Handoffs older than this record name none. + */ +export function pinnedWindowsPublishers(handoffs: ReadonlyArray): ReadonlyArray { + const pinned = new Set(); + for (const handoff of handoffs) { + const record = readHandoffWindowsSigning(handoff); + if (record === null) continue; + if (record.publisherName !== null) pinned.add(record.publisherName); + } + return [...pinned]; +} + +/** + * Clients switch between Stable and Beta, so a pin on either binds both, and electron-updater + * rejects an installer whose signer misses the pin: no exception may override it. Before any + * pin, an unsigned build still needs the explicit unsigned-Windows exception. + */ +export function decideWindowsPublication(input: { + readonly candidate: WindowsSigningRecord; + readonly pinned: ReadonlyArray; + readonly allowUnsigned: boolean; +}): { readonly warning: string | null } { + const { candidate, pinned, allowUnsigned } = input; + if (pinned.length > 0) { + const names = pinned.map((name) => `'${name}'`).join(", "); + if (!candidate.signed) { + throw new Error( + `Installed Windows clients pin publisher ${names}; an unsigned installer would fail their update signature check on Stable and Beta. allow_unsigned_windows cannot override this.`, + ); + } + if (pinned.some((publisher) => publisher !== candidate.publisherName)) { + throw new Error( + `Installed Windows clients pin publisher ${names}; this installer is signed by '${candidate.publisherName}' and would fail their update signature check.`, + ); + } + return { warning: null }; + } + if (candidate.signed) return { warning: null }; + if (!allowUnsigned) { + throw new Error( + "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true.", + ); + } + return { warning: "Publishing an unsigned Windows installer by explicit release approval." }; +} + +function values(args: ReadonlyArray, flag: string): ReadonlyArray { + return args.flatMap((arg, index) => (arg === flag && args[index + 1] ? [args[index + 1]!] : [])); +} + +function readJson(path: string): unknown { + return JSON.parse(NodeFS.readFileSync(path, "utf8")) as unknown; +} + +function readHandoffWindowsSigning(handoff: unknown): WindowsSigningRecord | null { + if (!isRecord(handoff)) throw new Error("Invalid release handoff."); + if (handoff.windows === undefined) return null; + if (!isRecord(handoff.windows)) throw new Error("Invalid Windows signing record in handoff."); + return readWindowsSigningAttestation({ platform: "win", arch: "x64", ...handoff.windows }); +} + +function githubJson(args: ReadonlyArray): unknown { + return JSON.parse( + NodeChildProcess.execFileSync("gh", args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 16 * 1024 * 1024, + }), + ) as unknown; +} + +function newestPublishedHandoffs(stableRepository: string, betaRepository: string): unknown[] { + const stable = githubJson(["api", `repos/${stableRepository}/releases/latest`]); + if (!isRecord(stable) || typeof stable.tag_name !== "string" || !stable.tag_name) { + throw new Error("Cannot identify the newest published Stable release."); + } + const pages = githubJson([ + "api", + "--paginate", + "--slurp", + `repos/${betaRepository}/releases?per_page=100`, + ]); + if (!Array.isArray(pages) || !pages.every(Array.isArray)) { + throw new Error("Cannot read published Beta releases."); + } + const releases: unknown[] = pages.flat(); + let betaTag: string | null = null; + for (const release of releases) { + if ( + !isRecord(release) || + typeof release.draft !== "boolean" || + typeof release.tag_name !== "string" + ) { + throw new Error("Invalid published Beta release record."); + } + if (!release.draft && release.tag_name.includes("-beta.")) { + betaTag = release.tag_name; + break; + } + } + const download = (repository: string, tag: string) => + githubJson([ + "release", + "download", + tag, + "--repo", + repository, + "--pattern", + "scient-release-handoff.json", + "--output", + "-", + ]); + return [ + download(stableRepository, stable.tag_name), + ...(betaTag === null ? [] : [download(betaRepository, betaTag)]), + ]; +} + +export function runWindowsPublicationGate(args: ReadonlyArray): string | null { + const [attestation] = values(args, "--attestation"); + const [candidateHandoff] = values(args, "--candidate-handoff"); + if (Boolean(attestation) === Boolean(candidateHandoff)) { + throw new Error("Exactly one of --attestation or --candidate-handoff is required."); + } + const candidate = attestation + ? readWindowsSigningAttestation(readJson(attestation)) + : readHandoffWindowsSigning(readJson(candidateHandoff!)); + if (candidate === null) throw new Error("The candidate handoff must record Windows signing."); + const [stableRepository] = values(args, "--stable-repository"); + const [betaRepository] = values(args, "--beta-repository"); + if (!stableRepository || !betaRepository) { + throw new Error("--stable-repository and --beta-repository are required."); + } + const allowUnsigned = values(args, "--allow-unsigned")[0] === "true"; + const { warning } = decideWindowsPublication({ + candidate, + pinned: pinnedWindowsPublishers(newestPublishedHandoffs(stableRepository, betaRepository)), + allowUnsigned, + }); + return warning; +} + +// Real paths, so an indirect invocation still runs the gate rather than silently passing it. +const isMain = process.argv[1] + ? NodeFS.realpathSync(process.argv[1]) === + NodeFS.realpathSync(NodeURL.fileURLToPath(import.meta.url)) + : false; +if (isMain) { + try { + const warning = runWindowsPublicationGate(process.argv.slice(2)); + if (warning !== null) process.stdout.write(`WARNING: ${warning}\n`); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +}