From f10d7da9abd7df324c4d61d802c5f9d81de8113f Mon Sep 17 00:00:00 2001 From: hkimw <54717101+hkimw@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:03:51 +0900 Subject: [PATCH] feat: add verified Worker OPFS checkpoint recovery --- CHANGELOG.md | 6 +- PUBLIC_SURFACE.json | 45 ++-- README.md | 16 +- docs/BROWSER_ARTIFACTS.md | 21 +- docs/BROWSER_RESUME.md | 129 +++++++++ docs/COMPATIBILITY.md | 18 +- docs/TRANSFER_QUEUE.md | 14 +- docs/UPLOADS.md | 6 +- docs/api.md | 20 +- examples/browser-resume/browser.html | 7 + examples/browser-resume/browser.mjs | 43 +++ examples/browser-resume/host.mjs | 37 +++ examples/browser-resume/serve.mjs | 22 ++ examples/browser-resume/worker.mjs | 24 ++ package.json | 7 +- .../browser-artifact-checkpoint.schema.json | 245 ++++++++++++++++++ scripts/benchmark-browser-artifacts.mjs | 1 + scripts/pack-check.mjs | 2 +- src/artifact-resume-browser.d.mts | 24 ++ src/artifact-resume-browser.mjs | 175 +++++++++++++ src/artifact-transfer-core.mjs | 45 +++- src/artifact-transfer.d.mts | 9 +- src/schemas.d.mts | 2 +- src/schemas.mjs | 4 + src/sha256-stream.mjs | 12 + src/transfer-queue.mjs | 4 +- tests/artifact-resume-browser.test.mjs | 167 ++++++++++++ tests/types/artifact-resume-browser.mts | 15 ++ 28 files changed, 1048 insertions(+), 72 deletions(-) create mode 100644 docs/BROWSER_RESUME.md create mode 100644 examples/browser-resume/browser.html create mode 100644 examples/browser-resume/browser.mjs create mode 100644 examples/browser-resume/host.mjs create mode 100644 examples/browser-resume/serve.mjs create mode 100644 examples/browser-resume/worker.mjs create mode 100644 schemas/browser-artifact-checkpoint.schema.json create mode 100644 src/artifact-resume-browser.d.mts create mode 100644 src/artifact-resume-browser.mjs create mode 100644 tests/artifact-resume-browser.test.mjs create mode 100644 tests/types/artifact-resume-browser.mts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d6b91e..97e6b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.10.0 — in development (not yet released) +## 0.10.0 — 2026-10-04 - Browser artifact streaming with caller-owned file/OPFS sinks and distinct received versus stored verification. @@ -8,6 +8,10 @@ authority, chunk/full hashes, explicit restart recovery and conditional commit. - Caller-owned mixed transfer queue with chunk scheduling, connection/resource/rate limits, pause/resume, bounded transient chunk retries and distinct verified progress. +- Worker OPFS checkpoints with full stored identity, fresh host authority, independent + prefix rehash, metadata crash recovery and exclusive ownership across tabs. +- Native browser examples cover reload, host restart, cross-tab conflict and recovery + after owner-tab termination. Other browser/storage environments require qualification. - Keep project, text-file, job and artifact v1 contracts and released 0.3–0.9 plugin package metadata verification. See [uploads](docs/UPLOADS.md). diff --git a/PUBLIC_SURFACE.json b/PUBLIC_SURFACE.json index d59a924..4c03fcc 100644 --- a/PUBLIC_SURFACE.json +++ b/PUBLIC_SURFACE.json @@ -2,43 +2,39 @@ "schemaVersion": 1, "description": "Explicitly reviewed files for the public DDS Plugin SDK package. Changes require publication and rights review.", "packageFiles": [ + "bin/dds-plugin.mjs", "CHANGELOG.md", "CONTRIBUTING.md", - "LICENSE", - "NOTICE", - "PUBLIC_SURFACE.json", - "README.md", - "SECURITY.md", - "bin/dds-plugin.mjs", + "docs/api.md", "docs/ARTIFACT_STORAGE.md", "docs/BINARY_ARTIFACTS.md", "docs/BROWSER_ARTIFACTS.md", + "docs/BROWSER_RESUME.md", "docs/COMPATIBILITY.md", "docs/CONSENT.md", "docs/DEVTOOLS.md", "docs/FILE_REVISIONS.md", - "docs/JOBS.md", + "docs/host-contract.md", "docs/JOB_HISTORY.md", "docs/JOB_STORAGE.md", + "docs/JOBS.md", "docs/LANGUAGE.md", "docs/LICENSING.md", "docs/OBSERVATION.md", - "docs/PROJECTS.md", + "docs/performance.md", "docs/PROJECT_QUERIES.md", "docs/PROJECT_TOOLS.md", "docs/PROJECT_WATCH.md", + "docs/PROJECTS.md", "docs/PUBLIC_SCOPE.md", "docs/PUBLISHING.md", + "docs/testing.md", "docs/THEMES.md", "docs/TRANSFER_QUEUE.md", "docs/UPLOADS.md", "docs/VERIFYING.md", - "docs/WORKSPACES.md", - "docs/api.md", - "docs/host-contract.md", - "docs/performance.md", - "docs/testing.md", "docs/workspace-protocol.md", + "docs/WORKSPACES.md", "examples/binary-artifacts/run.mjs", "examples/browser-artifacts/browser.html", "examples/browser-artifacts/browser.mjs", @@ -46,9 +42,14 @@ "examples/browser-artifacts/run.mjs", "examples/browser-artifacts/serve.mjs", "examples/browser-artifacts/worker.mjs", + "examples/browser-resume/browser.html", + "examples/browser-resume/browser.mjs", + "examples/browser-resume/host.mjs", + "examples/browser-resume/serve.mjs", + "examples/browser-resume/worker.mjs", "examples/command-jobs/run.mjs", - "examples/consent/README.md", "examples/consent/prompt.mjs", + "examples/consent/README.md", "examples/durable-jobs/run.mjs", "examples/hello-diagnostics/plugin.mjs", "examples/hello-diagnostics/run.mjs", @@ -66,13 +67,13 @@ "examples/project-tools/run.mjs", "examples/project-tools/serve.mjs", "examples/project-watch/run.mjs", - "examples/publishable-plugin/LICENSE", - "examples/publishable-plugin/NOTICE", - "examples/publishable-plugin/README.md", "examples/publishable-plugin/dds-package.json", "examples/publishable-plugin/disclosure.json", + "examples/publishable-plugin/LICENSE", + "examples/publishable-plugin/NOTICE", "examples/publishable-plugin/plugin.json", "examples/publishable-plugin/plugin.mjs", + "examples/publishable-plugin/README.md", "examples/release-verification/README.md", "examples/release-verification/run.mjs", "examples/stored-artifacts/run.mjs", @@ -85,15 +86,19 @@ "examples/uploads/browser.mjs", "examples/uploads/run.mjs", "examples/uploads/serve.mjs", - "examples/user-workspace/LICENSE", "examples/user-workspace/client.mjs", + "examples/user-workspace/LICENSE", "examples/user-workspace/plugin.mjs", "examples/user-workspace/private-manifest.json", "examples/user-workspace/project/design.sv", "examples/user-workspace/server.mjs", "examples/user-workspace/tool.mjs", "examples/workspace-observation/run.mjs", + "LICENSE", + "NOTICE", "package.json", + "PUBLIC_SURFACE.json", + "README.md", "schemas/artifact-sink-capabilities.schema.json", "schemas/artifact-storage-capabilities.schema.json", "schemas/artifact-transfer-receipt.schema.json", @@ -102,6 +107,7 @@ "schemas/binary-artifact-list.schema.json", "schemas/binary-artifact-reference.schema.json", "schemas/binary-artifact.schema.json", + "schemas/browser-artifact-checkpoint.schema.json", "schemas/command.schema.json", "schemas/diagnostics-request.schema.json", "schemas/diagnostics-result.schema.json", @@ -146,8 +152,11 @@ "schemas/workspace-file-revision.schema.json", "schemas/workspace-project-capabilities.schema.json", "schemas/workspace-project-poll.schema.json", + "SECURITY.md", "src/artifact-download-node.d.mts", "src/artifact-download-node.mjs", + "src/artifact-resume-browser.d.mts", + "src/artifact-resume-browser.mjs", "src/artifact-storage-node.d.mts", "src/artifact-storage-node.mjs", "src/artifact-storage-session.mjs", diff --git a/README.md b/README.md index 6ee5c86..a430a14 100644 --- a/README.md +++ b/README.md @@ -10,21 +10,21 @@ a small ESM package, TypeScript declarations and versioned contracts. Connect your own workspace and tools with the Node.js host and HTTP client. The SDK has **zero runtime dependencies**. -The next 0.10 line is in development: [browser artifact streaming](docs/BROWSER_ARTIFACTS.md) -adds bounded sinks, incremental whole-file hashes, explicit storage readback and -main-thread/Worker examples and [selected-file uploads](docs/UPLOADS.md) with private +The 0.10 line adds [browser artifact streaming](docs/BROWSER_ARTIFACTS.md) +with bounded sinks, incremental whole-file hashes, explicit storage readback and +main-thread/Worker examples, and [selected-file uploads](docs/UPLOADS.md) with private staging, explicit restart recovery and verified conditional commit. A caller-owned [transfer queue](docs/TRANSFER_QUEUE.md) schedules mixed uploads/downloads with -chunk fairness, pause/resume, rate budgets and bounded safe retries. The checkout -uses 0.10.0 for protocol development; these additions are not in the published -0.9 archive. +chunk fairness, pause/resume, rate budgets and bounded safe retries. +[Worker OPFS checkpoints](docs/BROWSER_RESUME.md) allow explicit recovery after +reload or host restart, with prefix hashes and cross-tab exclusive ownership. This is the intentionally public SDK for external plugin developers. The DDS application, private engines/services and production signing systems are outside this repository. Read the [public scope](docs/PUBLIC_SCOPE.md) and [compatibility table](docs/COMPATIBILITY.md) for exact boundaries. -Version **0.9.0** supports Node 22 and 24. Manifest v1 diagnostic plugins remain +Version **0.10.0** supports Node 22 and 24. Manifest v1 diagnostic plugins remain compatible. Manifest v2 adds commands, permission-scoped workspace and backend APIs, and open-source or proprietary distribution metadata. Themes export to the XML format already supported by DDS. The workspace host runs trusted plugins @@ -84,7 +84,7 @@ mkdir my-dds-plugin cd my-dds-plugin npm init -y npm pkg set type=module -npm install --ignore-scripts --save-exact https://github.com/Altifigence/dds-plugin-sdk/releases/download/v0.9.0/altifigence-dds-plugin-sdk-0.9.0.tgz +npm install --ignore-scripts --save-exact https://github.com/Altifigence/dds-plugin-sdk/releases/download/v0.10.0/altifigence-dds-plugin-sdk-0.10.0.tgz ``` Copy the example's `plugin.mjs` and `run.mjs` into this directory, then run: diff --git a/docs/BROWSER_ARTIFACTS.md b/docs/BROWSER_ARTIFACTS.md index 2516f3a..2fc1f78 100644 --- a/docs/BROWSER_ARTIFACTS.md +++ b/docs/BROWSER_ARTIFACTS.md @@ -1,7 +1,7 @@ -# Browser artifact streaming (0.10 development) +# Browser artifact streaming (0.10) -The public 0.9 archive does not contain these APIs yet. This foundation is part of -the next 0.10 release. Existing Node file downloads remain available unchanged. +These APIs are included in 0.10. Existing Node file downloads remain available. +For reload-safe downloads, use the [Worker checkpoint sink](BROWSER_RESUME.md). `/artifact-transfer` imports no Node built-ins. `streamJobBinaryArtifact()` uses the existing 0.7 binary protocol; `streamStoredJobArtifact()` uses the 0.8 snapshot @@ -54,8 +54,9 @@ event and returned receipt set `verified: true`. Callback exceptions fail the operation, even if commit has already happened. The trusted caller sink contract advertises `seek`, `readback`, `persistence` -(`none`, `on-commit`, `per-checkpoint`) and abort behavior. The current core writes -sequentially from zero; advertising seek or checkpoints does not enable resume. +(`none`, `on-commit`, `per-checkpoint`) and abort behavior. Explicit `resume: true` +requires seek, readback and per-checkpoint persistence plus a session `recovery` +reader. The core independently verifies its prefix before requesting remaining bytes. Implementations must bound retained data and cooperate with cancellation. The helper waits for in-flight writes before cleanup so late writes are not detached. @@ -72,8 +73,8 @@ staging stream on pre-commit failure. They do not provide a persistent partial download after a reload. Normal intervening file changes are checked using size and last-modified values; browser APIs provide no atomic compare-and-swap against external programs. Reservations prevent concurrent use of the same entry only -within this module instance/realm. Cross-tab persistent recovery is a separate -0.10 integration step and is not claimed by this foundation. +within this module instance/realm. Use the separate [checkpoint adapter](BROWSER_RESUME.md) +for persistent recovery and cross-tab exclusion. ## Examples, support and resource evidence @@ -124,10 +125,10 @@ Semantics follow the [File System Standard](https://fs.spec.whatwg.org/) and the ## 한국어 요약 -0.10 개발 단계 기능이다. 호출자가 선택한 파일/OPFS 디렉터리에만 저장하며 +0.10에 포함된 기능이다. 호출자가 선택한 파일/OPFS 디렉터리에만 저장하며 권한을 자동 요청하거나 다른 위치로 우회하지 않는다. 청크별 검증·저장을 기다린 뒤 다음 청크를 읽고, 수신 전체 해시와 저장 파일 재읽기 해시를 구분한다. 재읽기 없는 sink는 디스크 검증을 주장하지 않는다. 전체 바이트를 받았다는 진행률만으로 완료 처리하지 않는다. 취소·용량 초과·권한 철회·해시 오류에는 -부분 상태와 확정 여부를 반환한다. 이 foundation의 비동기 저장은 새로고침 후 -부분 파일 재개를 제공하지 않으며 영속 복구·탭 간 독점은 뒤의 0.10 통합 범위다. +부분 상태와 확정 여부를 반환한다. 비동기 저장은 확정 전에 중단되면 staging을 +버리며, 새로고침 후 재개·탭 간 독점에는 별도 Worker checkpoint adapter를 쓴다. diff --git a/docs/BROWSER_RESUME.md b/docs/BROWSER_RESUME.md new file mode 100644 index 0000000..fe781cc --- /dev/null +++ b/docs/BROWSER_RESUME.md @@ -0,0 +1,129 @@ +# Browser checkpoint recovery (0.10) + +`/artifact-resume-browser` supplies a persistent OPFS sink for **stored artifacts**. +Use it in a dedicated Worker with a directory and UUID explicitly selected by the +application. A new or reconnected client must first obtain the same retained +snapshot with current host authority. The SDK never restores tokens, grants, +connections, commands, pickers or a previous host generation from local storage. + +```js +// Inside an application-owned dedicated Worker: +import {createOpfsArtifactSink} from '@altifigence/dds-plugin-sdk/artifact-resume-browser'; +import {streamStoredJobArtifact} from '@altifigence/dds-plugin-sdk/artifact-transfer'; + +await client.connect(); +const reference = await client.getStoredJobArtifact(pluginId, jobId, artifactId, pluginPin); +const sink = createOpfsArtifactSink({directory: selectedDirectory, partialId, reference}); +const receipt = await streamStoredJobArtifact(client, reference, { + sink, + resume: true, // false/omitted creates a new partial; an existing ID is a conflict + signal: controller.signal, +}); +showVerifiedHash(receipt.storedSha256); +``` + +The same options work with `queue.enqueueStoredDownload(client, reference, options)`. +The application selects its own directory and partial ID again after reload; the +sink does not enumerate origin storage or guess a previous download. The example +remembers only those two selections. An application must decide its own consent, +retention, storage persistence requests and deletion UI. + +## Checkpoint format and ordering + +`getOpfsArtifactPartialNames(partialId)` returns exactly two names: +`dds-partial-.data` and `dds-partial-.json`. Creation refuses an existing +data or metadata entry. Explicit recovery requires both entries; eviction or +deletion never causes a silent replacement download. +An interrupted first creation can leave an empty or orphan entry. Explicitly +inspect and remove that selected fixture, or choose a new ID; it is never reused +as an implicitly empty download. + +The data file uses an exclusive `FileSystemSyncAccessHandle`. Each accepted chunk +is at most 64 KiB and is flushed before the small checksummed metadata is saved. +Metadata uses a separate writable stream, closed before acknowledging the chunk. +It records full immutable store/workspace/plugin-pin/job/snapshot/artifact +identity, timestamps, state and current/previous prefix offsets and hashes. It +contains no token, grants or old connection scope. `parseBrowserArtifactCheckpoint` +validates this bounded format; the JSON schema describes its structural contract. +The checksum detects accidental damage, not malicious changes by origin scripts. + +On recovery the sink rehashes the recorded prefix in bounded slices. If the file +has extra uncheckpointed bytes, it truncates them only after that prefix matches. +If the current checkpoint is beyond the actual file size, the previous checkpoint +is considered only when its complete bytes exist and its hash matches. A corrupt +prefix, missing file, mismatched identity or invalid metadata fails without +resetting the download. The host's current reference must match the complete saved +snapshot; a new host generation is permitted through a fresh connection and grant. + +The transfer helper independently rehashes the recovery prefix before asking for +remaining network bytes. Its final hash covers prefix plus new bytes, and its +stored verification re-reads the complete saved file. Recovering an already +complete partial still validates the host's EOF and rechecks saved bytes. +`resumedBytes` counts verified local prefix bytes; `receivedBytes` counts only this +attempt's network bytes. `verifying-prefix` is distinct from receive/commit/readback. +Only the returned verified receipt means that the whole transfer completed. + +## Ownership, cancellation and bounds + +An origin-wide exclusive Web Lock named `dds-artifact-partial:` is requested +with `ifAvailable`. A competing tab receives `conflict`; no waiting queue, lock +stealing or takeover is attempted. The native data handle is also exclusive. Both +remain held through final readback, and cancellation closes them while retaining +the last checkpoint. Worker/tab termination releases native ownership; a later +explicit recovery validates the files again. This is cooperative same-origin +coordination, not an isolation boundary against other code executing in the origin. + +Four open sink sessions per module instance, 1 GiB per file, 64 KiB per data +operation and 16 KiB per metadata record are hard maxima. The default local +retention is one hour; callers may choose up to 24 hours, capped by the retained +host snapshot's expiry. Every operation rechecks expiry. The transfer's separate +30-minute deadline includes queue wait and pause. Paused sessions retain locks, +and the fifth open checkpoint sink is rejected until a session closes. + +There is no total origin-disk quota manager. Browser quota/eviction policy still +applies and can remove data. A data write or metadata-close failure poisons that +session; explicitly reopen to inspect whichever old/new checkpoint actually +persisted. Cancellation is not rollback. Cleanup belongs to the caller after +closing its session; no error path deletes unknown files or expired partials. + +## Runtime and evidence + +`getBrowserArtifactResumeSupport()` reports dedicated Worker, sync-access and Web +Locks presence. Native main-thread use is unsupported. Structural directory and +lock ports are trusted implementations useful for deterministic contract tests; +they do not establish native browser acceptance. + +Run `npm run example:browser-resume`. The disposable host captures a 2 MiB result +and deletes its original source. In the page: + +1. Create a checkpoint and stop; reload, restart the host, then resume. +2. Remove that completed fixture, create a new partial and hold its lock. +3. Open the same URL in a second tab and try recovery: it must report conflict. +4. Close the owning tab; retry recovery in the second tab and verify the full hash. +5. Remove the selected fixture files and stop the host. + +Windows embedded Chromium 154 was exercised for checkpoint cancellation, page +reload, host restart, remaining-byte recovery, two-tab contention and owner-tab +termination. Other browsers and power-loss durability are not qualified by these +checks. Deterministic tests cover quota failures, metadata-close ambiguity, +truncated or damaged data, missing entries, expiry and identity changes; real HTTP +tests also cover fresh authority, revoked grants and source deletion. +The 2,097,169-byte reload/restart case reused 65,536 bytes and received 2,031,633 +bytes in 32 chunks, with matching full SHA-256 in 3.67 seconds. The owner-tab +termination case took 4.31 seconds after selecting recovery. These are local +fixture observations, not portable latency guarantees. + +The storage primitives follow the [File System Standard](https://fs.spec.whatwg.org/) +and [Web Locks specification](https://w3c.github.io/web-locks/). Writable metadata +becomes visible at close; sync data writes operate in place. These API semantics +do not promise operating-system power-loss durability or permanent browser storage. + +## 한국어 요약 + +전용 Worker에서 사용자가 선택한 OPFS 디렉터리와 partial UUID로 중단된 +다운로드를 재개한다. 새 연결이 현재 권한으로 얻은 저장 결과 참조와 로컬 +메타데이터의 전체 identity를 비교하고, 실제 prefix를 다시 해시한 뒤 남은 +바이트만 받는다. 손상·삭제·만료·다른 결과는 자동 초기화 없이 거부한다. +동일 partial의 다른 탭은 conflict를 받고, 소유 탭 종료 뒤 명시적으로 다시 +복구한다. 취소는 검증된 체크포인트를 보관하며 token이나 이전 grants를 +저장하지 않는다. 전체 수신·저장 재검증이 끝나야 완료로 표시한다. diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 035d8e2..53bd765 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -11,14 +11,15 @@ | Browser-only DDS host | Native workspace transport is unavailable | | Language providers | Diagnostics plus completion, hover, definition, references and document symbols in SDK hosts since 0.3.0; no released DDS editor bridge is implied | | Project edit sessions | Client helpers in 0.3.0 over unchanged workspace protocol v1, including 0.2.x servers | -| Download verification | Node API/CLI since 0.3.1; 0.9.0 accepts exact 0.3.x, 0.4.x, 0.5.x, 0.6.x, 0.7.x, 0.8.x and 0.9.x package metadata; no extraction, code execution or publisher authentication | +| Download verification | Node API/CLI since 0.3.1; 0.10.0 accepts exact 0.3.x–0.10.x package metadata; no extraction, code execution or publisher authentication | | Development tools | `init`, `doctor`, `dev` since 0.4.0; local Node tools, explicit trusted-code execution | | Command jobs | Opt-in SDK host/client extension since 0.4.0; progress/logs/UTF-8 artifacts; v1 hello unchanged | | File and job observation | Client helpers since 0.5.0; 0.6 uses revision reads when supported; job reads need enabled 0.4+ hosts | | Binary job results | Optional 0.7.0 extension; explicit host opt-in; 1 GiB files and 64 KiB chunks; Node resume/full-file verification | | Durable jobs, history and results | Optional 0.8.0 host stores; current authorization after restart; explicit retry and verified snapshot downloads | | File revision / conditional read | Optional methods since 0.6.0; bounded full-read fallback on old hosts; v1 hello unchanged | -| Browser streaming and selected-file uploads | In the 0.10 development checkout; explicit file/OPFS sinks and optional server upload store; not in the published 0.9 archive | +| Browser streaming and selected-file uploads | Since 0.10.0; explicit file/OPFS sinks and optional server upload store; independent current authority and full hashes | +| Browser checkpoint recovery | Since 0.10.0; dedicated Worker + OPFS sync access + Web Locks, currently approved stored references and explicit partial selection; Chromium 154 on Windows exercised | This is a developer preview. Pin the exact package release and read the changelog before updating. SDK, protocol, plugin and DDS product versions are distinct. @@ -110,3 +111,16 @@ See [project tools](PROJECT_TOOLS.md) for leases, resync and preserved drafts. New 0.9 plugin archives declare `>=0.9.0 <0.10.0`. Verification also preserves exact 0.8 and earlier metadata; installation still requires publisher validation and a newly packed plugin version for the selected SDK line. + +0.10.0 adds opt-in `uploads.*` methods and browser-safe bounded streaming, +mixed transfer scheduling and explicit Worker OPFS recovery. Existing file, +project, command, job and stored-artifact v1 contracts retain their wire shapes. +Known 0.3–0.9 hosts report uploads disabled without a new-method probe. Browser +live-result streaming needs a 0.7+ binary host; stored recovery needs a 0.8+ +snapshot host with current grants. Feature presence alone is not browser +qualification. See [streaming](BROWSER_ARTIFACTS.md), [uploads](UPLOADS.md), +[queue](TRANSFER_QUEUE.md) and [checkpoints](BROWSER_RESUME.md). + +New 0.10 plugin archives declare `>=0.10.0 <0.11.0`. Exact previous metadata is +accepted for verification without changing peer ranges. Validate and repack a new +plugin version for this SDK; none of these APIs deploys or changes a DDS product. diff --git a/docs/TRANSFER_QUEUE.md b/docs/TRANSFER_QUEUE.md index 14ed0b0..f3cd736 100644 --- a/docs/TRANSFER_QUEUE.md +++ b/docs/TRANSFER_QUEUE.md @@ -1,7 +1,7 @@ # Caller-owned transfer queue -The 0.10 development checkout adds `/transfer-queue` for uploads, live binary job -downloads and stored artifact downloads. The published 0.9 archive does not include +The 0.10 line adds `/transfer-queue` for uploads, live binary job +downloads and stored artifact downloads. Earlier archives do not include it. Node and browsers share the queue and the same chunk/whole-file verifiers used by `uploadFile`, `streamJobBinaryArtifact` and `streamStoredJobArtifact`. @@ -98,8 +98,9 @@ Queue pause/resume is scoped to this live JavaScript instance. After cancellatio host restart or page reload, create a new approved client and a new transfer. For uploads, explicitly choose `recover: true` with the same selection. The helper queries current authority and rehashes the source's saved prefix before sending -new bytes. Persistent browser download recovery is a separate 0.10 integration -feature; this queue alone does not persist sink handles or checkpoint metadata. +new bytes. Persistent browser recovery uses the [Worker checkpoint sink](BROWSER_RESUME.md) +and explicit `resume: true` on a newly enqueued stored download. This queue alone +does not persist sink handles or checkpoint metadata. The optional per-transfer `timeoutMs` is at most 30 minutes and includes queue wait, retry backoff and pause. Request timeouts are at most 30 seconds. Callers @@ -122,7 +123,7 @@ recovery proves the durable result. Retrying the whole transfer is not implicit. | Field | Meaning | | --- | --- | | `acknowledgedBytes` | Upload bytes saved by the host, or download bytes accepted by the sink | -| `resumedBytes` | Saved upload prefix found at explicit recovery | +| `resumedBytes` | Verified upload or download prefix found at explicit recovery | | `transferredBytes` | Newly acknowledged bytes, excluding that prefix | | `attemptedBytes` | Raw payload bytes admitted for network attempts, including failed attempts | | `retriedBytes` | Portion admitted after a chunk's first attempt | @@ -170,4 +171,5 @@ not a DDS/Cloud product deployment or a performance guarantee. 429/503·네트워크 오류·요청 timeout으로 실패한 동일 청크만 제한적으로 재시도합니다. 인증·권한·해시·범위·충돌 오류와 begin/commit은 자동 반복하지 않습니다. 진행률 100%와 `verified: true`를 구분하고 전송 시도·새 저장 바이트·재시도·기존 prefix를 -별도로 표시합니다. 이 문서는 0.10 개발분이며 공개 0.9 archive에는 포함되지 않습니다. +별도로 표시합니다. 이 기능은 0.10 archive에 포함되며 새로고침 후 다운로드 +복구에는 Worker checkpoint sink와 명시적 resume 옵션을 함께 사용합니다. diff --git a/docs/UPLOADS.md b/docs/UPLOADS.md index 910fc83..0043bc9 100644 --- a/docs/UPLOADS.md +++ b/docs/UPLOADS.md @@ -1,7 +1,7 @@ # Selected-file uploads -The 0.10 development line adds opt-in uploads to a server-owned Node workspace. -The public 0.9 archive does not contain these APIs. Use a local Windows/Linux +The 0.10 line adds opt-in uploads to a server-owned Node workspace. +Use a local Windows/Linux filesystem with Node 22 or 24. The operator selects both the workspace and a separate private staging directory; a client cannot choose a host absolute path. This is an SDK host contract, not evidence of a DDS or Cloud product rollout. @@ -155,7 +155,7 @@ native pickers, Firefox and Safari were not qualified by this run. ## 한국어 요약 -0.10 개발 API는 운영자가 선택한 workspace·별도 staging·root·principal을 +0.10 API는 운영자가 선택한 workspace·별도 staging·root·principal을 사용합니다. 현재 플러그인 패키지 해시와 선언/승인된 workspace.write를 매번 확인하고, 부분 파일은 작업 파일로 노출하지 않습니다. Node에서 선택한 파일 또는 브라우저 File을 전체 사전 해시한 뒤 최대 64 KiB씩 전송합니다. diff --git a/docs/api.md b/docs/api.md index 41f5b2c..4bf24f5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,4 +1,4 @@ -# API reference — 0.9.0 +# API reference — 0.10.0 SDK 0.8 adds optional [job storage](JOB_STORAGE.md), [history and retry](JOB_HISTORY.md) and [stored result files](ARTIFACT_STORAGE.md). Existing v1 response shapes remain valid. @@ -80,7 +80,7 @@ license eligibility or agreement acceptance. See [LICENSING](LICENSING.md). | Activation context member | Behavior | | --- | --- | -| `host` | `{id: 'test-host' \| 'workspace-host', version: '0.9.0', protocolVersion: 1}` | +| `host` | `{id: 'test-host' \| 'workspace-host', version: '0.10.0', protocolVersion: 1}` | | `pluginId`, `scope` | Manifest ID and opaque host `{projectId, sessionId}` | | `grants` | Frozen intersection of host grants and manifest permissions | | `signal` | Aborted on deactivation, host disposal or activation timeout | @@ -295,3 +295,19 @@ capability/poll validators. The server's `revokeProjects()` closes project resou and requires a new generation to re-enable them. Edit sessions retain their own snapshots. See [project tools](PROJECT_TOOLS.md), [observation](PROJECT_WATCH.md) and [tree/search](PROJECT_QUERIES.md). + +## Bidirectional transfers (SDK 0.10.0) + +`/artifact-transfer` exports browser-safe `streamJobBinaryArtifact` and +`streamStoredJobArtifact`, incremental SHA-256 and typed sink/verification contracts. +`/artifact-transfer-browser` supplies selected-file and OPFS writable sinks; +`/artifact-resume-browser` adds `createOpfsArtifactSink`, explicit partial names, +checkpoint validation and Worker capability detection. Recovery requires +`resume: true` and independently rehashed local prefix bytes. + +`/uploads` provides contracts and `uploadFile`; `/uploads-node` provides the +private Node upload store and selected-file source; `/uploads-browser` adapts an +explicit `File`. The HTTP client adds capability/begin/query/write/commit/abort +methods. `/transfer-queue` exports `createTransferQueue`, options/snapshot validators, +limits, states and phases. See [uploads](UPLOADS.md), [streaming](BROWSER_ARTIFACTS.md), +[queue](TRANSFER_QUEUE.md) and [persistent recovery](BROWSER_RESUME.md). diff --git a/examples/browser-resume/browser.html b/examples/browser-resume/browser.html new file mode 100644 index 0000000..109fdfe --- /dev/null +++ b/examples/browser-resume/browser.html @@ -0,0 +1,7 @@ +SDK browser checkpoint recovery + +

Recover a browser download

Save a checkpoint from the generated 2 MiB trace. Reload this page or open it in a second tab, then recover the selected partial with a fresh host connection.

+

Only this fixture's folder name and partial ID are remembered. Cancelling retains the checkpoint; another tab cannot take an active download's lock.

+ + +

Ready

Waiting for an explicit operation.
diff --git a/examples/browser-resume/browser.mjs b/examples/browser-resume/browser.mjs new file mode 100644 index 0000000..32283af --- /dev/null +++ b/examples/browser-resume/browser.mjs @@ -0,0 +1,43 @@ +let config=await (await fetch('/config.json')).json();const key='dds-resume-fixture:'+config.fixtureId; +const elements=Object.fromEntries(['checkpoint','hold','resume','continue','cancel','restart','reload','cleanup','close','selection','progress','status','output'].map(id=>[id,document.getElementById(id)])); +let worker,busy=false,lastProgress,selection=JSON.parse(localStorage.getItem(key)??'null'); +const renderSelection=()=>{elements.selection.textContent=selection?`Selected partial: ${selection.partialId}`:'No partial selected. Create a checkpoint first.';};renderSelection(); +window.addEventListener('storage',event=>{if(event.key===key&&!busy){selection=JSON.parse(event.newValue??'null');renderSelection();}}); +function state(running){busy=running;for(const id of ['checkpoint','hold','resume','restart','cleanup','close'])elements[id].disabled=running;elements.cancel.disabled=!running;elements.continue.disabled=!running;} +async function run(action){ + if(busy)return; + if(action!=='resume'){ + if(selection){elements.status.textContent='Remove the selected fixture files before creating another partial.';return;} + selection={partialId:crypto.randomUUID(),directory:'dds-resume-fixture-'+config.fixtureId};localStorage.setItem(key,JSON.stringify(selection));renderSelection(); + } + if(!selection){elements.status.textContent='Create or select a checkpoint first.';return;} + state(true);config=await (await fetch('/config.json')).json();worker?.terminate();worker=new Worker('/worker.mjs',{type:'module'}); + elements.status.textContent='Connecting with current authority…'; + worker.onmessage=({data})=>{ + if(data.progress){lastProgress=data.progress;elements.progress.value=(data.progress.totalBytes?data.progress.acknowledgedBytes/data.progress.totalBytes:0);elements.status.textContent=`${data.progress.state} · ${data.progress.phase} · ${data.progress.acknowledgedBytes} / ${data.progress.totalBytes} bytes`;return;} + if(data.idle){state(false);return;} + elements.output.textContent=JSON.stringify(data,null,2); + if(data.holding)elements.status.textContent='PAUSED — checkpoint saved; this Worker still owns the lock. A second tab must receive conflict.'; + else if(data.checkpointRetained)elements.status.textContent='CHECKPOINT RETAINED — reload or reopen, then resume.'; + else elements.status.textContent=data.passed?'PASSED — whole saved file SHA-256 verified':`FAILED — ${data.code}: ${data.message}`; + }; + worker.onerror=event=>{elements.status.textContent='Worker failed: '+event.message;state(false);};worker.postMessage({action,selection,config}); +} +for(const id of ['checkpoint','hold','resume'])elements[id].onclick=()=>void run(id).catch(error=>{elements.status.textContent=error.message;state(false);}); +elements.continue.onclick=()=>worker?.postMessage({action:'continue'});elements.cancel.onclick=()=>worker?.postMessage({action:'cancel'}); +elements.reload.onclick=()=>location.reload(); +elements.restart.onclick=async()=>{config=await(await fetch('/config.json')).json();const r=await fetch('/restart',{method:'POST',headers:{'x-fixture-token':config.token}});elements.status.textContent=r.ok?'HOST RESTARTED — resume obtains a new approved reference.':'Host restart failed';}; +elements.cleanup.onclick=async()=>{ + if(!selection)return;try{ + await navigator.locks.request('dds-artifact-partial:'+selection.partialId,{mode:'exclusive',ifAvailable:true},async lock=>{ + if(!lock)throw Error('Another tab owns this partial; close or cancel that transfer first.'); + worker?.terminate();const root=await navigator.storage.getDirectory(); + if(selection.directory!=='dds-resume-fixture-'+config.fixtureId||!/^[0-9a-f-]{36}$/.test(selection.partialId))throw Error('Unexpected fixture selection'); + const directory=await root.getDirectoryHandle(selection.directory);for(const suffix of ['data','json'])await directory.removeEntry('dds-partial-'+selection.partialId+'.'+suffix).catch(error=>{if(error.name!=='NotFoundError')throw error;}); + // Remove only an empty fixture directory; preserve any unrelated entry. + await root.removeEntry(selection.directory).catch(error=>{if(!['InvalidModificationError','NotFoundError'].includes(error.name))throw error;}); + localStorage.removeItem(key);selection=null;renderSelection();elements.status.textContent='Selected fixture files removed.'; + }); + }catch(error){elements.status.textContent=error.message;} +}; +elements.close.onclick=async()=>{config=await(await fetch('/config.json')).json();worker?.terminate();const r=await fetch('/close',{method:'POST',headers:{'x-fixture-token':config.token}});elements.status.textContent=r.ok?'Fixture stopped.':'Fixture stop failed.';}; diff --git a/examples/browser-resume/host.mjs b/examples/browser-resume/host.mjs new file mode 100644 index 0000000..bf62085 --- /dev/null +++ b/examples/browser-resume/host.mjs @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import * as fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {createHash,randomUUID,randomBytes} from 'node:crypto'; +import {definePlugin} from '@altifigence/dds-plugin-sdk'; +import {createWorkspaceServer} from '@altifigence/dds-plugin-sdk/workspace-node'; +import {createWorkspaceClient} from '@altifigence/dds-plugin-sdk/workspace-client'; +import {createNodeJobStore} from '@altifigence/dds-plugin-sdk/job-storage-node'; +import {createNodeArtifactStore} from '@altifigence/dds-plugin-sdk/artifact-storage-node'; + +export async function createResumeFixture({fileBytes=2_097_169,allowedOrigins=[]}={}){ + assert.ok(Number.isSafeInteger(fileBytes)&&fileBytes>=0&&fileBytes<=16_777_233); + const directory=await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(),'dds-resume-example-'))),root=path.join(directory,'workspace'),workspaceId=randomUUID(),fixtureId=randomUUID(),jobId=randomUUID(); + let server,client,jobs,artifacts,config,closing,executions=0; + async function stopHost(){client?.dispose();await server?.close();await artifacts?.close();await jobs?.close();client=server=artifacts=jobs=undefined;} + const close=()=>closing??=(async()=>{await stopHost();assert.equal(path.dirname(directory),await fs.realpath(os.tmpdir()));assert.ok(path.basename(directory).startsWith('dds-resume-example-'));await fs.rm(directory,{recursive:true});})(); + try{ + await fs.mkdir(root);const file=await fs.open(path.join(root,'trace.bin'),'wx'),hash=createHash('sha256'),block=Buffer.from(Array.from({length:65536},(_,n)=>(n*17+3)&255)); + try{for(let at=0;atctx.registerCommand({id:'capture',title:'Capture selected fixture'},async(_,{job})=>{executions++;await job.addBinaryArtifact({id:'binary',path:'trace.bin'});return{};})); + async function startHost({grants=['workspace.read']}={}){ + const token=randomBytes(32).toString('hex'); + jobs=await createNodeJobStore({directory:path.join(directory,'store'),workspaceRoot:root,workspaceId});artifacts=await createNodeArtifactStore({jobStore:jobs}); + server=await createWorkspaceServer({root,workspaceId,token,notice:{id:'resume-example',version:'1',text:'Explicit checkpoint and host-restart example.'},plugins:[{plugin,artifactSha256}],grants,jobs:true,binaryArtifacts:true,allowedOrigins,jobStorage:{store:jobs,artifacts}}); + client=createWorkspaceClient({url:server.url,token});await client.connect();config={fixtureId,url:server.url,token,pluginId,artifactSha256,jobId,artifactId:'binary',fileBytes,sha256,generation:client.binding.workspace.generation}; + } + await startHost();await client.startCommandJob(pluginId,'capture',{},artifactSha256,{jobId});assert.equal((await client.waitForJob(jobId,{intervalMs:250})).state,'succeeded'); + // Live job completion can precede durable snapshot capture. Wait for the saved record. + const deadline=Date.now()+30_000; + for(;;){const recovery=await client.recoverJob(pluginId,jobId,artifactSha256);if(recovery.record?.settled)break;assert.ok(Date.now()setTimeout(resolve,25));} + await client.getStoredJobArtifact(pluginId,jobId,'binary',artifactSha256);await fs.unlink(path.join(root,'trace.bin')); + return {directory,close,get client(){return client;},get config(){return {...config};},get executions(){return executions;},async restart(options){await stopHost();await startHost(options);return {...config};}}; + }catch(error){await close();throw error;} +} diff --git a/examples/browser-resume/serve.mjs b/examples/browser-resume/serve.mjs new file mode 100644 index 0000000..5331dfc --- /dev/null +++ b/examples/browser-resume/serve.mjs @@ -0,0 +1,22 @@ +import * as fs from 'node:fs/promises'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {createServer} from 'node:http'; +import {createResumeFixture} from './host.mjs'; +const here=path.dirname(fileURLToPath(import.meta.url)),source=path.resolve(here,'../../src');let fixture,origin,closing=false,restarting=false; +const web=createServer(async(req,res)=>{ + try{ + const authorized=req.method==='POST'&&req.headers.origin===origin&&req.headers['x-fixture-token']===fixture?.config.token; + if(authorized&&req.url==='/close'){res.writeHead(200);res.end('Closing');setImmediate(()=>void close());return;} + if(authorized&&req.url==='/restart'){ + if(restarting){res.writeHead(409);res.end();return;}restarting=true; + try{await fixture.restart();res.writeHead(200,{'content-type':'application/json','cache-control':'no-store'});res.end(JSON.stringify({restarted:true,generation:fixture.config.generation}));}finally{restarting=false;}return; + } + if(req.method!=='GET'){res.writeHead(405);res.end();return;} + if(req.url==='/config.json'&&fixture){res.writeHead(200,{'content-type':'application/json','cache-control':'no-store'});res.end(JSON.stringify(fixture.config));return;} + let file,type='text/javascript; charset=utf-8';if(req.url==='/'){file=path.join(here,'browser.html');type='text/html; charset=utf-8';}else if(['/browser.mjs','/worker.mjs'].includes(req.url))file=path.join(here,req.url.slice(1));else if(/^\/sdk\/[a-z0-9-]+\.mjs$/.test(req.url))file=path.join(source,req.url.slice(5));else{res.writeHead(404);res.end();return;} + res.writeHead(200,{'content-type':type,'cache-control':'no-store','x-content-type-options':'nosniff'});res.end(await fs.readFile(file)); + }catch{res.writeHead(500);res.end('Fixture operation failed');} +}); +async function close(){if(closing)return;closing=true;await fixture?.close();await new Promise(resolve=>{web.close(resolve);web.closeAllConnections();});console.log('Browser resume fixture cleaned');} +try{await new Promise((resolve,reject)=>{web.once('error',reject);web.listen(0,'127.0.0.1',resolve);});origin='http://127.0.0.1:'+web.address().port;fixture=await createResumeFixture({allowedOrigins:[origin]});console.log('Browser resume example: '+origin);for(const signal of ['SIGINT','SIGTERM'])process.once(signal,()=>void close());}catch(error){await close();throw error;} diff --git a/examples/browser-resume/worker.mjs b/examples/browser-resume/worker.mjs new file mode 100644 index 0000000..0f21d01 --- /dev/null +++ b/examples/browser-resume/worker.mjs @@ -0,0 +1,24 @@ +import {createWorkspaceClient} from '/sdk/workspace-client.mjs'; +import {createTransferQueue} from '/sdk/transfer-queue.mjs'; +import {createOpfsArtifactSink,getBrowserArtifactResumeSupport,parseBrowserArtifactCheckpoint} from '/sdk/artifact-resume-browser.mjs'; +let queue,job,running=false; +onmessage=async({data})=>{ + if(data.action==='cancel'){job?.cancel();return;} + if(data.action==='continue'){job?.resume();return;} + if(running)return;running=true; + const {config,selection,action}=data,client=createWorkspaceClient({url:config.url,token:config.token}); + try{ + const support=getBrowserArtifactResumeSupport();if(!support.supported)throw Object.assign(new Error('Dedicated Worker OPFS sync access and Web Locks required'),{code:'unsupported'}); + // The UI explicitly selects this fixture-owned directory and UUID. No credential is persisted. + const root=await navigator.storage.getDirectory(),directory=await root.getDirectoryHandle(selection.directory,{create:action!=='resume'}); + await client.connect();const reference=await client.getStoredJobArtifact(config.pluginId,config.jobId,config.artifactId,config.artifactSha256),sink=createOpfsArtifactSink({directory,partialId:selection.partialId,reference}); + queue=createTransferQueue({concurrency:1,perConnection:1});let stopped=false; + queue.subscribe(value=>{postMessage({progress:value});}); + job=queue.enqueueStoredDownload(client,reference,{sink,resume:action==='resume',onProgress:p=>{ + if(!stopped&&p.receivedBytes>=65536&&(action==='checkpoint'||action==='hold')){stopped=true;if(action==='checkpoint')job.cancel();else{job.pause();postMessage({holding:true,partialId:selection.partialId,generation:reference.scope.sessionId,checkpointBytes:p.writtenBytes});}} + }}); + const receipt=await job.result,metadata=parseBrowserArtifactCheckpoint(JSON.parse(await (await directory.getFileHandle(sink.names.metadata)).getFile().then(file=>file.text()))); + postMessage({passed:receipt.verification==='stored'&&receipt.storedSha256===config.sha256,support,runtime:navigator.userAgent,partialId:selection.partialId,generation:reference.scope.sessionId,receipt,metadata}); + }catch(error){postMessage({passed:false,code:error.code??error.name,message:error.message,partial:error.partial,checkpointRetained:error.code==='cancelled'&&error.partial?.disposition==='retained'});} + finally{await queue?.dispose();queue=job=undefined;client.dispose();running=false;postMessage({idle:true});} +}; diff --git a/package.json b/package.json index 12e71b8..548240a 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,10 @@ "types": "./src/uploads-browser.d.mts", "import": "./src/uploads-browser.mjs" }, + "./artifact-resume-browser": { + "types": "./src/artifact-resume-browser.d.mts", + "import": "./src/artifact-resume-browser.mjs" + }, "./transfer-queue": { "types": "./src/transfer-queue.d.mts", "import": "./src/transfer-queue.mjs" @@ -175,7 +179,8 @@ "example:uploads-browser": "node examples/uploads/serve.mjs", "example:transfer-queue": "node examples/transfer-queue/run.mjs", "example:transfer-queue-browser": "node examples/transfer-queue/serve.mjs", - "benchmark:transfer-queue": "node scripts/benchmark-transfer-queue.mjs" + "benchmark:transfer-queue": "node scripts/benchmark-transfer-queue.mjs", + "example:browser-resume": "node examples/browser-resume/serve.mjs" }, "devDependencies": { "typescript": "5.9.3" diff --git a/schemas/browser-artifact-checkpoint.schema.json b/schemas/browser-artifact-checkpoint.schema.json new file mode 100644 index 0000000..052f69d --- /dev/null +++ b/schemas/browser-artifact-checkpoint.schema.json @@ -0,0 +1,245 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Altifigence/dds-plugin-sdk/blob/v0.10.0/schemas/browser-artifact-checkpoint.schema.json", + "title": "browser-artifact-checkpoint", + "type": "object", + "properties": { + "schemaVersion": { + "const": 1 + }, + "partialId": { + "type": "string", + "minLength": 1, + "maxLength": 36, + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "snapshot": { + "type": "object", + "properties": { + "schemaVersion": { + "const": 1 + }, + "storeId": { + "type": "string", + "minLength": 1, + "maxLength": 36, + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "workspaceId": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "workspaceIdentity": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotId": { + "type": "string", + "minLength": 1, + "maxLength": 36, + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "jobId": { + "type": "string", + "minLength": 1, + "maxLength": 36, + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + }, + "pluginId": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "pluginArtifactSha256": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "enum": [ + "text", + "binary" + ] + }, + "artifact": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "$comment": "Runtime enforces the protected workspace file policy." + }, + "revision": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-f0-9]{64}$" + }, + "byteLength": { + "type": "integer", + "minimum": 0, + "maximum": 1073741824 + } + }, + "required": [ + "id", + "path", + "revision", + "byteLength" + ], + "additionalProperties": false + }, + "capturedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "storeId", + "workspaceId", + "workspaceIdentity", + "snapshotId", + "jobId", + "pluginId", + "pluginArtifactSha256", + "kind", + "artifact", + "capturedAt", + "expiresAt" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "kind": { + "const": "text" + } + } + }, + "then": { + "properties": { + "artifact": { + "properties": { + "byteLength": { + "type": "integer", + "minimum": 0, + "maximum": 262144 + } + } + } + } + } + } + ] + }, + "createdAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "state": { + "enum": [ + "partial", + "complete" + ] + }, + "checkpoint": { + "type": "object", + "properties": { + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 1073741824 + }, + "sha256": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "offset", + "sha256" + ], + "additionalProperties": false + }, + "previousCheckpoint": { + "oneOf": [ + { + "type": "object", + "properties": { + "offset": { + "type": "integer", + "minimum": 0, + "maximum": 1073741824 + }, + "sha256": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "offset", + "sha256" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "checksum": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "schemaVersion", + "partialId", + "snapshot", + "createdAt", + "updatedAt", + "expiresAt", + "state", + "checkpoint", + "previousCheckpoint", + "checksum" + ], + "additionalProperties": false, + "x-maxUtf8Bytes": 16384, + "$comment": "Runtime checks canonical JSON SHA-256 excluding checksum, full stored identity, checkpoint offsets and empty prefix SHA, previous offset ordering, completed full hash, and retention <=24h capped by the stored artifact expiry. No token, grants or old connection scope are stored." +} diff --git a/scripts/benchmark-browser-artifacts.mjs b/scripts/benchmark-browser-artifacts.mjs index 481f7bd..1cbe75d 100644 --- a/scripts/benchmark-browser-artifacts.mjs +++ b/scripts/benchmark-browser-artifacts.mjs @@ -12,6 +12,7 @@ async function visit(url){ assert.ok(!/\b(?:require\s*\(|import\s*\(\s*['"]node:)/.test(source),'No Node runtime bridge in browser modules'); } await visit(new URL('../src/artifact-transfer.mjs',import.meta.url));await visit(new URL('../src/artifact-transfer-browser.mjs',import.meta.url)); +await visit(new URL('../src/artifact-resume-browser.mjs',import.meta.url)); const transferSource=[...visited.values()].join('\n'),rawBytes=Buffer.byteLength(transferSource),gzipBytes=gzipSync(transferSource).length; assert.ok(rawBytes<250_000&&gzipBytes<60_000,'Browser transfer graph exceeded its initial regression budget'); const small=await runStreamingArtifactExample({fileBytes:2_097_169}); diff --git a/scripts/pack-check.mjs b/scripts/pack-check.mjs index 115f118..74f4c26 100644 --- a/scripts/pack-check.mjs +++ b/scripts/pack-check.mjs @@ -196,7 +196,7 @@ try { await cp(join(root, 'tests', 'types', 'jobs-devtools.mts'), join(consumer, 'jobs-devtools.mts')); await cp(join(root, 'tests', 'types', 'workspace-observation.mts'), join(consumer, 'workspace-observation.mts')); await cp(join(root, 'tests', 'types', 'artifacts.mts'), join(consumer, 'artifacts.mts')); - for(const name of ['job-storage.mts','job-history.mts','artifact-storage.mts','project-watch.mts','project-query.mts','workspace-project-tools.mts','artifact-transfer.mts']) await cp(join(root,'tests','types',name),join(consumer,name)); + for(const name of ['job-storage.mts','job-history.mts','artifact-storage.mts','project-watch.mts','project-query.mts','workspace-project-tools.mts','artifact-transfer.mts','artifact-resume-browser.mts']) await cp(join(root,'tests','types',name),join(consumer,name)); run(process.execPath, [join(installed, 'examples', 'binary-artifacts', 'run.mjs')], consumer); await cp(join(root, 'tests', 'types', 'tsconfig.json'), join(consumer, 'tsconfig.json')); const typescript = join(root, 'node_modules', 'typescript', 'bin', 'tsc'); diff --git a/src/artifact-resume-browser.d.mts b/src/artifact-resume-browser.d.mts new file mode 100644 index 0000000..b32847c --- /dev/null +++ b/src/artifact-resume-browser.d.mts @@ -0,0 +1,24 @@ +import type {StoredArtifact,StoredArtifactReference} from './artifact-storage.mjs'; +import type {ArtifactSink} from './artifact-transfer.mjs'; +export const BROWSER_ARTIFACT_RESUME_LIMITS:Readonly<{fileBytes:1073741824;chunkBytes:65536;metadataBytes:16384;concurrent:4;retentionMs:86400000}>; +export const DEFAULT_BROWSER_ARTIFACT_RETENTION_MS:3600000; +export interface BrowserArtifactCheckpoint { + readonly schemaVersion:1;readonly partialId:string;readonly snapshot:StoredArtifact; + readonly createdAt:number;readonly updatedAt:number;readonly expiresAt:number;readonly state:'partial'|'complete'; + readonly checkpoint:Readonly<{offset:number;sha256:string}>;readonly previousCheckpoint:Readonly<{offset:number;sha256:string}>|null;readonly checksum:string; +} +/** Structural ports are trusted. Native sync access is available only in a dedicated Worker. */ +export interface OpfsSyncAccessHandle {read(buffer:Uint8Array,options:{at:number}):number;write(buffer:Uint8Array,options:{at:number}):number;getSize():number;truncate(size:number):void;flush():void;close():void;} +export interface OpfsCheckpointFileHandle { + readonly kind:'file';getFile():Promise<{readonly size:number;arrayBuffer():Promise}>; + createSyncAccessHandle?():Promise; + createWritable(options:{keepExistingData:boolean}):Promise<{write(data:Uint8Array):Promise;close():Promise;abort():Promise}>; +} +export interface OpfsCheckpointDirectory {readonly kind:'directory';getFileHandle(name:string,options?:{create?:boolean}):Promise;} +export interface BrowserCheckpointLocks {request(name:string,options:{mode:'exclusive';ifAvailable:true},callback:(lock:unknown|null)=>Promise):Promise;} +export interface OpfsArtifactSinkOptions {readonly directory:OpfsCheckpointDirectory;readonly partialId:string;readonly reference:StoredArtifactReference;readonly retentionMs?:number;readonly locks?:BrowserCheckpointLocks;} +export interface OpfsArtifactSink extends ArtifactSink {readonly partialId:string;readonly names:Readonly<{data:string;metadata:string}>;} +export function createOpfsArtifactSink(options:OpfsArtifactSinkOptions):OpfsArtifactSink; +export function getOpfsArtifactPartialNames(partialId:string):Readonly<{data:string;metadata:string}>; +export function getBrowserArtifactResumeSupport():Readonly<{supported:boolean;dedicatedWorker:boolean;syncAccess:boolean;webLocks:boolean}>; +export function parseBrowserArtifactCheckpoint(value:unknown):BrowserArtifactCheckpoint; diff --git a/src/artifact-resume-browser.mjs b/src/artifact-resume-browser.mjs new file mode 100644 index 0000000..014b4cc --- /dev/null +++ b/src/artifact-resume-browser.mjs @@ -0,0 +1,175 @@ +import {parseStoredArtifact,parseStoredArtifactReference} from './artifact-storage.mjs'; +import {parseBinaryArtifact} from './artifacts.mjs'; +import {parseArtifactSinkCapabilities} from './artifact-transfer.mjs'; +import {createIncrementalSha256,snapshotSha256} from './sha256-stream.mjs'; +import {storageEqual} from './job-storage-validation.mjs'; +import {workspaceFailure,exactObject,copyWorkspaceJson,requireUuid,requireSha256} from './workspace-values.mjs'; + +export const BROWSER_ARTIFACT_RESUME_LIMITS=Object.freeze({fileBytes:1_073_741_824,chunkBytes:65_536,metadataBytes:16_384,concurrent:4,retentionMs:86_400_000}); +export const DEFAULT_BROWSER_ARTIFACT_RETENTION_MS=3_600_000; +const encoder=new TextEncoder(),decoder=new TextDecoder('utf-8',{fatal:true}); +const active=new Set(),emptySha=createIncrementalSha256().digest(); +const integer=(v,min=0,max=Number.MAX_SAFE_INTEGER)=>{if(!Number.isSafeInteger(v)||vmax)throw workspaceFailure('invalid_request','Invalid browser checkpoint number');return v;}; +const canonical=value=>value===null||typeof value!=='object'?JSON.stringify(value):Array.isArray(value)?`[${value.map(canonical).join(',')}]`:`{${Object.keys(value).sort().map(key=>`${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; +const checksum=value=>createIncrementalSha256().update(encoder.encode(canonical(value))).digest(); +const signalCheck=signal=>{if(signal?.aborted)throw workspaceFailure('cancelled','Browser checkpoint operation cancelled');}; +const yieldTask=()=>new Promise(resolve=>setTimeout(resolve,0)); +function checkpoint(value,size){exactObject(value,['offset','sha256']);integer(value.offset,0,size);requireSha256(value.sha256);if(value.offset===0&&value.sha256!==emptySha)throw workspaceFailure('invalid_request','Invalid empty checkpoint');} +export function parseBrowserArtifactCheckpoint(value){ + const v=copyWorkspaceJson(value,{maxBytes:BROWSER_ARTIFACT_RESUME_LIMITS.metadataBytes}); + exactObject(v,['schemaVersion','partialId','snapshot','createdAt','updatedAt','expiresAt','state','checkpoint','previousCheckpoint','checksum']); + if(v.schemaVersion!==1||!['partial','complete'].includes(v.state))throw workspaceFailure('invalid_request','Invalid browser checkpoint state'); + requireUuid(v.partialId);const s=parseStoredArtifact(v.snapshot);requireSha256(v.checksum); + integer(v.createdAt);integer(v.updatedAt,v.createdAt);integer(v.expiresAt,v.createdAt+1,Math.min(v.createdAt+BROWSER_ARTIFACT_RESUME_LIMITS.retentionMs,s.expiresAt)); + if(v.updatedAt>=v.expiresAt)throw workspaceFailure('invalid_request','Invalid browser checkpoint time'); + checkpoint(v.checkpoint,s.artifact.byteLength); + if(v.previousCheckpoint!==null){checkpoint(v.previousCheckpoint,s.artifact.byteLength);if(v.previousCheckpoint.offset>=v.checkpoint.offset)throw workspaceFailure('invalid_request','Invalid previous checkpoint');} + if(v.state==='complete'&&(v.checkpoint.offset!==s.artifact.byteLength||v.checkpoint.sha256!==s.artifact.revision))throw workspaceFailure('invalid_request','Invalid completed checkpoint'); + const {checksum:expected,...body}=v; + if(checksum(body)!==expected||encoder.encode(JSON.stringify(v)).length>BROWSER_ARTIFACT_RESUME_LIMITS.metadataBytes)throw workspaceFailure('conflict','Browser checkpoint checksum mismatch'); + return v; +} +export function getOpfsArtifactPartialNames(partialId){requireUuid(partialId);return Object.freeze({data:`dds-partial-${partialId}.data`,metadata:`dds-partial-${partialId}.json`});} +export function getBrowserArtifactResumeSupport(){ + const dedicatedWorker=typeof DedicatedWorkerGlobalScope!=='undefined'&&globalThis instanceof DedicatedWorkerGlobalScope; + const syncAccess=typeof FileSystemFileHandle!=='undefined'&&typeof FileSystemFileHandle.prototype.createSyncAccessHandle==='function'; + const webLocks=typeof globalThis.navigator?.locks?.request==='function'; + return Object.freeze({supported:dedicatedWorker&&syncAccess&&webLocks,dedicatedWorker,syncAccess,webLocks}); +} +async function findFile(directory,name){try{return await directory.getFileHandle(name);}catch(error){if(error?.name==='NotFoundError')return null;throw error;}} +async function readMetadata(handle){ + const file=await handle.getFile(); + if(file.size<1||file.size>BROWSER_ARTIFACT_RESUME_LIMITS.metadataBytes)throw workspaceFailure('conflict','Missing or oversized browser checkpoint'); + const bytes=new Uint8Array(await file.arrayBuffer()); + if(bytes.length!==file.size)throw workspaceFailure('conflict','Browser checkpoint size changed'); + try{const text=decoder.decode(bytes);return{text,value:parseBrowserArtifactCheckpoint(JSON.parse(text))};} + catch(error){if(error?.code)throw error;throw workspaceFailure('conflict','Malformed browser checkpoint');} +} +async function holdLock(locks,name,signal){ + let resolveAcquired,rejectAcquired,release; + const acquired=new Promise((resolve,reject)=>{resolveAcquired=resolve;rejectAcquired=reject;}); + const held=new Promise(resolve=>{release=resolve;}); + // Web Locks forbids combining ifAvailable and signal. No waiting queue or lock stealing. + const task=Promise.resolve().then(()=>locks.request(name,{mode:'exclusive',ifAvailable:true},async lock=>{ + if(!lock)throw workspaceFailure('conflict','Another tab or worker owns this partial'); + signalCheck(signal);resolveAcquired();await held; + })); + task.catch(rejectAcquired); + await acquired; + return async()=>{release();await task;}; +} + +/** Native use requires an OPFS directory in a dedicated Worker. Structural ports are trusted. */ +export function createOpfsArtifactSink(options){ + exactObject(options,['directory','partialId','reference'],['retentionMs','locks']); + const {directory,partialId,retentionMs=DEFAULT_BROWSER_ARTIFACT_RETENTION_MS}=options; + const reference=parseStoredArtifactReference(options.reference),snapshot=reference.snapshot,names=getOpfsArtifactPartialNames(partialId),locks=options.locks??globalThis.navigator?.locks; + integer(retentionMs,1,BROWSER_ARTIFACT_RESUME_LIMITS.retentionMs); + if(!directory||directory.kind!=='directory'||typeof directory.getFileHandle!=='function'||typeof locks?.request!=='function')throw workspaceFailure('unsupported','OPFS directory and Web Locks are required'); + if(typeof Window!=='undefined'&&globalThis instanceof Window||typeof FileSystemFileHandle!=='undefined'&&typeof FileSystemFileHandle.prototype.createSyncAccessHandle!=='function')throw workspaceFailure('unsupported','Persistent OPFS downloads require a dedicated Worker'); + const capabilities=parseArtifactSinkCapabilities({kind:'opfs',seek:true,readback:true,persistence:'per-checkpoint',abort:'retain'}); + return Object.freeze({capabilities,partialId,names,async open(input){ + exactObject(input,['artifact','signal','storedReference'],['resume']); + const {signal,resume=false}=input; + if(typeof resume!=='boolean'||!(signal instanceof AbortSignal))throw workspaceFailure('invalid_request','Invalid checkpoint open options'); + if(!storageEqual(parseBinaryArtifact(input.artifact),snapshot.artifact)||!storageEqual(parseStoredArtifactReference(input.storedReference),reference))throw workspaceFailure('conflict','Current approved artifact reference differs from the selected partial'); + signalCheck(signal); + if(Date.now()>=snapshot.expiresAt)throw workspaceFailure('unavailable','Stored artifact has expired'); + if(active.size>=BROWSER_ARTIFACT_RESUME_LIMITS.concurrent)throw workspaceFailure('budget_exceeded','Browser checkpoint capacity exceeded'); + const ticket={};active.add(ticket); + let releaseLock,access,metadataHandle,dataHandle,record,expectedText,hash,offset=0,closed=false,poisoned=false,busy=false,closing=false,closePromise,pendingOperation; + const close=()=>{ + if(closePromise)return closePromise;closing=true; + closePromise=(async()=>{ + // A metadata close already in flight may still publish. Retain ownership until it settles. + try{await pendingOperation;}catch{} + closed=true;try{access?.close();}finally{try{await releaseLock?.();}finally{active.delete(ticket);}} + })();return closePromise; + }; + function current(operationSignal){ + signalCheck(signal);signalCheck(operationSignal); + if(closed||closing||poisoned)throw workspaceFailure('disposed','Browser checkpoint session is closed or requires recovery'); + if(Date.now()>=(record?.expiresAt??snapshot.expiresAt))throw workspaceFailure('unavailable','Browser checkpoint has expired'); + } + async function unchanged(){ + if(expectedText===undefined)return; + if((await readMetadata(metadataHandle)).text!==expectedText)throw workspaceFailure('conflict','Browser checkpoint changed outside this session'); + } + async function save(body,operationSignal){ + current(operationSignal);await unchanged();current(operationSignal); + const next=parseBrowserArtifactCheckpoint({...body,checksum:checksum(body)}),text=JSON.stringify(next),bytes=encoder.encode(text); + let writable; + try{ + writable=await metadataHandle.createWritable({keepExistingData:false});current(operationSignal); + await writable.write(bytes);current(operationSignal);await writable.close(); + const observed=await readMetadata(metadataHandle); + if(observed.text!==text)throw workspaceFailure('conflict','Browser checkpoint did not persist the expected bytes'); + record=next;expectedText=text;current(operationSignal); + }catch(error){poisoned=true;try{await writable?.abort();}catch{}throw error;} + } + const read=(at,length,maximum)=>{ + integer(at,0,maximum);integer(length,0,BROWSER_ARTIFACT_RESUME_LIMITS.chunkBytes); + if(at+length>maximum)throw workspaceFailure('invalid_request','Checkpoint read is out of range'); + const bytes=new Uint8Array(length); + if(access.read(bytes,{at})!==length)throw workspaceFailure('conflict','Browser partial size changed');return bytes; + }; + const operation=(operationSignal,run)=>{ + try{current(operationSignal);if(busy)throw workspaceFailure('conflict','Concurrent checkpoint operations are not supported');}catch(error){return Promise.reject(error);}busy=true; + pendingOperation=(async()=>{try{await unchanged();current(operationSignal);return await run();}finally{busy=false;}})();return pendingOperation; + }; + try{ + releaseLock=await holdLock(locks,`dds-artifact-partial:${partialId}`,signal);current(signal); + dataHandle=await findFile(directory,names.data);metadataHandle=await findFile(directory,names.metadata);current(signal); + if(resume&&(!dataHandle||!metadataHandle))throw workspaceFailure('not_found','The selected browser partial or checkpoint is missing'); + if(!resume&&(dataHandle||metadataHandle))throw workspaceFailure('conflict','This partial ID already exists; select explicit recovery or a new ID'); + if(!resume){dataHandle=await directory.getFileHandle(names.data,{create:true});metadataHandle=await directory.getFileHandle(names.metadata,{create:true});} + if(typeof dataHandle.createSyncAccessHandle!=='function')throw workspaceFailure('unsupported','Selected file does not support Worker sync access'); + access=await dataHandle.createSyncAccessHandle();current(signal); + hash=createIncrementalSha256(); + if(resume){ + const saved=await readMetadata(metadataHandle);record=saved.value;expectedText=saved.text;current(signal); + if(record.partialId!==partialId||!storageEqual(record.snapshot,snapshot))throw workspaceFailure('conflict','Browser partial belongs to a different stored artifact'); + const size=integer(access.getSize(),0,BROWSER_ARTIFACT_RESUME_LIMITS.fileBytes); + let selected=record.checkpoint,fallback=false; + if(selected.offset>size){selected=record.previousCheckpoint;fallback=true;if(!selected||selected.offset>size)throw workspaceFailure('conflict','Browser partial is shorter than its recoverable checkpoints');} + while(offsetread(at,length,recovered.offset));}}), + async write({offset:at,bytes,signal:writeSignal}){return operation(writeSignal,async()=>{ + if(at!==offset||!(bytes instanceof Uint8Array)||typeof SharedArrayBuffer!=='undefined'&&bytes.buffer instanceof SharedArrayBuffer||bytes.length>BROWSER_ARTIFACT_RESUME_LIMITS.chunkBytes||offset+bytes.length>snapshot.artifact.byteLength)throw workspaceFailure('invalid_request','Invalid sequential checkpoint write'); + if(access.getSize()!==offset)throw workspaceFailure('conflict','Browser partial size changed'); + if(bytes.length===0)return; + if(record.state==='complete')throw workspaceFailure('conflict','Browser partial is already complete'); + try{ + if(access.write(bytes,{at})!==bytes.length)throw workspaceFailure('unavailable','Browser partial write was incomplete'); + access.flush();current(writeSignal);hash.update(bytes); + const {checksum:_,...body}=record; + await save({...body,updatedAt:Math.max(record.updatedAt,Date.now()),checkpoint:{offset:offset+bytes.length,sha256:snapshotSha256(hash)},previousCheckpoint:record.checkpoint},writeSignal); + offset+=bytes.length; + }catch(error){poisoned=true;throw error;} + });}, + async commit({signal:commitSignal}){return operation(commitSignal,async()=>{ + if(offset!==snapshot.artifact.byteLength||access.getSize()!==offset||snapshotSha256(hash)!==snapshot.artifact.revision)throw workspaceFailure('conflict','Cannot complete an unverified browser partial'); + access.flush();const {checksum:_,...body}=record; + await save({...body,state:'complete',updatedAt:Math.max(record.updatedAt,Date.now())},commitSignal); + });}, + async readback({signal:readSignal}){return operation(readSignal,()=>{ + if(record.state!=='complete'||access.getSize()!==snapshot.artifact.byteLength)throw workspaceFailure('conflict','Browser partial is not complete'); + return Object.freeze({byteLength:offset,async read(at,length,{signal:sliceSignal}){return operation(sliceSignal,()=>read(at,length,offset));}}); + });}, + async abort(){await close();},close, + }); + }catch(error){await close();throw error;} + }}); +} diff --git a/src/artifact-transfer-core.mjs b/src/artifact-transfer-core.mjs index ac34975..73a140c 100644 --- a/src/artifact-transfer-core.mjs +++ b/src/artifact-transfer-core.mjs @@ -2,7 +2,7 @@ import {BINARY_ARTIFACT_LIMITS,parseBinaryArtifact,parseBinaryArtifactReference, import {parseStoredArtifactReference,parseStoredArtifactChunk,parseArtifactStorageCapabilities} from './artifact-storage.mjs'; import {storageEqual} from './job-storage-validation.mjs'; import {WORKSPACE_LIMITS,WorkspaceError,workspaceFailure,exactObject,copyWorkspaceJson,requireSha256} from './workspace-values.mjs'; -import {createIncrementalSha256} from './sha256-stream.mjs'; +import {createIncrementalSha256,snapshotSha256} from './sha256-stream.mjs'; import {PluginSdkError} from './limits.mjs'; export {createIncrementalSha256} from './sha256-stream.mjs'; @@ -42,13 +42,14 @@ const yieldTask=()=>new Promise(resolve=>setTimeout(resolve,0)); /** Browser-safe, bounded sequential download. Sink ports are trusted caller implementations. */ export function streamJobBinaryArtifact(client,value,options){return runJobBinaryArtifact(client,value,options);} -export async function runJobBinaryArtifact(client,value,options,control){ +export async function runJobBinaryArtifact(client,value,options,control,storedReference){ const reference=parseBinaryArtifactReference(value); - exactObject(options,['sink'],['signal','timeoutMs','requestTimeoutMs','onProgress']); - const {sink,signal,timeoutMs=ARTIFACT_TRANSFER_LIMITS.timeoutMs,requestTimeoutMs=WORKSPACE_LIMITS.defaultTimeoutMs,onProgress}=options; + exactObject(options,['sink'],['resume','signal','timeoutMs','requestTimeoutMs','onProgress']); + const {sink,resume=false,signal,timeoutMs=ARTIFACT_TRANSFER_LIMITS.timeoutMs,requestTimeoutMs=WORKSPACE_LIMITS.defaultTimeoutMs,onProgress}=options; integer(timeoutMs,ARTIFACT_TRANSFER_LIMITS.timeoutMs,1);integer(requestTimeoutMs,WORKSPACE_LIMITS.maxTimeoutMs,1); - if(signal!==undefined&&!(signal instanceof AbortSignal)||onProgress!==undefined&&typeof onProgress!=='function'||!sink||typeof sink.open!=='function'||!client||['getBinaryArtifactCapabilities','getJobBinaryArtifact','readJobBinaryArtifactChunk'].some(key=>typeof client[key]!=='function'))throw workspaceFailure('invalid_request','Invalid stream download options'); + if(typeof resume!=='boolean'||signal!==undefined&&!(signal instanceof AbortSignal)||onProgress!==undefined&&typeof onProgress!=='function'||!sink||typeof sink.open!=='function'||!client||['getBinaryArtifactCapabilities','getJobBinaryArtifact','readJobBinaryArtifactChunk'].some(key=>typeof client[key]!=='function'))throw workspaceFailure('invalid_request','Invalid stream download options'); const capabilities=parseArtifactSinkCapabilities(sink.capabilities),binding=client.binding,controller=new AbortController(); + if(resume&&(!capabilities.seek||!capabilities.readback||capabilities.persistence!=='per-checkpoint'))throw workspaceFailure('unsupported','This sink does not support verified checkpoint recovery'); const abort=()=>controller.abort(workspaceFailure('cancelled','Artifact transfer cancelled')); if(signal?.aborted)abort();else signal?.addEventListener('abort',abort,{once:true}); const expiresAt=Date.now()+timeoutMs; @@ -61,7 +62,7 @@ export async function runJobBinaryArtifact(client,value,options,control){ }; const requestOptions={signal:controller.signal,timeoutMs:requestTimeoutMs},started=performance.now(); const step=(run,metadata={})=>control?control.step(async()=>{current();return run();},metadata,controller.signal):run(); - const partial={receivedBytes:0,writtenBytes:0,receivedVerified:false,storedVerified:false,commit:'not-committed',disposition:'unknown'}; + const partial={receivedBytes:0,resumedBytes:0,writtenBytes:0,receivedVerified:false,storedVerified:false,commit:'not-committed',disposition:'unknown'}; const metrics={chunks:0,peakQueuedChunks:0,peakDecodedBytes:0,maxChunkWorkMs:0,elapsedMs:0}; let session,registered=false; try{ @@ -74,9 +75,27 @@ export async function runJobBinaryArtifact(client,value,options,control){ if(reference.artifact.byteLength>caps.limits.fileBytes)throw workspaceFailure('budget_exceeded','Artifact exceeds the host file budget'); const fresh=parseBinaryArtifactReference(await step(()=>client.getJobBinaryArtifact(reference.jobId,reference.artifact.id,requestOptions),{phase:'preparing'}));current(); if(!storageEqual(fresh,reference))throw workspaceFailure('conflict','Artifact reference changed'); - session=await step(()=>sink.open({artifact:reference.artifact,signal:controller.signal}),{phase:'preparing'});current(); + session=await step(()=>sink.open({artifact:reference.artifact,signal:controller.signal,...(resume?{resume:true}:{}),...(storedReference&&capabilities.persistence==='per-checkpoint'?{storedReference}:{})}),{phase:'preparing'});current(); if(!session||['write','commit','abort'].some(key=>typeof session[key]!=='function')||capabilities.readback&&typeof session.readback!=='function')throw workspaceFailure('invalid_request','Invalid artifact sink session'); const hash=createIncrementalSha256();let offset=0; + if(resume){ + const recovery=session.recovery; + if(!recovery||typeof recovery.read!=='function')throw workspaceFailure('unsupported','The sink did not provide checkpoint recovery'); + integer(recovery.offset,reference.artifact.byteLength);requireSha256(recovery.sha256); + const checkpoint={offset:recovery.offset,sha256:recovery.sha256}; + progress(onProgress,{phase:'verifying-prefix',receivedBytes:0,writtenBytes:0,totalBytes:reference.artifact.byteLength,resumedBytes:0,verified:false});current(); + while(offset{ + const length=Math.min(ARTIFACT_TRANSFER_LIMITS.chunkBytes,checkpoint.offset-offset),bytes=await recovery.read(offset,length,{signal:controller.signal});current(); + if(!(bytes instanceof Uint8Array)||bytes.length!==length)throw workspaceFailure('conflict','Recovered artifact prefix size changed'); + const workStarted=performance.now();hash.update(bytes);metrics.maxChunkWorkMs=Math.max(metrics.maxChunkWorkMs,performance.now()-workStarted);offset+=bytes.length; + },{phase:'verifying-prefix'});await yieldTask();current(); + } + // Independently validate the saved bytes without trusting the sink's offset/hash assertion. + if(snapshotSha256(hash)!==checkpoint.sha256)throw workspaceFailure('conflict','Recovered artifact prefix SHA-256 mismatch'); + partial.resumedBytes=offset;partial.writtenBytes=offset; + progress(onProgress,{phase:'verifying-prefix',receivedBytes:0,writtenBytes:offset,totalBytes:reference.artifact.byteLength,resumedBytes:offset,verified:false});current(); + } for(;;){ let eof,retryError; await step(async()=>{ @@ -89,18 +108,18 @@ export async function runJobBinaryArtifact(client,value,options,control){ hash.update(bytes);partial.receivedBytes+=bytes.length;metrics.chunks++;metrics.peakQueuedChunks=1;metrics.peakDecodedBytes=Math.max(metrics.peakDecodedBytes,bytes.length);metrics.maxChunkWorkMs=Math.max(metrics.maxChunkWorkMs,performance.now()-workStarted); // Do not request the next chunk until the sink has accepted this one. await session.write({offset,bytes,signal:controller.signal});partial.writtenBytes+=bytes.length;current();offset=chunk.nextOffset; - progress(onProgress,{phase:'receiving',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:artifact.byteLength,resumedBytes:0,verified:false});current(); + progress(onProgress,{phase:'receiving',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:artifact.byteLength,resumedBytes:partial.resumedBytes,verified:false});current(); eof=chunk.eof; },{phase:'receiving',bytes:Math.min(caps.limits.chunkBytes,reference.artifact.byteLength-offset),canRetry:error=>error===retryError}); await yieldTask();current();if(eof)break; } const receivedSha256=hash.digest(); if(receivedSha256!==reference.artifact.revision)throw workspaceFailure('conflict','Whole artifact SHA-256 mismatch'); - partial.receivedVerified=true;progress(onProgress,{phase:'committing',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:reference.artifact.byteLength,resumedBytes:0,verified:false});current(); + partial.receivedVerified=true;progress(onProgress,{phase:'committing',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:reference.artifact.byteLength,resumedBytes:partial.resumedBytes,verified:false});current(); await step(async()=>{partial.commit='unknown';await session.commit({signal:controller.signal});partial.commit='committed';current();},{phase:'committing'}); let storedSha256=null; if(capabilities.readback){ - progress(onProgress,{phase:'verifying-storage',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:reference.artifact.byteLength,resumedBytes:0,verified:false});current(); + progress(onProgress,{phase:'verifying-storage',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:reference.artifact.byteLength,resumedBytes:partial.resumedBytes,verified:false});current(); const reader=await step(()=>session.readback({signal:controller.signal}),{phase:'verifying-storage'});current(); if(!reader||reader.byteLength!==reference.artifact.byteLength||typeof reader.read!=='function')throw workspaceFailure('conflict','Stored artifact size changed'); const storedHash=createIncrementalSha256();let offset=0; @@ -115,8 +134,8 @@ export async function runJobBinaryArtifact(client,value,options,control){ } if(typeof session.close==='function'){await step(()=>session.close(),{phase:'verifying-storage'});current();} metrics.elapsedMs=performance.now()-started; - const receipt=parseArtifactTransferReceipt({protocolVersion:1,artifact:reference.artifact,receivedBytes:partial.receivedBytes,resumedBytes:0,receivedSha256,storedSha256,verification:storedSha256===null?'received':'stored',committed:true,sink:capabilities,metrics}); - progress(onProgress,{phase:'completed',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:reference.artifact.byteLength,resumedBytes:0,verified:true});current();return receipt; + const receipt=parseArtifactTransferReceipt({protocolVersion:1,artifact:reference.artifact,receivedBytes:partial.receivedBytes,resumedBytes:partial.resumedBytes,receivedSha256,storedSha256,verification:storedSha256===null?'received':'stored',committed:true,sink:capabilities,metrics}); + progress(onProgress,{phase:'completed',receivedBytes:partial.receivedBytes,writtenBytes:partial.writtenBytes,totalBytes:reference.artifact.byteLength,resumedBytes:partial.resumedBytes,verified:true});current();return receipt; }catch(error){ if(session&&typeof session.abort==='function'){ try{await session.abort({reason:controller.signal.aborted?controller.signal.reason:error});partial.disposition=partial.commit==='not-committed'?({discard:'discarded',retain:'retained',unknown:'unknown'}[capabilities.abort]):'retained';}catch{partial.disposition='unknown';} @@ -137,5 +156,5 @@ export async function runStoredJobArtifact(client,value,options,control){ async getJobBinaryArtifact(_jobId,_artifactId,requestOptions){const current=parseStoredArtifactReference(await client.getStoredJobArtifact(s.pluginId,s.jobId,s.artifact.id,s.pluginArtifactSha256,requestOptions));if(!storageEqual(current,reference))throw workspaceFailure('conflict','Stored artifact identity changed');return legacy;}, async readJobBinaryArtifactChunk(_reference,offset,requestOptions){const result=parseStoredArtifactChunk(await client.readStoredJobArtifactChunk(reference,offset,requestOptions));if(!storageEqual(result.reference,reference))throw workspaceFailure('conflict','Stored artifact identity changed');const {reference:_,...chunk}=result;return{...legacy,...chunk};}, }; - return runJobBinaryArtifact(bridge,legacy,options,control); + return runJobBinaryArtifact(bridge,legacy,options,control,reference); } diff --git a/src/artifact-transfer.d.mts b/src/artifact-transfer.d.mts index 31e8c10..d010cf1 100644 --- a/src/artifact-transfer.d.mts +++ b/src/artifact-transfer.d.mts @@ -8,6 +8,7 @@ export function createIncrementalSha256():IncrementalSha256; export interface ArtifactSinkCapabilities {readonly kind:'caller'|'file-system-access'|'opfs';readonly seek:boolean;readonly readback:boolean;readonly persistence:'none'|'on-commit'|'per-checkpoint';readonly abort:'discard'|'retain'|'unknown';} export interface ArtifactSinkReader {readonly byteLength:number;read(offset:number,length:number,options:{readonly signal:AbortSignal}):Promise;} export interface ArtifactSinkSession { + readonly recovery?:{readonly offset:number;readonly sha256:string;read(offset:number,length:number,options:{readonly signal:AbortSignal}):Promise}; write(input:{readonly offset:number;readonly bytes:Uint8Array;readonly signal:AbortSignal}):Promise; commit(options:{readonly signal:AbortSignal}):Promise; abort(options:{readonly reason:unknown}):Promise; @@ -15,13 +16,13 @@ export interface ArtifactSinkSession { close?():Promise; } /** A trusted caller port must not retain unbounded chunks and must cooperate with cancellation. */ -export interface ArtifactSink {readonly capabilities:ArtifactSinkCapabilities;open(options:{readonly artifact:BinaryArtifact;readonly signal:AbortSignal}):Promise;} -export interface ArtifactTransferProgress {readonly phase:'receiving'|'committing'|'verifying-storage'|'completed';readonly receivedBytes:number;readonly writtenBytes:number;readonly totalBytes:number;readonly resumedBytes:number;readonly verified:boolean;} +export interface ArtifactSink {readonly capabilities:ArtifactSinkCapabilities;open(options:{readonly artifact:BinaryArtifact;readonly signal:AbortSignal;readonly resume?:boolean;readonly storedReference?:StoredArtifactReference}):Promise;} +export interface ArtifactTransferProgress {readonly phase:'verifying-prefix'|'receiving'|'committing'|'verifying-storage'|'completed';readonly receivedBytes:number;readonly writtenBytes:number;readonly totalBytes:number;readonly resumedBytes:number;readonly verified:boolean;} export interface ArtifactTransferMetrics {readonly chunks:number;readonly peakQueuedChunks:number;readonly peakDecodedBytes:number;readonly maxChunkWorkMs:number;readonly elapsedMs:number;} export interface ArtifactTransferReceipt {readonly protocolVersion:1;readonly artifact:BinaryArtifact;readonly receivedBytes:number;readonly resumedBytes:number;readonly receivedSha256:string;readonly storedSha256:string|null;readonly verification:'received'|'stored';readonly committed:true;readonly sink:ArtifactSinkCapabilities;readonly metrics:ArtifactTransferMetrics;} -export interface ArtifactTransferPartial {readonly receivedBytes:number;readonly writtenBytes:number;readonly receivedVerified:boolean;readonly storedVerified:boolean;readonly commit:'not-committed'|'unknown'|'committed';readonly disposition:'discarded'|'retained'|'unknown';} +export interface ArtifactTransferPartial {readonly receivedBytes:number;readonly resumedBytes?:number;readonly writtenBytes:number;readonly receivedVerified:boolean;readonly storedVerified:boolean;readonly commit:'not-committed'|'unknown'|'committed';readonly disposition:'discarded'|'retained'|'unknown';} export class ArtifactTransferError extends WorkspaceError {constructor(code:string,message:string,partial:ArtifactTransferPartial);readonly partial:ArtifactTransferPartial;} -export interface ArtifactTransferOptions {readonly sink:ArtifactSink;readonly signal?:AbortSignal;readonly timeoutMs?:number;readonly requestTimeoutMs?:number;readonly onProgress?:(progress:ArtifactTransferProgress)=>void;} +export interface ArtifactTransferOptions {readonly sink:ArtifactSink;readonly resume?:boolean;readonly signal?:AbortSignal;readonly timeoutMs?:number;readonly requestTimeoutMs?:number;readonly onProgress?:(progress:ArtifactTransferProgress)=>void;} export type BinaryArtifactClient=Pick; export type StoredArtifactClient=Pick; export function parseArtifactSinkCapabilities(value:unknown):ArtifactSinkCapabilities; diff --git a/src/schemas.d.mts b/src/schemas.d.mts index c50b3ff..46dd46a 100644 --- a/src/schemas.d.mts +++ b/src/schemas.d.mts @@ -1 +1 @@ -export const SCHEMAS: Readonly>>>; +export const SCHEMAS: Readonly>>>; diff --git a/src/schemas.mjs b/src/schemas.mjs index 5fc211e..be405ab 100644 --- a/src/schemas.mjs +++ b/src/schemas.mjs @@ -103,6 +103,10 @@ const historyItem = object({jobId,commandId:text(128),state:{enum:JOB_STATES},di export const SCHEMAS = Object.freeze({ ...Object.fromEntries(Object.entries({...PROJECT_WATCH_SCHEMAS,...PROJECT_QUERY_SCHEMAS,...WORKSPACE_PROJECT_SCHEMAS,...ARTIFACT_TRANSFER_SCHEMAS,...UPLOAD_SCHEMAS,...TRANSFER_QUEUE_SCHEMAS}).map(([name,body])=>[name,define(name,body)])), 'stored-artifact':define('stored-artifact',{...storedArtifact,$comment:'Runtime checks identity, labels are excluded, and capturedAt/expiresAt ordering and maximum retention.'}), + 'browser-artifact-checkpoint':define('browser-artifact-checkpoint',{ + ...object({schemaVersion:{const:1},partialId:jobId,snapshot:storedArtifact,createdAt:integer(Number.MAX_SAFE_INTEGER),updatedAt:integer(Number.MAX_SAFE_INTEGER),expiresAt:integer(Number.MAX_SAFE_INTEGER,1),state:{enum:['partial','complete']},checkpoint:object({offset:integer(BINARY_ARTIFACT_LIMITS.fileBytes),sha256:storageSha}),previousCheckpoint:{oneOf:[object({offset:integer(BINARY_ARTIFACT_LIMITS.fileBytes),sha256:storageSha}),{type:'null'}]},checksum:storageSha}), + 'x-maxUtf8Bytes':16384,$comment:'Runtime checks canonical JSON SHA-256 excluding checksum, full stored identity, checkpoint offsets and empty prefix SHA, previous offset ordering, completed full hash, and retention <=24h capped by the stored artifact expiry. No token, grants or old connection scope are stored.' + }), 'stored-artifact-reference':define('stored-artifact-reference',{...storedReference,$comment:'Current response scope binds the workspace but does not reuse an old session.'}), 'stored-artifact-chunk':define('stored-artifact-chunk',{...object({reference:storedReference,...binaryChunk}),$comment:'Runtime validates exact offsets/EOF, canonical base64 and content size. Hosts and clients verify chunk SHA-256; download publication verifies the whole file.'}), 'stored-artifact-list':define('stored-artifact-list',{...object({protocolVersion:{const:1},scope,storeId:jobId,jobId,pluginId:text(128),pluginArtifactSha256:storageSha,disposition:{enum:['live','completed','interrupted','expired','missing','corrupt','unsupported']},artifacts:list(storedListEntry,JOB_LIMITS.artifacts)}),$comment:'Runtime enforces unique artifact IDs, matching embedded snapshot bindings and empty artifacts for unavailable jobs. Retained describes indexed metadata; get performs byte verification.'}), diff --git a/src/sha256-stream.mjs b/src/sha256-stream.mjs index 03c3ad5..3f42f21 100644 --- a/src/sha256-stream.mjs +++ b/src/sha256-stream.mjs @@ -12,6 +12,13 @@ const K = new Uint32Array([ 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2, ]); const rotate = (x,n) => (x>>>n)|(x<<(32-n)); +const checkpoints = new WeakMap(); +// Private checkpoint support for persistent sinks; the public digest() remains finalizing. +export function snapshotSha256(hash) { + const checkpoint=checkpoints.get(hash); + if(!checkpoint)throw workspaceFailure('invalid_request','Unknown incremental hash'); + return checkpoint(); +} export function createIncrementalSha256() { const state = new Uint32Array([0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a,0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19]); const block = new Uint8Array(64), words = new Uint32Array(64); @@ -48,5 +55,10 @@ export function createIncrementalSha256() { const result=Array.from(state,value=>value.toString(16).padStart(8,'0')).join('');block.fill(0);words.fill(0);state.fill(0);return result; }, }); + checkpoints.set(api,()=>{ + if(finished)throw workspaceFailure('disposed','SHA-256 has already been finalized'); + const savedState=state.slice(),savedBlock=block.slice(),savedUsed=used; + try{return api.digest();}finally{state.set(savedState);block.set(savedBlock);used=savedUsed;finished=false;} + }); return api; } diff --git a/src/transfer-queue.mjs b/src/transfer-queue.mjs index be03d61..4eeed55 100644 --- a/src/transfer-queue.mjs +++ b/src/transfer-queue.mjs @@ -88,7 +88,7 @@ export function createTransferQueue(configuration={}){ if(!input||typeof input.read!=='function')throw workspaceFailure('invalid_request','Select an upload source');totalBytes=queueInteger(input.byteLength,1_073_741_824); parseUploadSpec({uploadId:options.uploadId,pluginId:options.pluginId,artifactSha256:options.artifactSha256,path:options.path,expectedRevision:options.expectedRevision,byteLength:totalBytes,sha256:EMPTY_UPLOAD_SHA256}); }else{ - exactObject(options,['sink'],['signal','timeoutMs','requestTimeoutMs','onProgress']); + exactObject(options,['sink'],['resume','signal','timeoutMs','requestTimeoutMs','onProgress']); reference=kind==='download'?parseBinaryArtifactReference(input):parseStoredArtifactReference(input);totalBytes=kind==='download'?reference.artifact.byteLength:reference.snapshot.artifact.byteLength; } if(options.signal!==undefined&&!(options.signal instanceof AbortSignal)||options.onProgress!==undefined&&typeof options.onProgress!=='function')throw workspaceFailure('invalid_request','Invalid transfer callbacks'); @@ -103,7 +103,7 @@ export function createTransferQueue(configuration={}){ if(p.status&&!job.prefixKnown){job.view.resumedBytes=p.status.offset;job.prefixKnown=true;} if(p.status){job.view.acknowledgedBytes=p.status.offset;job.view.transferredBytes=p.status.offset-job.view.resumedBytes;job.view.partialDisposition='retained';} if(p.phase==='committed')job.view.commit='committed'; - }else{job.view.acknowledgedBytes=p.writtenBytes;job.view.transferredBytes=p.writtenBytes;if(['verifying-storage','completed'].includes(p.phase))job.view.commit='committed';} + }else{job.view.acknowledgedBytes=p.writtenBytes;job.view.resumedBytes=p.resumedBytes;job.view.transferredBytes=p.writtenBytes-p.resumedBytes;if(['verifying-storage','completed'].includes(p.phase))job.view.commit='committed';} job.view.remainingBytes=totalBytes-job.view.acknowledgedBytes; if(p.phase!=='completed'&&p.phase!=='committed')job.view.phase=p.phase; emit(job);invoke(options.onProgress,p); diff --git a/tests/artifact-resume-browser.test.mjs b/tests/artifact-resume-browser.test.mjs new file mode 100644 index 0000000..6fcc558 --- /dev/null +++ b/tests/artifact-resume-browser.test.mjs @@ -0,0 +1,167 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createHash,randomUUID,randomBytes} from 'node:crypto'; +import {setTimeout as delay} from 'node:timers/promises'; +import {createOpfsArtifactSink,getOpfsArtifactPartialNames,getBrowserArtifactResumeSupport,parseBrowserArtifactCheckpoint} from '../src/artifact-resume-browser.mjs'; +import {streamStoredJobArtifact} from '../src/artifact-transfer.mjs'; +import {createIncrementalSha256,snapshotSha256} from '../src/sha256-stream.mjs'; +import {createTransferQueue} from '../src/transfer-queue.mjs'; +import {ARTIFACT_STORE_LIMITS} from '../src/artifact-storage.mjs'; +import {workspaceFailure} from '../src/workspace-values.mjs'; +import {createResumeFixture} from '../examples/browser-resume/host.mjs'; + +const digest=bytes=>createHash('sha256').update(bytes).digest('hex'); +const canonical=value=>value===null||typeof value!=='object'?JSON.stringify(value):Array.isArray(value)?`[${value.map(canonical).join(',')}]`:`{${Object.keys(value).sort().map(key=>`${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; +const code=expected=>error=>error.code===expected; +function fakeStorage(){ + const files=new Map(),held=new Set(),state={created:0,sync:0,maxRead:0,maxWrite:0,failMetadata:null,failData:null}; + function file(name){ + let entry=files.get(name); + if(!entry){entry={bytes:Buffer.alloc(0),locked:false};files.set(name,entry);state.created++;} + return {kind:'file',async getFile(){return new Blob([entry.bytes]);},async createWritable(){ + let buffer;return{async write(bytes){buffer=Buffer.from(bytes);},async close(){ + if(state.closeGate){const gate=state.closeGate;state.closeGate=null;gate.enter();await gate.wait;} + const mode=state.failMetadata;state.failMetadata=null; + if(mode==='before')throw new DOMException('Full','QuotaExceededError'); + entry.bytes=buffer; + if(mode==='after')throw new DOMException('Close reply lost','UnknownError'); + },async abort(){}}; + },async createSyncAccessHandle(){ + if(entry.locked)throw new DOMException('Locked','NoModificationAllowedError');entry.locked=true;state.sync++; + let closed=false;const current=()=>{if(closed)throw new DOMException('Closed','InvalidStateError');}; + return{getSize(){current();return entry.bytes.length;},read(bytes,{at}){current();state.maxRead=Math.max(state.maxRead,bytes.length);const part=entry.bytes.subarray(at,at+bytes.length);bytes.set(part);return part.length;}, + write(bytes,{at}){current();state.maxWrite=Math.max(state.maxWrite,bytes.length);const mode=state.failData;state.failData=null;const count=mode==='short'?Math.floor(bytes.length/2):bytes.length; + if(at+count>entry.bytes.length){const extended=Buffer.alloc(at+count);entry.bytes.copy(extended);entry.bytes=extended;}entry.bytes.set(bytes.subarray(0,count),at);return count;}, + truncate(size){current();entry.bytes=Buffer.from(entry.bytes.subarray(0,size));},flush(){current();},close(){closed=true;entry.locked=false;}, + }; + }}; + } + const directory={kind:'directory',async getFileHandle(name,{create=false}={}){if(!files.has(name)&&!create)throw new DOMException('Missing','NotFoundError');return file(name);}}; + const locks={async request(name,options,callback){assert.deepEqual(options,{mode:'exclusive',ifAvailable:true});if(held.has(name))return callback(null);held.add(name);try{return await callback({name});}finally{held.delete(name);}}}; + return {directory,locks,files,held,state}; +} +function fixture(length=196_625){ + const storage=fakeStorage(),bytes=randomBytes(length),now=Date.now(),projectId=randomUUID(),jobId=randomUUID(); + let reference={protocolVersion:1,storage:'snapshot',scope:{projectId,sessionId:randomUUID()},snapshot:{schemaVersion:1,storeId:randomUUID(),workspaceId:projectId,workspaceIdentity:'d'.repeat(64),snapshotId:randomUUID(),jobId,pluginId:'example',pluginArtifactSha256:'a'.repeat(64),kind:'binary',artifact:{id:'binary',path:'out/result.bin',revision:digest(bytes),byteLength:length},capturedAt:now,expiresAt:now+3_600_000}}; + const reads=[],partialId=randomUUID(),names=getOpfsArtifactPartialNames(partialId);let allowed=true; + const client={binding:{workspace:{id:projectId,generation:reference.scope.sessionId}},async getArtifactStorageCapabilities(){return{protocolVersion:1,enabled:true,identity:{schemaVersion:1,storeId:reference.snapshot.storeId,workspaceId:projectId,workspaceIdentity:reference.snapshot.workspaceIdentity},limits:ARTIFACT_STORE_LIMITS};},async getStoredJobArtifact(){if(!allowed)throw workspaceFailure('permission_denied','Denied');return reference;},async readStoredJobArtifactChunk(_ref,offset,{length}){if(!allowed)throw workspaceFailure('permission_denied','Denied');reads.push(offset);const part=bytes.subarray(offset,offset+length);return{reference,offset,nextOffset:offset+part.length,eof:offset+part.length===bytes.length,data:part.toString('base64'),sha256:digest(part)};}}; + const sink=(extra={})=>createOpfsArtifactSink({directory:storage.directory,locks:storage.locks,partialId,reference,...extra}); + const metadata=()=>JSON.parse(storage.files.get(names.metadata).bytes.toString()); + const rewrite=mutate=>{const value=metadata();mutate(value);delete value.checksum;value.checksum=digest(canonical(value));storage.files.get(names.metadata).bytes=Buffer.from(JSON.stringify(value));}; + const partial=async(chunks=1)=>{const controller=new AbortController();await assert.rejects(streamStoredJobArtifact(client,reference,{sink:sink(),signal:controller.signal,onProgress:p=>{if(p.receivedBytes>=chunks*65536)controller.abort();}}),code('cancelled'));assert.equal(storage.held.size,0);}; + return {storage,bytes,client,reads,partialId,names,sink,metadata,rewrite,partial,get reference(){return reference;},rebind(){reference={...reference,scope:{...reference.scope,sessionId:randomUUID()}};client.binding={workspace:{id:projectId,generation:reference.scope.sessionId}};},deny(){allowed=false;},allow(){allowed=true;}}; +} + +test('checkpoint SHA snapshots are independent and preserve finalizing digest semantics',()=>{ + const bytes=randomBytes(131_073),hash=createIncrementalSha256(); + for(const end of [1,55,56,63,64,65,65536,131073]){hash.update(bytes.subarray(hash.byteLength,end));assert.equal(snapshotSha256(hash),digest(bytes.subarray(0,end)));assert.equal(snapshotSha256(hash),digest(bytes.subarray(0,end)));} + assert.equal(hash.digest(),digest(bytes));assert.throws(()=>snapshotSha256(hash),code('disposed')); +}); +test('checkpoint cancellation and fresh generation recover only remaining bytes with independent full hash',async()=>{ + const f=fixture(1_048_593);await f.partial();const before=f.metadata();assert.equal(before.checkpoint.offset,65536);assert.equal(before.checkpoint.sha256,digest(f.bytes.subarray(0,65536)));assert.ok(!JSON.stringify(before).match(/token|grant|sessionId/)); + f.rebind();f.reads.length=0;const updates=[];const receipt=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true,onProgress:p=>updates.push(p)}); + assert.equal(f.reads[0],65536);assert.equal(receipt.resumedBytes,65536);assert.equal(receipt.receivedBytes,f.bytes.length-65536);assert.equal(receipt.verification,'stored');assert.equal(receipt.storedSha256,digest(f.bytes));assert.deepEqual(f.storage.files.get(f.names.data).bytes,f.bytes);assert.equal(f.metadata().state,'complete');assert.equal(f.storage.held.size,0);assert.ok(f.storage.state.maxRead<=65536&&f.storage.state.maxWrite<=65536);assert.equal(updates.at(-1).verified,true); +}); +test('uncheckpointed data tail is trimmed only after valid saved prefix; metadata-ahead falls back one verified checkpoint',async()=>{ + for(const mode of ['tail','ahead']){const f=fixture();await f.partial(2);const data=f.storage.files.get(f.names.data); + if(mode==='tail')data.bytes=Buffer.concat([data.bytes,Buffer.alloc(17,9)]);else data.bytes=Buffer.from(data.bytes.subarray(0,65536+9)); + f.reads.length=0;const receipt=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true});assert.equal(receipt.resumedBytes,mode==='tail'?131072:65536);assert.equal(f.reads[0],receipt.resumedBytes);assert.equal(receipt.storedSha256,digest(f.bytes)); + } +}); +test('corrupt prefix never falls back and missing or truncated files never silently restart',async()=>{ + for(const mode of ['corrupt','short','missing-data','missing-metadata']){const f=fixture();await f.partial(2);const data=f.storage.files.get(f.names.data); + if(mode==='corrupt')data.bytes[0]^=1;else if(mode==='short')data.bytes=Buffer.alloc(1);else f.storage.files.delete(f.names[mode==='missing-data'?'data':'metadata']); + f.reads.length=0;const count=f.storage.state.created; + await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true}),code(mode.startsWith('missing')?'not_found':'conflict'));assert.equal(f.reads.length,0);assert.equal(f.storage.state.created,count);assert.equal(f.storage.held.size,0); + } +}); +test('metadata atomic close failure conservatively recovers old or newly persisted checkpoint',async()=>{ + for(const mode of ['before','after']){const f=fixture();await f.partial();f.storage.state.failMetadata=mode; + await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true}),code(mode==='before'?'budget_exceeded':'unavailable')); + assert.equal(f.metadata().checkpoint.offset,mode==='before'?65536:131072);f.reads.length=0; + const receipt=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true});assert.equal(receipt.resumedBytes,mode==='before'?65536:131072);assert.equal(receipt.storedSha256,digest(f.bytes)); + } +}); +test('partial data writes preserve the last flushed metadata and repair only the uncheckpointed tail',async()=>{ + const f=fixture();await f.partial();f.storage.state.failData='short';await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true}),code('unavailable'));assert.equal(f.metadata().checkpoint.offset,65536); + const receipt=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true});assert.equal(receipt.resumedBytes,65536);assert.equal(receipt.storedSha256,digest(f.bytes)); +}); +test('exclusive Web Lock rejects other sink instances until owner closes and is not stolen',async()=>{ + const f=fixture(),signal=new AbortController().signal,input={artifact:f.reference.snapshot.artifact,storedReference:f.reference,signal}; + const first=await f.sink().open(input);await first.write({offset:0,bytes:f.bytes.subarray(0,65536),signal}); + await assert.rejects(f.sink().open({...input,resume:true}),code('conflict'));assert.equal(f.storage.held.size,1);await first.abort({reason:'selected cancel'});assert.equal(f.storage.held.size,0); + assert.equal((await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true})).resumedBytes,65536); +}); +test('current authorization and complete stored identity are required before opening or reusing a partial',async()=>{ + const f=fixture();await f.partial();f.deny();const sync=f.storage.state.sync; + await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true}),code('permission_denied'));assert.equal(f.storage.state.sync,sync);f.allow(); + const oldSink=f.sink();f.rebind();await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:oldSink,resume:true}),code('conflict')); + for(const field of ['storeId','workspaceIdentity','snapshotId','jobId','pluginId','pluginArtifactSha256']){ + const reference={...f.reference,snapshot:{...f.reference.snapshot,[field]:field.endsWith('Sha256')||field==='workspaceIdentity'?'f'.repeat(64):field==='pluginId'?'another-plugin':randomUUID()}}; + const changed={...f.client,async getStoredJobArtifact(){return reference;}}; + await assert.rejects(streamStoredJobArtifact(changed,reference,{sink:f.sink({reference}),resume:true}),code('conflict')); + } +}); +test('expired local retention is enforced even with a current host reference; no files are deleted',async()=>{ + const f=fixture();await f.partial();const time=Date.now; + Date.now=()=>f.metadata().expiresAt+1; + try{await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true}),code('unavailable'));assert.equal(f.storage.files.size,2);}finally{Date.now=time;} +}); +test('malformed, oversized, checksummed identity changes and unexpected metadata writes fail closed',async()=>{ + for(const mode of ['checksum','oversized','identity','unknown-field']){const f=fixture();await f.partial();const meta=f.storage.files.get(f.names.metadata); + if(mode==='checksum'){const value=f.metadata();value.checkpoint.sha256='e'.repeat(64);meta.bytes=Buffer.from(JSON.stringify(value));} + if(mode==='oversized')meta.bytes=Buffer.alloc(16385,32); + if(mode==='identity')f.rewrite(value=>{value.snapshot.storeId=randomUUID();}); + if(mode==='unknown-field')f.rewrite(value=>{value.token='not-a-token';}); + await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true}));assert.equal(f.storage.held.size,0); + } + const f=fixture(),signal=new AbortController().signal,s=await f.sink().open({artifact:f.reference.snapshot.artifact,storedReference:f.reference,signal});f.rewrite(v=>{v.expiresAt--;}); + await assert.rejects(s.write({offset:0,bytes:f.bytes.subarray(0,65536),signal}),code('conflict'));await s.abort({reason:'done'});assert.equal(f.storage.files.get(f.names.data).bytes.length,0); +}); +test('new partial never overwrites an existing ID; unsupported setup and bounds do not request permissions',async()=>{ + const f=fixture();await f.partial();await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink()}),code('conflict')); + assert.throws(()=>f.sink({retentionMs:86400001}),code('invalid_request'));assert.throws(()=>f.sink({partialId:'../outside'}),code('invalid_request'));assert.throws(()=>f.sink({locks:{}}),code('unsupported')); + assert.equal(getBrowserArtifactResumeSupport().supported,false); + const v=f.metadata();assert.deepEqual(parseBrowserArtifactCheckpoint(v),v);assert.throws(()=>parseBrowserArtifactCheckpoint({...v,checksum:'a'.repeat(64)}),code('conflict')); +}); +test('empty and already completed recovery still ask the host for a validated EOF without downloading the prefix',async()=>{ + for(const size of [0,65536]){const f=fixture(size),first=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink()});assert.equal(first.verification,'stored');f.reads.length=0; + const second=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true});assert.equal(second.resumedBytes,size);assert.equal(second.receivedBytes,0);assert.deepEqual(f.reads,[size]);assert.equal(second.storedSha256,digest(f.bytes)); + } +}); +test('core independently rejects forged sink prefix before network bytes, even when sink claims recovery',async()=>{ + const f=fixture(),base=f.sink();await f.partial();let aborted=0;const forged={capabilities:base.capabilities,async open(input){const session=await base.open(input);return{...session,recovery:{...session.recovery,sha256:'0'.repeat(64)},async abort(input){aborted++;await session.abort(input);}};}}; + f.reads.length=0;await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:forged,resume:true}),code('conflict'));assert.equal(f.reads.length,0);assert.equal(aborted,1);assert.equal(f.storage.held.size,0); +}); +test('queue progress separates recovered prefix from newly transferred bytes and verifies final storage',async()=>{ + const f=fixture();await f.partial();const queue=createTransferQueue({concurrency:1,perConnection:1}),events=[];queue.subscribe(event=>events.push(event)); + try{const job=queue.enqueueStoredDownload(f.client,f.reference,{sink:f.sink(),resume:true});const receipt=await job.result;assert.equal(receipt.resumedBytes,65536);assert.equal(job.snapshot.resumedBytes,65536);assert.equal(job.snapshot.transferredBytes,f.bytes.length-65536);assert.equal(job.snapshot.acknowledgedBytes,f.bytes.length);assert.equal(job.snapshot.attemptedBytes,f.bytes.length-65536);assert.equal(job.snapshot.verified,true);assert.ok(events.some(e=>e.phase==='verifying-prefix'&&!e.verified));}finally{await queue.dispose();} +}); +test('four open partial sessions are bounded and cancelled pause releases their locks',async()=>{ + const sessions=[];try{for(let i=0;i<4;i++){const f=fixture(),signal=new AbortController().signal;sessions.push(await f.sink().open({artifact:f.reference.snapshot.artifact,storedReference:f.reference,signal}));} + const f=fixture();await assert.rejects(streamStoredJobArtifact(f.client,f.reference,{sink:f.sink()}),code('budget_exceeded'));assert.equal(f.storage.state.created,0); + }finally{for(const session of sessions)await session.close();} + const f=fixture(),queue=createTransferQueue();let paused=false; + const job=queue.enqueueStoredDownload(f.client,f.reference,{sink:f.sink(),onProgress:p=>{if(p.receivedBytes===65536){paused=true;job.pause();}}}); + for(let n=0;n<200&&!paused;n++)await delay(5);assert.ok(paused);await queue.dispose();await assert.rejects(job.result,code('cancelled'));assert.equal(f.storage.held.size,0);assert.equal(f.metadata().checkpoint.offset,65536); +}); +test('real HTTP stored snapshot resumes with a fresh host generation after source deletion and rejects revoked grants',async()=>{ + const host=await createResumeFixture({fileBytes:262_161}),storage=fakeStorage(),partialId=randomUUID(); + try{ + const get=()=>host.client.getStoredJobArtifact(host.config.pluginId,host.config.jobId,host.config.artifactId,host.config.artifactSha256),reference=await get(); + const sink=reference=>createOpfsArtifactSink({directory:storage.directory,locks:storage.locks,partialId,reference}),controller=new AbortController(); + await assert.rejects(streamStoredJobArtifact(host.client,reference,{sink:sink(reference),signal:controller.signal,onProgress:p=>{if(p.receivedBytes>=65536)controller.abort();}}),code('cancelled')); + await host.restart({grants:[]});await assert.rejects(get(),code('permission_denied')); + await host.restart();const fresh=await get();assert.notEqual(fresh.scope.sessionId,reference.scope.sessionId);assert.deepEqual(fresh.snapshot,reference.snapshot); + await assert.rejects(streamStoredJobArtifact(host.client,reference,{sink:sink(reference),resume:true}),code('generation_mismatch')); + const receipt=await streamStoredJobArtifact(host.client,fresh,{sink:sink(fresh),resume:true});assert.equal(receipt.resumedBytes,65536);assert.equal(receipt.storedSha256,host.config.sha256);assert.equal(host.executions,1); + }finally{await host.close();} +}); +test('abort drains an in-flight metadata close before releasing cross-tab ownership',async()=>{ + const f=fixture(),signal=new AbortController().signal,input={artifact:f.reference.snapshot.artifact,storedReference:f.reference,signal},session=await f.sink().open(input); + let entered,release;const started=new Promise(resolve=>{entered=resolve;}),wait=new Promise(resolve=>{release=resolve;});f.storage.state.closeGate={enter:entered,wait}; + const writing=session.write({offset:0,bytes:f.bytes.subarray(0,65536),signal});const rejected=assert.rejects(writing,code('disposed'));await started; + const closing=session.abort({reason:'close during metadata commit'});assert.equal(f.storage.held.size,1); + await assert.rejects(f.sink().open({...input,resume:true}),code('conflict'));release();await rejected;await closing;assert.equal(f.storage.held.size,0); + const receipt=await streamStoredJobArtifact(f.client,f.reference,{sink:f.sink(),resume:true});assert.equal(receipt.resumedBytes,65536);assert.equal(receipt.storedSha256,digest(f.bytes)); +}); diff --git a/tests/types/artifact-resume-browser.mts b/tests/types/artifact-resume-browser.mts new file mode 100644 index 0000000..ece2425 --- /dev/null +++ b/tests/types/artifact-resume-browser.mts @@ -0,0 +1,15 @@ +import {createOpfsArtifactSink,getBrowserArtifactResumeSupport,getOpfsArtifactPartialNames,parseBrowserArtifactCheckpoint} from '@altifigence/dds-plugin-sdk/artifact-resume-browser'; +import type {OpfsCheckpointDirectory,BrowserCheckpointLocks,BrowserArtifactCheckpoint} from '@altifigence/dds-plugin-sdk/artifact-resume-browser'; +import type {StoredArtifactReference} from '@altifigence/dds-plugin-sdk/artifact-storage'; +import {streamStoredJobArtifact} from '@altifigence/dds-plugin-sdk/artifact-transfer'; +import type {StoredArtifactClient} from '@altifigence/dds-plugin-sdk/artifact-transfer'; +import {createTransferQueue} from '@altifigence/dds-plugin-sdk/transfer-queue'; +declare const directory:OpfsCheckpointDirectory,locks:BrowserCheckpointLocks,reference:StoredArtifactReference,client:StoredArtifactClient; +const sink=createOpfsArtifactSink({directory,locks,reference,partialId:'12345678-1234-4234-8234-123456789abc'}); +void streamStoredJobArtifact(client,reference,{sink,resume:true,onProgress:p=>{const phase:string=p.phase;void phase;}}); +createTransferQueue().enqueueStoredDownload(client,reference,{sink,resume:true}); +const checkpoint:BrowserArtifactCheckpoint=parseBrowserArtifactCheckpoint({});void checkpoint; +const supported:boolean=getBrowserArtifactResumeSupport().supported;void supported; +const metadata:string=getOpfsArtifactPartialNames(sink.partialId).metadata;void metadata; +// @ts-expect-error recovery requires a boolean, not inferred truthiness +void streamStoredJobArtifact(client,reference,{sink,resume:'yes'});