From 8f40edd294fe9d9eb6982b0b8c3068dbd7d5a99b Mon Sep 17 00:00:00 2001 From: Jerry Yuan Date: Thu, 6 Aug 2026 22:41:39 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(fs):=20the=20fs=20module=20=E2=80=94?= =?UTF-8?q?=20a=20per-app=20file=20tree=20behind=20a=20nine-op=20spec,=20s?= =?UTF-8?q?im=20host,=20pocket-fs=20reference=20core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fifth module-shaped vertical slice (after ui, strike, audio and db), built spec-first per the module discipline: contracts/spec/fs.ts pins nine synchronous ops (read/write/remove/list/stat/mkdir/rename/usage/ lastError), the payload encoding (text as a JSON string, bytes as the db module's {"$b": base64} spelling), the path grammar, and the resource ceilings (64 KiB per payload crossing, 256 entries per paged list call). Names are UNIVERSAL: a segment is any well-formed Unicode — Chinese names, dot-prefixed names, spaces — except "." and ".." (the escape hatches), control characters, and oversize (64 UTF-8 bytes). No name is reserved to the host. Isolation never depended on names: every path is relative and resolves under the root the host binds at mount, ".."/ absolute/"/"-in-name are unrepresentable, and the reference core lstat-refuses symlinks planted by host-side actors — so apps cannot spell each other's trees, the ATTACH-refusal principle generalized. Privilege is the binding: on Pocket Pi the device agent is the same module bound at /workspace, apps at /workspace/apps//data/. Truncate writes are atomic (O_EXCL temp beside the target + rename — the power-loss contract LittleFS hosts inherit from their atomic rename). Entries list in Unicode code point order (= UTF-8 byte order; the sim host carries the comparator since JS sorts UTF-16 code units). The module owns no clock, emits no events, and stat carries NO mtime — a timestamp is the fs spelling of Date.now, excluded for the same golden-test reason. - gen-rust emits pub mod fs into engine/core/src/spec.rs (drift-guarded) - data.fs capability registered ahead of any stock TARGET advertising it, the audio.pcm/data.sqlite precedent - @pocketjs/framework/fs SDK: the Bun shape — file()/write() plus the node:fs sync subset Bun implements — so Bun file code migrates unchanged (await unwraps the sync returns); payloads chunk transparently past FS_MAX_IO_BYTES; throws where the namespace is unmounted, like db - framework/src/bytes.ts: the base64 codec extracted from db-api plus a strict UTF-8 decoder (QuickJS has no TextDecoder), shared by both SDKs - hosts/sim/fs.ts: in-memory tree behind the op namespace, injected via bootWorld extraGlobals; tests/fs.test.ts runs the op contract and the SDK - engine/crates/pocket-fs: the reference core, Storage::Memory/Dir over std::fs — root confinement, symlink refusal, atomic truncate writes, per-app quota. mount is a default feature; default-features = false drops the pocket-mod dependency for firmware with its own QuickJS wiring — verified to cargo check clean for riscv32imafc-esp-espidf - docs/FS.md maps the boundary, the shared-root layout with db, and the three-move adoption path Verified: bun run test 11/11 stages green (tests/fs.test.ts 15 pass), cargo test -p pocket-fs 12/12 including a live QuickJS guest round-trip and universal-name round-trips, cargo check --workspace clean, clippy clean. Co-Authored-By: Claude Fable 5 --- contracts/spec/fs.ts | 222 ++++++ contracts/spec/gen-rust.ts | 41 ++ contracts/spec/platforms.ts | 10 + docs/FS.md | 159 +++++ engine/Cargo.lock | 11 + engine/Cargo.toml | 2 + engine/core/src/spec.rs | 30 + engine/crates/pocket-fs/Cargo.toml | 22 + engine/crates/pocket-fs/src/lib.rs | 1044 ++++++++++++++++++++++++++++ framework/compiler/subpaths.ts | 1 + framework/src/bytes.ts | 94 +++ framework/src/db-api.ts | 43 +- framework/src/fs-api.ts | 325 +++++++++ hosts/sim/fs.ts | 253 +++++++ hosts/sim/sim.ts | 1 + package.json | 3 + tests/fs.test.ts | 270 +++++++ 17 files changed, 2491 insertions(+), 40 deletions(-) create mode 100644 contracts/spec/fs.ts create mode 100644 docs/FS.md create mode 100644 engine/crates/pocket-fs/Cargo.toml create mode 100644 engine/crates/pocket-fs/src/lib.rs create mode 100644 framework/src/bytes.ts create mode 100644 framework/src/fs-api.ts create mode 100644 hosts/sim/fs.ts create mode 100644 tests/fs.test.ts diff --git a/contracts/spec/fs.ts b/contracts/spec/fs.ts new file mode 100644 index 00000000..f570d23b --- /dev/null +++ b/contracts/spec/fs.ts @@ -0,0 +1,222 @@ +// PocketJS fs spec — the boundary of the FS module (`globalThis.fs`). +// +// This is a MODULE spec in the docs/RUNTIMES.md §5 sense: a vertical slice +// with its own vocabulary, mounted as its own namespace, pinned here as data. +// It is deliberately NOT part of the `ui` op table — fs evolves append-only +// in its own op space, and a host adopts it independently of the UI surface +// (capability id `data.fs` in contracts/spec/platforms.ts). +// +// The module is a per-app file tree behind nine synchronous ops. The SDK +// (@pocketjs/framework/fs) is the Bun shape — `file()`/`write()` plus the +// node:fs sync subset Bun implements — so file code written against Bun runs +// against the mounted module with the async wrappers dropped. +// +// The four parts of the boundary: +// +// ops guest -> core intent (numeric codes below, append-only) +// events none — every op is synchronous; the module owns no clock. +// There is no watch(): watching needs events and a clock, +// and a per-tick guest can poll stat() when it must. +// data contract the path grammar + payload encoding below +// frame contract every op completes inside the guest's single per-tick +// turn (law 3 holds unchanged). stat() carries NO mtime — +// a timestamp is the fs spelling of Date.now, and a +// golden-tested app must not depend on one. An app that +// needs a timestamp writes it into content it controls. +// +// Storage rule: every path is RELATIVE and resolves under the app's own +// data root; the host binds that root when it mounts the module, and the +// guest never sees a real path. There is no op that names another app's +// tree — isolation is by construction, not by permission check (the same +// principle as db's "open(name) is the only path to a database" and its +// ATTACH refusal). Hosts MUST NOT follow a symlink out of the root: the +// guest cannot create symlinks through this API, but a host-side actor may +// have (on Pocket Pi the device agent owns the whole workspace and every +// app root under it — that asymmetry is host layout policy, above this +// boundary, see docs/FS.md), so the reference core lstat-checks every +// segment. +// +// If you change ANY value here: run `bun contracts/spec/gen-rust.ts`, commit +// the regenerated engine/core/src/spec.rs (tests/contract.ts byte-compares). + +// --------------------------------------------------------------------------- +// Fs ops (the `fs.*` native contract) +// --------------------------------------------------------------------------- +// Numeric codes are the FFI ABI identity of each op. 0 is reserved +// (invalid/nop). Codes are append-only: never renumber, never reuse. +// +// Signatures (authoritative; hosts marshal them however they like). Ops +// returning 0 | 1 report detail through lastError(); ops returning a JSON +// line carry their own {"error": "..."} shape (and set lastError too). +// +// read(path, offset:number, maxBytes:number) -> string +// [one JSON line: {"data":{"$b":""},"size":N, +// "eof":bool} or {"error":...}. Reads up to maxBytes +// bytes at byte offset; maxBytes must be 1..FS_MAX_IO_BYTES +// or the op fails. `size` is the file's total byte size, +// `eof` is true when offset+data reaches it. Reading a +// directory fails] +// write(path, data:string, mode:number) -> 0 | 1 +// [data is the payload encoding below, decoded byte length +// <= FS_MAX_IO_BYTES per call (the SDK chunks larger +// writes). mode FS_WRITE_TRUNCATE replaces the file +// ATOMICALLY — the old content or the new, never a torn +// middle (temp + rename; the power-loss contract device +// hosts inherit from LittleFS's atomic rename. Temps +// live in a host directory outside the bound root, so +// the app's tree never shows host machinery). +// FS_WRITE_APPEND appends and is not atomic. Parent +// directories are created automatically (Bun.write +// semantics). Writing over a directory fails] +// remove(path, recursive:number) -> 0 | 1 +// [removes a file, or a directory when empty; recursive=1 +// removes a directory tree. A missing path fails with +// "not found" (the SDK's rmSync force option swallows +// that one). remove("") — the root — always fails] +// list(path, offset:number) -> string +// [{"entries":[{"name":"a.txt","kind":"file","size":N}, +// {"name":"sub","kind":"dir","size":0}],"eof":bool} or +// {"error":...}. Entries sort by name in Unicode code +// point order (= UTF-8 byte order; NOT UTF-16 code unit +// order — hosts written in JS must sort by code point) — +// deterministic across hosts — and one call returns at +// most FS_MAX_DIR_ENTRIES of them starting at `offset` +// in that order; `eof` false means page again. list("") +// lists the root] +// stat(path) -> string +// [{"kind":"file","size":N} | {"kind":"dir","size":0} or +// {"error":"not found"}. stat("") is the root: always +// {"kind":"dir","size":0}. No mtime — see the frame +// contract above] +// mkdir(path) -> 0 | 1 +// [recursive (every missing ancestor is created) and +// idempotent (an existing directory is success). A file +// anywhere on the path fails] +// rename(from, to) -> 0 | 1 +// [moves a file or directory within the root. An existing +// file at `to` is replaced atomically; an existing +// directory at `to` fails; a missing parent of `to` +// fails (mkdir first — rename does not create). Renaming +// a directory into its own subtree fails] +// usage() -> string +// [{"usedBytes":N,"quotaBytes":N} — usedBytes sums every +// file's size under the root; quotaBytes is the host's +// configured budget for this app, 0 = unmetered. When a +// quota is set, a write/append that would exceed it +// fails with "quota exceeded"] +// lastError() -> string +// [detail for the last failed op on this module; "" when +// the last op succeeded. Module-scoped — there are no +// handles in this vocabulary] + +export const FS_OP = { + read: 1, + write: 2, + remove: 3, + list: 4, + stat: 5, + mkdir: 6, + rename: 7, + usage: 8, + lastError: 9, +} as const; + +/** write() modes. */ +export const FS_WRITE_TRUNCATE = 0; +export const FS_WRITE_APPEND = 1; + +// --------------------------------------------------------------------------- +// Data contract — payload encoding (write data in, read data out) +// --------------------------------------------------------------------------- +// A payload crossing the boundary is one JSON value: +// +// text <-> a JSON string [stored as its UTF-8 bytes] +// bytes <-> { "$b": "" } [the db module's blob spelling] +// +// write() accepts either; read() always returns bytes — the file does not +// remember which spelling wrote it, and the SDK's .text() decodes UTF-8 +// guest-side (QuickJS has no TextDecoder; the SDK carries the codec). + +/** Marker key for a bytes payload (same spelling as db's DB_BLOB_KEY). */ +export const FS_BLOB_KEY = "$b"; + +// --------------------------------------------------------------------------- +// Data contract — the path grammar +// --------------------------------------------------------------------------- +// A path is 1..FS_MAX_DEPTH segments joined by "/": no leading or trailing +// slash, no empty segment. A segment is ANY well-formed Unicode string — +// Chinese names, dot-prefixed names, whatever the app wants — except the +// four things no filesystem can or this sandbox may allow: +// +// "." and ".." the escape hatches (this is the security rule); +// "/" in a name unrepresentable — it IS the separator, on every +// filesystem on earth; +// control chars C0 (U+0000..U+001F) and DEL (U+007F); +// oversize a segment > FS_MAX_SEGMENT_BYTES of UTF-8. +// +// No name is reserved to the host. "" names the root and is valid only +// where an op says so (list, stat). Total path <= FS_MAX_PATH_BYTES of +// UTF-8. +// +// Identity: segments are byte-for-byte identities (compared as UTF-8, no +// case folding, no Unicode normalization). Some host filesystems fold case +// or normalize (macOS APFS); two sibling names differing only by case or +// normalization form are therefore NOT portable — an app must never create +// both. The deterministic hosts (sim, the reference core's Memory storage) +// are byte-exact, so a golden test catches the collision early. + +/** Maximum UTF-8 bytes in one segment. */ +export const FS_MAX_SEGMENT_BYTES = 64; + +/** Maximum segments in a path (root = depth 0). */ +export const FS_MAX_DEPTH = 8; + +/** Maximum total path length in UTF-8 bytes (segments + separators). */ +export const FS_MAX_PATH_BYTES = 160; + +/** UTF-8 byte length of a JS string (QuickJS has no TextEncoder). */ +function utf8Bytes(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++) { + const c = s.codePointAt(i)!; + n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4; + if (c >= 0x10000) i++; + } + return n; +} + +/** True when `segment` is one valid path segment under the grammar above. */ +export function fsValidSegment(segment: string): boolean { + if (segment.length === 0 || segment === "." || segment === "..") return false; + // eslint-disable-next-line no-control-regex + if (/[\u0000-\u001f\u007f]/.test(segment)) return false; + return utf8Bytes(segment) <= FS_MAX_SEGMENT_BYTES; +} + +/** True when `path` is a valid non-root path under the grammar above. + * The SAME predicate every host implements; exported so hosts and tests + * share one spelling. */ +export function fsValidPath(path: string): boolean { + if (path.length === 0 || utf8Bytes(path) > FS_MAX_PATH_BYTES) return false; + const segments = path.split("/"); + if (segments.length > FS_MAX_DEPTH) return false; + return segments.every(fsValidSegment); +} + +// --------------------------------------------------------------------------- +// Data contract — resource ceilings +// --------------------------------------------------------------------------- + +/** + * Payload ceiling per read()/write() call, in bytes. One call's payload + * must fit a device heap comfortably; the SDK loops for larger files, so + * the ceiling bounds marshaling, not file size. + */ +export const FS_MAX_IO_BYTES = 65536; + +/** + * Entries per list() call. list() pages (offset + eof), so a big directory + * is slower to enumerate, never impossible — a ceiling an app cannot get + * stuck behind, unlike an unpaged cap. + */ +export const FS_MAX_DIR_ENTRIES = 256; diff --git a/contracts/spec/gen-rust.ts b/contracts/spec/gen-rust.ts index ddafe5a8..3fd2935c 100644 --- a/contracts/spec/gen-rust.ts +++ b/contracts/spec/gen-rust.ts @@ -24,6 +24,17 @@ import { DB_MEMORY, DB_OP, } from "./db.ts"; +import { + FS_BLOB_KEY, + FS_MAX_DEPTH, + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_MAX_PATH_BYTES, + FS_MAX_SEGMENT_BYTES, + FS_OP, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, +} from "./fs.ts"; import { NET_DEFAULT_RESPONSE_BYTES, NET_DEFAULT_TIMEOUT_MS, @@ -511,6 +522,36 @@ export function generateRust(): string { put("}"); put(""); + // --- fs module ----------------------------------------------------------- + // The fs MODULE's boundary (contracts/spec/fs.ts): a per-app file tree + // behind nine synchronous ops, mounted as `globalThis.fs`, independent of + // the ui surface. A native host implementing it reads these constants; the + // reference implementation is engine/crates/pocket-fs. + put("/// FS module boundary (contracts/spec/fs.ts — `globalThis.fs`)."); + put("/// A per-app file tree behind nine synchronous ops; every path resolves"); + put("/// under the app's own data root. No clock, no events, no mtime."); + put("pub mod fs {"); + for (const [name, v] of Object.entries(FS_OP)) { + put(` pub const OP_${screaming(name)}: u8 = ${v};`); + } + put(` /// write() modes.`); + put(` pub const WRITE_TRUNCATE: u32 = ${FS_WRITE_TRUNCATE};`); + put(` pub const WRITE_APPEND: u32 = ${FS_WRITE_APPEND};`); + put(` /// Marker key for a bytes payload (db's blob spelling).`); + put(` pub const BLOB_KEY: &str = ${JSON.stringify(FS_BLOB_KEY)};`); + put(` /// Maximum UTF-8 bytes in one path segment.`); + put(` pub const MAX_SEGMENT_BYTES: usize = ${FS_MAX_SEGMENT_BYTES};`); + put(` /// Maximum segments in a path.`); + put(` pub const MAX_DEPTH: usize = ${FS_MAX_DEPTH};`); + put(` /// Maximum total path length in bytes.`); + put(` pub const MAX_PATH_BYTES: usize = ${FS_MAX_PATH_BYTES};`); + put(` /// Payload ceiling per read()/write() call, in bytes.`); + put(` pub const MAX_IO_BYTES: usize = ${FS_MAX_IO_BYTES};`); + put(` /// Entries per list() call (paged via offset + eof).`); + put(` pub const MAX_DIR_ENTRIES: usize = ${FS_MAX_DIR_ENTRIES};`); + put("}"); + put(""); + // --- net module --------------------------------------------------------------- put("/// NET module boundary (contracts/spec/net.ts — `globalThis.net`)."); put("/// Bounded whole-response HTTP; completions batch to tick boundaries."); diff --git a/contracts/spec/platforms.ts b/contracts/spec/platforms.ts index 66fc1590..d3b6d079 100644 --- a/contracts/spec/platforms.ts +++ b/contracts/spec/platforms.ts @@ -158,6 +158,16 @@ export const POCKET_CAPABILITIES = defineCapabilityRegistry([ // device target appends the id to its profile only when its native host // ships the module. "data.sqlite", + // A per-app file tree behind the fs module's own namespace + // (`globalThis.fs`, contracts/spec/fs.ts): nine synchronous ops, every + // path confined to the app's own data root — apps cannot name, let alone + // reach, each other's trees. Registered ahead of any stock TARGET + // advertising it: the sim host and the engine/crates/pocket-fs reference + // core implement and test the whole contract, so apps can already declare + // the requirement and fail admission where the module is absent. A device + // target appends the id to its profile only when its native host ships + // the module. + "data.fs", // Copy/cut/paste round-trips with the OS clipboard. "host.clipboard", // The logical viewport is runtime-mutable: the app is told about live diff --git a/docs/FS.md b/docs/FS.md new file mode 100644 index 00000000..b9599faf --- /dev/null +++ b/docs/FS.md @@ -0,0 +1,159 @@ +# The FS Module + +FS is PocketJS's fifth module (after `ui`, `strike`, `audio` and `db`): a +per-app file tree mounted as `globalThis.fs` behind nine synchronous ops. +Like db it was written spec-first — the boundary existed before any host +code, every host implements the same pinned protocol, and a developer +adding a storage feature extends the spec instead of forking a host. +`contracts/spec/fs.ts` is normative; this page is the map. + +``` +platform storage (POSIX dir · LittleFS · memory) Host / substrate + ↑ the app's own data root is the port point +fs core: path grammar + confinement + atomic writes the module +fs spec: ops (read, write, remove, list, stat, mkdir, + rename, usage, lastError) + events (none — every op is synchronous) + data contract (path grammar · payload encoding · ceilings) + frame contract (no module clock; no mtime; ops complete in the turn) +SDK: @pocketjs/framework/fs (file/write + the node:fs sync subset — the Bun shape) + ↓ +app: notes in files, assets in dirs, config in json Guest +``` + +## The boundary in one page + +**Mount.** The module is its own namespace: `globalThis.fs`, one method per +op (`FS_OP` codes are the ABI identity, append-only). Capability id +`data.fs`. Like db, absence does **not** degrade to a no-op — file code +that silently drops writes is a corruption bug, so the SDK throws where the +namespace is unmounted, and an app declares `data.fs` in `pocket.json` +`requires` so admission catches the gap before eval does. + +**Ops** (guest → core, all synchronous): `read(path, offset, maxBytes)` → +one JSON line (`{data:{"$b":…}, size, eof}`), `write(path, data, mode)` +with truncate/append modes, `remove(path, recursive)`, `list(path, offset)` +→ name-sorted, paged entries, `stat(path)` → `{kind, size}`, +`mkdir(path)` (recursive, idempotent), `rename(from, to)`, `usage()` → +`{usedBytes, quotaBytes}`, and `lastError()`. + +**Payloads** cross as one JSON value: text as a JSON string (stored as its +UTF-8 bytes), bytes as `{"$b": ""}` — the db module's blob +spelling. `read` always returns bytes; the SDK's `.text()` decodes UTF-8 +guest-side (QuickJS has no TextDecoder; the SDK carries the codec). + +**The storage rule — isolation by construction.** Every path is relative +and resolves under the app's own data root, bound by the host at mount. +Names are universal — any well-formed Unicode a filesystem can hold, +dot-prefixed included; nothing in the app's tree is reserved to the host. +Isolation never depended on names: `..`, absolute paths, and `/` inside a +name are unrepresentable, so there is no way to *spell* another app's +tree — the same principle as db's "open(name) is the only path to a +database" and its ATTACH refusal. Hosts must not follow a symlink out of +the root; the reference core lstat-checks every segment and treats any +symlink as absent. + +The confinement binds the **guest**, not the host. On Pocket Pi the device +agent's home is the whole workspace — with every app root laid out under +it (`/workspace/apps//data/`), the agent reads and writes every +app's tree through the same module, bound wider, while apps still cannot +reach each other. Privilege is the binding, not the code. + +One data root serves both data modules: a database is an ordinary file +(`/.sqlite`) in the app's home — its own asset, visible +like any of its files (backup = a file copy). Overwriting it corrupts the +app's own data, the same trust class as deleting its own files; SQLite +fails loudly on a corrupt image. + +**Atomicity.** A truncate `write` lands completely or not at all: the +payload lands in the module's own temp directory — outside the bound +root, same filesystem — then renames over the target, so after power +loss the file holds the old content or the new, never a torn middle, and +the app's tree never shows host machinery (the module owns the temp +directory and clears it on construction, so a crash orphan cannot +outlive the next boot). Append is not atomic. LittleFS's rename is +atomic, so device hosts inherit the contract by the same moves. + +**Ceilings.** `FS_MAX_IO_BYTES` (64 KiB) per read/write payload — the SDK +chunks larger files, so the ceiling bounds marshaling, not file size. +`FS_MAX_DIR_ENTRIES` (256) per `list()` call, paged via offset + eof — a +big directory is slower to enumerate, never impossible. Paths: +`FS_MAX_DEPTH` (8) segments of `FS_MAX_SEGMENT_BYTES` (64) each, +`FS_MAX_PATH_BYTES` (160) total. A per-app byte quota is host policy, +reported by `usage()` (0 = unmetered) and enforced on write. + +**Frame contract.** The module owns no clock and emits no events: every op +completes inside the guest's single per-tick turn (law 3 unchanged). There +is no `watch()` — watching needs events and a clock; a per-tick guest +polls `stat()` when it must. `stat` carries **no mtime**: a timestamp is +the fs spelling of `Date.now`, and a golden-tested app must not depend on +one. An app that needs a timestamp writes it into content it controls. + +**Identity.** Segments are byte-for-byte identities (UTF-8, no case +folding, no Unicode normalization), but some host filesystems fold or +normalize (macOS APFS). Two sibling names differing only by case or +normalization form are not portable — never create both. The +deterministic hosts (sim, the reference core's Memory storage) are +byte-exact, so a golden test catches the collision before a folding +device filesystem hides it. + +## The SDK + +`@pocketjs/framework/fs` is the Bun shape — `file()`/`write()` plus the +node:fs sync subset Bun implements — so file code written against Bun runs +against the mounted module unchanged. Methods return values synchronously +(the frame contract), and `await` unwraps a plain value, so Bun-idiomatic +`await file(p).text()` needs no edits: + +```ts +import { file, write, readdirSync, mkdirSync, rmSync, usage } from "@pocketjs/framework/fs"; + +write("notes/today.md", "# Today\n- ship the fs module"); // atomic, mkdir -p +const f = file("notes/today.md"); +f.exists(); // true +f.size; // bytes +f.text(); // the string (await f.text() works too) +f.bytes(); // Uint8Array +f.json(); // parsed JSON (for config files) + +mkdirSync("assets/img"); +readdirSync("notes", { withFileTypes: true }); // name-sorted entries +rmSync("notes", { recursive: true }); +usage(); // { usedBytes, quotaBytes } +``` + +Also exported: `readFileSync`, `writeFileSync`, `appendFileSync`, +`renameSync`, `statSync`, `existsSync` — each the node spelling Bun also +serves. Files larger than one payload chunk transparently +(`FS_MAX_IO_BYTES` per op crossing). + +Choosing between fs and db: rows, queries and transactions belong in +`data.sqlite`; documents, assets and configs belong here. A key-value need +is one db table, not a third module. + +## Host status + +| Host | Implementation | Status | +|---|---|---| +| sim (`hosts/sim/fs.ts`) | in-memory tree behind the op namespace, injected via `bootWorld` `extraGlobals` | ships with the test host; `tests/fs.test.ts` runs the contract, the SDK, and an oracle comparison against Bun's real fs | +| reference core (`engine/crates/pocket-fs`) | `Storage::Memory`/`Storage::Dir` over std::fs — grammar confinement, symlink refusal, atomic truncate writes, quota — mountable on any `pocket-mod` guest as `globalThis.fs` | tested including a live QuickJS guest round-trip | +| consoles / devices | — | a target appends `data.fs` to its profile when its native host ships the module; the port point is the data root (POSIX dir on desktop, a LittleFS directory on MCU hosts) | + +## Adoption path + +A device host that wants the module makes three moves, none of which touch +the spec, the SDK, or any app: + +1. pick the app's data root and a sibling temp directory on the platform + filesystem (Pocket Pi: `/workspace/apps//data/` — the SAME + root the db module binds — and `/workspace/apps//tmp/`) and + construct the module bound to them — + `pocket_fs::FsModule::new(Storage::Dir { root, tmp })`, one instance + per app; +2. mount the namespace beside `ui` — `pocket_fs::mount(&guest, module)` on + `pocket-mod` hosts, or the raw-QuickJS spelling of the same nine + functions elsewhere (depend with `default-features = false` to drop the + pocket-mod dependency; verified to `cargo check` clean for + `riscv32imafc-esp-espidf`); +3. append `data.fs` to the target's profile in + `contracts/spec/platforms.ts`. diff --git a/engine/Cargo.lock b/engine/Cargo.lock index 8913a7f6..d33204ab 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -1646,6 +1646,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "pocket-fs" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.23.1", + "pocket-mod", + "pocketjs-core", + "serde_json", +] + [[package]] name = "pocket-mod" version = "0.1.0" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 206ad691..38eb5f5e 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -12,6 +12,7 @@ resolver = "2" members = [ "crates/pocket-db", + "crates/pocket-fs", "crates/pocket-mod", "crates/pocket-net", "crates/pocket-ui-surface", @@ -45,6 +46,7 @@ repository = "https://github.com/pocket-stack/pocketjs" pocket3d = { path = "pocket3d/crates/pocket3d" } pocket3d-bsp = { path = "pocket3d/crates/pocket3d-bsp" } pocket-db = { path = "crates/pocket-db" } +pocket-fs = { path = "crates/pocket-fs" } pocket-mod = { path = "crates/pocket-mod" } pocket-net = { path = "crates/pocket-net" } pocket-ui-surface = { path = "crates/pocket-ui-surface" } diff --git a/engine/core/src/spec.rs b/engine/core/src/spec.rs index ffcba9c0..bec774d2 100644 --- a/engine/core/src/spec.rs +++ b/engine/core/src/spec.rs @@ -519,6 +519,36 @@ pub mod db { pub const MAX_RESULT_ROWS: usize = 4096; } +/// FS module boundary (contracts/spec/fs.ts — `globalThis.fs`). +/// A per-app file tree behind nine synchronous ops; every path resolves +/// under the app's own data root. No clock, no events, no mtime. +pub mod fs { + pub const OP_READ: u8 = 1; + pub const OP_WRITE: u8 = 2; + pub const OP_REMOVE: u8 = 3; + pub const OP_LIST: u8 = 4; + pub const OP_STAT: u8 = 5; + pub const OP_MKDIR: u8 = 6; + pub const OP_RENAME: u8 = 7; + pub const OP_USAGE: u8 = 8; + pub const OP_LAST_ERROR: u8 = 9; + /// write() modes. + pub const WRITE_TRUNCATE: u32 = 0; + pub const WRITE_APPEND: u32 = 1; + /// Marker key for a bytes payload (db's blob spelling). + pub const BLOB_KEY: &str = "$b"; + /// Maximum UTF-8 bytes in one path segment. + pub const MAX_SEGMENT_BYTES: usize = 64; + /// Maximum segments in a path. + pub const MAX_DEPTH: usize = 8; + /// Maximum total path length in bytes. + pub const MAX_PATH_BYTES: usize = 160; + /// Payload ceiling per read()/write() call, in bytes. + pub const MAX_IO_BYTES: usize = 65536; + /// Entries per list() call (paged via offset + eof). + pub const MAX_DIR_ENTRIES: usize = 256; +} + /// NET module boundary (contracts/spec/net.ts — `globalThis.net`). /// Bounded whole-response HTTP; completions batch to tick boundaries. pub mod net { diff --git a/engine/crates/pocket-fs/Cargo.toml b/engine/crates/pocket-fs/Cargo.toml new file mode 100644 index 00000000..b9c4f2c7 --- /dev/null +++ b/engine/crates/pocket-fs/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pocket-fs" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The fs module's reference core: a per-app file tree behind the nine-op contracts/spec/fs.ts boundary, mountable as globalThis.fs via pocket-mod" + +[features] +# `mount` brings pocket-mod (and its QuickJS embedding) for the one-line +# globalThis.fs install. A device host with its own QuickJS wiring turns +# it off (`default-features = false`) and drives FsModule directly — the +# MCU build then never compiles an engine it doesn't use. +default = ["mount"] +mount = ["dep:pocket-mod", "dep:anyhow"] + +[dependencies] +pocketjs-core = { workspace = true } +pocket-mod = { workspace = true, optional = true } +anyhow = { workspace = true, optional = true } +serde_json = { workspace = true } +base64 = { workspace = true } diff --git a/engine/crates/pocket-fs/src/lib.rs b/engine/crates/pocket-fs/src/lib.rs new file mode 100644 index 00000000..6e847590 --- /dev/null +++ b/engine/crates/pocket-fs/src/lib.rs @@ -0,0 +1,1044 @@ +//! pocket-fs — the fs module's reference core. +//! +//! A per-app file tree behind the nine-op boundary pinned in +//! contracts/spec/fs.ts (`pocketjs_core::spec::fs` is the generated +//! mirror): read / write / remove / list / stat / mkdir / rename / usage / +//! lastError, mounted as `globalThis.fs` through [`mount`]. Payloads cross +//! as one JSON value (a string for text, `{"$b": base64}` for bytes); +//! results cross as one JSON line. +//! +//! Storage policy is the host's: [`Storage::Memory`] for tests and +//! throwaway guests, [`Storage::Dir`] to bind the module to the app's own +//! data root on a real filesystem. Names are universal — any well-formed +//! Unicode, dot-prefixed included; nothing in the app's tree is reserved +//! to the host. Isolation is by construction and never depended on names: +//! every path is relative, `..`/absolute/`/`-in-segment are +//! unrepresentable, so the bound root is the sandbox boundary the way +//! db's ATTACH refusal keeps its data root one. The guest cannot create +//! symlinks through this API, but a host-side actor may have (on Pocket +//! Pi the device agent owns the whole workspace), so the Dir backend +//! lstat-checks every segment and treats any symlink as absent. +//! +//! Truncate writes are ATOMIC (temp + rename): after power loss a file +//! holds the old content or the new, never a torn middle. Temps land in +//! the module's own `tmp` directory — OUTSIDE the bound root, on the +//! same filesystem (cross-directory rename stays atomic) — so the app's +//! tree never shows host machinery, and a crash orphan cannot outlive +//! the next construction: the module OWNS `tmp` and clears it on +//! construction, which is provably safe precisely because nothing else +//! may live there. Porting note (the ESP32/LittleFS path): LittleFS's +//! rename is atomic, so a device host keeps the same contract by the +//! same moves. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read as _, Seek as _, SeekFrom, Write as _}; +use std::path::{Path, PathBuf}; + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use pocketjs_core::spec::fs as spec; +use serde_json::{json, Value as Json}; + +/// Where the app's file tree lives. +pub enum Storage { + /// The whole tree in memory (tests, previews). Byte-exact names — the + /// deterministic twin of the sim host. + Memory, + /// The tree under `root` — the app's own data root; the guest never + /// sees the path. `tmp` is a host-private directory for atomic-write + /// temps: same filesystem, outside `root` (Pocket Pi layout: + /// `apps//data` and `apps//tmp`). The module OWNS `tmp` and + /// clears it on construction. + Dir { root: PathBuf, tmp: PathBuf }, +} + +enum Backend { + Memory { + files: BTreeMap>, + dirs: BTreeSet, + }, + Dir { + root: PathBuf, + tmp: PathBuf, + tmp_counter: u64, + }, +} + +/// The fs module: every op as a method, [`mount`] to install the namespace. +pub struct FsModule { + backend: Backend, + /// Byte budget for the tree; 0 = unmetered. Enforced on write. + quota_bytes: u64, + last_error: String, +} + +impl FsModule { + pub fn new(storage: Storage) -> FsModule { + FsModule::with_quota(storage, 0) + } + + pub fn with_quota(storage: Storage, quota_bytes: u64) -> FsModule { + FsModule { + backend: match storage { + Storage::Memory => Backend::Memory { + files: BTreeMap::new(), + dirs: BTreeSet::new(), + }, + Storage::Dir { root, tmp } => { + // Best-effort sweep: any leftover temp is an orphan + // from a crash mid-write; nothing else lives here. + let _ = std::fs::remove_dir_all(&tmp); + Backend::Dir { + root, + tmp, + tmp_counter: 0, + } + } + }, + quota_bytes, + last_error: String::new(), + } + } + + fn ok_line(&mut self, line: String) -> String { + self.last_error.clear(); + line + } + + fn err_line(&mut self, message: &str) -> String { + self.last_error = message.to_owned(); + json!({ "error": message }).to_string() + } + + fn status(&mut self, result: Result<(), String>) -> i32 { + match result { + Ok(()) => { + self.last_error.clear(); + 0 + } + Err(message) => { + self.last_error = message; + 1 + } + } + } + + /// `read(path, offset, maxBytes) -> json line` (spec OP_READ). + pub fn read(&mut self, path: &str, offset: i64, max_bytes: i64) -> String { + if !valid_path(path) { + return self.err_line("invalid path"); + } + if max_bytes < 1 || max_bytes as usize > spec::MAX_IO_BYTES { + return self.err_line("read maxBytes out of range"); + } + if offset < 0 { + return self.err_line("read offset out of range"); + } + let result = match &mut self.backend { + Backend::Memory { files, dirs } => { + match files.get(path) { + Some(bytes) => { + let start = (offset as usize).min(bytes.len()); + let end = (start + max_bytes as usize).min(bytes.len()); + Ok((bytes[start..end].to_vec(), bytes.len() as u64, end >= bytes.len())) + } + None if dirs.contains(path) => Err("is a directory".to_owned()), + None => Err("not found".to_owned()), + } + } + Backend::Dir { root, .. } => dir_read(root, path, offset as u64, max_bytes as usize), + }; + match result { + Ok((chunk, size, eof)) => self.ok_line( + json!({ + "data": { spec::BLOB_KEY: BASE64.encode(&chunk) }, + "size": size, + "eof": eof, + }) + .to_string(), + ), + Err(message) => self.err_line(&message), + } + } + + /// `write(path, data, mode) -> 0 | 1` (spec OP_WRITE). + pub fn write(&mut self, path: &str, data: &str, mode: u32) -> i32 { + let result = self.write_inner(path, data, mode); + self.status(result) + } + + fn write_inner(&mut self, path: &str, data: &str, mode: u32) -> Result<(), String> { + if !valid_path(path) { + return Err("invalid path".to_owned()); + } + if mode != spec::WRITE_TRUNCATE && mode != spec::WRITE_APPEND { + return Err("invalid write mode".to_owned()); + } + let payload = decode_payload(data)?; + if payload.len() > spec::MAX_IO_BYTES { + return Err("write exceeds FS_MAX_IO_BYTES".to_owned()); + } + let quota = self.quota_bytes; + match &mut self.backend { + Backend::Memory { files, dirs } => { + if dirs.contains(path) { + return Err("is a directory".to_owned()); + } + for ancestor in ancestors_of(path) { + if files.contains_key(&ancestor) { + return Err(format!("not a directory: {ancestor}")); + } + dirs.insert(ancestor); + } + let existing = files.get(path).map(Vec::len).unwrap_or(0) as u64; + let next = if mode == spec::WRITE_APPEND { + existing + payload.len() as u64 + } else { + payload.len() as u64 + }; + let used: u64 = files.values().map(|b| b.len() as u64).sum(); + if quota > 0 && used - existing + next > quota { + return Err("quota exceeded".to_owned()); + } + if mode == spec::WRITE_APPEND { + files.entry(path.to_owned()).or_default().extend_from_slice(&payload); + } else { + files.insert(path.to_owned(), payload); + } + Ok(()) + } + Backend::Dir { + root, + tmp, + tmp_counter, + } => { + *tmp_counter += 1; + dir_write(root, tmp, path, &payload, mode, quota, *tmp_counter) + } + } + } + + /// `remove(path, recursive) -> 0 | 1` (spec OP_REMOVE). + pub fn remove(&mut self, path: &str, recursive: u32) -> i32 { + let result = (|| { + if !valid_path(path) { + return Err("invalid path".to_owned()); + } + match &mut self.backend { + Backend::Memory { files, dirs } => { + if files.remove(path).is_some() { + return Ok(()); + } + if !dirs.contains(path) { + return Err("not found".to_owned()); + } + let prefix = format!("{path}/"); + let occupied = files.keys().any(|k| k.starts_with(&prefix)) + || dirs.iter().any(|k| k.starts_with(&prefix)); + if occupied && recursive != 1 { + return Err("directory not empty".to_owned()); + } + files.retain(|k, _| !k.starts_with(&prefix)); + dirs.retain(|k| !k.starts_with(&prefix)); + dirs.remove(path); + Ok(()) + } + Backend::Dir { root, .. } => dir_remove(root, path, recursive == 1), + } + })(); + self.status(result) + } + + /// `list(path, offset) -> json line` (spec OP_LIST). + pub fn list(&mut self, path: &str, offset: i64) -> String { + if !path.is_empty() && !valid_path(path) { + return self.err_line("invalid path"); + } + let offset = offset.max(0) as usize; + let result = match &mut self.backend { + Backend::Memory { files, dirs } => { + if files.contains_key(path) { + Err("not a directory".to_owned()) + } else if !path.is_empty() && !dirs.contains(path) { + Err("not found".to_owned()) + } else { + let mut names: BTreeSet = BTreeSet::new(); + let prefix = if path.is_empty() { String::new() } else { format!("{path}/") }; + for key in files.keys().chain(dirs.iter()) { + if let Some(rest) = key.strip_prefix(&prefix) { + if key == path || rest.is_empty() { + continue; + } + names.insert(rest.split('/').next().unwrap().to_owned()); + } + } + Ok(names + .into_iter() + .map(|name| { + let full = + if path.is_empty() { name.clone() } else { format!("{path}/{name}") }; + match files.get(&full) { + Some(bytes) => (name, "file", bytes.len() as u64), + None => (name, "dir", 0), + } + }) + .collect::>()) + } + } + Backend::Dir { root, .. } => dir_list(root, path), + }; + match result { + Ok(all) => { + let page: Vec = all + .iter() + .skip(offset) + .take(spec::MAX_DIR_ENTRIES) + .map(|(name, kind, size)| json!({ "name": name, "kind": kind, "size": size })) + .collect(); + let eof = offset + page.len() >= all.len(); + self.ok_line(json!({ "entries": page, "eof": eof }).to_string()) + } + Err(message) => self.err_line(&message), + } + } + + /// `stat(path) -> json line` (spec OP_STAT). + pub fn stat(&mut self, path: &str) -> String { + if path.is_empty() { + return self.ok_line(json!({ "kind": "dir", "size": 0 }).to_string()); + } + if !valid_path(path) { + return self.err_line("invalid path"); + } + let result = match &mut self.backend { + Backend::Memory { files, dirs } => match files.get(path) { + Some(bytes) => Some(("file", bytes.len() as u64)), + None if dirs.contains(path) => Some(("dir", 0)), + None => None, + }, + Backend::Dir { root, .. } => dir_stat(root, path), + }; + match result { + Some((kind, size)) => self.ok_line(json!({ "kind": kind, "size": size }).to_string()), + None => self.err_line("not found"), + } + } + + /// `mkdir(path) -> 0 | 1` (spec OP_MKDIR) — recursive, idempotent. + pub fn mkdir(&mut self, path: &str) -> i32 { + let result = (|| { + if !valid_path(path) { + return Err("invalid path".to_owned()); + } + match &mut self.backend { + Backend::Memory { files, dirs } => { + if files.contains_key(path) { + return Err(format!("not a directory: {path}")); + } + for ancestor in ancestors_of(path) { + if files.contains_key(&ancestor) { + return Err(format!("not a directory: {ancestor}")); + } + dirs.insert(ancestor); + } + dirs.insert(path.to_owned()); + Ok(()) + } + Backend::Dir { root, .. } => dir_mkdir(root, path), + } + })(); + self.status(result) + } + + /// `rename(from, to) -> 0 | 1` (spec OP_RENAME). + pub fn rename(&mut self, from: &str, to: &str) -> i32 { + let result = (|| { + if !valid_path(from) || !valid_path(to) { + return Err("invalid path".to_owned()); + } + if from == to { + return Ok(()); + } + if to.starts_with(&format!("{from}/")) { + return Err("cannot rename into own subtree".to_owned()); + } + match &mut self.backend { + Backend::Memory { files, dirs } => { + let to_parent = parent_of(to); + if !to_parent.is_empty() && !dirs.contains(to_parent) { + return Err("not found".to_owned()); + } + if dirs.contains(to) { + return Err("destination exists".to_owned()); + } + if let Some(bytes) = files.remove(from) { + files.insert(to.to_owned(), bytes); + return Ok(()); + } + if !dirs.contains(from) { + return Err("not found".to_owned()); + } + if files.contains_key(to) { + return Err("destination exists".to_owned()); + } + let prefix = format!("{from}/"); + let moved_files: Vec<(String, Vec)> = files + .iter() + .filter(|(k, _)| k.starts_with(&prefix)) + .map(|(k, v)| (format!("{to}/{}", &k[prefix.len()..]), v.clone())) + .collect(); + files.retain(|k, _| !k.starts_with(&prefix)); + files.extend(moved_files); + let moved_dirs: Vec = dirs + .iter() + .filter(|k| k.starts_with(&prefix)) + .map(|k| format!("{to}/{}", &k[prefix.len()..])) + .collect(); + dirs.retain(|k| !k.starts_with(&prefix)); + dirs.extend(moved_dirs); + dirs.remove(from); + dirs.insert(to.to_owned()); + Ok(()) + } + Backend::Dir { root, .. } => dir_rename(root, from, to), + } + })(); + self.status(result) + } + + /// `usage() -> json line` (spec OP_USAGE). + pub fn usage(&mut self) -> String { + let used: u64 = match &self.backend { + Backend::Memory { files, .. } => files.values().map(|b| b.len() as u64).sum(), + Backend::Dir { root, .. } => dir_used_bytes(root), + }; + let quota = self.quota_bytes; + self.ok_line(json!({ "usedBytes": used, "quotaBytes": quota }).to_string()) + } + + /// `lastError() -> string` (spec OP_LAST_ERROR) — module-scoped. + pub fn last_error(&self) -> String { + self.last_error.clone() + } +} + +// --------------------------------------------------------------------------- +// The path grammar (contracts/spec/fs.ts, spelled out — no regex dependency) +// --------------------------------------------------------------------------- + +/// Universal names: any well-formed Unicode (a Rust `&str` already is) +/// except the escape hatches ("." and ".."), control characters, and +/// oversize segments. "/" inside a name is unrepresentable — the caller +/// split on it. +fn valid_segment(segment: &str) -> bool { + if segment.is_empty() || segment.len() > spec::MAX_SEGMENT_BYTES { + return false; + } + if segment == "." || segment == ".." { + return false; + } + !segment.bytes().any(|b| b < 0x20 || b == 0x7f) +} + +/// fsValidSegment / FS_MAX_DEPTH / FS_MAX_PATH_BYTES, one predicate. +fn valid_path(path: &str) -> bool { + if path.is_empty() || path.len() > spec::MAX_PATH_BYTES { + return false; + } + let segments: Vec<&str> = path.split('/').collect(); + segments.len() <= spec::MAX_DEPTH && segments.iter().all(|s| valid_segment(s)) +} + +/// Ancestor paths of a valid path, nearest last ("a/b/c" -> ["a", "a/b"]). +fn ancestors_of(path: &str) -> Vec { + let mut out = Vec::new(); + for (i, b) in path.bytes().enumerate() { + if b == b'/' { + out.push(path[..i].to_owned()); + } + } + out +} + +fn parent_of(path: &str) -> &str { + match path.rfind('/') { + Some(i) => &path[..i], + None => "", + } +} + +/// JSON payload -> bytes (a string stores as UTF-8; {"$b": base64} as-is). +fn decode_payload(data: &str) -> Result, String> { + let parsed: Json = + serde_json::from_str(data).map_err(|e| format!("malformed payload: {e}"))?; + match parsed { + Json::String(text) => Ok(text.into_bytes()), + Json::Object(map) => match map.get(spec::BLOB_KEY) { + Some(Json::String(b64)) => { + BASE64.decode(b64).map_err(|e| format!("malformed payload: {e}")) + } + _ => Err("malformed payload: a JSON string or {\"$b\": base64}".to_owned()), + }, + _ => Err("malformed payload: a JSON string or {\"$b\": base64}".to_owned()), + } +} + +// --------------------------------------------------------------------------- +// The Dir backend — std::fs under the app root, symlinks treated as absent +// --------------------------------------------------------------------------- + +/// Resolve `path` under `root`, refusing any symlink component. The grammar +/// already forbids `..`/absolute paths; this guards against a HOST-side +/// actor having planted a link inside the root. +fn resolve(root: &Path, path: &str) -> Result { + let mut current = root.to_path_buf(); + for segment in path.split('/') { + current.push(segment); + if std::fs::symlink_metadata(¤t).is_ok_and(|md| md.file_type().is_symlink()) { + return Err("not found".to_owned()); + } + } + Ok(current) +} + +fn dir_read(root: &Path, path: &str, offset: u64, max_bytes: usize) -> Result<(Vec, u64, bool), String> { + let full = resolve(root, path)?; + let md = std::fs::metadata(&full).map_err(|_| "not found".to_owned())?; + if md.is_dir() { + return Err("is a directory".to_owned()); + } + let size = md.len(); + let mut file = std::fs::File::open(&full).map_err(|e| e.to_string())?; + file.seek(SeekFrom::Start(offset.min(size))).map_err(|e| e.to_string())?; + let mut chunk = vec![0u8; max_bytes]; + let mut filled = 0; + while filled < max_bytes { + let n = file.read(&mut chunk[filled..]).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + filled += n; + } + chunk.truncate(filled); + let eof = offset.min(size) + filled as u64 >= size; + Ok((chunk, size, eof)) +} + +fn dir_write( + root: &Path, + tmp_dir: &Path, + path: &str, + payload: &[u8], + mode: u32, + quota: u64, + tmp_counter: u64, +) -> Result<(), String> { + let full = resolve(root, path)?; + if full.is_dir() { + return Err("is a directory".to_owned()); + } + // Refuse a file on the ancestor chain with the memory backend's message. + for ancestor in ancestors_of(path) { + let p = resolve(root, &ancestor)?; + if p.is_file() { + return Err(format!("not a directory: {ancestor}")); + } + } + let parent = full.parent().expect("resolved path always has a parent"); + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + if quota > 0 { + let existing = std::fs::metadata(&full).map(|m| m.len()).unwrap_or(0); + let next = if mode == spec::WRITE_APPEND { + existing + payload.len() as u64 + } else { + payload.len() as u64 + }; + if dir_used_bytes(root) - existing + next > quota { + return Err("quota exceeded".to_owned()); + } + } + if mode == spec::WRITE_APPEND { + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&full) + .map_err(|e| e.to_string())?; + file.write_all(payload).map_err(|e| e.to_string())?; + file.sync_all().map_err(|e| e.to_string())?; + return Ok(()); + } + // The atomicity contract: land the payload in the host-owned temp + // directory, sync, then rename over the target (same filesystem — + // cross-directory rename is atomic). + std::fs::create_dir_all(tmp_dir).map_err(|e| e.to_string())?; + let mut suffix = tmp_counter; + let (tmp, mut file) = loop { + let candidate = tmp_dir.join(suffix.to_string()); + match std::fs::OpenOptions::new().write(true).create_new(true).open(&candidate) { + Ok(file) => break (candidate, file), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => suffix += 1, + Err(e) => return Err(e.to_string()), + } + }; + let landed = file + .write_all(payload) + .and_then(|()| file.sync_all()) + .map_err(|e| e.to_string()); + drop(file); + landed + .and_then(|()| std::fs::rename(&tmp, &full).map_err(|e| e.to_string())) + .inspect_err(|_| { + let _ = std::fs::remove_file(&tmp); + }) +} + +fn dir_remove(root: &Path, path: &str, recursive: bool) -> Result<(), String> { + let full = resolve(root, path)?; + let md = std::fs::symlink_metadata(&full).map_err(|_| "not found".to_owned())?; + if md.is_file() { + return std::fs::remove_file(&full).map_err(|e| e.to_string()); + } + if !recursive { + return match std::fs::remove_dir(&full) { + Ok(()) => Ok(()), + Err(_) if std::fs::read_dir(&full).map(|mut d| d.next().is_some()).unwrap_or(false) => { + Err("directory not empty".to_owned()) + } + Err(e) => Err(e.to_string()), + }; + } + std::fs::remove_dir_all(&full).map_err(|e| e.to_string()) +} + +fn dir_list(root: &Path, path: &str) -> Result, String> { + let full = if path.is_empty() { root.to_path_buf() } else { resolve(root, path)? }; + let md = std::fs::metadata(&full).map_err(|_| "not found".to_owned())?; + if md.is_file() { + return Err("not a directory".to_owned()); + } + let mut out: Vec<(String, &'static str, u64)> = Vec::new(); + for entry in std::fs::read_dir(&full).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = match entry.file_name().into_string() { + Ok(name) => name, + Err(_) => continue, + }; + // A name the vocabulary cannot address (control chars, oversize) + // does not exist to the guest — it could be listed but never read. + if !valid_segment(&name) { + continue; + } + let emd = entry.metadata().map_err(|e| e.to_string())?; + if emd.file_type().is_symlink() { + continue; + } + if emd.is_dir() { + out.push((name, "dir", 0)); + } else { + out.push((name, "file", emd.len())); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(out) +} + +fn dir_stat(root: &Path, path: &str) -> Option<(&'static str, u64)> { + let full = resolve(root, path).ok()?; + let md = std::fs::symlink_metadata(&full).ok()?; + if md.file_type().is_symlink() { + return None; + } + if md.is_dir() { + Some(("dir", 0)) + } else { + Some(("file", md.len())) + } +} + +fn dir_mkdir(root: &Path, path: &str) -> Result<(), String> { + for ancestor in ancestors_of(path).into_iter().chain([path.to_owned()]) { + let p = resolve(root, &ancestor)?; + if p.is_file() { + return Err(format!("not a directory: {ancestor}")); + } + } + let full = resolve(root, path)?; + std::fs::create_dir_all(&full).map_err(|e| e.to_string()) +} + +fn dir_rename(root: &Path, from: &str, to: &str) -> Result<(), String> { + let from_full = resolve(root, from)?; + let from_md = std::fs::symlink_metadata(&from_full).map_err(|_| "not found".to_owned())?; + let to_full = resolve(root, to)?; + let to_parent = to_full.parent().expect("resolved path always has a parent"); + if !to_parent.is_dir() { + return Err("not found".to_owned()); + } + if std::fs::symlink_metadata(&to_full).is_ok_and(|to_md| to_md.is_dir() || from_md.is_dir()) { + return Err("destination exists".to_owned()); + } + std::fs::rename(&from_full, &to_full).map_err(|e| e.to_string()) +} + +fn dir_used_bytes(root: &Path) -> u64 { + fn walk(dir: &Path) -> u64 { + let Ok(entries) = std::fs::read_dir(dir) else { + return 0; + }; + let mut total = 0; + for entry in entries.flatten() { + let Ok(md) = entry.metadata() else { continue }; + if md.file_type().is_symlink() { + continue; + } + if md.is_dir() { + total += walk(&entry.path()); + } else { + total += md.len(); + } + } + total + } + walk(root) +} + +// --------------------------------------------------------------------------- +// Mount +// --------------------------------------------------------------------------- + +#[cfg(feature = "mount")] +use std::cell::RefCell; +#[cfg(feature = "mount")] +use std::rc::Rc; + +/// Mount the module as `globalThis.fs` on a pocket-mod [`Guest`] — one JS +/// function per spec op, marshaled as (String, f64) -> i32/String. +/// Feature `mount` (default); a host with its own QuickJS wiring turns it +/// off and spells these nine functions itself. +#[cfg(feature = "mount")] +pub fn mount(guest: &pocket_mod::Guest, module: Rc>) -> anyhow::Result<()> { + use pocket_mod::qjs::Function; + guest.mount("fs", |ctx, ns| { + let m = module.clone(); + ns.set( + "read", + Function::new( + ctx.clone(), + move |path: String, offset: f64, max_bytes: f64| -> String { + m.borrow_mut().read(&path, offset as i64, max_bytes as i64) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "write", + Function::new( + ctx.clone(), + move |path: String, data: String, mode: f64| -> i32 { + m.borrow_mut().write(&path, &data, mode as u32) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "remove", + Function::new(ctx.clone(), move |path: String, recursive: f64| -> i32 { + m.borrow_mut().remove(&path, recursive as u32) + })?, + )?; + let m = module.clone(); + ns.set( + "list", + Function::new(ctx.clone(), move |path: String, offset: f64| -> String { + m.borrow_mut().list(&path, offset as i64) + })?, + )?; + let m = module.clone(); + ns.set( + "stat", + Function::new(ctx.clone(), move |path: String| -> String { + m.borrow_mut().stat(&path) + })?, + )?; + let m = module.clone(); + ns.set( + "mkdir", + Function::new(ctx.clone(), move |path: String| -> i32 { + m.borrow_mut().mkdir(&path) + })?, + )?; + let m = module.clone(); + ns.set( + "rename", + Function::new(ctx.clone(), move |from: String, to: String| -> i32 { + m.borrow_mut().rename(&from, &to) + })?, + )?; + let m = module.clone(); + ns.set( + "usage", + Function::new(ctx.clone(), move || -> String { m.borrow_mut().usage() })?, + )?; + let m = module.clone(); + ns.set( + "lastError", + Function::new(ctx.clone(), move || -> String { m.borrow().last_error() })?, + )?; + Ok(()) + }) +} + +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn module() -> FsModule { + FsModule::new(Storage::Memory) + } + + fn line(s: &str) -> Json { + serde_json::from_str(s).unwrap() + } + + fn text(s: &str) -> String { + json!(s).to_string() + } + + #[test] + fn path_grammar_refuses_escapes_and_nothing_else() { + // The security rule: escapes and malformed shapes. + for bad in ["", "/abs", "a//b", "a/", "../up", "a/../b", "a/.", "a\x07b"] { + assert!(!valid_path(bad), "{bad:?} should be invalid"); + } + assert!(valid_path(&vec!["a"; spec::MAX_DEPTH].join("/"))); + assert!(!valid_path(&vec!["a"; spec::MAX_DEPTH + 1].join("/"))); + assert!(!valid_path(&format!("{}x", "a".repeat(spec::MAX_PATH_BYTES)))); + assert!(!valid_path(&"名".repeat(22)), "22 CJK chars = 66 bytes > segment cap"); + // Universal names: anything an app wants to call its own files. + for good in [ + "a", + "notes/today.md", + "A1._-x", + ".config", + "notes/.drafts/今日笔记.md", + "-lead", + "a\\b", + "space in name.txt", + ] { + assert!(valid_path(good), "{good:?} should be valid"); + } + } + + #[test] + fn universal_names_round_trip() { + let mut m = module(); + assert_eq!(m.write("笔记/今天.md", &text("你好"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write(".config", &text("k=v"), spec::WRITE_TRUNCATE), 0); + assert_eq!(line(&m.stat("笔记/今天.md"))["size"], 6); + let listing = line(&m.list("", 0)); + let names: Vec<&str> = listing["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["name"].as_str().unwrap()) + .collect(); + // Code point order: "." (U+002E) < "笔" (U+7B14). + assert_eq!(names, [".config", "笔记"]); + } + + #[test] + fn write_read_round_trip_text_and_bytes() { + let mut m = module(); + assert_eq!(m.write("notes/today.md", &text("# 今天"), spec::WRITE_TRUNCATE), 0); + let read = line(&m.read("notes/today.md", 0, spec::MAX_IO_BYTES as i64)); + let bytes = BASE64.decode(read["data"][spec::BLOB_KEY].as_str().unwrap()).unwrap(); + assert_eq!(String::from_utf8(bytes).unwrap(), "# 今天"); + assert_eq!(read["eof"], true); + + let payload = json!({ spec::BLOB_KEY: BASE64.encode([0u8, 1, 255]) }).to_string(); + assert_eq!(m.write("raw.bin", &payload, spec::WRITE_TRUNCATE), 0); + assert_eq!(line(&m.stat("raw.bin"))["size"], 3); + } + + #[test] + fn append_and_chunked_read() { + let mut m = module(); + assert_eq!(m.write("log.txt", &text("aaa"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write("log.txt", &text("bbb"), spec::WRITE_APPEND), 0); + let first = line(&m.read("log.txt", 0, 4)); + assert_eq!(first["size"], 6); + assert_eq!(first["eof"], false); + let rest = line(&m.read("log.txt", 4, 4)); + assert_eq!(rest["eof"], true); + let bytes = BASE64.decode(rest["data"][spec::BLOB_KEY].as_str().unwrap()).unwrap(); + assert_eq!(bytes, b"bb"); + } + + #[test] + fn write_creates_parents_and_refuses_file_ancestors() { + let mut m = module(); + assert_eq!(m.write("a/b/c.txt", &text("x"), spec::WRITE_TRUNCATE), 0); + assert_eq!(line(&m.stat("a/b"))["kind"], "dir"); + assert_eq!(m.write("a/b/c.txt/d.txt", &text("x"), spec::WRITE_TRUNCATE), 1); + assert!(m.last_error().contains("not a directory")); + } + + #[test] + fn remove_semantics() { + let mut m = module(); + m.write("dir/f.txt", &text("x"), spec::WRITE_TRUNCATE); + assert_eq!(m.remove("missing.txt", 0), 1); + assert_eq!(m.last_error(), "not found"); + assert_eq!(m.remove("dir", 0), 1); + assert_eq!(m.last_error(), "directory not empty"); + assert_eq!(m.remove("dir", 1), 0); + assert!(line(&m.stat("dir"))["error"].as_str().is_some()); + } + + #[test] + fn list_is_sorted_and_pages() { + let mut m = module(); + for i in 0..(spec::MAX_DIR_ENTRIES + 3) { + m.write(&format!("d/f{i:04}.txt"), &text("x"), spec::WRITE_TRUNCATE); + } + let first = line(&m.list("d", 0)); + assert_eq!(first["entries"].as_array().unwrap().len(), spec::MAX_DIR_ENTRIES); + assert_eq!(first["eof"], false); + assert_eq!(first["entries"][0]["name"], "f0000.txt"); + let second = line(&m.list("d", spec::MAX_DIR_ENTRIES as i64)); + assert_eq!(second["entries"].as_array().unwrap().len(), 3); + assert_eq!(second["eof"], true); + } + + #[test] + fn rename_semantics() { + let mut m = module(); + m.write("a.txt", &text("A"), spec::WRITE_TRUNCATE); + m.write("b.txt", &text("B"), spec::WRITE_TRUNCATE); + assert_eq!(m.rename("a.txt", "b.txt"), 0, "file over file replaces"); + assert_eq!(line(&m.stat("a.txt"))["error"], "not found"); + + m.mkdir("sub"); + assert_eq!(m.rename("b.txt", "sub"), 1); + assert_eq!(m.last_error(), "destination exists"); + assert_eq!(m.rename("b.txt", "ghost/x.txt"), 1, "missing parent fails"); + assert_eq!(m.rename("sub", "sub/inner"), 1); + assert_eq!(m.last_error(), "cannot rename into own subtree"); + + m.write("sub/deep/f.txt", &text("x"), spec::WRITE_TRUNCATE); + assert_eq!(m.rename("sub", "moved"), 0); + assert_eq!(line(&m.stat("moved/deep/f.txt"))["kind"], "file"); + } + + #[test] + fn quota_is_enforced_and_usage_reports() { + let mut m = FsModule::with_quota(Storage::Memory, 10); + assert_eq!(m.write("a.txt", &text("12345678"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write("b.txt", &text("123"), spec::WRITE_TRUNCATE), 1); + assert_eq!(m.last_error(), "quota exceeded"); + assert_eq!(m.write("a.txt", &text("1"), spec::WRITE_TRUNCATE), 0, "shrink fits"); + let usage = line(&m.usage()); + assert_eq!(usage["usedBytes"], 1); + assert_eq!(usage["quotaBytes"], 10); + } + + #[test] + fn io_ceiling_fails_loudly() { + let mut m = module(); + let too_big = "x".repeat(spec::MAX_IO_BYTES + 1); + assert_eq!(m.write("big.txt", &text(&too_big), spec::WRITE_TRUNCATE), 1); + assert!(m.last_error().contains("FS_MAX_IO_BYTES")); + m.write("ok.txt", &text("x"), spec::WRITE_TRUNCATE); + let over = line(&m.read("ok.txt", 0, spec::MAX_IO_BYTES as i64 + 1)); + assert!(over["error"].as_str().unwrap().contains("maxBytes")); + } + + #[test] + fn dir_storage_round_trip_atomicity_and_symlink_refusal() { + let base = std::env::temp_dir().join(format!("pocket-fs-test-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + let root = base.join("data"); + let tmp = base.join("tmp"); + let dir = || Storage::Dir { root: root.clone(), tmp: tmp.clone() }; + std::fs::create_dir_all(&root).unwrap(); + // A leftover orphan from a "crash" is swept on construction. + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::write(tmp.join("7"), b"orphan").unwrap(); + { + let mut m = FsModule::new(dir()); + assert!(!tmp.join("7").exists(), "orphan swept on construction"); + assert_eq!(m.write("notes/a.md", &text("hello"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write("notes/a.md", &text(" world"), spec::WRITE_APPEND), 0); + m.mkdir("empty"); + let listing = line(&m.list("", 0)); + let names: Vec<&str> = listing["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["empty", "notes"]); + } + { + // A fresh module over the same root sees the persisted tree. + let mut m = FsModule::new(dir()); + let read = line(&m.read("notes/a.md", 0, 64)); + let bytes = BASE64.decode(read["data"][spec::BLOB_KEY].as_str().unwrap()).unwrap(); + assert_eq!(bytes, b"hello world"); + // The app tree holds ONLY app names — temps live in `tmp`, + // outside the bound root. + let names: Vec = std::fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!(names.iter().all(|n| n == "notes" || n == "empty"), "{names:?}"); + } + #[cfg(unix)] + { + let outside = root.parent().unwrap().join("pocket-fs-outside.txt"); + std::fs::write(&outside, b"secret").unwrap(); + std::os::unix::fs::symlink(&outside, root.join("link.txt")).unwrap(); + let mut m = FsModule::new(dir()); + let read = line(&m.read("link.txt", 0, 64)); + assert_eq!(read["error"], "not found", "a symlink is invisible"); + assert_eq!(line(&m.stat("link.txt"))["error"], "not found"); + let listing = line(&m.list("", 0)); + assert!(!listing["entries"] + .as_array() + .unwrap() + .iter() + .any(|e| e["name"] == "link.txt")); + std::fs::remove_file(&outside).unwrap(); + } + std::fs::remove_dir_all(&base).unwrap(); + } + + #[cfg(feature = "mount")] + #[test] + fn mounted_namespace_serves_a_quickjs_guest() { + let guest = pocket_mod::Guest::new().unwrap(); + let module = Rc::new(RefCell::new(module())); + mount(&guest, module).unwrap(); + guest + .eval( + "boot", + r#" + if (fs.write("notes/hi.txt", JSON.stringify("from-guest"), 0) !== 0) { + throw new Error(fs.lastError()); + } + const stat = JSON.parse(fs.stat("notes/hi.txt")); + if (stat.kind !== "file" || stat.size !== 10) throw new Error("bad stat"); + const read = JSON.parse(fs.read("notes/hi.txt", 0, 64)); + if (!read.eof) throw new Error("expected eof"); + const escape = JSON.parse(fs.read("../../etc/passwd", 0, 64)); + if (escape.error !== "invalid path") throw new Error("traversal not refused"); + globalThis.result = read.data["$b"]; + "#, + ) + .unwrap(); + let result: String = guest.with(|ctx| ctx.globals().get("result").unwrap()); + assert_eq!(BASE64.decode(result).unwrap(), b"from-guest"); + } +} diff --git a/framework/compiler/subpaths.ts b/framework/compiler/subpaths.ts index 117b0368..493d81dc 100644 --- a/framework/compiler/subpaths.ts +++ b/framework/compiler/subpaths.ts @@ -66,6 +66,7 @@ export const SUBPATHS: Record = { }, devtools: { file: "framework/src/devtools.ts" }, effects: { file: "framework/src/effects.ts", aliases: TWINS }, + fs: { file: "framework/src/fs-api.ts", aliases: TWINS }, gesture: { file: { solid: "framework/src/gesture.ts" } }, host: { file: "framework/src/host.ts" }, lifecycle: { diff --git a/framework/src/bytes.ts b/framework/src/bytes.ts new file mode 100644 index 00000000..a1dd1311 --- /dev/null +++ b/framework/src/bytes.ts @@ -0,0 +1,94 @@ +// Byte codecs shared by the data-module SDKs (db, fs). Internal — not a +// framework subpath. QuickJS has no btoa/Buffer/TextEncoder/TextDecoder, so +// the codecs are spelled out; every caller is a cold path (payloads cross +// the boundary far less often than draw ops). + +const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +export function bytesToBase64(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]; + const b = i + 1 < bytes.length ? bytes[i + 1] : 0; + const c = i + 2 < bytes.length ? bytes[i + 2] : 0; + out += B64[a >> 2] + B64[((a & 3) << 4) | (b >> 4)]; + out += i + 1 < bytes.length ? B64[((b & 15) << 2) | (c >> 6)] : "="; + out += i + 2 < bytes.length ? B64[c & 63] : "="; + } + return out; +} + +const B64_INV: Record = {}; +for (let i = 0; i < B64.length; i++) B64_INV[B64[i]] = i; + +export function base64ToBytes(s: string): Uint8Array { + let pad = 0; + while (s.endsWith("=")) { + pad++; + s = s.slice(0, -1); + } + const out = new Uint8Array(Math.floor((s.length * 3) / 4)); + let o = 0; + for (let i = 0; i < s.length; i += 4) { + const n = + (B64_INV[s[i]] << 18) | + ((B64_INV[s[i + 1]] ?? 0) << 12) | + ((B64_INV[s[i + 2]] ?? 0) << 6) | + (B64_INV[s[i + 3]] ?? 0); + out[o++] = n >> 16; + if (o < out.length) out[o++] = (n >> 8) & 0xff; + if (o < out.length) out[o++] = n & 0xff; + } + return out; +} + +/** UTF-8 decode, strict: malformed sequences throw (a file that fails + * .text() is a bytes file — read it with .bytes()). */ +export function utf8ToString(bytes: Uint8Array): string { + let out = ""; + let i = 0; + while (i < bytes.length) { + const a = bytes[i++]; + if (a < 0x80) { + out += String.fromCharCode(a); + continue; + } + let n: number; + let extra: number; + if ((a & 0xe0) === 0xc0) { + n = a & 0x1f; + extra = 1; + } else if ((a & 0xf0) === 0xe0) { + n = a & 0x0f; + extra = 2; + } else if ((a & 0xf8) === 0xf0) { + n = a & 0x07; + extra = 3; + } else { + throw new Error("invalid UTF-8"); + } + if (i + extra > bytes.length) throw new Error("invalid UTF-8"); + for (let k = 0; k < extra; k++) { + const b = bytes[i++]; + if ((b & 0xc0) !== 0x80) throw new Error("invalid UTF-8"); + n = (n << 6) | (b & 0x3f); + } + // Reject overlong encodings and surrogate-range codepoints. + if ( + n > 0x10ffff || + (n >= 0xd800 && n <= 0xdfff) || + (extra === 1 && n < 0x80) || + (extra === 2 && n < 0x800) || + (extra === 3 && n < 0x10000) + ) { + throw new Error("invalid UTF-8"); + } + if (n < 0x10000) { + out += String.fromCharCode(n); + } else { + n -= 0x10000; + out += String.fromCharCode(0xd800 + (n >> 10), 0xdc00 + (n & 0x3ff)); + } + } + return out; +} diff --git a/framework/src/db-api.ts b/framework/src/db-api.ts index 8e64d0f1..04a2bc92 100644 --- a/framework/src/db-api.ts +++ b/framework/src/db-api.ts @@ -22,6 +22,9 @@ // `data.sqlite` in pocket.json `requires` so admission catches it first. import { DB_BLOB_KEY, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; +// QuickJS has no btoa/Buffer; the codec lives in bytes.ts (cold path), +// shared with the fs SDK. +import { base64ToBytes, bytesToBase64 } from "./bytes.ts"; export { DB_MAX_RESULT_ROWS, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; @@ -51,46 +54,6 @@ export function dbHost(): DbOps | null { export type SqlValue = null | number | string | boolean | Uint8Array; export type SqlParams = readonly SqlValue[] | Readonly>; -const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -/** QuickJS has no btoa/Buffer; the codec is spelled out (cold path). */ -function bytesToBase64(bytes: Uint8Array): string { - let out = ""; - for (let i = 0; i < bytes.length; i += 3) { - const a = bytes[i]; - const b = i + 1 < bytes.length ? bytes[i + 1] : 0; - const c = i + 2 < bytes.length ? bytes[i + 2] : 0; - out += B64[a >> 2] + B64[((a & 3) << 4) | (b >> 4)]; - out += i + 1 < bytes.length ? B64[((b & 15) << 2) | (c >> 6)] : "="; - out += i + 2 < bytes.length ? B64[c & 63] : "="; - } - return out; -} - -const B64_INV: Record = {}; -for (let i = 0; i < B64.length; i++) B64_INV[B64[i]] = i; - -function base64ToBytes(s: string): Uint8Array { - let pad = 0; - while (s.endsWith("=")) { - pad++; - s = s.slice(0, -1); - } - const out = new Uint8Array(Math.floor((s.length * 3) / 4)); - let o = 0; - for (let i = 0; i < s.length; i += 4) { - const n = - (B64_INV[s[i]] << 18) | - ((B64_INV[s[i + 1]] ?? 0) << 12) | - ((B64_INV[s[i + 2]] ?? 0) << 6) | - (B64_INV[s[i + 3]] ?? 0); - out[o++] = n >> 16; - if (o < out.length) out[o++] = (n >> 8) & 0xff; - if (o < out.length) out[o++] = n & 0xff; - } - return out; -} - function encodeValue(v: SqlValue): unknown { if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: bytesToBase64(v) }; if (typeof v === "number" && !Number.isFinite(v)) { diff --git a/framework/src/fs-api.ts b/framework/src/fs-api.ts new file mode 100644 index 00000000..3b9a1730 --- /dev/null +++ b/framework/src/fs-api.ts @@ -0,0 +1,325 @@ +// Fs module SDK — the thin guest-side algebra over the `fs` spec +// (contracts/spec/fs.ts). Framework-agnostic (no solid-js, no JSX): the +// same file serves ./fs, ./vue-vapor/fs and ./octane/fs. +// +// The API is the Bun shape — `file(path)` returning a lazy handle with +// `.text()/.bytes()/.json()/.size/.exists()`, `write(path, data)` — plus +// the node:fs sync subset Bun implements (readFileSync, writeFileSync, +// appendFileSync, mkdirSync, readdirSync, rmSync, renameSync, statSync, +// existsSync) — so file code written against Bun runs against the mounted +// module with the async wrappers dropped. One deliberate deviation, from +// the module family's frame contract: everything is synchronous (every op +// completes inside the guest's per-tick turn), so `.text()` returns the +// string, not a Promise. Migration stays painless anyway: `await` unwraps +// a plain value, so Bun-idiomatic code — `await Bun.file(p).text()`, +// `await Bun.write(p, data)` — runs against this SDK unchanged. +// +// Paths are RELATIVE to the app's own data root — the host binds the root +// at mount; there is no way to spell another app's tree (or an absolute +// path) in this vocabulary. The SDK chunks payloads larger than +// FS_MAX_IO_BYTES, so file size is bounded by storage (and any host +// quota), not by the marshaling ceiling. +// +// Like db (and unlike audio), absence does NOT degrade to a no-op: file +// code that silently drops writes is a corruption bug, not a missing +// enhancement. Every entry point throws where `globalThis.fs` is +// unmounted — declare `data.fs` in pocket.json `requires` so admission +// catches it first. + +import { + FS_BLOB_KEY, + FS_MAX_IO_BYTES, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, +} from "../../contracts/spec/fs.ts"; +import { base64ToBytes, bytesToBase64, utf8ToString } from "./bytes.ts"; + +export { + FS_MAX_DEPTH, + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_MAX_PATH_BYTES, + fsValidPath, +} from "../../contracts/spec/fs.ts"; + +/** The mounted fs namespace — one method per spec op (FS_OP codes). */ +export interface FsOps { + read(path: string, offset: number, maxBytes: number): string; + write(path: string, data: string, mode: number): number; + remove(path: string, recursive: number): number; + list(path: string, offset: number): string; + stat(path: string): string; + mkdir(path: string): number; + rename(from: string, to: string): number; + usage(): string; + lastError(): string; +} + +/** The fs module namespace, or null where the host doesn't mount one. + * A live lookup (not cached): hosts install `globalThis.fs` before eval + * and reset it per app load, exactly like `globalThis.ui`. */ +export function fsHost(): FsOps | null { + const ns = (globalThis as { fs?: unknown }).fs; + if (!ns || typeof ns !== "object") return null; + return typeof (ns as FsOps).read === "function" ? (ns as FsOps) : null; +} + +function host(): FsOps { + const ops = fsHost(); + if (!ops) { + throw new Error("fs: globalThis.fs is not mounted — declare `data.fs` in pocket.json requires"); + } + return ops; +} + +function fail(ops: FsOps, op: string): never { + throw new Error(`fs: ${op}: ${ops.lastError()}`); +} + +// --------------------------------------------------------------------------- +// Payload encoding (the contracts/spec/fs.ts data contract) +// --------------------------------------------------------------------------- + +interface ReadResult { + data?: { [FS_BLOB_KEY]: string }; + size?: number; + eof?: boolean; + error?: string; +} + +/** Read the whole file as bytes, chunking past FS_MAX_IO_BYTES. */ +function readAll(ops: FsOps, path: string): Uint8Array { + const chunks: Uint8Array[] = []; + let offset = 0; + for (;;) { + const result = JSON.parse(ops.read(path, offset, FS_MAX_IO_BYTES)) as ReadResult; + if (result.error !== undefined) throw new Error(`fs: read ${path}: ${result.error}`); + const chunk = base64ToBytes(result.data![FS_BLOB_KEY]); + chunks.push(chunk); + offset += chunk.length; + if (result.eof) break; + } + if (chunks.length === 1) return chunks[0]; + const out = new Uint8Array(offset); + let o = 0; + for (const c of chunks) { + out.set(c, o); + o += c.length; + } + return out; +} + +/** Write `data` in <= FS_MAX_IO_BYTES payloads: one truncate, then appends. + * A string payload crosses as the JSON string itself (stored as UTF-8); + * bytes cross base64. Returns bytes written. */ +function writeAll(ops: FsOps, path: string, data: string | Uint8Array, mode: number): number { + if (typeof data === "string") { + // JS string length bounds UTF-8 length only within 3x; slice by + // codepoint-safe chunks conservatively sized so the encoded payload + // stays under the ceiling. + const step = Math.floor(FS_MAX_IO_BYTES / 3); + if (data.length <= step && mode === FS_WRITE_TRUNCATE) { + if (ops.write(path, JSON.stringify(data), mode) !== 0) fail(ops, `write ${path}`); + return utf8Length(data); + } + let m = mode; + let i = 0; + do { + let end = Math.min(i + step, data.length); + // Never split a surrogate pair across payloads. + if (end < data.length && isHighSurrogate(data.charCodeAt(end - 1))) end--; + if (ops.write(path, JSON.stringify(data.slice(i, end)), m) !== 0) { + fail(ops, `write ${path}`); + } + i = end; + m = FS_WRITE_APPEND; + } while (i < data.length); + return utf8Length(data); + } + let m = mode; + let i = 0; + do { + const chunk = data.subarray(i, Math.min(i + FS_MAX_IO_BYTES, data.length)); + const payload = JSON.stringify({ [FS_BLOB_KEY]: bytesToBase64(chunk) }); + if (ops.write(path, payload, m) !== 0) fail(ops, `write ${path}`); + i += chunk.length; + m = FS_WRITE_APPEND; + } while (i < data.length); + return data.length; +} + +function isHighSurrogate(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff; +} + +function utf8Length(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++) { + const c = s.codePointAt(i)!; + n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4; + if (c >= 0x10000) i++; + } + return n; +} + +// --------------------------------------------------------------------------- +// file / write (the Bun shape) +// --------------------------------------------------------------------------- + +interface StatResult { + kind?: "file" | "dir"; + size?: number; + error?: string; +} + +function statOf(ops: FsOps, path: string): StatResult { + return JSON.parse(ops.stat(path)) as StatResult; +} + +/** A lazy handle on one path — the Bun.file shape, synchronous. */ +export class PocketFile { + constructor(readonly path: string) {} + + /** File size in bytes; 0 when the file does not exist (Bun's behavior). */ + get size(): number { + const s = statOf(host(), this.path); + return s.kind === "file" ? s.size! : 0; + } + + exists(): boolean { + return statOf(host(), this.path).kind === "file"; + } + + bytes(): Uint8Array { + return readAll(host(), this.path); + } + + text(): string { + return utf8ToString(this.bytes()); + } + + json(): unknown { + return JSON.parse(this.text()); + } + + /** Delete the file (Bun.file(...).delete()). */ + delete(): void { + const ops = host(); + if (ops.remove(this.path, 0) !== 0) fail(ops, `remove ${this.path}`); + } +} + +/** `file(path)` — a lazy handle; nothing is read until a method call. */ +export function file(path: string): PocketFile { + return new PocketFile(path); +} + +/** `write(path, data)` — replace the file atomically, creating parent + * directories (Bun.write semantics). Returns bytes written. */ +export function write(path: string, data: string | Uint8Array): number { + return writeAll(host(), path, data, FS_WRITE_TRUNCATE); +} + +/** `usage()` — the app's storage footprint and budget (0 = unmetered). */ +export function usage(): { usedBytes: number; quotaBytes: number } { + return JSON.parse(host().usage()) as { usedBytes: number; quotaBytes: number }; +} + +// --------------------------------------------------------------------------- +// The node:fs sync subset (the spelling Bun also implements) +// --------------------------------------------------------------------------- + +export function readFileSync(path: string): Uint8Array; +export function readFileSync(path: string, encoding: "utf8" | "utf-8"): string; +export function readFileSync(path: string, encoding?: string): Uint8Array | string { + const bytes = readAll(host(), path); + return encoding === "utf8" || encoding === "utf-8" ? utf8ToString(bytes) : bytes; +} + +export function writeFileSync(path: string, data: string | Uint8Array): void { + writeAll(host(), path, data, FS_WRITE_TRUNCATE); +} + +export function appendFileSync(path: string, data: string | Uint8Array): void { + writeAll(host(), path, data, FS_WRITE_APPEND); +} + +/** Always recursive (every missing ancestor is created), idempotent. */ +export function mkdirSync(path: string): void { + const ops = host(); + if (ops.mkdir(path) !== 0) fail(ops, `mkdir ${path}`); +} + +export interface DirEntry { + name: string; + kind: "file" | "dir"; + size: number; + isFile(): boolean; + isDirectory(): boolean; +} + +interface ListResult { + entries?: { name: string; kind: "file" | "dir"; size: number }[]; + eof?: boolean; + error?: string; +} + +export function readdirSync(path: string): string[]; +export function readdirSync(path: string, options: { withFileTypes: true }): DirEntry[]; +export function readdirSync( + path: string, + options?: { withFileTypes?: boolean }, +): string[] | DirEntry[] { + const ops = host(); + const entries: DirEntry[] = []; + let offset = 0; + for (;;) { + const result = JSON.parse(ops.list(path, offset)) as ListResult; + if (result.error !== undefined) throw new Error(`fs: readdir ${path}: ${result.error}`); + for (const e of result.entries!) { + entries.push({ + ...e, + isFile: () => e.kind === "file", + isDirectory: () => e.kind === "dir", + }); + } + offset += result.entries!.length; + if (result.eof) break; + } + return options?.withFileTypes ? entries : entries.map((e) => e.name); +} + +/** `force` swallows "not found" (node semantics); `recursive` removes a + * directory tree. */ +export function rmSync(path: string, options?: { recursive?: boolean; force?: boolean }): void { + const ops = host(); + if (ops.remove(path, options?.recursive ? 1 : 0) !== 0) { + if (options?.force && ops.lastError() === "not found") return; + fail(ops, `rm ${path}`); + } +} + +export function renameSync(from: string, to: string): void { + const ops = host(); + if (ops.rename(from, to) !== 0) fail(ops, `rename ${from} -> ${to}`); +} + +export interface Stats { + size: number; + isFile(): boolean; + isDirectory(): boolean; +} + +export function statSync(path: string): Stats { + const s = statOf(host(), path); + if (s.error !== undefined) throw new Error(`fs: stat ${path}: ${s.error}`); + return { + size: s.size!, + isFile: () => s.kind === "file", + isDirectory: () => s.kind === "dir", + }; +} + +export function existsSync(path: string): boolean { + return statOf(host(), path).kind !== undefined; +} diff --git a/hosts/sim/fs.ts b/hosts/sim/fs.ts new file mode 100644 index 00000000..b4ab1d74 --- /dev/null +++ b/hosts/sim/fs.ts @@ -0,0 +1,253 @@ +// hosts/sim/fs.ts — the in-memory implementation of the fs module +// (contracts/spec/fs.ts) for the headless sim host. +// +// Storage policy is sim-shaped: the whole tree lives in memory (no disk, +// no cleanup) and persists for the life of the host object, so an app +// reload inside one scenario keeps its files, the way a device keeps its +// flash. The tree is case-sensitive — the deterministic host that catches +// a case-only collision before a case-folding device filesystem hides it. +// +// Inject via bootWorld's extraGlobals: { fs: host.ns }, the way a device +// host mounts the namespace beside `ui`. One host per guest = one app's +// data root, which is the isolation model: a second app gets a second +// host object, and neither vocabulary can name the other's tree. + +import { + FS_BLOB_KEY, + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, + fsValidPath, +} from "../../contracts/spec/fs.ts"; + +export interface SimFsHost { + /** The `globalThis.fs` namespace (one method per FS_OP). */ + ns: Record; + /** Every op call in order (for trace assertions). */ + log: string[]; + /** Drop the whole tree (end of scenario). */ + dispose(): void; +} + +/** Parent path of a valid path ("" = the root). */ +function parentOf(path: string): string { + const i = path.lastIndexOf("/"); + return i < 0 ? "" : path.slice(0, i); +} + +function decodePayload(data: string): Uint8Array | string { + const parsed = JSON.parse(data) as unknown; + if (typeof parsed === "string") return new Uint8Array(Buffer.from(parsed, "utf8")); + if (parsed !== null && typeof parsed === "object") { + const b64 = (parsed as Record)[FS_BLOB_KEY]; + if (typeof b64 === "string") return new Uint8Array(Buffer.from(b64, "base64")); + } + return "malformed payload: a JSON string or {\"$b\": base64}"; +} + +export function createSimFsHost(options?: { quotaBytes?: number }): SimFsHost { + const files = new Map(); + const dirs = new Set(); // the root "" is implicit + const log: string[] = []; + const quota = options?.quotaBytes ?? 0; + let lastError = ""; + + const ok = (value: T): T => { + lastError = ""; + return value; + }; + const err = (message: string): 1 => { + lastError = message; + return 1; + }; + const errLine = (message: string): string => { + lastError = message; + return JSON.stringify({ error: message }); + }; + + const isDir = (path: string): boolean => path === "" || dirs.has(path); + + /** Sorted child names of a directory — Unicode CODE POINT order (= UTF-8 + * byte order, the spec's order). JS default sort compares UTF-16 code + * units, which disagrees for astral-plane names, hence the comparator. */ + function childrenOf(path: string): string[] { + const prefix = path === "" ? "" : `${path}/`; + const names = new Set(); + for (const key of [...files.keys(), ...dirs]) { + if (!key.startsWith(prefix) || key === path) continue; + const rest = key.slice(prefix.length); + const slash = rest.indexOf("/"); + names.add(slash < 0 ? rest : rest.slice(0, slash)); + } + return [...names].sort((a, b) => { + const as = [...a]; + const bs = [...b]; + for (let i = 0; i < Math.min(as.length, bs.length); i++) { + const d = as[i].codePointAt(0)! - bs[i].codePointAt(0)!; + if (d !== 0) return d; + } + return as.length - bs.length; + }); + } + + /** Create every missing ancestor of `path`; error string if one is a file. */ + function ensureParents(path: string): string | null { + for (let p = parentOf(path); p !== ""; p = parentOf(p)) { + if (files.has(p)) return `not a directory: ${p}`; + dirs.add(p); + } + return null; + } + + function usedBytes(): number { + let n = 0; + for (const bytes of files.values()) n += bytes.length; + return n; + } + + const ns = { + read(path: string, offset: number, maxBytes: number): string { + log.push(`op read ${path} ${offset} ${maxBytes}`); + if (!fsValidPath(path)) return errLine("invalid path"); + if (maxBytes < 1 || maxBytes > FS_MAX_IO_BYTES) { + return errLine("read maxBytes out of range"); + } + if (offset < 0) return errLine("read offset out of range"); + const bytes = files.get(path); + if (!bytes) return errLine(isDir(path) ? "is a directory" : "not found"); + const chunk = bytes.subarray(offset, offset + maxBytes); + return ok( + JSON.stringify({ + data: { [FS_BLOB_KEY]: Buffer.from(chunk).toString("base64") }, + size: bytes.length, + eof: offset + chunk.length >= bytes.length, + }), + ); + }, + write(path: string, data: string, mode: number): number { + log.push(`op write ${path} ${mode}`); + if (!fsValidPath(path)) return err("invalid path"); + if (mode !== FS_WRITE_TRUNCATE && mode !== FS_WRITE_APPEND) { + return err("invalid write mode"); + } + const payload = decodePayload(data); + if (typeof payload === "string") return err(payload); + if (payload.length > FS_MAX_IO_BYTES) return err("write exceeds FS_MAX_IO_BYTES"); + if (isDir(path)) return err("is a directory"); + const parentProblem = ensureParents(path); + if (parentProblem) return err(parentProblem); + const existing = mode === FS_WRITE_APPEND ? files.get(path) : undefined; + const nextSize = (existing?.length ?? 0) + payload.length; + if (quota > 0 && usedBytes() - (files.get(path)?.length ?? 0) + nextSize > quota) { + return err("quota exceeded"); + } + if (existing) { + const joined = new Uint8Array(nextSize); + joined.set(existing, 0); + joined.set(payload, existing.length); + files.set(path, joined); + } else { + files.set(path, payload.slice()); + } + return ok(0); + }, + remove(path: string, recursive: number): number { + log.push(`op remove ${path} ${recursive}`); + if (!fsValidPath(path)) return err("invalid path"); + if (files.delete(path)) return ok(0); + if (!dirs.has(path)) return err("not found"); + if (childrenOf(path).length > 0 && recursive !== 1) return err("directory not empty"); + const prefix = `${path}/`; + for (const key of [...files.keys()]) if (key.startsWith(prefix)) files.delete(key); + for (const key of [...dirs]) if (key.startsWith(prefix)) dirs.delete(key); + dirs.delete(path); + return ok(0); + }, + list(path: string, offset: number): string { + log.push(`op list ${path} ${offset}`); + if (path !== "" && !fsValidPath(path)) return errLine("invalid path"); + if (files.has(path)) return errLine("not a directory"); + if (!isDir(path)) return errLine("not found"); + const names = childrenOf(path); + const page = names.slice(offset, offset + FS_MAX_DIR_ENTRIES); + return ok( + JSON.stringify({ + entries: page.map((name) => { + const full = path === "" ? name : `${path}/${name}`; + const bytes = files.get(full); + return bytes + ? { name, kind: "file", size: bytes.length } + : { name, kind: "dir", size: 0 }; + }), + eof: offset + page.length >= names.length, + }), + ); + }, + stat(path: string): string { + log.push(`op stat ${path}`); + if (path === "") return ok(JSON.stringify({ kind: "dir", size: 0 })); + if (!fsValidPath(path)) return errLine("invalid path"); + const bytes = files.get(path); + if (bytes) return ok(JSON.stringify({ kind: "file", size: bytes.length })); + if (dirs.has(path)) return ok(JSON.stringify({ kind: "dir", size: 0 })); + return errLine("not found"); + }, + mkdir(path: string): number { + log.push(`op mkdir ${path}`); + if (!fsValidPath(path)) return err("invalid path"); + if (files.has(path)) return err(`not a directory: ${path}`); + const parentProblem = ensureParents(path); + if (parentProblem) return err(parentProblem); + dirs.add(path); + return ok(0); + }, + rename(from: string, to: string): number { + log.push(`op rename ${from} ${to}`); + if (!fsValidPath(from) || !fsValidPath(to)) return err("invalid path"); + if (from === to) return ok(0); + if (!isDir(parentOf(to))) return err("not found"); + if (dirs.has(to)) return err("destination exists"); + const fromFile = files.get(from); + if (fromFile) { + if (files.has(to)) files.delete(to); + files.delete(from); + files.set(to, fromFile); + return ok(0); + } + if (!dirs.has(from)) return err("not found"); + if (files.has(to)) return err("destination exists"); + if (to.startsWith(`${from}/`)) return err("cannot rename into own subtree"); + const prefix = `${from}/`; + for (const [key, bytes] of [...files.entries()]) { + if (!key.startsWith(prefix)) continue; + files.delete(key); + files.set(`${to}/${key.slice(prefix.length)}`, bytes); + } + for (const key of [...dirs]) { + if (!key.startsWith(prefix)) continue; + dirs.delete(key); + dirs.add(`${to}/${key.slice(prefix.length)}`); + } + dirs.delete(from); + dirs.add(to); + return ok(0); + }, + usage(): string { + log.push("op usage"); + return ok(JSON.stringify({ usedBytes: usedBytes(), quotaBytes: quota })); + }, + lastError(): string { + return lastError; + }, + }; + + return { + ns, + log, + dispose(): void { + files.clear(); + dirs.clear(); + }, + }; +} diff --git a/hosts/sim/sim.ts b/hosts/sim/sim.ts index b986b8e5..f05e6e45 100644 --- a/hosts/sim/sim.ts +++ b/hosts/sim/sim.ts @@ -242,6 +242,7 @@ export async function bootWorld( g.frame = undefined; g.audio = undefined; // audio module namespace: absent unless extraGlobals mounts one g.db = undefined; // db module namespace: absent unless extraGlobals mounts one + g.fs = undefined; // fs module namespace: absent unless extraGlobals mounts one g.__pocketApp = app; g.__simHz = hz; g.__pocketEffectTrace = (e: EffectEvent) => effects.push(e); diff --git a/package.json b/package.json index ac98fd34..53900ec8 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "./components": "./framework/src/components.ts", "./devtools": "./framework/src/devtools.ts", "./effects": "./framework/src/effects.ts", + "./fs": "./framework/src/fs-api.ts", "./gesture": "./framework/src/gesture.ts", "./host": "./framework/src/host.ts", "./lifecycle": "./framework/src/lifecycle.ts", @@ -118,6 +119,7 @@ "./vue-vapor/db": "./framework/src/db-api.ts", "./vue-vapor/components": "./framework/src/components-vue-vapor.ts", "./vue-vapor/effects": "./framework/src/effects.ts", + "./vue-vapor/fs": "./framework/src/fs-api.ts", "./vue-vapor/lifecycle": "./framework/src/lifecycle-vue-vapor.ts", "./vue-vapor/input": "./framework/src/input-api.ts", "./vue-vapor/net": "./framework/src/net-api.ts", @@ -129,6 +131,7 @@ "./octane/db": "./framework/src/db-api.ts", "./octane/components": "./framework/src/components-octane.tsx", "./octane/effects": "./framework/src/effects.ts", + "./octane/fs": "./framework/src/fs-api.ts", "./octane/lifecycle": "./framework/src/lifecycle-octane.ts", "./octane/input": "./framework/src/input-api.ts", "./octane/net": "./framework/src/net-api.ts", diff --git a/tests/fs.test.ts b/tests/fs.test.ts new file mode 100644 index 00000000..76b620fc --- /dev/null +++ b/tests/fs.test.ts @@ -0,0 +1,270 @@ +// Fs module unit tests: the sim host against the pinned op contract, and +// the Bun-shaped SDK over it. Runs entirely in-process; no disk. + +import { afterEach, describe, expect, test } from "bun:test"; +import { + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, + fsValidPath, +} from "../contracts/spec/fs.ts"; +import { + appendFileSync, + existsSync, + file, + fsHost, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + statSync, + usage, + write, + writeFileSync, +} from "../framework/src/fs-api.ts"; +import { createSimFsHost, type SimFsHost } from "../hosts/sim/fs.ts"; + +const g = globalThis as { fs?: unknown }; +let host: SimFsHost | null = null; + +/** Mount a fresh sim host as globalThis.fs, the way bootWorld's + * extraGlobals does for a scenario. */ +function mount(options?: { quotaBytes?: number }): SimFsHost { + host = createSimFsHost(options); + g.fs = host.ns; + return host; +} + +afterEach(() => { + host?.dispose(); + host = null; + g.fs = undefined; +}); + +type Ns = { + read(path: string, offset: number, maxBytes: number): string; + write(path: string, data: string, mode: number): number; + remove(path: string, recursive: number): number; + list(path: string, offset: number): string; + stat(path: string): string; + mkdir(path: string): number; + rename(from: string, to: string): number; + usage(): string; + lastError(): string; +}; + +const text = (s: string) => JSON.stringify(s); + +// --- the namespace contract (ops, straight through) ------------------------- + +describe("sim host ops", () => { + test("the path grammar refuses traversal and escapes — and nothing else", () => { + const ns = mount().ns as Ns; + // The predicate itself is covered in the spec-constants block; here the + // point is that the HOST enforces it on every op. + for (const bad of ["", "/etc/passwd", "../up", "a/../b", "a//b", "a/", "a/.."]) { + expect(JSON.parse(ns.read(bad, 0, 16)).error).toBe("invalid path"); + expect(ns.write(bad, text("x"), FS_WRITE_TRUNCATE)).toBe(1); + } + // Universal names: an app calls its files whatever it wants. + for (const ok of ["notes/today.md", ".config", "笔记/今日笔记.md", "space in name.txt"]) { + expect(ns.write(ok, text("ok"), FS_WRITE_TRUNCATE)).toBe(0); + expect(JSON.parse(ns.stat(ok)).kind).toBe("file"); + } + }); + + test("write creates parents; truncate replaces; append appends", () => { + const ns = mount().ns as Ns; + expect(ns.write("a/b/c.txt", text("one"), FS_WRITE_TRUNCATE)).toBe(0); + expect(JSON.parse(ns.stat("a/b")).kind).toBe("dir"); + expect(ns.write("a/b/c.txt", text("two"), FS_WRITE_TRUNCATE)).toBe(0); + expect(ns.write("a/b/c.txt", text("+"), FS_WRITE_APPEND)).toBe(0); + const read = JSON.parse(ns.read("a/b/c.txt", 0, FS_MAX_IO_BYTES)); + expect(read.size).toBe(4); + expect(read.eof).toBe(true); + }); + + test("read pages with offset/eof and refuses out-of-range maxBytes", () => { + const ns = mount().ns as Ns; + ns.write("f.txt", text("abcdef"), FS_WRITE_TRUNCATE); + const first = JSON.parse(ns.read("f.txt", 0, 4)); + expect(first.eof).toBe(false); + const rest = JSON.parse(ns.read("f.txt", 4, 4)); + expect(rest.eof).toBe(true); + expect(JSON.parse(ns.read("f.txt", 0, 0)).error).toContain("maxBytes"); + expect(JSON.parse(ns.read("f.txt", 0, FS_MAX_IO_BYTES + 1)).error).toContain("maxBytes"); + }); + + test("a payload beyond FS_MAX_IO_BYTES fails loudly", () => { + const ns = mount().ns as Ns; + expect(ns.write("big.bin", text("x".repeat(FS_MAX_IO_BYTES + 1)), FS_WRITE_TRUNCATE)).toBe(1); + expect(ns.lastError()).toContain("FS_MAX_IO_BYTES"); + }); + + test("list is name-sorted, pages, and stats carry kind/size", () => { + const ns = mount().ns as Ns; + ns.write("d/b.txt", text("xx"), FS_WRITE_TRUNCATE); + ns.write("d/a.txt", text("x"), FS_WRITE_TRUNCATE); + ns.mkdir("d/sub"); + const listing = JSON.parse(ns.list("d", 0)); + expect(listing.entries).toEqual([ + { name: "a.txt", kind: "file", size: 1 }, + { name: "b.txt", kind: "file", size: 2 }, + { name: "sub", kind: "dir", size: 0 }, + ]); + expect(listing.eof).toBe(true); + + for (let i = 0; i < FS_MAX_DIR_ENTRIES + 2; i++) { + ns.write(`many/f${String(i).padStart(4, "0")}`, text("x"), FS_WRITE_TRUNCATE); + } + const page1 = JSON.parse(ns.list("many", 0)); + expect(page1.entries.length).toBe(FS_MAX_DIR_ENTRIES); + expect(page1.eof).toBe(false); + const page2 = JSON.parse(ns.list("many", FS_MAX_DIR_ENTRIES)); + expect(page2.entries.length).toBe(2); + expect(page2.eof).toBe(true); + }); + + test("stat('') is the root; a missing path is 'not found'", () => { + const ns = mount().ns as Ns; + expect(JSON.parse(ns.stat(""))).toEqual({ kind: "dir", size: 0 }); + expect(JSON.parse(ns.stat("ghost.txt")).error).toBe("not found"); + expect(JSON.parse(ns.list("", 0)).entries).toEqual([]); + }); + + test("remove: files, empty dirs, recursive trees; root refused", () => { + const ns = mount().ns as Ns; + ns.write("tree/deep/f.txt", text("x"), FS_WRITE_TRUNCATE); + expect(ns.remove("tree", 0)).toBe(1); + expect(ns.lastError()).toBe("directory not empty"); + expect(ns.remove("tree", 1)).toBe(0); + expect(JSON.parse(ns.stat("tree")).error).toBe("not found"); + expect(ns.remove("", 0)).toBe(1); + expect(ns.remove("ghost", 0)).toBe(1); + expect(ns.lastError()).toBe("not found"); + }); + + test("rename: atomic file replace, dir moves, guarded destinations", () => { + const ns = mount().ns as Ns; + ns.write("a.txt", text("A"), FS_WRITE_TRUNCATE); + ns.write("b.txt", text("B"), FS_WRITE_TRUNCATE); + expect(ns.rename("a.txt", "b.txt")).toBe(0); + expect(JSON.parse(ns.stat("a.txt")).error).toBe("not found"); + ns.mkdir("sub"); + expect(ns.rename("b.txt", "sub")).toBe(1); + expect(ns.lastError()).toBe("destination exists"); + expect(ns.rename("b.txt", "ghost/c.txt")).toBe(1); + ns.write("sub/deep/f.txt", text("x"), FS_WRITE_TRUNCATE); + expect(ns.rename("sub", "sub/inner")).toBe(1); + expect(ns.rename("sub", "moved")).toBe(0); + expect(JSON.parse(ns.stat("moved/deep/f.txt")).kind).toBe("file"); + }); + + test("quota: writes beyond the budget fail; usage() reports", () => { + const ns = mount({ quotaBytes: 10 }).ns as Ns; + expect(ns.write("a.txt", text("12345678"), FS_WRITE_TRUNCATE)).toBe(0); + expect(ns.write("b.txt", text("123"), FS_WRITE_TRUNCATE)).toBe(1); + expect(ns.lastError()).toBe("quota exceeded"); + expect(ns.write("a.txt", text("1"), FS_WRITE_TRUNCATE)).toBe(0); + expect(JSON.parse(ns.usage())).toEqual({ usedBytes: 1, quotaBytes: 10 }); + }); +}); + +// --- the SDK (the Bun shape over the mounted namespace) ---------------------- + +describe("fs SDK", () => { + test("throws where the module is unmounted", () => { + expect(fsHost()).toBeNull(); + expect(() => write("a.txt", "x")).toThrow("data.fs"); + expect(() => file("a.txt").text()).toThrow("data.fs"); + }); + + test("file()/write() round-trip text, bytes and json", () => { + mount(); + expect(write("notes/today.md", "# 今天 🚀")).toBe(Buffer.byteLength("# 今天 🚀")); + const f = file("notes/today.md"); + expect(f.exists()).toBe(true); + expect(f.size).toBe(Buffer.byteLength("# 今天 🚀")); + expect(f.text()).toBe("# 今天 🚀"); + + const bytes = new Uint8Array([0, 1, 2, 253, 254, 255]); + write("raw.bin", bytes); + expect(Array.from(file("raw.bin").bytes())).toEqual(Array.from(bytes)); + + write("config.json", JSON.stringify({ theme: "dark", volume: 7 })); + expect(file("config.json").json()).toEqual({ theme: "dark", volume: 7 }); + + file("raw.bin").delete(); + expect(file("raw.bin").exists()).toBe(false); + }); + + test("payloads larger than FS_MAX_IO_BYTES chunk transparently", () => { + mount(); + const big = new Uint8Array(FS_MAX_IO_BYTES * 2 + 123); + for (let i = 0; i < big.length; i++) big[i] = i % 251; + write("big.bin", big); + expect(file("big.bin").size).toBe(big.length); + const back = file("big.bin").bytes(); + expect(back.length).toBe(big.length); + expect(back[FS_MAX_IO_BYTES + 7]).toBe((FS_MAX_IO_BYTES + 7) % 251); + + const bigText = "样🚀x".repeat(40_000); // multi-byte, crosses chunk seams + write("big.txt", bigText); + expect(file("big.txt").text()).toBe(bigText); + }); + + test("the node:fs sync subset behaves like node", () => { + mount(); + mkdirSync("a/b"); + writeFileSync("a/b/f.txt", "one"); + appendFileSync("a/b/f.txt", "+two"); + expect(readFileSync("a/b/f.txt", "utf8")).toBe("one+two"); + expect(readFileSync("a/b/f.txt")).toBeInstanceOf(Uint8Array); + + expect(readdirSync("a")).toEqual(["b"]); + const entries = readdirSync("a/b", { withFileTypes: true }); + expect(entries[0].name).toBe("f.txt"); + expect(entries[0].isFile()).toBe(true); + expect(statSync("a/b/f.txt").size).toBe(7); + expect(statSync("a").isDirectory()).toBe(true); + expect(existsSync("a/b/f.txt")).toBe(true); + + renameSync("a/b/f.txt", "a/g.txt"); + expect(existsSync("a/b/f.txt")).toBe(false); + + expect(() => rmSync("ghost.txt")).toThrow("not found"); + rmSync("ghost.txt", { force: true }); // node semantics: force swallows + rmSync("a", { recursive: true }); + expect(existsSync("a")).toBe(false); + + expect(usage().usedBytes).toBe(0); + }); + + test("errors surface as thrown Errors with the op detail", () => { + mount(); + expect(() => readFileSync("missing.txt")).toThrow("not found"); + expect(() => readdirSync("missing")).toThrow("not found"); + expect(() => statSync("missing")).toThrow("not found"); + mkdirSync("d"); + expect(() => writeFileSync("d", "x")).toThrow("is a directory"); + }); +}); + +// --- spec sanity -------------------------------------------------------------- + +describe("spec constants", () => { + test("fsValidPath allows universal names and refuses only escapes", () => { + for (const good of ["a", "notes/today.md", ".config", "笔记/今天.md", "a b", "a\\b"]) { + expect(fsValidPath(good)).toBe(true); + } + for (const bad of ["", "/a", "a//b", "../x", "a/..", "a/.", "a/", "a\u0007b"]) { + expect(fsValidPath(bad)).toBe(false); + } + expect(fsValidPath(Array(9).fill("a").join("/"))).toBe(false); + expect(fsValidPath(Array(8).fill("a").join("/"))).toBe(true); + expect(fsValidPath("名".repeat(22))).toBe(false); // 66 UTF-8 bytes > segment cap + expect(fsValidPath("名".repeat(21))).toBe(true); + }); +}); From c24b05cd2509f61acc4387bc729ce4d2a72ea253 Mon Sep 17 00:00:00 2001 From: Jerry Yuan Date: Thu, 6 Aug 2026 22:41:39 +0800 Subject: [PATCH 2/3] =?UTF-8?q?test(esp32p4):=20data-smoke=20=E2=80=94=20t?= =?UTF-8?q?he=20data=20modules=20verified=20on=20real=20hardware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hosts/esp32p4 grows its second example, following the ppa-smoke pattern (one building block, one on-device verifier): pocket-fs and pocket-db driven directly (default-features = false — no QuickJS, the way a device host with its own guest wiring consumes the cores) against a LittleFS partition on the ESP32-P4. The smoke runs both contracts' hardware-facing edges — fs write/append/ chunked read/sorted list/rename/recursive-remove, universal names (dot-prefixed and CJK) on real LittleFS, and the traversal refusal; db DDL, a 288-row single-transaction insert, the ATTACH refusal, and the database as an ordinary file in the shared data root — and proves persistence rather than asserting it: an fs boot counter survives resets, and the sample count stays a multiple of 288 across power cycles, SQLite's transaction atomicity witnessed through the module (an interrupted run contributes exactly zero rows). Measured on a Waveshare ESP32-P4 rev 1.3 (UART transcript): DATA-SMOKE: fs ok in 453.626ms; boot 1; usedBytes 11 DATA-SMOKE: db ok in 557.143ms (288-row tx 338.076ms) DATA-SMOKE: PASS boot=1 ...and PASS boot=2 after another reset (576 rows verified). The example is self-contained: pinned nightly + build-std, its own partition table (an 8 MB LittleFS "workspace"), the SQLite build recipe from docs/DB.md as committed .cargo config, and cc/ar wrappers that find the esp-idf-sys-installed toolchain (IDF_TOOLS_PATH reuses an existing install). Co-Authored-By: Claude Fable 5 --- .../examples/data-smoke/.cargo/config.toml | 22 ++ hosts/esp32p4/examples/data-smoke/.gitignore | 4 + hosts/esp32p4/examples/data-smoke/Cargo.toml | 39 +++ hosts/esp32p4/examples/data-smoke/README.md | 42 +++ hosts/esp32p4/examples/data-smoke/build.rs | 4 + .../data-smoke/components_esp32p4.lock | 20 ++ .../examples/data-smoke/partitions.csv | 5 + .../examples/data-smoke/rust-toolchain.toml | 5 + .../examples/data-smoke/sdkconfig.defaults | 8 + .../examples/data-smoke/shim/sys/ioctl.h | 0 hosts/esp32p4/examples/data-smoke/src/main.rs | 267 ++++++++++++++++++ .../examples/data-smoke/tools/data-smoke-ar | 13 + .../examples/data-smoke/tools/data-smoke-cc | 17 ++ 13 files changed, 446 insertions(+) create mode 100644 hosts/esp32p4/examples/data-smoke/.cargo/config.toml create mode 100644 hosts/esp32p4/examples/data-smoke/.gitignore create mode 100644 hosts/esp32p4/examples/data-smoke/Cargo.toml create mode 100644 hosts/esp32p4/examples/data-smoke/README.md create mode 100644 hosts/esp32p4/examples/data-smoke/build.rs create mode 100644 hosts/esp32p4/examples/data-smoke/components_esp32p4.lock create mode 100644 hosts/esp32p4/examples/data-smoke/partitions.csv create mode 100644 hosts/esp32p4/examples/data-smoke/rust-toolchain.toml create mode 100644 hosts/esp32p4/examples/data-smoke/sdkconfig.defaults create mode 100644 hosts/esp32p4/examples/data-smoke/shim/sys/ioctl.h create mode 100644 hosts/esp32p4/examples/data-smoke/src/main.rs create mode 100755 hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar create mode 100755 hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc diff --git a/hosts/esp32p4/examples/data-smoke/.cargo/config.toml b/hosts/esp32p4/examples/data-smoke/.cargo/config.toml new file mode 100644 index 00000000..17f32957 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/.cargo/config.toml @@ -0,0 +1,22 @@ +[build] +target = "riscv32imafc-esp-espidf" + +[target.'cfg(target_os = "espidf")'] +linker = "ldproxy" +runner = "espflash flash --monitor" +rustflags = ["--cfg", "espidf_time64"] + +[unstable] +build-std = ["std", "panic_abort"] + +[env] +MCU = "esp32p4" +ESP_IDF_VERSION = "v5.5.3" +# Tools install into this example's .embuild by default; set IDF_TOOLS_PATH +# to reuse an existing espressif tools dir (the cc/ar wrappers honor it). +ESP_IDF_TOOLS_INSTALL_DIR = "workspace" +CC_riscv32imafc_esp_espidf = { value = "tools/data-smoke-cc", relative = true } +AR_riscv32imafc_esp_espidf = { value = "tools/data-smoke-ar", relative = true } +CFLAGS_riscv32imafc_esp_espidf = "-mabi=ilp32f -march=rv32imafc_zicsr_zifencei_xesppie -fno-pic -fno-PIC -Wno-error=incompatible-pointer-types" +# The vendored sqlite3.c, tuned for the device (docs/DB.md "ESP32 / ESP-IDF"). +LIBSQLITE3_FLAGS = "-DSQLITE_TEMP_STORE=3 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -Dlstat=stat" diff --git a/hosts/esp32p4/examples/data-smoke/.gitignore b/hosts/esp32p4/examples/data-smoke/.gitignore new file mode 100644 index 00000000..dfaa254a --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/.gitignore @@ -0,0 +1,4 @@ +target/ +.embuild/ +sdkconfig +Cargo.lock diff --git a/hosts/esp32p4/examples/data-smoke/Cargo.toml b/hosts/esp32p4/examples/data-smoke/Cargo.toml new file mode 100644 index 00000000..51654e15 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/Cargo.toml @@ -0,0 +1,39 @@ +# On-device conformance smoke for the data modules (pocket-db, pocket-fs) +# over a LittleFS partition — the hardware half of the verification story +# (the contract half lives in tests/{db,fs}.test.ts and the crates' own +# tests). Build with the local toolchain this directory pins, flash, and +# watch UART for "DATA-SMOKE: PASS". Power-cycle and run again: the boot +# counter proves persistence through real power loss. +[package] +name = "data-smoke" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "data-smoke" +harness = false + +[workspace] + +[profile.release] +opt-level = "s" +lto = "thin" +codegen-units = 1 + +[dependencies] +anyhow = "1" +log = "0.4" +serde_json = "1" +esp-idf-svc = { version = "0.52.1", features = ["critical-section"] } +# default-features = false: the firmware brings no QuickJS here — the smoke +# drives the module cores directly, exactly like a device host that has its +# own guest wiring. +pocket-db = { path = "../../../../engine/crates/pocket-db", default-features = false } +pocket-fs = { path = "../../../../engine/crates/pocket-fs", default-features = false } + +[build-dependencies] +embuild = "0.33" + +[[package.metadata.esp-idf-sys.extra_components]] +remote_component = { name = "joltwallet/littlefs", version = "1.14.*" } diff --git a/hosts/esp32p4/examples/data-smoke/README.md b/hosts/esp32p4/examples/data-smoke/README.md new file mode 100644 index 00000000..780dd0f1 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/README.md @@ -0,0 +1,42 @@ +# data-smoke + +On-device conformance smoke for the data modules — `pocket-db` and +`pocket-fs` driven directly (no `mount` feature, the way a device host +with its own guest wiring consumes them) against a LittleFS partition on +an ESP32-P4. + +The contract semantics are verified host-side (`tests/{db,fs}.test.ts`, +the crates' unit tests). This binary asks only the questions hardware can +answer: does everything compile and link here, does LittleFS behave +(atomic rename, persistence), and what does it cost (heap, timings). + +## Run + +```sh +cargo build --release +espflash flash --port --partition-table partitions.csv --monitor \ + target/riscv32imafc-esp-espidf/release/data-smoke +``` + +The first build bootstraps ESP-IDF v5.5.3 into `.embuild/` (set +`IDF_TOOLS_PATH` to reuse an existing espressif tools directory — the +`tools/data-smoke-cc` wrapper honors it). The SQLite build recipe this +directory pins (`LIBSQLITE3_FLAGS`, the empty `sys/ioctl.h` shim) is +documented in docs/DB.md "ESP32 / ESP-IDF". + +Watch UART for: + +``` +DATA-SMOKE: fs ok in ...; boot N; usedBytes ... +DATA-SMOKE: db ok in ... (288-row tx ...) +DATA-SMOKE: PASS boot=N +``` + +A boot counter (an fs truncate-write) and a per-boot 288-row transaction +persist across runs; power-cycle the board and `boot` increments while the +row count is verified as `boots × 288` — flash persistence proven through +both modules, not just asserted. + +Measured on a Waveshare ESP32-P4 (rev 1.3, LittleFS on 8 MB partition): +fs contract pass ~0.5 s, 288-row insert transaction ~0.5–0.7 s, steady +heap delta ~1 KB across the whole run. diff --git a/hosts/esp32p4/examples/data-smoke/build.rs b/hosts/esp32p4/examples/data-smoke/build.rs new file mode 100644 index 00000000..1ef09435 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/build.rs @@ -0,0 +1,4 @@ +fn main() { + println!("cargo:rerun-if-changed=Cargo.toml"); + embuild::espidf::sysenv::output(); +} diff --git a/hosts/esp32p4/examples/data-smoke/components_esp32p4.lock b/hosts/esp32p4/examples/data-smoke/components_esp32p4.lock new file mode 100644 index 00000000..71aba26d --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/components_esp32p4.lock @@ -0,0 +1,20 @@ +dependencies: + idf: + source: + type: idf + version: 5.5.3 + joltwallet/littlefs: + component_hash: 362f1f5beb5087b0c60169aff82676d2d0ffc991ead975212b0cba95959181c5 + dependencies: + - name: idf + require: private + version: '>=4.3' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.14.8 +direct_dependencies: +- joltwallet/littlefs +manifest_hash: 63c4596bdefd7c81424d6f1f243201e4c3ff7245cb13401c292c520c662859c9 +target: esp32p4 +version: 2.0.0 diff --git a/hosts/esp32p4/examples/data-smoke/partitions.csv b/hosts/esp32p4/examples/data-smoke/partitions.csv new file mode 100644 index 00000000..f1ae316f --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 0xfa0000, +workspace, data, littlefs,0xfb0000, 0x800000, diff --git a/hosts/esp32p4/examples/data-smoke/rust-toolchain.toml b/hosts/esp32p4/examples/data-smoke/rust-toolchain.toml new file mode 100644 index 00000000..d772578a --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +# Pin the cross-compiled std ABI (nightly for -Zbuild-std). Newer nightlies +# broke on ESP-IDF in 2026-08 (std began requiring libc::AT_FDCWD). +channel = "nightly-2026-05-01" +components = ["rust-src"] diff --git a/hosts/esp32p4/examples/data-smoke/sdkconfig.defaults b/hosts/esp32p4/examples/data-smoke/sdkconfig.defaults new file mode 100644 index 00000000..ada13f43 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/sdkconfig.defaults @@ -0,0 +1,8 @@ +# Waveshare ESP32-P4 (ESP32-P4NRW32) baseline — the vendor's flash/PSRAM +# values, trimmed to what the data smoke needs (no display, no Wi-Fi). +CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +CONFIG_ESP32P4_REV_MIN_1=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHSIZE_32MB=y +CONFIG_ESPTOOLPY_FLASHSIZE="32MB" +CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384 diff --git a/hosts/esp32p4/examples/data-smoke/shim/sys/ioctl.h b/hosts/esp32p4/examples/data-smoke/shim/sys/ioctl.h new file mode 100644 index 00000000..e69de29b diff --git a/hosts/esp32p4/examples/data-smoke/src/main.rs b/hosts/esp32p4/examples/data-smoke/src/main.rs new file mode 100644 index 00000000..361180ff --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/src/main.rs @@ -0,0 +1,267 @@ +//! data-smoke — on-device conformance check for the data modules. +//! +//! Runs pocket-fs and pocket-db (no `mount` feature — the module cores +//! directly, the way a device host with its own guest wiring drives them) +//! against a LittleFS partition on real hardware, and reports over UART: +//! +//! DATA-SMOKE: PASS boot= +//! +//! A boot counter persists across runs, so flashing once and power-cycling +//! twice proves both modules keep data through real power loss. The +//! contract semantics themselves are verified host-side (tests/*.test.ts, +//! the crates' unit tests); this binary only asks the questions hardware +//! can answer: does it compile here, does LittleFS behave, what does it +//! cost. + +use std::time::Instant; + +use esp_idf_svc::fs::littlefs::Littlefs; +use esp_idf_svc::io::vfs::MountedLittlefs; +use pocket_db::{DbModule, Storage as DbStorage}; +use pocket_fs::{FsModule, Storage as FsStorage}; +use serde_json::Value as Json; + +const WORKSPACE_ROOT: &str = "/workspace"; +const DATA_ROOT: &str = "/workspace/apps/smoke/data"; +const TMP_DIR: &str = "/workspace/apps/smoke/tmp"; + +fn main() { + esp_idf_svc::sys::link_patches(); + esp_idf_svc::log::EspLogger::initialize_default(); + match run() { + Ok(boot) => log::info!("DATA-SMOKE: PASS boot={boot}"), + Err(error) => log::error!("DATA-SMOKE: FAIL: {error:#}"), + } + loop { + std::thread::sleep(std::time::Duration::from_secs(10)); + log::info!("DATA-SMOKE: idle"); + } +} + +fn run() -> anyhow::Result { + let _mount = mount_workspace()?; + std::fs::create_dir_all(DATA_ROOT)?; + let heap_before = unsafe { esp_idf_svc::sys::esp_get_free_heap_size() }; + + let boot = fs_smoke()?; + db_smoke()?; + + let heap_after = unsafe { esp_idf_svc::sys::esp_get_free_heap_size() }; + log::info!( + "DATA-SMOKE: heap before {heap_before} after {heap_after} (delta {})", + heap_before as i64 - heap_after as i64 + ); + Ok(boot) +} + +fn expect(condition: bool, what: &str) -> anyhow::Result<()> { + anyhow::ensure!(condition, "expectation failed: {what}"); + Ok(()) +} + +// --- fs: the nine-op contract against real LittleFS ------------------------ + +fn fs_smoke() -> anyhow::Result { + let started = Instant::now(); + let mut fs = FsModule::new(FsStorage::Dir { + root: DATA_ROOT.into(), + tmp: TMP_DIR.into(), + }); + + // Boot counter: truncate-write on every boot; its value is the proof + // that atomic writes and LittleFS persistence survive power cycling. + let boot = match parse(&fs.read("boot.txt", 0, 64)) { + Ok(line) => { + let b64 = line["data"]["$b"].as_str().unwrap_or_default(); + String::from_utf8(base64_decode(b64))?.trim().parse::()? + 1 + } + Err(_) => 0, // first boot on a fresh partition + }; + let write = fs.write("boot.txt", &format!("{:?}", boot.to_string()), 0); + expect(write == 0, "boot counter write")?; + + // Text + append round-trip. + expect(fs.write("notes/hello.md", "\"# hi\"", 0) == 0, "write text")?; + expect(fs.write("notes/hello.md", "\" there\"", 1) == 0, "append text")?; + let read = parse(&fs.read("notes/hello.md", 0, 64))?; + expect(read["size"].as_i64() == Some(10), "size after append")?; + expect(read["eof"].as_bool() == Some(true), "eof")?; + + // Bytes round-trip via the {"$b": base64} spelling. + expect( + fs.write("raw.bin", r#"{"$b":"AAEC/w=="}"#, 0) == 0, + "write bytes", + )?; + let stat = parse(&fs.stat("raw.bin"))?; + expect(stat["size"].as_i64() == Some(4), "bytes size")?; + + // list is name-sorted; mkdir/rename/remove behave. (Listing a fresh + // subdirectory, not the root — the root also holds the db module's + // ordinary files, main.sqlite and a transient journal.) + expect(fs.mkdir("assets/img") == 0, "mkdir -p")?; + expect(fs.rename("raw.bin", "assets/raw.bin") == 0, "rename")?; + let listing = parse(&fs.list("assets", 0))?; + let names: Vec<&str> = listing["entries"] + .as_array() + .map(|entries| entries.iter().filter_map(|e| e["name"].as_str()).collect()) + .unwrap_or_default(); + anyhow::ensure!(names == ["img", "raw.bin"], "listing sorted: got {names:?}"); + expect(fs.remove("assets", 0) == 1, "non-recursive remove of full dir refused")?; + expect(fs.remove("assets", 1) == 0, "recursive remove")?; + + // The sandbox refusal holds on-device exactly as in the goldens, and + // universal names (dot-prefixed, CJK) round-trip on real LittleFS. + expect( + parse(&fs.read("../../etc/passwd", 0, 16)).is_err(), + "traversal refused", + )?; + expect(fs.write(".config", "\"k=v\"", 0) == 0, "dot name allowed")?; + expect(fs.write("笔记/今天.md", "\"你好\"", 0) == 0, "CJK name allowed")?; + expect(fs.remove("笔记", 1) == 0 && fs.remove(".config", 0) == 0, "cleanup")?; + + let usage = parse(&fs.usage())?; + log::info!( + "DATA-SMOKE: fs ok in {:?}; boot {boot}; usedBytes {}", + started.elapsed(), + usage["usedBytes"] + ); + Ok(boot) +} + +// --- db: SQLite through the module core over the same data root ------------ + +fn db_smoke() -> anyhow::Result<()> { + let started = Instant::now(); + let mut db = DbModule::new(DbStorage::Dir(DATA_ROOT.into())); + let handle = db.open("main"); + anyhow::ensure!(handle > 0, "db open failed"); + + expect( + db.exec( + handle, + "CREATE TABLE IF NOT EXISTS samples ( + captured_at INTEGER PRIMARY KEY, + total_cents INTEGER NOT NULL + );", + ) == 0, + "ddl", + )?; + + // Prior completed runs' rows must still be there, and ONLY whole + // transactions: a run interrupted mid-transaction (reset, power loss) + // contributes exactly zero rows. The %288 invariant is SQLite's + // atomicity witnessed across power cycles, through the module. + let prior = parse(&db.query(handle, "SELECT COUNT(*) FROM samples", "[]"))?; + let prior_rows = prior["rows"][0][0].as_i64().unwrap_or(-1); + expect(prior_rows >= 0 && prior_rows % 288 == 0, "whole transactions only")?; + + // One day of 5-minute samples in one transaction — the flash-wear shape. + let tx_started = Instant::now(); + expect(db.exec(handle, "BEGIN") == 0, "begin")?; + for i in 0..288i64 { + let at = (prior_rows + i) * 300; + let cents = 1_500_000 + (i % 97) * 137; + let line = db.query( + handle, + "INSERT INTO samples (captured_at, total_cents) VALUES (?, ?)", + &format!("[{at}, {cents}]"), + ); + parse(&line)?; + } + expect(db.exec(handle, "COMMIT") == 0, "commit")?; + let tx_elapsed = tx_started.elapsed(); + + let agg = parse(&db.query(handle, "SELECT COUNT(*) FROM samples", "[]"))?; + expect( + agg["rows"][0][0].as_i64() == Some(prior_rows + 288), + "aggregate row count", + )?; + + // The ATTACH refusal holds on-device; the database is an ordinary + // file in the app's data root. + expect( + db.exec(handle, "ATTACH DATABASE '/workspace/x' AS other") == 1, + "attach refused", + )?; + expect( + std::path::Path::new(DATA_ROOT).join("main.sqlite").is_file(), + "db is an ordinary file in the data root", + )?; + + log::info!( + "DATA-SMOKE: db ok in {:?} (288-row tx {tx_elapsed:?})", + started.elapsed() + ); + Ok(()) +} + +// --- small helpers ---------------------------------------------------------- + +/// Parse one op result line; an {"error": ...} shape becomes an Err. +fn parse(line: &str) -> anyhow::Result { + let value: Json = serde_json::from_str(line)?; + match value.get("error").and_then(Json::as_str) { + Some(error) => anyhow::bail!("op error: {error}"), + None => Ok(value), + } +} + +/// Minimal base64 decode (standard alphabet, padded) — enough for the boot +/// counter without pulling a crate into the example. +fn base64_decode(s: &str) -> Vec { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let value = |c: u8| ALPHABET.iter().position(|&a| a == c).unwrap_or(0) as u32; + let s = s.trim_end_matches('=').as_bytes(); + let mut out = Vec::with_capacity(s.len() * 3 / 4); + for chunk in s.chunks(4) { + let mut n = 0u32; + for (i, &c) in chunk.iter().enumerate() { + n |= value(c) << (18 - 6 * i); + } + for i in 0..chunk.len().saturating_sub(1) { + out.push((n >> (16 - 8 * i)) as u8); + } + } + out +} + +// --- LittleFS mount (the pocket-pi firmware's semantics: format only a +// blank partition, never a corrupted one) ----------------------------------- + +type WorkspaceMount = MountedLittlefs>; + +fn mount_workspace() -> anyhow::Result { + let fs = unsafe { Littlefs::<()>::new_partition("workspace")? }; + match MountedLittlefs::mount(fs, WORKSPACE_ROOT) { + Ok(mounted) => Ok(mounted), + Err(_mount_error) if partition_is_blank()? => { + let mut fs = unsafe { Littlefs::<()>::new_partition("workspace")? }; + fs.format()?; + MountedLittlefs::mount(fs, WORKSPACE_ROOT).map_err(Into::into) + } + Err(mount_error) => Err(anyhow::anyhow!( + "LittleFS workspace mount failed; preserving non-blank partition: {mount_error}" + )), + } +} + +fn partition_is_blank() -> anyhow::Result { + let partition = unsafe { + esp_idf_svc::sys::esp_partition_find_first( + esp_idf_svc::sys::esp_partition_type_t_ESP_PARTITION_TYPE_DATA, + esp_idf_svc::sys::esp_partition_subtype_t_ESP_PARTITION_SUBTYPE_DATA_LITTLEFS, + c"workspace".as_ptr(), + ) + }; + if partition.is_null() { + anyhow::bail!("LittleFS workspace partition is missing"); + } + let mut prefix = [0u8; 4096]; + let status = unsafe { + esp_idf_svc::sys::esp_partition_read(partition, 0, prefix.as_mut_ptr().cast(), prefix.len()) + }; + if status != esp_idf_svc::sys::ESP_OK { + anyhow::bail!("read LittleFS workspace partition: ESP error {status}"); + } + Ok(prefix.iter().all(|byte| *byte == 0xff)) +} diff --git a/hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar new file mode 100755 index 00000000..d552c91f --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +example_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +tools_root=${IDF_TOOLS_PATH:-$example_root/.embuild/espressif} +for ar in \ + "$tools_root"/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-ar +do + if [ -x "$ar" ]; then + exec "$ar" "$@" + fi +done +echo "ESP32 ar is missing; build once so esp-idf-sys installs the toolchain (or set IDF_TOOLS_PATH)" >&2 +exit 1 diff --git a/hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc new file mode 100755 index 00000000..49c466ef --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc @@ -0,0 +1,17 @@ +#!/bin/sh +# ESP32 gcc wrapper for the C sources this example vendors (SQLite via +# libsqlite3-sys). Finds the toolchain esp-idf-sys installed — locally in +# .embuild by default, or wherever IDF_TOOLS_PATH points — and injects the +# empty sys/ioctl.h shim (newlib has no such header; SQLite includes it). +set -eu +example_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +tools_root=${IDF_TOOLS_PATH:-$example_root/.embuild/espressif} +for compiler in \ + "$tools_root"/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-gcc +do + if [ -x "$compiler" ]; then + exec "$compiler" -isystem "$example_root/shim" "$@" + fi +done +echo "ESP32 gcc is missing; build once so esp-idf-sys installs the toolchain (or set IDF_TOOLS_PATH)" >&2 +exit 1 From 01bfd0ebbd1dbcce4f0e03c39933bb09cfbd70b1 Mon Sep 17 00:00:00 2001 From: "Yifeng \"Evan\" Wang" <7312949+doodlewind@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:01:15 +0800 Subject: [PATCH 3/3] fix(fs): wire the fs tests into the gate and enforce the well-formed-Unicode claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on #238: - tests/fs.test.ts was never added to the tools/test.ts unit stage, so `bun run test` ran 316 tests, not the 331+ the PR body claims — the suite list is hand-maintained and the PR missed its own entry. Wired in; the stage now runs 331. - The path grammar documents "any well-formed Unicode" but fsValidSegment accepted unpaired surrogates, which have no UTF-8 spelling: a JS host stores one byte-exactly while the QuickJS-to- native bridge mangles it into a DIFFERENT name — silent identity divergence between sim and a native core. The shared predicate now refuses them (pinned in tests); the spec also documents that a text PAYLOAD with a lone surrogate is host-dependent, so arbitrary bytes belong in the {"$b"} spelling. - sim list() passed a negative offset straight to Array.slice, which wraps to slice-from-the-end where the reference core clamps to 0 — clamped to match (cross-host parity probe now byte-agrees on the whole op matrix modulo JSON key order). - docs/FS.md now states the chunking caveat: the op is the atomic unit, so an SDK write above FS_MAX_IO_BYTES crosses as truncate + appends and power loss between chunks can keep only the leading chunks; whole-file atomicity above 64 KiB is write-sibling + rename. - site/content/docs/concepts.md's module diagram enumerated ui/audio/ strike; db and fs join it (both PRs describe themselves as the 4th and 5th modules but neither updated the enumerating docs page). - bytes.ts: dropped a dead `pad` counter carried over from db-api. cargo test -p pocket-fs 12/12, clippy clean, cargo check --workspace clean, tests/fs.test.ts + db.test.ts 33/33, bunx tsc --noEmit clean, unit stage 330/331 (the 1 fail is the pre-existing Gatekeeper first- launch stall in symbian-runtime.test.ts, present on main). Co-Authored-By: Claude Fable 5 --- contracts/spec/fs.ts | 27 ++++++++++++++++++++++++++- docs/FS.md | 6 ++++++ framework/src/bytes.ts | 6 +----- hosts/sim/fs.ts | 3 +++ site/content/docs/concepts.md | 15 ++++++++------- tests/fs.test.ts | 7 +++++++ tools/test.ts | 1 + 7 files changed, 52 insertions(+), 13 deletions(-) diff --git a/contracts/spec/fs.ts b/contracts/spec/fs.ts index f570d23b..df4bc896 100644 --- a/contracts/spec/fs.ts +++ b/contracts/spec/fs.ts @@ -136,6 +136,12 @@ export const FS_WRITE_APPEND = 1; // write() accepts either; read() always returns bytes — the file does not // remember which spelling wrote it, and the SDK's .text() decodes UTF-8 // guest-side (QuickJS has no TextDecoder; the SDK carries the codec). +// +// A text payload must be well-formed Unicode, like a path segment: an +// unpaired surrogate has no UTF-8 spelling, so what happens to one is +// host-dependent (a JS host lossily encodes U+FFFD where a JSON-parsing +// native core fails the op). Arbitrary byte data belongs in the bytes +// spelling, never in a string. /** Marker key for a bytes payload (same spelling as db's DB_BLOB_KEY). */ export const FS_BLOB_KEY = "$b"; @@ -152,7 +158,11 @@ export const FS_BLOB_KEY = "$b"; // "/" in a name unrepresentable — it IS the separator, on every // filesystem on earth; // control chars C0 (U+0000..U+001F) and DEL (U+007F); -// oversize a segment > FS_MAX_SEGMENT_BYTES of UTF-8. +// oversize a segment > FS_MAX_SEGMENT_BYTES of UTF-8; +// lone surrogates ill-formed Unicode has no UTF-8 spelling — a JS host +// could store one byte-exactly while the QuickJS-to- +// native bridge mangles it into a DIFFERENT name, so +// the shared predicate refuses it on every host. // // No name is reserved to the host. "" names the root and is valid only // where an op says so (list, stat). Total path <= FS_MAX_PATH_BYTES of @@ -185,9 +195,24 @@ function utf8Bytes(s: string): number { return n; } +/** True when `s` is well-formed Unicode (every surrogate is paired). */ +function wellFormed(s: string): boolean { + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 0xdc00 && c <= 0xdfff) return false; // low with no high before it + if (c >= 0xd800 && c <= 0xdbff) { + const next = s.charCodeAt(i + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return false; + i++; + } + } + return true; +} + /** True when `segment` is one valid path segment under the grammar above. */ export function fsValidSegment(segment: string): boolean { if (segment.length === 0 || segment === "." || segment === "..") return false; + if (!wellFormed(segment)) return false; // eslint-disable-next-line no-control-regex if (/[\u0000-\u001f\u007f]/.test(segment)) return false; return utf8Bytes(segment) <= FS_MAX_SEGMENT_BYTES; diff --git a/docs/FS.md b/docs/FS.md index b9599faf..df162e77 100644 --- a/docs/FS.md +++ b/docs/FS.md @@ -74,6 +74,12 @@ directory and clears it on construction, so a crash orphan cannot outlive the next boot). Append is not atomic. LittleFS's rename is atomic, so device hosts inherit the contract by the same moves. +**The op is the atomic unit.** A file larger than `FS_MAX_IO_BYTES` +crosses as one truncate plus appends (the SDK's chunking), so power loss +between chunks can leave the leading chunks only. An app that needs +whole-file atomicity above 64 KiB writes to a sibling name and +`rename`s over the target — the same move the module itself makes. + **Ceilings.** `FS_MAX_IO_BYTES` (64 KiB) per read/write payload — the SDK chunks larger files, so the ceiling bounds marshaling, not file size. `FS_MAX_DIR_ENTRIES` (256) per `list()` call, paged via offset + eof — a diff --git a/framework/src/bytes.ts b/framework/src/bytes.ts index a1dd1311..681d64d7 100644 --- a/framework/src/bytes.ts +++ b/framework/src/bytes.ts @@ -22,11 +22,7 @@ const B64_INV: Record = {}; for (let i = 0; i < B64.length; i++) B64_INV[B64[i]] = i; export function base64ToBytes(s: string): Uint8Array { - let pad = 0; - while (s.endsWith("=")) { - pad++; - s = s.slice(0, -1); - } + while (s.endsWith("=")) s = s.slice(0, -1); const out = new Uint8Array(Math.floor((s.length * 3) / 4)); let o = 0; for (let i = 0; i < s.length; i += 4) { diff --git a/hosts/sim/fs.ts b/hosts/sim/fs.ts index b4ab1d74..d60da0ef 100644 --- a/hosts/sim/fs.ts +++ b/hosts/sim/fs.ts @@ -170,6 +170,9 @@ export function createSimFsHost(options?: { quotaBytes?: number }): SimFsHost { if (files.has(path)) return errLine("not a directory"); if (!isDir(path)) return errLine("not found"); const names = childrenOf(path); + // Clamp like the reference core: a negative offset must not wrap to + // slice-from-the-end. + offset = Math.max(0, offset); const page = names.slice(offset, offset + FS_MAX_DIR_ENTRIES); return ok( JSON.stringify({ diff --git a/site/content/docs/concepts.md b/site/content/docs/concepts.md index d88ff391..f5525799 100644 --- a/site/content/docs/concepts.md +++ b/site/content/docs/concepts.md @@ -16,8 +16,8 @@ Runtime = Host + mounted Modules + Guest ┌────────────────────────── Runtime ──────────────────────────┐ │ Guest product code (QuickJS bundle / wasm host eval) │ │ ───────── one namespace per mounted module ───────────── │ -│ Modules ui audio net strike │ -│ core+spec core+spec core+spec core+spec │ +│ Modules ui · audio · db · fs · net · strike │ +│ core+spec, one per module │ │ Substrate pocket3d · platform drivers (no guest API) │ │ Host PSP EBOOT · Vita · browser · headless sim │ └──────────────────────────────────────────────────────────────┘ @@ -45,11 +45,12 @@ Core native side: owns the domain's state and clock The **core** owns the domain's state and its clock; per-entity, per-frame work happens only there, and the core never calls into the guest. The **SDK** is ordinary guest code shaped for its domain — JSX components for -`ui`, `decodeWav` and a `WavPlayer` for `audio`, `fetch` and buffered -responses for `net`, a mod API for OpenStrike's `strike`. The two sides can -be replaced independently because the **spec** between them does not move: -swap Solid for Vue Vapor, or rewrite the layout engine, and the other side -cannot tell. +`ui`, `decodeWav` and a `WavPlayer` for `audio`, a `Database` with prepared +statements for `db`, `file()` and the node:fs sync subset for `fs`, `fetch` +and buffered responses for `net`, a mod API for OpenStrike's `strike`. The +two sides can be replaced independently because the **spec** between them +does not move: swap Solid for Vue Vapor, or rewrite the layout engine, and +the other side cannot tell. `ui` (pocketjs-core + the `ui.*` ops + the JSX SDK) was the first module. `strike` was the second. `audio` — credit-based PCM streaming — is the diff --git a/tests/fs.test.ts b/tests/fs.test.ts index 76b620fc..257f460f 100644 --- a/tests/fs.test.ts +++ b/tests/fs.test.ts @@ -266,5 +266,12 @@ describe("spec constants", () => { expect(fsValidPath(Array(8).fill("a").join("/"))).toBe(true); expect(fsValidPath("名".repeat(22))).toBe(false); // 66 UTF-8 bytes > segment cap expect(fsValidPath("名".repeat(21))).toBe(true); + // Ill-formed Unicode has no UTF-8 spelling: a lone surrogate would be + // byte-exact on a JS host but mangled by the QuickJS-to-native bridge, + // so the shared predicate refuses it; the paired form stays valid. + expect(fsValidPath("a\uD800b")).toBe(false); + expect(fsValidPath("a\uDC00b")).toBe(false); + expect(fsValidPath("tail\uDBFF")).toBe(false); + expect(fsValidPath("😀.txt")).toBe(true); // a real surrogate pair }); }); diff --git a/tools/test.ts b/tools/test.ts index 6594fa83..679b3f47 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -60,6 +60,7 @@ const SUITE: readonly Stage[] = [ "tests/osk-controller.test.ts", "tests/audio.test.ts", "tests/db.test.ts", + "tests/fs.test.ts", "tests/net.test.ts", "tests/net-web.test.js", "tests/vita-package.test.ts",