From 1e0f8b34bf15ae76264ab85d6181b41fb233e9f1 Mon Sep 17 00:00:00 2001 From: cccat6 <22387156+cccat6@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:56:21 +0800 Subject: [PATCH 1/5] fix: restore transactional provider-byte updates for POSIX pilot Restore the equal-length provider optimization from cccat6 PR #51 (7231881, cdcde35, 84a60d3) following the v0.4 transaction refactor in #71. Persist v3 mutation evidence before applying; recover same-inode writes through the existing journal, including bounded partial writes and append-preserving rollback. MOSS trial scope only. Windows retains the prior worker; .NET/Windows parity and formal PR review are deferred. References #51, #71, #69 and the active-writer inode review finding. --- docs/TRANSACTIONAL_IN_PLACE_PILOT.md | 77 ++++ .../contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md | 5 + src/backup.js | 33 +- src/service.js | 4 +- src/session-files.js | 428 ++++++++++++++++-- test/in-place-transaction.test.js | 279 ++++++++++++ test/sync-service.test.js | 13 +- 7 files changed, 784 insertions(+), 55 deletions(-) create mode 100644 docs/TRANSACTIONAL_IN_PLACE_PILOT.md create mode 100644 test/in-place-transaction.test.js diff --git a/docs/TRANSACTIONAL_IN_PLACE_PILOT.md b/docs/TRANSACTIONAL_IN_PLACE_PILOT.md new file mode 100644 index 0000000..af50c5c --- /dev/null +++ b/docs/TRANSACTIONAL_IN_PLACE_PILOT.md @@ -0,0 +1,77 @@ +# Transactional in-place provider writes: POSIX pilot + +This branch restores the optimization introduced by cccat6 in PR #51 +(`7231881`, `cdcde35`, `84a60d3`). PR #71's transaction refactor removed the +production in-place path; v0.5.0 still counts `APPLIED_IN_PLACE` but does not +produce it. This pilot does not change authentication or deploy to other hosts. + +## Implemented and tested + +- Node POSIX only; Windows retains the existing exclusive replacement worker. + The .NET implementation is unchanged. Neither is claimed to have parity yet. +- A non-empty, equal-length ASCII provider ID with one unescaped, unambiguous + `session_meta.payload.model_provider` field can be replaced in place. A + `turn_context.model` rewrite or an ineligible header uses the existing path. +- The plan captures device/inode, size, mtime, the original header, byte offset, + and both byte sequences. An immutable managed backup contains this descriptor + before the coordinator durably appends `applying` and starts the write. +- Apply revalidates the path, handle identity, snapshot, header and bytes. A + stale precondition is skipped, never used as a reason for a full rewrite. +- Short writes loop; write/fsync/read-back failure attempts byte restoration + through the same handle. There is no post-mutation fallback to rename. +- `applying` and `applied` targets recover from the immutable manifest. A torn + journal conservatively selects all manifest candidates. Recovery failure + leaves the existing `recoveryRequired` state and evidence intact. +- Both backup metadata and session manifest use version 3 when any entry is + in-place, so old readers reject before restoring config or SQLite. Version + 1/2 backups still use their old recovery semantics. Non-in-place backups stay + version 2. New backups must be restored using this pilot or a compatible tool. + +## Recovery and writer contract + +Recovery verifies device/inode **and** all surrounding original header bytes +and requires a file at least as large as the original. Identity alone is not a +permanent guarantee against inode reuse. It accepts original bytes, replacement +bytes, or a single contiguous run of replacement bytes among original bytes +at differing positions (`old* new* old*`). That last case covers a sequential +short write and an interrupted sequential rollback. Unknown bytes, disjoint +tears, replaced paths, or truncation fail closed. This is conditional evidence +under the append-only writer model, not proof against a third party rewriting +the header to an indistinguishable value. + +The supported writer leaves existing bytes alone and appends after the guarded +metadata operation. Pre-apply growth is skipped. Later appends remain visible +through the existing fd and survive rollback without truncation. Recovery +preserves the newer mtime of an already-appended file rather than applying the +scan-time mtime. No POSIX cooperative lock can force Codex to participate: this +does not promise atomic visibility to concurrent readers, or protection from +non-cooperating writers replacing/truncating/editing the header during the +small check/write window. Full replacement paths still have the active-fd risk +identified in PR #71; this pilot removes that risk only for eligible writes. + +## Validation and follow-up + +`test/in-place-transaction.test.js` covers eligibility, short/zero writes, +fsync failure, immediate restoration failure, immutable manifests, A/B failure, +crashes before `applied` and before commit, torn journals, idempotence, +conflicts, active fds and appends. A 32 MiB disposable fixture records only +8 rollout bytes written and verifies the unchanged tail hash and inode. +The full Node suite must pass before MOSS installation. No real history is +used by these tests, and no Codex/API calls are needed. + +Before a formal PR: add and actually run Windows worker fault/recovery tests; +decide .NET transition support with the maintainer; review recovery portability +and the append-only contract. No public PR/comment/release is authorized yet. + +References: [#51](https://github.com/Dailin521/codex-provider-sync/pull/51), +[#71](https://github.com/Dailin521/codex-provider-sync/pull/71), +[#69](https://github.com/Dailin521/codex-provider-sync/issues/69), +[active-fd finding](https://github.com/Dailin521/codex-provider-sync/pull/71#discussion_r3711178450), +[Codex #38149](https://github.com/openai/codex/issues/38149). + +For frequent switching, use equal-length ASCII provider IDs, preferably six +characters because `openai` has six (for example `provider_a` as `prov_a`). +Different lengths require whole-file rewriting; large histories can multiply +disk writes and elapsed time. The original user's rollout collection was +approximately 53 GiB. In-place updates do not convert `encrypted_content` or +make histories portable between providers/accounts. diff --git a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md index 319a3d3..33908f0 100644 --- a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md +++ b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md @@ -10,6 +10,11 @@ ## 1. 文档目的 +当前修复分支的 POSIX 原地写试运行扩展见 +[事务化原地更新试运行说明](../../TRANSACTIONAL_IN_PLACE_PILOT.md)：等长 provider +使用带 mutation descriptor 的 v3 备份；旧 v1/v2 备份继续可读，旧客户端必须拒绝 v3。 +此扩展不是 v0.5.0 已发布行为，也不声称 Windows/.NET 已具备对应原地恢复能力。 + 本文冻结 vNext 迁移开始时 Node 实现已经提供的外部行为。这里的“外部”不仅指 npm 最终用户，也包括当前 CLI 与 Local Web UI 对 Node service 的真实依赖。 本文解决三个问题： diff --git a/src/backup.js b/src/backup.js index fbbc20e..020f4ca 100644 --- a/src/backup.js +++ b/src/backup.js @@ -10,7 +10,7 @@ import { GLOBAL_STATE_BACKUP_FILE_BASENAME, GLOBAL_STATE_FILE_BASENAME } from "./constants.js"; -import { restoreSessionChanges } from "./session-files.js"; +import { restoreSessionChanges, validateProviderMutationDescriptor } from "./session-files.js"; import { assertSqliteWritable, createSqliteOnlineBackup, @@ -156,6 +156,12 @@ async function validateSessionManifestEntries(entries, codexHome) { throw new Error(`Backup session manifest contains a duplicate rollout target: ${entry.path}`); } seen.add(key); + if (entry.mutation) { + if (entry.modelOnlyChange || entry.originalTurnContextModels?.length) { + throw new Error(`Provider byte mutation cannot also restore model fields: ${entry.path}`); + } + validateProviderMutationDescriptor(entry.mutation, entry.path, entry.originalFirstLine, entry.originalSeparator); + } await assertNoLinkedPathSegments(lexicalRoot, target); const [canonicalRoot, canonicalTarget] = await Promise.all([ fs.realpath(lexicalRoot), @@ -289,8 +295,11 @@ export async function createBackup({ } const globalStateFiles = await backupGlobalStateFiles(codexHome, backupDir); + // Both versions must advance so old readers reject before restoring any + // config/SQLite data, even when rollout restore was disabled by the caller. + const backupVersion = sessionChanges.some((change) => change.inPlaceMutation) ? 3 : 2; const sessionManifest = { - version: 2, + version: backupVersion, namespace: BACKUP_NAMESPACE, codexHome, targetProvider, @@ -305,6 +314,7 @@ export async function createBackup({ originalFirstLine: change.originalFirstLine, originalSeparator: change.originalSeparator, originalMtimeMs: change.originalMtimeMs, + mutation: change.inPlaceMutation ?? null, // Per-line record of the original turn_context.model values // so a failed rollback can put the per-turn model back to // what it was before the sync. Without this, a restore @@ -323,7 +333,7 @@ export async function createBackup({ ); await writeMetadataWithInventory(backupDir, { - version: 2, + version: backupVersion, namespace: BACKUP_NAMESPACE, codexHome, sqliteHome: actualSqliteHome, @@ -344,11 +354,10 @@ export async function updateSessionBackupManifest(backupDir, sessionChanges, opt const sessionManifest = JSON.parse(await fs.readFile(manifestPath, "utf8")); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - // Promote older manifests to the v2 schema so restoreSessionChanges - // can rely on the per-line `originalTurnContextModels` field. - if (sessionManifest.version !== 2) { - sessionManifest.version = 2; - } + // Promote older manifests to the v3 schema. Existing entries remain valid; + // only newly collected equal-length provider changes carry a mutation + // descriptor for in-place recovery. + sessionManifest.version = Math.max(2, sessionManifest.version); const filesByPath = new Map( (sessionManifest.files ?? []).map((entry) => [pathComparisonKey(entry.path), entry]) @@ -383,7 +392,7 @@ export async function refreshBackupInventory(backupDir, options = {}) { const normalizedBackupDir = path.resolve(backupDir); const metadataPath = path.join(normalizedBackupDir, "metadata.json"); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - if (metadata?.namespace !== BACKUP_NAMESPACE || !new Set([1, 2]).has(metadata.version)) { + if (metadata?.namespace !== BACKUP_NAMESPACE || !new Set([1, 2, 3]).has(metadata.version)) { throw new Error(`Unsupported backup metadata in ${metadataPath}.`); } await writeMetadataWithInventory(normalizedBackupDir, metadata, options); @@ -484,7 +493,7 @@ async function selectSessionRestoreEntries(backupDir, sessionManifest) { async function readValidatedBackupMetadata(backupDir, codexHome) { const metadataPath = path.join(backupDir, "metadata.json"); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - if (metadata.namespace !== BACKUP_NAMESPACE || ![1, 2].includes(metadata.version)) { + if (metadata.namespace !== BACKUP_NAMESPACE || ![1, 2, 3].includes(metadata.version)) { throw new Error(`Unsupported backup metadata in ${metadataPath}.`); } if (typeof metadata.codexHome !== "string" || !storagePathsEqual(metadata.codexHome, codexHome)) { @@ -526,7 +535,7 @@ export async function getBackupRecoveryCoverage(backupDir, storageOrCodexHome) { const sessionManifestPath = path.join(backupDir, "session-meta-backup.json"); const sessionManifest = JSON.parse(await fs.readFile(sessionManifestPath, "utf8")); - if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2].includes(sessionManifest.version)) { + if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2, 3].includes(sessionManifest.version)) { throw new Error(`Unsupported session backup manifest in ${sessionManifestPath}.`); } if (typeof sessionManifest.codexHome !== "string" @@ -591,7 +600,7 @@ export async function restoreBackup(backupDir, storageOrCodexHome, options = {}) if (restoreSessions) { const sessionManifestPath = path.join(backupDir, "session-meta-backup.json"); sessionManifest = JSON.parse(await fs.readFile(sessionManifestPath, "utf8")); - if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2].includes(sessionManifest.version)) { + if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2, 3].includes(sessionManifest.version)) { throw new Error(`Unsupported session backup manifest in ${sessionManifestPath}.`); } if (typeof sessionManifest.codexHome !== "string" diff --git a/src/service.js b/src/service.js index d524be5..5f91339 100644 --- a/src/service.js +++ b/src/service.js @@ -759,7 +759,9 @@ async function runSyncCore({ restoreFailures.push(`transaction journal read: ${journalError.message}`); } const startedRolloutTargets = journalSnapshot - ? getStartedJournalTargets(journalSnapshot, "rollout") + ? (journalSnapshot.invalidTail || journalSnapshot.events.length === 0 + ? writableChanges.map((change) => change.path) + : getStartedJournalTargets(journalSnapshot, "rollout")) : (sessionRestoreNeeded ? appliedSessionChanges.map((change) => change.path) : writableChanges.map((change) => change.path)); diff --git a/src/session-files.js b/src/session-files.js index e147eb7..0483373 100644 --- a/src/session-files.js +++ b/src/session-files.js @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import readline from "node:readline"; import { promisify } from "node:util"; +import { isDeepStrictEqual } from "node:util"; import { SESSION_DIRS } from "./constants.js"; import { syncDirectory } from "./atomic-file.js"; @@ -41,16 +42,28 @@ function wrapRolloutFileBusyError(error, filePath, action) { async function getFileSnapshot(filePath) { const stat = await fsp.stat(filePath); + const identity = await fsp.stat(filePath, { bigint: true }); return { size: stat.size, mtimeMs: stat.mtimeMs, - mode: stat.mode + mode: stat.mode, + dev: String(identity.dev), + ino: String(identity.ino) }; } function snapshotMatches(change, snapshot) { - return change.originalSize === snapshot.size - && change.originalMtimeMs === snapshot.mtimeMs; + if (change.originalSize !== snapshot.size + || change.originalMtimeMs !== snapshot.mtimeMs) { + return false; + } + if (change.originalDev !== undefined && String(change.originalDev) !== String(snapshot.dev)) { + return false; + } + if (change.originalIno !== undefined && String(change.originalIno) !== String(snapshot.ino)) { + return false; + } + return true; } function emptyEncryptedContentCounts() { @@ -224,36 +237,40 @@ async function listJsonlFiles(rootDir) { return files; } +async function readFirstLineRecordFromHandle(handle) { + let position = 0; + let collected = Buffer.alloc(0); + while (true) { + const chunk = Buffer.alloc(64 * 1024); + const { bytesRead } = await handle.read(chunk, 0, chunk.length, position); + if (bytesRead === 0) { + break; + } + position += bytesRead; + collected = Buffer.concat([collected, chunk.subarray(0, bytesRead)]); + const newlineIndex = collected.indexOf(0x0a); + if (newlineIndex !== -1) { + const crlf = newlineIndex > 0 && collected[newlineIndex - 1] === 0x0d; + const lineBuffer = crlf ? collected.subarray(0, newlineIndex - 1) : collected.subarray(0, newlineIndex); + return { + firstLine: lineBuffer.toString("utf8"), + separator: crlf ? "\r\n" : "\n", + offset: newlineIndex + 1 + }; + } + } + return { + firstLine: collected.toString("utf8"), + separator: "", + offset: collected.length + }; +} + async function readFirstLineRecord(filePath) { let handle; try { handle = await fsp.open(filePath, "r"); - let position = 0; - let collected = Buffer.alloc(0); - while (true) { - const chunk = Buffer.alloc(64 * 1024); - const { bytesRead } = await handle.read(chunk, 0, chunk.length, position); - if (bytesRead === 0) { - break; - } - position += bytesRead; - collected = Buffer.concat([collected, chunk.subarray(0, bytesRead)]); - const newlineIndex = collected.indexOf(0x0a); - if (newlineIndex !== -1) { - const crlf = newlineIndex > 0 && collected[newlineIndex - 1] === 0x0d; - const lineBuffer = crlf ? collected.subarray(0, newlineIndex - 1) : collected.subarray(0, newlineIndex); - return { - firstLine: lineBuffer.toString("utf8"), - separator: crlf ? "\r\n" : "\n", - offset: newlineIndex + 1 - }; - } - } - return { - firstLine: collected.toString("utf8"), - separator: "", - offset: collected.length - }; + return await readFirstLineRecordFromHandle(handle); } catch (error) { throw wrapRolloutFileBusyError(error, filePath, "read"); } finally { @@ -480,6 +497,315 @@ async function restoreOriginalMtime(filePath, mtimeMs) { } } +const SAFE_IN_PLACE_PROVIDER_ID_RE = /^[A-Za-z0-9._-]+$/; +const PROVIDER_MUTATION_STRATEGY = "provider_bytes_in_place"; + +function getInPlaceProviderMutation(change) { + // POSIX pilot: Windows keeps its existing exclusive replacement worker + // until its file-ID and crash-recovery implementation is platform-tested. + if (process.platform === "win32" || !change + || change.modelRewriteRequired + || change.modelOnlyChange + || typeof change.originalFirstLine !== "string" + || typeof change.originalProvider !== "string" + || typeof change.updatedProvider !== "string" + || change.originalProvider === change.updatedProvider + || !SAFE_IN_PLACE_PROVIDER_ID_RE.test(change.originalProvider) + || !SAFE_IN_PLACE_PROVIDER_ID_RE.test(change.updatedProvider)) { + return null; + } + + const originalLiteral = JSON.stringify(change.originalProvider); + const replacementLiteral = JSON.stringify(change.updatedProvider); + const originalBytes = Buffer.from(originalLiteral, "utf8"); + const replacementBytes = Buffer.from(replacementLiteral, "utf8"); + if (originalBytes.length === 0 || originalBytes.length !== replacementBytes.length) { + return null; + } + + // Tokenize strings first: a regex on raw field text can match inside a JSON + // string or miss an escaped duplicate key. Only one literal provider key and + // one payload key anywhere in the header are eligible. + const keys = [...change.originalFirstLine.matchAll(/"(?:[^"\\]|\\.)*"/g)] + .filter((token) => /^\s*:/.test(change.originalFirstLine.slice(token.index + token[0].length))); + const named = (name) => keys.filter((key) => JSON.parse(key[0]) === name); + const fields = named("model_provider"); + if (fields.length !== 1 || named("payload").length !== 1 + || !fields[0][0].startsWith('"model_provider"')) { + return null; + } + const field = fields[0]; + const valueOffset = field.index + field[0].length + + change.originalFirstLine.slice(field.index + field[0].length).match(/^\s*:\s*/)[0].length; + if (!change.originalFirstLine.startsWith(originalLiteral, valueOffset)) { + return null; + } + const nextCharacter = change.originalFirstLine[valueOffset + originalLiteral.length]; + if (nextCharacter !== undefined && !/[\s,}]/.test(nextCharacter)) { + return null; + } + const original = parseSessionMetaRecord(change.originalFirstLine); + const replaced = change.originalFirstLine.slice(0, valueOffset) + replacementLiteral + + change.originalFirstLine.slice(valueOffset + originalLiteral.length); + if (original?.payload.model_provider !== change.originalProvider + || !isDeepStrictEqual(JSON.parse(replaced), JSON.parse(change.updatedFirstLine))) { + return null; + } + + return { + strategy: PROVIDER_MUTATION_STRATEGY, + byteOffset: Buffer.byteLength(change.originalFirstLine.slice(0, valueOffset), "utf8"), + originalBase64: originalBytes.toString("base64"), + replacementBase64: replacementBytes.toString("base64"), + originalSize: change.originalSize, + originalMtimeMs: change.originalMtimeMs, + originalDev: change.originalDev, + originalIno: change.originalIno + }; +} + +function decodeCanonicalBase64(value) { + if (typeof value !== "string" || value.length === 0 || value.length % 4 !== 0 + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) { + return null; + } + const decoded = Buffer.from(value, "base64"); + return decoded.toString("base64") === value ? decoded : null; +} + +export function validateProviderMutationDescriptor(mutation, targetPath, firstLine, separator = "") { + if (!mutation || mutation.strategy !== PROVIDER_MUTATION_STRATEGY + || !Number.isSafeInteger(mutation.byteOffset) || mutation.byteOffset < 0 + || !Number.isSafeInteger(mutation.originalSize) || mutation.originalSize < 0 + || !Number.isFinite(mutation.originalMtimeMs)) { + throw new Error(`Invalid provider in-place mutation descriptor for ${targetPath}.`); + } + const originalBytes = decodeCanonicalBase64(mutation.originalBase64); + const replacementBytes = decodeCanonicalBase64(mutation.replacementBase64); + if (!originalBytes || !replacementBytes || originalBytes.length === 0 + || originalBytes.length !== replacementBytes.length + || mutation.byteOffset + originalBytes.length > mutation.originalSize + || !/^"[A-Za-z0-9._-]+"$/.test(originalBytes.toString("utf8")) + || !/^"[A-Za-z0-9._-]+"$/.test(replacementBytes.toString("utf8"))) { + throw new Error(`Invalid provider in-place mutation bytes for ${targetPath}.`); + } + if (typeof firstLine !== "string" || !["", "\n", "\r\n"].includes(separator) + || typeof mutation.originalDev !== "string" || !/^\d+$/.test(mutation.originalDev) + || typeof mutation.originalIno !== "string" || !/^\d+$/.test(mutation.originalIno)) { + throw new Error(`Incomplete provider in-place recovery evidence for ${targetPath}.`); + } + const header = Buffer.from(firstLine, "utf8"); + const end = mutation.byteOffset + originalBytes.length; + if (!header.subarray(mutation.byteOffset, end).equals(originalBytes) + || header.length + Buffer.byteLength(separator) > mutation.originalSize) { + throw new Error(`Provider mutation does not match the original header: ${targetPath}`); + } + const replaced = Buffer.concat([header.subarray(0, mutation.byteOffset), replacementBytes, header.subarray(end)]).toString("utf8"); + const expected = getInPlaceProviderMutation({ + originalFirstLine: firstLine, + originalProvider: JSON.parse(originalBytes.toString()), + updatedProvider: JSON.parse(replacementBytes.toString()), + updatedFirstLine: replaced + }); + if (!expected || expected.byteOffset !== mutation.byteOffset) { + throw new Error(`Provider mutation targets an ambiguous JSON field: ${targetPath}`); + } + return { originalBytes, replacementBytes }; +} + +async function readBytesFully(handle, length, position) { + const buffer = Buffer.alloc(length); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await handle.read( + buffer, + offset, + buffer.length - offset, + position + offset + ); + if (bytesRead <= 0) { + return null; + } + offset += bytesRead; + } + return buffer; +} + +async function defaultInPlaceWrite(handle, buffer, offset, length, position) { + return handle.write(buffer, offset, length, position); +} + +async function writeBytesFully(handle, bytes, position, writeImpl) { + let offset = 0; + while (offset < bytes.length) { + const result = await writeImpl( + handle, + bytes, + offset, + bytes.length - offset, + position + offset + ); + const bytesWritten = typeof result === "number" ? result : result?.bytesWritten; + if (!Number.isInteger(bytesWritten) || bytesWritten <= 0 + || bytesWritten > bytes.length - offset) { + throw new Error("Provider in-place write made no valid forward progress."); + } + offset += bytesWritten; + } +} + +async function finishInPlaceWrite(handle, entry, expectedBytes, options = {}) { + const mutation = entry.mutation ?? entry.inPlaceMutation; + await (options.inPlaceSync ?? ((h) => h.sync()))(handle); + const actual = await readBytesFully(handle, expectedBytes.length, mutation.byteOffset); + if (!actual?.equals(expectedBytes)) { + throw new Error(`Provider in-place write verification failed: ${entry.path}`); + } + const stat = await handle.stat(); + await assertInPlaceIdentity(handle, entry.path, mutation); + // Never reset an append-only writer's new timestamp or truncate its tail. + if (stat.size === mutation.originalSize) { + await handle.utimes(stat.atime, mutation.originalMtimeMs / 1000); + await handle.sync(); + } else if (options.previousStat?.size === stat.size) { + await handle.utimes(stat.atime, options.previousStat.mtimeMs / 1000); + await handle.sync(); + } +} + +async function assertInPlaceIdentity(handle, filePath, mutation) { + const [opened, current] = await Promise.all([handle.stat({ bigint: true }), fsp.lstat(filePath, { bigint: true })]); + if (!current.isFile() || current.isSymbolicLink() + || String(opened.dev) !== mutation.originalDev || String(opened.ino) !== mutation.originalIno + || opened.dev !== current.dev || opened.ino !== current.ino) { + throw new Error(`Rollout identity changed before provider byte access: ${filePath}`); + } +} + +function isRecoverableProviderBytes(current, original, replacement) { + // Forward short writes and interrupted rollback produce old* new* old* at + // the differing positions. This excludes arbitrary edits and disjoint tears. + let phase = 0; + for (let i = 0; i < current.length; i += 1) { + if (original[i] === replacement[i]) { + if (current[i] !== original[i]) return false; + } else if (current[i] === replacement[i]) { + if (phase === 2) return false; + phase = 1; + } else if (current[i] === original[i]) { + if (phase === 1) phase = 2; + } else return false; + } + return true; +} + +async function restoreProviderOnHandle(handle, entry, options = {}) { + const mutation = entry.mutation ?? entry.inPlaceMutation; + const { originalBytes, replacementBytes } = validateProviderMutationDescriptor( + mutation, entry.path, entry.originalFirstLine, entry.originalSeparator); + await assertInPlaceIdentity(handle, entry.path, mutation); + const stat = await handle.stat(); + const expected = Buffer.from(entry.originalFirstLine + entry.originalSeparator, "utf8"); + const header = await readBytesFully(handle, expected.length, 0); + if (stat.size < mutation.originalSize || !header) { + throw new Error(`Rollout truncated before provider recovery: ${entry.path}`); + } + const end = mutation.byteOffset + originalBytes.length; + const current = header.subarray(mutation.byteOffset, end); + if (!header.subarray(0, mutation.byteOffset).equals(expected.subarray(0, mutation.byteOffset)) + || !header.subarray(end).equals(expected.subarray(end)) + || !isRecoverableProviderBytes(current, originalBytes, replacementBytes)) { + throw new Error(`Unknown rollout bytes during provider recovery: ${entry.path}`); + } + if (!current.equals(originalBytes)) { + await writeBytesFully(handle, originalBytes, mutation.byteOffset, options.inPlaceRestoreWrite ?? defaultInPlaceWrite); + } + await finishInPlaceWrite(handle, entry, originalBytes, { previousStat: stat }); +} + +async function tryRewriteProviderInPlace(change, options = {}) { + const mutation = change.inPlaceMutation; + const { originalBytes, replacementBytes } = validateProviderMutationDescriptor( + mutation, change.path, change.originalFirstLine, change.originalSeparator); + const writeImpl = options.inPlaceWrite ?? defaultInPlaceWrite; + let handle; + let writeAttempted = false; + try { + const pathStat = await fsp.lstat(change.path); + if (pathStat.isSymbolicLink() || !pathStat.isFile()) { + return "SKIP_CHANGED"; + } + handle = await fsp.open(change.path, "r+"); + const stat = await handle.stat(); + const identity = await handle.stat({ bigint: true }); + const snapshot = { + size: stat.size, + mtimeMs: stat.mtimeMs, + dev: String(identity.dev), + ino: String(identity.ino) + }; + if (!snapshotMatches(change, snapshot) + || mutation.originalSize !== change.originalSize + || mutation.originalMtimeMs !== change.originalMtimeMs) { + return "SKIP_CHANGED"; + } + const current = await readFirstLineRecordFromHandle(handle); + if (current.firstLine !== change.originalFirstLine || current.offset !== change.originalOffset) { + return "SKIP_CHANGED"; + } + const currentBytes = await readBytesFully(handle, originalBytes.length, mutation.byteOffset); + if (!currentBytes?.equals(originalBytes)) { + return "SKIP_CHANGED"; + } + try { + await assertInPlaceIdentity(handle, change.path, mutation); + if (!snapshotMatches(change, await getFileSnapshot(change.path))) return "SKIP_CHANGED"; + } catch { + return "SKIP_CHANGED"; + } + + try { + writeAttempted = true; + await writeBytesFully(handle, replacementBytes, mutation.byteOffset, writeImpl); + await finishInPlaceWrite(handle, change, replacementBytes, options); + } catch (error) { + if (writeAttempted) { + try { + await restoreProviderOnHandle(handle, change, options); + } catch (restoreError) { + const failure = new AggregateError( + [error, restoreError], + `Provider in-place write and immediate byte restoration both failed for ${change.path}.` + ); + failure.code = "IN_PLACE_RESTORE_FAILED"; + throw failure; + } + } + throw error; + } + return "APPLIED_IN_PLACE"; + } catch (error) { + throw wrapRolloutFileBusyError(error, change.path, "rewrite provider bytes in place"); + } finally { + await handle?.close(); + } +} + +async function restoreProviderBytesInPlace(entry, options = {}) { + let handle; + try { + const pathStat = await fsp.lstat(entry.path); + if (pathStat.isSymbolicLink() || !pathStat.isFile()) { + throw new Error(`Rollout path changed before in-place recovery: ${entry.path}`); + } + handle = await fsp.open(entry.path, "r+"); + await restoreProviderOnHandle(handle, entry, options); + return "RESTORED_IN_PLACE"; + } finally { + await handle?.close(); + } +} + const WINDOWS_REWRITE_PROTOCOL_VERSION = 1; const WINDOWS_REWRITE_READY_TIMEOUT_MS = 15_000; @@ -953,7 +1279,11 @@ async function rewriteFirstLine(filePath, nextFirstLine, separator) { } } -async function tryRewriteCollectedFirstLine(change) { +async function tryRewriteCollectedFirstLine(change, options = {}) { + if (change.inPlaceMutation?.strategy === PROVIDER_MUTATION_STRATEGY) { + return tryRewriteProviderInPlace(change, options); + } + const beforeSnapshot = await getFileSnapshot(change.path); if (!snapshotMatches(change, beforeSnapshot)) { return "SKIP_CHANGED"; @@ -1213,7 +1543,9 @@ export async function collectSessionChanges(codexHome, targetProvider, options = const rolloutPaths = await listJsonlFiles(rootDir); for (const rolloutPath of rolloutPaths) { let record; + let scanStart; try { + scanStart = await getFileSnapshot(rolloutPath); record = await readFirstLineRecord(rolloutPath); } catch (error) { if (skipLockedReads && isRolloutFileBusyError(error)) { @@ -1276,10 +1608,15 @@ export async function collectSessionChanges(codexHome, targetProvider, options = if (providerChanged || modelChanged) { const snapshot = await getFileSnapshot(rolloutPath); + if (snapshot.size !== scanStart.size || snapshot.mtimeMs !== scanStart.mtimeMs + || snapshot.dev !== scanStart.dev || snapshot.ino !== scanStart.ino) { + lockedPaths.push(rolloutPath); + continue; + } if (providerChanged) { parsed.payload.model_provider = targetProvider; } - summaries.push({ + const change = { path: rolloutPath, threadId: parsed.payload.id ?? null, directory: dirName, @@ -1288,6 +1625,8 @@ export async function collectSessionChanges(codexHome, targetProvider, options = originalOffset: record.offset, originalSize: snapshot.size, originalMtimeMs: snapshot.mtimeMs, + originalDev: snapshot.dev, + originalIno: snapshot.ino, originalProvider: currentProvider, updatedProvider: targetProvider, originalModel, @@ -1295,7 +1634,9 @@ export async function collectSessionChanges(codexHome, targetProvider, options = modelRewriteRequired: modelChanged, modelOnlyChange: !providerChanged && modelChanged, updatedFirstLine: providerChanged ? JSON.stringify(parsed) : record.firstLine - }); + }; + change.inPlaceMutation = getInPlaceProviderMutation(change); + summaries.push(change); } } } @@ -1311,7 +1652,10 @@ export async function applySessionChanges(changes, options = {}) { onMutation, onApplied, onSkipped, - windowsRewriteWorkerFactory = createWindowsExclusiveRewriteWorker + windowsRewriteWorkerFactory = createWindowsExclusiveRewriteWorker, + inPlaceWrite, + inPlaceRestoreWrite, + inPlaceSync } = options ?? {}; const skippedPaths = []; const appliedPaths = []; @@ -1354,7 +1698,7 @@ export async function applySessionChanges(changes, options = {}) { await onMutation?.(change, { stage: "model", result: "APPLIED" }); } } - await restoreOriginalMtime(change.path, change.originalMtimeMs); + if (result !== "APPLIED_IN_PLACE") await restoreOriginalMtime(change.path, change.originalMtimeMs); await onApplied?.(change); } else { skippedPaths.push(change.path); @@ -1378,7 +1722,11 @@ export async function applySessionChanges(changes, options = {}) { } else { for (const change of firstLineChanges) { await onBeforeApply?.(change); - const result = await tryRewriteCollectedFirstLine(change); + const result = await tryRewriteCollectedFirstLine(change, { + inPlaceWrite, + inPlaceRestoreWrite, + inPlaceSync + }); if (result === "APPLIED" || result === "APPLIED_IN_PLACE") { appliedChanges += 1; inPlaceChanges += result === "APPLIED_IN_PLACE" ? 1 : 0; @@ -1392,7 +1740,7 @@ export async function applySessionChanges(changes, options = {}) { await onMutation?.(change, { stage: "model", result: "APPLIED" }); } } - await restoreOriginalMtime(change.path, change.originalMtimeMs); + if (result !== "APPLIED_IN_PLACE") await restoreOriginalMtime(change.path, change.originalMtimeMs); await onApplied?.(change); } else { skippedPaths.push(change.path); @@ -1518,7 +1866,11 @@ export async function restoreSessionChanges(manifestEntries, options = {}) { for (const entry of manifestEntries) { try { await options.onBeforeRestore?.(entry); - if (!entry.modelOnlyChange) { + if (entry.mutation) { + validateProviderMutationDescriptor(entry.mutation, entry.path, entry.originalFirstLine, entry.originalSeparator); + if (process.platform === "win32") throw new Error("POSIX provider-byte backups require recovery on their original POSIX host."); + await restoreProviderBytesInPlace(entry, options); + } else if (!entry.modelOnlyChange) { if (process.platform === "win32") { const [result] = await invokeWindowsExclusiveRewriteBatch([{ path: entry.path, @@ -1538,7 +1890,7 @@ export async function restoreSessionChanges(manifestEntries, options = {}) { if (entry.originalTurnContextModels?.length) { await restoreTurnContextModelsInFile(entry.path, entry.originalTurnContextModels, entry.originalSeparator); } - await restoreOriginalMtime(entry.path, entry.originalMtimeMs); + if (!entry.mutation) await restoreOriginalMtime(entry.path, entry.originalMtimeMs); restoredPaths.push(entry.path); await options.onRestored?.(entry); } catch (error) { diff --git a/test/in-place-transaction.test.js b/test/in-place-transaction.test.js new file mode 100644 index 0000000..b8cbaf0 --- /dev/null +++ b/test/in-place-transaction.test.js @@ -0,0 +1,279 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; +import { applySessionChanges, collectSessionChanges, restoreSessionChanges } from "../src/session-files.js"; +import { createBackup, restoreBackup } from "../src/backup.js"; +import { runRestore, runSync } from "../src/service.js"; +import { TransactionJournal, readTransactionJournal, findPendingTransactions } from "../src/transaction-journal.js"; + +const repo = fileURLToPath(new URL("..", import.meta.url)); +const posix = { skip: process.platform === "win32" }; +const header = '{"type":"session_meta","payload":{"id":"fixture","cwd":"\u4e2d\u6587","model_provider" : "openai"}}'; +const tail = '\n{"type":"event_msg","payload":{"type":"user_message","message":"fixture"}}\n'; + +async function fixture(t, line = header, suffix = tail) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "provider-in-place-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const codexHome = path.join(root, "codex"); + await fs.mkdir(path.join(codexHome, "sessions"), { recursive: true }); + const file = path.join(codexHome, "sessions", "rollout-fixture.jsonl"); + const configPath = path.join(codexHome, "config.toml"); + await fs.writeFile(configPath, 'model_provider = "prov_a"\n'); + await fs.writeFile(file, line + suffix); + const mtime = new Date("2026-01-02T03:04:05Z"); + await fs.utimes(file, mtime, mtime); + return { codexHome, file, configPath, original: Buffer.from(line + suffix), mtime }; +} + +async function prepare(f) { + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + const backup = await createBackup({ codexHome: f.codexHome, targetProvider: "prov_a", sessionChanges: changes, configPath: f.configPath }); + const manifest = JSON.parse(await fs.readFile(path.join(backup, "session-meta-backup.json"), "utf8")); + return { changes, backup, entry: manifest.files[0] }; +} + +async function setBytes(file, mutation, bytes) { + const h = await fs.open(file, "r+"); + try { await h.write(bytes, 0, bytes.length, mutation.byteOffset); await h.sync(); } + finally { await h.close(); } +} + +test("in-place scan rejects ambiguous, escaped, non-ASCII and model-changing inputs", async (t) => { + const lines = [ + header.replace('"model_provider" : "openai"', '"model_provider":"openai","model_provider":"openai"'), + header.replace('"model_provider"', '"model_\\u0070rovider"'), + header.replace('"openai"', '"ope\\u006eai"'), + header.replace('"model_provider" : "openai"', '"model_provider":"openai","nested":{"model_provider":"openai"}'), + header.replace('"payload":', '"payload":{},"payload":') + ]; + for (const line of lines) { + const f = await fixture(t, line); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + assert.equal(changes[0].inPlaceMutation, null); + assert.equal((await applySessionChanges(changes)).inPlaceChanges, 0); + } + for (const target of ["provider_a", "", "\u4e00\u4e8c", 'bad"id']) { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, target); + assert.equal(changes[0].inPlaceMutation, null); + } + const f = await fixture(t, header, '\n{"type":"turn_context","payload":{"model":"before"}}\n'); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a", { targetModel: "after" }); + assert.equal(changes[0].inPlaceMutation, null); + assert.equal((await applySessionChanges(changes, { targetModel: "after" })).inPlaceChanges, 0); +}); + +test("provider-looking text inside a string is not a duplicate field", posix, async (t) => { + const f = await fixture(t, header.replace('"cwd":"\u4e2d\u6587"', '"cwd":"\\\"model_provider\\\":\\\"elsewhere\\\""')); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + assert.ok(changes[0].inPlaceMutation); +}); + +test("short writes loop to completion, preserve inode, bytes, size and mtime", posix, async (t) => { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + const before = await fs.stat(f.file); + let writes = 0; + const result = await applySessionChanges(changes, { inPlaceWrite(h, b, o, n, p) { writes++; return h.write(b, o, Math.min(n, 2), p); } }); + assert.equal(result.inPlaceChanges, 1); + assert.equal(writes, 4); + const after = await fs.stat(f.file); + assert.equal(after.ino, before.ino); + assert.equal(after.size, before.size); + assert.equal(Math.round(after.mtimeMs), f.mtime.getTime()); + assert.equal(await fs.readFile(f.file, "utf8"), f.original.toString().replace('"openai"', '"prov_a"')); +}); + +test("short-write exception, zero progress and fsync failure restore original bytes", posix, async (t) => { + for (const kind of ["short", "zero", "sync"]) { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + const before = await fs.stat(f.file); + let writes = 0; + const options = kind === "sync" ? { inPlaceSync: () => { throw new Error("fsync fault"); } } : { + async inPlaceWrite(h, b, o, n, p) { + if (kind === "zero") return { bytesWritten: 0 }; + if (writes++) throw new Error("write fault"); + return h.write(b, o, 3, p); + } + }; + await assert.rejects(applySessionChanges(changes, options)); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.equal((await fs.stat(f.file)).ino, before.ino); + } +}); + +test("failed immediate restoration never falls back and remains recoverable", posix, async (t) => { + const f = await fixture(t); + const { changes, entry } = await prepare(f); + let writes = 0; + await assert.rejects(applySessionChanges(changes, { + async inPlaceWrite(h, b, o, n, p) { if (writes++) throw new Error("write fault"); return h.write(b, o, 4, p); }, + inPlaceRestoreWrite() { throw new Error("restore fault"); } + }), { code: "IN_PLACE_RESTORE_FAILED" }); + assert.notDeepEqual(await fs.readFile(f.file), f.original); + await restoreSessionChanges([entry]); + assert.deepEqual(await fs.readFile(f.file), f.original); +}); + +test("in-place recovery accepts old/new/contiguous partial, rejects unknown bytes", posix, async (t) => { + for (const state of ["old", "new", "prefix", "middle", "unknown", "disjoint", "header", "truncate", "replace"]) { + const f = await fixture(t); + const { entry } = await prepare(f); + const m = entry.mutation; + const original = Buffer.from(m.originalBase64, "base64"), replacement = Buffer.from(m.replacementBase64, "base64"); + const bytes = Buffer.from(original); + if (state === "new") replacement.copy(bytes); + if (state === "prefix") replacement.copy(bytes, 0, 0, 4); + if (state === "middle") replacement.copy(bytes, 2, 2, 5); + if (state === "unknown") bytes[2] = 33; + if (state === "disjoint") { bytes[1] = replacement[1]; bytes[5] = replacement[5]; } + await setBytes(f.file, m, bytes); + if (state === "header") { const h = await fs.open(f.file, "r+"); await h.write(Buffer.from("!"), 0, 1, 0); await h.close(); } + if (state === "truncate") await fs.truncate(f.file, f.original.length - 1); + if (state === "replace") { await fs.writeFile(f.file + ".other", f.original); await fs.rename(f.file + ".other", f.file); } + const before = await fs.readFile(f.file), stat = await fs.stat(f.file); + if (["unknown", "disjoint", "header", "truncate", "replace"].includes(state)) { + await assert.rejects(restoreSessionChanges([entry]), AggregateError); + assert.deepEqual(await fs.readFile(f.file), before); + } else { + await restoreSessionChanges([entry]); + await restoreSessionChanges([entry]); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.equal((await fs.stat(f.file)).ino, stat.ino); + } + } +}); + +test("pre-write replaced path or append is skipped without fallback", posix, async (t) => { + for (const state of ["replace", "append"]) { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + if (state === "replace") { + await fs.writeFile(f.file + ".other", f.original); + await fs.utimes(f.file + ".other", f.mtime, f.mtime); + await fs.rename(f.file + ".other", f.file); + } else await fs.appendFile(f.file, tail); + const before = await fs.readFile(f.file); + const result = await applySessionChanges(changes); + assert.equal(result.appliedChanges, 0); + assert.deepEqual(result.skippedPaths, [f.file]); + assert.deepEqual(await fs.readFile(f.file), before); + } +}); + +test("active fd appends remain visible and rollback preserves appended tail and mtime", posix, async (t) => { + const f = await fixture(t); + const { changes, entry } = await prepare(f); + const writer = await fs.open(f.file, "a"); + t.after(() => writer.close()); + const before = await writer.stat(); + await applySessionChanges(changes); + await writer.write(tail); + const appended = await writer.stat(); + await restoreSessionChanges([entry]); + const after = await fs.stat(f.file); + assert.equal(after.ino, before.ino); + assert.ok(Math.abs(after.mtimeMs - appended.mtimeMs) < 0.01); + assert.equal(await fs.readFile(f.file, "utf8"), f.original + tail); +}); + +test("durable manifest and applying precede mutation; observer failure rolls back without rename", posix, async (t) => { + const f = await fixture(t); + const before = await fs.stat(f.file); + let backup, immutable; + await assert.rejects(runSync({ codexHome: f.codexHome, faultInjector: async ({ point, path: file, mutation }) => { + if (point === "before_rollout_apply") { + const pending = await findPendingTransactions(f.codexHome); + backup = pending[0].backupDir; + immutable = await fs.readFile(path.join(backup, "session-meta-backup.json")); + assert.equal(pending[0].events.at(-1).state, "applying"); + const manifest = JSON.parse(immutable); + assert.equal(manifest.version, 3); + assert.ok(manifest.files[0].mutation.originalBase64); + assert.deepEqual(await fs.readFile(file), f.original); + } + if (point === "after_rollout_mutation_before_applied") { + assert.equal(mutation.result, "APPLIED_IN_PLACE"); + throw new Error("observer fault"); + } + } }), (e) => e.code === "SYNC_FAILED_ROLLED_BACK"); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.equal((await fs.stat(f.file)).ino, before.ino); + assert.deepEqual(await fs.readFile(path.join(backup, "session-meta-backup.json")), immutable); +}); + +test("A applied, B fails: both in-place targets roll back (#69)", posix, async (t) => { + const f = await fixture(t); + const second = path.join(f.codexHome, "sessions", "rollout-z.jsonl"); + await fs.writeFile(second, f.original); + const before = await fs.stat(f.file); + await assert.rejects(runSync({ codexHome: f.codexHome, faultInjector: ({ point, targetIndex }) => { + if (point === "before_rollout_apply" && targetIndex === 2) throw new Error("B failed"); + } }), (e) => e.code === "SYNC_FAILED_ROLLED_BACK"); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.deepEqual(await fs.readFile(second), f.original); + assert.equal((await fs.stat(f.file)).ino, before.ino); +}); + +test("unknown bytes leave a recoveryRequired journal and block later writes", posix, async (t) => { + const f = await fixture(t); + await assert.rejects(runSync({ codexHome: f.codexHome, faultInjector: async ({ point }) => { + if (point === "after_rollout_mutation_before_applied") { + const { changes } = await collectSessionChanges(f.codexHome, "openai"); + await setBytes(f.file, changes[0].inPlaceMutation, Buffer.from('"??????"')); + throw new Error("unknown writer"); + } + } }), (e) => e.code === "RECOVERY_REQUIRED" && e.recoveryRequired); + assert.equal((await findPendingTransactions(f.codexHome))[0].state, "recoveryRequired"); + await assert.rejects(runSync({ codexHome: f.codexHome }), { code: "RECOVERY_REQUIRED" }); +}); + +test("actual process exit at applying/applied boundary recovers in place", posix, async (t) => { + for (const point of ["after_rollout_mutation_before_applied", "after_rollout_apply"]) { + const f = await fixture(t); + const before = await fs.stat(f.file); + const child = spawnSync(process.execPath, ["--input-type=module", "-e", ` + import { runSync } from './src/service.js'; + await runSync({codexHome: process.argv[1], faultInjector: ({point}) => {if(point === ${JSON.stringify(point)}) process.exit(91);}}); + `, f.codexHome], { cwd: repo, encoding: "utf8" }); + assert.equal(child.status, 91, child.stderr); + const [pending] = await findPendingTransactions(f.codexHome); + assert.ok(pending); + await runRestore({ codexHome: f.codexHome, backupDir: pending.backupDir, restoreConfig: false, restoreDatabase: false }); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.equal((await fs.stat(f.file)).ino, before.ino); + } +}); + +test("applying-only partial crash and torn journal recover from immutable manifest", posix, async (t) => { + for (const torn of [false, true]) { + const f = await fixture(t); + const { backup, entry } = await prepare(f); + const journal = await TransactionJournal.create(backup, { codexHome: f.codexHome, targetProvider: "prov_a", potentialTargets: [f.file] }); + await journal.applying("rollout", f.file); + await setBytes(f.file, entry.mutation, Buffer.from(entry.mutation.replacementBase64, "base64").subarray(0, 4)); + if (torn) await fs.appendFile(journal.filePath, '{"torn":'); + await runRestore({ codexHome: f.codexHome, backupDir: backup, restoreConfig: torn, restoreDatabase: false }); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.equal((await readTransactionJournal(journal.filePath)).state, "rolledBack"); + } +}); + +test("large fixture: actual rollout writes are bounded by provider bytes, not tail size", posix, async (t) => { + const f = await fixture(t, header, tail + ("x".repeat(65535) + "\n").repeat(512)); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + const hash = (b) => createHash("sha256").update(b.subarray(b.indexOf(10) + 1)).digest("hex"); + let written = 0; + const start = performance.now(); + await applySessionChanges(changes, { async inPlaceWrite(h, b, o, n, p) { const r = await h.write(b, o, n, p); written += r.bytesWritten; return r; } }); + assert.equal(written, 8); + assert.equal(hash(await fs.readFile(f.file)), hash(f.original)); + t.diagnostic(`32 MiB fixture: ${written} rollout bytes written in ${(performance.now() - start).toFixed(1)} ms (including verification/hash read).`); +}); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index b64794a..3bc2697 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -1703,7 +1703,7 @@ test("runSync rewrites rollout files and sqlite, then restore reverts both", asy assert.deepEqual(syncResult.skippedLockedRolloutFiles, []); assert.equal(syncResult.sqliteRowsUpdated, 2); const backupMetadata = JSON.parse(await fs.readFile(path.join(syncResult.backupDir, "metadata.json"), "utf8")); - assert.equal(backupMetadata.version, 2); + assert.equal(backupMetadata.version, process.platform === "win32" ? 2 : 3); assert.equal(backupMetadata.sqliteHome, path.join(codexHome, SQLITE_DIR_BASENAME)); assert.deepEqual(backupMetadata.sqliteDbFiles, [DB_FILE_BASENAME]); assert.ok(Number.isSafeInteger(backupMetadata.sizeBytes)); @@ -1835,7 +1835,7 @@ test("runSync uses an explicit SQLite home and never touches a stale Codex Home } const metadata = JSON.parse(await fs.readFile(path.join(result.backupDir, "metadata.json"), "utf8")); - assert.equal(metadata.version, 2); + assert.equal(metadata.version, process.platform === "win32" ? 2 : 3); assert.equal(metadata.sqliteHome, sqliteHome); assert.deepEqual(metadata.dbFiles, []); assert.deepEqual(metadata.sqliteDbFiles, [DB_FILE_BASENAME]); @@ -3263,7 +3263,7 @@ test("applySessionChanges preserves large UTF-8 session metadata", async () => { assert.match(rollout, /"large_blob":"数据块数据块/); }); -test("applySessionChanges atomically replaces equal-length provider IDs", async () => { +test("applySessionChanges updates equal-length provider IDs without replacing the inode", async () => { const { codexHome } = await makeTempCodexHome(); const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-in-place.jsonl"); await writeRollout(sessionPath, "thread-in-place", "openai"); @@ -3278,6 +3278,7 @@ test("applySessionChanges atomically replaces equal-length provider IDs", async await fs.writeFile(sessionPath, original, "utf8"); const originalTime = new Date("2026-01-02T03:04:05.000Z"); await fs.utimes(sessionPath, originalTime, originalTime); + const before = await fs.stat(sessionPath); const { changes } = await collectSessionChanges(codexHome, "prov_a"); const result = await applySessionChanges(changes); @@ -3285,7 +3286,11 @@ test("applySessionChanges atomically replaces equal-length provider IDs", async const rollout = await fs.readFile(sessionPath, "utf8"); assert.equal(result.appliedChanges, 1); - assert.equal(result.inPlaceChanges, 0); + assert.equal(result.inPlaceChanges, process.platform === "win32" ? 0 : 1); + if (process.platform !== "win32") { + assert.equal(after.ino, before.ino); + assert.equal(after.size, before.size); + } assert.equal(Math.round(after.mtimeMs), originalTime.getTime()); const firstNewline = rollout.indexOf("\n"); assert.equal(JSON.parse(rollout.slice(0, firstNewline)).payload.model_provider, "prov_a"); From 10cf5a908dca2ef86ea4bae6a40fd9a32f5e33ab Mon Sep 17 00:00:00 2001 From: cccat6 <22387156+cccat6@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:21:03 +0800 Subject: [PATCH 2/5] fix: harden transactional provider byte updates and add fast scans Extend the PR #51 optimization restored by 1e0f8b3 with Windows exclusive-handle writes and recovery, post-write guards, and explicit metadata-only sync. Keep backup-first per-target journal semantics from #71/#69 and reject unknown recovery bytes. Fuse default body diagnostics, cover failures on Node 16/24 and native Windows, and document proposed mtime/schema compatibility boundaries. No production deployment or upstream publication. --- CHANGELOG.md | 6 + README.md | 4 + docs/README_EN.md | 4 + docs/TRANSACTIONAL_IN_PLACE_PILOT.md | 99 +++++- .../proposed-transactional-provider-bytes.md | 48 +++ .../architecture/contracts/CLI_CONTRACT_ZH.md | 2 + .../contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md | 14 +- docs/architecture/contracts/ERROR_CODES_ZH.md | 2 + docs/migration/BEHAVIOR_FIXTURES_ZH.md | 2 + scripts/benchmark-provider-io.mjs | 69 ++++ src/backup.js | 16 +- src/cli.js | 23 +- src/service.js | 23 +- src/session-files.js | 300 ++++++++---------- src/windows-provider-bytes.cs | 134 ++++++++ test/fast-sync.test.js | 184 +++++++++++ test/in-place-transaction.test.js | 57 +++- test/sync-service.test.js | 15 +- test/windows-provider-bytes.ps1 | 164 ++++++++++ test/windows-rewrite-worker.test.js | 48 ++- 20 files changed, 993 insertions(+), 221 deletions(-) create mode 100644 docs/adr/proposed-transactional-provider-bytes.md create mode 100644 scripts/benchmark-provider-io.mjs create mode 100644 src/windows-provider-bytes.cs create mode 100644 test/fast-sync.test.js create mode 100644 test/windows-provider-bytes.ps1 diff --git a/CHANGELOG.md b/CHANGELOG.md index bdee749..be8a5c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ 本文件记录面向用户和集成方的重要变化。完整的发布叙事、升级说明和下载入口见对应版本的中文发布说明；实现证据和测试门禁见技术发布说明。 +## 未发布（候选） + +- 恢复 #51 等长 provider 原地更新，将字节恢复纳入 #71 逐目标事务；覆盖 Node POSIX 和 Windows worker。 +- 合并正文扫描；新增显式 `sync --fast` / `switch --fast`，只读首行、保留模型、提示未执行检查，不支持原地更新时不隐式重写全文。 +- 原地/快速备份升级为 v3，旧工具不能恢复。POSIX 原地路径保留实际写入 mtime，避免回拨并发追加时间；依赖文件时间的 History 排序可能受影响。详见[兼容边界](docs/TRANSACTIONAL_IN_PLACE_PILOT.md)。 + ## [0.5.0] - 2026-08-15 ### 新增 diff --git a/README.md b/README.md index 7e78d4c..f378b98 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,10 @@ codex-provider sync `switch` 默认会在目标 Provider section 定义了 `model` 时同步根级 `model`。使用 `--keep-root-model` 保留当前值，或使用 `--model ` 显式指定。 +本分支新增 `sync --fast` / `switch --fast`：只读 rollout 首行，保留模型，不检查历史用户消息和加密内容；不支持原地更新时在写入前报错，不自动全量重写。备份、事务和恢复检查仍然执行；新备份需兼容 v3 的工具恢复，旧 .NET GUI 不支持。详见[实现与兼容边界](docs/TRANSACTIONAL_IN_PLACE_PILOT.md)。 + +建议使用统一长度的 ASCII Provider ID，优先 6 个字符（如将 `provider_a` 写作 `prov_a`），因为内置 `openai` 是 6 个字符，相对最通用；历史文件很多或很大时，不等长替换会重写整个 rollout、产生大量硬盘写入，等长且符合条件时可 in-place 替换。POSIX 原地路径保留实际写入 mtime，不回拨并发追加的时间；`updated_at` 不变。 + SQLite Home 解析顺序：`--sqlite-home` → `config.toml` 根级 `sqlite_home` → `CODEX_SQLITE_HOME` → `/sqlite`。只有默认布局会回退到 `/state_5.sqlite`。 ## 当前架构 diff --git a/docs/README_EN.md b/docs/README_EN.md index 954005a..8197201 100644 --- a/docs/README_EN.md +++ b/docs/README_EN.md @@ -109,6 +109,10 @@ codex-provider sync By default, `switch` also updates the root-level `model` when the target provider section defines one. Use `--keep-root-model` to preserve the current value, or `--model ` to set it explicitly. +This branch adds `sync --fast` / `switch --fast`: read only rollout headers, preserve models, and leave historical user-message/encryption checks unperformed. Unsupported in-place updates fail before mutation, without automatic full rewriting. Backups, transactions and recovery checks remain enabled; new backups need a v3-compatible restore tool, not the old .NET GUI. See [implementation and compatibility](TRANSACTIONAL_IN_PLACE_PILOT.md). + +We recommend uniform-length ASCII provider IDs, preferably six characters (for example, `provider_a` as `prov_a`), because built-in `openai` has six characters and is the most common compatibility target. With many or large histories, different lengths require whole-rollout rewrites and substantial disk writes; eligible equal-length values can be replaced in place. POSIX in-place writes retain their actual mtime rather than backdating concurrent appends; `updated_at` is unchanged. + SQLite Home resolution order: `--sqlite-home` → root-level `sqlite_home` in `config.toml` → `CODEX_SQLITE_HOME` → `/sqlite`. Only the default layout falls back to `/state_5.sqlite`. ## Current Architecture diff --git a/docs/TRANSACTIONAL_IN_PLACE_PILOT.md b/docs/TRANSACTIONAL_IN_PLACE_PILOT.md index af50c5c..abb5047 100644 --- a/docs/TRANSACTIONAL_IN_PLACE_PILOT.md +++ b/docs/TRANSACTIONAL_IN_PLACE_PILOT.md @@ -1,14 +1,18 @@ -# Transactional in-place provider writes: POSIX pilot +# Transactional provider byte updates (implementation candidate) This branch restores the optimization introduced by cccat6 in PR #51 (`7231881`, `cdcde35`, `84a60d3`). PR #71's transaction refactor removed the production in-place path; v0.5.0 still counts `APPLIED_IN_PLACE` but does not -produce it. This pilot does not change authentication or deploy to other hosts. +produce it. This candidate does not manage authentication, profiles or processes. +It is based on upstream main `c7ff852`, not the unmerged V1 migration (#90). ## Implemented and tested -- Node POSIX only; Windows retains the existing exclusive replacement worker. - The .NET implementation is unchanged. Neither is claimed to have parity yet. +- Node POSIX and the existing Windows exclusive PowerShell worker support the + byte strategy. The small C# helper is compiled by that worker for native file + identity and handle operations; it is not a second sync service or SDK dependency. + The published .NET application is unchanged and rejects metadata v3 before + writing any restore target. Cross-runtime recovery parity is NOT claimed. - A non-empty, equal-length ASCII provider ID with one unescaped, unambiguous `session_meta.payload.model_provider` field can be replaced in place. A `turn_context.model` rewrite or an ineligible header uses the existing path. @@ -17,6 +21,8 @@ produce it. This pilot does not change authentication or deploy to other hosts. before the coordinator durably appends `applying` and starts the write. - Apply revalidates the path, handle identity, snapshot, header and bytes. A stale precondition is skipped, never used as a reason for a full rewrite. + Hardlinked targets are ineligible. Post-write checks include the complete + header, identity and minimum file size, not just the replacement bytes. - Short writes loop; write/fsync/read-back failure attempts byte restoration through the same handle. There is no post-mutation fallback to rename. - `applying` and `applied` targets recover from the immutable manifest. A torn @@ -25,7 +31,8 @@ produce it. This pilot does not change authentication or deploy to other hosts. - Both backup metadata and session manifest use version 3 when any entry is in-place, so old readers reject before restoring config or SQLite. Version 1/2 backups still use their old recovery semantics. Non-in-place backups stay - version 2. New backups must be restored using this pilot or a compatible tool. + version 2. Fast-mode backups always use version 3, including no-op rollouts. + New backups require this version or a compatible restore implementation. ## Recovery and writer contract @@ -41,27 +48,91 @@ the header to an indistinguishable value. The supported writer leaves existing bytes alone and appends after the guarded metadata operation. Pre-apply growth is skipped. Later appends remain visible -through the existing fd and survive rollback without truncation. Recovery -preserves the newer mtime of an already-appended file rather than applying the -scan-time mtime. No POSIX cooperative lock can force Codex to participate: this +through the existing fd and survive rollback without truncation. POSIX in-place +apply/restore leaves the actual write mtime: there is no race-free stat/utimes +sequence against an uncooperative appender. Windows restores mtime while holding +the exclusive handle; no-op recovery does not alter it. Thread `updated_at` +is never changed. History views using filesystem mtime may reorder; this +deliberate safety/compatibility tradeoff requires maintainer acceptance. +No POSIX cooperative lock can force Codex to participate: this does not promise atomic visibility to concurrent readers, or protection from non-cooperating writers replacing/truncating/editing the header during the small check/write window. Full replacement paths still have the active-fd risk identified in PR #71; this pilot removes that risk only for eligible writes. +## Fast scope and reading cost + +`sync --fast` and `switch --fast` enumerate both rollout roots but read +only metadata headers (bounded to 1 MiB per header) plus file attributes. +Every changed rollout must qualify for in-place replacement. An ineligible or +invalid header fails preflight, before backup or config/SQLite mutation; there +is no implicit full rewrite. Busy/changed targets retain the existing partial +outcome semantics and must not be described as completely aligned. + +The scope preserves root and historical models, leaves `has_user_event` +unchanged, and reports encrypted-content/model/user-event checks as unchecked. +Provider and header-derived cwd/workspace repair retain the existing transaction. +`--model` conflicts with `--fast`; `--keep-root-model` is redundant, allowed. +The managed manifest records `scanScope: metadata`; restore does not invent +historical model snapshots or scan/copy message bodies. Existing config and +SQLite backup/restore behavior is retained, so cost is not strictly header-only. + +Default scans keep their diagnostics but compute encryption presence, positive +user-event evidence and model snapshots in one streaming pass. No persistent +cache is added. The default full rewrite remains for noneligible operations. + ## Validation and follow-up +Validated on 2026-08-28: + +- Linux Node 24.16.0: full suite, 280 passed / 6 platform skips / 0 failed. +- Linux Node 16.20.2 with the existing optional better-sqlite3 8.7.0 driver: + all 19 test files passed (the older runner reports file-level totals). +- Existing Windows PowerShell: real production worker protocol, busy response, + native file identity, in-place apply/restore, timestamp retention, short-write + exception, Flush failure, failed immediate undo followed by recovery, + idempotence, appended-tail preservation and unknown-byte rejection passed. +- Web production build, package dry-run (including the native helper source), + and `git diff --check` passed. +- Not run: full Node/SQLite suite on Windows, macOS native tests, real WSL UNC + tests, and cross-runtime v3 restore (the old .NET reader rejects v3). + +`node scripts/benchmark-provider-io.mjs 32` uses disposable data. On ext4 with +warm page cache, one ~32 MiB fixture produced these process-level measurements: + +| Mode | Logical reads | Kernel-accounted writes | Elapsed | +| --- | ---: | ---: | ---: | +| Full, equal IDs | 34,044,759 B | 45,056 B | 127 ms | +| Fast, equal IDs | 405,780 B | 45,056 B | 55 ms | +| Full, unequal IDs | 67,689,493 B | 33,681,408 B | 247 ms | + +Equal-ID cases retained inode and tail hash. Numbers include managed backup +and journal overhead, not just provider bytes. They exclude SSD-internal write +amplification and are not a prediction for cold storage or a large SQLite DB. + `test/in-place-transaction.test.js` covers eligibility, short/zero writes, fsync failure, immediate restoration failure, immutable manifests, A/B failure, crashes before `applied` and before commit, torn journals, idempotence, conflicts, active fds and appends. A 32 MiB disposable fixture records only 8 rollout bytes written and verifies the unchanged tail hash and inode. -The full Node suite must pass before MOSS installation. No real history is -used by these tests, and no Codex/API calls are needed. +`test/fast-sync.test.js` guards against rollout body streams throughout switch +and restore, tests preflight failures, CLI parsing, models, SQLite and rollback. +`test/windows-provider-bytes.ps1` also tests the native helper using the existing +PowerShell runtime without a Node installation. No real history or API calls +are used. This work does not authorize production deployment. + +Before a formal PR: run the full native Windows Node suite; agree the POSIX +mtime policy, metadata v3 transition and fast-scope semantics with the maintainer. +See the [proposed ADR](adr/proposed-transactional-provider-bytes.md). No public +PR/comment/release is authorized by this development work. -Before a formal PR: add and actually run Windows worker fault/recovery tests; -decide .NET transition support with the maintainer; review recovery portability -and the append-only contract. No public PR/comment/release is authorized yet. +If #90 lands first, rebase through its shared Core, plan ledger, dual locks and +Restore v2. Its full-content revisions and final status refresh must become +scope-aware for fast operations, not be bypassed. Bind scope, original header, +file identity, target bytes and config/DB revisions in the plan; retain full +hash verification for full-scope operations (streaming rather than readFile). +Restore-v2 pre-snapshots, target digests and compensation must use the same byte +strategy. Those V1-specific changes are NOT implemented on this main-based branch. References: [#51](https://github.com/Dailin521/codex-provider-sync/pull/51), [#71](https://github.com/Dailin521/codex-provider-sync/pull/71), @@ -69,7 +140,7 @@ References: [#51](https://github.com/Dailin521/codex-provider-sync/pull/51), [active-fd finding](https://github.com/Dailin521/codex-provider-sync/pull/71#discussion_r3711178450), [Codex #38149](https://github.com/openai/codex/issues/38149). -For frequent switching, use equal-length ASCII provider IDs, preferably six +For frequent switching, we recommend equal-length ASCII provider IDs, preferably six characters because `openai` has six (for example `provider_a` as `prov_a`). Different lengths require whole-file rewriting; large histories can multiply disk writes and elapsed time. The original user's rollout collection was diff --git a/docs/adr/proposed-transactional-provider-bytes.md b/docs/adr/proposed-transactional-provider-bytes.md new file mode 100644 index 0000000..29873c5 --- /dev/null +++ b/docs/adr/proposed-transactional-provider-bytes.md @@ -0,0 +1,48 @@ +# Proposed: transactional provider byte updates and explicit fast scope + +- Status: Proposed, not maintainer-approved or released +- Date: 2026-08-28 +- Base: upstream main c7ff852 (v0.5.0) + +## Context + +PR #51 saved whole-rollout writes for equal-length IDs. PR #71 removed that +path while adding durable per-target recovery. Restore the optimization inside +the same journal, not around it. A separate fast scope addresses body reading. + +## Proposal + +- Keep current default synchronization semantics and full-rewrite fallback. +- Add a manifest-bound provider-byte strategy, durable before `applying`. +- Revalidate identity/header/bytes, write through the same handle, flush and + verify; undo through that handle/strategy without rename or tail truncation. +- Preserve POSIX write mtime to avoid racing appenders; Windows can preserve + original mtime under its exclusive handle. This is an explicit compatibility + change, not an assertion that all default observable behavior is identical. +- Metadata and manifest v3 prevent old readers from silently applying the + wrong restore strategy. Old v1/v2 backups retain their existing semantics. +- Opt-in `--fast` narrows diagnostics/model scope, never durability or recovery. + Unsupported headers fail preflight rather than silently copying the body. +- No credentials, account tooling, process management, new database or cache. + +## Acceptance questions + +The mtime policy can affect History's filesystem-time fallback, so it needs +explicit acceptance. Do not claim safe concurrent append and unconditional old +mtime restoration simultaneously. Do not replace this policy with a timing +heuristic and call it exclusive access. + +Current .NET rejects v3 safely but cannot recover it. Maintainer approval is +required for this migration boundary, or a compatible recovery reader must be +added before release. Windows Node support is not .NET application parity. + +Fast scope is new behavior: discuss it separately from restoring #51. If the +unmerged #90 becomes main, integrate scope into its Plan/Revision/Restore v2 +contracts before claiming support. No new public method, lock protocol or +second transaction system is necessary on the current base. + +## Evidence + +See [implementation and tests](../TRANSACTIONAL_IN_PLACE_PILOT.md). Native, +simulated and unrun coverage must be reported separately. Process-exit tests +do not establish arbitrary power-loss or non-cooperating-writer guarantees. diff --git a/docs/architecture/contracts/CLI_CONTRACT_ZH.md b/docs/architecture/contracts/CLI_CONTRACT_ZH.md index 219bb45..c3f1fb3 100644 --- a/docs/architecture/contracts/CLI_CONTRACT_ZH.md +++ b/docs/architecture/contracts/CLI_CONTRACT_ZH.md @@ -1,5 +1,7 @@ # CLI 命令兼容合同 +> 本分支候选增量（未发布）：`sync --fast` 和 `switch --fast` 只扫描首行，要求原地 provider 更新，保留模型并提示未执行的历史检查。`--fast` 不接收值、不支持其他命令，与 `--model` 互斥，允许冗余的 `--keep-root-model`。默认同步范围不变；摘要增加 `In-place rollout updates`。不支持的快速操作在业务写入前失败，不自动全量重写。详见[候选 ADR](../../adr/proposed-transactional-provider-bytes.md)。 + > 状态：Phase 0 兼容基线 > > 基线版本：`@dailin521/codex-provider-sync` v0.5.0 diff --git a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md index 33908f0..5fe0b84 100644 --- a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md +++ b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md @@ -10,10 +10,16 @@ ## 1. 文档目的 -当前修复分支的 POSIX 原地写试运行扩展见 -[事务化原地更新试运行说明](../../TRANSACTIONAL_IN_PLACE_PILOT.md)：等长 provider -使用带 mutation descriptor 的 v3 备份；旧 v1/v2 备份继续可读，旧客户端必须拒绝 v3。 -此扩展不是 v0.5.0 已发布行为，也不声称 Windows/.NET 已具备对应原地恢复能力。 +本分支候选增量（未发布，待上游评审）： + +- Node `runSync` / `runSwitch` 新增 `fast=false`。默认保留原检查范围，合并为一次正文扫描。 +- 合格的等长 provider 使用 manifest v3 描述的原地 mutation；执行、回滚和崩溃恢复保留同一 inode/file ID，开始写入后不得回退全文重写。 +- POSIX 原地路径不恢复旧 mtime，Windows 在独占句柄内保留 mtime；`updated_at` 不变，但依赖文件 mtime 的 History 排序可能变化，需维护者确认。 +- `fast=true` 只读首行（上限 1 MiB）及属性，保留根级/历史模型，不检查用户事件与加密内容；保留 provider、首行 cwd、workspace 修复及数据库事务。 +- 快速模式静态不支持项在备份和业务写入前以 `FAST_MODE_UNSUPPORTED` 失败，不隐式全量重写；运行时 busy/changed 沿用 partial 分类。 +- 结果增加 `inPlaceSessionFiles`；快速结果另含 `scanScope="metadata"`、`unchecked=["historyModels","userEventFlags","encryptedContent"]`、`encryptedContentCounts=null` 和未检查警告。 +- 原地/快速备份使用 metadata 和 manifest v3；旧 v1/v2 保持原恢复语义，旧 .NET 仅安全拒绝 v3，不宣称互相恢复。 +- 详见[候选 ADR](../../adr/proposed-transactional-provider-bytes.md)和[实现说明](../../TRANSACTIONAL_IN_PLACE_PILOT.md)。V1/#90 的 Plan/Revision/Restore v2 接入不属于本分支已完成能力。 本文冻结 vNext 迁移开始时 Node 实现已经提供的外部行为。这里的“外部”不仅指 npm 最终用户，也包括当前 CLI 与 Local Web UI 对 Node service 的真实依赖。 diff --git a/docs/architecture/contracts/ERROR_CODES_ZH.md b/docs/architecture/contracts/ERROR_CODES_ZH.md index a1938b8..3285370 100644 --- a/docs/architecture/contracts/ERROR_CODES_ZH.md +++ b/docs/architecture/contracts/ERROR_CODES_ZH.md @@ -10,6 +10,8 @@ ## 1. 目的 +本分支候选增量（未发布）：`FAST_MODE_UNSUPPORTED` 表示首行无效/超过快速读取上限，或待修改文件无法采用等长原地策略。发生在备份和业务 mutation 之前，修正输入或显式选择完整同步后可重试，不要求 recovery。`IN_PLACE_RESTORE_FAILED` 仅作为内部原因；服务层仍以既有 `SYNC_FAILED_ROLLED_BACK` / `RECOVERY_REQUIRED` 表达最终恢复结果。不得将未知字节、截断或文件替换伪装成安全回退。 + 本文冻结 vNext 的错误分类、兼容映射和演进规则，使调用方依据稳定的 `code` 决策，而不是解析自然语言 `message`、异常类型名或堆栈。 本文不表示当前代码已经完成统一。当前 Node、Web 与 .NET 仍存在不同大小写、命名和结构；这些现状被列为 Legacy Surface，由后续结构化错误 PR 通过 Adapter 渐进收口。 diff --git a/docs/migration/BEHAVIOR_FIXTURES_ZH.md b/docs/migration/BEHAVIOR_FIXTURES_ZH.md index bfc2ca1..08894e1 100644 --- a/docs/migration/BEHAVIOR_FIXTURES_ZH.md +++ b/docs/migration/BEHAVIOR_FIXTURES_ZH.md @@ -1,5 +1,7 @@ # vNext 行为兼容 Fixture 清单 +本分支候选夹具：`test/in-place-transaction.test.js` 验证短写、崩溃、损坏 journal、冲突及活跃追加；`test/fast-sync.test.js` 验证快速范围、无正文流、模型/SQLite/cwd、前置失败和回滚；`test/windows-rewrite-worker.test.js` 与 `test/windows-provider-bytes.ps1` 验证协议和 Windows 原生独占句柄。POSIX 原地路径保留实际写入 mtime，原全量路径仍保留旧 mtime；该区别是待评审合同，不是隐式测试豁免。 + > **状态：Accepted（阶段 0 语义清单；共享 Corpus 尚未创建）** > > **日期：2026-08-24** diff --git a/scripts/benchmark-provider-io.mjs b/scripts/benchmark-provider-io.mjs new file mode 100644 index 0000000..d54c7d2 --- /dev/null +++ b/scripts/benchmark-provider-io.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { runSync } from "../src/service.js"; + +// Disposable, bounded-memory benchmark. Never use an existing Codex Home. +delete process.env.CODEX_SQLITE_HOME; +const mib = Number(process.argv[2] ?? 32); +if (!Number.isInteger(mib) || mib < 1 || mib > 256) throw new Error("Size must be 1..256 MiB."); +const root = await fsp.mkdtemp(path.join(os.tmpdir(), "provider-io-benchmark-")); +const line = JSON.stringify({ type: "event_msg", payload: { text: "x".repeat(4000) } }) + "\n"; +const block = Buffer.from(line.repeat(128)); + +async function counters() { + if (process.platform !== "linux") return null; + const text = await fsp.readFile("/proc/self/io", "utf8"); + return Object.fromEntries(text.trim().split("\n").map(line => { + const [key, value] = line.split(": "); + return [key, Number(value)]; + })); +} + +async function hashTail(file, offset) { + const hash = createHash("sha256"); + for await (const chunk of fs.createReadStream(file, { start: offset })) hash.update(chunk); + return hash.digest("hex"); +} + +try { + const results = []; + for (const mode of ["full-equal", "fast-equal", "full-unequal"]) { + const home = path.join(root, mode); + await fsp.mkdir(path.join(home, "sessions"), { recursive: true }); + await fsp.writeFile(path.join(home, "config.toml"), 'model_provider="prov_a"\n'); + const file = path.join(home, "sessions", "rollout-fixture.jsonl"); + const header = JSON.stringify({ type: "session_meta", payload: { + id: "fixture", model_provider: mode === "full-unequal" ? "provider_old" : "openai" + } }) + "\n"; + const h = await fsp.open(file, "w"); + try { + await h.writeFile(header); + await h.writeFile('{"type":"event_msg","payload":{"type":"user_message","message":"fixture"}}\n'); + for (let n = 0; n < mib * 1024 * 1024; n += block.length) await h.writeFile(block); + await h.sync(); + } finally { await h.close(); } + const beforeStat = await fsp.stat(file, { bigint: true }); + const beforeHash = await hashTail(file, Buffer.byteLength(header)); + const before = await counters(); + const start = performance.now(); + const result = await runSync({ codexHome: home, fast: mode === "fast-equal" }); + const ms = performance.now() - start; + const after = await counters(); + const afterStat = await fsp.stat(file, { bigint: true }); + const headerAfter = header.replace(mode === "full-unequal" ? "provider_old" : "openai", "prov_a"); + assert.equal(await hashTail(file, Buffer.byteLength(headerAfter)), beforeHash); + const delta = before && Object.fromEntries(["rchar", "wchar", "read_bytes", "write_bytes"].map(k => [k, after[k] - before[k]])); + results.push({ mode, rolloutBytes: Number(beforeStat.size), ms: Math.round(ms), + inPlace: result.inPlaceSessionFiles, sameInode: beforeStat.ino === afterStat.ino, + ...(delta ? { processIo: delta } : {}) }); + } + console.log(JSON.stringify({ platform: process.platform, results, + note: "Warm-cache synthetic benchmark. rchar/wchar are logical process I/O, not SSD wear; kernel write_bytes excludes device-internal amplification." }, null, 2)); +} finally { + await fsp.rm(root, { recursive: true, force: true }); +} diff --git a/src/backup.js b/src/backup.js index 020f4ca..55601f0 100644 --- a/src/backup.js +++ b/src/backup.js @@ -10,7 +10,7 @@ import { GLOBAL_STATE_BACKUP_FILE_BASENAME, GLOBAL_STATE_FILE_BASENAME } from "./constants.js"; -import { restoreSessionChanges, validateProviderMutationDescriptor } from "./session-files.js"; +import { restoreSessionChanges, validateProviderMutationDescriptor, validateProviderByteRestore } from "./session-files.js"; import { assertSqliteWritable, createSqliteOnlineBackup, @@ -248,6 +248,7 @@ export async function createBackup({ codexHome, targetProvider, sessionChanges, + fast = false, configPath, configBackupText }) { @@ -297,12 +298,13 @@ export async function createBackup({ // Both versions must advance so old readers reject before restoring any // config/SQLite data, even when rollout restore was disabled by the caller. - const backupVersion = sessionChanges.some((change) => change.inPlaceMutation) ? 3 : 2; + const backupVersion = fast || sessionChanges.some((change) => change.inPlaceMutation) ? 3 : 2; const sessionManifest = { version: backupVersion, namespace: BACKUP_NAMESPACE, codexHome, targetProvider, + ...(fast ? { scanScope: "metadata" } : {}), createdAt: new Date().toISOString(), // Keep the full pre-mutation source of truth for the lifetime of the // backup. appliedPaths is only a compatibility hint for backups without a @@ -338,6 +340,7 @@ export async function createBackup({ codexHome, sqliteHome: actualSqliteHome, targetProvider, + ...(fast ? { scanScope: "metadata" } : {}), createdAt: sessionManifest.createdAt, dbFiles: copiedDbFiles, sqliteDbFiles: copiedSqliteDbFiles, @@ -354,9 +357,7 @@ export async function updateSessionBackupManifest(backupDir, sessionChanges, opt const sessionManifest = JSON.parse(await fs.readFile(manifestPath, "utf8")); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - // Promote older manifests to the v3 schema. Existing entries remain valid; - // only newly collected equal-length provider changes carry a mutation - // descriptor for in-place recovery. + // Legacy bookkeeping must not retroactively invent in-place undo evidence. sessionManifest.version = Math.max(2, sessionManifest.version); const filesByPath = new Map( @@ -615,6 +616,11 @@ export async function restoreBackup(backupDir, storageOrCodexHome, options = {}) } else { sessionRestoreEntries = await selectSessionRestoreEntries(backupDir, sessionManifest); } + // Detect known byte conflicts before rewinding config or SQLite. Apply + // still rechecks through the mutation handle; preflight is not a lock. + for (const entry of sessionRestoreEntries) { + if (entry.mutation) await validateProviderByteRestore(entry); + } } let stateDb = null; diff --git a/src/cli.js b/src/cli.js index 271b28d..235380e 100755 --- a/src/cli.js +++ b/src/cli.js @@ -16,14 +16,17 @@ function printHelp() { Usage: codex-provider status [--codex-home PATH] [--sqlite-home PATH] - codex-provider sync [--provider ID] [--keep N] [--codex-home PATH] [--sqlite-home PATH] - codex-provider switch [--model NAME] [--keep-root-model] [--keep N] [--codex-home PATH] [--sqlite-home PATH] + codex-provider sync [--fast] [--provider ID] [--keep N] [--codex-home PATH] [--sqlite-home PATH] + codex-provider switch [--fast | --model NAME] [--keep-root-model] [--keep N] [--codex-home PATH] [--sqlite-home PATH] codex-provider watch [--codex-home PATH] [--sqlite-home PATH] [--debounce-ms N] [--once] [--no-state-db] codex-provider web [--port N] [--no-open] [--reset-access] [--codex-home PATH] [--sqlite-home PATH] codex-provider prune-backups [--keep N] [--codex-home PATH] codex-provider restore [--no-config] [--no-db] [--no-sessions] [--allow-sqlite-home-relocation] [--codex-home PATH] [--sqlite-home PATH] codex-provider install-windows-launcher [--dir PATH] [--codex-home PATH] [--sqlite-home PATH] +sync / switch flags: + --fast read metadata only; require in-place provider updates; preserve models + switch flags: --model NAME override root-level model field with NAME (e.g. "MiniMax-M3") --keep-root-model do not touch the root-level model field; only switch model_provider @@ -56,6 +59,11 @@ function parseArgs(argv) { } const [flagName, inlineValue] = value.split("=", 2); const normalizedName = flagName.slice(2); + if (normalizedName === "fast") { + if (inlineValue !== undefined) throw new Error("--fast does not take a value."); + flags.fast = true; + continue; + } if (inlineValue !== undefined) { flags[normalizedName] = inlineValue; continue; @@ -85,6 +93,7 @@ function summarizeSync(result, label) { if (result.sqliteUserEventRowsUpdated) { lines.push(`Updated SQLite user-event flags: ${result.sqliteUserEventRowsUpdated}`); } + lines.push(`In-place rollout updates: ${result.inPlaceSessionFiles ?? 0}`); if (result.sqliteCwdRowsUpdated) { lines.push(`Updated SQLite cwd paths: ${result.sqliteCwdRowsUpdated}`); } @@ -205,6 +214,12 @@ async function main() { } assertSupportedNodeVersion(); + if (flags.fast && !["sync", "switch"].includes(command)) { + throw new Error("--fast is supported only by sync and switch."); + } + if (flags.fast && flags.model !== undefined) { + throw new Error("--fast and --model cannot be combined."); + } if (command === "status") { const { getStatus, renderStatus } = await loadService(); @@ -238,7 +253,8 @@ async function main() { provider: flags.provider, keepCount: parseKeepCount(flags.keep), onProgress: createSyncProgressReporter(), - model: rootModel + model: flags.fast ? null : rootModel, + fast: Boolean(flags.fast) }); console.log(summarizeSync(result, "Synchronized")); return; @@ -252,6 +268,7 @@ async function main() { sqliteHome: flags["sqlite-home"], provider, model: flags.model, + fast: Boolean(flags.fast), keepRootModel: Boolean(flags["keep-root-model"]), keepCount: parseKeepCount(flags.keep), onProgress: createSyncProgressReporter() diff --git a/src/service.js b/src/service.js index 5f91339..fec174d 100644 --- a/src/service.js +++ b/src/service.js @@ -420,10 +420,14 @@ async function runSyncCore({ sqliteBusyTimeoutMs, onProgress, model = null, + fast = false, platform, faultInjector, signal } = {}, { afterBackup } = {}) { + if (typeof fast !== "boolean" || (fast && model !== null)) { + throw new Error("Fast sync preserves historical models; do not supply a model."); + } if (!Number.isInteger(keepCount) || keepCount < 1) { throw new Error(`Invalid automatic keep count: ${keepCount}. Expected an integer greater than or equal to 1.`); } @@ -459,9 +463,11 @@ async function runSyncCore({ encryptedContentCounts, userEventThreadIds, threadCwdById - } = await collectSessionChanges(codexHome, targetProvider, { skipLockedReads: true, targetModel: model }); + } = await collectSessionChanges(codexHome, targetProvider, { skipLockedReads: true, targetModel: model, fast }); const cwdStats = await readThreadCwdStats(storage); - const encryptedContentWarning = buildEncryptedContentWarning(encryptedContentCounts, targetProvider); + const encryptedContentWarning = fast + ? "Fast mode: history models, user-event flags and encrypted content were not checked. Metadata alignment does not guarantee continuation with another provider." + : buildEncryptedContentWarning(encryptedContentCounts, targetProvider); emitProgress(onProgress, { stage: "scan_rollout_files", status: "complete", @@ -501,7 +507,8 @@ async function runSyncCore({ targetProvider, sessionChanges: writableChanges, configPath, - configBackupText + configBackupText, + fast }); backupDurationMs = Date.now() - backupStartedAt; emitProgress(onProgress, { @@ -710,6 +717,8 @@ async function runSyncCore({ backupDir, backupDurationMs, changedSessionFiles: applyResult.appliedChanges, + inPlaceSessionFiles: applyResult.inPlaceChanges ?? 0, + ...(fast ? { scanScope: "metadata", unchecked: ["historyModels", "userEventFlags", "encryptedContent"] } : {}), skippedLockedRolloutFiles, sqliteRowsUpdated: sqliteResult.updatedRows, sqliteProviderRowsUpdated: sqliteResult.providerRowsUpdated, @@ -881,12 +890,17 @@ export async function runSwitch({ provider, model, keepRootModel = false, + fast = false, keepCount = DEFAULT_BACKUP_RETENTION_COUNT, onProgress, platform, faultInjector, signal }) { + if (typeof fast !== "boolean" || (fast && model !== undefined && model !== null)) { + throw new Error("Fast switch preserves root and historical models; --fast and --model cannot be combined."); + } + if (fast) keepRootModel = true; if (!provider) { throw new Error("Missing provider id. Usage: codex-provider switch "); } @@ -950,7 +964,8 @@ export async function runSwitch({ configBackupText: originalConfigText, keepCount, onProgress, - model: modelForThreads, + model: fast ? null : modelForThreads, + fast, faultInjector, signal }, diff --git a/src/session-files.js b/src/session-files.js index 0483373..1467dbd 100644 --- a/src/session-files.js +++ b/src/session-files.js @@ -41,14 +41,14 @@ function wrapRolloutFileBusyError(error, filePath, action) { } async function getFileSnapshot(filePath) { - const stat = await fsp.stat(filePath); - const identity = await fsp.stat(filePath, { bigint: true }); + const stat = await fsp.stat(filePath, { bigint: true }); return { - size: stat.size, - mtimeMs: stat.mtimeMs, - mode: stat.mode, - dev: String(identity.dev), - ino: String(identity.ino) + size: Number(stat.size), + mtimeMs: Number(stat.mtimeNs) / 1e6, + mode: Number(stat.mode), + nlink: Number(stat.nlink), + dev: String(stat.dev), + ino: String(stat.ino) }; } @@ -77,60 +77,6 @@ function incrementPlainCount(counts, directory, provider) { counts[directory][provider] = (counts[directory][provider] ?? 0) + 1; } -function streamContainsText(filePath, text, startOffset) { - const needle = Buffer.from(text); - const safeStartOffset = Math.max(0, startOffset ?? 0); - - return new Promise((resolve, reject) => { - let previous = Buffer.alloc(0); - let settled = false; - const stream = fs.createReadStream(filePath, { - start: safeStartOffset, - highWaterMark: ROLLOUT_SCAN_CHUNK_BYTES - }); - - function settle(value, error) { - if (settled) { - return; - } - settled = true; - if (error) { - reject(wrapRolloutFileBusyError(error, filePath, "scan")); - return; - } - resolve(value); - } - - stream.on("data", (chunk) => { - const buffer = previous.length ? Buffer.concat([previous, chunk]) : chunk; - if (buffer.indexOf(needle) !== -1) { - settle(true); - stream.destroy(); - return; - } - - const keepBytes = Math.max(0, needle.length - 1); - previous = keepBytes > 0 - ? buffer.subarray(Math.max(0, buffer.length - keepBytes)) - : Buffer.alloc(0); - }); - stream.on("end", () => settle(false)); - stream.on("error", (error) => { - if (settled) { - return; - } - settle(false, error); - }); - }); -} - -async function fileHasEncryptedContent(filePath, firstLine, startOffset) { - if (firstLine.includes("encrypted_content")) { - return true; - } - return streamContainsText(filePath, "encrypted_content", startOffset); -} - function recordHasUserEvent(record) { if (!record || typeof record !== "object") { return false; @@ -179,47 +125,6 @@ function toDesktopWorkspacePath(value) { return value; } -async function fileHasUserEvent(filePath, firstLine, startOffset) { - try { - if (recordHasUserEvent(JSON.parse(firstLine))) { - return true; - } - } catch { - // Keep scanning the rest of the rollout below. - } - - const stream = fs.createReadStream(filePath, { - encoding: "utf8", - start: Math.max(0, startOffset ?? 0), - highWaterMark: ROLLOUT_SCAN_CHUNK_BYTES - }); - const lines = readline.createInterface({ - input: stream, - crlfDelay: Infinity - }); - - try { - for await (const line of lines) { - if (!line) { - continue; - } - try { - if (recordHasUserEvent(JSON.parse(line))) { - return true; - } - } catch { - // Ignore malformed non-metadata lines; provider sync only needs positive evidence. - } - } - return false; - } catch (error) { - throw wrapRolloutFileBusyError(error, filePath, "scan"); - } finally { - lines.close(); - stream.destroy(); - } -} - async function listJsonlFiles(rootDir) { const entries = await fsp.readdir(rootDir, { withFileTypes: true }); entries.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0); @@ -237,11 +142,16 @@ async function listJsonlFiles(rootDir) { return files; } -async function readFirstLineRecordFromHandle(handle) { +async function readFirstLineRecordFromHandle(handle, maxBytes = Infinity) { let position = 0; let collected = Buffer.alloc(0); while (true) { - const chunk = Buffer.alloc(64 * 1024); + if (position >= maxBytes) { + const error = new Error("Fast mode requires a session metadata header smaller than 1 MiB; use full sync."); + error.code = "FAST_MODE_UNSUPPORTED"; + throw error; + } + const chunk = Buffer.alloc(Math.min(64 * 1024, maxBytes - position)); const { bytesRead } = await handle.read(chunk, 0, chunk.length, position); if (bytesRead === 0) { break; @@ -266,11 +176,11 @@ async function readFirstLineRecordFromHandle(handle) { }; } -async function readFirstLineRecord(filePath) { +async function readFirstLineRecord(filePath, maxBytes) { let handle; try { handle = await fsp.open(filePath, "r"); - return await readFirstLineRecordFromHandle(handle); + return await readFirstLineRecordFromHandle(handle, maxBytes); } catch (error) { throw wrapRolloutFileBusyError(error, filePath, "read"); } finally { @@ -293,44 +203,24 @@ function parseSessionMetaRecord(firstLine) { } } -// Scan the start of a rollout file looking for the first `turn_context` -// event and return its `payload.model` field. This is the field that the -// Codex GUI bottom-right uses to label old conversations, so we have to -// rewrite it (along with `payload.collaboration_mode.settings.model`) on -// every sync in addition to the per-thread SQLite `model` column. -// -// We stream line-by-line because individual `turn_context` lines can -// easily exceed 64 KB once Codex includes the `developer_instructions` -// blob — the previous code that capped the read at 64 KB silently -// missed those, which made the rollout model rewrite a no-op for -// sessions whose first turn was a long planning step. We stop as -// soon as we find a `turn_context` line, so the scan is O(1) for the -// common case and we never load multi-MB rollouts into memory just -// to read a header. -// -// For each line we find, we do a regex on the raw text instead of -// `JSON.parse`-ing the entire payload: Codex writes opaque multi-KB -// strings (`developer_instructions`, raw tool output, …) into the -// payload, and round-tripping those through `JSON.parse` -> `JSON.stringify` -// would silently mangle embedded escape sequences. Anchoring on -// `"type":"turn_context"` and grabbing the first `"model":""` -// that follows is enough for the first `turn_context` of the file, -// because rollout lines are single JSON objects. +// One streaming pass supplies all body-dependent diagnostics and model undo +// evidence. Keep the existing detection rules; never reserialize message data. const ROLLOUT_TURNCONTEXT_TYPE_RE = /"type"\s*:\s*"turn_context"/; -async function readTurnContextModelSnapshot( +async function scanRolloutBody( rolloutPath, - { firstLineOffset, firstLineLength, targetModel = null } = {} + { firstLine, firstLineLength, targetModel = null } = {} ) { - const headerSkip = Math.max(0, firstLineOffset ?? 0); const headerLength = Math.max(0, firstLineLength ?? 0); const models = []; const originalTurnContextModels = []; + let hasEncryptedContent = firstLine.includes("encrypted_content"); + let hasUserEvent = recordHasUserEvent(JSON.parse(firstLine)); let lineIndex = 0; const stream = fs.createReadStream(rolloutPath, { encoding: "utf8", - start: headerSkip + headerLength, + start: headerLength, highWaterMark: ROLLOUT_SCAN_CHUNK_BYTES }); const lines = readline.createInterface({ @@ -341,6 +231,11 @@ async function readTurnContextModelSnapshot( try { for await (const line of lines) { lineIndex += 1; + hasEncryptedContent ||= line.includes("encrypted_content"); + if (!hasUserEvent) { + try { hasUserEvent = recordHasUserEvent(JSON.parse(line)); } + catch { /* Malformed body lines provide no positive user-event evidence. */ } + } if (!line.includes('"turn_context"')) { continue; } @@ -368,7 +263,7 @@ async function readTurnContextModelSnapshot( } } } - return { models, originalTurnContextModels }; + return { models, originalTurnContextModels, hasEncryptedContent, hasUserEvent }; } catch (error) { throw wrapRolloutFileBusyError(error, rolloutPath, "read"); } finally { @@ -501,9 +396,8 @@ const SAFE_IN_PLACE_PROVIDER_ID_RE = /^[A-Za-z0-9._-]+$/; const PROVIDER_MUTATION_STRATEGY = "provider_bytes_in_place"; function getInPlaceProviderMutation(change) { - // POSIX pilot: Windows keeps its existing exclusive replacement worker - // until its file-ID and crash-recovery implementation is platform-tested. - if (process.platform === "win32" || !change + if (!change + || (change.originalNlink !== undefined && change.originalNlink !== 1) || change.modelRewriteRequired || change.modelOnlyChange || typeof change.originalFirstLine !== "string" @@ -657,25 +551,20 @@ async function writeBytesFully(handle, bytes, position, writeImpl) { async function finishInPlaceWrite(handle, entry, expectedBytes, options = {}) { const mutation = entry.mutation ?? entry.inPlaceMutation; await (options.inPlaceSync ?? ((h) => h.sync()))(handle); - const actual = await readBytesFully(handle, expectedBytes.length, mutation.byteOffset); - if (!actual?.equals(expectedBytes)) { + const expected = Buffer.from(entry.originalFirstLine + entry.originalSeparator, "utf8"); + expectedBytes.copy(expected, mutation.byteOffset); + const actual = await readBytesFully(handle, expected.length, 0); + if (!actual?.equals(expected) || (await handle.stat()).size < mutation.originalSize) { throw new Error(`Provider in-place write verification failed: ${entry.path}`); } - const stat = await handle.stat(); await assertInPlaceIdentity(handle, entry.path, mutation); - // Never reset an append-only writer's new timestamp or truncate its tail. - if (stat.size === mutation.originalSize) { - await handle.utimes(stat.atime, mutation.originalMtimeMs / 1000); - await handle.sync(); - } else if (options.previousStat?.size === stat.size) { - await handle.utimes(stat.atime, options.previousStat.mtimeMs / 1000); - await handle.sync(); - } + // POSIX has no exclusive handle here. Even stat followed by utimes races an + // append, so retain the actual write time instead of overwriting newer mtime. } async function assertInPlaceIdentity(handle, filePath, mutation) { const [opened, current] = await Promise.all([handle.stat({ bigint: true }), fsp.lstat(filePath, { bigint: true })]); - if (!current.isFile() || current.isSymbolicLink() + if (!current.isFile() || current.isSymbolicLink() || opened.nlink !== 1n || String(opened.dev) !== mutation.originalDev || String(opened.ino) !== mutation.originalIno || opened.dev !== current.dev || opened.ino !== current.ino) { throw new Error(`Rollout identity changed before provider byte access: ${filePath}`); @@ -699,7 +588,7 @@ function isRecoverableProviderBytes(current, original, replacement) { return true; } -async function restoreProviderOnHandle(handle, entry, options = {}) { +async function inspectProviderRecovery(handle, entry) { const mutation = entry.mutation ?? entry.inPlaceMutation; const { originalBytes, replacementBytes } = validateProviderMutationDescriptor( mutation, entry.path, entry.originalFirstLine, entry.originalSeparator); @@ -717,10 +606,22 @@ async function restoreProviderOnHandle(handle, entry, options = {}) { || !isRecoverableProviderBytes(current, originalBytes, replacementBytes)) { throw new Error(`Unknown rollout bytes during provider recovery: ${entry.path}`); } + return { current, originalBytes }; +} + +export async function validateProviderByteRestore(entry) { + const handle = await fsp.open(entry.path, "r"); + try { await inspectProviderRecovery(handle, entry); } + finally { await handle.close(); } +} + +async function restoreProviderOnHandle(handle, entry, options = {}) { + const mutation = entry.mutation ?? entry.inPlaceMutation; + const { current, originalBytes } = await inspectProviderRecovery(handle, entry); if (!current.equals(originalBytes)) { await writeBytesFully(handle, originalBytes, mutation.byteOffset, options.inPlaceRestoreWrite ?? defaultInPlaceWrite); } - await finishInPlaceWrite(handle, entry, originalBytes, { previousStat: stat }); + await finishInPlaceWrite(handle, entry, originalBytes); } async function tryRewriteProviderInPlace(change, options = {}) { @@ -736,11 +637,10 @@ async function tryRewriteProviderInPlace(change, options = {}) { return "SKIP_CHANGED"; } handle = await fsp.open(change.path, "r+"); - const stat = await handle.stat(); const identity = await handle.stat({ bigint: true }); const snapshot = { - size: stat.size, - mtimeMs: stat.mtimeMs, + size: Number(identity.size), + mtimeMs: Number(identity.mtimeNs) / 1e6, dev: String(identity.dev), ino: String(identity.ino) }; @@ -817,6 +717,10 @@ const WINDOWS_EXCLUSIVE_REWRITE_WORKER_SCRIPT = ` [Console]::InputEncoding = $utf8 [Console]::OutputEncoding = $utf8 + Add-Type -TypeDefinition @' +${fs.readFileSync(new URL("./windows-provider-bytes.cs", import.meta.url), "utf8")} +'@ + function Write-ProtocolMessage($value) { $json = $value | ConvertTo-Json -Compress -Depth 8 [Console]::Out.WriteLine($json) @@ -874,6 +778,16 @@ const WINDOWS_EXCLUSIVE_REWRITE_WORKER_SCRIPT = ` return "SKIP_CHANGED" } + if ($null -ne $change.inPlaceMutation) { + $m = $change.inPlaceMutation + $header = $encoding.GetBytes([string]$change.originalFirstLine + [string]$change.originalSeparator) + return [ProviderByteFile]::Apply($source, $header, + [Convert]::FromBase64String([string]$m.originalBase64), + [Convert]::FromBase64String([string]$m.replacementBase64), + [int]$m.byteOffset, [long]$m.originalSize, [double]$m.originalMtimeMs, + [string]$m.originalDev, [string]$m.originalIno, [bool]$change.restoreProviderBytes) + } + if ([bool]$change.requireOriginalMatch) { if ($source.Length -ne [int64]$change.originalSize) { return "SKIP_CHANGED" @@ -1119,6 +1033,10 @@ export async function createWindowsExclusiveRewriteWorker(options = {}) { if (!change || typeof change.path !== "string" || !path.isAbsolute(change.path)) { throw new Error(`Windows rewrite worker requires an absolute rollout path: ${change?.path ?? "(missing)"}`); } + if (change.inPlaceMutation) { + validateProviderMutationDescriptor(change.inPlaceMutation, change.path, + change.originalFirstLine, change.originalSeparator); + } const id = nextRequestId; nextRequestId += 1; @@ -1136,7 +1054,8 @@ export async function createWindowsExclusiveRewriteWorker(options = {}) { || response?.type !== "result" || response?.id !== id || response?.path !== change.path - || !isValidWindowsRewriteResult(response?.result)) { + || !isValidWindowsRewriteResult(response?.result) + || (change.inPlaceMutation && response.result === "APPLIED")) { throw new Error(`Unexpected Windows rewrite worker response for ${change.path}: ${JSON.stringify(response)}`); } return response.result; @@ -1521,16 +1440,20 @@ async function findLockedFilesOnWindows(filePaths) { export async function collectSessionChanges(codexHome, targetProvider, options = {}) { const { skipLockedReads = false, - targetModel = null + targetModel = null, + fast = false } = options; + if (typeof fast !== "boolean" || (fast && targetModel !== null)) { + throw new Error("Fast mode requires a boolean fast option and no historical model rewrite."); + } const summaries = []; const lockedPaths = []; const providerCounts = { sessions: new Map(), archived_sessions: new Map() }; - const encryptedContentCounts = emptyEncryptedContentCounts(); - const userEventThreadIds = new Set(); + const encryptedContentCounts = fast ? null : emptyEncryptedContentCounts(); + const userEventThreadIds = fast ? null : new Set(); const threadCwdById = new Map(); for (const dirName of SESSION_DIRS) { @@ -1546,7 +1469,7 @@ export async function collectSessionChanges(codexHome, targetProvider, options = let scanStart; try { scanStart = await getFileSnapshot(rolloutPath); - record = await readFirstLineRecord(rolloutPath); + record = await readFirstLineRecord(rolloutPath, fast ? 1024 * 1024 : undefined); } catch (error) { if (skipLockedReads && isRolloutFileBusyError(error)) { lockedPaths.push(rolloutPath); @@ -1556,6 +1479,11 @@ export async function collectSessionChanges(codexHome, targetProvider, options = } const parsed = parseSessionMetaRecord(record.firstLine); if (!parsed) { + if (fast) { + const error = new Error(`Fast mode cannot validate session metadata: ${rolloutPath}`); + error.code = "FAST_MODE_UNSUPPORTED"; + throw error; + } continue; } const currentProvider = parsed.payload.model_provider ?? "(missing)"; @@ -1566,11 +1494,15 @@ export async function collectSessionChanges(codexHome, targetProvider, options = && parsed.payload.cwd.trim()) { threadCwdById.set(parsed.payload.id, toDesktopWorkspacePath(parsed.payload.cwd)); } + let modelSnapshot = { models: [], originalTurnContextModels: [] }; try { - if (await fileHasEncryptedContent(rolloutPath, record.firstLine, record.offset)) { + if (!fast) modelSnapshot = await scanRolloutBody(rolloutPath, { + firstLine: record.firstLine, firstLineLength: record.offset, targetModel + }); + if (modelSnapshot.hasEncryptedContent) { incrementPlainCount(encryptedContentCounts, dirName, currentProvider); } - if (parsed.payload.id && await fileHasUserEvent(rolloutPath, record.firstLine, record.offset)) { + if (parsed.payload.id && modelSnapshot.hasUserEvent) { userEventThreadIds.add(parsed.payload.id); } } catch (error) { @@ -1581,16 +1513,6 @@ export async function collectSessionChanges(codexHome, targetProvider, options = throw error; } - // Peek at the first `turn_context` event to capture the - // per-turn model that the Codex GUI bottom-right reads. We - // keep this on the summary so the rewrite step knows what - // value to swap out, without making collectSessionChanges - // require a target model. - const modelSnapshot = await readTurnContextModelSnapshot(rolloutPath, { - firstLineOffset: 0, - firstLineLength: record.offset, - targetModel - }); const currentModels = modelSnapshot.models; const originalModel = currentModels[0] ?? null; @@ -1627,6 +1549,7 @@ export async function collectSessionChanges(codexHome, targetProvider, options = originalMtimeMs: snapshot.mtimeMs, originalDev: snapshot.dev, originalIno: snapshot.ino, + originalNlink: snapshot.nlink, originalProvider: currentProvider, updatedProvider: targetProvider, originalModel, @@ -1636,6 +1559,11 @@ export async function collectSessionChanges(codexHome, targetProvider, options = updatedFirstLine: providerChanged ? JSON.stringify(parsed) : record.firstLine }; change.inPlaceMutation = getInPlaceProviderMutation(change); + if (fast && !change.inPlaceMutation) { + const error = new Error(`Fast mode requires an unambiguous equal-length provider byte replacement: ${rolloutPath}. Run full sync explicitly for this file.`); + error.code = "FAST_MODE_UNSUPPORTED"; + throw error; + } summaries.push(change); } } @@ -1863,21 +1791,38 @@ export async function restoreSessionChanges(manifestEntries, options = {}) { const restoredPaths = []; const failures = []; + let windowsWorker = null; + async function restoreWindows(change) { + windowsWorker ??= await (options.windowsRewriteWorkerFactory ?? createWindowsExclusiveRewriteWorker)(); + try { + return await windowsWorker.rewrite(change, { requireOriginalMatch: false }); + } catch (error) { + await windowsWorker.close().catch(() => {}); + windowsWorker = null; + throw error; + } + } for (const entry of manifestEntries) { try { await options.onBeforeRestore?.(entry); if (entry.mutation) { validateProviderMutationDescriptor(entry.mutation, entry.path, entry.originalFirstLine, entry.originalSeparator); - if (process.platform === "win32") throw new Error("POSIX provider-byte backups require recovery on their original POSIX host."); - await restoreProviderBytesInPlace(entry, options); + if (process.platform === "win32") { + const result = await restoreWindows({ + ...entry, inPlaceMutation: entry.mutation, restoreProviderBytes: true + }); + if (result !== "APPLIED_IN_PLACE") throw new Error(`Provider byte recovery failed: ${result}`); + } else { + await restoreProviderBytesInPlace(entry, options); + } } else if (!entry.modelOnlyChange) { if (process.platform === "win32") { - const [result] = await invokeWindowsExclusiveRewriteBatch([{ + const result = await restoreWindows({ path: entry.path, separator: entry.originalSeparator ?? "\n", updatedFirstLine: entry.originalFirstLine, originalMtimeMs: entry.originalMtimeMs - }], { requireOriginalMatch: false }); + }); if (result !== "APPLIED") { throw new Error( `Unable to rewrite rollout file because it is currently in use. Close Codex and the Codex app, then retry. Locked file: ${entry.path}` @@ -1908,6 +1853,11 @@ export async function restoreSessionChanges(manifestEntries, options = {}) { } } + if (windowsWorker) { + try { await windowsWorker.close(); } + catch (error) { failures.push(error); } + } + if (failures.length > 0) { const aggregate = new AggregateError( failures, diff --git a/src/windows-provider-bytes.cs b/src/windows-provider-bytes.cs new file mode 100644 index 0000000..6dc5cca --- /dev/null +++ b/src/windows-provider-bytes.cs @@ -0,0 +1,134 @@ +using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; + +// Loaded by the existing exclusive PowerShell worker, not a second sync engine. +public static class ProviderByteFile +{ + [StructLayout(LayoutKind.Sequential, Pack = 4)] + struct Info + { + public uint Attributes; + public long CreationTime, AccessTime, WriteTime; + public uint Volume, SizeHigh, SizeLow, Links, IndexHigh, IndexLow; + } + + [DllImport("kernel32.dll", SetLastError = true)] + static extern bool GetFileInformationByHandle(IntPtr handle, out Info info); + + [DllImport("kernel32.dll", SetLastError = true)] + static extern bool SetFileTime(IntPtr handle, IntPtr creation, IntPtr access, ref long write); + + static Info Inspect(FileStream stream, string dev, string ino) + { + Info info; + if (!GetFileInformationByHandle(stream.SafeFileHandle.DangerousGetHandle(), out info)) + throw new Win32Exception(Marshal.GetLastWin32Error()); + ulong index = ((ulong)info.IndexHigh << 32) | info.IndexLow; + if (info.Volume.ToString() != dev || index.ToString() != ino || info.Links != 1 + || (info.Attributes & (0x400u | 0x10u)) != 0) + throw new IOException("Rollout identity changed before provider byte access."); + return info; + } + + static byte[] Read(FileStream stream, int count) + { + var bytes = new byte[count]; + stream.Position = 0; + for (int offset = 0; offset < count;) + { + int n = stream.Read(bytes, offset, count - offset); + if (n == 0) throw new IOException("Rollout header was truncated."); + offset += n; + } + return bytes; + } + + static bool Equal(byte[] a, byte[] b) + { + if (a.Length != b.Length) return false; + for (int i = 0; i < a.Length; i++) if (a[i] != b[i]) return false; + return true; + } + + static bool Recoverable(byte[] current, byte[] header, byte[] oldBytes, byte[] newBytes, int offset) + { + int phase = 0; + for (int i = 0; i < header.Length; i++) + { + int j = i - offset; + if (j < 0 || j >= oldBytes.Length || oldBytes[j] == newBytes[j]) + { + if (current[i] != header[i]) return false; + } + else if (current[i] == newBytes[j]) + { + if (phase == 2) return false; + phase = 1; + } + else if (current[i] == oldBytes[j]) + { + if (phase == 1) phase = 2; + } + else return false; + } + return true; + } + + static void Write(FileStream stream, byte[] bytes, int offset, byte[] expected, + long size, string dev, string ino, long mtime) + { + stream.Position = offset; + stream.Write(bytes, 0, bytes.Length); + stream.Flush(true); + if (stream.Length < size || !Equal(Read(stream, expected.Length), expected)) + throw new IOException("Provider byte write verification failed."); + Inspect(stream, dev, ino); + if (!SetFileTime(stream.SafeFileHandle.DangerousGetHandle(), IntPtr.Zero, IntPtr.Zero, ref mtime)) + throw new Win32Exception(Marshal.GetLastWin32Error()); + stream.Flush(true); + } + + public static string Apply(FileStream stream, byte[] header, byte[] oldBytes, byte[] newBytes, + int offset, long size, double mtimeMs, string dev, string ino, bool restore) + { + var before = Inspect(stream, dev, ino); + if (stream.Length < size) throw new IOException("Rollout truncated before provider byte access."); + var current = Read(stream, header.Length); + var expected = (byte[])header.Clone(); + Array.Copy(newBytes, 0, expected, offset, newBytes.Length); + // FILETIME and libuv's Unix timestamp have different epochs. + double currentMs = (before.WriteTime - 116444736000000000L) / 10000.0; + if (!restore && (stream.Length != size || Math.Abs(currentMs - mtimeMs) > 0.001 + || !Equal(current, header))) return "SKIP_CHANGED"; + if (restore) + { + if (!Recoverable(current, header, oldBytes, newBytes, offset)) + throw new IOException("Unknown rollout bytes during provider recovery."); + if (Equal(current, header)) return "APPLIED_IN_PLACE"; + Write(stream, oldBytes, offset, header, size, dev, ino, before.WriteTime); + return "APPLIED_IN_PLACE"; + } + try + { + Write(stream, newBytes, offset, expected, size, dev, ino, before.WriteTime); + } + catch (Exception failure) + { + try + { + Inspect(stream, dev, ino); + if (stream.Length < size || !Recoverable(Read(stream, header.Length), header, oldBytes, newBytes, offset)) + throw new IOException("Cannot verify bytes for immediate provider recovery."); + Write(stream, oldBytes, offset, header, size, dev, ino, before.WriteTime); + } + catch (Exception recovery) + { + throw new AggregateException("Provider write and immediate recovery failed.", failure, recovery); + } + throw; + } + return "APPLIED_IN_PLACE"; + } +} diff --git a/test/fast-sync.test.js b/test/fast-sync.test.js new file mode 100644 index 0000000..eddd330 --- /dev/null +++ b/test/fast-sync.test.js @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test, { afterEach } from "node:test"; +import { fileURLToPath } from "node:url"; +import { collectSessionChanges } from "../src/session-files.js"; +import { runRestore, runSwitch, runSync } from "../src/service.js"; +import { openDatabase } from "../src/sqlite.js"; +import { findPendingTransactions } from "../src/transaction-journal.js"; + +const cli = fileURLToPath(new URL("../src/cli.js", import.meta.url)); +const cleanups = []; +afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); }); + +async function fixture(t, provider = "openai") { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "provider-fast-")); + cleanups.push(() => fs.rm(home, { recursive: true, force: true })); + await fs.mkdir(path.join(home, "sessions")); + await fs.mkdir(path.join(home, "sqlite")); + const config = 'model_provider = "openai"\nmodel = "root-model"\n[model_providers.prov_a]\nmodel = "provider-model"\n'; + await fs.writeFile(path.join(home, "config.toml"), config); + const file = path.join(home, "sessions", "rollout-test.jsonl"); + const header = JSON.stringify({ type: "session_meta", payload: { id: "test", cwd: "/workspace/test", model_provider: provider } }); + const body = [ + { type: "turn_context", payload: { model: "history-model" } }, + { type: "event_msg", payload: { type: "user_message", message: "fixture" } }, + { type: "response_item", payload: { encrypted_content: "fixture-not-a-secret" } } + ].map(JSON.stringify).join("\n") + "\n"; + await fs.writeFile(file, header + "\n" + body); + const dbPath = path.join(home, "sqlite", "state_5.sqlite"); + const db = await openDatabase(dbPath); + db.exec(`CREATE TABLE threads (id TEXT PRIMARY KEY, model_provider TEXT, model TEXT, + cwd TEXT, archived INTEGER DEFAULT 0, has_user_event INTEGER DEFAULT 0, + first_user_message TEXT DEFAULT '', updated_at INTEGER DEFAULT 123); + INSERT INTO threads (id, model_provider, model, cwd) VALUES ('test', 'openai', 'history-model', '/old');`); + db.close(); + return { home, file, body, config, dbPath }; +} + +async function row(f) { + const db = await openDatabase(f.dbPath); + try { return { ...db.prepare("SELECT * FROM threads WHERE id = 'test'").get() }; } + finally { db.close(); } +} + +test("fast switch/restore never open a rollout body stream and preserve models", async (t) => { + const f = await fixture(t); + const before = await fs.stat(f.file); + const original = await fs.readFile(f.file); + const originalRow = await row(f); + const createStream = fsSync.createReadStream; + const readFile = fs.readFile; + let guarded = true; + fs.readFile = function (file, ...args) { + if (guarded) assert.notEqual(String(file), f.file, "fast operation must not read the entire rollout"); + return readFile.call(this, file, ...args); + }; + fsSync.createReadStream = function (file, ...args) { + assert.notEqual(String(file), f.file, "fast operation must not scan the rollout body"); + return createStream.call(this, file, ...args); + }; + cleanups.push(() => { fsSync.createReadStream = createStream; fs.readFile = readFile; }); + const result = await runSwitch({ codexHome: f.home, provider: "prov_a", fast: true }); + guarded = false; + assert.equal(result.inPlaceSessionFiles, 1); + assert.equal(result.scanScope, "metadata"); + assert.equal(result.encryptedContentCounts, null); + assert.deepEqual(result.unchecked, ["historyModels", "userEventFlags", "encryptedContent"]); + assert.match(result.encryptedContentWarning, /not checked/); + assert.equal((await fs.stat(f.file)).ino, before.ino); + assert.equal(await fs.readFile(f.file, "utf8"), original.toString().replace("openai", "prov_a")); + assert.deepEqual(await row(f), { ...originalRow, model_provider: "prov_a", cwd: "/workspace/test" }); + assert.equal(await fs.readFile(path.join(f.home, "config.toml"), "utf8"), f.config.replace('"openai"', '"prov_a"')); + const metadata = JSON.parse(await fs.readFile(path.join(result.backupDir, "metadata.json"))); + assert.equal(metadata.version, 3); + assert.equal(metadata.scanScope, "metadata"); + guarded = true; + await runRestore({ codexHome: f.home, backupDir: result.backupDir }); + guarded = false; + assert.deepEqual(await fs.readFile(f.file), original); + assert.deepEqual(await row(f), originalRow); + assert.equal((await fs.stat(f.file)).ino, before.ino); +}); + +test("full scan reads each body once and retains all three diagnostics", async (t) => { + const f = await fixture(t); + const createStream = fsSync.createReadStream; + let streams = 0; + fsSync.createReadStream = function (file, ...args) { + if (String(file) === f.file) streams++; + return createStream.call(this, file, ...args); + }; + cleanups.push(() => { fsSync.createReadStream = createStream; }); + const scan = await collectSessionChanges(f.home, "prov_a", { targetModel: "new-model" }); + assert.equal(streams, 1); + assert.equal(scan.encryptedContentCounts.sessions.openai, 1); + assert.ok(scan.userEventThreadIds.has("test")); + assert.equal(scan.changes[0].modelRewriteRequired, true); + assert.equal(scan.changes[0].originalTurnContextModels[0].originalModel, "history-model"); + assert.equal(scan.changes[0].inPlaceMutation, null); +}); + +test("full mode still repairs models and user-event flags", async (t) => { + const f = await fixture(t); + const result = await runSwitch({ codexHome: f.home, provider: "prov_a" }); + assert.equal(result.inPlaceSessionFiles, 0); + assert.equal((await row(f)).model, "provider-model"); + assert.equal((await row(f)).has_user_event, 1); + assert.equal((await row(f)).updated_at, 123); + assert.match(await fs.readFile(f.file, "utf8"), /"model":"provider-model"/); +}); + +test("fast preflight rejects ineligible files and model intents before any mutation", async (t) => { + for (const kind of ["length", "duplicate", "oversized", "model"]) { + const f = await fixture(t, kind === "length" ? "provider_old" : "openai"); + if (kind === "duplicate") { + const text = (await fs.readFile(f.file, "utf8")).replace('"model_provider":"openai"', '"model_provider":"openai","model_provider":"openai"'); + await fs.writeFile(f.file, text); + } + if (kind === "oversized") await fs.writeFile(f.file, "x".repeat(2 * 1024 * 1024)); + const original = await fs.readFile(f.file); + const before = await row(f); + await assert.rejects(runSwitch({ codexHome: f.home, provider: "prov_a", fast: true, + ...(kind === "model" ? { model: "other" } : {}) })); + assert.deepEqual(await fs.readFile(f.file), original); + assert.deepEqual(await row(f), before); + assert.equal(await fs.readFile(path.join(f.home, "config.toml"), "utf8"), f.config); + assert.deepEqual(await findPendingTransactions(f.home), []); + await assert.rejects(fs.access(path.join(f.home, "backups_state")), { code: "ENOENT" }); + } +}); + +test("fast transaction restores config, database and bytes after mutation-before-applied", async (t) => { + const f = await fixture(t); + const original = await fs.readFile(f.file); + const before = await row(f); + await assert.rejects(runSwitch({ codexHome: f.home, provider: "prov_a", fast: true, + faultInjector({ point }) { if (point === "after_rollout_mutation_before_applied") throw new Error("fault"); } + }), { code: "SYNC_FAILED_ROLLED_BACK" }); + assert.deepEqual(await fs.readFile(f.file), original); + assert.deepEqual(await row(f), before); + assert.equal(await fs.readFile(path.join(f.home, "config.toml"), "utf8"), f.config); +}); + +test("CLI --fast is a flag in either position and explicitly preserves models", async (t) => { + for (const args of [["switch", "--fast", "prov_a"], ["switch", "prov_a", "--fast"], ["sync", "--provider", "prov_a", "--fast"]]) { + const f = await fixture(t); + const child = spawnSync(process.execPath, [cli, ...args, "--codex-home", f.home], { + encoding: "utf8", env: { ...process.env, CODEX_SQLITE_HOME: "" } + }); + assert.equal(child.status, 0, child.stderr); + assert.match(child.stdout, /In-place rollout updates: 1/); + assert.match(child.stdout, /not checked/); + assert.equal((await row(f)).model, "history-model"); + } + for (const args of [["status", "--fast"], ["sync", "--fast=false"], ["sync", "--fast", "--model", "other"]]) { + const f = await fixture(t); + const child = spawnSync(process.execPath, [cli, ...args, "--codex-home", f.home], { encoding: "utf8" }); + assert.equal(child.status, 1); + assert.match(child.stderr, /--fast/); + } +}); + +test("fast sync rejects a model at the service boundary", async (t) => { + const f = await fixture(t); + await assert.rejects(runSync({ codexHome: f.home, fast: true, model: "other" }), /preserves historical models/); +}); + +test("manual restore preflights unknown provider bytes before changing config or SQLite", async (t) => { + const f = await fixture(t); + const result = await runSwitch({ codexHome: f.home, provider: "prov_a", fast: true }); + const manifest = JSON.parse(await fs.readFile(path.join(result.backupDir, "session-meta-backup.json"))); + const h = await fs.open(f.file, "r+"); + try { await h.write(Buffer.from("!"), 0, 1, manifest.files[0].mutation.byteOffset + 1); } + finally { await h.close(); } + const config = await fs.readFile(path.join(f.home, "config.toml")); + const before = await row(f); + await assert.rejects(runRestore({ codexHome: f.home, backupDir: result.backupDir }), /Unknown rollout bytes/); + assert.deepEqual(await fs.readFile(path.join(f.home, "config.toml")), config); + assert.deepEqual(await row(f), before); +}); diff --git a/test/in-place-transaction.test.js b/test/in-place-transaction.test.js index b8cbaf0..415206f 100644 --- a/test/in-place-transaction.test.js +++ b/test/in-place-transaction.test.js @@ -4,7 +4,7 @@ import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import test from "node:test"; +import test, { afterEach } from "node:test"; import { performance } from "node:perf_hooks"; import { fileURLToPath } from "node:url"; import { applySessionChanges, collectSessionChanges, restoreSessionChanges } from "../src/session-files.js"; @@ -13,13 +13,15 @@ import { runRestore, runSync } from "../src/service.js"; import { TransactionJournal, readTransactionJournal, findPendingTransactions } from "../src/transaction-journal.js"; const repo = fileURLToPath(new URL("..", import.meta.url)); +const cleanups = []; +afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); }); const posix = { skip: process.platform === "win32" }; const header = '{"type":"session_meta","payload":{"id":"fixture","cwd":"\u4e2d\u6587","model_provider" : "openai"}}'; const tail = '\n{"type":"event_msg","payload":{"type":"user_message","message":"fixture"}}\n'; async function fixture(t, line = header, suffix = tail) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "provider-in-place-")); - t.after(() => fs.rm(root, { recursive: true, force: true })); + cleanups.push(() => fs.rm(root, { recursive: true, force: true })); const codexHome = path.join(root, "codex"); await fs.mkdir(path.join(codexHome, "sessions"), { recursive: true }); const file = path.join(codexHome, "sessions", "rollout-fixture.jsonl"); @@ -75,7 +77,7 @@ test("provider-looking text inside a string is not a duplicate field", posix, as assert.ok(changes[0].inPlaceMutation); }); -test("short writes loop to completion, preserve inode, bytes, size and mtime", posix, async (t) => { +test("short writes loop to completion, preserve inode/size and retain actual write time", posix, async (t) => { const f = await fixture(t); const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); const before = await fs.stat(f.file); @@ -86,7 +88,7 @@ test("short writes loop to completion, preserve inode, bytes, size and mtime", p const after = await fs.stat(f.file); assert.equal(after.ino, before.ino); assert.equal(after.size, before.size); - assert.equal(Math.round(after.mtimeMs), f.mtime.getTime()); + assert.ok(after.mtimeMs > f.mtime.getTime()); assert.equal(await fs.readFile(f.file, "utf8"), f.original.toString().replace('"openai"', '"prov_a"')); }); @@ -168,11 +170,11 @@ test("pre-write replaced path or append is skipped without fallback", posix, asy } }); -test("active fd appends remain visible and rollback preserves appended tail and mtime", posix, async (t) => { +test("active fd appends remain visible and rollback never backdates the appended tail", posix, async (t) => { const f = await fixture(t); const { changes, entry } = await prepare(f); const writer = await fs.open(f.file, "a"); - t.after(() => writer.close()); + cleanups.push(() => writer.close()); const before = await writer.stat(); await applySessionChanges(changes); await writer.write(tail); @@ -180,10 +182,51 @@ test("active fd appends remain visible and rollback preserves appended tail and await restoreSessionChanges([entry]); const after = await fs.stat(f.file); assert.equal(after.ino, before.ino); - assert.ok(Math.abs(after.mtimeMs - appended.mtimeMs) < 0.01); + assert.ok(after.mtimeMs >= appended.mtimeMs); assert.equal(await fs.readFile(f.file, "utf8"), f.original + tail); }); +test("write-time truncation or surrounding-header edits cannot report success", posix, async (t) => { + for (const kind of ["truncate", "header"]) { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + await assert.rejects(applySessionChanges(changes, { + async inPlaceWrite(h, b, o, n, p) { + if (kind === "truncate") await h.truncate(p); + else await h.write(Buffer.from("!"), 0, 1, 0); + return h.write(b, o, n, p); + } + }), { code: "IN_PLACE_RESTORE_FAILED" }); + } +}); + +test("append between patch and fsync remains visible and never calls utimes", posix, async (t) => { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + const writer = await fs.open(f.file, "a"); + cleanups.push(() => writer.close()); + let appended; + await applySessionChanges(changes, { + async inPlaceSync(h) { + h.utimes = () => { throw new Error("in-place must not backdate an unlocked file"); }; + await writer.write(tail); + appended = await writer.stat(); + await h.sync(); + } + }); + assert.equal((await fs.stat(f.file)).mtimeMs, appended.mtimeMs); + assert.equal(await fs.readFile(f.file, "utf8"), f.original.toString().replace("openai", "prov_a") + tail); +}); + +test("hardlinked files are not eligible and late links prevent byte mutation", posix, async (t) => { + const f = await fixture(t); + const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); + await fs.link(f.file, f.file + ".link"); + assert.equal((await collectSessionChanges(f.codexHome, "prov_a")).changes[0].inPlaceMutation, null); + assert.equal((await applySessionChanges(changes)).appliedChanges, 0); + assert.deepEqual(await fs.readFile(f.file), f.original); +}); + test("durable manifest and applying precede mutation; observer failure rolls back without rename", posix, async (t) => { const f = await fixture(t); const before = await fs.stat(f.file); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 3bc2697..8ca3a0d 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -1703,7 +1703,7 @@ test("runSync rewrites rollout files and sqlite, then restore reverts both", asy assert.deepEqual(syncResult.skippedLockedRolloutFiles, []); assert.equal(syncResult.sqliteRowsUpdated, 2); const backupMetadata = JSON.parse(await fs.readFile(path.join(syncResult.backupDir, "metadata.json"), "utf8")); - assert.equal(backupMetadata.version, process.platform === "win32" ? 2 : 3); + assert.equal(backupMetadata.version, 3); assert.equal(backupMetadata.sqliteHome, path.join(codexHome, SQLITE_DIR_BASENAME)); assert.deepEqual(backupMetadata.sqliteDbFiles, [DB_FILE_BASENAME]); assert.ok(Number.isSafeInteger(backupMetadata.sizeBytes)); @@ -1835,7 +1835,7 @@ test("runSync uses an explicit SQLite home and never touches a stale Codex Home } const metadata = JSON.parse(await fs.readFile(path.join(result.backupDir, "metadata.json"), "utf8")); - assert.equal(metadata.version, process.platform === "win32" ? 2 : 3); + assert.equal(metadata.version, 3); assert.equal(metadata.sqliteHome, sqliteHome); assert.deepEqual(metadata.dbFiles, []); assert.deepEqual(metadata.sqliteDbFiles, [DB_FILE_BASENAME]); @@ -3286,12 +3286,11 @@ test("applySessionChanges updates equal-length provider IDs without replacing th const rollout = await fs.readFile(sessionPath, "utf8"); assert.equal(result.appliedChanges, 1); - assert.equal(result.inPlaceChanges, process.platform === "win32" ? 0 : 1); - if (process.platform !== "win32") { - assert.equal(after.ino, before.ino); - assert.equal(after.size, before.size); - } - assert.equal(Math.round(after.mtimeMs), originalTime.getTime()); + assert.equal(result.inPlaceChanges, 1); + assert.equal(after.ino, before.ino); + assert.equal(after.size, before.size); + if (process.platform === "win32") assert.equal(Math.round(after.mtimeMs), originalTime.getTime()); + else assert.ok(after.mtimeMs > originalTime.getTime()); const firstNewline = rollout.indexOf("\n"); assert.equal(JSON.parse(rollout.slice(0, firstNewline)).payload.model_provider, "prov_a"); assert.equal(rollout.slice(firstNewline + 1), original.slice(original.indexOf("\n") + 1)); diff --git a/test/windows-provider-bytes.ps1 b/test/windows-provider-bytes.ps1 new file mode 100644 index 0000000..0ccd8d3 --- /dev/null +++ b/test/windows-provider-bytes.ps1 @@ -0,0 +1,164 @@ +param([string]$Source = "$PSScriptRoot/../src/windows-provider-bytes.cs", [string]$WorkerScript) +$ErrorActionPreference = "Stop" +Add-Type -Path $Source +Add-Type -TypeDefinition @' +using System; +using System.IO; +public sealed class FaultingProviderStream : FileStream { + readonly string kind; + int writes, syncs; + bool wrote; + public FaultingProviderStream(string path, string kind) + : base(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None) { this.kind = kind; } + public override void Write(byte[] buffer, int offset, int count) { + wrote = true; + if (kind == "write" || kind == "restore") { + if (++writes == 1) { + base.Write(buffer, offset, Math.Min(3, count)); + throw new IOException("Injected partial write failure"); + } + if (kind == "restore") throw new IOException("Injected recovery failure"); + } + base.Write(buffer, offset, count); + } + public override void Flush(bool disk) { + if (kind == "flush" && wrote && ++syncs == 1) throw new IOException("Injected flush failure"); + base.Flush(disk); + } +} +'@ +$root = Join-Path ([IO.Path]::GetTempPath()) ("provider-bytes-native-" + [Guid]::NewGuid()) +[IO.Directory]::CreateDirectory($root) | Out-Null +try { + $file = Join-Path $root "rollout-fixture.jsonl" + $utf8 = [Text.UTF8Encoding]::new($false) + $header = $utf8.GetBytes('{"type":"session_meta","payload":{"model_provider":"openai"}}' + "`n") + $old = $utf8.GetBytes('"openai"') + $new = $utf8.GetBytes('"prov_a"') + $offset = $utf8.GetString($header).IndexOf('"openai"') + $tail = $utf8.GetBytes("fixture tail`n") + [IO.File]::WriteAllBytes($file, [byte[]]($header + $tail)) + [IO.File]::SetLastWriteTimeUtc($file, [DateTime]::new(2026, 1, 2, 3, 4, 5, [DateTimeKind]::Utc)) + $s = [IO.File]::Open($file, "Open", "ReadWrite", "None") + try { + # Read the same native identity used by the worker, without a Node install. + $native = [ProviderByteFile].GetMethod("GetFileInformationByHandle", [Reflection.BindingFlags]"Static,NonPublic") + $args = [object[]]@($s.SafeFileHandle.DangerousGetHandle(), $null) + if (-not $native.Invoke($null, $args)) { throw "Native file identity unavailable" } + $info = $args[1] + $dev = [string]$info.Volume + $ino = [string](([uint64]$info.IndexHigh -shl 32) -bor [uint64]$info.IndexLow) + $mtime = ([double]($info.WriteTime - 116444736000000000L)) / 10000 + $size = $s.Length + $result = [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $false) + if ($result -ne "APPLIED_IN_PLACE") { throw "Apply did not use in-place: $result" } + $native.Invoke($null, $args) | Out-Null + if ($args[1].WriteTime -ne $info.WriteTime) { throw "Exclusive apply changed mtime" } + try { + $other = [IO.File]::Open($file, "Open", "ReadWrite", "None") + $other.Dispose() + throw "Exclusive handle did not block another writer" + } catch [IO.IOException] { } + # Model a crashed forward write or interrupted rollback, then append. + $s.Position = $offset + $s.Write($old, 0, 3) + $s.Position = $s.Length + $s.Write($tail, 0, $tail.Length) + $s.Flush($true) + [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null + [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null + $s.Position = 0 + $bytes = New-Object byte[] $s.Length + $s.Read($bytes, 0, $bytes.Length) | Out-Null + if ($utf8.GetString($bytes) -ne $utf8.GetString([byte[]]($header + $tail + $tail))) { throw "Recovery changed body or lost append" } + $s.Position = $offset + 1 + $s.WriteByte(33) + $s.Flush($true) + $rejected = $false + try { [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null } + catch { $rejected = $true } + if (-not $rejected) { throw "Unknown bytes were overwritten" } + } finally { $s.Dispose() } + Write-Output "PASS: native identity, exclusive handle, in-place write, mtime, partial recovery, idempotence, append, unknown-byte rejection" + foreach ($kind in @("write", "flush", "restore")) { + [IO.File]::WriteAllBytes($file, [byte[]]($header + $tail)) + $s = [FaultingProviderStream]::new($file, $kind) + try { + $args = [object[]]@($s.SafeFileHandle.DangerousGetHandle(), $null) + $native.Invoke($null, $args) | Out-Null + $info = $args[1] + $dev = [string]$info.Volume + $ino = [string](([uint64]$info.IndexHigh -shl 32) -bor [uint64]$info.IndexLow) + $mtime = ([double]($info.WriteTime - 116444736000000000L)) / 10000 + $failed = $false + try { [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $false) | Out-Null } + catch { $failed = $true } + if (-not $failed) { throw "Injected $kind failure was ignored" } + } finally { $s.Dispose() } + $isOriginal = $utf8.GetString([IO.File]::ReadAllBytes($file)) -eq $utf8.GetString([byte[]]($header + $tail)) + if (($kind -ne "restore") -and (-not $isOriginal)) { throw "Immediate recovery failed for $kind" } + if ($kind -eq "restore") { + if ($isOriginal) { throw "Recovery failure did not leave the expected partial bytes" } + $s = [IO.File]::Open($file, "Open", "ReadWrite", "None") + try { [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null } + finally { $s.Dispose() } + if ($utf8.GetString([IO.File]::ReadAllBytes($file)) -ne $utf8.GetString([byte[]]($header + $tail))) { throw "Later recovery failed" } + } + } + Write-Output "PASS: native partial-write exception, Flush failure, failed immediate recovery and later recovery" + if ($WorkerScript) { + [IO.File]::WriteAllBytes($file, [byte[]]($header + $tail)) + $s = [IO.File]::Open($file, "Open", "ReadWrite", "None") + try { + $args = [object[]]@($s.SafeFileHandle.DangerousGetHandle(), $null) + $native.Invoke($null, $args) | Out-Null + $info = $args[1] + $m = @{ + strategy = "provider_bytes_in_place"; byteOffset = $offset + originalBase64 = [Convert]::ToBase64String($old); replacementBase64 = [Convert]::ToBase64String($new) + originalSize = $s.Length; originalMtimeMs = ([double]($info.WriteTime - 116444736000000000L)) / 10000 + originalDev = [string]$info.Volume + originalIno = [string](([uint64]$info.IndexHigh -shl 32) -bor [uint64]$info.IndexLow) + } + } finally { $s.Dispose() } + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = "powershell.exe" + $start.Arguments = '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File "' + $WorkerScript + '"' + $start.UseShellExecute = $false + $start.RedirectStandardInput = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + $p = [Diagnostics.Process]::Start($start) + try { + $ready = $p.StandardOutput.ReadLine() | ConvertFrom-Json + if ($ready.type -ne "ready") { throw "Worker did not become ready" } + $request = @{ + protocolVersion = 1; type = "rewrite"; id = 1; path = $file + originalFirstLine = $utf8.GetString($header).TrimEnd([char]10); originalSeparator = "`n" + originalOffset = $header.Length; originalSize = $m.originalSize + originalMtimeMs = $m.originalMtimeMs; inPlaceMutation = $m; requireOriginalMatch = $true + } + foreach ($mode in @("busy", "apply", "restore")) { + $lock = $null + try { + if ($mode -eq "busy") { $lock = [IO.File]::Open($file, "Open", "ReadWrite", "None") } + $request.restoreProviderBytes = ($mode -eq "restore") + $p.StandardInput.WriteLine(($request | ConvertTo-Json -Compress -Depth 5)) + $p.StandardInput.Flush() + $response = $p.StandardOutput.ReadLine() | ConvertFrom-Json + $expected = if ($mode -eq "busy") { "SKIP_BUSY" } else { "APPLIED_IN_PLACE" } + if ($response.result -ne $expected) { throw "Worker $mode failed: $($response | ConvertTo-Json -Compress)" } + $request.id++ + } finally { if ($lock) { $lock.Dispose() } } + } + $p.StandardInput.Close() + if (-not $p.WaitForExit(30000)) { throw "Worker failed to exit" } + if ($p.ExitCode -ne 0) { throw $p.StandardError.ReadToEnd() } + if ($utf8.GetString([IO.File]::ReadAllBytes($file)) -ne $utf8.GetString([byte[]]($header + $tail))) { throw "Worker roundtrip mismatch" } + Write-Output "PASS: production worker protocol, busy, in-place apply and restore roundtrip" + } finally { + if (-not $p.HasExited) { $p.Kill(); $p.WaitForExit() } + $p.Dispose() + } + } +} finally { Remove-Item -LiteralPath $root -Recurse -Force } diff --git a/test/windows-rewrite-worker.test.js b/test/windows-rewrite-worker.test.js index 80a6397..e3ae419 100644 --- a/test/windows-rewrite-worker.test.js +++ b/test/windows-rewrite-worker.test.js @@ -1,13 +1,20 @@ import { EventEmitter } from "node:events"; +import { spawnSync } from "node:child_process"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { PassThrough } from "node:stream"; -import test from "node:test"; +import test, { afterEach } from "node:test"; import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; + +const cleanups = []; +afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); }); import { applySessionChanges, + collectSessionChanges, + restoreSessionChanges, createWindowsExclusiveRewriteWorker } from "../src/session-files.js"; @@ -66,6 +73,15 @@ function createFakeSpawn({ ready = { protocolVersion: 1, type: "ready" }, respon return { spawnImpl, getSpawnCount: () => spawnCount }; } +test("native Windows helper recovers short writes and flush failures", { + skip: process.platform !== "win32" +}, () => { + const result = spawnSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", + "-File", fileURLToPath(new URL("./windows-provider-bytes.ps1", import.meta.url))], { encoding: "utf8", timeout: 60000 }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /PASS: native partial-write exception/); +}); + test("Windows rewrite worker reuses one process and preserves the closed result set", async () => { const expectedResults = ["APPLIED", "APPLIED_IN_PLACE", "SKIP_BUSY", "SKIP_CHANGED"]; const fake = createFakeSpawn({ @@ -96,6 +112,36 @@ test("Windows rewrite worker reuses one process and preserves the closed result assert.deepEqual(results, expectedResults); }); +test("real Windows in-place apply/recovery retains file ID and appended data", { + skip: process.platform !== "win32" +}, async (t) => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "provider-bytes-windows-")); + cleanups.push(() => fs.rm(home, { recursive: true, force: true })); + await fs.mkdir(path.join(home, "sessions")); + const file = path.join(home, "sessions", "rollout-[fixture].jsonl"); + const firstLine = JSON.stringify({ type: "session_meta", payload: { id: "fixture", model_provider: "openai" } }); + const original = firstLine + '\r\n{"type":"event_msg","payload":{}}\r\n'; + await fs.writeFile(file, original); + const { changes } = await collectSessionChanges(home, "prov_a", { fast: true }); + const before = await fs.stat(file, { bigint: true }); + assert.equal((await applySessionChanges(changes)).inPlaceChanges, 1); + assert.equal((await fs.stat(file, { bigint: true })).ino, before.ino); + const entry = { ...changes[0], mutation: changes[0].inPlaceMutation }; + const h = await fs.open(file, "r+"); + const partial = Buffer.from(entry.mutation.originalBase64, "base64").subarray(0, 3); + await h.write(partial, 0, partial.length, entry.mutation.byteOffset); + await h.close(); + await fs.appendFile(file, "appended\r\n"); + await restoreSessionChanges([entry]); + await restoreSessionChanges([entry]); + assert.equal((await fs.stat(file, { bigint: true })).ino, before.ino); + assert.equal(await fs.readFile(file, "utf8"), original + "appended\r\n"); + const unknown = await fs.open(file, "r+"); + await unknown.write(Buffer.from("!"), 0, 1, entry.mutation.byteOffset + 1); + await unknown.close(); + await assert.rejects(restoreSessionChanges([entry]), AggregateError); +}); + test("Windows rewrite worker rejects mismatched and malformed protocol responses", async (t) => { await t.test("invalid ready message", async () => { const fake = createFakeSpawn({ From dcb104198d4d6438ff913f67cdeb13098dc0ea89 Mon Sep 17 00:00:00 2001 From: cccat6 <22387156+cccat6@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:54:32 +0800 Subject: [PATCH 3/5] fix: preserve history ordering and isolate recovery conflicts Restore stable-file mtime without backdating concurrent appends, including interrupted timestamp repair. Skip stale Windows preconditions before mutation, compensate valid rollout targets despite other byte conflicts, and keep v2 for backups without byte descriptors. Bound header verification, remove redundant state and duplicate review/worker scaffolding, and cover the audit findings with regression tests. No production deployment. --- CHANGELOG.md | 2 +- README.md | 4 +- docs/README_EN.md | 4 +- docs/TRANSACTIONAL_IN_PLACE_PILOT.md | 148 ------------------ .../proposed-transactional-provider-bytes.md | 55 +++++-- .../contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md | 6 +- docs/migration/BEHAVIOR_FIXTURES_ZH.md | 2 +- src/backup.js | 12 +- src/session-files.js | 61 +++++--- src/windows-provider-bytes.cs | 28 +++- test/fast-sync.test.js | 11 ++ test/in-place-transaction.test.js | 84 +++++++++- test/sync-service.test.js | 3 +- test/windows-provider-bytes.ps1 | 75 +++------ 14 files changed, 222 insertions(+), 273 deletions(-) delete mode 100644 docs/TRANSACTIONAL_IN_PLACE_PILOT.md diff --git a/CHANGELOG.md b/CHANGELOG.md index be8a5c5..5433a5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - 恢复 #51 等长 provider 原地更新，将字节恢复纳入 #71 逐目标事务；覆盖 Node POSIX 和 Windows worker。 - 合并正文扫描；新增显式 `sync --fast` / `switch --fast`，只读首行、保留模型、提示未执行检查，不支持原地更新时不隐式重写全文。 -- 原地/快速备份升级为 v3，旧工具不能恢复。POSIX 原地路径保留实际写入 mtime，避免回拨并发追加时间；依赖文件时间的 History 排序可能受影响。详见[兼容边界](docs/TRANSACTIONAL_IN_PLACE_PILOT.md)。 +- 含原地变更的备份使用 v3，旧工具不能恢复；其余保持 v2。稳定文件保留原 mtime，并处理并发追加与回滚，保持 History 原有排序和去重规则。详见[兼容边界](docs/adr/proposed-transactional-provider-bytes.md)。 ## [0.5.0] - 2026-08-15 diff --git a/README.md b/README.md index f378b98..757ace2 100644 --- a/README.md +++ b/README.md @@ -107,9 +107,9 @@ codex-provider sync `switch` 默认会在目标 Provider section 定义了 `model` 时同步根级 `model`。使用 `--keep-root-model` 保留当前值，或使用 `--model ` 显式指定。 -本分支新增 `sync --fast` / `switch --fast`：只读 rollout 首行，保留模型，不检查历史用户消息和加密内容；不支持原地更新时在写入前报错，不自动全量重写。备份、事务和恢复检查仍然执行；新备份需兼容 v3 的工具恢复，旧 .NET GUI 不支持。详见[实现与兼容边界](docs/TRANSACTIONAL_IN_PLACE_PILOT.md)。 +本分支新增 `sync --fast` / `switch --fast`：只读 rollout 首行，保留模型，不检查历史用户消息和加密内容；不支持原地更新时在写入前报错，不自动全量重写。备份、事务和恢复检查仍然执行；含原地变更的备份需兼容 v3 的工具恢复，旧 .NET GUI 不支持。详见[设计与兼容边界](docs/adr/proposed-transactional-provider-bytes.md)。 -建议使用统一长度的 ASCII Provider ID，优先 6 个字符（如将 `provider_a` 写作 `prov_a`），因为内置 `openai` 是 6 个字符，相对最通用；历史文件很多或很大时，不等长替换会重写整个 rollout、产生大量硬盘写入，等长且符合条件时可 in-place 替换。POSIX 原地路径保留实际写入 mtime，不回拨并发追加的时间；`updated_at` 不变。 +建议使用统一长度的 ASCII Provider ID，优先 6 个字符（如将 `provider_a` 写作 `prov_a`），因为内置 `openai` 是 6 个字符，相对最通用；历史文件很多或很大时，不等长替换会重写整个 rollout、产生大量硬盘写入，等长且符合条件时可 in-place 替换。 SQLite Home 解析顺序：`--sqlite-home` → `config.toml` 根级 `sqlite_home` → `CODEX_SQLITE_HOME` → `/sqlite`。只有默认布局会回退到 `/state_5.sqlite`。 diff --git a/docs/README_EN.md b/docs/README_EN.md index 8197201..463fc4c 100644 --- a/docs/README_EN.md +++ b/docs/README_EN.md @@ -109,9 +109,9 @@ codex-provider sync By default, `switch` also updates the root-level `model` when the target provider section defines one. Use `--keep-root-model` to preserve the current value, or `--model ` to set it explicitly. -This branch adds `sync --fast` / `switch --fast`: read only rollout headers, preserve models, and leave historical user-message/encryption checks unperformed. Unsupported in-place updates fail before mutation, without automatic full rewriting. Backups, transactions and recovery checks remain enabled; new backups need a v3-compatible restore tool, not the old .NET GUI. See [implementation and compatibility](TRANSACTIONAL_IN_PLACE_PILOT.md). +This branch adds `sync --fast` / `switch --fast`: read only rollout headers, preserve models, and leave historical user-message/encryption checks unperformed. Unsupported in-place updates fail before mutation, without automatic full rewriting. Backups, transactions and recovery checks remain enabled; backups containing in-place mutations need a v3-compatible restore tool, not the old .NET GUI. See [design and compatibility](adr/proposed-transactional-provider-bytes.md). -We recommend uniform-length ASCII provider IDs, preferably six characters (for example, `provider_a` as `prov_a`), because built-in `openai` has six characters and is the most common compatibility target. With many or large histories, different lengths require whole-rollout rewrites and substantial disk writes; eligible equal-length values can be replaced in place. POSIX in-place writes retain their actual mtime rather than backdating concurrent appends; `updated_at` is unchanged. +We recommend uniform-length ASCII provider IDs, preferably six characters (for example, `provider_a` as `prov_a`), because built-in `openai` has six characters and is the most common compatibility target. With many or large histories, different lengths require whole-rollout rewrites and substantial disk writes; eligible equal-length values can be replaced in place. SQLite Home resolution order: `--sqlite-home` → root-level `sqlite_home` in `config.toml` → `CODEX_SQLITE_HOME` → `/sqlite`. Only the default layout falls back to `/state_5.sqlite`. diff --git a/docs/TRANSACTIONAL_IN_PLACE_PILOT.md b/docs/TRANSACTIONAL_IN_PLACE_PILOT.md deleted file mode 100644 index abb5047..0000000 --- a/docs/TRANSACTIONAL_IN_PLACE_PILOT.md +++ /dev/null @@ -1,148 +0,0 @@ -# Transactional provider byte updates (implementation candidate) - -This branch restores the optimization introduced by cccat6 in PR #51 -(`7231881`, `cdcde35`, `84a60d3`). PR #71's transaction refactor removed the -production in-place path; v0.5.0 still counts `APPLIED_IN_PLACE` but does not -produce it. This candidate does not manage authentication, profiles or processes. -It is based on upstream main `c7ff852`, not the unmerged V1 migration (#90). - -## Implemented and tested - -- Node POSIX and the existing Windows exclusive PowerShell worker support the - byte strategy. The small C# helper is compiled by that worker for native file - identity and handle operations; it is not a second sync service or SDK dependency. - The published .NET application is unchanged and rejects metadata v3 before - writing any restore target. Cross-runtime recovery parity is NOT claimed. -- A non-empty, equal-length ASCII provider ID with one unescaped, unambiguous - `session_meta.payload.model_provider` field can be replaced in place. A - `turn_context.model` rewrite or an ineligible header uses the existing path. -- The plan captures device/inode, size, mtime, the original header, byte offset, - and both byte sequences. An immutable managed backup contains this descriptor - before the coordinator durably appends `applying` and starts the write. -- Apply revalidates the path, handle identity, snapshot, header and bytes. A - stale precondition is skipped, never used as a reason for a full rewrite. - Hardlinked targets are ineligible. Post-write checks include the complete - header, identity and minimum file size, not just the replacement bytes. -- Short writes loop; write/fsync/read-back failure attempts byte restoration - through the same handle. There is no post-mutation fallback to rename. -- `applying` and `applied` targets recover from the immutable manifest. A torn - journal conservatively selects all manifest candidates. Recovery failure - leaves the existing `recoveryRequired` state and evidence intact. -- Both backup metadata and session manifest use version 3 when any entry is - in-place, so old readers reject before restoring config or SQLite. Version - 1/2 backups still use their old recovery semantics. Non-in-place backups stay - version 2. Fast-mode backups always use version 3, including no-op rollouts. - New backups require this version or a compatible restore implementation. - -## Recovery and writer contract - -Recovery verifies device/inode **and** all surrounding original header bytes -and requires a file at least as large as the original. Identity alone is not a -permanent guarantee against inode reuse. It accepts original bytes, replacement -bytes, or a single contiguous run of replacement bytes among original bytes -at differing positions (`old* new* old*`). That last case covers a sequential -short write and an interrupted sequential rollback. Unknown bytes, disjoint -tears, replaced paths, or truncation fail closed. This is conditional evidence -under the append-only writer model, not proof against a third party rewriting -the header to an indistinguishable value. - -The supported writer leaves existing bytes alone and appends after the guarded -metadata operation. Pre-apply growth is skipped. Later appends remain visible -through the existing fd and survive rollback without truncation. POSIX in-place -apply/restore leaves the actual write mtime: there is no race-free stat/utimes -sequence against an uncooperative appender. Windows restores mtime while holding -the exclusive handle; no-op recovery does not alter it. Thread `updated_at` -is never changed. History views using filesystem mtime may reorder; this -deliberate safety/compatibility tradeoff requires maintainer acceptance. -No POSIX cooperative lock can force Codex to participate: this -does not promise atomic visibility to concurrent readers, or protection from -non-cooperating writers replacing/truncating/editing the header during the -small check/write window. Full replacement paths still have the active-fd risk -identified in PR #71; this pilot removes that risk only for eligible writes. - -## Fast scope and reading cost - -`sync --fast` and `switch --fast` enumerate both rollout roots but read -only metadata headers (bounded to 1 MiB per header) plus file attributes. -Every changed rollout must qualify for in-place replacement. An ineligible or -invalid header fails preflight, before backup or config/SQLite mutation; there -is no implicit full rewrite. Busy/changed targets retain the existing partial -outcome semantics and must not be described as completely aligned. - -The scope preserves root and historical models, leaves `has_user_event` -unchanged, and reports encrypted-content/model/user-event checks as unchecked. -Provider and header-derived cwd/workspace repair retain the existing transaction. -`--model` conflicts with `--fast`; `--keep-root-model` is redundant, allowed. -The managed manifest records `scanScope: metadata`; restore does not invent -historical model snapshots or scan/copy message bodies. Existing config and -SQLite backup/restore behavior is retained, so cost is not strictly header-only. - -Default scans keep their diagnostics but compute encryption presence, positive -user-event evidence and model snapshots in one streaming pass. No persistent -cache is added. The default full rewrite remains for noneligible operations. - -## Validation and follow-up - -Validated on 2026-08-28: - -- Linux Node 24.16.0: full suite, 280 passed / 6 platform skips / 0 failed. -- Linux Node 16.20.2 with the existing optional better-sqlite3 8.7.0 driver: - all 19 test files passed (the older runner reports file-level totals). -- Existing Windows PowerShell: real production worker protocol, busy response, - native file identity, in-place apply/restore, timestamp retention, short-write - exception, Flush failure, failed immediate undo followed by recovery, - idempotence, appended-tail preservation and unknown-byte rejection passed. -- Web production build, package dry-run (including the native helper source), - and `git diff --check` passed. -- Not run: full Node/SQLite suite on Windows, macOS native tests, real WSL UNC - tests, and cross-runtime v3 restore (the old .NET reader rejects v3). - -`node scripts/benchmark-provider-io.mjs 32` uses disposable data. On ext4 with -warm page cache, one ~32 MiB fixture produced these process-level measurements: - -| Mode | Logical reads | Kernel-accounted writes | Elapsed | -| --- | ---: | ---: | ---: | -| Full, equal IDs | 34,044,759 B | 45,056 B | 127 ms | -| Fast, equal IDs | 405,780 B | 45,056 B | 55 ms | -| Full, unequal IDs | 67,689,493 B | 33,681,408 B | 247 ms | - -Equal-ID cases retained inode and tail hash. Numbers include managed backup -and journal overhead, not just provider bytes. They exclude SSD-internal write -amplification and are not a prediction for cold storage or a large SQLite DB. - -`test/in-place-transaction.test.js` covers eligibility, short/zero writes, -fsync failure, immediate restoration failure, immutable manifests, A/B failure, -crashes before `applied` and before commit, torn journals, idempotence, -conflicts, active fds and appends. A 32 MiB disposable fixture records only -8 rollout bytes written and verifies the unchanged tail hash and inode. -`test/fast-sync.test.js` guards against rollout body streams throughout switch -and restore, tests preflight failures, CLI parsing, models, SQLite and rollback. -`test/windows-provider-bytes.ps1` also tests the native helper using the existing -PowerShell runtime without a Node installation. No real history or API calls -are used. This work does not authorize production deployment. - -Before a formal PR: run the full native Windows Node suite; agree the POSIX -mtime policy, metadata v3 transition and fast-scope semantics with the maintainer. -See the [proposed ADR](adr/proposed-transactional-provider-bytes.md). No public -PR/comment/release is authorized by this development work. - -If #90 lands first, rebase through its shared Core, plan ledger, dual locks and -Restore v2. Its full-content revisions and final status refresh must become -scope-aware for fast operations, not be bypassed. Bind scope, original header, -file identity, target bytes and config/DB revisions in the plan; retain full -hash verification for full-scope operations (streaming rather than readFile). -Restore-v2 pre-snapshots, target digests and compensation must use the same byte -strategy. Those V1-specific changes are NOT implemented on this main-based branch. - -References: [#51](https://github.com/Dailin521/codex-provider-sync/pull/51), -[#71](https://github.com/Dailin521/codex-provider-sync/pull/71), -[#69](https://github.com/Dailin521/codex-provider-sync/issues/69), -[active-fd finding](https://github.com/Dailin521/codex-provider-sync/pull/71#discussion_r3711178450), -[Codex #38149](https://github.com/openai/codex/issues/38149). - -For frequent switching, we recommend equal-length ASCII provider IDs, preferably six -characters because `openai` has six (for example `provider_a` as `prov_a`). -Different lengths require whole-file rewriting; large histories can multiply -disk writes and elapsed time. The original user's rollout collection was -approximately 53 GiB. In-place updates do not convert `encrypted_content` or -make histories portable between providers/accounts. diff --git a/docs/adr/proposed-transactional-provider-bytes.md b/docs/adr/proposed-transactional-provider-bytes.md index 29873c5..795b9a9 100644 --- a/docs/adr/proposed-transactional-provider-bytes.md +++ b/docs/adr/proposed-transactional-provider-bytes.md @@ -6,9 +6,11 @@ ## Context -PR #51 saved whole-rollout writes for equal-length IDs. PR #71 removed that -path while adding durable per-target recovery. Restore the optimization inside -the same journal, not around it. A separate fast scope addresses body reading. +[PR #51](https://github.com/Dailin521/codex-provider-sync/pull/51), by cccat6 +(7231881, cdcde35, 84a60d3), avoided whole-rollout writes for equal-length IDs. +[PR #71](https://github.com/Dailin521/codex-provider-sync/pull/71) removed that +path while adding durable per-target recovery for #69. Restore the optimization +inside that journal, not around it. A separate fast scope addresses body reads. ## Proposal @@ -16,21 +18,40 @@ the same journal, not around it. A separate fast scope addresses body reading. - Add a manifest-bound provider-byte strategy, durable before `applying`. - Revalidate identity/header/bytes, write through the same handle, flush and verify; undo through that handle/strategy without rename or tail truncation. -- Preserve POSIX write mtime to avoid racing appenders; Windows can preserve - original mtime under its exclusive handle. This is an explicit compatibility - change, not an assertion that all default observable behavior is identical. +- Preserve original mtime for unchanged-size files. If an append races POSIX + stat/utimes, reassert only the guarded bytes for a fresh kernel write time; + never backdate again. Recovery also repairs this interrupted timestamp step. + Windows preserves timestamps under its exclusive handle. `updated_at` stays + unchanged; History's existing ordering and duplicate selection remain intact. - Metadata and manifest v3 prevent old readers from silently applying the - wrong restore strategy. Old v1/v2 backups retain their existing semantics. + wrong restore strategy. Only backups containing byte mutations require v3; + all others, including fast no-op backups, keep v2. Old backups remain readable. - Opt-in `--fast` narrows diagnostics/model scope, never durability or recovery. Unsupported headers fail preflight rather than silently copying the body. +- Full mode fuses encryption, user-event and model checks in one body pass. +- Fast mode caps headers at 1 MiB, preserves root/history models and user-event + flags, retains provider/cwd/workspace repair, and reports unchecked diagnostics. + `--model` conflicts with `--fast`. Busy/changed targets retain partial outcomes. - No credentials, account tooling, process management, new database or cache. -## Acceptance questions +## Recovery boundaries + +Only a unique unescaped ASCII provider value, equal encoded length and no +model rewrite qualifies. Validate file identity, size, header and target bytes +before writing; a stale precondition is skipped, never a full-rewrite fallback. +Normal undo and crash recovery use the same byte strategy. Old/new bytes or +one contiguous run of new bytes among old bytes are recoverable under the +append-only writer model. Unknown/disjoint edits, truncation and replaced +identities fail closed. Hardlinked targets are ineligible. Inode identity is +not a permanent proof against reuse; writes are not atomic to concurrent readers. -The mtime policy can affect History's filesystem-time fallback, so it needs -explicit acceptance. Do not claim safe concurrent append and unconditional old -mtime restoration simultaneously. Do not replace this policy with a timing -heuristic and call it exclusive access. +Cross-store restore preflights byte conflicts before changing config/SQLite; +rollout-only compensation attempts all targets even when one conflicts. Applying +and applied targets recover from the immutable manifest; damaged journals use +all candidates. Failure retains recoveryRequired and evidence. Full rewrite and +old-backup paths retain their existing active-writer limitations. + +## Acceptance questions Current .NET rejects v3 safely but cannot recover it. Maintainer approval is required for this migration boundary, or a compatible recovery reader must be @@ -43,6 +64,10 @@ second transaction system is necessary on the current base. ## Evidence -See [implementation and tests](../TRANSACTIONAL_IN_PLACE_PILOT.md). Native, -simulated and unrun coverage must be reported separately. Process-exit tests -do not establish arbitrary power-loss or non-cooperating-writer guarantees. +Focused tests: `in-place-transaction.test.js`, `fast-sync.test.js`, and the +Windows worker/native-helper tests. They cover short writes, flush failure, +crashes, torn journals, rollback conflicts, append/mtime races, inode and tail +preservation, fast preflight and default behavior. `scripts/benchmark-provider-io.mjs` +measures disposable fixtures; report logical I/O separately from kernel writes. +Native, simulated and unrun coverage belong in the PR validation report. +Process-exit tests do not prove arbitrary power-loss or hostile-writer safety. diff --git a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md index 5fe0b84..535c80f 100644 --- a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md +++ b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md @@ -14,12 +14,12 @@ - Node `runSync` / `runSwitch` 新增 `fast=false`。默认保留原检查范围，合并为一次正文扫描。 - 合格的等长 provider 使用 manifest v3 描述的原地 mutation；执行、回滚和崩溃恢复保留同一 inode/file ID，开始写入后不得回退全文重写。 -- POSIX 原地路径不恢复旧 mtime，Windows 在独占句柄内保留 mtime；`updated_at` 不变，但依赖文件 mtime 的 History 排序可能变化，需维护者确认。 +- 稳定文件保留原 mtime；POSIX 检测并修复 stat/utimes 期间的追加竞争，Windows 使用独占句柄；`updated_at` 和 History 的去重/排序规则不变。 - `fast=true` 只读首行（上限 1 MiB）及属性，保留根级/历史模型，不检查用户事件与加密内容；保留 provider、首行 cwd、workspace 修复及数据库事务。 - 快速模式静态不支持项在备份和业务写入前以 `FAST_MODE_UNSUPPORTED` 失败，不隐式全量重写；运行时 busy/changed 沿用 partial 分类。 - 结果增加 `inPlaceSessionFiles`；快速结果另含 `scanScope="metadata"`、`unchecked=["historyModels","userEventFlags","encryptedContent"]`、`encryptedContentCounts=null` 和未检查警告。 -- 原地/快速备份使用 metadata 和 manifest v3；旧 v1/v2 保持原恢复语义，旧 .NET 仅安全拒绝 v3，不宣称互相恢复。 -- 详见[候选 ADR](../../adr/proposed-transactional-provider-bytes.md)和[实现说明](../../TRANSACTIONAL_IN_PLACE_PILOT.md)。V1/#90 的 Plan/Revision/Restore v2 接入不属于本分支已完成能力。 +- 仅含原地变更的备份使用 metadata 和 manifest v3；其余保持 v2。旧备份保留原恢复语义，旧 .NET 仅安全拒绝 v3，不宣称互相恢复。 +- 详见[候选 ADR](../../adr/proposed-transactional-provider-bytes.md)。V1/#90 的 Plan/Revision/Restore v2 接入不属于本分支已完成能力。 本文冻结 vNext 迁移开始时 Node 实现已经提供的外部行为。这里的“外部”不仅指 npm 最终用户，也包括当前 CLI 与 Local Web UI 对 Node service 的真实依赖。 diff --git a/docs/migration/BEHAVIOR_FIXTURES_ZH.md b/docs/migration/BEHAVIOR_FIXTURES_ZH.md index 08894e1..cd752f4 100644 --- a/docs/migration/BEHAVIOR_FIXTURES_ZH.md +++ b/docs/migration/BEHAVIOR_FIXTURES_ZH.md @@ -1,6 +1,6 @@ # vNext 行为兼容 Fixture 清单 -本分支候选夹具：`test/in-place-transaction.test.js` 验证短写、崩溃、损坏 journal、冲突及活跃追加；`test/fast-sync.test.js` 验证快速范围、无正文流、模型/SQLite/cwd、前置失败和回滚；`test/windows-rewrite-worker.test.js` 与 `test/windows-provider-bytes.ps1` 验证协议和 Windows 原生独占句柄。POSIX 原地路径保留实际写入 mtime，原全量路径仍保留旧 mtime；该区别是待评审合同，不是隐式测试豁免。 +本分支候选夹具：`test/in-place-transaction.test.js` 验证短写、崩溃、损坏 journal、逐目标回滚、追加/mtime 竞争与 History 副本选择；`test/fast-sync.test.js` 验证快速范围、无正文流、模型/SQLite/cwd、前置失败和格式兼容；`test/windows-rewrite-worker.test.js` 与 `test/windows-provider-bytes.ps1` 验证协议和原生独占句柄。 > **状态：Accepted（阶段 0 语义清单；共享 Corpus 尚未创建）** > diff --git a/src/backup.js b/src/backup.js index 55601f0..e10f4da 100644 --- a/src/backup.js +++ b/src/backup.js @@ -298,7 +298,7 @@ export async function createBackup({ // Both versions must advance so old readers reject before restoring any // config/SQLite data, even when rollout restore was disabled by the caller. - const backupVersion = fast || sessionChanges.some((change) => change.inPlaceMutation) ? 3 : 2; + const backupVersion = sessionChanges.some((change) => change.inPlaceMutation) ? 3 : 2; const sessionManifest = { version: backupVersion, namespace: BACKUP_NAMESPACE, @@ -616,10 +616,12 @@ export async function restoreBackup(backupDir, storageOrCodexHome, options = {}) } else { sessionRestoreEntries = await selectSessionRestoreEntries(backupDir, sessionManifest); } - // Detect known byte conflicts before rewinding config or SQLite. Apply - // still rechecks through the mutation handle; preflight is not a lock. - for (const entry of sessionRestoreEntries) { - if (entry.mutation) await validateProviderByteRestore(entry); + // Preflight before rewinding other stores. Rollout-only compensation must + // attempt every target, even if another target has conflicting bytes. + if (restoreConfig || restoreGlobalState || restoreDatabase) { + for (const entry of sessionRestoreEntries) { + if (entry.mutation) await validateProviderByteRestore(entry); + } } } diff --git a/src/session-files.js b/src/session-files.js index 1467dbd..b260e2a 100644 --- a/src/session-files.js +++ b/src/session-files.js @@ -548,18 +548,34 @@ async function writeBytesFully(handle, bytes, position, writeImpl) { } } -async function finishInPlaceWrite(handle, entry, expectedBytes, options = {}) { +async function verifyInPlaceWrite(handle, entry, expectedBytes) { const mutation = entry.mutation ?? entry.inPlaceMutation; - await (options.inPlaceSync ?? ((h) => h.sync()))(handle); const expected = Buffer.from(entry.originalFirstLine + entry.originalSeparator, "utf8"); expectedBytes.copy(expected, mutation.byteOffset); const actual = await readBytesFully(handle, expected.length, 0); - if (!actual?.equals(expected) || (await handle.stat()).size < mutation.originalSize) { + const stat = await handle.stat(); + if (!actual?.equals(expected) || stat.size < mutation.originalSize) { throw new Error(`Provider in-place write verification failed: ${entry.path}`); } await assertInPlaceIdentity(handle, entry.path, mutation); - // POSIX has no exclusive handle here. Even stat followed by utimes races an - // append, so retain the actual write time instead of overwriting newer mtime. + return stat; +} + +async function finishInPlaceWrite(handle, entry, expectedBytes, options = {}) { + const mutation = entry.mutation ?? entry.inPlaceMutation; + await (options.inPlaceSync ?? ((h) => h.sync()))(handle); + const stat = await verifyInPlaceWrite(handle, entry, expectedBytes); + const grew = stat.size !== mutation.originalSize; + if (grew && Math.round(stat.mtimeMs) !== Math.round(mutation.originalMtimeMs)) return; + if (!grew) await handle.utimes(stat.atime, mutation.originalMtimeMs / 1000); + const after = await verifyInPlaceWrite(handle, entry, expectedBytes); + if (after.size !== mutation.originalSize) { + // An append raced stat/utimes (possibly in interrupted recovery). Reassert + // only the guarded bytes for a kernel write time; never backdate again. + await writeBytesFully(handle, expectedBytes, mutation.byteOffset, options.inPlaceWrite ?? defaultInPlaceWrite); + } + await handle.sync(); + await verifyInPlaceWrite(handle, entry, expectedBytes); } async function assertInPlaceIdentity(handle, filePath, mutation) { @@ -621,16 +637,15 @@ async function restoreProviderOnHandle(handle, entry, options = {}) { if (!current.equals(originalBytes)) { await writeBytesFully(handle, originalBytes, mutation.byteOffset, options.inPlaceRestoreWrite ?? defaultInPlaceWrite); } - await finishInPlaceWrite(handle, entry, originalBytes); + await finishInPlaceWrite(handle, entry, originalBytes, { inPlaceWrite: options.inPlaceRestoreWrite }); } async function tryRewriteProviderInPlace(change, options = {}) { const mutation = change.inPlaceMutation; - const { originalBytes, replacementBytes } = validateProviderMutationDescriptor( + const { replacementBytes } = validateProviderMutationDescriptor( mutation, change.path, change.originalFirstLine, change.originalSeparator); const writeImpl = options.inPlaceWrite ?? defaultInPlaceWrite; let handle; - let writeAttempted = false; try { const pathStat = await fsp.lstat(change.path); if (pathStat.isSymbolicLink() || !pathStat.isFile()) { @@ -649,12 +664,9 @@ async function tryRewriteProviderInPlace(change, options = {}) { || mutation.originalMtimeMs !== change.originalMtimeMs) { return "SKIP_CHANGED"; } - const current = await readFirstLineRecordFromHandle(handle); - if (current.firstLine !== change.originalFirstLine || current.offset !== change.originalOffset) { - return "SKIP_CHANGED"; - } - const currentBytes = await readBytesFully(handle, originalBytes.length, mutation.byteOffset); - if (!currentBytes?.equals(originalBytes)) { + const expectedHeader = Buffer.from(change.originalFirstLine + change.originalSeparator, "utf8"); + const current = await readBytesFully(handle, expectedHeader.length, 0); + if (!current?.equals(expectedHeader)) { return "SKIP_CHANGED"; } try { @@ -665,21 +677,18 @@ async function tryRewriteProviderInPlace(change, options = {}) { } try { - writeAttempted = true; await writeBytesFully(handle, replacementBytes, mutation.byteOffset, writeImpl); await finishInPlaceWrite(handle, change, replacementBytes, options); } catch (error) { - if (writeAttempted) { - try { - await restoreProviderOnHandle(handle, change, options); - } catch (restoreError) { - const failure = new AggregateError( - [error, restoreError], - `Provider in-place write and immediate byte restoration both failed for ${change.path}.` - ); - failure.code = "IN_PLACE_RESTORE_FAILED"; - throw failure; - } + try { + await restoreProviderOnHandle(handle, change, options); + } catch (restoreError) { + const failure = new AggregateError( + [error, restoreError], + `Provider in-place write and immediate byte restoration both failed for ${change.path}.` + ); + failure.code = "IN_PLACE_RESTORE_FAILED"; + throw failure; } throw error; } diff --git a/src/windows-provider-bytes.cs b/src/windows-provider-bytes.cs index 6dc5cca..de7178e 100644 --- a/src/windows-provider-bytes.cs +++ b/src/windows-provider-bytes.cs @@ -20,14 +20,19 @@ struct Info [DllImport("kernel32.dll", SetLastError = true)] static extern bool SetFileTime(IntPtr handle, IntPtr creation, IntPtr access, ref long write); - static Info Inspect(FileStream stream, string dev, string ino) + static bool Matches(Info info, string dev, string ino) + { + ulong index = ((ulong)info.IndexHigh << 32) | info.IndexLow; + return info.Volume.ToString() == dev && index.ToString() == ino && info.Links == 1 + && (info.Attributes & (0x400u | 0x10u)) == 0; + } + + static Info Inspect(FileStream stream, string dev, string ino, bool requireMatch = true) { Info info; if (!GetFileInformationByHandle(stream.SafeFileHandle.DangerousGetHandle(), out info)) throw new Win32Exception(Marshal.GetLastWin32Error()); - ulong index = ((ulong)info.IndexHigh << 32) | info.IndexLow; - if (info.Volume.ToString() != dev || index.ToString() != ino || info.Links != 1 - || (info.Attributes & (0x400u | 0x10u)) != 0) + if (requireMatch && !Matches(info, dev, ino)) throw new IOException("Rollout identity changed before provider byte access."); return info; } @@ -93,8 +98,12 @@ static void Write(FileStream stream, byte[] bytes, int offset, byte[] expected, public static string Apply(FileStream stream, byte[] header, byte[] oldBytes, byte[] newBytes, int offset, long size, double mtimeMs, string dev, string ino, bool restore) { - var before = Inspect(stream, dev, ino); - if (stream.Length < size) throw new IOException("Rollout truncated before provider byte access."); + var before = Inspect(stream, dev, ino, false); + if (!Matches(before, dev, ino) || stream.Length < Math.Max(size, header.Length)) + { + if (!restore) return "SKIP_CHANGED"; + throw new IOException("Rollout identity changed or file truncated before provider recovery."); + } var current = Read(stream, header.Length); var expected = (byte[])header.Clone(); Array.Copy(newBytes, 0, expected, offset, newBytes.Length); @@ -106,8 +115,11 @@ public static string Apply(FileStream stream, byte[] header, byte[] oldBytes, by { if (!Recoverable(current, header, oldBytes, newBytes, offset)) throw new IOException("Unknown rollout bytes during provider recovery."); - if (Equal(current, header)) return "APPLIED_IN_PLACE"; - Write(stream, oldBytes, offset, header, size, dev, ino, before.WriteTime); + if (Equal(current, header) && (stream.Length != size || Math.Abs(currentMs - mtimeMs) <= 0.001)) + return "APPLIED_IN_PLACE"; + long restoreTime = stream.Length == size + ? 116444736000000000L + (long)Math.Round(mtimeMs * 10000.0) : before.WriteTime; + Write(stream, oldBytes, offset, header, size, dev, ino, restoreTime); return "APPLIED_IN_PLACE"; } try diff --git a/test/fast-sync.test.js b/test/fast-sync.test.js index eddd330..b7b0891 100644 --- a/test/fast-sync.test.js +++ b/test/fast-sync.test.js @@ -169,6 +169,17 @@ test("fast sync rejects a model at the service boundary", async (t) => { await assert.rejects(runSync({ codexHome: f.home, fast: true, model: "other" }), /preserves historical models/); }); +test("fast mode without a byte mutation keeps the compatible v2 backup format", async (t) => { + const f = await fixture(t); + const result = await runSync({ codexHome: f.home, fast: true }); + assert.equal(result.inPlaceSessionFiles, 0); + assert.equal(result.changedSessionFiles, 0); + for (const file of ["metadata.json", "session-meta-backup.json"]) { + assert.equal(JSON.parse(await fs.readFile(path.join(result.backupDir, file))).version, 2); + } + await runRestore({ codexHome: f.home, backupDir: result.backupDir }); +}); + test("manual restore preflights unknown provider bytes before changing config or SQLite", async (t) => { const f = await fixture(t); const result = await runSwitch({ codexHome: f.home, provider: "prov_a", fast: true }); diff --git a/test/in-place-transaction.test.js b/test/in-place-transaction.test.js index 415206f..40d7eb6 100644 --- a/test/in-place-transaction.test.js +++ b/test/in-place-transaction.test.js @@ -10,6 +10,7 @@ import { fileURLToPath } from "node:url"; import { applySessionChanges, collectSessionChanges, restoreSessionChanges } from "../src/session-files.js"; import { createBackup, restoreBackup } from "../src/backup.js"; import { runRestore, runSync } from "../src/service.js"; +import { listHistory } from "../src/history.js"; import { TransactionJournal, readTransactionJournal, findPendingTransactions } from "../src/transaction-journal.js"; const repo = fileURLToPath(new URL("..", import.meta.url)); @@ -77,7 +78,7 @@ test("provider-looking text inside a string is not a duplicate field", posix, as assert.ok(changes[0].inPlaceMutation); }); -test("short writes loop to completion, preserve inode/size and retain actual write time", posix, async (t) => { +test("short writes preserve inode, size, content and original mtime", posix, async (t) => { const f = await fixture(t); const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); const before = await fs.stat(f.file); @@ -88,7 +89,7 @@ test("short writes loop to completion, preserve inode/size and retain actual wri const after = await fs.stat(f.file); assert.equal(after.ino, before.ino); assert.equal(after.size, before.size); - assert.ok(after.mtimeMs > f.mtime.getTime()); + assert.equal(Math.round(after.mtimeMs), f.mtime.getTime()); assert.equal(await fs.readFile(f.file, "utf8"), f.original.toString().replace('"openai"', '"prov_a"')); }); @@ -153,14 +154,17 @@ test("in-place recovery accepts old/new/contiguous partial, rejects unknown byte } }); -test("pre-write replaced path or append is skipped without fallback", posix, async (t) => { - for (const state of ["replace", "append"]) { +test("pre-write replaced path, header or append is skipped without fallback", posix, async (t) => { + for (const state of ["replace", "header", "append"]) { const f = await fixture(t); const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); if (state === "replace") { await fs.writeFile(f.file + ".other", f.original); await fs.utimes(f.file + ".other", f.mtime, f.mtime); await fs.rename(f.file + ".other", f.file); + } else if (state === "header") { + await fs.writeFile(f.file, f.original.toString().replace(/\n/g, " ")); + await fs.utimes(f.file, f.mtime, f.mtime); } else await fs.appendFile(f.file, tail); const before = await fs.readFile(f.file); const result = await applySessionChanges(changes); @@ -218,6 +222,63 @@ test("append between patch and fsync remains visible and never calls utimes", po assert.equal(await fs.readFile(f.file, "utf8"), f.original.toString().replace("openai", "prov_a") + tail); }); +test("append racing mtime restoration is never backdated, including rollback", posix, async (t) => { + for (const restore of [false, true]) for (const timing of ["before", "after"]) { + const f = await fixture(t); + const { changes, entry } = await prepare(f); + const writer = await fs.open(f.file, "a"); + cleanups.push(() => writer.close()); + let appended; + const write = async (h, b, o, n, p) => { + if (!appended) { + const utimes = h.utimes.bind(h); + h.utimes = async (...args) => { + if (timing === "after") await utimes(...args); + await writer.write(tail); + appended = await writer.stat(); + if (timing === "before") await utimes(...args); + }; + } + return h.write(b, o, n, p); + }; + if (restore) { + await applySessionChanges(changes); + await restoreSessionChanges([entry], { inPlaceRestoreWrite: write }); + } else await applySessionChanges(changes, { inPlaceWrite: write }); + assert.ok(appended); + assert.ok((await fs.stat(f.file)).mtimeMs >= appended.mtimeMs); + const expected = restore ? f.original.toString() : f.original.toString().replace("openai", "prov_a"); + assert.equal(await fs.readFile(f.file, "utf8"), expected + tail); + } +}); + +test("provider-only updates do not make History select an older duplicate", async (t) => { + for (const fast of [false, true]) { + const f = await fixture(t); + const newer = path.join(f.codexHome, "sessions", "rollout-newer.jsonl"); + await fs.writeFile(newer, f.original.toString().replace("openai", "prov_a")); + const date = new Date(f.mtime.getTime() + 10000); + await fs.utimes(newer, date, date); + assert.equal((await listHistory(f.codexHome)).sessions[0].rolloutPath, newer); + await runSync({ codexHome: f.codexHome, fast }); + assert.equal((await listHistory(f.codexHome)).sessions[0].rolloutPath, newer); + assert.equal(Math.round((await fs.stat(f.file)).mtimeMs), f.mtime.getTime()); + } +}); + +test("recovery repairs a crash after backdating an append even when bytes are already old", posix, async (t) => { + const f = await fixture(t); + const { entry } = await prepare(f); + await fs.appendFile(f.file, tail); + await fs.utimes(f.file, f.mtime, f.mtime); + await restoreSessionChanges([entry]); + const restored = await fs.stat(f.file); + assert.ok(restored.mtimeMs > f.mtime.getTime()); + await restoreSessionChanges([entry]); + assert.equal((await fs.stat(f.file)).mtimeMs, restored.mtimeMs); + assert.equal(await fs.readFile(f.file, "utf8"), f.original + tail); +}); + test("hardlinked files are not eligible and late links prevent byte mutation", posix, async (t) => { const f = await fixture(t); const { changes } = await collectSessionChanges(f.codexHome, "prov_a"); @@ -278,6 +339,21 @@ test("unknown bytes leave a recoveryRequired journal and block later writes", po await assert.rejects(runSync({ codexHome: f.codexHome }), { code: "RECOVERY_REQUIRED" }); }); +test("a conflict on B does not prevent compensation of A", async (t) => { + const f = await fixture(t); + const second = path.join(f.codexHome, "sessions", "rollout-z.jsonl"); + await fs.writeFile(second, f.original); + await assert.rejects(runSync({ codexHome: f.codexHome, faultInjector: async ({ point, path: file }) => { + if (point === "after_rollout_mutation_before_applied" && file === second) { + const text = await fs.readFile(second, "utf8"); + await fs.writeFile(second, text.replace("prov_a", "??????")); + throw new Error("B conflict"); + } + } }), { code: "RECOVERY_REQUIRED" }); + assert.deepEqual(await fs.readFile(f.file), f.original); + assert.match(await fs.readFile(second, "utf8"), /\?{6}/); +}); + test("actual process exit at applying/applied boundary recovers in place", posix, async (t) => { for (const point of ["after_rollout_mutation_before_applied", "after_rollout_apply"]) { const f = await fixture(t); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 8ca3a0d..3d8e69c 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -3289,8 +3289,7 @@ test("applySessionChanges updates equal-length provider IDs without replacing th assert.equal(result.inPlaceChanges, 1); assert.equal(after.ino, before.ino); assert.equal(after.size, before.size); - if (process.platform === "win32") assert.equal(Math.round(after.mtimeMs), originalTime.getTime()); - else assert.ok(after.mtimeMs > originalTime.getTime()); + assert.equal(Math.round(after.mtimeMs), originalTime.getTime()); const firstNewline = rollout.indexOf("\n"); assert.equal(JSON.parse(rollout.slice(0, firstNewline)).payload.model_provider, "prov_a"); assert.equal(rollout.slice(firstNewline + 1), original.slice(original.indexOf("\n") + 1)); diff --git a/test/windows-provider-bytes.ps1 b/test/windows-provider-bytes.ps1 index 0ccd8d3..b7669eb 100644 --- a/test/windows-provider-bytes.ps1 +++ b/test/windows-provider-bytes.ps1 @@ -1,4 +1,4 @@ -param([string]$Source = "$PSScriptRoot/../src/windows-provider-bytes.cs", [string]$WorkerScript) +param([string]$Source = "$PSScriptRoot/../src/windows-provider-bytes.cs") $ErrorActionPreference = "Stop" Add-Type -Path $Source Add-Type -TypeDefinition @' @@ -54,6 +54,12 @@ try { if ($result -ne "APPLIED_IN_PLACE") { throw "Apply did not use in-place: $result" } $native.Invoke($null, $args) | Out-Null if ($args[1].WriteTime -ne $info.WriteTime) { throw "Exclusive apply changed mtime" } + $setTime = [ProviderByteFile].GetMethod("SetFileTime", [Reflection.BindingFlags]"Static,NonPublic") + $setTime.Invoke($null, [object[]]@($s.SafeFileHandle.DangerousGetHandle(), [IntPtr]::Zero, [IntPtr]::Zero, [DateTime]::UtcNow.ToFileTimeUtc())) | Out-Null + [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null + $native.Invoke($null, $args) | Out-Null + if ($args[1].WriteTime -ne $info.WriteTime) { throw "Crash recovery did not restore original mtime" } + [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $false) | Out-Null try { $other = [IO.File]::Open($file, "Open", "ReadWrite", "None") $other.Dispose() @@ -78,6 +84,18 @@ try { try { [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null } catch { $rejected = $true } if (-not $rejected) { throw "Unknown bytes were overwritten" } + $otherIno = [string]([uint64]::Parse($ino) + 1) + if ([ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $otherIno, $false) -ne "SKIP_CHANGED") { + throw "Pre-write identity change was not skipped" + } + $s.SetLength($size - 1) + if ([ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $false) -ne "SKIP_CHANGED") { + throw "Pre-write truncation was not skipped" + } + $rejected = $false + try { [ProviderByteFile]::Apply($s, $header, $old, $new, $offset, $size, $mtime, $dev, $ino, $true) | Out-Null } + catch { $rejected = $true } + if (-not $rejected) { throw "Truncated recovery was not rejected" } } finally { $s.Dispose() } Write-Output "PASS: native identity, exclusive handle, in-place write, mtime, partial recovery, idempotence, append, unknown-byte rejection" foreach ($kind in @("write", "flush", "restore")) { @@ -106,59 +124,4 @@ try { } } Write-Output "PASS: native partial-write exception, Flush failure, failed immediate recovery and later recovery" - if ($WorkerScript) { - [IO.File]::WriteAllBytes($file, [byte[]]($header + $tail)) - $s = [IO.File]::Open($file, "Open", "ReadWrite", "None") - try { - $args = [object[]]@($s.SafeFileHandle.DangerousGetHandle(), $null) - $native.Invoke($null, $args) | Out-Null - $info = $args[1] - $m = @{ - strategy = "provider_bytes_in_place"; byteOffset = $offset - originalBase64 = [Convert]::ToBase64String($old); replacementBase64 = [Convert]::ToBase64String($new) - originalSize = $s.Length; originalMtimeMs = ([double]($info.WriteTime - 116444736000000000L)) / 10000 - originalDev = [string]$info.Volume - originalIno = [string](([uint64]$info.IndexHigh -shl 32) -bor [uint64]$info.IndexLow) - } - } finally { $s.Dispose() } - $start = [Diagnostics.ProcessStartInfo]::new() - $start.FileName = "powershell.exe" - $start.Arguments = '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File "' + $WorkerScript + '"' - $start.UseShellExecute = $false - $start.RedirectStandardInput = $true - $start.RedirectStandardOutput = $true - $start.RedirectStandardError = $true - $p = [Diagnostics.Process]::Start($start) - try { - $ready = $p.StandardOutput.ReadLine() | ConvertFrom-Json - if ($ready.type -ne "ready") { throw "Worker did not become ready" } - $request = @{ - protocolVersion = 1; type = "rewrite"; id = 1; path = $file - originalFirstLine = $utf8.GetString($header).TrimEnd([char]10); originalSeparator = "`n" - originalOffset = $header.Length; originalSize = $m.originalSize - originalMtimeMs = $m.originalMtimeMs; inPlaceMutation = $m; requireOriginalMatch = $true - } - foreach ($mode in @("busy", "apply", "restore")) { - $lock = $null - try { - if ($mode -eq "busy") { $lock = [IO.File]::Open($file, "Open", "ReadWrite", "None") } - $request.restoreProviderBytes = ($mode -eq "restore") - $p.StandardInput.WriteLine(($request | ConvertTo-Json -Compress -Depth 5)) - $p.StandardInput.Flush() - $response = $p.StandardOutput.ReadLine() | ConvertFrom-Json - $expected = if ($mode -eq "busy") { "SKIP_BUSY" } else { "APPLIED_IN_PLACE" } - if ($response.result -ne $expected) { throw "Worker $mode failed: $($response | ConvertTo-Json -Compress)" } - $request.id++ - } finally { if ($lock) { $lock.Dispose() } } - } - $p.StandardInput.Close() - if (-not $p.WaitForExit(30000)) { throw "Worker failed to exit" } - if ($p.ExitCode -ne 0) { throw $p.StandardError.ReadToEnd() } - if ($utf8.GetString([IO.File]::ReadAllBytes($file)) -ne $utf8.GetString([byte[]]($header + $tail))) { throw "Worker roundtrip mismatch" } - Write-Output "PASS: production worker protocol, busy, in-place apply and restore roundtrip" - } finally { - if (-not $p.HasExited) { $p.Kill(); $p.WaitForExit() } - $p.Dispose() - } - } } finally { Remove-Item -LiteralPath $root -Recurse -Force } From eb890c1678b3ee93ca432e0f5b19eae8ba79e38b Mon Sep 17 00:00:00 2001 From: cccat6 <22387156+cccat6@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:14:12 +0800 Subject: [PATCH 4/5] fix: keep provider-byte backups interoperable with official v2 readers Write the official v2 metadata and manifest with optional byte-undo records while retaining all standard restore fields and v1/v2 support. Remove the unpublished v3 format branch. Verified both directions against unmodified upstream Node and .NET Core, including applying-only partial writes. --- CHANGELOG.md | 2 +- README.md | 2 +- docs/README_EN.md | 2 +- .../proposed-transactional-provider-bytes.md | 15 ++++++----- .../contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md | 4 +-- src/backup.js | 23 ++++++++--------- test/fast-sync.test.js | 25 +++++++++++++++++-- test/in-place-transaction.test.js | 2 +- test/sync-service.test.js | 4 +-- 9 files changed, 51 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5433a5b..4e59e2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - 恢复 #51 等长 provider 原地更新，将字节恢复纳入 #71 逐目标事务；覆盖 Node POSIX 和 Windows worker。 - 合并正文扫描；新增显式 `sync --fast` / `switch --fast`，只读首行、保留模型、提示未执行检查，不支持原地更新时不隐式重写全文。 -- 含原地变更的备份使用 v3，旧工具不能恢复；其余保持 v2。稳定文件保留原 mtime，并处理并发追加与回滚，保持 History 原有排序和去重规则。详见[兼容边界](docs/adr/proposed-transactional-provider-bytes.md)。 +- 备份保持官方 v2，并继续支持 v1/v2 恢复；字节记录是可选扩展，不改变旧工具所需的标准恢复数据。稳定文件保留原 mtime，并处理并发追加与回滚。详见[兼容边界](docs/adr/proposed-transactional-provider-bytes.md)。 ## [0.5.0] - 2026-08-15 diff --git a/README.md b/README.md index 757ace2..d4ee6bb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ codex-provider sync `switch` 默认会在目标 Provider section 定义了 `model` 时同步根级 `model`。使用 `--keep-root-model` 保留当前值，或使用 `--model ` 显式指定。 -本分支新增 `sync --fast` / `switch --fast`：只读 rollout 首行，保留模型，不检查历史用户消息和加密内容；不支持原地更新时在写入前报错，不自动全量重写。备份、事务和恢复检查仍然执行；含原地变更的备份需兼容 v3 的工具恢复，旧 .NET GUI 不支持。详见[设计与兼容边界](docs/adr/proposed-transactional-provider-bytes.md)。 +本分支新增 `sync --fast` / `switch --fast`：只读 rollout 首行，保留模型，不检查历史用户消息和加密内容；不支持原地更新时在写入前报错，不自动全量重写。备份继续使用官方 v2 格式并支持恢复 v1/v2；旧工具按原有方式恢复，新代码可利用可选字节记录原地恢复。详见[设计与兼容边界](docs/adr/proposed-transactional-provider-bytes.md)。 建议使用统一长度的 ASCII Provider ID，优先 6 个字符（如将 `provider_a` 写作 `prov_a`），因为内置 `openai` 是 6 个字符，相对最通用；历史文件很多或很大时，不等长替换会重写整个 rollout、产生大量硬盘写入，等长且符合条件时可 in-place 替换。 diff --git a/docs/README_EN.md b/docs/README_EN.md index 463fc4c..73a98b6 100644 --- a/docs/README_EN.md +++ b/docs/README_EN.md @@ -109,7 +109,7 @@ codex-provider sync By default, `switch` also updates the root-level `model` when the target provider section defines one. Use `--keep-root-model` to preserve the current value, or `--model ` to set it explicitly. -This branch adds `sync --fast` / `switch --fast`: read only rollout headers, preserve models, and leave historical user-message/encryption checks unperformed. Unsupported in-place updates fail before mutation, without automatic full rewriting. Backups, transactions and recovery checks remain enabled; backups containing in-place mutations need a v3-compatible restore tool, not the old .NET GUI. See [design and compatibility](adr/proposed-transactional-provider-bytes.md). +This branch adds `sync --fast` / `switch --fast`: read only rollout headers, preserve models, and leave historical user-message/encryption checks unperformed. Unsupported in-place updates fail before mutation, without automatic full rewriting. Backups retain the official v2 format and v1/v2 restore support; older tools use their existing restore path, while updated code can use optional byte records for in-place recovery. See [design and compatibility](adr/proposed-transactional-provider-bytes.md). We recommend uniform-length ASCII provider IDs, preferably six characters (for example, `provider_a` as `prov_a`), because built-in `openai` has six characters and is the most common compatibility target. With many or large histories, different lengths require whole-rollout rewrites and substantial disk writes; eligible equal-length values can be replaced in place. diff --git a/docs/adr/proposed-transactional-provider-bytes.md b/docs/adr/proposed-transactional-provider-bytes.md index 795b9a9..d25d027 100644 --- a/docs/adr/proposed-transactional-provider-bytes.md +++ b/docs/adr/proposed-transactional-provider-bytes.md @@ -23,9 +23,10 @@ inside that journal, not around it. A separate fast scope addresses body reads. never backdate again. Recovery also repairs this interrupted timestamp step. Windows preserves timestamps under its exclusive handle. `updated_at` stays unchanged; History's existing ordering and duplicate selection remain intact. -- Metadata and manifest v3 prevent old readers from silently applying the - wrong restore strategy. Only backups containing byte mutations require v3; - all others, including fast no-op backups, keep v2. Old backups remain readable. +- Keep official v2 metadata/manifests and v1/v2 restore support. `mutation` is + optional: all standard original header/model/time/DB fields remain intact. + Older readers may ignore it and use their existing restore path; updated + readers use it for guarded byte-level undo. No private format version is added. - Opt-in `--fast` narrows diagnostics/model scope, never durability or recovery. Unsupported headers fail preflight rather than silently copying the body. - Full mode fuses encryption, user-event and model checks in one body pass. @@ -53,9 +54,11 @@ old-backup paths retain their existing active-writer limitations. ## Acceptance questions -Current .NET rejects v3 safely but cannot recover it. Maintainer approval is -required for this migration boundary, or a compatible recovery reader must be -added before release. Windows Node support is not .NET application parity. +Format interoperability does not retrofit in-place recovery into older tools: +their original full-rewrite and active-writer restrictions still apply. Verify +both directions with unmodified upstream Node/.NET readers and producers, as +well as recovery from applying-only partial writes. This does not claim that +old binaries implement the new inode-preserving strategy. Fast scope is new behavior: discuss it separately from restoring #51. If the unmerged #90 becomes main, integrate scope into its Plan/Revision/Restore v2 diff --git a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md index 535c80f..f6e7083 100644 --- a/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md +++ b/docs/architecture/contracts/CORE_EXTERNAL_BEHAVIOR_ZH.md @@ -13,12 +13,12 @@ 本分支候选增量（未发布，待上游评审）： - Node `runSync` / `runSwitch` 新增 `fast=false`。默认保留原检查范围，合并为一次正文扫描。 -- 合格的等长 provider 使用 manifest v3 描述的原地 mutation；执行、回滚和崩溃恢复保留同一 inode/file ID，开始写入后不得回退全文重写。 +- 合格的等长 provider 使用官方 v2 manifest 中的可选 mutation 记录；新代码执行、回滚和崩溃恢复保留同一 inode/file ID，开始写入后不得回退全文重写。 - 稳定文件保留原 mtime；POSIX 检测并修复 stat/utimes 期间的追加竞争，Windows 使用独占句柄；`updated_at` 和 History 的去重/排序规则不变。 - `fast=true` 只读首行（上限 1 MiB）及属性，保留根级/历史模型，不检查用户事件与加密内容；保留 provider、首行 cwd、workspace 修复及数据库事务。 - 快速模式静态不支持项在备份和业务写入前以 `FAST_MODE_UNSUPPORTED` 失败，不隐式全量重写；运行时 busy/changed 沿用 partial 分类。 - 结果增加 `inPlaceSessionFiles`；快速结果另含 `scanScope="metadata"`、`unchecked=["historyModels","userEventFlags","encryptedContent"]`、`encryptedContentCounts=null` 和未检查警告。 -- 仅含原地变更的备份使用 metadata 和 manifest v3；其余保持 v2。旧备份保留原恢复语义，旧 .NET 仅安全拒绝 v3，不宣称互相恢复。 +- metadata/manifest 均保持官方 v2，继续读取 v1/v2；标准恢复字段不变。旧 Node/.NET 可忽略新增记录并按原流程恢复，不代表旧版本自动具备原地恢复能力。 - 详见[候选 ADR](../../adr/proposed-transactional-provider-bytes.md)。V1/#90 的 Plan/Revision/Restore v2 接入不属于本分支已完成能力。 本文冻结 vNext 迁移开始时 Node 实现已经提供的外部行为。这里的“外部”不仅指 npm 最终用户，也包括当前 CLI 与 Local Web UI 对 Node service 的真实依赖。 diff --git a/src/backup.js b/src/backup.js index e10f4da..7095186 100644 --- a/src/backup.js +++ b/src/backup.js @@ -296,11 +296,8 @@ export async function createBackup({ } const globalStateFiles = await backupGlobalStateFiles(codexHome, backupDir); - // Both versions must advance so old readers reject before restoring any - // config/SQLite data, even when rollout restore was disabled by the caller. - const backupVersion = sessionChanges.some((change) => change.inPlaceMutation) ? 3 : 2; const sessionManifest = { - version: backupVersion, + version: 2, namespace: BACKUP_NAMESPACE, codexHome, targetProvider, @@ -316,7 +313,9 @@ export async function createBackup({ originalFirstLine: change.originalFirstLine, originalSeparator: change.originalSeparator, originalMtimeMs: change.originalMtimeMs, - mutation: change.inPlaceMutation ?? null, + // Optional byte-level undo; the standard v2 fields remain sufficient + // for older readers to use their existing full-header restore path. + ...(change.inPlaceMutation ? { mutation: change.inPlaceMutation } : {}), // Per-line record of the original turn_context.model values // so a failed rollback can put the per-turn model back to // what it was before the sync. Without this, a restore @@ -335,7 +334,7 @@ export async function createBackup({ ); await writeMetadataWithInventory(backupDir, { - version: backupVersion, + version: 2, namespace: BACKUP_NAMESPACE, codexHome, sqliteHome: actualSqliteHome, @@ -357,8 +356,8 @@ export async function updateSessionBackupManifest(backupDir, sessionChanges, opt const sessionManifest = JSON.parse(await fs.readFile(manifestPath, "utf8")); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - // Legacy bookkeeping must not retroactively invent in-place undo evidence. - sessionManifest.version = Math.max(2, sessionManifest.version); + // Preserve the official v1-to-v2 model-snapshot upgrade. + if (sessionManifest.version !== 2) sessionManifest.version = 2; const filesByPath = new Map( (sessionManifest.files ?? []).map((entry) => [pathComparisonKey(entry.path), entry]) @@ -393,7 +392,7 @@ export async function refreshBackupInventory(backupDir, options = {}) { const normalizedBackupDir = path.resolve(backupDir); const metadataPath = path.join(normalizedBackupDir, "metadata.json"); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - if (metadata?.namespace !== BACKUP_NAMESPACE || !new Set([1, 2, 3]).has(metadata.version)) { + if (metadata?.namespace !== BACKUP_NAMESPACE || !new Set([1, 2]).has(metadata.version)) { throw new Error(`Unsupported backup metadata in ${metadataPath}.`); } await writeMetadataWithInventory(normalizedBackupDir, metadata, options); @@ -494,7 +493,7 @@ async function selectSessionRestoreEntries(backupDir, sessionManifest) { async function readValidatedBackupMetadata(backupDir, codexHome) { const metadataPath = path.join(backupDir, "metadata.json"); const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); - if (metadata.namespace !== BACKUP_NAMESPACE || ![1, 2, 3].includes(metadata.version)) { + if (metadata.namespace !== BACKUP_NAMESPACE || ![1, 2].includes(metadata.version)) { throw new Error(`Unsupported backup metadata in ${metadataPath}.`); } if (typeof metadata.codexHome !== "string" || !storagePathsEqual(metadata.codexHome, codexHome)) { @@ -536,7 +535,7 @@ export async function getBackupRecoveryCoverage(backupDir, storageOrCodexHome) { const sessionManifestPath = path.join(backupDir, "session-meta-backup.json"); const sessionManifest = JSON.parse(await fs.readFile(sessionManifestPath, "utf8")); - if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2, 3].includes(sessionManifest.version)) { + if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2].includes(sessionManifest.version)) { throw new Error(`Unsupported session backup manifest in ${sessionManifestPath}.`); } if (typeof sessionManifest.codexHome !== "string" @@ -601,7 +600,7 @@ export async function restoreBackup(backupDir, storageOrCodexHome, options = {}) if (restoreSessions) { const sessionManifestPath = path.join(backupDir, "session-meta-backup.json"); sessionManifest = JSON.parse(await fs.readFile(sessionManifestPath, "utf8")); - if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2, 3].includes(sessionManifest.version)) { + if (sessionManifest.namespace !== BACKUP_NAMESPACE || ![1, 2].includes(sessionManifest.version)) { throw new Error(`Unsupported session backup manifest in ${sessionManifestPath}.`); } if (typeof sessionManifest.codexHome !== "string" diff --git a/test/fast-sync.test.js b/test/fast-sync.test.js index b7b0891..178022e 100644 --- a/test/fast-sync.test.js +++ b/test/fast-sync.test.js @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import test, { afterEach } from "node:test"; import { fileURLToPath } from "node:url"; -import { collectSessionChanges } from "../src/session-files.js"; +import { collectSessionChanges, restoreSessionChanges } from "../src/session-files.js"; import { runRestore, runSwitch, runSync } from "../src/service.js"; import { openDatabase } from "../src/sqlite.js"; import { findPendingTransactions } from "../src/transaction-journal.js"; @@ -75,7 +75,7 @@ test("fast switch/restore never open a rollout body stream and preserve models", assert.deepEqual(await row(f), { ...originalRow, model_provider: "prov_a", cwd: "/workspace/test" }); assert.equal(await fs.readFile(path.join(f.home, "config.toml"), "utf8"), f.config.replace('"openai"', '"prov_a"')); const metadata = JSON.parse(await fs.readFile(path.join(result.backupDir, "metadata.json"))); - assert.equal(metadata.version, 3); + assert.equal(metadata.version, 2); assert.equal(metadata.scanScope, "metadata"); guarded = true; await runRestore({ codexHome: f.home, backupDir: result.backupDir }); @@ -180,6 +180,27 @@ test("fast mode without a byte mutation keeps the compatible v2 backup format", await runRestore({ codexHome: f.home, backupDir: result.backupDir }); }); +test("standard v2 fields suffice when a reader ignores optional byte-undo records", async (t) => { + const f = await fixture(t); + const original = await fs.readFile(f.file); + const originalRow = await row(f); + const result = await runSwitch({ codexHome: f.home, provider: "prov_a", fast: true }); + const manifest = JSON.parse(await fs.readFile(path.join(result.backupDir, "session-meta-backup.json"))); + assert.equal(manifest.version, 2); + assert.ok(manifest.files[0].mutation); + // Project only the official v2 fields, as older readers do. Do not modify + // the immutable managed backup just to simulate an older restore path. + const entries = manifest.files.map(({ path, originalFirstLine, originalSeparator, + originalMtimeMs, originalTurnContextModels, modelOnlyChange }) => ({ + path, originalFirstLine, originalSeparator, originalMtimeMs, originalTurnContextModels, modelOnlyChange + })); + await restoreSessionChanges(entries); + await runRestore({ codexHome: f.home, backupDir: result.backupDir, restoreSessions: false }); + assert.deepEqual(await fs.readFile(f.file), original); + assert.deepEqual(await row(f), originalRow); + assert.equal(await fs.readFile(path.join(f.home, "config.toml"), "utf8"), f.config); +}); + test("manual restore preflights unknown provider bytes before changing config or SQLite", async (t) => { const f = await fixture(t); const result = await runSwitch({ codexHome: f.home, provider: "prov_a", fast: true }); diff --git a/test/in-place-transaction.test.js b/test/in-place-transaction.test.js index 40d7eb6..6a80625 100644 --- a/test/in-place-transaction.test.js +++ b/test/in-place-transaction.test.js @@ -299,7 +299,7 @@ test("durable manifest and applying precede mutation; observer failure rolls bac immutable = await fs.readFile(path.join(backup, "session-meta-backup.json")); assert.equal(pending[0].events.at(-1).state, "applying"); const manifest = JSON.parse(immutable); - assert.equal(manifest.version, 3); + assert.equal(manifest.version, 2); assert.ok(manifest.files[0].mutation.originalBase64); assert.deepEqual(await fs.readFile(file), f.original); } diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 3d8e69c..298680d 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -1703,7 +1703,7 @@ test("runSync rewrites rollout files and sqlite, then restore reverts both", asy assert.deepEqual(syncResult.skippedLockedRolloutFiles, []); assert.equal(syncResult.sqliteRowsUpdated, 2); const backupMetadata = JSON.parse(await fs.readFile(path.join(syncResult.backupDir, "metadata.json"), "utf8")); - assert.equal(backupMetadata.version, 3); + assert.equal(backupMetadata.version, 2); assert.equal(backupMetadata.sqliteHome, path.join(codexHome, SQLITE_DIR_BASENAME)); assert.deepEqual(backupMetadata.sqliteDbFiles, [DB_FILE_BASENAME]); assert.ok(Number.isSafeInteger(backupMetadata.sizeBytes)); @@ -1835,7 +1835,7 @@ test("runSync uses an explicit SQLite home and never touches a stale Codex Home } const metadata = JSON.parse(await fs.readFile(path.join(result.backupDir, "metadata.json"), "utf8")); - assert.equal(metadata.version, 3); + assert.equal(metadata.version, 2); assert.equal(metadata.sqliteHome, sqliteHome); assert.deepEqual(metadata.dbFiles, []); assert.deepEqual(metadata.sqliteDbFiles, [DB_FILE_BASENAME]); From 782a90128b817c56b3d559ca4a4d686b1d353159 Mon Sep 17 00:00:00 2001 From: cccat6 <22387156+cccat6@users.noreply.github.com> Date: Fri, 28 Aug 2026 18:50:36 +0800 Subject: [PATCH 5/5] test: isolate provider fixtures from external SQLite homes --- test/fast-sync.test.js | 2 ++ test/in-place-transaction.test.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/fast-sync.test.js b/test/fast-sync.test.js index 178022e..9c6fab0 100644 --- a/test/fast-sync.test.js +++ b/test/fast-sync.test.js @@ -11,6 +11,8 @@ import { runRestore, runSwitch, runSync } from "../src/service.js"; import { openDatabase } from "../src/sqlite.js"; import { findPendingTransactions } from "../src/transaction-journal.js"; +delete process.env.CODEX_SQLITE_HOME; + const cli = fileURLToPath(new URL("../src/cli.js", import.meta.url)); const cleanups = []; afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); }); diff --git a/test/in-place-transaction.test.js b/test/in-place-transaction.test.js index 6a80625..e4e055f 100644 --- a/test/in-place-transaction.test.js +++ b/test/in-place-transaction.test.js @@ -13,6 +13,8 @@ import { runRestore, runSync } from "../src/service.js"; import { listHistory } from "../src/history.js"; import { TransactionJournal, readTransactionJournal, findPendingTransactions } from "../src/transaction-journal.js"; +delete process.env.CODEX_SQLITE_HOME; + const repo = fileURLToPath(new URL("..", import.meta.url)); const cleanups = []; afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); });