Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 164 additions & 0 deletions apps/server/src/atomicWrite.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { describe, expect, it } from "@effect/vitest";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as PlatformError from "effect/PlatformError";
import * as Fiber from "effect/Fiber";
import * as TestClock from "effect/testing/TestClock";
import { writeFileStringAtomically } from "./atomicWrite.ts";

describe("durable atomic text replacement", () => {
Expand Down Expand Up @@ -74,4 +77,165 @@ describe("durable atomic text replacement", () => {
expect(yield* fs.readDirectory(directory)).toEqual(["document.md"]);
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("retries a transient Windows destination lock without exposing a partial file", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const directory = yield* fs.makeTempDirectoryScoped();
const target = `${directory}/document.md`;
yield* fs.writeFileString(target, "original");
const locked = PlatformError.systemError({
_tag: "Unknown",
module: "FileSystem",
method: "rename",
cause: Object.assign(new Error("destination is temporarily locked"), { code: "EPERM" }),
});
let attempts = 0;
const firstRename = yield* Deferred.make<void>();
const observed = FileSystem.make({
...fs,
rename: (from, to) => {
attempts += 1;
return Deferred.succeed(firstRename, undefined).pipe(
Effect.andThen(attempts < 3 ? Effect.fail(locked) : fs.rename(from, to)),
);
},
});
const writing = yield* writeFileStringAtomically({
filePath: target,
contents: "replacement",
}).pipe(
Effect.provideService(FileSystem.FileSystem, observed),
Effect.provideService(HostProcessPlatform, "win32"),
Effect.forkChild,
);
yield* Deferred.await(firstRename);
yield* TestClock.adjust("1 second");
yield* Fiber.join(writing);
expect(attempts).toBe(3);
expect(yield* fs.readFileString(target)).toBe("replacement");
expect(yield* fs.readDirectory(directory)).toEqual(["document.md"]);
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("does not retry a permanent rename error or a transient error off Windows", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const directory = yield* fs.makeTempDirectoryScoped();
const denied = PlatformError.systemError({
_tag: "PermissionDenied",
module: "FileSystem",
method: "rename",
cause: Object.assign(new Error("denied"), { code: "EPERM" }),
});
for (const [platform, error] of [
["linux", denied],
[
"win32",
PlatformError.systemError({
_tag: "Unknown",
module: "FileSystem",
method: "rename",
cause: Object.assign(new Error("different volume"), { code: "EXDEV" }),
}),
],
] as const) {
let attempts = 0;
const failing = FileSystem.make({
...fs,
rename: () => {
attempts += 1;
return Effect.fail(error);
},
});
const outcome = yield* writeFileStringAtomically({
filePath: `${directory}/${platform}.md`,
contents: "replacement",
}).pipe(
Effect.provideService(FileSystem.FileSystem, failing),
Effect.provideService(HostProcessPlatform, platform),
Effect.exit,
);
expect(outcome._tag).toBe("Failure");
expect(attempts).toBe(1);
}
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("bounds retries when a Windows destination lock does not clear", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const directory = yield* fs.makeTempDirectoryScoped();
const target = `${directory}/document.md`;
yield* fs.writeFileString(target, "original");
const firstRename = yield* Deferred.make<void>();
let attempts = 0;
const locked = PlatformError.systemError({
_tag: "Unknown",
module: "FileSystem",
method: "rename",
cause: Object.assign(new Error("destination remains locked"), { code: "EACCES" }),
});
const failing = FileSystem.make({
...fs,
rename: () => {
attempts += 1;
return Deferred.succeed(firstRename, undefined).pipe(Effect.andThen(Effect.fail(locked)));
},
});
const writing = yield* writeFileStringAtomically({
filePath: target,
contents: "replacement",
}).pipe(
Effect.provideService(FileSystem.FileSystem, failing),
Effect.provideService(HostProcessPlatform, "win32"),
Effect.exit,
Effect.forkChild,
);
yield* Deferred.await(firstRename);
yield* TestClock.adjust("6 seconds");
const outcome = yield* Fiber.join(writing);
expect(outcome._tag).toBe("Failure");
expect(attempts).toBeGreaterThan(1);
expect(yield* fs.readFileString(target)).toBe("original");
expect(yield* fs.readDirectory(directory)).toEqual(["document.md"]);
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("remains cancellable while a transient Windows lock is backing off", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const directory = yield* fs.makeTempDirectoryScoped();
const target = `${directory}/document.md`;
yield* fs.writeFileString(target, "original");
const firstRename = yield* Deferred.make<void>();
let attempts = 0;
const locked = PlatformError.systemError({
_tag: "Unknown",
module: "FileSystem",
method: "rename",
cause: Object.assign(new Error("destination is temporarily locked"), { code: "EBUSY" }),
});
const failing = FileSystem.make({
...fs,
rename: () => {
attempts += 1;
return Deferred.succeed(firstRename, undefined).pipe(Effect.andThen(Effect.fail(locked)));
},
});
const writing = yield* writeFileStringAtomically({
filePath: target,
contents: "replacement",
}).pipe(
Effect.provideService(FileSystem.FileSystem, failing),
Effect.provideService(HostProcessPlatform, "win32"),
Effect.forkChild,
);
yield* Deferred.await(firstRename);
yield* Fiber.interrupt(writing);
expect(attempts).toBe(1);
expect(yield* fs.readFileString(target)).toBe("original");
expect(yield* fs.readDirectory(directory)).toEqual(["document.md"]);
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);
});
48 changes: 42 additions & 6 deletions apps/server/src/atomicWrite.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,29 @@
import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
import * as Clock from "effect/Clock";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import type * as PlatformError from "effect/PlatformError";

const WINDOWS_RENAME_RETRY_BUDGET_MS = 5_000;

function platformErrorCode(error: PlatformError.PlatformError): unknown {
const cause = error.reason.cause;
return typeof cause === "object" && cause !== null && "code" in cause ? cause.code : undefined;
}

function windowsRenameRetryDelay(
error: PlatformError.PlatformError,
elapsedMs: number,
attempt: number,
): number | undefined {
if (
elapsedMs >= WINDOWS_RENAME_RETRY_BUDGET_MS ||
!["EPERM", "EACCES", "EBUSY"].includes(String(platformErrorCode(error)))
)
return undefined;
return Math.min(25 * 2 ** Math.min(attempt, 5), 500, WINDOWS_RENAME_RETRY_BUDGET_MS - elapsedMs);
}

export const writeFileStringAtomically = (input: {
readonly filePath: string;
Expand Down Expand Up @@ -34,7 +56,25 @@ export const writeFileStringAtomically = (input: {
}),
);
else if (input.mode !== undefined) yield* fs.chmod(tempPath, input.mode);
yield* fs.rename(tempPath, input.filePath);
// Windows can temporarily deny replacement while another process has the
// destination open. Keep the staged file private and retry only that
// narrow class of errors; all other failures preserve fail-fast behavior.
const started = yield* Clock.currentTimeMillis;
let attempt = 0;
for (;;) {
const renamed = yield* Effect.result(fs.rename(tempPath, input.filePath));
if (renamed._tag === "Success") break;
const delay =
platform === "win32"
? windowsRenameRetryDelay(
renamed.failure,
(yield* Clock.currentTimeMillis) - started,
attempt++,
)
: undefined;
if (delay === undefined) return yield* renamed.failure;
yield* Effect.sleep(delay);
}
// Windows does not support opening directory handles through this API.
// On POSIX, persist the rename as well as the temporary file's contents.
if (input.durable && platform !== "win32") {
Expand All @@ -45,11 +85,7 @@ export const writeFileStringAtomically = (input: {
}),
).pipe(
Effect.catch((error) => {
const cause = error.reason.cause;
const code =
typeof cause === "object" && cause !== null && "code" in cause
? cause.code
: undefined;
const code = platformErrorCode(error);
return code === "EINVAL" || code === "ENOTSUP" || code === "EOPNOTSUPP"
? Effect.void
: Effect.fail(error);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,16 @@ describe.runIf(Boolean(TEST_PYTHON) && NodeProcess.platform !== "win32")(
]);
}
});
const waitForLsofProcess = Effect.fn("PortScannerPythonCompute.waitForLsofProcess")(
(processId: number) =>
eventually(
scanConfiguredFixture(1).pipe(
Effect.map(() =>
lsofObservedProcess(lsofSnapshots, processId) ? true : null,
),
),
),
);
const waitForExecution = (executionId: ComputeExecutionId) =>
eventually(
compute.listExecutions({ projectId: PROJECT, sessionId: SESSION }).pipe(
Expand Down Expand Up @@ -302,7 +312,7 @@ describe.runIf(Boolean(TEST_PYTHON) && NodeProcess.platform !== "win32")(
label: "FIRST",
});
yield* scanConfiguredFixture(3);
expect(lsofObservedProcess(lsofSnapshots, firstRuntimeProcessId)).toBe(true);
yield* waitForLsofProcess(firstRuntimeProcessId);
expectOnlyFixtureTraffic();
expect(
yield* compute.listOutputs({
Expand Down Expand Up @@ -333,8 +343,7 @@ describe.runIf(Boolean(TEST_PYTHON) && NodeProcess.platform !== "win32")(
yield* eventually(
Effect.sync(() => (!processExists(firstRuntimeProcessId) ? true : null)),
);
yield* scanConfiguredFixture(2);
expect(lsofObservedProcess(lsofSnapshots, restartedRuntimeProcessId)).toBe(true);
yield* waitForLsofProcess(restartedRuntimeProcessId);
expect((yield* computeOwnedEndpoints.snapshot).protectedLoopbackTcpPorts.size).toBe(
5,
);
Expand Down
Loading
Loading