diff --git a/.github/fork-seams.json b/.github/fork-seams.json index b0a43c3945cae..bf92cac396202 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -92,21 +92,13 @@ ] }, "apps/server/src/cli/connect.ts": { - "why": "A server held with `serve --hold-work` shares the live server's environment id, so `t3 connect --disconnect` refuses before the relay delete (cloud/holdWorkConnect.ts). Announced for #398: that call becomes one line, and the CLI's Connect-on check (requireConnectAllowed) refuses on a held copy too.", + "why": "`serve --hold-work`: a held copy's home shares the live server's T3 Connect environment id and sign-in. The CLI's unlink path (disconnectCloud) and its turn-Connect-on check (requireConnectAllowed) each call the refusal in cloud/holdWorkConnect.ts once.", "added": [ "import * as HoldWorkConnect from \"../cloud/holdWorkConnect.ts\";", "// Fork: a held copy shares the live server's environment id; refuse before the relay delete.", - "yield* HoldWorkConnect.refuseCliUnlinkWhileHeld(", - "(yield* ServerConfig.ServerConfig).serverRuntimeStatePath,", - ");" - ], - "announced": { - "since": "2026-11-18", - "added": [ - "yield* HoldWorkConnect.refuseCliOnHeldCopy(yield* ServerConfig.ServerConfig);", - "yield* HoldWorkConnect.refuseCliOnHeldCopy(yield* ServerConfig.ServerConfig); // Fork: a held copy" - ] - } + "yield* HoldWorkConnect.refuseCliOnHeldCopy(yield* ServerConfig.ServerConfig);", + "yield* HoldWorkConnect.refuseCliOnHeldCopy(yield* ServerConfig.ServerConfig); // Fork: a held copy" + ] }, "apps/server/src/cloud/CloudLink.ts": { "why": "unlink's error type takes the hold-work refusal (cloud/holdWorkConnect.ts); the one upstream line is split so the fork's error sits on its own line.", @@ -217,6 +209,7 @@ ".gitleaks.toml", ".gitleaksignore", "apps/desktop/src/settings/DesktopClientSettings.removedKeys.test.ts", + "apps/server/src/cli/connect.holdWork.test.ts", "apps/server/src/cli/server.holdWork.test.ts", "apps/server/src/cloud/autoUpdate*", "apps/server/src/cloud/holdWorkConnect*", diff --git a/apps/server/src/cli/connect.holdWork.test.ts b/apps/server/src/cli/connect.holdWork.test.ts new file mode 100644 index 0000000000000..76ddb30ceabf4 --- /dev/null +++ b/apps/server/src/cli/connect.holdWork.test.ts @@ -0,0 +1,188 @@ +// Fork (`serve --hold-work`): `t3 connect` and `t3 up` on a held copy's home, where they are called. +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as NetService from "@t3tools/shared/Net"; +import * as Cause from "effect/Cause"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as TestConsole from "effect/testing/TestConsole"; +import { Command } from "effect/cli"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import { vi } from "vite-plus/test"; + +import packageJson from "../../package.json" with { type: "json" }; +import { makeCli } from "../binCli.ts"; +import * as BootService from "../cloud/bootService.ts"; +import * as HoldWorkConnect from "../cloud/holdWorkConnect.ts"; +import { upCommand } from "./up.ts"; + +// `t3 up` offers T3 Connect only in builds with its public config. +vi.mock("../cloud/publicConfig.ts", async (importOriginal) => ({ + ...(await importOriginal()), + hasCloudPublicConfig: true, +})); + +const RELAY = "https://relay.hold-work.test"; + +/** What a full copy of a linked home carries: its link and a CLI sign-in far from expiry. */ +const LINKED_SECRETS: Record = { + "cloud-cli-desired-link": "managed", + "cloud-linked-user-id": "user-hold-work", + "cloud-relay-url": RELAY, + "cloud-cli-oauth-token": JSON.stringify({ + accessToken: "access-hold-work", + refreshToken: "refresh-hold-work", + expiresAtEpochMs: 4_102_444_800_000, + }), +}; + +type Run = "unlink" | "logout" | "link" | "login" | "publish" | "up" | "up-off"; + +const ARGS: Record, ReadonlyArray> = { + unlink: ["connect", "unlink"], + logout: ["connect", "logout"], + link: ["connect", "link", "--publish-only", "--headless"], + login: ["connect", "login", "--headless"], + // Turning publishing on writes `cloud-publish-agent-activity`, which the copy has not set. + publish: ["connect", "publish"], +}; + +const runOnHome = (run: Run, options: { readonly marked: boolean }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-connect-held-copy-" }); + const secretsDir = `${baseDir}/userdata/secrets`; + yield* fs.makeDirectory(secretsDir, { recursive: true }); + for (const [name, value] of Object.entries(LINKED_SECRETS)) { + // `t3 up --connect` only links a home whose Connect is off. + if (run === "up" && name === "cloud-cli-desired-link") continue; + yield* fs.writeFileString(`${secretsDir}/${name}.bin`, value); + } + if (options.marked) { + yield* fs.writeFileString(`${baseDir}/${HoldWorkConnect.HELD_COPY_MARKER_FILE}`, "{}"); + } + // The Connect link and sign-in; any CLI run makes the server signing key when it is missing. + const secrets = Effect.gen(function* () { + const names = (yield* fs.readDirectory(secretsDir)) + .filter((name) => name.startsWith("cloud-")) + .toSorted(); + const entries: Array<[string, string]> = []; + for (const name of names) { + entries.push([name, yield* fs.readFileString(`${secretsDir}/${name}`)]); + } + return Object.fromEntries(entries); + }); + const before = yield* secrets; + + // Every request the CLI makes: relay, sign-in and relay client download alike. + const requests: Array = []; + const fetch: typeof globalThis.fetch = (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + requests.push(`${method} ${url}`); + return Promise.resolve(Response.json({ ok: true })); + }; + const restarts: Array = []; + vi.spyOn(BootService, "layer").mockReturnValue( + Layer.succeed( + BootService.BootService, + BootService.BootService.of({ + status: Effect.succeed({ + supported: true, + installed: true, + current: true, + unitPath: "/home/me/.config/systemd/user/t3code.service", + logPath: "/home/me/.t3/userdata/logs/boot-service.log", + }), + withStopped: (operation) => operation, + restart: Effect.sync(() => restarts.push("restart")).pipe(Effect.as("skipped" as const)), + install: () => Effect.die("a current service is not reinstalled"), + uninstall: Effect.succeed(false), + }), + ), + ); + if (run === "up" || run === "up-off") { + yield* fs.writeFileString(`${baseDir}/service.env`, "T3CODE_LAN=false\n"); + } + + const command = + run === "up" || run === "up-off" + ? Command.runWith(upCommand, { version: packageJson.version })([ + "--base-dir", + baseDir, + run === "up" ? "--connect" : "--no-connect", + ]) + : Command.runWith(makeCli({ cloudEnabled: true }), { version: "0.0.0" })([ + ...ARGS[run], + "--base-dir", + baseDir, + ]); + const exit = yield* command.pipe( + Effect.provideService(FetchHttpClient.Fetch, fetch), + Effect.provideService(HostProcess.Environment, {}), + Effect.exit, + ); + return { + error: Exit.isFailure(exit) ? Cause.squash(exit.cause) : null, + requests, + restarts, + secretsBefore: before, + secretsAfter: yield* secrets, + }; + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer( + ConfigProvider.fromEnv({ + env: { + T3CODE_RELAY_URL: RELAY, + // No managed policy, whatever this machine has: a missing file means none. + T3CODE_DEV_MANAGED_POLICY_PATH: "/nonexistent/t3-hold-work-policy.json", + }, + }), + ), + Layer.fresh(TestConsole.layer), + ), + ), + ); + +it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer))( + "t3 connect and t3 up on a held copy's home", + (it) => { + it.effect("unlink on an unmarked home sends the relay delete (what the refusals stop)", () => + Effect.gen(function* () { + const result = yield* runOnHome("unlink", { marked: false }); + expect(result.error).toBeNull(); + expect(result.requests).toHaveLength(1); + expect(result.requests[0]).toMatch( + new RegExp(`^DELETE ${RELAY}/v1/client/environment-links/[^/]+$`), + ); + }), + ); + + it.effect.each([ + { run: "unlink" as const }, + { run: "logout" as const }, + { run: "link" as const }, + { run: "login" as const }, + { run: "publish" as const }, + { run: "up" as const }, + // `t3 up` turning Connect off: its unlink ends in the relay delete. + { run: "up-off" as const }, + ])("$run on a marked home is refused before any request or change", ({ run }) => + Effect.gen(function* () { + const result = yield* runOnHome(run, { marked: true }); + expect(result.error).toBeInstanceOf(HoldWorkConnect.HeldCopyRefusedError); + expect(result.requests).toEqual([]); + expect(result.restarts).toEqual([]); + expect(result.secretsAfter).toEqual(result.secretsBefore); + }), + ); + }, +); diff --git a/apps/server/src/cli/connect.ts b/apps/server/src/cli/connect.ts index 6853ce81b49d2..f0f5c22f88ece 100644 --- a/apps/server/src/cli/connect.ts +++ b/apps/server/src/cli/connect.ts @@ -509,9 +509,7 @@ export const disconnectCloud = Effect.fn("cloud.cli.disconnect")(function* (opti readonly clearAuthorization: boolean; }) { // Fork: a held copy shares the live server's environment id; refuse before the relay delete. - yield* HoldWorkConnect.refuseCliUnlinkWhileHeld( - (yield* ServerConfig.ServerConfig).serverRuntimeStatePath, - ); + yield* HoldWorkConnect.refuseCliOnHeldCopy(yield* ServerConfig.ServerConfig); yield* CliState.setCliDesiredCloudLink(false); const liveResult = yield* runLiveCloudUnlink(); const relayResult = yield* Effect.exit(unlinkRelayEnvironment()); @@ -582,6 +580,7 @@ export const runCloudCommand = Effect.fn("cloud.cli.run_cloud_command")(function /** Status, unlink, and logout stay available so a machine can leave Connect the policy turned off. */ export const requireConnectAllowed = Effect.gen(function* () { + yield* HoldWorkConnect.refuseCliOnHeldCopy(yield* ServerConfig.ServerConfig); // Fork: a held copy const policy = yield* ManagedPolicy.ManagedPolicy; if (!ManagedLimits.allowsConnect(policy)) { return yield* ManagedLimits.refuseConnect(policy); diff --git a/apps/server/src/cloud/holdWorkConnect.test.ts b/apps/server/src/cloud/holdWorkConnect.test.ts index ad772977d0617..50f7d04338b1a 100644 --- a/apps/server/src/cloud/holdWorkConnect.test.ts +++ b/apps/server/src/cloud/holdWorkConnect.test.ts @@ -7,6 +7,7 @@ import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as ServerConfig from "../config.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as WorkAdmission from "../workAdmission.ts"; import * as CloudLink from "./CloudLink.ts"; @@ -29,7 +30,11 @@ const recordingCloudLink = () => { return { calls, service }; }; -const cloudLinkFor = (heldFromBoot: boolean) => +/** A temp home, or `baseDir` when given (a marked copy's home, say). */ +const homeConfig = (baseDir?: string) => + ServerConfig.layerTest(process.cwd(), baseDir ?? { prefix: "t3-hold-work-link-" }); + +const cloudLinkFor = (heldFromBoot: boolean, baseDir?: string) => Effect.gen(function* () { const recording = recordingCloudLink(); const gate = yield* WorkAdmission.makeWorkAdmissionGate({ heldFromBoot }); @@ -37,6 +42,8 @@ const cloudLinkFor = (heldFromBoot: boolean) => Effect.provide( HoldWorkConnect.layer(Layer.succeed(CloudLink.CloudLink, recording.service)).pipe( Layer.provide(Layer.succeed(WorkAdmission.WorkAdmission, gate)), + Layer.provide(homeConfig(baseDir)), + Layer.provide(NodeServices.layer), ), ), ); @@ -91,7 +98,7 @@ describe("holdWorkConnect", () => { }); /** An agent activity publisher whose every method records its call. */ -const relayFor = (heldFromBoot: boolean) => +const relayFor = (heldFromBoot: boolean, baseDir?: string) => Effect.gen(function* () { const calls: Array = []; const record = (name: string) => () => Effect.sync(() => void calls.push(name)); @@ -108,7 +115,11 @@ const relayFor = (heldFromBoot: boolean) => start: record("start"), }), ), - ).pipe(Layer.provide(Layer.succeed(WorkAdmission.WorkAdmission, gate))), + ).pipe( + Layer.provide(Layer.succeed(WorkAdmission.WorkAdmission, gate)), + Layer.provide(homeConfig(baseDir)), + Layer.provide(NodeServices.layer), + ), ), ); return { calls, relay }; @@ -140,14 +151,164 @@ describe("holdWorkConnect agent activity", () => { ); }); -describe("holdWorkConnect CLI unlink", () => { - const refusalFor = (state: Record | null) => +describe("holdWorkConnect held-copy marker", () => { + /** Builds the held or normal CloudLink layer on `baseDir`; the fake CloudLink notes whether the marker was there when it was built. */ + const buildOn = (baseDir: string, heldFromBoot: boolean, markerAtBuild: Array = []) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const dir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-hold-work-cli-" }); - const statePath = `${dir}/server-runtime.json`; - if (state !== null) yield* fs.writeFileString(statePath, JSON.stringify(state)); - return yield* HoldWorkConnect.refuseCliUnlinkWhileHeld(statePath).pipe(Effect.exit); + const marker = `${baseDir}/${HoldWorkConnect.HELD_COPY_MARKER_FILE}`; + const gate = yield* WorkAdmission.makeWorkAdmissionGate({ heldFromBoot }); + const fake = Layer.effect( + CloudLink.CloudLink, + Effect.gen(function* () { + markerAtBuild.push(yield* fs.exists(marker)); + return recordingCloudLink().service; + }), + ); + yield* Effect.service(CloudLink.CloudLink).pipe( + Effect.provide( + HoldWorkConnect.layer(fake).pipe( + Layer.provide(Layer.succeed(WorkAdmission.WorkAdmission, gate)), + Layer.provide(ServerConfig.layerTest(process.cwd(), baseDir)), + ), + ), + ); + return { marker, markerAtBuild }; + }); + + it.effect( + "a held boot marks its home before CloudLink is built; a normal boot keeps the mark", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-hold-work-mark-" }); + const normal = yield* buildOn(baseDir, false); + expect(yield* fs.exists(normal.marker)).toBe(false); + + const held = yield* buildOn(baseDir, true); + expect(held.markerAtBuild).toEqual([true]); + const written = yield* fs.readFileString(held.marker); + expect(written).toContain("--hold-work"); + + // A normal boot of the copy later does not clear it: that boot is itself the unsafe step. + yield* buildOn(baseDir, false); + expect(yield* fs.readFileString(held.marker)).toBe(written); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("a held boot that cannot write its marker fails before CloudLink is built", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-hold-work-mark-" }); + // A normal boot first lays out the home, so only the marker write meets the read-only home. + yield* buildOn(baseDir, false); + yield* fs.chmod(baseDir, 0o500); + const built: Array = []; + const exit = yield* buildOn(baseDir, true, built).pipe(Effect.exit); + yield* fs.chmod(baseDir, 0o700); + // A held copy that booted unmarked would be unguarded once it stops. + expect(Exit.isFailure(exit) ? Cause.squash(exit.cause) : null).toMatchObject({ + reason: { _tag: "PermissionDenied" }, + }); + expect(built).toEqual([]); + expect(yield* fs.exists(`${baseDir}/${HoldWorkConnect.HELD_COPY_MARKER_FILE}`)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect( + "a second held start keeps the marker as it is, even one cut off empty or unreadable", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + for (const mode of [0o644, 0o000]) { + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-hold-work-mark-" }); + const marker = `${baseDir}/${HoldWorkConnect.HELD_COPY_MARKER_FILE}`; + // What a start stopped right after creating the file leaves behind. + yield* fs.writeFileString(marker, ""); + yield* fs.chmod(marker, mode); + const again = yield* buildOn(baseDir, true); + expect(again.markerAtBuild).toEqual([true]); + yield* fs.chmod(marker, 0o644); + expect(yield* fs.readFileString(marker)).toBe(""); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("holdWorkConnect normal boot of a marked copy", () => { + it.effect.each([ + { label: "a marker", marker: "file" as const }, + { label: "a marker a start cut off empty", marker: "empty" as const }, + { label: "a dangling symlink in the marker's place", marker: "dangling" as const }, + ])("holds Connect and agent activity on $label, as a held boot does", ({ marker }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-hold-work-marked-" }); + const markerPath = `${baseDir}/${HoldWorkConnect.HELD_COPY_MARKER_FILE}`; + if (marker === "dangling") yield* fs.symlink(`${baseDir}/gone`, markerPath); + else yield* fs.writeFileString(markerPath, marker === "empty" ? "" : "marked"); + + const { calls, cloudLink } = yield* cloudLinkFor(false, baseDir); + expect(yield* cloudLink.reconcileDesiredLinkIfStillDesired(ORIGIN)).toBeNull(); + expect(yield* cloudLink.registerManagedTunnelRecovery(ORIGIN)).toEqual({ + status: "not_linked", + }); + expect(yield* cloudLink.startManagedTunnelIfOriginConfirmed(ORIGIN)).toBe(false); + // The shutdown release would delete the running server's tunnel under the shared id. + expect(yield* cloudLink.releaseManagedTunnelOnShutdown()).toBe(false); + const refused = yield* Effect.flip(cloudLink.unlink()); + expect(refused).toEqual(HoldWorkConnect.heldConnectRefusal(markerPath)); + expect(refused.message).toContain(markerPath); + expect(calls).toEqual([]); + + const activity = yield* relayFor(false, baseDir); + yield* useRelay(activity.relay); + expect(activity.calls).toEqual([]); + // Nothing touched the marker. + if (marker !== "dangling") { + expect(yield* fs.readFileString(markerPath)).toBe(marker === "empty" ? "" : "marked"); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + +describe("holdWorkConnect CLI refusal on a held copy's home", () => { + const refusalFor = (home: { + readonly state?: Record; + readonly marker?: "file" | "empty" | "unreadable" | "directory" | "dangling"; + readonly secrets?: boolean; + readonly missing?: boolean; + readonly unsearchable?: boolean; + }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-hold-work-cli-" }); + const baseDir = home.missing ? `${root}/never-made` : root; + const serverRuntimeStatePath = `${baseDir}/userdata/server-runtime.json`; + const marker = `${baseDir}/${HoldWorkConnect.HELD_COPY_MARKER_FILE}`; + if (!home.missing) yield* fs.makeDirectory(`${baseDir}/userdata`, { recursive: true }); + if (home.state) yield* fs.writeFileString(serverRuntimeStatePath, JSON.stringify(home.state)); + if (home.secrets) { + // A linked home copied before markers existed: its link files, and no marker. + yield* fs.makeDirectory(`${baseDir}/userdata/secrets`, { recursive: true }); + yield* fs.writeFileString( + `${baseDir}/userdata/secrets/cloud-cli-desired-link.bin`, + "managed", + ); + } + if (home.marker === "directory") yield* fs.makeDirectory(marker); + else if (home.marker === "dangling") yield* fs.symlink(`${baseDir}/gone`, marker); + else if (home.marker !== undefined) { + yield* fs.writeFileString(marker, home.marker === "empty" ? "" : "{not json"); + if (home.marker === "unreadable") yield* fs.chmod(marker, 0o000); + } + if (home.unsearchable) yield* fs.chmod(baseDir, 0o600); + const exit = yield* HoldWorkConnect.refuseCliOnHeldCopy({ + baseDir, + serverRuntimeStatePath, + }).pipe(Effect.exit); + if (home.unsearchable) yield* fs.chmod(baseDir, 0o700); + return { marker, exit }; }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)); const running = { version: 1, @@ -156,25 +317,57 @@ describe("holdWorkConnect CLI unlink", () => { origin: ORIGIN, startedAt: "2026-01-01T00:00:00.000Z", }; + const refusal = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Cause.squash(exit.cause) : null; it.effect("refuses while this home's server runs held", () => Effect.gen(function* () { - const exit = yield* refusalFor({ ...running, workHeld: true }); - expect(Exit.isFailure(exit)).toBe(true); - const error = Exit.isFailure(exit) ? Cause.squash(exit.cause) : null; - expect(error).toBeInstanceOf(HoldWorkConnect.HeldCliUnlinkRefusedError); + const { exit } = yield* refusalFor({ state: { ...running, workHeld: true } }); + const error = refusal(exit); + expect(error).toBeInstanceOf(HoldWorkConnect.HeldCopyRefusedError); expect(String((error as Error).message)).toContain("--hold-work"); }), ); - it.effect("lets it run for a normal server, no server, or a held server that is gone", () => + it.effect.each([ + { label: "a marked home with no server", marker: "file" as const }, + { label: "a marker a start cut off empty", marker: "empty" as const }, + { label: "an unreadable marker", marker: "unreadable" as const }, + { label: "a directory in the marker's place", marker: "directory" as const }, + // The held boot's exclusive create sees it as there and keeps it, so the CLI must too. + { label: "a dangling symlink in the marker's place", marker: "dangling" as const }, + // Checking fails (no search permission on the home): counted as there. + { label: "a marker that cannot be checked", marker: "file" as const, unsearchable: true }, + ])("refuses on $label, and says how to clear it", ({ marker, unsearchable }) => Effect.gen(function* () { - expect(Exit.isSuccess(yield* refusalFor(running))).toBe(true); - expect(Exit.isSuccess(yield* refusalFor(null))).toBe(true); - // A pid that cannot be alive: the held server stopped and left its record behind. - expect( - Exit.isSuccess(yield* refusalFor({ ...running, pid: 2 ** 22 + 1, workHeld: true })), - ).toBe(true); + const result = yield* refusalFor({ + marker, + unsearchable: unsearchable === true, + state: running, + }); + const error = refusal(result.exit); + expect(error).toBeInstanceOf(HoldWorkConnect.HeldCopyRefusedError); + const message = String((error as Error).message); + expect(message).toContain(result.marker); + expect(message).toContain("delete"); }), ); + + it.effect( + "lets it run for a normal server, no server, a held server that is gone, an empty home, or a home linked before markers", + () => + Effect.gen(function* () { + expect(Exit.isSuccess((yield* refusalFor({ state: running })).exit)).toBe(true); + expect(Exit.isSuccess((yield* refusalFor({})).exit)).toBe(true); + // A pid that cannot be alive: an older held server stopped and left its record behind. + expect( + Exit.isSuccess( + (yield* refusalFor({ state: { ...running, pid: 2 ** 22 + 1, workHeld: true } })).exit, + ), + ).toBe(true); + expect(Exit.isSuccess((yield* refusalFor({ missing: true })).exit)).toBe(true); + // Not protected: nothing tells such a copy from the live home until it is started held. + expect(Exit.isSuccess((yield* refusalFor({ secrets: true })).exit)).toBe(true); + }), + ); }); diff --git a/apps/server/src/cloud/holdWorkConnect.ts b/apps/server/src/cloud/holdWorkConnect.ts index 750bf20817fd9..dd6f4a6826859 100644 --- a/apps/server/src/cloud/holdWorkConnect.ts +++ b/apps/server/src/cloud/holdWorkConnect.ts @@ -1,34 +1,58 @@ -// Fork (`serve --hold-work`): a held copy shares the live server's Connect link and environment id, -// so it starts, registers and releases no tunnel, publishes no agent activity, and refuses to -// link, change or unlink (an unlink is followed by a relay delete of that id). It only reads. -// Every method is listed by name, never spread: a new upstream method is a type error here. +// Fork (`serve --hold-work`, or any boot of a home a held boot marked): a copy shares the live +// server's Connect environment id, so it starts, registers and releases no tunnel, publishes no +// agent activity, and refuses to link, change or unlink (an unlink ends in a relay delete of that +// id). Every method is listed by name, never spread: a new upstream method is a type error here. +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as ServerConfig from "../config.ts"; import * as ManagedLimits from "../policy/managedLimits.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { readLiveServerRuntimeState } from "../serverRuntimeState.ts"; import * as WorkAdmission from "../workAdmission.ts"; import * as CloudLink from "./CloudLink.ts"; -const HOLD_CONNECT_LOG = - "T3 Connect held (--hold-work): the stored link is kept as it is; no tunnel starts, registers or is released"; +/** Why Connect is held: `--hold-work` (`markerPath` null), or the held-copy marker in this home. */ +interface ConnectHold { + readonly markerPath: string | null; +} -/** What a client sees when it asks a held server to link or change its link. */ -export const heldConnectRefusal = () => - new ManagedLimits.ManagedPolicyRefusedError({ - organization: "This held server", - action: "T3 Connect", - detail: "while it runs with --hold-work; restart it without the flag to link it", - }); +const markedCopy = (markerPath: string) => + `${markerPath} marks this home as a copy that shares another server's T3 Connect link`; + +const holdLog = (hold: ConnectHold, part: string, effect: string) => + hold.markerPath === null + ? `${part} held (--hold-work): ${effect}` + : `${part} held: ${markedCopy(hold.markerPath)}, so ${effect}. Delete it only if this home is not a copy`; -const refuse = () => Effect.fail(heldConnectRefusal()); +/** What a client sees when it asks a held server, or one on a marked copy, to link or change its link. */ +export const heldConnectRefusal = (markerPath?: string) => + new ManagedLimits.ManagedPolicyRefusedError( + markerPath === undefined + ? { + organization: "This held server", + action: "T3 Connect", + detail: "while it runs with --hold-work; restart it without the flag to link it", + } + : { + organization: "This copy", + action: "T3 Connect", + detail: `as ${markedCopy(markerPath)}; if it is not a copy, delete that file and restart`, + }, + ); /** The CloudLink a held server uses: it reads the link state, and starts or releases nothing. */ -const holdConnect = (cloudLink: CloudLink.CloudLink["Service"]): CloudLink.CloudLink["Service"] => - CloudLink.CloudLink.of({ +const holdConnect = ( + cloudLink: CloudLink.CloudLink["Service"], + hold: ConnectHold, +): CloudLink.CloudLink["Service"] => { + const refuse = () => Effect.fail(heldConnectRefusal(hold.markerPath ?? undefined)); + return CloudLink.CloudLink.of({ linkState: cloudLink.linkState, // The web Unlink sends a relay delete for this environment id once the server agrees, and a // copy shares the live server's id: refusing here stops it before that delete. @@ -54,36 +78,83 @@ const holdConnect = (cloudLink: CloudLink.CloudLink["Service"]): CloudLink.Cloud Effect.succeed(false), ), }); +}; + +/** Held from boot, or a normal boot of a home a held boot marked; read once, as the layer builds. */ +const connectHold = Effect.gen(function* () { + if ((yield* WorkAdmission.WorkAdmission).heldFromBoot) { + return Option.some({ markerPath: null }); + } + const marker = yield* heldCopyMarker((yield* ServerConfig.ServerConfig).baseDir); + return Option.map(marker, (markerPath): ConnectHold => ({ markerPath })); +}); + +/** + * A held boot marks its home as a copy before the real CloudLink is built, so the CLI refusals + * hold once it stops too. Created only when missing: a marker already there stays as it is. + */ +const markHeldCopy = Layer.effectDiscard( + Effect.gen(function* () { + if (!(yield* WorkAdmission.WorkAdmission).heldFromBoot) return; + const markerPath = (yield* Path.Path).join( + (yield* ServerConfig.ServerConfig).baseDir, + HELD_COPY_MARKER_FILE, + ); + const note = [ + `A server ran on this T3 home with --hold-work (first at ${DateTime.formatIso(yield* DateTime.now)}),`, + "so it is taken to be a copy that shares the live server's T3 Connect link. While this file", + "exists, `t3 connect` and `t3 up` refuse to sign in, link, unlink or sign out here.", + "Delete it only if this home is not a copy.", + "", + ].join("\n"); + yield* (yield* FileSystem.FileSystem).writeFileString(markerPath, note, { flag: "wx" }).pipe( + Effect.catchIf( + (error) => error.reason._tag === "AlreadyExists", + () => Effect.void, + ), + ); + }), +); -/** `layer` as is, or held when the shared work gate is held from boot. */ +/** `layer` as is, or held when the server is held from boot or its home is a marked copy. */ export const layer = (cloudLinkLayer: Layer.Layer) => Layer.effect( CloudLink.CloudLink, Effect.gen(function* () { const cloudLink = yield* CloudLink.CloudLink; - if (!(yield* WorkAdmission.WorkAdmission).heldFromBoot) return cloudLink; - yield* Effect.logInfo(HOLD_CONNECT_LOG); - return holdConnect(cloudLink); + const hold = yield* connectHold; + if (Option.isNone(hold)) return cloudLink; + yield* Effect.logInfo( + holdLog( + hold.value, + "T3 Connect", + "the stored link is kept as it is; no tunnel starts, registers or is released", + ), + ); + return holdConnect(cloudLink, hold.value); }), - ).pipe(Layer.provide(cloudLinkLayer)); - -const HOLD_AGENT_ACTIVITY_LOG = - "Agent activity publishing held (--hold-work): no thread state is sent to the T3 Connect relay"; + ).pipe(Layer.provide(cloudLinkLayer), Layer.provide(markHeldCopy)); /** The publisher a held server uses: it logs that it is held and sends nothing. */ -const holdAgentActivity = (): AgentAwarenessRelay.AgentAwarenessRelay["Service"] => +const holdAgentActivity = (hold: ConnectHold): AgentAwarenessRelay.AgentAwarenessRelay["Service"] => AgentAwarenessRelay.AgentAwarenessRelay.of({ publishThread: Effect.fn("HoldWorkConnect.publishThread")(() => Effect.void), drain: Effect.void, requestCatchUp: Effect.fn("HoldWorkConnect.requestCatchUp")(() => Effect.void), start: Effect.fn("HoldWorkConnect.startAgentActivity")(() => - Effect.logInfo(HOLD_AGENT_ACTIVITY_LOG), + Effect.logInfo( + holdLog( + hold, + "Agent activity publishing", + "no thread state is sent to the T3 Connect relay", + ), + ), ), }); /** - * `layer` as is, or held when the shared work gate is held from boot. The publisher reads the - * copied relay address, credential and signing key itself, not through CloudLink. + * `layer` as is, or held like `layer`. The publisher reads the copied relay address, credential + * and signing key itself, not through CloudLink. */ export const agentActivityLayer = ( relayLayer: Layer.Layer, @@ -92,31 +163,72 @@ export const agentActivityLayer = ( AgentAwarenessRelay.AgentAwarenessRelay, Effect.gen(function* () { const relay = yield* AgentAwarenessRelay.AgentAwarenessRelay; - return (yield* WorkAdmission.WorkAdmission).heldFromBoot ? holdAgentActivity() : relay; + const hold = yield* connectHold; + return Option.isSome(hold) ? holdAgentActivity(hold.value) : relay; }), ).pipe(Layer.provide(relayLayer)); +/** Written in a home by a held boot. Every boot and the `t3 connect` refusals read it; nothing removes it. */ +export const HELD_COPY_MARKER_FILE = "hold-work-copy.txt"; + /** - * `t3 connect unlink` / `logout`: refused while this home's server runs held. They also send the - * relay delete for the environment id, which a copy shares with the live server. A copy that is - * not running cannot be told apart from the live home; docs/fork/updates.md warns about that. + * The marker's path when `baseDir` is marked. Anything at the path counts, a dangling symlink too + * (the held boot's exclusive create keeps one), and so does a check that fails. */ -export const refuseCliUnlinkWhileHeld = Effect.fn("HoldWorkConnect.refuseCliUnlinkWhileHeld")( - function* (serverRuntimeStatePath: string) { - const running = yield* readLiveServerRuntimeState(serverRuntimeStatePath).pipe( - Effect.orElseSucceed(() => Option.none()), - ); - if (Option.isSome(running) && running.value.workHeld === true) { - return yield* new HeldCliUnlinkRefusedError(); - } - }, -); +const heldCopyMarker = Effect.fn("HoldWorkConnect.heldCopyMarker")(function* (baseDir: string) { + const fs = yield* FileSystem.FileSystem; + const markerPath = (yield* Path.Path).join(baseDir, HELD_COPY_MARKER_FILE); + const marked = yield* fs.exists(markerPath).pipe( + Effect.flatMap((exists) => + exists + ? Effect.succeed(true) + : fs.readLink(markerPath).pipe( + Effect.as(true), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(false), + ), + ), + ), + Effect.orElseSucceed(() => true), + ); + return marked ? Option.some(markerPath) : Option.none(); +}); + +/** + * `t3 connect` sign-in, link, unlink and logout, and `t3 up` turning Connect on or off: refused on + * a held copy's home. A copy shares the live server's environment id and CLI sign-in. + */ +export const refuseCliOnHeldCopy = Effect.fn("HoldWorkConnect.refuseCliOnHeldCopy")(function* ( + config: Pick, +) { + const marker = yield* heldCopyMarker(config.baseDir); + if (Option.isSome(marker)) { + return yield* new HeldCopyRefusedError({ markerPath: marker.value }); + } + const running = yield* readLiveServerRuntimeState(config.serverRuntimeStatePath).pipe( + Effect.orElseSucceed(() => Option.none()), + ); + if (Option.isSome(running) && running.value.workHeld === true) { + return yield* new HeldCopyRefusedError({ markerPath: null }); + } +}); -export class HeldCliUnlinkRefusedError extends Schema.TaggedError()( - "HeldCliUnlinkRefusedError", - {}, +export class HeldCopyRefusedError extends Schema.TaggedError()( + "HeldCopyRefusedError", + { markerPath: Schema.NullOr(Schema.String) }, ) { override get message(): string { - return "Refused: this home's server runs with --hold-work, so it is a copy that shares the live server's T3 Connect environment id. Unlinking or signing out here would also remove the live server's link on the relay. Nothing was changed."; + const why = + this.markerPath === null + ? "this home's server runs with --hold-work, so it is a copy" + : `${this.markerPath} marks this home as a copy that ran with --hold-work. A copy`; + return [ + `Refused: ${why} that shares the live server's T3 Connect environment id and sign-in.`, + "Signing in, linking, unlinking or signing out here would act on the live server's link too. Nothing was sent, and the link and sign-in are as they were.", + ...(this.markerPath === null + ? [] + : [`If this home is not a copy, delete ${this.markerPath} and run the command again.`]), + ].join("\n"); } } diff --git a/apps/server/src/holdWork.boot.test.ts b/apps/server/src/holdWork.boot.test.ts index 9ec2b1a2603eb..e8937b67548ca 100644 --- a/apps/server/src/holdWork.boot.test.ts +++ b/apps/server/src/holdWork.boot.test.ts @@ -1,8 +1,9 @@ // @effect-diagnostics nodeBuiltinImport:off - Boots the real CLI in a child process on a temp home. /** - * Fork (`serve --hold-work`): boots the real `t3 serve` held, then normally, on one seeded temp home - * (work, a Connect link and analytics on a local fake server, a task in a linked environment). The - * held boot must log each hold and start none of it; the normal boot must then run it all. + * Fork (`serve --hold-work`): boots the real `t3 serve` on one seeded temp home (work, a Connect link + * and analytics on a local fake server, a task in a linked environment): held, which starts none of + * it and marks the home; then normally, which runs the work but still holds Connect on the marked + * home; then normally with the marker deleted, which releases the tunnel. */ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; @@ -34,6 +35,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import { afterEach, describe, expect, it } from "vite-plus/test"; +import * as HoldWorkConnect from "./cloud/holdWorkConnect.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as SqlitePersistence from "./persistence/Sqlite.ts"; @@ -176,6 +178,8 @@ const boot = async ( T3CODE_CLERK_PUBLISHABLE_KEY: "pk_test_hold_work", T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: "client-hold-work", NO_COLOR: "1", + // No managed policy, whatever this machine has: a missing file means none. + T3CODE_DEV_MANAGED_POLICY_PATH: NodePath.join(root, "no-policy.json"), }, stdio: ["ignore", "pipe", "pipe"], }, @@ -299,16 +303,30 @@ const HELD_CONNECT_STEPS = [ "HoldWorkConnect.reconcileDesiredLink", ]; -/** The spans that show a hold was not kept: each runs only when its part starts. */ -const HELD_SPANS = [ - "autoPullProjects", - "RemoteDelegation.followOnce", +/** The real CloudLink's spans: a held boot, or any boot of a marked copy, runs none of them. */ +const CONNECT_SPANS = [ "environment.cloud.registerManagedCloudTunnelRecovery", "environment.cloud.releaseManagedTunnelOnShutdown", "environment.cloud.startManagedCloudTunnelIfOriginConfirmed", "environment.cloud.reconcileDesiredLinkIfStillDesired", ]; +/** The spans that show a hold was not kept: each runs only when its part starts. */ +const HELD_SPANS = ["autoPullProjects", "RemoteDelegation.followOnce", ...CONNECT_SPANS]; + +/** What a normal boot of a marked copy logs instead of starting Connect and agent activity. */ +const markedLines = (marker: string) => + ["T3 Connect held: ", "Agent activity publishing held: "].map( + (part) => `${part}${marker} marks this home as a copy`, + ); + +const tunnelReleases = (remote: FakeRemote) => + relayRequests(remote).filter( + (request) => + request.method === "DELETE" && + /^\/v1\/client\/environment-links\/[^/]+\/tunnel$/.test(request.path), + ); + const seedThreads = (dbPath: string, nowMs: number, activityRoot: string) => { const at = (offsetMs: number) => DateTime.makeUnsafe(nowMs + offsetMs); const iso = (offsetMs: number) => DateTime.formatIso(at(offsetMs)); @@ -694,7 +712,7 @@ const snapshot = (paths: { stateDir: string; dbPath: string }): Snapshot => { }; describe("serve --hold-work on a seeded home", () => { - it("a held boot consumes no seeded work; a normal boot on the same home then runs it", async () => { + it("a held boot consumes no seeded work; a normal boot on the same home then runs it, Connect only once unmarked", async () => { const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-hold-work-boot-")); const remote = await fakeRemote(); try { @@ -709,6 +727,9 @@ describe("serve --hold-work on a seeded home", () => { expect(Object.keys(seeded.connect)).toHaveLength(8); expect(seeded.remoteTask).toBeDefined(); + // A home linked before held copies were marked: no marker until it boots held. + const marker = NodePath.join(root, "t3", HoldWorkConnect.HELD_COPY_MARKER_FILE); + expect(NodeFS.existsSync(marker)).toBe(false); const held = await boot(root, remote, ["--hold-work"]); const runtime = await ready(root, held); expect(runtime.workHeld).toBe(true); @@ -742,6 +763,9 @@ describe("serve --hold-work on a seeded home", () => { ); }); await stop(held.child); + // The stopped copy stays marked, so `t3 connect` refuses to unlink or link it. + const markerText = NodeFS.readFileSync(marker, "utf8"); + expect(markerText).toContain("--hold-work"); const heldSpans = spansFrom(paths.stateDir, 0); // The publisher logs its state when it starts, before any held line: a held one never does. expect(held.output()).not.toMatch(PUBLISHER_STARTED); @@ -778,34 +802,76 @@ describe("serve --hold-work on a seeded home", () => { if (effect.status === "pending") expect(effect.attempt_count).toBe(0); } + // A normal boot of the marked copy runs the work, but Connect and agent activity stay held: + // its stop would release the running server's tunnel under the shared environment id. + const marked = await boot(root, remote, []); + const markedRuntime = await ready(root, marked); + expect(markedRuntime.workHeld).toBeUndefined(); + await when( + "the marked copy's normal boot to pick up the seeded work", + marked, + paths.stateDir, + () => { + const now = snapshot(paths); + const spans = spansFrom(paths.stateDir, heldSpans.end).names; + // A leak ends the wait at once, so the checks below name it instead of a timeout. + if ( + activityPublishes(remote).length > 0 || + CONNECT_SPANS.some((name) => spans.has(name)) + ) { + return true; + } + return ( + now.task.last_run_status !== "never" && + now.waits.every((wait) => wait.state !== "waiting") && + now.limitedThread.limit_recovery !== null && + now.effects.some( + (row) => row.effect_id === "effect:hold-work:continue" && row.status !== "pending", + ) && + heartbeats(remote).length > 0 && + spans.has("RemoteDelegation.followOnce") && + markedLines(marker).every((line) => marked.output().includes(line)) && + // Past the random wait before registration, and past the desired-link reconcile. + HELD_CONNECT_STEPS.every((name) => spans.has(name)) + ); + }, + ); + await stop(marked.child); + const markedSpans = spansFrom(paths.stateDir, heldSpans.end); + // No tunnel registered, started or released, and no agent activity sent. + expect(tunnelReleases(remote)).toEqual([]); + expect(activityPublishes(remote)).toEqual([]); + expect(CONNECT_SPANS.filter((name) => markedSpans.names.has(name))).toEqual([]); + expect(relayRequests(remote)).toEqual([]); + expect(marked.output()).not.toMatch(PUBLISHER_STARTED); + expect(marked.output()).not.toContain("--hold-work"); + expect(markedSpans.names).toContain("HoldWorkConnect.releaseManagedTunnel"); + expect(markedSpans.names).toContain("autoPullProjects"); + expect(markedSpans.names).toContain("RemoteDelegation.followOnce"); + // A normal boot of the copy leaves its marker as it is. + expect(NodeFS.readFileSync(marker, "utf8")).toBe(markerText); + + // With the marker deleted, the same home is no longer taken to be a copy. + NodeFS.rmSync(marker); const normal = await boot(root, remote, []); - const normalRuntime = await ready(root, normal); - expect(normalRuntime.workHeld).toBeUndefined(); - await when("the normal boot to pick up the seeded work", normal, paths.stateDir, () => { - const now = snapshot(paths); - return ( - now.task.last_run_status !== "never" && - now.waits.every((wait) => wait.state !== "waiting") && - now.limitedThread.limit_recovery !== null && - now.effects.some( - (row) => row.effect_id === "effect:hold-work:continue" && row.status !== "pending", - ) && - heartbeats(remote).length > 0 && - // The finished thread's activity reaches the relay, so the held boot had it to send. - activityPublishes(remote).length > 0 && - spansFrom(paths.stateDir, heldSpans.end).names.has("RemoteDelegation.followOnce") - ); - }); + await ready(root, normal); + await when( + "the unmarked home's normal boot to publish", + normal, + paths.stateDir, + () => + // The finished thread's activity reaches the relay, so the held boots had it to send. + activityPublishes(remote).length > 0, + ); expect(normal.output()).not.toContain("--hold-work"); + expect(normal.output()).not.toContain(HoldWorkConnect.HELD_COPY_MARKER_FILE); await stop(normal.child); - // The same home without the flag does each held thing. Its tunnel registration waits a - // random part of half a minute, but stopping releases the tunnel at once, so the relay has - // been reached by now either way. - const normalSpans = spansFrom(paths.stateDir, heldSpans.end).names; - expect(normalSpans).toContain("autoPullProjects"); - expect(normalSpans).toContain("RemoteDelegation.followOnce"); + // Its tunnel registration waits a random part of half a minute, but stopping releases the + // tunnel at once: the relay has the release itself. + const normalSpans = spansFrom(paths.stateDir, markedSpans.end).names; expect(normalSpans).toContain("environment.cloud.releaseManagedTunnelOnShutdown"); - expect(relayRequests(remote).length).toBeGreaterThan(0); + expect(tunnelReleases(remote)).toHaveLength(1); + expect(NodeFS.existsSync(marker)).toBe(false); } finally { for (const child of children) await stop(child); await remote.close(); diff --git a/docs/fork/updates.md b/docs/fork/updates.md index 0fb6cd4ad370a..c46ac0dc0634f 100644 --- a/docs/fork/updates.md +++ b/docs/fork/updates.md @@ -256,7 +256,7 @@ settings files may still exist; it moves with the add-ons. threads, settings and migrations before it replaces a running T3. It is refused for the background service. -Held (nothing of it starts until the server restarts without the flag): +Held (nothing of it starts until the server restarts without the flag, except T3 Connect: see below): - agent turns, steers and rollbacks (outbox effects stay pending, unclaimed); - thread launches and worktree preparation; @@ -270,14 +270,34 @@ Held (nothing of it starts until the server restarts without the flag): and tunnel). The held server starts no tunnel, registers none with the relay and releases none when it stops, so remote access stays with the running server. It publishes no agent activity (thread state) to the relay under that id. A client's request to link, change the link, unlink or answer - the relay is refused with the reason, and so is every CLI unlink on that home (`t3 connect unlink`, - `t3 connect logout`, `t3 up` turning Connect off) while the held server runs: unlinking ends in a - relay delete of the shared environment id, which would remove the running server's link too. - The stored link files are kept as they are (`apps/server/src/cloud/holdWorkConnect.ts`). - - The CLI can only tell a copy is held while its server runs. Never run `t3 connect unlink` or - `t3 connect logout` against a stopped copy's home: it removes the running server's link. Delete - the copy's home instead. + the relay is refused with the reason. The stored link files are kept as they are + (`apps/server/src/cloud/holdWorkConnect.ts`), with one exception: a managed policy that turns + T3 Connect off makes the server unlink while it starts, before the hold applies. That removes the + copy's own link files and sends nothing to the relay; the policy has unlinked the running server + on that machine already. + + The first held boot writes `hold-work-copy.txt` in the copy's home, and nothing removes it, not + a normal boot either. While it is there: + - Every boot of that home holds T3 Connect and agent activity publishing as above, with or without + `--hold-work`, and logs the marker's path. A copy started normally (`t3 serve` or `t3 up` on its + home) runs its work but never registers or releases the tunnel under the shared environment id. + The server reads the marker once as it starts: one added while it runs counts from its next + start. + - `t3 connect login`, `link`, `unlink` and `logout`, `t3 connect publish` turning publishing on, + and `t3 up` turning T3 Connect on or off are refused on that home, running or not, before they + send anything or change the link or sign-in (the CLI may still create the home's folders and + signing key, as any command there does). Unlinking ends in a relay delete of the shared + environment id, which would remove the running server's link too, and signing in can refresh + the copied sign-in, which the running server shares. + + Anything at that path counts, even an empty file or a dangling symlink, and so does a path that + cannot be checked. If the home is not a copy, delete the file and run the command or start the + server again. + + A copy with no marker is not protected: one that has never booted held, or one held-booted only by + a build from before the marker (its first held boot on a newer build marks it). Neither the CLI + nor the server can tell it from the running server's home: never run those commands against it or + start it normally. Delete the copy's home instead. Still runs: @@ -292,6 +312,10 @@ Still runs: - reading the copy's Connect status; - the periodic Connect reachability check: read-only requests to the stored public address and the tunnel network. It restarts only a tunnel this server started, and a held server starts none. +- the held-hooks waker, which reads the copied relay credential but sends only when a tunnel + connects, and a held server connects none; +- the relay client ledger's clean-up at start, which stops only a recorded tunnel client whose + owner process is gone (a running server's own is skipped). Local only. Startup refuses to serve when the gate is not held or a part that starts work (outbox, thread launch, recovery, scheduler) does not use it. The check runs before activation, so nothing parked has run.