From a170622b306a2efe0104bfdc2200e5ca2f8e1081 Mon Sep 17 00:00:00 2001 From: Franco Mendez Date: Sun, 26 Jul 2026 08:05:56 -0400 Subject: [PATCH] fix(codex): avoid runtime shadow reindex on TUI startup Run interactive TUI sessions against the canonical Codex home so its state database and thread index remain reusable. Route both the TUI and its app-server through ephemeral non-secret proxy overrides, preserving account rotation without mutating config or exposing credentials. --- scripts/codex.js | 110 +++++++++++++++++++++++++++++---- test/codex-bin-wrapper.test.ts | 95 ++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 13 deletions(-) diff --git a/scripts/codex.js b/scripts/codex.js index 29cec6be2..b7459e308 100755 --- a/scripts/codex.js +++ b/scripts/codex.js @@ -74,6 +74,10 @@ const APP_RUNTIME_HELPER_OWNER_PID_ENV = "CODEX_MULTI_AUTH_APP_ROTATION_OWNER_PID"; const APP_RUNTIME_HELPER_REAL_CODEX_HOME_ENV = "CODEX_MULTI_AUTH_REAL_CODEX_HOME"; +const APP_RUNTIME_HELPER_USE_CANONICAL_HOME_ENV = + "CODEX_MULTI_AUTH_APP_ROTATION_USE_CANONICAL_HOME"; +const APP_SERVER_CONFIG_ARGS_ENV = + "CODEX_MULTI_AUTH_APP_SERVER_CONFIG_ARGS_JSON"; const APP_RUNTIME_HELPER_STATUS_FILE = RUNTIME_CONSTANTS.APP_RUNTIME_HELPER_STATUS_FILE; const DEFAULT_APP_RUNTIME_HELPER_IDLE_MS = 12 * 60 * 60 * 1000; @@ -3490,6 +3494,38 @@ function createRuntimeRotationProxyCodexHome( }; } +function createRuntimeRotationProxyCanonicalCodexHome( + baseEnv, + proxyBaseUrl, + clientApiKey, + configTomlModule, +) { + const originalCodexHome = resolveRuntimeRotationProxyOriginalCodexHome(baseEnv); + const providerTable = `model_providers.${RUNTIME_ROTATION_PROXY_PROVIDER_ID}`; + const configArgs = [ + `${providerTable}.name=${configTomlModule.tomlStringLiteral(APP_SERVER_ACCOUNT_DISPLAY_NAME)}`, + `${providerTable}.base_url=${configTomlModule.tomlStringLiteral(proxyBaseUrl)}`, + `${providerTable}.env_key=${configTomlModule.tomlStringLiteral("OPENAI_API_KEY")}`, + `${providerTable}.requires_openai_auth=false`, + `${providerTable}.wire_api=${configTomlModule.tomlStringLiteral("responses")}`, + "disable_response_storage=false", + ].flatMap((assignment) => ["-c", assignment]); + + return { + args: configArgs, + env: { + ...baseEnv, + CODEX_HOME: originalCodexHome, + OPENAI_API_KEY: clientApiKey, + CODEX_MULTI_AUTH_DIR: resolveRuntimeRotationOriginalMultiAuthDir( + originalCodexHome, + baseEnv, + ), + }, + cleanup: () => {}, + }; +} + function appendNodeImportOption(nodeOptions, preloadPath) { const importOption = `--import=${pathToFileURL(preloadPath).href}`; const trimmed = (nodeOptions ?? "").trim(); @@ -3504,15 +3540,26 @@ function createRuntimeRotationAppServerPreloadSource(wrapperScriptPath) { "", `const wrapperScriptPath = ${JSON.stringify(wrapperScriptPath)};`, `const accountLabelEnv = ${JSON.stringify(APP_SERVER_ACCOUNT_LABEL_ENV)};`, + `const configArgsEnv = ${JSON.stringify(APP_SERVER_CONFIG_ARGS_ENV)};`, + "let configArgs = [];", + "try {", + ' const parsed = JSON.parse(process.env[configArgsEnv] ?? "[]");', + " if (Array.isArray(parsed)) {", + ' configArgs = parsed.filter((arg) => typeof arg === "string");', + " }", + "} catch {", + " // Ignore malformed internal metadata and preserve app-server startup.", + "}", "const rawArgs = process.argv.slice(1);", "const firstArg = rawArgs[0] ?? \"\";", 'if (basename(firstArg).toLowerCase() === "app-server") {', - ' const args = ["app-server", ...rawArgs.slice(1)];', + ' const args = ["app-server", ...rawArgs.slice(1), ...configArgs];', " const env = {", " ...process.env,", ' CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY: "0",', " [accountLabelEnv]: \"1\",", " };", + " delete env[configArgsEnv];", " const child = spawn(process.execPath, [wrapperScriptPath, ...args], {", " stdio: \"inherit\",", " env,", @@ -3558,7 +3605,7 @@ function sweepStaleRuntimeRotationAppServerShimDirs(shimRootDir) { } } -function installRuntimeRotationAppServerCliShim(forwardedEnv) { +function installRuntimeRotationAppServerCliShim(forwardedEnv, configArgs = []) { const shadowCodexHome = forwardedEnv.CODEX_HOME; if (!shadowCodexHome) { throw new Error("runtime app-server shim requires CODEX_HOME"); @@ -3615,6 +3662,11 @@ function installRuntimeRotationAppServerCliShim(forwardedEnv) { ); forwardedEnv.CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY = "0"; forwardedEnv[APP_SERVER_ACCOUNT_LABEL_ENV] = "1"; + if (configArgs.length > 0) { + forwardedEnv[APP_SERVER_CONFIG_ARGS_ENV] = JSON.stringify(configArgs); + } else { + delete forwardedEnv[APP_SERVER_CONFIG_ARGS_ENV]; + } return shimDir; } @@ -3715,6 +3767,7 @@ function pickRuntimeRotationAppHelperEnv(env) { "CODEX_MULTI_AUTH_REAL_CODEX_BIN", "CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY", APP_SERVER_ACCOUNT_LABEL_ENV, + APP_SERVER_CONFIG_ARGS_ENV, ]) { if (env[name]) { picked[name] = env[name]; @@ -3776,7 +3829,7 @@ function createRuntimeRotationAppHelperStatus({ async function runRuntimeRotationAppHelper() { let proxyServer = null; - let shadowContext = null; + let runtimeContext = null; let appServerShimDir = null; let statusTimer = null; let closing = false; @@ -3813,7 +3866,7 @@ async function runRuntimeRotationAppHelper() { } appServerShimDir = null; } - shadowContext?.cleanup?.(); + runtimeContext?.cleanup?.(); } finally { try { await proxyServer?.close?.(); @@ -3844,13 +3897,33 @@ async function runRuntimeRotationAppHelper() { } const clientApiKey = createRuntimeRotationProxyClientApiKey(); proxyServer = await proxyModule.startRuntimeRotationProxy({ clientApiKey }); - shadowContext = createRuntimeRotationProxyCodexHome( - process.env, - proxyServer.baseUrl, - clientApiKey, - configTomlModule, + const useCanonicalHome = + (process.env[APP_RUNTIME_HELPER_USE_CANONICAL_HOME_ENV] ?? "").trim() === + "1"; + runtimeContext = useCanonicalHome + ? createRuntimeRotationProxyCanonicalCodexHome( + process.env, + proxyServer.baseUrl, + clientApiKey, + configTomlModule, + ) + : createRuntimeRotationProxyCodexHome( + process.env, + proxyServer.baseUrl, + clientApiKey, + configTomlModule, + ); + const appServerConfigArgs = useCanonicalHome + ? [ + ...(runtimeContext.args ?? []), + "-c", + `model_provider=${configTomlModule.tomlStringLiteral(RUNTIME_ROTATION_PROXY_PROVIDER_ID)}`, + ] + : []; + appServerShimDir = installRuntimeRotationAppServerCliShim( + runtimeContext.env, + appServerConfigArgs, ); - appServerShimDir = installRuntimeRotationAppServerCliShim(shadowContext.env); lastRequestCount = proxyServer.getStatus?.().totalRequests ?? 0; publishStatus("running"); process.stdout.write( @@ -3859,7 +3932,8 @@ async function runRuntimeRotationAppHelper() { pid: process.pid, baseUrl: proxyServer.baseUrl, statusPath: resolveRuntimeRotationAppHelperStatusPath(), - env: pickRuntimeRotationAppHelperEnv(shadowContext.env), + args: runtimeContext.args ?? [], + env: pickRuntimeRotationAppHelperEnv(runtimeContext.env), })}\n`, ); @@ -3920,7 +3994,7 @@ function stopRuntimeRotationAppHelper(helper) { return waitForRuntimeRotationAppHelperExit(helper); } -function startRuntimeRotationAppHelper(baseContext) { +function startRuntimeRotationAppHelper(baseContext, options = {}) { const realCodexHome = baseContext.originalCodexHome ?? resolveRuntimeRotationProxyOriginalCodexHome(baseContext.env); @@ -3940,6 +4014,8 @@ function startRuntimeRotationAppHelper(baseContext) { ), [APP_RUNTIME_HELPER_OWNER_PID_ENV]: String(process.pid), [APP_RUNTIME_HELPER_REAL_CODEX_HOME_ENV]: realCodexHome, + [APP_RUNTIME_HELPER_USE_CANONICAL_HOME_ENV]: + options.useCanonicalHome === true ? "1" : "0", }, stdio: ["ignore", "pipe", "pipe"], detached: true, @@ -4005,8 +4081,14 @@ async function createRuntimeRotationAppHelperContext( options = {}, ) { const startedAt = Date.now(); - const { helper, message } = await startRuntimeRotationAppHelper(baseContext); + const { helper, message } = await startRuntimeRotationAppHelper( + baseContext, + options, + ); const helperEnv = message.env ?? {}; + const helperArgs = Array.isArray(message.args) + ? message.args.filter((arg) => typeof arg === "string") + : []; const detachGraceMs = resolveRuntimeRotationAppHelperDetachGraceMs(baseContext.env); const cleanup = async ({ exitCode } = {}) => { @@ -4023,6 +4105,7 @@ async function createRuntimeRotationAppHelperContext( return { args: [ ...baseContext.args, + ...helperArgs, "-c", `model_provider=${configTomlModule.tomlStringLiteral(RUNTIME_ROTATION_PROXY_PROVIDER_ID)}`, ], @@ -4063,6 +4146,7 @@ async function createRuntimeRotationProxyContextIfEnabled( if (isCodexInteractiveTuiCommand(rawArgs)) { return createRuntimeRotationAppHelperContext(baseContext, configTomlModule, { detachOnExit: true, + useCanonicalHome: true, }); } diff --git a/test/codex-bin-wrapper.test.ts b/test/codex-bin-wrapper.test.ts index 1a44f8bbf..448504056 100644 --- a/test/codex-bin-wrapper.test.ts +++ b/test/codex-bin-wrapper.test.ts @@ -2760,6 +2760,101 @@ describe("codex bin wrapper", () => { expect(marker).toContain("close\n"); }); + it("uses the canonical Codex home for interactive TUI runtime routing", async () => { + const fixtureRoot = createWrapperFixture(); + createRuntimeRotationProxyFixtureModule(fixtureRoot); + const fakeBin = createCustomFakeCodexBin(fixtureRoot, [ + "#!/usr/bin/env node", + 'const { spawnSync } = require("node:child_process");', + 'const fs = require("node:fs");', + 'const path = require("node:path");', + 'const args = process.argv.slice(2);', + 'if (args[0] === "app-server") {', + ' console.log(`APP_SERVER_FORWARDED:${args.join(" ")}`);', + " process.exit(0);", + "}", + 'const statePath = path.join(process.env.CODEX_HOME ?? "", "state_5.sqlite");', + 'const originalState = path.join(process.env.ORIGINAL_CODEX_HOME ?? "", "state_5.sqlite");', + 'console.log(`TUI_HOME_IS_ORIGINAL:${process.env.CODEX_HOME === process.env.ORIGINAL_CODEX_HOME}`);', + 'console.log(`TUI_STATE_EXISTED:${fs.existsSync(statePath)}`);', + 'fs.writeFileSync(statePath, "first-run-state\\n", "utf8");', + 'console.log(`TUI_STATE_PERSISTED:${fs.readFileSync(originalState, "utf8").includes("first-run-state")}`);', + 'console.log(`TUI_HAS_BASE_URL_OVERRIDE:${args.some((arg) => arg.includes("model_providers.codex-multi-auth-runtime-proxy.base_url="))}`);', + 'console.log(`TUI_HAS_ENV_KEY_OVERRIDE:${args.includes(\'model_providers.codex-multi-auth-runtime-proxy.env_key="OPENAI_API_KEY"\')}`);', + 'console.log(`TUI_HAS_AUTH_OVERRIDE:${args.includes("model_providers.codex-multi-auth-runtime-proxy.requires_openai_auth=false")}`);', + 'console.log(`TUI_HAS_WIRE_OVERRIDE:${args.includes(\'model_providers.codex-multi-auth-runtime-proxy.wire_api="responses"\')}`);', + 'console.log(`TUI_HAS_STORAGE_OVERRIDE:${args.includes("disable_response_storage=false")}`);', + 'console.log(`TUI_KEY_IN_ARGS:${args.includes(process.env.OPENAI_API_KEY ?? "__missing__")}`);', + 'const shimExe = path.join(process.env.CODEX_CLI_PATH ?? "", process.platform === "win32" ? "codex.exe" : "codex");', + 'const shimResult = spawnSync(shimExe, ["app-server", "--canonical-shim"], { encoding: "utf8", env: process.env });', + 'console.log(`TUI_SHIM_STATUS:${shimResult.status}`);', + 'console.log(`TUI_SHIM_STDOUT:${(shimResult.stdout ?? "").trim()}`);', + "process.exit(0);", + ]); + const originalHome = join(fixtureRoot, "codex-home"); + const markerPath = join(fixtureRoot, "proxy-marker.txt"); + mkdirSync(originalHome, { recursive: true }); + writeFileSync( + join(originalHome, "config.toml"), + 'model_provider = "openai"\n', + "utf8", + ); + + const result = runWrapper(fixtureRoot, [], { + CODEX_MULTI_AUTH_REAL_CODEX_BIN: fakeBin, + CODEX_HOME: originalHome, + ORIGINAL_CODEX_HOME: originalHome, + CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY: "1", + CODEX_MULTI_AUTH_APP_ROTATION_IDLE_MS: "200", + CODEX_MULTI_AUTH_TEST_PROXY_MARKER: markerPath, + OPENAI_API_KEY: undefined, + }); + + const output = combinedOutput(result); + if (result.status !== 0) { + throw new Error(output); + } + expect(output).toContain("TUI_HOME_IS_ORIGINAL:true"); + expect(output).toContain("TUI_STATE_EXISTED:false"); + expect(output).toContain("TUI_STATE_PERSISTED:true"); + expect(output).toContain("TUI_HAS_BASE_URL_OVERRIDE:true"); + expect(output).toContain("TUI_HAS_ENV_KEY_OVERRIDE:true"); + expect(output).toContain("TUI_HAS_AUTH_OVERRIDE:true"); + expect(output).toContain("TUI_HAS_WIRE_OVERRIDE:true"); + expect(output).toContain("TUI_HAS_STORAGE_OVERRIDE:true"); + expect(output).toContain("TUI_KEY_IN_ARGS:false"); + expect(output).toContain("TUI_SHIM_STATUS:0"); + expect(output).toContain( + "APP_SERVER_FORWARDED:app-server --canonical-shim", + ); + expect(output).toContain( + "model_providers.codex-multi-auth-runtime-proxy.base_url=", + ); + expect(output).toContain( + 'model_providers.codex-multi-auth-runtime-proxy.env_key="OPENAI_API_KEY"', + ); + expect(output).toContain( + "model_providers.codex-multi-auth-runtime-proxy.requires_openai_auth=false", + ); + expect(output).toContain( + 'model_providers.codex-multi-auth-runtime-proxy.wire_api="responses"', + ); + expect(output).toContain("disable_response_storage=false"); + expect(output).toContain( + 'model_provider="codex-multi-auth-runtime-proxy"', + ); + expect(readFileSync(join(originalHome, "state_5.sqlite"), "utf8")).toBe( + "first-run-state\n", + ); + expect(readFileSync(join(originalHome, "config.toml"), "utf8")).toBe( + 'model_provider = "openai"\n', + ); + await waitForFileText( + markerPath, + "start:http://127.0.0.1:4567\nclose\n", + ); + }); + it("stops detached TUI app helpers after failed launches", async () => { const fixtureRoot = createWrapperFixture(); createRuntimeRotationProxyFixtureModule(fixtureRoot);