diff --git a/.github/workflows/sandbox-isolation.yml b/.github/workflows/sandbox-isolation.yml index c80c71f63..ed6ef0f89 100644 --- a/.github/workflows/sandbox-isolation.yml +++ b/.github/workflows/sandbox-isolation.yml @@ -234,8 +234,8 @@ jobs: executed=$(grep -oE 'executed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') passed=$(grep -oE 'passed="[0-9]+"' "$trx" | head -1 | grep -oE '[0-9]+') echo "executed=${executed:-0} passed=${passed:-0}" - if [ "${executed:-0}" -lt 107 ]; then - echo "::error::Expected >=107 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal + an allowlist run's veth guarded both ways, a flow the worker opened before the run included + forwarding a root worker may not write named before an allowlist is planned + an allowlist run's port 53 open only at the resolvers of the resolv.conf its namespace reads, and still to a resolver address the worker's own NAT rewrites before its forward and its input hooks + a target repository's own CLI config kept out of the run with the real CLIs, every repository's memory read in a multi-repo workspace included + a repository whose memory links outside the workspace left out of a real Claude run, against the CLI following the link once that repository is added + nested memory loaded in place by a real Claude run, single- and multi-repo, and none of it past the in-place budget, where a read below a directory attaches only that directory's pointer + a path-scoped rule's pointer reaching a real Claude run only after a read its glob covers, single- and multi-repo, carrying none of the rule's text, against the rule copied into the config home as its control + pointers attaching on exactly the reads the unpinned CLI attached the repository's own rules on, single- and multi-repo + an Explore subagent pointed at nested memory loaded in place, which the CLI keeps out of its context + a multi-repo Codex run starting at a workspace root that is no git repository and loading no config from it + a repo-less Codex run starting in a scratch directory that is no git repository + a goal handed to the real Claude CLI as text it cannot act on, fresh and resumed, against the text channel as its control + a repository clean filter the capture runs with its egress severed + a real Codex agent that tampers its clone's .git while the platform publishes the branch from a clean repo, its Claude counterpart running in the non-root lane), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR." + if [ "${executed:-0}" -lt 108 ]; then + echo "::error::Expected >=108 sandbox isolation tests to run (bwrap/prlimit confinement + cap-drop + cgroup-namespace re-root + egress-allowlist filter + cgroup resource cap + durable-launch cgroup wiring + argv/envp per-string kernel ceiling + a prompt past it riding stdin + a read-only workspace mount + a network-off run reaching its broker through the relay and nothing else + an allowlist run relayed to its broker + a read-only reviewer reading its diff with the real CLIs + a bwrap probe that runs the launch argv + the MCP helper bound file by file behind a read-only socket dir + a CLI reaching its broker socket through the relay + a severed child reaching its broker over the lease socket across a worker restart + a pre-relay namespaced run re-bound at its gateway and torn down with its seal + an allowlist run's veth guarded both ways, a flow the worker opened before the run included + forwarding a root worker may not write named before an allowlist is planned + an allowlist run's port 53 open only at the resolvers of the resolv.conf its namespace reads, and still to a resolver address the worker's own NAT rewrites before its forward and its input hooks + a target repository's own CLI config kept out of the run with the real CLIs, every repository's memory read in a multi-repo workspace included + a repository whose memory links outside the workspace left out of a real Claude run, against the CLI following the link once that repository is added + nested memory loaded in place by a real Claude run, single- and multi-repo, and none of it past the in-place budget, where a read below a directory attaches only that directory's pointer + a path-scoped rule's pointer reaching a real Claude run only after a read its glob covers, single- and multi-repo, carrying none of the rule's text, against the rule copied into the config home as its control + pointers attaching on exactly the reads the unpinned CLI attached the repository's own rules on, single- and multi-repo + an Explore subagent pointed at nested memory loaded in place, which the CLI keeps out of its context + a multi-repo Codex run starting at a workspace root that is no git repository and loading no config from it + every repository's AGENTS.md reaching a real multi-repo Codex run once, after the operating contract, an override before the doc beside it and none that links outside the workspace + a repo-less Codex run starting in a scratch directory that is no git repository + a goal handed to the real Claude CLI as text it cannot act on, fresh and resumed, against the text channel as its control + a repository clean filter the capture runs with its egress severed + a real Codex agent that tampers its clone's .git while the platform publishes the branch from a clean repo, its Claude counterpart running in the non-root lane), but only ${executed:-0} did — the Category=Sandbox filter matched too few (trait regression?). If a case was deliberately removed, lower this number in the same PR." exit 1 fi @@ -268,10 +268,10 @@ jobs: print(f'All {len(arms)} reviewer E2E arms ran and passed.') # The repository-config E2E is armed by the same CLI pins and returns early the same way; require each arm's marker. - repo_config_arms = ('claude-code single-repo Confined', 'claude-code multi-repo Confined', 'memory-link-outside claude-code multi-repo Confined', 'nested-in-place claude-code single-repo Confined', 'nested-in-place claude-code multi-repo Confined', 'nested-over-budget claude-code single-repo Confined', 'scoped-rule-pointer claude-code single-repo Confined', 'scoped-rule-pointer claude-code multi-repo Confined', 'pointer-differential claude-code single-repo Confined', 'pointer-differential claude-code multi-repo Confined', 'nested-subagent-pointer claude-code single-repo Confined', 'codex-cli single-repo', 'codex-cli multi-repo', 'codex-cli scratch') + repo_config_arms = ('claude-code single-repo Confined', 'claude-code multi-repo Confined', 'memory-link-outside claude-code multi-repo Confined', 'nested-in-place claude-code single-repo Confined', 'nested-in-place claude-code multi-repo Confined', 'nested-over-budget claude-code single-repo Confined', 'scoped-rule-pointer claude-code single-repo Confined', 'scoped-rule-pointer claude-code multi-repo Confined', 'pointer-differential claude-code single-repo Confined', 'pointer-differential claude-code multi-repo Confined', 'nested-subagent-pointer claude-code single-repo Confined', 'codex-cli single-repo', 'codex-cli multi-repo', 'agents-md codex-cli multi-repo', 'codex-cli scratch') for arm in repo_config_arms: assert f'[repo-config-e2e] ran {arm}' in text, f'repository-config E2E arm "{arm}" did not run — check CODESPACE_REQUIRE_REVIEW_CLIS and the CLI install step' - repo_config_methods = (('A_claude_run_ignores_the_settings_its_repository_commits_and_still_reads_its_memory', 1), ('A_multi_repo_claude_run_reads_every_repositorys_memory_and_none_of_its_settings', 1), ('A_claude_run_leaves_out_repository_memory_that_links_outside_the_workspace', 1), ('A_claude_run_reads_nested_memory_in_place_and_nothing_it_must_not', 2), ('A_claude_run_over_the_in_place_budget_loads_no_nested_memory_up_front', 1), ('An_explore_subagent_is_pointed_at_nested_memory_loaded_in_place', 1), ('A_scoped_rule_reaches_the_model_only_after_a_read_it_matches', 2), ('Pointer_rules_attach_where_the_unpinned_cli_attached_project_rules', 2), ('A_codex_run_ignores_the_config_and_hooks_its_repository_commits_and_still_reads_its_agents_md', 1), ('A_multi_repo_codex_run_starts_at_a_workspace_root_that_is_no_repository_and_loads_no_config_from_it', 1), ('A_repo_less_codex_run_starts_in_a_scratch_directory_that_is_no_repository', 1)) + repo_config_methods = (('A_claude_run_ignores_the_settings_its_repository_commits_and_still_reads_its_memory', 1), ('A_multi_repo_claude_run_reads_every_repositorys_memory_and_none_of_its_settings', 1), ('A_claude_run_leaves_out_repository_memory_that_links_outside_the_workspace', 1), ('A_claude_run_reads_nested_memory_in_place_and_nothing_it_must_not', 2), ('A_claude_run_over_the_in_place_budget_loads_no_nested_memory_up_front', 1), ('An_explore_subagent_is_pointed_at_nested_memory_loaded_in_place', 1), ('A_scoped_rule_reaches_the_model_only_after_a_read_it_matches', 2), ('Pointer_rules_attach_where_the_unpinned_cli_attached_project_rules', 2), ('A_codex_run_ignores_the_config_and_hooks_its_repository_commits_and_still_reads_its_agents_md', 1), ('A_multi_repo_codex_run_starts_at_a_workspace_root_that_is_no_repository_and_loads_no_config_from_it', 1), ('A_multi_repo_codex_run_reads_every_repositorys_agents_md', 1), ('A_repo_less_codex_run_starts_in_a_scratch_directory_that_is_no_repository', 1)) for method, rows in repo_config_methods: cases = [r for r in results if 'RepositoryConfigE2ETests.' + method in r.get('testName', '')] assert len(cases) == rows and all(r.get('outcome') == 'Passed' for r in cases), f'{method}: all {rows} case(s) must pass' diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs index 1ca124950..b81a8f00c 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexHarness.cs @@ -163,6 +163,8 @@ public SandboxSpec BuildInvocation(AgentTask task) { EnsureWithinInputCap(task.Goal); + var guides = CodexRepositoryGuides.For(task); + // P3.2: a CONTINUE re-stage rewrites the `exec --json` seed to `exec resume --json` so Codex picks up the // prior thread. The subcommand must follow `exec` directly; --model, the `-c` overrides (incl. the sandbox on // the resume path — see AppendSandbox), and the stdin `-` positional follow. Null (a fresh run) → the plain seed. @@ -223,12 +225,14 @@ public SandboxSpec BuildInvocation(AgentTask task) // Codex's native loader discovers them there (the same Agent-Skills format + SkillProjection as Claude — // only the root differs, which is why it's CODEX_HOME's, not CLAUDE_CONFIG_DIR's). On a CONTINUE the prior // session's rollout is restored alongside them under sessions/ (see BuildConfigHomeFiles). - ConfigHomeFiles = BuildConfigHomeFiles(task), + ConfigHomeFiles = BuildConfigHomeFiles(task, guides.Appendix), // The agent reaches the network only when its permissions allow it (the sandbox severs egress otherwise). AllowNetwork = task.Permissions.Network == AgentNetworkAccess.On, // Codex's own sandbox is a nested bubblewrap that cannot start inside ours, so where our runner confines // the run it stands that sandbox down and ours bounds every command instead (see SandboxStandDown). WhenRunnerConfines = SandboxStandDown(task), + // Each repository doc the run's AGENTS.md left out or cut, for the run's timeline. + LaunchNotices = guides.Notices, }; } @@ -333,16 +337,19 @@ private static void EnsureWithinInputCap(string goal) /// /// The config-home files the runner materializes: (1) B1 — AGENTS.md carrying the persona + the always-on /// operating contract (Codex's native instruction channel, since exec has no system-prompt flag; codex loads - /// $CODEX_HOME/AGENTS.md and merges it with any workspace AGENTS.md — verified against 0.142.2), ALWAYS present; - /// (2) the persona's projected skills; PLUS (3) — on a CONTINUE — the prior session's restored rollout at - /// sessions/rollout-<sessionId>.jsonl where codex exec resume finds it (codex scans sessions/ - /// at any depth and matches the id in the rollout-… filename, so a deterministic id-named rollout suffices). + /// $CODEX_HOME/AGENTS.md and merges it with any workspace AGENTS.md — verified against 0.142.2), ALWAYS present, + /// and after them , the AGENTS.md of each repository below a multi-repo + /// run's cwd, which Codex never reads from there (; empty for every other run, whose + /// file is byte-identical); (2) the persona's projected skills; PLUS (3) — on a CONTINUE — the prior session's + /// restored rollout at sessions/rollout-<sessionId>.jsonl where codex exec resume finds it (codex + /// scans sessions/ at any depth and matches the id in the rollout-… filename, so a deterministic + /// id-named rollout suffices). /// - private static IReadOnlyList BuildConfigHomeFiles(AgentTask task) + private static IReadOnlyList BuildConfigHomeFiles(AgentTask task, string repositoryDocs) { var files = new List(SkillProjection.ToConfigHomeFiles(task.Skills, SkillsRoot)) { - new() { RelativePath = AgentsFile, Content = AgentOperatingContract.Compose(task.SystemPrompt) }, + new() { RelativePath = AgentsFile, Content = AgentOperatingContract.Compose(task.SystemPrompt) + repositoryDocs }, }; // On a CONTINUE, restore the prior rollout so `codex exec resume` re-opens the thread. diff --git a/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexRepositoryGuides.cs b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexRepositoryGuides.cs new file mode 100644 index 000000000..a48c4a47f --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Agents/Harnesses/Codex/CodexRepositoryGuides.cs @@ -0,0 +1,153 @@ +using System.Text; +using System.Text.RegularExpressions; +using CodeSpace.Core.Services.Agents.Workspace; +using CodeSpace.Messages.Agents; + +namespace CodeSpace.Core.Services.Agents.Harnesses.Codex; + +/// +/// The AGENTS.md of every repository below a multi-repo run's cwd, for the run's own CODEX_HOME/AGENTS.md +/// (see CodexHarness.BuildConfigHomeFiles). Codex reads a project doc from its cwd and the directories above it, +/// never from one below (observed against 0.142.2), and a multi-repo run's cwd is the workspace root, which holds each +/// repository in a folder of its own and is no repository itself, so none of theirs reached the model. A run whose cwd +/// is a repository — single-repo, or at the primary repository — gets nothing here: Codex loads that doc itself, and +/// appending it would hand the model the doc twice. +/// +/// Each repository strictly inside the cwd gives the doc Codex would pick at its root: AGENTS.override.md +/// whenever that is a file or a link, even one that holds nothing, else AGENTS.md. It is read as Codex reads a +/// project doc — its first decoded lossily, nothing at all when that is only +/// whitespace — and appended after the persona and the operating contract under a separator that names it, +/// --- project-doc (<repository>/AGENTS.md) ---, beside Codex's own --- project-doc ---. That is the +/// instruction block a single-repo run's doc reaches, at the same authority. Codex gives the file no import or mention +/// semantics (observed against 0.142.2: an @ path, a ~ path or a $skill in it attaches nothing), +/// so the repository's text reaches the model and nothing it names does. +/// +/// A doc that resolves outside the workspace () is left out, and so is one whose +/// repository's path below the cwd holds anything but 's characters, since the separator +/// repeats it; the run's timeline says so, and says when a doc was cut. A doc that dangles or is no regular file gives +/// nothing. Every file is opened no-follow, non-blocking and only if regular, so a FIFO cannot hang the build. The read +/// runs on every build, a revise round's included, when no agent process is running. On a host that is neither Linux nor +/// macOS nothing is appended. +/// +internal static partial class CodexRepositoryGuides +{ + /// The most bytes of one repository's doc a run is given: Codex's own project_doc_max_bytes default, past which 0.142.2 hands the model no more of a project doc. Pinned by a test. + internal const int MaxProjectDocBytes = 32 * 1024; + + /// The docs Codex looks for at a repository's root, the one it picks first. + private static readonly string[] DocNames = ["AGENTS.override.md", "AGENTS.md"]; + + private static readonly Plan None = new("", []); + + private static readonly Doc Nothing = new("", []); + + /// The characters a repository's path below the cwd may hold to be named in a separator. + [GeneratedRegex(@"^[A-Za-z0-9._@+\-/]+\z")] + private static partial Regex SafeRelativePath(); + + [GeneratedRegex(@"[^A-Za-z0-9._@+\-/]")] + private static partial Regex UnsafeCharacter(); + + /// What the run's AGENTS.md gains after the persona and the operating contract — empty for a run whose cwd is a repository — and one sentence for each doc left out or cut. + internal sealed record Plan(string Appendix, IReadOnlyList Notices); + + /// One repository's block, empty when it gives nothing, and what the timeline says about it. + private sealed record Doc(string Block, IReadOnlyList Notices); + + public static Plan For(AgentTask task) + { + if (!OperatingSystem.IsLinux() && !OperatingSystem.IsMacOS()) return None; + + var repositories = RepositoriesBelow(task); + + if (repositories.Count == 0) return None; + + var cwd = Normalized(task.WorkspaceDirectory!); + var workspace = PhysicalPath.File(cwd) ?? cwd; + var docs = repositories.Select(repository => DocOf(cwd, workspace, repository)).ToList(); + + return new Plan(string.Concat(docs.Select(doc => doc.Block)), docs.SelectMany(doc => doc.Notices).ToList()); + } + + /// Every repository strictly inside the cwd, once each, in the order the executor names them; none when the cwd is itself a repository or no workspace was materialised. + private static List RepositoriesBelow(AgentTask task) + { + if (string.IsNullOrWhiteSpace(task.WorkspaceDirectory)) return []; + + var cwd = Normalized(task.WorkspaceDirectory); + var repositories = (task.WorkspaceRepositoryDirectories ?? []).Select(Normalized).Distinct(StringComparer.Ordinal).ToList(); + + if (repositories.Contains(cwd, StringComparer.Ordinal)) return []; + + return repositories.Where(directory => directory.StartsWith(cwd + Path.DirectorySeparatorChar, StringComparison.Ordinal)).ToList(); + } + + private static string Normalized(string directory) => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)); + + /// The doc one repository gives, as the physical bounds it. + private static Doc DocOf(string cwd, string workspace, string repository) + { + if (Pick(repository) is not { } name) return Nothing; + + var relative = Path.GetRelativePath(cwd, repository); + + if (!SafeRelativePath().IsMatch(relative)) return LeftOut(relative, name, "its path holds a character other than a letter, a digit or one of . _ @ + - /"); + + if (PhysicalPath.File(Path.Combine(repository, name)) is not { } physical) return Nothing; + + if (!PhysicalPath.StaysInside(workspace, physical)) return LeftOut(relative, name, "it resolves outside the workspace"); + + return Read(physical) is { } bytes ? Rendered(relative, name, bytes) : Nothing; + } + + /// The doc Codex picks at the repository's root: the first of that is a file or a link, whatever it holds or leads to; null when there is neither. + private static string? Pick(string repository) => DocNames.FirstOrDefault(name => IsFileOrLink(Path.Combine(repository, name))); + + private static bool IsFileOrLink(string path) + { + try + { + var entry = new FileInfo(path); + + return entry.LinkTarget is not null || entry.Exists; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// A regular file's bytes, opened no-follow and non-blocking, read to one past so a longer doc is known to be cut; null for anything else. + private static byte[]? Read(string physical) + { + try + { + using var stream = new FileStream(LocalAcceptanceFileIdentity.Open(physical, directory: false), FileAccess.Read); + var buffer = new byte[MaxProjectDocBytes + 1]; + var total = 0; + + for (int read; total < buffer.Length && (read = stream.Read(buffer, total, buffer.Length - total)) > 0;) total += read; + + return buffer[..total]; + } + catch (IOException) + { + return null; + } + } + + /// The doc's block: its first decoded as Codex decodes them, a character split at the cut included, under the separator naming it; nothing for a doc of only whitespace. + private static Doc Rendered(string relative, string name, byte[] bytes) + { + var text = Encoding.UTF8.GetString(bytes, 0, Math.Min(bytes.Length, MaxProjectDocBytes)); + + if (string.IsNullOrWhiteSpace(text)) return Nothing; + + var cut = bytes.Length > MaxProjectDocBytes ? [$"Left all but the first {MaxProjectDocBytes} bytes of the {name} of '{relative}' out of this run: Codex reads no more of a project doc."] : Array.Empty(); + + return new Doc($"\n\n--- project-doc ({relative}/{name}) ---\n\n{text}", cut); + } + + /// No block, and the one sentence that says why; the repository's path as the sentence repeats it has every character the separator could not carry replaced. + private static Doc LeftOut(string relative, string name, string why) => new("", [$"Left the {name} of '{UnsafeCharacter().Replace(relative, "?")}' out of this run: {why}."]); +} diff --git a/backend/src/CodeSpace.Messages/Agents/AgentTask.cs b/backend/src/CodeSpace.Messages/Agents/AgentTask.cs index 763420c96..106640a02 100644 --- a/backend/src/CodeSpace.Messages/Agents/AgentTask.cs +++ b/backend/src/CodeSpace.Messages/Agents/AgentTask.cs @@ -243,7 +243,8 @@ public sealed record AgentTask /// The directory of every repository materialised in the workspace, stamped by the executor beside /// at launch: a single-repo workspace's one entry is that directory, a multi-repo /// workspace's sit below its root. A harness that names directories to its CLI reads it — Claude Code adds each one - /// inside the workspace so every repository's memory loads. Empty for a scratch workspace that holds no repository; + /// inside the workspace so every repository's memory loads, and Codex, whose cwd at a multi-repo root is no repository, + /// is handed the AGENTS.md of each one below it. Empty for a scratch workspace that holds no repository; /// null when no workspace was materialised, including a task that names its own . /// [JsonIgnore(WhenWritingNull)]. /// diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/RealHarnessCodexMultiRepoGuidesTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/RealHarnessCodexMultiRepoGuidesTests.cs new file mode 100644 index 000000000..de25d27f3 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/RealHarnessCodexMultiRepoGuidesTests.cs @@ -0,0 +1,260 @@ +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Workflows.Infrastructure; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// Each repository's AGENTS.md run through the production pipeline of a multi-repo Codex run: does the file Codex +/// reads its instructions from, $CODEX_HOME/AGENTS.md, carry every repository's doc after the operating contract +/// when the run works at the workspace root, and does the run's timeline say when a doc was cut? +/// +/// 🟡 Medium-mock (Rule 12): the real DI-wired , the real , +/// LocalGitWorkspaceProvider cloning each repository from a file:// bare remote into a folder of its own +/// below the workspace root, the real writing the run's config home, and real Postgres. +/// Only the CLI is a fake: a /bin/sh script armed through that reads the +/// AGENTS.md in the CODEX_HOME it was really spawned with and fails the run unless the contract comes first +/// and each repository's doc after it, in order. That the real CLI hands that file to the model at a workspace root, and +/// reads no repository's doc from below it on its own, is pinned by RepositoryConfigE2ETests. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public sealed class RealHarnessCodexMultiRepoGuidesTests +{ + private const string NoticePrefix = "Left "; + + /// The related repository's alias, and the folder below the workspace root it is cloned into. + private const string RelatedAlias = "api"; + + private readonly PostgresFixture _fixture; + + public RealHarnessCodexMultiRepoGuidesTests(PostgresFixture fixture) { _fixture = fixture; } + + [Theory] + [InlineData(false)] // each doc whole: nothing to say + [InlineData(true)] // the primary's doc runs past Codex's cap: cut where Codex cuts it, and said once + public async Task A_multi_repo_codex_run_reads_every_repositorys_agents_md_after_the_operating_contract(bool oversized) + { + if (OperatingSystem.IsWindows()) return; // the fake CLI is a /bin/sh script + + var nonce = Guid.NewGuid().ToString("N"); + var primaryDoc = $"PRIMARY-DOC-{nonce}\n" + (oversized ? new string('x', CodexRepositoryGuides.MaxProjectDocBytes) + $"\nPRIMARY-TAIL-{nonce}\n" : ""); + using var primary = new BareRemote(); + using var related = new BareRemote(); + await primary.SeedAsync(new Dictionary { ["AGENTS.md"] = primaryDoc }); + await related.SeedAsync(new Dictionary { ["AGENTS.override.md"] = $"OVERRIDE-DOC-{nonce}\n", ["AGENTS.md"] = $"PLAIN-DOC-{nonce}\n" }); + using var cli = new GuideCheckingFakeCli(AgentOperatingContract.Compose(null).Split('\n')[0], $"PRIMARY-DOC-{nonce}", $"OVERRIDE-DOC-{nonce}", forbidden: [$"PLAIN-DOC-{nonce}", $"PRIMARY-TAIL-{nonce}"]); + + var (teamId, userId) = await SeedTeamAsync(); + var (primaryId, relatedId) = await SeedBoundRepositoriesAsync(teamId, primary.Url, related.Url); + var runId = await CreateRunAsync(teamId, userId, primaryId, relatedId, cli.Env()); + + await ExecuteRealAsync(runId); + + var run = await LoadAsync(runId); + var notices = (await LoadEventsAsync(runId)).Where(e => e.Text.StartsWith(NoticePrefix, StringComparison.Ordinal)).ToList(); + var read = cli.Read(); + + run.Status.ShouldBe(AgentRunStatus.Succeeded, $"the fake codex fails a run whose $CODEX_HOME/AGENTS.md does not carry the contract and then each repository's doc, in order; error: {run.Error}; it read: {read}"); + read.ShouldContain($"\n\n--- project-doc ({WorkspaceSpec.DefaultAlias}/AGENTS.md) ---\n\nPRIMARY-DOC-{nonce}\n", customMessage: "the primary's doc rides under a separator naming the folder it was cloned into"); + read.ShouldContain($"\n\n--- project-doc ({RelatedAlias}/AGENTS.override.md) ---\n\nOVERRIDE-DOC-{nonce}\n", customMessage: "the related repository's override is the doc Codex itself would pick there"); + notices.Select(e => (e.Kind, e.Text)).ShouldBe(oversized ? new[] { (AgentEventKind.Warning, $"Left all but the first {CodexRepositoryGuides.MaxProjectDocBytes} bytes of the AGENTS.md of '{WorkspaceSpec.DefaultAlias}' out of this run: Codex reads no more of a project doc.") } : [], "a Warning when the run is given less of a doc than its repository holds, and nothing otherwise"); + } + + private async Task CreateRunAsync(Guid teamId, Guid userId, Guid primaryId, Guid relatedId, IReadOnlyDictionary env) + { + using var scope = _fixture.BeginScopeAs(userId, teamId); + + var workspace = new WorkspaceSpec + { + PrimaryAlias = WorkspaceSpec.DefaultAlias, + Repositories = new[] + { + new WorkspaceRepositorySpec { Alias = WorkspaceSpec.DefaultAlias, RepositoryId = primaryId, Access = WorkspaceAccess.Write, IsPrimary = true }, + new WorkspaceRepositorySpec { Alias = RelatedAlias, RepositoryId = relatedId, Access = WorkspaceAccess.Read }, + }, + }; + var task = new AgentTask { Goal = "read every repository's instructions", Harness = CodexHarness.HarnessKind, Model = null, Workspace = workspace, Environment = env, TimeoutSeconds = 120 }; + var run = await scope.Resolve().CreateAsync(task, teamId, null, null, iterationKey: "", cancellationToken: CancellationToken.None); + + return run.Id; + } + + private async Task ExecuteRealAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + await scope.Resolve().ExecuteAsync(runId, CancellationToken.None); + } + + private async Task LoadAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + return await scope.Resolve().GetAsync(runId, CancellationToken.None); + } + + private async Task> LoadEventsAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + var run = await scope.Resolve().GetAsync(runId, CancellationToken.None); + return await scope.Resolve().GetEventsAsync(runId, run.TeamId, afterSequence: 0, CancellationToken.None); + } + + private async Task<(Guid TeamId, Guid UserId)> SeedTeamAsync() + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + + var userId = Guid.NewGuid(); + db.User.Add(new User { Id = userId, Email = $"guides-{userId:N}@test.local", Name = $"guides-{userId:N}" }); + + var teamId = Guid.NewGuid(); + db.Team.Add(new Team { Id = teamId, Slug = $"guides-{teamId:N}", Name = "Codex Guides Team", Kind = TeamKind.Workspace }); + db.TeamMembership.Add(new TeamMembership { Id = Guid.NewGuid(), TeamId = teamId, UserId = userId, Role = TeamRole.Owner }); + + await db.SaveChangesAsync(); + return (teamId, userId); + } + + /// Two repositories bound through one provider instance and PAT credential, so each clone carries a token as production's does — as RealHarnessWorkspaceMemoryTests seeds one. + private async Task<(Guid Primary, Guid Related)> SeedBoundRepositoriesAsync(Guid teamId, string primaryUrl, string relatedUrl) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + + var instanceId = Guid.NewGuid(); + db.ProviderInstance.Add(new ProviderInstance { Id = instanceId, TeamId = teamId, Provider = ProviderKind.GitHub, DisplayName = "local", BaseUrl = "https://local" }); + + var payloadJson = scope.Resolve().Serialize(new PatPayload { Token = "agent-clone-token" }); + + var credentialId = Guid.NewGuid(); + db.Credential.Add(new Credential + { + Id = credentialId, TeamId = teamId, ProviderInstanceId = instanceId, + AuthType = AuthType.Pat, DisplayName = "clone cred", + EncryptedPayload = scope.Resolve().Encrypt(payloadJson), Status = CredentialStatus.Active, + }); + + Repository Bound(string name, string cloneUrlHttps) => new() + { + Id = Guid.NewGuid(), TeamId = teamId, ProviderInstanceId = instanceId, CredentialId = credentialId, + ExternalId = Guid.NewGuid().ToString(), NamespacePath = "org", Name = name, FullPath = $"org/{name}", + DefaultBranch = "main", CloneUrlHttps = cloneUrlHttps, WebUrl = $"https://local/org/{name}", + }; + + var (primary, related) = (Bound("primary", primaryUrl), Bound("related", relatedUrl)); + db.Repository.AddRange(primary, related); + + await db.SaveChangesAsync(); + return (primary.Id, related.Id); + } + + /// A bare local remote whose one commit holds the files given. GUID-suffixed; best-effort cleanup. + private sealed class BareRemote : IDisposable + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "cs-guides-remote-" + Guid.NewGuid().ToString("N")); + private readonly string _bare; + + public BareRemote() + { + Directory.CreateDirectory(_root); + _bare = Path.Combine(_root, "remote.git"); + } + + public string Url => new Uri(_bare).AbsoluteUri; + + public async Task SeedAsync(IReadOnlyDictionary files) + { + await Git(_root, "init", "--bare", "-b", "main", _bare); + + var seed = Path.Combine(_root, "seed"); + Directory.CreateDirectory(seed); + await Git(seed, "clone", _bare, seed); + await Git(seed, "config", "user.email", "test@codespace.dev"); + await Git(seed, "config", "user.name", "Test"); + await Git(seed, "config", "commit.gpgsign", "false"); + + foreach (var (relative, content) in files) await File.WriteAllTextAsync(Path.Combine(seed, relative), content); + + await Git(seed, "add", "-A"); + await Git(seed, "commit", "-m", "seed"); + await Git(seed, "push", "origin", "main"); + } + + private static async Task Git(string workdir, params string[] args) + { + var result = await new LocalProcessRunner().RunAsync(new SandboxSpec { Command = "git", Args = args, WorkingDirectory = workdir, TimeoutSeconds = 60 }, CancellationToken.None); + + if (result.Status != SandboxStatus.Success) + throw new InvalidOperationException($"git {string.Join(' ', args)} failed: {result.Stderr}"); + } + + public void Dispose() + { + try { Directory.Delete(_root, recursive: true); } catch { /* best-effort */ } + } + } + + /// + /// The fake codex: copies the $CODEX_HOME/AGENTS.md it was spawned with aside for the test, then exits 9 + /// unless that file carries the contract's first line, then the first doc's sentinel, then the second's, each on a + /// later line than the one before, and none of the forbidden sentinels; otherwise prints a successful Codex stream. + /// Named and staged with the markers, so a real-CLI gate elsewhere in the process + /// sees it for a fake. Arms the process-wide ; restores it and deletes its + /// directory on dispose. + /// + private sealed class GuideCheckingFakeCli : IDisposable + { + private readonly string? _original; + private readonly string _directory = Path.Combine(Path.GetTempPath(), "cs-guides" + FakeAgentCliMarker.DirectoryMarker + Guid.NewGuid().ToString("N")); + private readonly string _read; + private readonly Dictionary _env; + + public GuideCheckingFakeCli(string contract, string first, string second, IReadOnlyList forbidden) + { + Directory.CreateDirectory(_directory); + _read = Path.Combine(_directory, "agents-read.md"); + _env = new Dictionary { ["FAKE_READ"] = _read, ["FAKE_CONTRACT"] = contract, ["FAKE_FIRST"] = first, ["FAKE_SECOND"] = second, ["FAKE_FORBIDDEN"] = string.Join(' ', forbidden) }; + + var script = Path.Combine(_directory, FakeAgentCliMarker.ScriptNamePrefix + "codex.sh"); + File.WriteAllText(script, """ + #!/bin/sh + cat > /dev/null + doc="$CODEX_HOME/AGENTS.md" + cp "$doc" "$FAKE_READ" || { echo "no AGENTS.md in CODEX_HOME '$CODEX_HOME'" >&2; exit 9; } + line() { grep -n -F -- "$1" "$doc" | head -1 | cut -d: -f1; } + contract=$(line "$FAKE_CONTRACT"); first=$(line "$FAKE_FIRST"); second=$(line "$FAKE_SECOND") + [ -n "$contract" ] && [ -n "$first" ] && [ -n "$second" ] && [ "$contract" -lt "$first" ] && [ "$first" -lt "$second" ] || { echo "AGENTS.md does not carry the contract (line '$contract'), then '$FAKE_FIRST' (line '$first'), then '$FAKE_SECOND' (line '$second')" >&2; exit 9; } + for word in $FAKE_FORBIDDEN; do + ! grep -q -F -- "$word" "$doc" || { echo "AGENTS.md carries '$word', which the run must not be given" >&2; exit 9; } + done + printf '%s\n' '{"type":"agent_message","message":"read every guide"}' '{"type":"task_complete","message":"completed"}' + + """); + File.SetUnixFileMode(script, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute); + + _original = Environment.GetEnvironmentVariable(CodexHarness.CommandEnvVar); + Environment.SetEnvironmentVariable(CodexHarness.CommandEnvVar, script); + } + + public IReadOnlyDictionary Env() => _env; + + /// The AGENTS.md the fake found in its CODEX_HOME; empty when it never ran. + public string Read() => File.Exists(_read) ? File.ReadAllText(_read) : ""; + + public void Dispose() + { + Environment.SetEnvironmentVariable(CodexHarness.CommandEnvVar, _original); + try { Directory.Delete(_directory, recursive: true); } catch { /* best-effort */ } + } + } +} diff --git a/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.CodexGuides.cs b/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.CodexGuides.cs new file mode 100644 index 000000000..42167cef5 --- /dev/null +++ b/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.CodexGuides.cs @@ -0,0 +1,90 @@ +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Core.Services.Agents.Sandbox.Isolation; +using CodeSpace.Messages.Agents; +using Shouldly; + +namespace CodeSpace.SandboxTests; + +/// +/// Every repository's own AGENTS.md in a multi-repo Codex run. The pinned 0.142.2 reads a project doc from its +/// cwd and the directories above it, never from one below, and a multi-repo run's cwd is the workspace root, so before +/// the harness appended them (CodexRepositoryGuides) no repository's doc reached the model. The arm pins both +/// halves against the real binary: what the harness appends to the run's CODEX_HOME/AGENTS.md reaches the +/// model in the instruction block the operating contract opens, and the CLI reads none of those docs on its own, so +/// each reaches it once. +/// +/// Same fidelity as the class: the pinned binary, the production argv and runner, the production broker; only the +/// model is scripted. Root lane, Standard: the posture a shipped Codex run has, which the CLI runs as uid 0. +/// +public sealed partial class RepositoryConfigE2ETests +{ + /// What every request carries in the instruction block CODEX_HOME/AGENTS.md opens: the operating contract the harness writes first. + private const string OperatingContractText = "UNATTENDED agent"; + + /// + /// Three repositories below a workspace root that is no repository: the first commits an AGENTS.md, the second + /// an AGENTS.override.md beside one, and the third an AGENTS.md linked to a file outside the workspace, + /// where a confined run could still read it, so its absence is the containment check's doing. Every repository also + /// commits hostile Codex config. The first request's instruction block must carry the + /// operating contract, then the first repository's doc, then the second's override, each once; the second's plain + /// doc and the outside file must reach no request; the third must be named on the launch; and none of the config + /// may load. + /// + [Fact] + public async Task A_multi_repo_codex_run_reads_every_repositorys_agents_md() + { + const string harnessKind = CodexHarness.HarnessKind; + var harness = ReviewerReadsItsDiffE2ETests.HarnessFor(harnessKind); + + if (!ReviewerReadsItsDiffE2ETests.Armed(harnessKind) || OperatingSystem.IsWindows()) return; + + await ReviewerReadsItsDiffE2ETests.RequirePinnedBinaryAsync(harness, harnessKind); + + using var hostile = new ConnectionCounter(); + var workspace = NewWorkspace(repositories: 3); + var (plain, overridden, linked) = (workspace.Repositories[0], workspace.Repositories[1], workspace.Repositories[2]); + var markers = workspace.Repositories.Select(repo => new Markers(repo, "mcp-server", "session", "prompt", "stop")).ToList(); + var outside = Path.Combine(NewOutsideDirectory(), "AGENTS.md"); + + foreach (var (repo, marked) in workspace.Repositories.Zip(markers)) PlantCodexConfig(repo, hostile, marked); + + File.WriteAllText(outside, $"{Mention(linked, "OUTSIDE-DOC")}\n"); + plain.Commit("AGENTS.md", $"{Mention(plain, "PROJECT-DOC")}\n"); + overridden.Commit("AGENTS.md", $"{Mention(overridden, "PLAIN-DOC")}\n"); + overridden.Commit("AGENTS.override.md", $"{Mention(overridden, "OVERRIDE-DOC")}\n"); + linked.CommitLink("AGENTS.md", outside); + + GitRepositoryHolding(workspace.Directory).ShouldBeNull("fixture check: the workspace root must sit in no git repository, or the CLI's own project-doc walk could reach the repositories' docs and this says nothing"); + + var (spec, run, upstream) = await RunAsync(harness, workspace, AgentAutonomyLevel.Standard, task => task); + + spec.WorkingDirectory.ShouldBe(workspace.Directory, "fixture check: the run must start at the workspace root, where a multi-repo run's cwd is"); + BrokerViolations(run, upstream, hostile, workspace).Concat(markers.SelectMany(marked => marked.Ran())).ShouldBeEmpty(Diagnosis(harnessKind, spec, run, upstream)); + + var first = upstream.Requests.FirstOrDefault(r => r.Path.EndsWith("/responses", StringComparison.Ordinal))?.Body ?? ""; + var block = InstructionBlock(first); + var order = new[] { OperatingContractText, SurfaceText(plain, "PROJECT-DOC"), SurfaceText(overridden, "OVERRIDE-DOC") }.Select(text => block.IndexOf(text, StringComparison.Ordinal)).ToList(); + + order.ShouldAllBe(at => at >= 0, $"the first request's instruction block must carry the operating contract, the first repository's AGENTS.md and the second's AGENTS.override.md (positions {string.Join(", ", order)}). {Diagnosis(harnessKind, spec, run, upstream)}"); + order.ShouldBeInOrder(SortDirection.Ascending, $"each repository's doc must follow the operating contract, in the order the executor names the repositories. {Diagnosis(harnessKind, spec, run, upstream)}"); + Occurrences(first, SurfaceText(plain, "PROJECT-DOC")).ShouldBe(1, $"a repository's doc must reach the model once: the CLI reads none below its cwd on its own. {Diagnosis(harnessKind, spec, run, upstream)}"); + upstream.Requests.ShouldNotContain(r => r.Body.Contains(SurfaceText(overridden, "PLAIN-DOC"), StringComparison.Ordinal), $"an override beside it is the doc Codex picks, so the plain AGENTS.md must reach no request. {Diagnosis(harnessKind, spec, run, upstream)}"); + upstream.Requests.ShouldNotContain(r => r.Body.Contains(SurfaceText(linked, "OUTSIDE-DOC"), StringComparison.Ordinal), $"a doc linked outside the workspace must reach no request. {Diagnosis(harnessKind, spec, run, upstream)}"); + spec.LaunchNotices.ShouldBe(new[] { $"Left the AGENTS.md of '{Path.GetFileName(linked.Directory)}' out of this run: it resolves outside the workspace." }); + + output.WriteLine($"{RanMarker} agents-md codex-cli multi-repo Standard uid={NonRootWorker.EffectiveUid()} confined={BubblewrapSandbox.Available is not null} hostileConnections={hostile.Connections}"); + } + + /// The text of the request's input item that carries the operating contract — the instruction block the CLI builds from CODEX_HOME/AGENTS.md and any project doc — or empty when none does. + private static string InstructionBlock(string body) => + TryParse(body) is { } request ? Items(request, "input").SelectMany(item => Items(item, "content")).Select(part => Text(part, "text")).FirstOrDefault(text => text.Contains(OperatingContractText, StringComparison.Ordinal)) ?? "" : ""; + + private static int Occurrences(string text, string value) + { + var count = 0; + + for (var at = text.IndexOf(value, StringComparison.Ordinal); at >= 0; at = text.IndexOf(value, at + value.Length, StringComparison.Ordinal)) count++; + + return count; + } +} diff --git a/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.cs b/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.cs index 08d8f0cb7..881504f4b 100644 --- a/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.cs +++ b/backend/tests/CodeSpace.SandboxTests/RepositoryConfigE2ETests.cs @@ -22,11 +22,11 @@ namespace CodeSpace.SandboxTests; /// Codex's .codex/config.toml and .codex/hooks.json — and the target repository is untrusted input. A file /// committed there must not take the run's model call off its broker, and must not run a command the model never /// chose. The repository's own instructions (CLAUDE.md, AGENTS.md) are context, not config, and must -/// still reach the model. For Claude that holds for every repository of a multi-repo workspace. The multi-repo Codex -/// arm asserts less: the run starts at the workspace root, reaches its broker and loads no config from that root. It -/// does not assert that the AGENTS.md of a repository below that root reaches the model. A repo-less Codex run -/// must likewise start in its scratch directory, and where nothing of ours confines a multi-repo run, Codex's own -/// sandbox must keep every repository's .git and .codex read-only +/// still reach the model. For Claude that holds for every repository of a multi-repo workspace, and for Codex too +/// (); a multi-repo Codex run must also start at +/// the workspace root, reach its broker and load no config from that root. A repo-less Codex run must likewise start in +/// its scratch directory, and where nothing of ours confines a multi-repo run, Codex's own sandbox must keep every +/// repository's .git and .codex read-only /// (, run by the unconfined lane). The platform's own /// Claude Stop hook must still run beside the settings an Allowlist run carries on its argv /// (, run by the non-root lane). @@ -74,9 +74,11 @@ namespace CodeSpace.SandboxTests; /// directory it resolves as its cwd, so an entry keyed only by a workspace path under a symlink matched nothing. Codex /// also refused to start at a multi-repo workspace's root, or in a repo-less run's scratch directory, neither of which /// is a git repository: without --skip-git-repo-check it exits 1 before any model request. A single-repo arm -/// cannot see that. Once it started at a multi-repo root, its own sandbox kept .git and .codex read-only -/// only at that root, so where nothing of ours confined the run, each repository's .git/hooks and -/// .git/config were writable to the agent. Unpinned, the same Claude also named the repository's skill, command and +/// cannot see that. Started there, it read no repository's AGENTS.md: it reads a project doc from its cwd and +/// the directories above it, never from one below. Once it started at a multi-repo root, its own sandbox kept +/// .git and .codex read-only only at that root, so where nothing of ours confined the run, each +/// repository's .git/hooks and .git/config were writable to the agent. Unpinned, the same Claude also +/// named the repository's skill, command and /// agent in its first request and on its init line, put CLAUDE.local.md and the selected output style in /// front of the model, attached the scoped rule and sub/CLAUDE.md once the run read sub/notes.txt, and, at /// Standard, ran the skill's hook and the skill's and the command's shell once invoked. Codex started a repository @@ -186,6 +188,7 @@ public async Task A_codex_run_ignores_the_config_and_hooks_its_repository_commit violations.ShouldBeEmpty(Diagnosis(harnessKind, spec, run, upstream)); File.Exists(ownHook).ShouldBeTrue($"the platform's own Stop hook must still run with the repository's hooks shut out — a distrust that also silenced it would disable in-loop acceptance. {Diagnosis(harnessKind, spec, run, upstream)}"); upstream.Requests.ShouldContain(r => r.Body.Contains($"PROJECT-DOC-{repo.Nonce}", StringComparison.Ordinal), $"the repository's AGENTS.md is context, not config — it must still reach the model. {Diagnosis(harnessKind, spec, run, upstream)}"); + Occurrences(upstream.Requests.First(r => r.Path.EndsWith("/responses", StringComparison.Ordinal)).Body, $"PROJECT-DOC-{repo.Nonce}").ShouldBe(1, $"the CLI loads the doc of the repository it runs in itself, so the harness must not hand it over a second time. {Diagnosis(harnessKind, spec, run, upstream)}"); output.WriteLine($"{RanMarker} codex-cli single-repo confined={BubblewrapSandbox.Available is not null} hostileConnections={hostile.Connections}"); } @@ -199,7 +202,8 @@ public async Task A_codex_run_ignores_the_config_and_hooks_its_repository_commit /// config too. Codex never reads it from there, so those markers only catch a future CLI that reads config below its /// cwd; the distrust of a repository's own committed config is pinned by /// . Which - /// instructions reach the run from the repositories below its cwd is not asserted here. + /// instructions reach the run from the repositories below its cwd is pinned by + /// . /// [Fact] public async Task A_multi_repo_codex_run_starts_at_a_workspace_root_that_is_no_repository_and_loads_no_config_from_it() diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs index 8160cf51e..f24f88162 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/CodexHarnessTests.cs @@ -242,6 +242,61 @@ public void Persona_and_operating_contract_ride_agents_md_not_the_goal() CodexHarness.AgentsFile.ShouldBe("AGENTS.md"); // Rule 8: pin the native instruction file } + [Fact] + public void A_multi_repo_run_gets_each_repositorys_agents_md_after_the_persona_and_the_operating_contract() + { + // Codex reads a project doc from its cwd and above only, so at a workspace root none of the repositories' reached + // it (observed against 0.142.2). They ride the run's own AGENTS.md, after everything the platform says. + if (OperatingSystem.IsWindows()) return; + + using var tree = new TempTree(); + var workspace = tree.Directory("ws"); + tree.File("ws/api/AGENTS.md", "Keep the API stable.\n"); + tree.File("ws/web/AGENTS.md", "Keep the UI accessible.\n"); + + var spec = Harness.BuildInvocation(Task() with { SystemPrompt = "You are a meticulous reviewer.", WorkspaceDirectory = workspace, WorkspaceRepositoryDirectories = [Path.Combine(workspace, "api"), Path.Combine(workspace, "web")] }); + + spec.ConfigHomeFiles.Single(f => f.RelativePath == CodexHarness.AgentsFile).Content.ShouldBe(AgentOperatingContract.Compose("You are a meticulous reviewer.") + "\n\n--- project-doc (api/AGENTS.md) ---\n\nKeep the API stable.\n\n\n--- project-doc (web/AGENTS.md) ---\n\nKeep the UI accessible.\n"); + spec.LaunchNotices.ShouldBeEmpty(); + } + + [Fact] + public void A_single_repo_run_writes_the_same_agents_md_as_before_whatever_its_repository_holds() + { + // The cwd is the repository, whose AGENTS.md Codex loads itself: appending it would hand the model the doc twice. + if (OperatingSystem.IsWindows()) return; + + using var tree = new TempTree(); + var workspace = tree.Directory("ws"); + tree.File("ws/AGENTS.md", "Keep the change small.\n"); + tree.File("ws/pkg/AGENTS.md", "Keep the package pure.\n"); + + var spec = Harness.BuildInvocation(Task() with { SystemPrompt = "You are a meticulous reviewer.", WorkspaceDirectory = workspace, WorkspaceRepositoryDirectories = [workspace] }); + + spec.ConfigHomeFiles.Single(f => f.RelativePath == CodexHarness.AgentsFile).Content.ShouldBe(AgentOperatingContract.Compose("You are a meticulous reviewer.")); + spec.LaunchNotices.ShouldBeEmpty(); + } + + [Fact] + public void What_a_multi_repo_run_leaves_out_of_its_agents_md_is_on_its_launch_notices() + { + if (OperatingSystem.IsWindows()) return; + + using var tree = new TempTree(); + var workspace = tree.Directory("ws"); + tree.File("ws/api/AGENTS.md", new string('x', CodexRepositoryGuides.MaxProjectDocBytes + 1)); + tree.Link("ws/web/AGENTS.md", tree.File("outside/secret.md", "OUTSIDE")); + + var spec = Harness.BuildInvocation(Task() with { WorkspaceDirectory = workspace, WorkspaceRepositoryDirectories = [Path.Combine(workspace, "api"), Path.Combine(workspace, "web")] }); + + spec.LaunchNotices.ShouldBe(new[] + { + $"Left all but the first {CodexRepositoryGuides.MaxProjectDocBytes} bytes of the AGENTS.md of 'api' out of this run: Codex reads no more of a project doc.", + "Left the AGENTS.md of 'web' out of this run: it resolves outside the workspace.", + }); + spec.ConfigHomeFiles.Single(f => f.RelativePath == CodexHarness.AgentsFile).Content.ShouldNotContain("OUTSIDE"); + } + [Fact] public void SkillsRoot_is_pinned_to_the_codex_home_relative_skills_dir() { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/CodexRepositoryGuidesTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/CodexRepositoryGuidesTests.cs new file mode 100644 index 000000000..167670df4 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/CodexRepositoryGuidesTests.cs @@ -0,0 +1,279 @@ +using System.Diagnostics; +using System.Text; +using CodeSpace.Core.Services.Agents.Harnesses.Codex; +using CodeSpace.Messages.Agents; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// Pins what a multi-repo Codex run is given of each repository's own AGENTS.md (). +/// Codex reads a project doc only from its cwd and above, and a multi-repo run's cwd is the workspace root, so the +/// harness appends each repository's doc to the run's own CODEX_HOME/AGENTS.md: the one Codex itself would pick +/// there, cut where Codex cuts a project doc, under a separator naming it. Each case lays out a real workspace under a +/// GUID temp root spelled through 's root link, so every host also compares a workspace reached +/// through a symlink. +/// +[Trait("Category", "Unit")] +public sealed class CodexRepositoryGuidesTests : IDisposable +{ + private readonly TempTree _tree = new(); + private readonly string _workspace; + private readonly string _secret; + + public CodexRepositoryGuidesTests() + { + _workspace = _tree.Directory("ws"); + _secret = _tree.File("outside/secret.md", "OUTSIDE"); + } + + public void Dispose() => _tree.Dispose(); + + [Fact] + public void MaxProjectDocBytes_is_pinned() + { + // Codex's own project_doc_max_bytes default: 0.142.2 hands the model the first 32768 bytes of a project doc and + // no more. If a Codex bump moves it, this moves by PR with the CLI. + CodexRepositoryGuides.MaxProjectDocBytes.ShouldBe(32 * 1024); + } + + [Theory] + [InlineData("a single repository, which is the cwd itself")] + [InlineData("a cwd at the primary repository, its sibling beside it")] + [InlineData("a scratch directory, which holds no repository")] + [InlineData("a task that names its own workspace and no repositories")] + [InlineData("a directory that only shares the cwd's prefix")] + public void A_run_whose_cwd_has_no_repository_strictly_inside_it_gets_no_appendix(string shape) + { + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/AGENTS.md", "ROOT"); + _tree.File("ws/a/AGENTS.md", "A"); + _tree.File("ws/b/AGENTS.md", "B"); + _tree.File("ws-other/AGENTS.md", "OTHER"); + + var guides = shape switch + { + "a single repository, which is the cwd itself" => Guides(_workspace, _workspace), + "a cwd at the primary repository, its sibling beside it" => Guides(Path.Combine(_workspace, "a"), Path.Combine(_workspace, "a"), Path.Combine(_workspace, "b")), + "a scratch directory, which holds no repository" => Guides(_workspace, Array.Empty()), + "a task that names its own workspace and no repositories" => CodexRepositoryGuides.For(Task(_workspace, null)), + _ => Guides(_workspace, _workspace + "-other"), + }; + + guides.Appendix.ShouldBeEmpty($"{shape}: Codex reads the cwd's own doc itself, and no repository below the cwd is hidden from it"); + guides.Notices.ShouldBeEmpty(shape); + } + + [Fact] + public void Each_repository_below_the_cwd_gives_its_doc_under_a_separator_naming_it_in_the_order_the_executor_names_them() + { + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/repo-a/AGENTS.md", "Keep a small.\n"); + _tree.File("ws/repo-b/AGENTS.md", "Keep b typed.\n"); + _tree.Directory("ws/repo-c"); + + var guides = Guides(_workspace, Repo("repo-b"), Repo("repo-c"), Repo("repo-a")); + + guides.Appendix.ShouldBe("\n\n--- project-doc (repo-b/AGENTS.md) ---\n\nKeep b typed.\n\n\n--- project-doc (repo-a/AGENTS.md) ---\n\nKeep a small.\n", customMessage: "a repository without a doc gives nothing, and the rest keep the executor's order"); + guides.Notices.ShouldBeEmpty(); + } + + [Theory] + [InlineData("an override beside the doc", "OVERRIDE", "AGENTS.override.md")] + [InlineData("an empty override beside the doc", null, null)] + [InlineData("an override of only whitespace beside the doc", null, null)] + [InlineData("an override that is a directory beside the doc", "PLAIN", "AGENTS.md")] + [InlineData("an override linked to the doc beside it", "PLAIN", "AGENTS.override.md")] + public void The_doc_is_the_one_codex_picks_an_override_first_whatever_it_holds(string shape, string? text, string? name) + { + // Codex 0.142.2 picks AGENTS.override.md whenever it is a file or a link, and gives nothing for one that holds + // nothing, even with an AGENTS.md beside it; a directory of that name it passes over. + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/repo-a/AGENTS.md", "PLAIN"); + + switch (shape) + { + case "an override beside the doc": _tree.File("ws/repo-a/AGENTS.override.md", "OVERRIDE"); break; + case "an empty override beside the doc": _tree.File("ws/repo-a/AGENTS.override.md", ""); break; + case "an override of only whitespace beside the doc": _tree.File("ws/repo-a/AGENTS.override.md", " \n\t\n"); break; + case "an override that is a directory beside the doc": _tree.Directory("ws/repo-a/AGENTS.override.md"); break; + default: _tree.Link("ws/repo-a/AGENTS.override.md", "AGENTS.md"); break; + } + + var guides = Guides(_workspace, Repo("repo-a")); + + guides.Appendix.ShouldBe(text is null ? "" : $"\n\n--- project-doc (repo-a/{name}) ---\n\n{text}", customMessage: shape); + guides.Notices.ShouldBeEmpty(shape); + } + + [Theory] + [InlineData(CodexRepositoryGuides.MaxProjectDocBytes, false)] + [InlineData(CodexRepositoryGuides.MaxProjectDocBytes + 1, true)] + public void A_doc_is_cut_where_codex_cuts_a_project_doc_and_the_cut_is_said(int length, bool cut) + { + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/repo-a/AGENTS.md", new string('x', CodexRepositoryGuides.MaxProjectDocBytes) + new string('T', length - CodexRepositoryGuides.MaxProjectDocBytes)); + + var guides = Guides(_workspace, Repo("repo-a")); + + guides.Appendix.ShouldBe($"\n\n--- project-doc (repo-a/AGENTS.md) ---\n\n{new string('x', CodexRepositoryGuides.MaxProjectDocBytes)}", customMessage: "the run is given the doc's first bytes and nothing past them"); + guides.Notices.ShouldBe(cut ? new[] { $"Left all but the first {CodexRepositoryGuides.MaxProjectDocBytes} bytes of the AGENTS.md of 'repo-a' out of this run: Codex reads no more of a project doc." } : Array.Empty()); + } + + [Fact] + public void A_doc_cut_through_a_character_is_handed_over_as_codex_hands_it_over() + { + // Codex cuts the bytes and decodes them lossily, so a character split at the cap becomes U+FFFD. + if (OperatingSystem.IsWindows()) return; + + var head = new string('x', CodexRepositoryGuides.MaxProjectDocBytes - 1); + + File.WriteAllBytes(Path.Combine(_tree.Directory("ws/repo-a"), "AGENTS.md"), Encoding.UTF8.GetBytes(head + "é")); + + Guides(_workspace, Repo("repo-a")).Appendix.ShouldBe($"\n\n--- project-doc (repo-a/AGENTS.md) ---\n\n{head}�"); + } + + [Theory] + [InlineData("a doc linked outside by its absolute path", "AGENTS.md")] + [InlineData("a doc linked outside by a relative target", "AGENTS.md")] + [InlineData("a link whose .. climbs out of a linked directory", "AGENTS.md")] + [InlineData("an override linked outside beside a doc inside", "AGENTS.override.md")] + [InlineData("a repository directory an earlier round replaced with a link outside", "AGENTS.md")] + public void A_doc_that_resolves_outside_the_workspace_is_left_out_and_said(string shape, string name) + { + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/repo-a/AGENTS.md", "A"); + _tree.Directory("ws/repo-b"); + + switch (shape) + { + case "a doc linked outside by its absolute path": _tree.Link("ws/repo-b/AGENTS.md", _secret); break; + case "a doc linked outside by a relative target": _tree.Link("ws/repo-b/AGENTS.md", "../../outside/secret.md"); break; + case "a link whose .. climbs out of a linked directory": + _tree.Directory("outside/deep/er"); + _tree.Link("ws/repo-b/deep", Path.Combine(_tree.Root, "outside", "deep", "er")); + _tree.Link("ws/repo-b/AGENTS.md", "deep/../../secret.md"); + break; + case "an override linked outside beside a doc inside": + _tree.File("ws/repo-b/AGENTS.md", "B"); + _tree.Link("ws/repo-b/AGENTS.override.md", _secret); + break; + default: + Directory.Delete(Path.Combine(_workspace, "repo-b")); + _tree.File("outside/repo/AGENTS.md", "OUTSIDE"); + _tree.Link("ws/repo-b", Path.Combine(_tree.Root, "outside", "repo")); + break; + } + + var guides = Guides(_workspace, Repo("repo-a"), Repo("repo-b")); + + guides.Appendix.ShouldBe("\n\n--- project-doc (repo-a/AGENTS.md) ---\n\nA", customMessage: $"{shape}: only the repository whose doc stays inside is given"); + guides.Appendix.ShouldNotContain("OUTSIDE"); + guides.Notices.ShouldBe(new[] { $"Left the {name} of 'repo-b' out of this run: it resolves outside the workspace." }, shape); + } + + [Theory] + [InlineData("a doc linked to a file of another repository in the workspace", "SIBLING")] + [InlineData("a doc linked to the README beside it", "README")] + [InlineData("a doc linked to nothing", null)] + [InlineData("a doc linked to a directory", null)] + public void A_doc_linked_inside_the_workspace_is_given_and_one_that_reaches_no_file_gives_nothing(string shape, string? text) + { + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/repo-b/notes.md", "SIBLING"); + _tree.File("ws/repo-a/README.md", "README"); + + switch (shape) + { + case "a doc linked to a file of another repository in the workspace": _tree.Link("ws/repo-a/AGENTS.md", "../repo-b/notes.md"); break; + case "a doc linked to the README beside it": _tree.Link("ws/repo-a/AGENTS.md", "README.md"); break; + case "a doc linked to nothing": _tree.Link("ws/repo-a/AGENTS.md", "missing.md"); break; + default: _tree.Link("ws/repo-a/AGENTS.md", "../repo-b"); break; + } + + var guides = Guides(_workspace, Repo("repo-a")); + + guides.Appendix.ShouldBe(text is null ? "" : $"\n\n--- project-doc (repo-a/AGENTS.md) ---\n\n{text}", customMessage: shape); + guides.Notices.ShouldBeEmpty(shape); + } + + [Fact] + public async Task A_fifo_doc_is_never_opened_for_reading() + { + // A blocking open of a FIFO waits for a writer that never comes, and the build would wait with it. + if (OperatingSystem.IsWindows()) return; + + await MakeFifoAsync(Path.Combine(_tree.Directory("ws/repo-a"), "AGENTS.md")); + + var guides = await System.Threading.Tasks.Task.Run(() => Guides(_workspace, Repo("repo-a"))).WaitAsync(TimeSpan.FromSeconds(1)); + + guides.Appendix.ShouldBeEmpty(); + guides.Notices.ShouldBeEmpty(); + } + + [Theory] + [InlineData("repo a", "repo?a")] + [InlineData("repo\na", "repo?a")] + [InlineData("repo)a", "repo?a")] + public void A_repository_whose_path_the_separator_cannot_carry_is_left_out_and_said(string directory, string printed) + { + // The separator repeats the repository's path below the cwd, which the workspace's author chose: a newline or a + // bracket in it could forge or end a separator line. + if (OperatingSystem.IsWindows()) return; + + _tree.File($"ws/{directory}/AGENTS.md", "FORGED"); + + var guides = Guides(_workspace, Repo(directory)); + + guides.Appendix.ShouldBeEmpty(); + guides.Notices.ShouldBe(new[] { $"Left the AGENTS.md of '{printed}' out of this run: its path holds a character other than a letter, a digit or one of . _ @ + - /." }); + } + + [Fact] + public void A_repository_with_no_doc_says_nothing_whatever_its_path_holds() + { + if (OperatingSystem.IsWindows()) return; + + _tree.Directory("ws/repo a"); + + var guides = Guides(_workspace, Repo("repo a")); + + guides.Appendix.ShouldBeEmpty(); + guides.Notices.ShouldBeEmpty(); + } + + [Fact] + public void A_repository_named_twice_or_spelled_two_ways_is_given_once() + { + if (OperatingSystem.IsWindows()) return; + + _tree.File("ws/repo-a/AGENTS.md", "A"); + + Guides(_workspace, Repo("repo-a"), Repo("repo-a") + "/", Path.Combine(_workspace, "repo-b", "..", "repo-a")).Appendix.ShouldBe("\n\n--- project-doc (repo-a/AGENTS.md) ---\n\nA"); + } + + private string Repo(string relative) => Path.Combine(_workspace, relative); + + private static CodexRepositoryGuides.Plan Guides(string workspace, params string[] repositories) => CodexRepositoryGuides.For(Task(workspace, repositories)); + + private static AgentTask Task(string workspace, IReadOnlyList? repositories) => new() + { + Goal = "Fix the failing billing tests", + Harness = CodexHarness.HarnessKind, + WorkspaceDirectory = workspace, + WorkspaceRepositoryDirectories = repositories, + }; + + private static async Task MakeFifoAsync(string path) + { + using var mkfifo = Process.Start("mkfifo", path)!; + await mkfifo.WaitForExitAsync(); + mkfifo.ExitCode.ShouldBe(0, $"fixture check: mkfifo {path}"); + } +}