Repository navigation
Pin the syberlabs CLI surface Syber's SyberWork builder calls (contract 2's receiving end today) #175
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Pin the syberlabs CLI surface Syber's SyberWork builder calls (contract 2's receiving end today) #175
Changes from all commits
c0f292b
56c8c9e
7d08a7f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,249 @@ | ||
| """Contract 2's receiving end as it runs today: the ``syberlabs`` steps Syber's SyberWork builder calls. | ||
|
|
||
| SyberLabs/syber ``lib/factory/syberwork-builder.mjs`` (read at 5a4667f, 2026-11-04) runs each job as a Build | ||
| Thread by starting this CLI in the job's worktree, with no ``--repo`` or ``--home``: | ||
|
|
||
| init | ||
| start <objective> --class R | ||
| propose --claude-code --bare --lane provider_3p --max-calls N --name syber-line0 --revision 1 | ||
| status --json reads thread, candidates[].id, candidates[].evaluation == "passed", candidates[].in_scope | ||
| check <id> --json reads acceptable, reason, checks[].name, checks[].state | ||
| diff <id> counts the unified diff's +/- lines when several candidates passed | ||
| accept <id> | ||
| status --json reads target (40 hex) and accepted == <id> | ||
| export writes stdout to a file as the thread's journal | ||
|
|
||
| These tests pin exactly what that builder relies on, and no more of the output, so a change here that would | ||
| break Syber fails in this repository first. They are not a protocol: the S0 Agent protocol (``spec/v1/``) is the | ||
| seam's specification, and no runtime implements it yet. | ||
| """ | ||
|
|
||
| import contextlib | ||
| import io | ||
| import json | ||
| import os | ||
| import re | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
| import unittest | ||
| from pathlib import Path | ||
| from unittest import mock | ||
|
|
||
| from syberlabs import Rejected | ||
| from syberlabs.cli import main as cli | ||
| from tests.test_build_thread import GOOD, REPORT, TEST_REPORT, RepoCase, git | ||
|
|
||
| CHECKOUT = Path(__file__).resolve().parents[1] | ||
|
|
||
| SYBER_OBJECTIVE = "Work order 1: add a CSV export for report rows" | ||
| SYBER_PROPOSE = ["propose", "--claude-code", "--bare", "--lane", "provider_3p", "--max-calls", "2", | ||
| "--name", "syber-line0", "--revision", "1"] | ||
| BROKEN = {**GOOD, "src/report.py": "def rows():\n return []\n"} # the existing report test fails | ||
|
|
||
|
|
||
| class SyberBuilderSeam(RepoCase): | ||
| def run_cli(self, *args): | ||
| out, err = io.StringIO(), io.StringIO() | ||
| with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): | ||
| code = cli(["--repo", str(self.root), "--home", str(self.root / ".syberlabs"), | ||
| "--actor", "syber@example.test", *args]) | ||
| return code, out.getvalue(), err.getvalue() | ||
|
|
||
| def json_of(self, *args): | ||
| code, out, err = self.run_cli(*args) | ||
| try: | ||
| return code, json.loads(out) | ||
| except ValueError: | ||
| self.fail(f"`syberlabs {' '.join(args)}` did not print JSON (exit {code}): {out[-400:]} {err[-400:]}") | ||
|
|
||
| def propose(self, changes): | ||
| for path, text in changes.items(): | ||
| (self.root / path).write_text(text) | ||
| code, out, err = self.run_cli("propose", "--from-worktree", "--name", "syber-line0", "--revision", "1") | ||
| self.assertEqual(code, 0, out + err) | ||
|
|
||
| def test_a_passing_candidate_reads_back_through_the_fields_syber_uses(self): | ||
| self.assertEqual(self.run_cli("init")[0], 0) | ||
| code, out, err = self.run_cli("start", SYBER_OBJECTIVE, "--class", "R") | ||
| self.assertEqual(code, 0, out + err) | ||
| self.propose(GOOD) | ||
|
|
||
| code, status = self.json_of("status", "--json") | ||
| self.assertEqual(code, 0) | ||
| thread = status["thread"] | ||
| self.assertIsInstance(thread, str) | ||
| self.assertTrue(thread) | ||
| [candidate] = status["candidates"] | ||
| self.assertEqual(candidate["id"], "c1") | ||
| self.assertIs(candidate["in_scope"], True) | ||
| self.assertNotEqual(candidate["evaluation"], "passed", "not passed before the host checked it") | ||
|
|
||
| code, verdict = self.json_of("check", "c1", "--json") | ||
| self.assertEqual(code, 0) | ||
| self.assertIs(verdict["acceptable"], True) | ||
| self.assertIsInstance(verdict["reason"], str) | ||
| self.assertTrue(verdict["reason"]) | ||
| self.assertIn(("tests", "passed"), [(c["name"], c["state"]) for c in verdict["checks"]]) | ||
|
|
||
| code, status = self.json_of("status", "--json") | ||
| self.assertEqual([(c["id"], c["evaluation"], c["in_scope"]) for c in status["candidates"]], | ||
| [("c1", "passed", True)], "Syber's passedCandidates filter: in_scope and evaluation passed") | ||
|
|
||
| code, diff, _ = self.run_cli("diff", "c1") | ||
| self.assertEqual(code, 0) | ||
| self.assertIn("+++ b/src/export.py", diff) | ||
| self.assertTrue(any(line.startswith("+") and not line.startswith("+++") for line in diff.splitlines()), | ||
| "Syber counts the added and removed lines below the headers") | ||
|
|
||
| code, out, err = self.run_cli("accept", "c1") | ||
| self.assertEqual(code, 0, out + err) | ||
| code, after = self.json_of("status", "--json") | ||
| self.assertEqual(after["accepted"], "c1") | ||
| self.assertRegex(after["target"], r"^[0-9a-f]{40}$") | ||
| self.assertEqual(after["target"], git(self.root, "rev-parse", after["target_ref"]), | ||
| "the commit Syber lays into the job's worktree is the thread branch's tip") | ||
|
|
||
| code, record = self.json_of("export") | ||
| self.assertEqual(code, 0) | ||
| self.assertIs(record["chain_valid"], True) | ||
| self.assertEqual(record["status"]["accepted"], "c1") | ||
|
|
||
| def test_a_failing_candidate_is_not_passed_and_its_check_says_why(self): | ||
| self.assertEqual(self.run_cli("init")[0], 0) | ||
| self.assertEqual(self.run_cli("start", SYBER_OBJECTIVE, "--class", "R")[0], 0) | ||
| self.propose(BROKEN) | ||
|
|
||
| code, verdict = self.json_of("check", "c1", "--json") | ||
| self.assertEqual(code, 1) | ||
| self.assertIs(verdict["acceptable"], False) | ||
| self.assertIsInstance(verdict["reason"], str) | ||
| self.assertTrue(verdict["reason"], "Syber reports the refusal as `<id>: <reason>`") | ||
|
|
||
| code, status = self.json_of("status", "--json") | ||
| [candidate] = status["candidates"] | ||
| self.assertNotEqual(candidate["evaluation"], "passed", "Syber would not take it") | ||
| self.assertIsNone(status["accepted"]) | ||
|
|
||
| def test_syber_propose_argv_builds_the_claude_code_provider_it_asks_for(self): | ||
| """Syber's propose line must parse and build a provider under today's refusals (#162 made ``--bare`` refuse | ||
| unscreened lookups): routed through a key proxy (``provider_3p``), bare, Syber's name and revision, no | ||
| GitHits. No Claude Code runs: the executable answers ``--version`` only, and the search is stopped before | ||
| any call.""" | ||
| self.assertEqual(self.run_cli("init")[0], 0) | ||
| self.assertEqual(self.run_cli("start", SYBER_OBJECTIVE, "--class", "R")[0], 0) | ||
| seen = [] | ||
|
|
||
| def stop(thread, provider, **kwargs): | ||
| seen.append(provider) | ||
| raise Rejected("seam_stop", "stopped before any call") | ||
|
|
||
| version_only = [sys.executable, "-c", "print('2.1.312 (Claude Code)')"] | ||
| with mock.patch("syberlabs.claude_code.claude_executable", return_value=version_only), \ | ||
| mock.patch("syberlabs.build.Thread.propose", stop): | ||
| code, _, err = self.run_cli(*SYBER_PROPOSE) | ||
| self.assertEqual(code, 2) | ||
| self.assertIn("seam_stop", err, "the provider was built; nothing refused Syber's argv") | ||
| [provider] = seen | ||
| self.assertEqual((provider.bare, provider.lane, provider.cost_basis), (True, "provider_3p", "estimated")) | ||
| self.assertEqual((provider.name, provider.revision, provider.max_calls), ("syber-line0", "1", 2)) | ||
| self.assertIs(provider.githits, False) | ||
| self.assertEqual(provider.mcp_config, None) | ||
| self.assertFalse(any(re.match(r"^(Bash|PowerShell|WebSearch|WebFetch)$", t) for t in provider.allowed_tools), | ||
| "Syber's builder relies on the default tools: no shell, no web lookups") | ||
|
Comment on lines
+152
to
+153
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The assertion only rejects exact tool names. An entry such as Check scoped entries too, or assert the expected tool list explicitly. Otherwise, adding restricted shell access could leave this no-shell test green. Prompt To Fix With AIThis is a comment left during a code review.
Path: tests/test_syber_builder_seam.py
Line: 146-147
Comment:
**Scoped shell tools escape detection**
The assertion only rejects exact tool names. An entry such as `Bash(git *)` passes, but `built_in_tools()` turns it into `Bash` when building the command. The constructor permits shell tools on this path because neither GitHits nor an MCP config is enabled.
Check scoped entries too, or assert the expected tool list explicitly. Otherwise, adding restricted shell access could leave this no-shell test green.
---
For each issue above, determine whether it is valid and should be fixed. If so, fix it directly. |
||
|
|
||
|
|
||
| class SyberBuilderDefaults(unittest.TestCase): | ||
| """The same steps as Syber runs them: a subprocess in the job worktree with no ``--repo``, ``--home`` or | ||
| ``--actor``, and the environment Syber builds (``syberwork-builder.mjs:350-364``: ``PATH``, a job ``HOME``, | ||
| ``LANG``, ``TERM``, ``CI``, ``PYTHONDONTWRITEBYTECODE``, ``PYTHONPATH`` at SyberWork). The repository discovery, | ||
| the default home and the default actor are then this repository's defaults, not a test's options. No git identity | ||
| is inherited: ``HOME``, ``USERPROFILE`` and the XDG directories are temporary, so no global config exists. The | ||
| test's own git calls also get an empty ``GIT_CONFIG_GLOBAL`` and ``GIT_CONFIG_NOSYSTEM``, but ``syberlabs`` drops | ||
| every ``GIT_*`` variable before it runs git (``gitspace.Repo._env``), so its view still includes the host's system | ||
| config, as a Syber host's does; the precondition below checks that view has no identity.""" | ||
|
|
||
| def setUp(self): | ||
| folder = tempfile.TemporaryDirectory() | ||
| self.addCleanup(folder.cleanup) | ||
| base = Path(folder.name) | ||
| self.worktree, home = base / "job", base / "home" | ||
| (self.worktree / "src").mkdir(parents=True) | ||
| (self.worktree / "tests").mkdir() | ||
| home.mkdir() | ||
| (self.worktree / "src" / "report.py").write_text(REPORT) | ||
| (self.worktree / "tests" / "test_report.py").write_text(TEST_REPORT) | ||
| empty = base / "gitconfig" | ||
| empty.write_text("") | ||
| self.env = {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), "HOME": str(home), "LANG": "C.UTF-8", | ||
| "TERM": "dumb", "CI": "1", "PYTHONDONTWRITEBYTECODE": "1", "PYTHONPATH": str(CHECKOUT), | ||
| "USERPROFILE": str(home), "XDG_CONFIG_HOME": str(home / ".config"), | ||
| "XDG_CACHE_HOME": str(home / ".cache"), "XDG_DATA_HOME": str(home / ".local" / "share"), | ||
| "GIT_CONFIG_GLOBAL": str(empty), "GIT_CONFIG_NOSYSTEM": "1"} | ||
| if os.name == "nt": # a Windows interpreter cannot start without these; none carries an identity | ||
| self.env.update({name: os.environ[name] for name in ("SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT") | ||
| if name in os.environ}) | ||
|
|
||
| def step(self, *args, cmd=None): | ||
| """One step, as ``runStep`` starts it (``syberwork-builder.mjs:370``): argv, ``cwd`` the worktree.""" | ||
| done = subprocess.run([*(cmd or [sys.executable, "-m", "syberlabs"]), *args], cwd=self.worktree, env=self.env, | ||
| stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=240) | ||
| return done.returncode, done.stdout, done.stderr | ||
|
|
||
| def json_step(self, *args): | ||
| code, out, err = self.step(*args) | ||
| try: | ||
| return code, json.loads(out) | ||
| except ValueError: | ||
| self.fail(f"`syberlabs {' '.join(args)}` did not print JSON (exit {code}): {out[-400:]} {err[-400:]}") | ||
|
|
||
| def test_the_steps_run_on_the_defaults_from_the_job_worktree(self): | ||
| # The disposable repository, exactly as Syber makes it (syberwork-builder.mjs:389-393). | ||
| for args in (["-c", "init.defaultBranch=job", "init", "-q"], ["add", "-A"], | ||
| ["-c", "user.name=Syber", "-c", "user.email=syber@syberlabs.io", "commit", "-q", "--allow-empty", | ||
| "-m", "syber: the worktree as this round starts"]): | ||
| code, out, err = self.step(*args, cmd=["git"]) | ||
| self.assertEqual(code, 0, out + err) | ||
| seen_by_syberlabs = {k: v for k, v in self.env.items() if not k.startswith("GIT_")} # gitspace.Repo._env | ||
| done = subprocess.run(["git", "config", "user.email"], cwd=self.worktree, env=seen_by_syberlabs, | ||
| stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=60) | ||
| self.assertEqual((done.returncode, done.stdout.strip()), (1, ""), | ||
| "no git identity in syberlabs' view (a host system gitconfig with user.email would set one)") | ||
|
Comment on lines
+210
to
+211
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
If the host has a system-level Make the fixture independent of system Git settings while keeping the CLI defaults under test. Prompt To Fix With AIThis is a comment left during a code review.
Path: tests/test_syber_builder_seam.py
Line: 210-211
Comment:
**Machine settings break the test**
If the host has a system-level `user.email`, this assertion fails before any CLI step runs, even though the CLI behaves correctly. `Repo._env()` removes `GIT_CONFIG_NOSYSTEM` and `GIT_CONFIG_GLOBAL`, so the temporary home does not isolate that setting.
Make the fixture independent of system Git settings while keeping the CLI defaults under test.
---
For each issue above, determine whether it is valid and should be fixed. If so, fix it directly. |
||
|
|
||
| self.assertEqual(self.step("init")[0], 0, "syberwork-builder.mjs:399") | ||
| self.assertTrue((self.worktree / ".syberlabs").is_dir(), "the default home is the worktree's .syberlabs") | ||
| code, out, err = self.step("start", SYBER_OBJECTIVE, "--class", "R") # syberwork-builder.mjs:402 | ||
| self.assertEqual(code, 0, out + err) | ||
| for path, text in GOOD.items(): # the proposer's work; Syber's propose line is pinned in SyberBuilderSeam | ||
| (self.worktree / path).write_text(text) | ||
| code, out, err = self.step("propose", "--from-worktree", "--name", "syber-line0", "--revision", "1") | ||
| self.assertEqual(code, 0, out + err) | ||
|
|
||
| code, status = self.json_step("status", "--json") # syberwork-builder.mjs:411 | ||
| self.assertEqual(code, 0) | ||
| thread = status["thread"] | ||
| self.assertEqual([(c["id"], c["in_scope"]) for c in status["candidates"]], [("c1", True)]) | ||
| code, verdict = self.json_step("check", "c1", "--json") # syberwork-builder.mjs:435 | ||
| self.assertEqual(code, 0) | ||
| self.assertIs(verdict["acceptable"], True) | ||
| code, out, err = self.step("accept", "c1") # syberwork-builder.mjs:455 | ||
| self.assertEqual(code, 0, out + err) | ||
| code, after = self.json_step("status", "--json") # syberwork-builder.mjs:460 | ||
| self.assertEqual((after["thread"], after["accepted"]), (thread, "c1")) | ||
| tip = self.step("rev-parse", after["target_ref"], cmd=["git"])[1].strip() | ||
| self.assertRegex(after["target"], r"^[0-9a-f]{40}$") | ||
| self.assertEqual(after["target"], tip, "the thread branch is in the worktree's own repository") | ||
|
|
||
| code, record = self.json_step("export") # syberwork-builder.mjs:417 | ||
| self.assertEqual(code, 0) | ||
| self.assertIs(record["chain_valid"], True) | ||
| self.assertEqual(record["status"]["thread"], thread) | ||
| [created] = [e for e in record["events"] if e["kind"] == "case_created"] | ||
| self.assertEqual(created["body"]["actor"], "developer", | ||
| "no git user.email: the default actor (Kit.local), not the commit's inline identity") | ||
| self.assertEqual(sorted(p.name for p in self.worktree.parent.iterdir() if p.name.startswith(".syberlabs")), [], | ||
| "nothing was written beside the worktree") | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| unittest.main() | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If the default tool list later includes a scoped entry such as
Bash(git log *), this exact-name regex will not recognize it, so the seam test can pass despite allowing a shell tool. Check the base tool names (or otherwise cover scoped forms) so the test catches that access widening.AGENTS.md reference: AGENTS.md:L212-L215
Useful? React with 👍 / 👎.