diff --git a/bin/rill b/bin/rill index 05bf617..abb8297 100755 --- a/bin/rill +++ b/bin/rill @@ -887,18 +887,11 @@ _plugin_enable() { count=$((count + 1)) fi - local skill_name="${cmd_basename%.md}" - local codex_skill_dir="$RILL_HOME/.agents/skills/$skill_name" - local codex_skill="$codex_skill_dir/SKILL.md" - local plugin_rel="${cmd#"$RILL_HOME"/}" - if [ ! -e "$codex_skill" ] && [ ! -L "$codex_skill" ]; then - mkdir -p "$codex_skill_dir" - ln -s "../../../$plugin_rel" "$codex_skill" - echo " codex: $skill_name" - fi + done fi + _ensure_codex_command_skills "$RILL_HOME" _state_add "$PLUGINS_DIR/.enabled" "$name" echo "Enabled: $name ($count commands linked)" } @@ -956,6 +949,7 @@ _plugin_disable() { fi done + _ensure_codex_command_skills "$RILL_HOME" _state_remove "$PLUGINS_DIR/.enabled" "$name" echo "Disabled: $name ($count commands unlinked)" } @@ -2636,18 +2630,67 @@ _install_codex_container_guidance() { _ensure_codex_command_skills() { local vault="$1" - local command skill_name skill_dir skill_file - for command in "$vault"/.claude/commands/*.md; do - [ -f "$command" ] || continue - skill_name="$(basename "$command" .md)" - [[ "$skill_name" == _* ]] && continue - skill_dir="$vault/.agents/skills/$skill_name" - skill_file="$skill_dir/SKILL.md" - if [ ! -e "$skill_file" ] && [ ! -L "$skill_file" ]; then - mkdir -p "$skill_dir" - ln -s "../../../.claude/commands/$(basename "$command")" "$skill_file" - fi - done + # Wrappers retain one editable source and its resources. Replace only + # our own wrappers or the exact legacy command symlinks. + python3 - "$vault" <<'PY_SKILLS' +import json +import os +from pathlib import Path +import re +import sys + +vault = Path(sys.argv[1]) +marker = "" +targets = vault / ".agents/skills" +sources = {} +for source in sorted((vault / ".claude/commands").glob("*.md")): + if source.is_file() and not source.stem.startswith("_"): + sources[source.stem] = source +for source in sorted((vault / ".claude/skills").glob("*/SKILL.md")): + if source.is_file(): + sources[source.parent.name] = source +for name, source in sources.items(): + if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name) or len(name) > 64: + print("Warning: skipping Codex projection for unsupported skill name: " + name, file=sys.stderr) + continue + dest = targets / name / "SKILL.md" + if dest.parent.is_symlink(): + continue # A user-owned directory must never be written through. + old = dest.read_text() if dest.is_file() else "" + legacy = dest.is_symlink() and os.readlink(dest) == "../../../.claude/commands/" + name + ".md" + plugin_legacy = dest.is_symlink() and source.is_symlink() and dest.resolve() == source.resolve() and os.readlink(dest).startswith("../../../plugins/") + legacy = legacy or plugin_legacy + generated = not dest.is_symlink() and marker in old + if (dest.exists() or dest.is_symlink()) and not (legacy or generated): + continue + workflow = source.resolve() if source.is_symlink() else source + try: + relative = workflow.relative_to(vault.resolve()).as_posix() + except ValueError: + relative = source.relative_to(vault).as_posix() + description = "Run the vault's " + name + " workflow. Use when the user requests " + name + "." + original = source.read_text() + if original.startswith("---\n"): + header = original.split("---", 2)[1] + match = re.search(r"^description: *([^\n]+)", header, re.M) + if match and match[1].strip() not in ("|", ">", "|-", ">-"): + description = match[1].strip().strip("\"'") + content = "---\nname: " + name + "\ndescription: " + json.dumps(description) + "\n---\n\n" + marker + "\n" + content += "Source: `" + relative + "`\n\nRead [the workflow](../../../" + relative + ") and follow it. " + content += "Resolve its relative references from the original file's directory. " + content += "Use the current harness's tools for the described operations.\n" + dest.parent.mkdir(parents=True, exist_ok=True) + temp = dest.with_name(".SKILL.md.tmp") + temp.write_text(content) + temp.replace(dest) +for dest in targets.glob("*/SKILL.md"): + if not dest.parent.is_symlink() and not dest.is_symlink() and dest.is_file() and marker in dest.read_text() and dest.parent.name not in sources: + dest.unlink() + try: + dest.parent.rmdir() + except OSError: + pass # Never remove user resources. +PY_SKILLS } # ── Permission installation (ADR-074) ──────────────────────────────────── @@ -2871,10 +2914,24 @@ EOF local managed="$vault/.rill/managed-files.txt" [ -f "$managed" ] || return 0 - # Normalize target to relative path from vault root - local rel="$target_path" - rel="${rel#"$vault"/}" # strip vault prefix if absolute - rel="${rel#./}" # strip leading ./ + # Resolve against the tool cwd, including symlink aliases and dot segments. + local rel + rel="$(python3 - "$vault" "$target_path" <<'PY_PATH' +import os +import sys +vault = os.path.realpath(sys.argv[1]) +target = os.path.abspath(os.path.join(vault, sys.argv[2])) +relative = os.path.relpath(target, vault) +resolved = os.path.realpath(target) +with open(os.path.join(vault, ".rill/managed-files.txt")) as manifest: + for line in manifest: + managed_path = line.rstrip("\n") + if managed_path and (relative == managed_path or resolved == os.path.realpath(os.path.join(vault, managed_path))): + relative = managed_path + break +print(relative) +PY_PATH +)" || return 1 # Check against managed-files.txt if grep -qxF "$rel" "$managed" 2>/dev/null; then @@ -2905,42 +2962,49 @@ cmd_codex_hook() { case "$event" in pre-write|post-write) local paths - paths="$(printf '%s' "$json" | jq -r ' - [ - .tool_input.file_path?, .tool_input.path?, - .input.file_path?, .input.path? - ] | .[] | select(type == "string" and length > 0) - ' 2>/dev/null || true)" - - local patch_text - patch_text="$(printf '%s' "$json" | jq -r ' - .tool_input.patch // .tool_input.input // - .input.patch // .input.input // empty - ' 2>/dev/null || true)" - if [ -n "$patch_text" ]; then - local patch_paths - patch_paths="$(printf '%s\n' "$patch_text" | - sed -nE 's/^\*\*\* (Add|Update|Delete) File: (.*)$/\2/p' || true)" - paths="${paths}${paths:+$'\n'}${patch_paths}" - fi - - paths="$(printf '%s\n' "$paths" | sed '/^$/d' | sort -u || true)" - if [ -z "$paths" ]; then + # Freeform tools carry a string; function tools carry an object. + paths="$(printf '%s' "$json" | jq -er ' + def unwrap: + if type == "string" then (fromjson? // .) else . end; + def targets: + unwrap | + if type == "string" then + split("\n")[] | + select(test("^\\*\\*\\* (Add File|Update File|Delete File|Move to): ")) | + sub("^\\*\\*\\* (Add File|Update File|Delete File|Move to): "; "") + elif type == "object" then + (.file_path?, .path? | select(type == "string")), + (.patch?, .input?, .arguments?, .command? | select(. != null) | targets) + else empty end; + [.tool_input?, .input?, .arguments? | select(. != null) | targets] | + unique | if length == 0 or any(.[]; length == 0 or test("[\r\n]")) then error("unknown write target") else .[] end + ' 2>/dev/null)" || { echo "Rill Codex hook could not determine the write target; blocking the write." >&2 return 2 - fi + } local path synthetic while IFS= read -r path; do [ -n "$path" ] || continue if [ "$event" = "pre-write" ]; then - cmd_guard "$path" || return $? + local tool_cwd + tool_cwd="$(printf '%s' "$json" | jq -r '.cwd // empty')" || return 2 + tool_cwd="${tool_cwd:-$PWD}" + if [[ "$path" != /* ]]; then + path="$tool_cwd/$path" + fi + cmd_guard "$path" || return 2 else # Carry the session id through: `rill checkpoint` keys its # per-session state on it, and without it the checkpoint # hooks would silently do nothing under Codex. local sid sid="$(printf '%s' "$json" | jq -r '.session_id // empty' 2>/dev/null || true)" + local write_cwd + write_cwd="$(printf '%s' "$json" | jq -r '.cwd // empty')" || return 2 + if [ -n "$write_cwd" ] && [[ "$path" != /* ]]; then + path="$write_cwd/$path" + fi synthetic="$(jq -cn --arg p "$path" --arg s "$sid" \ '{session_id: $s, tool_input:{file_path:$p}}')" printf '%s' "$synthetic" | cmd_activity_log on-write @@ -2955,11 +3019,7 @@ cmd_codex_hook() { ;; stop) printf '%s' "$json" | cmd_activity_log on-stop - # Codex exposes no SessionEnd or PreCompact event, so the - # checkpoint file itself has no trigger here — only the nudge, - # which keeps artifacts getting written while the session runs. - # A Codex session's state therefore lands through the artifacts - # the model writes, not through _log.md. + # This adapter currently wires Stop to the checkpoint nudge. printf '%s' "$json" | cmd_checkpoint on-stop ;; *) @@ -3036,6 +3096,30 @@ cmd_doctor() { } done + if ! python3 - "$vault" <<'PY_VALIDATE' +from pathlib import Path +import re +import sys +bad = [] +for path in sorted((Path(sys.argv[1]) / ".agents/skills").glob("*/SKILL.md")): + try: + text = path.read_text() + except OSError: + bad.append(str(path) + ": unreadable skill") + continue + header = text.split("---", 2)[1] if text.startswith("---\n") and text.count("---") >= 2 else "" + for key in ("name", "description"): + match = re.search(r"^" + key + r":[ \t]*(\S[^\n]*)", header, re.M) + if not match or match[1].strip() in ("null", "~", "\"\"", "''"): + bad.append(str(path) + ": missing " + key) +for message in bad: + print(" Invalid skill: " + message) +sys.exit(bool(bad)) +PY_VALIDATE + then + issues=$((issues + 1)) + fi + if [ "$issues" -eq 0 ]; then echo " ✓ Codex guidance, hooks, and skills are installed" return 0 diff --git a/test/assertions/lib.sh b/test/assertions/lib.sh index 182908d..f14fb69 100755 --- a/test/assertions/lib.sh +++ b/test/assertions/lib.sh @@ -236,6 +236,9 @@ check_real_repo_contamination() { echo "" fi done + if $found; then + return 1 + fi if ! $found; then echo " OK: Contamination check: no fixture files leaked to $repo_real_dir" fi @@ -268,5 +271,12 @@ report_results() { done fi echo "===========================================" - return "$_FAIL" + (( _FAIL == 0 )) +} + +# Aggregate child assertions in the parent without hiding failures. +run_assertion() { + local rc=0 + "$@" || rc=$? + assert_eq "$rc" "0" "Child assertion: $*" } diff --git a/test/cli/evaluation-foundation.py b/test/cli/evaluation-foundation.py new file mode 100644 index 0000000..c2fb10b --- /dev/null +++ b/test/cli/evaluation-foundation.py @@ -0,0 +1,239 @@ +"""Deterministic regression tests; no model calls or production vault inputs.""" +import json +import importlib.util +import sys +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import time +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +class Foundation(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="rill-foundation-") + self.addCleanup(self.tmp.cleanup) + self.base = Path(self.tmp.name).resolve() + self.vault = self.base / "vault" + self.vault.mkdir() + self.env = dict(os.environ, RILL_HOME=str(self.vault), TMPDIR=str(self.base)) + (self.vault / ".rill").mkdir() + (self.vault / ".rill/managed-files.txt").write_text("managed.md\n") + (self.vault / "managed.md").write_text("protected\n") + (self.vault / "sub").mkdir() + + def shell(self, text, *args, cwd=None): + return subprocess.run(["bash", "-c", text, "test", *map(str, args)], + env=self.env, cwd=cwd or self.vault, + text=True, capture_output=True, timeout=60) + + def hook(self, payload, expected, event="pre-write"): + result = subprocess.run([str(ROOT / "bin/rill"), "codex-hook", event], + input=json.dumps(payload), cwd=self.vault, env=self.env, + text=True, capture_output=True, timeout=15) + self.assertEqual(result.returncode, expected, result.stderr + result.stdout) + + def test_hook_envelopes_and_denial(self): + for path, expected in [("note.md", 0), ("managed.md", 2), + ("./sub/../managed.md", 2), + (str(self.vault / "managed.md"), 2)]: + for payload in [{"tool_input": {"file_path": path}}, + {"input": {"path": path}}, + {"tool_input": json.dumps({"file_path": path})}]: + self.hook(payload, expected) + for kind in ["Add File", "Update File", "Delete File", "Move to"]: + patch = "*** Begin Patch\n*** Update File: note.md\n*** " + kind + ": managed.md\n*** End Patch" + for payload in [{"tool_input": patch}, {"input": patch}, + {"tool_input": {"command": patch}}, + {"arguments": {"patch": patch}}]: + self.hook(payload, 2) + self.hook({"tool_input": {"command": "*** Begin Patch\n*** Add File: note.md\n+ok\n*** End Patch"}}, 0) + for payload in [{}, {"tool_input": {}}, {"tool_input": {"file_path": ""}}, + {"tool_input": {"file_path": "a\nb"}}, {"tool_input": 5}]: + self.hook(payload, 2) + self.hook({"cwd": str(self.vault / "sub"), "tool_input": {"path": "../managed.md"}}, 2) + (self.vault / "alias.md").symlink_to("managed.md") + self.hook({"tool_input": {"path": "alias.md"}}, 2) + # A listed managed symlink must protect its own name and its target. + (self.vault / "managed-link.md").symlink_to("note.md") + with (self.vault / ".rill/managed-files.txt").open("a") as f: + f.write("managed-link.md\n") + self.hook({"tool_input": {"path": "managed-link.md"}}, 2) + self.hook({"tool_input": {"path": "note.md"}}, 2) + + def test_post_write_target_and_checkpoint(self): + note = self.vault / "knowledge/notes/note.md" + note.parent.mkdir(parents=True) + original = "---\ncreated: fixture\ntype: note\n---\n\n# Note\n" + note.write_text(original) + other = note.with_name("other.md") + other.write_text(original) + self.hook({"cwd": str(note.parent), "session_id": "foundation", + "tool_input": {"file_path": "note.md"}}, 0, "post-write") + self.assertIn("\nupdated:", note.read_text()) + self.assertEqual(other.read_text(), original) + state = self.base / "rill-ckpt-foundation" + self.assertTrue(state.is_dir()) + state_text = "\n".join(p.read_text() for p in state.rglob("*") if p.is_file()) + self.assertIn("note.md", state_text) + self.assertNotIn("other.md", state_text) + + def test_child_failure_and_counter_overflow(self): + lib = ROOT / "test/assertions/lib.sh" + for command, expected in [("true", 0), ("false", 1), ("bash -c 'exit 42'", 1)]: + r = self.shell('source "$1"; run_assertion ' + command + '; report_results', lib) + self.assertEqual(r.returncode, expected, r.stdout + r.stderr) + r = self.shell('source "$1"; for i in {1..256}; do assert_eq x y injected; done; report_results', lib) + self.assertNotEqual(r.returncode, 0) + + def test_individual_skill_propagates_child_failure(self): + # Use the real inspect suite. Only replace its child assertion, so the + # suite must succeed for rc=0 and fail for rc=42 with identical inputs. + test = self.base / "test" + (test / "skills").mkdir(parents=True) + shutil.copytree(ROOT / "test/assertions", test / "assertions") + shutil.copy2(ROOT / "test/skills/test-inspect.sh", test / "skills/test-inspect.sh") + for directory in ["inbox", "knowledge/notes"]: + (self.vault / directory).mkdir(parents=True, exist_ok=True) + for rc in [0, 42]: + (test / "assertions/check-no-mutation.sh").write_text("exit " + str(rc) + "\n") + r = self.shell('bash "$1" --skip-execute "--vault=$2"', + test / "skills/test-inspect.sh", self.vault) + self.assertEqual(r.returncode == 0, rc == 0, r.stdout + r.stderr) + + def test_aggregate_and_ci_exit(self): + test = self.base / "suite/test" + test.mkdir(parents=True) + suite = (ROOT / "test/run-all.sh").read_text() + (test / "run-all.sh").write_text(suite) + children = re.findall(r'\$SCRIPT_DIR/([^"\s]+\.(?:sh|py))', suite) + self.assertGreater(len(children), 10) + for child in children: + p = test / child + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("" if p.suffix == ".py" else "exit 0\n") + self.assertEqual(self.shell('bash "$1"', test / "run-all.sh").returncode, 0) + (test / "skills/test-distill.sh").write_text("exit 42\n") + self.assertNotEqual(self.shell('bash "$1"', test / "run-all.sh").returncode, 0) + workflow = (ROOT / ".github/workflows/ci.yml").read_text() + block = workflow.split("name: pure-shell suites", 1)[1].split("run: |", 1)[1].split("\n guard:", 1)[0] + command = textwrap.dedent(block).strip() + self.assertIn('exit "$fail"', command) + self.assertEqual(self.shell(command, cwd=test.parent).returncode, 0) + (test / "cli/test-cli-smoke.sh").write_text("exit 42\n") + self.assertNotEqual(self.shell(command, cwd=test.parent).returncode, 0) + + +class Comparison(unittest.TestCase): + @classmethod + def setUpClass(cls): + sys.dont_write_bytecode = True + spec = importlib.util.spec_from_file_location("comparison", ROOT / "test/harness/compare.py") + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def test_execution_failures_never_pass(self): + with tempfile.TemporaryDirectory(prefix="rill-runner-test-") as temp: + base = Path(temp) + for code, expected in [("print('ok')", "finished"), ("raise SystemExit(7)", "failed"), + ("import time; time.sleep(10)", "timeout")]: + result = self.module.execute([sys.executable, "-c", code], base, os.environ, 0.1, base / "run") + self.assertEqual(result["status"], expected) + result = self.module.execute([str(base / "missing-cli")], base, os.environ, 1, base / "missing") + self.assertEqual(result["status"], "not-run") + + def test_timeout_kills_children_after_leader_exits(self): + with tempfile.TemporaryDirectory(prefix="rill-child-timeout-") as temp: + base = Path(temp) + code = """import os, signal, time +from pathlib import Path +if os.fork() == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + Path('ready').write_text('ready') + time.sleep(0.7) + Path('leaked').write_text('child survived') + os._exit(0) +time.sleep(10) +""" + result = self.module.execute([sys.executable, "-c", code], base, os.environ, 0.4, base / "run") + self.assertEqual(result["status"], "timeout") + self.assertTrue((base / "ready").exists()) + time.sleep(0.5) + self.assertFalse((base / "leaked").exists()) + + def test_deployment_excludes_private_and_untracked_files(self): + with tempfile.TemporaryDirectory(prefix="rill-copy-test-") as temp: + root = Path(temp) / "root" + root.mkdir() + subprocess.run(["git", "init", "-q", str(root)], check=True) + fixtures = ["plugins/demo/run.sh", "plugins/local/private/run.sh", + "plugins/demo/.config", "plugins/demo/.state/private.txt"] + for name in fixtures: + p = root / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("synthetic sentinel") + subprocess.run(["git", "-C", str(root), "add", "-f", "."], check=True) + (root / "plugins/demo/untracked-secret").write_text("synthetic secret") + source = Path(temp) / "source" + self.module.copy_deployment(root, source) + self.assertEqual([str(p.relative_to(source)) for p in source.rglob("*") if p.is_file()], + ["plugins/demo/run.sh"]) + + def test_snapshot_tracks_directory_symlink_changes(self): + with tempfile.TemporaryDirectory(prefix="rill-watch-test-") as temp: + root = Path(temp) + (root / "one").mkdir() + (root / "two").mkdir() + link = root / "alias" + before = self.module.snapshot(root) + link.symlink_to("one", target_is_directory=True) + added = self.module.snapshot(root) + self.assertNotEqual(before, added) + link.unlink() + link.symlink_to("two", target_is_directory=True) + self.assertNotEqual(added, self.module.snapshot(root)) + link.unlink() + self.assertEqual(before, self.module.snapshot(root)) + + def test_fixture_includes_installed_container_schemas(self): + with tempfile.TemporaryDirectory(prefix="rill-schema-test-") as temp: + vault, source = self.module.setup(Path(temp).resolve()) + for container in ["inbox/journal", "knowledge/notes", "tasks", "workspace"]: + self.assertEqual((vault / container / "CLAUDE.md").read_bytes(), + (ROOT / container / "CLAUDE.md").read_bytes()) + self.assertTrue((vault / container / "AGENTS.md").is_file()) + self.assertFalse((source / ".git").exists()) + self.assertEqual(list((source / "inbox/journal").iterdir()), + [source / "inbox/journal/CLAUDE.md"]) + + def test_grade_detects_fact_loss_duplicates_and_mutation(self): + with tempfile.TemporaryDirectory(prefix="rill-grader-test-") as temp: + vault = Path(temp) + journal = vault / "inbox/journal/2030-01-02.md" + journal.parent.mkdir(parents=True) + journal.write_text("original fixture") + original = self.module.digest(journal) + note = vault / "knowledge/notes/note.md" + note.parent.mkdir(parents=True) + valid = "---\ncreated: 2030-01-02T01:00+00:00\ntype: insight\nsource: inbox/journal/2030-01-02.md\n---\nviolet-orbit cache 120 45\n" + note.write_text(valid) + case = self.module.CASES["create-note"] + errors, names = self.module.grade(vault, case, original) + self.assertEqual(errors, []) + note.write_text(valid.replace("120", "")) + self.assertTrue(self.module.grade(vault, case, original)[0]) + note.write_text(valid) + note.with_name("duplicate.md").write_text(valid) + self.assertTrue(self.module.grade(vault, case, original, names)[0]) + journal.write_text("mutated") + self.assertIn("Original journal changed", self.module.grade(vault, case, original)[0]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/cli/pipefail-allowlist.txt b/test/cli/pipefail-allowlist.txt index d6a1738..444864a 100644 --- a/test/cli/pipefail-allowlist.txt +++ b/test/cli/pipefail-allowlist.txt @@ -21,9 +21,7 @@ now_iso="$(date +%Y-%m-%dT%H:%M%z | sed 's/\([0-9][0-9]\)$/:\1/')" now_iso="$(date +%Y-%m-%dT%H:%M%z | sed 's/\([0-9][0-9]\)$/:\1/')" num="$(basename "$f" | sed 's/^\([0-9]*\)-.*/\1/' | sed 's/^0*//')" num="$(basename "$f" | sed 's/^\([0-9]*\)-.*/\1/' | sed 's/^0*//')" -patch_paths="$(printf '%s\n' "$patch_text" | -patch_text="$(printf '%s' "$json" | jq -r ' -paths="$(printf '%s' "$json" | jq -r ' +paths="$(printf '%s' "$json" | jq -er ' pid="$(echo "$line" | cut -f1)" preview="$(sed '/^---$/,/^---$/d' "$f" | sed '/^[[:space:]]*$/d' | head -1)" read_array="$(_yaml_list_from_block "$scope_block" "read" | _json_string_array)" diff --git a/test/cli/test-codex-projection.sh b/test/cli/test-codex-projection.sh index f05bc40..a3c73c6 100755 --- a/test/cli/test-codex-projection.sh +++ b/test/cli/test-codex-projection.sh @@ -63,7 +63,69 @@ test "$agents_bytes" -lt 4096 printf '%s\n' '---' 'description: Personal test workflow' '---' '# Personal' \ > "$VAULT/.claude/commands/personal-test.md" "$RILL_BIN" update --vault codex-test >/dev/null -test -L "$VAULT/.agents/skills/personal-test/SKILL.md" +test -f "$VAULT/.agents/skills/personal-test/SKILL.md" +test ! -L "$VAULT/.agents/skills/personal-test/SKILL.md" +grep -q "^name: personal-test$" "$VAULT/.agents/skills/personal-test/SKILL.md" + +# Personal native skill resources remain next to the single editable source. +mkdir -p "$VAULT/.claude/skills/personal-native/assets" +printf '%s\n' '---' 'name: personal-native' 'description: Use for native testing' '---' \ + '[asset](assets/example.txt)' > "$VAULT/.claude/skills/personal-native/SKILL.md" +printf '%s\n' 'resource sentinel' > "$VAULT/.claude/skills/personal-native/assets/example.txt" +mkdir -p "$VAULT/.agents/skills/personal-owned" +printf '%s\n' '---' 'name: personal-owned' 'description: User maintained' '---' \ + 'Do not overwrite' > "$VAULT/.agents/skills/personal-owned/SKILL.md" +cp "$VAULT/.agents/skills/personal-owned/SKILL.md" "$TMP_ROOT/owned-before" +"$RILL_BIN" update --vault codex-test >/dev/null +cmp "$TMP_ROOT/owned-before" "$VAULT/.agents/skills/personal-owned/SKILL.md" +grep -q '.claude/skills/personal-native/SKILL.md' "$VAULT/.agents/skills/personal-native/SKILL.md" +grep -q 'resource sentinel' "$VAULT/.claude/skills/personal-native/assets/example.txt" + +# Plugin commands without Codex metadata receive a wrapper immediately. +mkdir -p "$VAULT/plugins/local/foundation/commands/assets" +printf '%s\n' '# Fixture workflow' '[asset](assets/example.txt)' \ + > "$VAULT/plugins/local/foundation/commands/foundation-command.md" +printf '%s\n' 'plugin sentinel' > "$VAULT/plugins/local/foundation/commands/assets/example.txt" +printf '%s\n' 'foundation' >> "$VAULT/plugins/.installed" +(cd "$VAULT" && "$RILL_BIN" plugin enable foundation >/dev/null) +PLUGIN_SKILL="$VAULT/.agents/skills/foundation-command/SKILL.md" +test -f "$PLUGIN_SKILL" +test ! -L "$PLUGIN_SKILL" +grep -q '^name: foundation-command$' "$PLUGIN_SKILL" +grep -q 'plugins/local/foundation/commands/foundation-command.md' "$PLUGIN_SKILL" +(cd "$VAULT" && "$RILL_BIN" plugin disable foundation >/dev/null) +test ! -e "$PLUGIN_SKILL" +test -f "$VAULT/plugins/local/foundation/commands/assets/example.txt" + +# Doctor must reject a discoverable skill with missing required metadata. +mkdir -p "$VAULT/.agents/skills/invalid-fixture" +printf '%s\n' '# Missing frontmatter' > "$VAULT/.agents/skills/invalid-fixture/SKILL.md" +if (cd "$VAULT" && "$RILL_BIN" doctor codex) > "$TMP_ROOT/doctor-invalid.log" 2>&1; then + echo "doctor accepted invalid skill metadata" >&2 + exit 1 +fi +grep -q 'Invalid skill:' "$TMP_ROOT/doctor-invalid.log" +# This is a test-owned fixture, restored to valid form for the final doctor. +printf '%s\n' '---' 'name: invalid-fixture' 'description: Repaired fixture' '---' \ + > "$VAULT/.agents/skills/invalid-fixture/SKILL.md" + +# Legacy command names unsupported by Codex must not abort unrelated updates +# or leave plugin lifecycle state half-written. Doctor still reports the gap. +printf '%s\n' '# Preserve this command' > "$VAULT/.claude/commands/unsupported_name.md" +"$RILL_BIN" update --vault codex-test > "$TMP_ROOT/unsupported-update.log" 2>&1 +grep -q 'unsupported skill name: unsupported_name' "$TMP_ROOT/unsupported-update.log" +grep -q 'Preserve this command' "$VAULT/.claude/commands/unsupported_name.md" +(cd "$VAULT" && "$RILL_BIN" plugin enable foundation >/dev/null 2>&1) +grep -qx foundation "$VAULT/plugins/.enabled" +(cd "$VAULT" && "$RILL_BIN" plugin disable foundation >/dev/null 2>&1) +if grep -qx foundation "$VAULT/plugins/.enabled"; then exit 1; fi +if (cd "$VAULT" && "$RILL_BIN" doctor codex) > "$TMP_ROOT/unsupported-doctor.log" 2>&1; then + echo "doctor failed to report unsupported command projection" >&2 + exit 1 +fi +grep -q 'command /unsupported_name has no Codex skill' "$TMP_ROOT/unsupported-doctor.log" +mv "$VAULT/.claude/commands/unsupported_name.md" "$VAULT/.claude/commands/supported-name.md" +"$RILL_BIN" update --vault codex-test >/dev/null # Deny rules must survive reprojection on `rill update` too. test -f "$VAULT/.codex/rules/rill-deny.rules" diff --git a/test/cli/test-evaluation-foundation.sh b/test/cli/test-evaluation-foundation.sh new file mode 100644 index 0000000..d40ee1a --- /dev/null +++ b/test/cli/test-evaluation-foundation.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +python3 "$SCRIPT_DIR/evaluation-foundation.py" diff --git a/test/harness/README.md b/test/harness/README.md new file mode 100644 index 0000000..2b3a445 --- /dev/null +++ b/test/harness/README.md @@ -0,0 +1,108 @@ +# Rill harness evaluation foundation + +This directory compares installed Rill workflows using the actual Codex and +Claude Code CLIs. It complements the deterministic tests in `test/cli/` and the +older Claude skill suites in `test/skills/`; it is not the knowledge-search +quality evaluation provided by `/eval`. + +## Run + +Log into both CLIs using their normal interactive login flows first. Then run: + +```bash +python3 test/harness/compare.py \ + --harness both --codex-model gpt-6-astra --claude-model opus +``` + +Model arguments are explicit. Choose model names supported by your installed +CLIs and account. `--case create-note` selects the small distribution/resource +case; `--case distill-repeat` runs the real `/distill` workflow twice. The default +runs both. `--timeout` is a per-invocation wall-clock limit in seconds (default +600), and the runner terminates the entire invocation process group on timeout. +This is opt-in: ordinary CI does not call either model or spend model credits. + +The runner prints a newly allocated temporary directory containing `results.json`, +raw CLI stdout/stderr and every resulting synthetic vault. It retains failures +for inspection. There is deliberately no existing-vault argument. Inputs are +constructed locally, never read from a production vault. Clean up retained test +directories yourself after reviewing them. + +Optional `--watch /path/to/a/vault` takes read-only content snapshots before and +after execution. A change anywhere in the watched directory (excluding Git +internals) fails the comparison. Concurrent legitimate edits also fail this +check; run comparisons when watched vaults are idle. An outside canary adds a +small additional contamination check. Neither check detects every possible +write outside the test area or replaces the OS sandbox. + +## What is measured + +`cases.json` contains expectations independently of the generated outputs: + +- `create-note`: invoke a projected personal command, read its adjacent asset, + create exactly one note with required facts, source and timestamp metadata. +- `distill-repeat`: extract one reusable cache benchmark finding from a journal; + preserve the original; record processing exactly once; run again and verify + that no duplicate or replacement note appears. + +A pass requires a zero CLI exit status, a successful structured completion +event, expected artifacts/facts and all invariants. An empty log, missing CLI, +authentication error, timeout or failed completion never counts as a pass. +The per-run record includes CLI version, requested model, arguments, elapsed +seconds, completion/usage events, note names and individual failures. Raw logs +retain resolved model details when the CLI reports them. The source revision, +dirty-state indicator and initial fixture digest identify the evaluated input. + +This is a small functional regression set. Keyword checks cannot establish +semantic quality, generalized reliability, or model superiority. Review the +retained notes as well. Repeated trials and a larger, held-out case set are the +next step before removing more rules or comparing quality statistically. + +## Isolation and scope + +Each harness/case receives a separate copy of the same initialized fixture. +Only allowlisted, Git-tracked deployment inputs are copied from the current checkout; source Git +metadata, worktrees, credentials and real vault contents are excluded. Local plugins, plugin +configuration/state and untracked files are excluded even if present in the checkout. +Symlink deployment inputs fail setup rather than following an external target. The +initialization registry has a separate home. Child processes use the fixture's +`RILL_HOME` and projected `rill` executable on PATH. Repositories have no remote. + +The CLIs reuse the operator's existing authentication; no credential file is +copied or printed. Codex ignores user configuration and uses `workspace-write` +with approvals disabled (a denied action fails rather than prompting). Claude +loads project settings only, disables external MCP servers, requires its Bash +sandbox, disallows unsandboxed retries and runs with edit acceptance. User-level +CLI runtime state and authentication stores are still owned by the CLIs; this +is not a separate virtual machine or a claim of identical system prompts. + +The runner does not bypass hook trust. Untrusted newly projected hooks may be +skipped by the CLI, so these model runs alone are not evidence that hook +protection was active. `test-evaluation-foundation.sh` independently replays +actual supported hook envelopes and asserts exact blocking exit codes and +post-write side effects. Deployment must still establish hook trust through +the harness's normal mechanism. + +Do not publish raw logs without inspecting them: CLI startup may include local +paths or user customization metadata even though all evaluation data is fake. +A public PR should include reviewed aggregate results and limitations. + +## Deterministic regression checks + +```bash +bash test/cli/test-evaluation-foundation.sh +bash test/cli/test-codex-projection.sh +bash test/cli/test-cli-smoke.sh +bash test/cli/test-track-managed-gitignore.sh +``` + +The foundation suite injects failures into child assertions, a real individual +skill suite, the aggregate runner and the exact shell block used by CI. It also +covers a 256-failure counter overflow, unknown hook inputs, string/object/command +patch envelopes, multiple targets, moves, relative paths, symlink aliases and +post-write recording. Distribution tests cover init/update, personal assets, +user-owned skills, plugin activation/deactivation and missing metadata. + +The legacy skill suites still support assertion-only modes for diagnosis; those +modes do not prove that a model ran. Use this comparison runner for structured +live-run evidence. The aggregate `test/run-all.sh` includes model-backed legacy +suites and must not be mistaken for a model-free CI command. diff --git a/test/harness/cases.json b/test/harness/cases.json new file mode 100644 index 0000000..34dc19a --- /dev/null +++ b/test/harness/cases.json @@ -0,0 +1,14 @@ +{ + "create-note": { + "prompt": "Run the /foundation-note workflow. Use the installed skill and its referenced resource. Complete the file creation in this synthetic vault.", + "repeats": 1, + "required_terms": ["violet-orbit", "120", "45"], + "expected_notes": 1 + }, + "distill-repeat": { + "prompt": "Run /distill on the single journal entry in this synthetic vault. Extract the reusable technical finding, preserve the original, and record processing state. There are no people, organizations, tasks, or external sources to resolve. Do not contact external services or push Git. Use English for the generated notes. Finish all applicable local steps without asking for preferences.", + "repeats": 2, + "required_terms": ["120", "45", "cache"], + "expected_notes": 1 + } +} diff --git a/test/harness/compare.py b/test/harness/compare.py new file mode 100644 index 0000000..c542768 --- /dev/null +++ b/test/harness/compare.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Opt-in CLI comparison in new synthetic vaults. Requires existing CLI login.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[2] +CASES = json.loads(Path(__file__).with_name("cases.json").read_text()) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def snapshot(root): + """Read-only content snapshot; exclude Git internals, never follow symlinks.""" + result = {} + for directory, dirs, files in os.walk(root, followlinks=False): + dirs[:] = [d for d in dirs if d != ".git"] + for name in list(dirs): + path = Path(directory) / name + if path.is_symlink(): + result[str(path.relative_to(root))] = "link:" + os.readlink(path) + dirs.remove(name) + for name in files: + path = Path(directory) / name + key = str(path.relative_to(root)) + if path.is_symlink(): + result[key] = "link:" + os.readlink(path) + elif path.is_file(): + result[key] = digest(path) + return result + + +def execute(argv, cwd, env, timeout, stem): + """Keep failure, empty output and timeout distinct; terminate the process group.""" + start = time.monotonic() + record = {"argv": argv, "status": "not-run", "exit_code": None} + with stem.with_suffix(".stdout.jsonl").open("w") as out, stem.with_suffix(".stderr.log").open("w") as err: + try: + proc = subprocess.Popen(argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=out, stderr=err, start_new_session=True) + try: + record["exit_code"] = proc.wait(timeout=timeout) + record["status"] = "finished" if proc.returncode == 0 else "failed" + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + finally: + # The group can outlive its leader. Kill remaining children + # even when wait() already returned for the CLI parent. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + record["status"] = "timeout" + record["exit_code"] = proc.returncode + except OSError as exc: + record["error"] = str(exc) + record["elapsed_seconds"] = round(time.monotonic() - start, 3) + return record + + +def checked(argv, cwd, env): + result = subprocess.run(list(map(str, argv)), cwd=cwd, env=env, + text=True, capture_output=True, timeout=120) + if result.returncode: + raise RuntimeError("Setup failed: " + result.stderr + result.stdout) + return result.stdout + + +CONTAINERS = ["inbox", "inbox/journal", "inbox/meetings", "inbox/tweets", + "inbox/web-clips", "inbox/sources", "knowledge/notes", "knowledge/people", + "knowledge/orgs", "knowledge/self", "projects", "workspace", "tasks", + "pages", "reports/daily", "reports/newsletter"] + + +def copy_deployment(root, source): + """Allowlisted, Git-tracked deployment inputs; never local plugin state.""" + tracked = checked(["git", "ls-files", "-z"], root, os.environ).split("\0") + schemas = {container + "/CLAUDE.md" for container in CONTAINERS} + documents = {"SPEC.md", "taxonomy.md", "VERSION", "CLAUDE.md", "AGENTS.md"} + prefixes = ("bin/", "lib/", "skills/", "templates/", "plugins/", "eval/", + ".claude/rules/", ".claude/commands/", ".claude/agents/") + excluded = {".config", ".state", ".env", ".installed", ".enabled", "__pycache__"} + for name in tracked: + if not name or not (name in schemas or name in documents or name.startswith(prefixes)): + continue + parts = Path(name).parts + if name.startswith("plugins/local/") or any(part in excluded for part in parts): + continue + original = root / name + if original.is_symlink() or any((root / parent).is_symlink() for parent in Path(name).parents): + raise RuntimeError("Symlink deployment input is not supported: " + name) + destination = source / name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(original, destination) + + +def setup(base): + source = base / "source" + source.mkdir() + copy_deployment(ROOT, source) + home = base / "registry-home" + home.mkdir() + env = dict(os.environ, RILL_SOURCE=str(source), HOME=str(home), + GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") + env.pop("RILL_HOME", None) + template = base / "template" + checked([source / "bin/rill", "init", template, "--name", "comparison", "--no-default"], base, env) + env["RILL_HOME"] = str(template) + command = template / ".claude/commands/foundation-note.md" + command.write_text("# Foundation note\n\nRead `foundation-assets/finding.txt` next to this workflow. " + "Create exactly one knowledge note using rill mkfile, type insight, " + "source inbox/journal/2030-01-02.md. Preserve its generated created field. " + "Include every factual detail and the marker from the resource. " + "Do not run distill, contact services, commit or push.\n") + assets = command.parent / "foundation-assets" + assets.mkdir() + assets.joinpath("finding.txt").write_text("Marker: violet-orbit. In a synthetic benchmark, " + "a cache reduced median latency from 120 ms to 45 ms.\n") + # Update exercises legacy command -> Codex skill distribution. + checked([source / "bin/rill", "update"], template, env) + checked([source / "bin/rill", "mkfile", "inbox/journal", "--date", "2030-01-02", "--type", "journal"], template, env) + journal = template / "inbox/journal/2030-01-02.md" + with journal.open("a") as f: + f.write("\n# Synthetic cache benchmark\n\nA cache reduced median latency from 120 ms to 45 ms " + "in a synthetic benchmark. This suggests caching repeated reads can reduce latency. " + "This is a technical observation, not a task or a commitment.\n") + # Remove only the known init-owned welcome note from this synthetic template. + welcome = template / "knowledge/notes/welcome-to-rill.md" + if welcome.exists(): + welcome.unlink() + for container in CONTAINERS: + for filename in ("CLAUDE.md", "AGENTS.md"): + if not (template / container / filename).is_file(): + raise RuntimeError("Missing installed container schema: " + container + "/" + filename) + checked([source / "bin/rill", "doctor", "codex"], template, env) + # The fixture does not need onboarding or external synchronization. + return template, source + + +def command_for(harness, model, prompt, vault): + if harness == "codex": + return ["codex", "exec", "--ignore-user-config", "--ephemeral", "--sandbox", "workspace-write", + "-c", 'approval_policy="never"', "-c", 'web_search="disabled"', + "-c", 'shell_environment_policy.inherit="all"', + "--json", "--color", "never", "--model", model, "--cd", str(vault), prompt] + settings = {"sandbox": {"enabled": True, "failIfUnavailable": True, + "autoAllowBashIfSandboxed": True, "allowUnsandboxedCommands": False}} + return ["claude", "-p", prompt, "--output-format", "stream-json", "--verbose", + "--model", model, "--max-turns", "80", "--no-session-persistence", + "--setting-sources", "project", "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}', + "--settings", json.dumps(settings), "--permission-mode", "acceptEdits", + "--disallowedTools", "WebSearch,WebFetch,Bash(git push *)"] + + +def grade(vault, case, original, previous_notes=None): + errors = [] + if digest(vault / "inbox/journal/2030-01-02.md") != original: + errors.append("Original journal changed") + notes = sorted(p for p in (vault / "knowledge/notes").glob("*.md") + if p.name not in ("AGENTS.md", "CLAUDE.md")) + if len(notes) != case["expected_notes"]: + errors.append("Expected exactly %s note(s), found %s" % (case["expected_notes"], len(notes))) + body = "\n".join(p.read_text() for p in notes) + for term in case["required_terms"]: + if term.lower() not in body.lower(): + errors.append("Missing fact: " + term) + for p in notes: + text = p.read_text() + match = re.match(r"\A---\n(.*?)\n---(?:\n|$)", text, re.S) + header = match[1] if match else "" + for key in ("created", "type", "source"): + if not re.search(r"^" + key + r":\s*\S", header, re.M): + errors.append(p.name + ": missing " + key) + if not re.search(r"^created: \d{4}-\d\d-\d\dT", header, re.M): + errors.append(p.name + ": invalid created timestamp") + if not re.search(r"^type: (record|insight|reference)\s*$", header, re.M): + errors.append(p.name + ": invalid note type") + if not re.search(r"^source: .*2030-01-02", header, re.M): + errors.append(p.name + ": wrong source") + if case.get("repeats", 1) > 1: + processed = vault / "inbox/journal/.processed" + lines = processed.read_text().splitlines() if processed.is_file() else [] + if lines.count("2030-01-02.md") != 1: + errors.append("Journal must be marked processed exactly once") + names = [p.name for p in notes] + if previous_notes is not None and names != previous_notes: + errors.append("Repeat changed the note set (duplicate or replacement)") + return errors, names + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--harness", choices=["codex", "claude", "both"], default="both") + parser.add_argument("--case", choices=list(CASES), action="append") + parser.add_argument("--codex-model", required=True) + parser.add_argument("--claude-model", required=True) + parser.add_argument("--timeout", type=int, default=600) + parser.add_argument("--watch", type=Path, action="append", default=[], help="Read-only contamination snapshot") + args = parser.parse_args() + if args.timeout < 1: + parser.error("timeout must be positive") + base = Path(tempfile.mkdtemp(prefix="rill-comparison-")).resolve() + print("Results: " + str(base), flush=True) + report = {"schema_version": 1, "source_revision": checked(["git", "rev-parse", "HEAD"], ROOT, os.environ).strip(), + "source_dirty": bool(checked(["git", "status", "--porcelain"], ROOT, os.environ)), + "case_sha256": digest(Path(__file__).with_name("cases.json")), "runs": []} + watched = {str(p.resolve()): snapshot(p.resolve()) for p in args.watch} + outside = base / "outside-canary.txt" + outside.write_text("comparison canary\n") + canary = digest(outside) + try: + template, source = setup(base) + report["source_sha256"] = hashlib.sha256(json.dumps(snapshot(source), sort_keys=True).encode()).hexdigest() + report["fixture_sha256"] = hashlib.sha256(json.dumps(snapshot(template), sort_keys=True).encode()).hexdigest() + harnesses = ["codex", "claude"] if args.harness == "both" else [args.harness] + for harness in harnesses: + model = getattr(args, harness + "_model") + version = subprocess.run([harness, "--version"], text=True, capture_output=True, timeout=30) + if version.returncode: + raise RuntimeError(harness + " --version failed") + for case_name in args.case or CASES: + case = CASES[case_name] + run_dir = base / (harness + "-" + case_name) + vault = run_dir / "vault" + shutil.copytree(template, vault) + env = dict(os.environ, RILL_HOME=str(vault), RILL_SOURCE=str(source), + PATH=str(vault / ".rill/bin") + os.pathsep + os.environ["PATH"], + GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") + checked(["git", "init", "-q"], vault, env) + # No remotes. The test identity is local to this repository. + checked(["git", "config", "user.name", "Fixture Runner"], vault, env) + checked(["git", "config", "user.email", "fixture@localhost"], vault, env) + checked(["git", "add", "."], vault, env) + checked(["git", "-c", "commit.gpgsign=false", "commit", "-qm", "Synthetic fixture"], vault, env) + original = digest(vault / "inbox/journal/2030-01-02.md") + previous_notes = None + for repeat in range(case["repeats"]): + prompt = case["prompt"] + "\nWork only in the current synthetic vault. Do not change harness settings or run rill update/init. Do not read other vaults. Use English for all generated content." + argv = command_for(harness, model, prompt, vault) + record = execute(argv, vault, env, args.timeout, run_dir / ("run-" + str(repeat + 1))) + record.update(harness=harness, model=model, version=version.stdout.strip(), case=case_name, repeat=repeat + 1) + errors, names = grade(vault, case, original, previous_notes) + output = (run_dir / ("run-" + str(repeat + 1) + ".stdout.jsonl")).read_text() + events = [] + for line in output.splitlines(): + try: + event = json.loads(line) + if isinstance(event, dict): + events.append(event) + except json.JSONDecodeError: + pass + if harness == "codex": + complete = any(e.get("type") == "turn.completed" for e in events) + else: + complete = any(e.get("type") == "result" and not e.get("is_error") and e.get("subtype") == "success" for e in events) + if not complete: + errors.append("No successful CLI completion event") + if record["status"] != "finished": + errors.append("CLI did not finish successfully") + if digest(outside) != canary: + errors.append("Outside canary changed") + for path, before in watched.items(): + if snapshot(Path(path)) != before: + errors.append("Watched directory changed: " + path) + record["usage_events"] = [e for e in events if e.get("type") in ("turn.completed", "result")] + record["reported_models"] = sorted({str(e.get("model")) for e in events if e.get("model")}) + record["errors"] = errors + record["passed"] = not errors + record["notes"] = names + report["runs"].append(record) + (base / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(harness, case_name, repeat + 1, "PASS" if not errors else "FAIL", errors, flush=True) + previous_notes = names + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + report["setup_error"] = str(exc) + report["passed"] = bool(report["runs"]) and "setup_error" not in report and all(r["passed"] for r in report["runs"]) + (base / "results.json").write_text(json.dumps(report, indent=2) + "\n") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/run-all.sh b/test/run-all.sh index d368b9c..e59edb6 100755 --- a/test/run-all.sh +++ b/test/run-all.sh @@ -65,6 +65,7 @@ run_test "/solve" "$SCRIPT_DIR/skills/test-solve.sh" run_test "/inspect" "$SCRIPT_DIR/skills/test-inspect.sh" run_test "/repair" "$SCRIPT_DIR/skills/test-repair.sh" run_test "/eval" "$SCRIPT_DIR/skills/test-eval.sh" +run_test "Evaluation foundation" "$SCRIPT_DIR/cli/test-evaluation-foundation.sh" run_test "Codex projection" "$SCRIPT_DIR/cli/test-codex-projection.sh" run_test "eval distribution" "$SCRIPT_DIR/cli/test-eval-distribution.sh" @@ -75,3 +76,6 @@ else echo " $TOTAL_FAIL test suite(s) failed" fi echo "============================================" + +# The summary must agree with the process status used by CI. +(( TOTAL_FAIL == 0 )) diff --git a/test/skills/test-briefing.sh b/test/skills/test-briefing.sh index 31fe198..42b6cb4 100755 --- a/test/skills/test-briefing.sh +++ b/test/skills/test-briefing.sh @@ -94,7 +94,7 @@ echo "" # 1. Inbox immutability (INV-01) echo "=== INV-01: Inbox immutability ===" -bash "$ASSERTIONS_DIR/check-no-mutation.sh" "$HASH_FILE" || true +run_assertion bash "$ASSERTIONS_DIR/check-no-mutation.sh" "$HASH_FILE" echo "" # 2. Daily Note existence and frontmatter (INV-02, INV-04) @@ -249,7 +249,7 @@ cat > "$RESULTS_DIR/summary.json" </dev/null || true) if [[ -n "$ORG_TAGS" && "$ORG_TAGS" != "[]" ]]; then - bash "$ASSERTIONS_DIR/check-taxonomy.sh" "$ORG_FILE" taxonomy.md || true + run_assertion bash "$ASSERTIONS_DIR/check-taxonomy.sh" "$ORG_FILE" taxonomy.md else echo " INFO: No tags field in organized file (skipping taxonomy check)" fi @@ -223,7 +223,7 @@ cat > "$RESULTS_DIR/summary.json" < "$RESULTS_DIR/summary.json" < "$RESULTS_DIR/summary.json" < "$RESULTS_DIR/summary.json" < "$RESULTS_DIR/summary.json" < "$RESULTS_DIR/summary.json" < "$RESULTS_DIR/summary.json" <