diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 8b4d8b6942..48320e405e 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -61,8 +61,12 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: "3.14" + - name: setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod - name: Validate skill and installer compatibility - run: ./hack/ci/validate-agent-skill.sh + run: go run ./hack/validate_agent_skill lint: name: Lint diff --git a/Taskfile.yml b/Taskfile.yml index ec689e9faf..c0210cdcb3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,7 +3,7 @@ version: "3" tasks: agent-skill:validate: desc: validate the end-user Devsy skill and installer discovery (requires Node 22.20+ and Python 3.11+) - cmd: ./hack/ci/validate-agent-skill.sh + cmd: go run ./hack/validate_agent_skill cli:tidy: desc: tidy go.mod and go.sum diff --git a/hack/ci/agent-skill-evaluation.md b/hack/ci/agent-skill-evaluation.md index 30b041b72b..6c26b9e168 100644 --- a/hack/ci/agent-skill-evaluation.md +++ b/hack/ci/agent-skill-evaluation.md @@ -33,7 +33,7 @@ Reconcile the tool schemas against `cmd/mcp`, and the CLI syntax in the skill ag ## Validation helper -`validate-agent-skill.sh` runs the upstream Agent Skills `skills-ref validate` command for format and naming, and installs the skill with the Vercel `skills` CLI to exercise discovery. It also checks that the reference files exist and that installed copies match the source. `skills` is pinned by package version and `skills-ref` by source commit. +`validate_agent_skill` (run as `go run ./hack/validate_agent_skill`) runs the upstream Agent Skills `skills-ref validate` command for format and naming, and installs the skill with the Vercel `skills` CLI to exercise discovery. It also checks that the reference files exist and that installed copies match the source. `skills` is pinned by package version and `skills-ref` by source commit. Requirements: Node.js 22.20+, Python 3.11+ (set `PYTHON` if the default is older), and network access to fetch dependencies. The helper uses temporary virtualenv and install directories that it removes on exit, and disables installer telemetry. CI runs it without provider infrastructure. diff --git a/hack/ci/validate-agent-skill.sh b/hack/ci/validate-agent-skill.sh deleted file mode 100755 index ca05582d3e..0000000000 --- a/hack/ci/validate-agent-skill.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -skill_dir="$repo_root/.agents/skills/devsy" -skills_version="1.7.0" -skills_ref_commit="69ef37e9424c0a7ea9dd2293b559e43ec8176379" -python_bin="${PYTHON:-python3}" - -for file in SKILL.md references/{mcp-tools,cli,workspace-workflows,providers,safety,troubleshooting}.md; do - test -s "$skill_dir/$file" || { - echo "Missing skill file: $file" >&2 - exit 1 - } -done -"$python_bin" -c 'import sys; assert sys.version_info >= (3, 11), "Python 3.11+ is required (set PYTHON to its executable)"' - -tmp_dir="$(mktemp -d)" -trap 'rm -rf "$tmp_dir"' EXIT -export DISABLE_TELEMETRY=1 -export NO_COLOR=1 - -"$python_bin" -m venv "$tmp_dir/validator" -"$tmp_dir/validator/bin/python" -m pip install --quiet --disable-pip-version-check \ - "skills-ref @ git+https://github.com/agentskills/agentskills.git@${skills_ref_commit}#subdirectory=skills-ref" -"$tmp_dir/validator/bin/skills-ref" validate "$skill_dir" - -for agent in codex claude-code; do - install_dir="$tmp_dir/$agent" - mkdir -p "$install_dir" - ( - cd "$install_dir" - npx --yes "skills@$skills_version" add "$repo_root" --skill devsy \ - --agent "$agent" --copy --yes --json >"$tmp_dir/$agent.json" - ) - case "$agent" in - codex) installed="$install_dir/.agents/skills/devsy" ;; - claude-code) installed="$install_dir/.claude/skills/devsy" ;; - esac - test -f "$installed/SKILL.md" - diff -r "$skill_dir" "$installed" -done - -echo "Devsy skill conformance, discovery, and Codex/Claude Code installation passed." diff --git a/hack/validate_agent_skill/main.go b/hack/validate_agent_skill/main.go new file mode 100644 index 0000000000..3c2f74d17a --- /dev/null +++ b/hack/validate_agent_skill/main.go @@ -0,0 +1,308 @@ +// Command validate_agent_skill checks the end-user Devsy agent skill. +// +// It runs the upstream skills-ref validator against the source skill, then +// installs the skill with the Vercel skills CLI for Codex and Claude Code and +// requires each installed copy to match the source byte for byte. +// +// Run from the repository root, or pass --repo-root: +// +// go run ./hack/validate_agent_skill +// +// Requires npx (Node.js), Python 3.11+ (PYTHON overrides python3), and network +// access to fetch the pinned validator and installer. +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" +) + +const ( + skillsVersion = "1.7.0" + skillsRefCommit = "69ef37e9424c0a7ea9dd2293b559e43ec8176379" + skillsRefSpec = "skills-ref @ git+https://github.com/agentskills/agentskills.git@" + + skillsRefCommit + "#subdirectory=skills-ref" + + pythonVersionCheck = "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)" + pythonRequired = "Python 3.11+ is required (set PYTHON to its executable)" + successMessage = "Devsy skill conformance, discovery, and Codex/Claude Code installation passed." +) + +var requiredSkillFiles = []string{ + "SKILL.md", + "references/mcp-tools.md", + "references/cli.md", + "references/workspace-workflows.md", + "references/providers.md", + "references/safety.md", + "references/troubleshooting.md", +} + +type agentInstall struct { + agent string + dir string // installed skill directory under the agent's temp install root +} + +var agentInstalls = []agentInstall{ + {agent: "codex", dir: filepath.Join(".agents", "skills", "devsy")}, + {agent: "claude-code", dir: filepath.Join(".claude", "skills", "devsy")}, +} + +// command is one external process invocation. +type command struct { + name string + args []string + dir string + env []string // nil inherits the parent environment + stdout io.Writer +} + +// runner executes a command. execCommand is the only real implementation. +type runner func(c command) error + +func execCommand(c command) error { + cmd := exec.Command(c.name, c.args...) + cmd.Dir = c.dir + cmd.Env = c.env + cmd.Stdout = c.stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +type validation struct { + repoRoot string + python string + goos string // selects the virtualenv layout + run runner + stdout io.Writer +} + +// missingSkillFile reports a required source skill file that is absent or empty. +type missingSkillFile string + +func (m missingSkillFile) Error() string { + return "Missing skill file: " + string(m) +} + +func main() { + os.Exit(runCLI(os.Args[1:], os.Stdout, os.Stderr, execCommand)) +} + +func runCLI(args []string, stdout, stderr io.Writer, run runner) int { + flags := flag.NewFlagSet("validate_agent_skill", flag.ContinueOnError) + flags.SetOutput(stderr) + repoRoot := flags.String("repo-root", ".", "repository root containing .agents/skills/devsy") + if err := flags.Parse(args); err != nil { + return 2 + } + root, err := filepath.Abs(*repoRoot) + if err != nil { + _, _ = fmt.Fprintln(stderr, err) + return 1 + } + err = validate(validation{ + repoRoot: root, + python: pythonBin(), + goos: runtime.GOOS, + run: run, + stdout: stdout, + }) + if err != nil { + _, _ = fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +func pythonBin() string { + if p := os.Getenv("PYTHON"); p != "" { + return p + } + return "python3" +} + +// validate runs the steps in order and stops at the first failure. +func validate(v validation) error { + skillDir := skillDirFor(v.repoRoot) + if err := checkSourceFiles(skillDir); err != nil { + return err + } + if err := v.run(command{name: v.python, args: []string{"-c", pythonVersionCheck}}); err != nil { + return fmt.Errorf("%s: %w", pythonRequired, err) + } + + tmp, err := os.MkdirTemp("", "validate-agent-skill-") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + + env := append(os.Environ(), "DISABLE_TELEMETRY=1", "NO_COLOR=1") + if err := validateWithSkillsRef(v, env, tmp, skillDir); err != nil { + return err + } + for _, in := range agentInstalls { + if err := checkInstall(v, env, tmp, in); err != nil { + return err + } + } + _, _ = fmt.Fprintln(v.stdout, successMessage) + return nil +} + +// validateWithSkillsRef creates a virtualenv with the pinned skills-ref +// validator and runs it against the source skill. +func validateWithSkillsRef(v validation, env []string, tmp, skillDir string) error { + venv := filepath.Join(tmp, "validator") + steps := []command{ + {name: v.python, args: []string{"-m", "venv", venv}}, + { + name: venvExecutable(venv, v.goos, "python"), + args: []string{ + "-m", "pip", "install", "--quiet", "--disable-pip-version-check", skillsRefSpec, + }, + }, + {name: venvExecutable(venv, v.goos, "skills-ref"), args: []string{"validate", skillDir}}, + } + for _, c := range steps { + c.env = env + c.stdout = v.stdout + if err := v.run(c); err != nil { + return fmt.Errorf("%s: %w", c.name, err) + } + } + return nil +} + +// checkInstall installs the skill for one agent with the pinned skills CLI and +// requires the installed copy to match the source tree. +func checkInstall(v validation, env []string, tmp string, in agentInstall) error { + dir := filepath.Join(tmp, in.agent) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + err := v.run(command{ + name: "npx", + args: []string{ + "--yes", "skills@" + skillsVersion, "add", v.repoRoot, + "--skill", "devsy", "--agent", in.agent, "--copy", "--yes", "--json", + }, + dir: dir, + env: env, + stdout: io.Discard, + }) + if err != nil { + return fmt.Errorf("%s install: %w", in.agent, err) + } + + installed := filepath.Join(dir, in.dir) + if !isNonEmptyFile(filepath.Join(installed, "SKILL.md")) { + return fmt.Errorf("%s install: SKILL.md missing or empty in %s", in.agent, installed) + } + if err := compareTrees(skillDirFor(v.repoRoot), installed); err != nil { + return fmt.Errorf("%s install: %w", in.agent, err) + } + return nil +} + +func skillDirFor(repoRoot string) string { + return filepath.Join(repoRoot, ".agents", "skills", "devsy") +} + +// venvExecutable returns the path of an executable in a virtualenv built for +// goos. Windows virtualenvs keep executables in Scripts with an .exe suffix. +func venvExecutable(venvDir, goos, name string) string { + if goos == "windows" { + return filepath.Join(venvDir, "Scripts", name+".exe") + } + return filepath.Join(venvDir, "bin", name) +} + +func checkSourceFiles(skillDir string) error { + for _, rel := range requiredSkillFiles { + if !isNonEmptyFile(filepath.Join(skillDir, filepath.FromSlash(rel))) { + return missingSkillFile(rel) + } + } + return nil +} + +func isNonEmptyFile(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() && info.Size() > 0 +} + +var errContentDiffers = errors.New("content differs") + +// compareTrees reports the first difference between the regular files under +// want and got: a file missing from got, a file whose bytes differ, or a file +// present only in got. Paths are compared relative to each root. +func compareTrees(want, got string) error { + wantFiles, err := listFiles(want) + if err != nil { + return err + } + gotFiles, err := listFiles(got) + if err != nil { + return err + } + for _, rel := range wantFiles { + if !slices.Contains(gotFiles, rel) { + return fmt.Errorf("installed copy missing %s", rel) + } + if err := sameBytes(filepath.Join(want, rel), filepath.Join(got, rel)); err != nil { + return fmt.Errorf("installed copy %s: %w", rel, err) + } + } + for _, rel := range gotFiles { + if !slices.Contains(wantFiles, rel) { + return fmt.Errorf("installed copy has extra file %s", rel) + } + } + return nil +} + +// listFiles returns the sorted slash-separated paths of regular files under root. +func listFiles(root string) ([]string, error) { + var files []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + slices.Sort(files) + return files, err +} + +func sameBytes(a, b string) error { + aData, err := os.ReadFile(a) + if err != nil { + return err + } + bData, err := os.ReadFile(b) + if err != nil { + return err + } + if !bytes.Equal(aData, bData) { + return errContentDiffers + } + return nil +} diff --git a/hack/validate_agent_skill/main_test.go b/hack/validate_agent_skill/main_test.go new file mode 100644 index 0000000000..9921f6b599 --- /dev/null +++ b/hack/validate_agent_skill/main_test.go @@ -0,0 +1,279 @@ +package main + +import ( + "bytes" + "errors" + "io/fs" + "maps" + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + testPython = "python-under-test" + cliRef = "references/cli.md" + safetyRef = "references/safety.md" + skillMD = "SKILL.md" +) + +var errBoom = errors.New("boom") + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// skillFixture returns a repository root holding a complete source skill. +func skillFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + for _, rel := range requiredSkillFiles { + writeFile(t, filepath.Join(skillDirFor(root), filepath.FromSlash(rel)), "content of "+rel) + } + return root +} + +func copyTree(t *testing.T, src, dst string) { + t.Helper() + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(filepath.Join(dst, rel)), 0o755); err != nil { + return err + } + return os.WriteFile(filepath.Join(dst, rel), data, 0o644) + }) + if err != nil { + t.Fatal(err) + } +} + +// fakeRunner records every command. failAt is the index of the call that +// returns errBoom (-1 for none). copyInstalls makes the fake npx install the +// source skill into its working directory, as the real installer does. +type fakeRunner struct { + t *testing.T + repoRoot string + failAt int + copyInstalls bool + calls []command +} + +func (f *fakeRunner) run(c command) error { + idx := len(f.calls) + f.calls = append(f.calls, c) + if idx == f.failAt { + return errBoom + } + if c.name == "npx" && f.copyInstalls { + agent := filepath.Base(c.dir) + for _, in := range agentInstalls { + if in.agent == agent { + copyTree(f.t, skillDirFor(f.repoRoot), filepath.Join(c.dir, in.dir)) + } + } + } + return nil +} + +func validationFor(root string, f *fakeRunner, out *bytes.Buffer) validation { + return validation{repoRoot: root, python: testPython, goos: "linux", run: f.run, stdout: out} +} + +func TestCheckSourceFiles(t *testing.T) { + t.Run("complete skill", func(t *testing.T) { + if err := checkSourceFiles(skillDirFor(skillFixture(t))); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("missing reference", func(t *testing.T) { + root := skillFixture(t) + safety := filepath.Join(skillDirFor(root), filepath.FromSlash(safetyRef)) + if err := os.Remove(safety); err != nil { + t.Fatal(err) + } + err := checkSourceFiles(skillDirFor(root)) + if err == nil || err.Error() != "Missing skill file: "+safetyRef { + t.Fatalf("got %v, want missing %s", err, safetyRef) + } + }) + + t.Run("empty SKILL.md", func(t *testing.T) { + root := skillFixture(t) + writeFile(t, filepath.Join(skillDirFor(root), skillMD), "") + err := checkSourceFiles(skillDirFor(root)) + if err == nil || err.Error() != "Missing skill file: "+skillMD { + t.Fatalf("got %v, want missing %s", err, skillMD) + } + }) +} + +func TestVenvExecutable(t *testing.T) { + venv := filepath.Join("tmp", "validator") + tests := []struct { + goos, name, want string + }{ + {"linux", "python", filepath.Join(venv, "bin", "python")}, + {"darwin", "skills-ref", filepath.Join(venv, "bin", "skills-ref")}, + {"windows", "python", filepath.Join(venv, "Scripts", "python.exe")}, + {"windows", "skills-ref", filepath.Join(venv, "Scripts", "skills-ref.exe")}, + } + for _, tt := range tests { + if got := venvExecutable(venv, tt.goos, tt.name); got != tt.want { + t.Errorf("venvExecutable(%s, %s) = %q, want %q", tt.goos, tt.name, got, tt.want) + } + } +} + +func baseTree() map[string]string { + return map[string]string{ + skillMD: "skill", + cliRef: "cli", + safetyRef: "safety", + } +} + +// tree returns baseTree with overrides applied and drop removed. +func tree(overrides map[string]string, drop string) map[string]string { + out := baseTree() + maps.Copy(out, overrides) + delete(out, drop) + return out +} + +func writeTree(t *testing.T, files map[string]string) string { + t.Helper() + root := t.TempDir() + for rel, content := range files { + writeFile(t, filepath.Join(root, filepath.FromSlash(rel)), content) + } + return root +} + +func assertErrContains(t *testing.T, err error, subs []string) { + t.Helper() + if len(subs) == 0 { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatal("expected error, got nil") + } + for _, sub := range subs { + if !strings.Contains(err.Error(), sub) { + t.Errorf("error %q does not contain %q", err, sub) + } + } +} + +func TestCompareTrees(t *testing.T) { + tests := []struct { + name string + got map[string]string + wantErr []string // substrings the error must contain; nil means no error + }{ + {name: "equal", got: tree(nil, "")}, + { + name: "changed bytes", + got: tree(map[string]string{cliRef: "changed"}, ""), + wantErr: []string{cliRef, "content differs"}, + }, + { + name: "missing file", + got: tree(nil, safetyRef), + wantErr: []string{"missing", safetyRef}, + }, + { + name: "extra file", + got: tree(map[string]string{"references/extra.md": "x"}, ""), + wantErr: []string{"extra", "references/extra.md"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := compareTrees(writeTree(t, baseTree()), writeTree(t, tt.got)) + assertErrContains(t, err, tt.wantErr) + }) + } +} + +// TestValidateStopsAtFirstFailure fails each step in turn and requires the +// error to surface and no later step to run. Call order: python version +// check, venv, pip install, skills-ref validate, codex install, claude install. +func TestValidateStopsAtFirstFailure(t *testing.T) { + steps := []struct { + name string + want string + }{ + {"python version check", pythonRequired}, + {"venv creation", ""}, + {"pip install", ""}, + {"skills-ref validate", ""}, + {"codex install", "codex install"}, + {"claude-code install", "claude-code install"}, + } + for i, step := range steps { + t.Run(step.name, func(t *testing.T) { + root := skillFixture(t) + f := &fakeRunner{t: t, repoRoot: root, failAt: i, copyInstalls: true} + var out bytes.Buffer + err := validate(validationFor(root, f, &out)) + if !errors.Is(err, errBoom) { + t.Fatalf("got %v, want wrapped errBoom", err) + } + if step.want != "" && !strings.Contains(err.Error(), step.want) { + t.Errorf("error %q does not contain %q", err, step.want) + } + if len(f.calls) != i+1 { + t.Errorf("ran %d commands, want %d (stop at failing step)", len(f.calls), i+1) + } + if out.Len() != 0 { + t.Errorf("success output written after failure: %q", out.String()) + } + }) + } +} + +func TestValidateInstalledSkillMissing(t *testing.T) { + root := skillFixture(t) + f := &fakeRunner{t: t, repoRoot: root, failAt: -1, copyInstalls: false} + var out bytes.Buffer + err := validate(validationFor(root, f, &out)) + if err == nil || !strings.Contains(err.Error(), "SKILL.md missing or empty") { + t.Fatalf("got %v, want installed SKILL.md failure", err) + } + if out.Len() != 0 { + t.Errorf("success output written after failure: %q", out.String()) + } +} + +func TestValidateSucceeds(t *testing.T) { + root := skillFixture(t) + f := &fakeRunner{t: t, repoRoot: root, failAt: -1, copyInstalls: true} + var out bytes.Buffer + if err := validate(validationFor(root, f, &out)); err != nil { + t.Fatalf("validate: %v", err) + } + if got := out.String(); got != successMessage+"\n" { + t.Errorf("stdout = %q, want %q", got, successMessage+"\n") + } +}