From 4deccb6ddba3ac7dfe06d80ade80fb40272bfe24 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 16 Oct 2026 12:29:58 -0600 Subject: [PATCH] refactor(hack): port agent-skill validator to Go --- .github/workflows/pr-ci.yml | 6 +- Taskfile.yml | 2 +- hack/ci/agent-skill-evaluation.md | 2 +- hack/ci/validate-agent-skill.sh | 44 ---- hack/validate_agent_skill/main.go | 308 +++++++++++++++++++++++++ hack/validate_agent_skill/main_test.go | 279 ++++++++++++++++++++++ 6 files changed, 594 insertions(+), 47 deletions(-) delete mode 100755 hack/ci/validate-agent-skill.sh create mode 100644 hack/validate_agent_skill/main.go create mode 100644 hack/validate_agent_skill/main_test.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 8b4d8b6942..48320e405e 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -61,8 +61,12 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: "3.14" + - name: setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod - name: Validate skill and installer compatibility - run: ./hack/ci/validate-agent-skill.sh + run: go run ./hack/validate_agent_skill lint: name: Lint diff --git a/Taskfile.yml b/Taskfile.yml index ec689e9faf..c0210cdcb3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,7 +3,7 @@ version: "3" tasks: agent-skill:validate: desc: validate the end-user Devsy skill and installer discovery (requires Node 22.20+ and Python 3.11+) - cmd: ./hack/ci/validate-agent-skill.sh + cmd: go run ./hack/validate_agent_skill cli:tidy: desc: tidy go.mod and go.sum diff --git a/hack/ci/agent-skill-evaluation.md b/hack/ci/agent-skill-evaluation.md index 10080817ac..7ad1cc0d35 100644 --- a/hack/ci/agent-skill-evaluation.md +++ b/hack/ci/agent-skill-evaluation.md @@ -41,7 +41,7 @@ The helper delegates format and naming validation to the upstream Agent Skills `skills` installations. Its repository-specific checks require the reference files and compare installed copies with the source; it does not implement a separate skill schema or parse installer display output. Both tools are pinned -in `validate-agent-skill.sh`: `skills` by package version and `skills-ref` by +in `hack/validate_agent_skill/main.go`: `skills` by package version and `skills-ref` by source commit. The helper requires Node.js 22.20+ and Python 3.11+ (set `PYTHON` if the default Python is older). It creates temporary virtualenv/install directories, removes diff --git a/hack/ci/validate-agent-skill.sh b/hack/ci/validate-agent-skill.sh deleted file mode 100755 index ca05582d3e..0000000000 --- a/hack/ci/validate-agent-skill.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -skill_dir="$repo_root/.agents/skills/devsy" -skills_version="1.7.0" -skills_ref_commit="69ef37e9424c0a7ea9dd2293b559e43ec8176379" -python_bin="${PYTHON:-python3}" - -for file in SKILL.md references/{mcp-tools,cli,workspace-workflows,providers,safety,troubleshooting}.md; do - test -s "$skill_dir/$file" || { - echo "Missing skill file: $file" >&2 - exit 1 - } -done -"$python_bin" -c 'import sys; assert sys.version_info >= (3, 11), "Python 3.11+ is required (set PYTHON to its executable)"' - -tmp_dir="$(mktemp -d)" -trap 'rm -rf "$tmp_dir"' EXIT -export DISABLE_TELEMETRY=1 -export NO_COLOR=1 - -"$python_bin" -m venv "$tmp_dir/validator" -"$tmp_dir/validator/bin/python" -m pip install --quiet --disable-pip-version-check \ - "skills-ref @ git+https://github.com/agentskills/agentskills.git@${skills_ref_commit}#subdirectory=skills-ref" -"$tmp_dir/validator/bin/skills-ref" validate "$skill_dir" - -for agent in codex claude-code; do - install_dir="$tmp_dir/$agent" - mkdir -p "$install_dir" - ( - cd "$install_dir" - npx --yes "skills@$skills_version" add "$repo_root" --skill devsy \ - --agent "$agent" --copy --yes --json >"$tmp_dir/$agent.json" - ) - case "$agent" in - codex) installed="$install_dir/.agents/skills/devsy" ;; - claude-code) installed="$install_dir/.claude/skills/devsy" ;; - esac - test -f "$installed/SKILL.md" - diff -r "$skill_dir" "$installed" -done - -echo "Devsy skill conformance, discovery, and Codex/Claude Code installation passed." diff --git a/hack/validate_agent_skill/main.go b/hack/validate_agent_skill/main.go new file mode 100644 index 0000000000..3c2f74d17a --- /dev/null +++ b/hack/validate_agent_skill/main.go @@ -0,0 +1,308 @@ +// Command validate_agent_skill checks the end-user Devsy agent skill. +// +// It runs the upstream skills-ref validator against the source skill, then +// installs the skill with the Vercel skills CLI for Codex and Claude Code and +// requires each installed copy to match the source byte for byte. +// +// Run from the repository root, or pass --repo-root: +// +// go run ./hack/validate_agent_skill +// +// Requires npx (Node.js), Python 3.11+ (PYTHON overrides python3), and network +// access to fetch the pinned validator and installer. +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" +) + +const ( + skillsVersion = "1.7.0" + skillsRefCommit = "69ef37e9424c0a7ea9dd2293b559e43ec8176379" + skillsRefSpec = "skills-ref @ git+https://github.com/agentskills/agentskills.git@" + + skillsRefCommit + "#subdirectory=skills-ref" + + pythonVersionCheck = "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)" + pythonRequired = "Python 3.11+ is required (set PYTHON to its executable)" + successMessage = "Devsy skill conformance, discovery, and Codex/Claude Code installation passed." +) + +var requiredSkillFiles = []string{ + "SKILL.md", + "references/mcp-tools.md", + "references/cli.md", + "references/workspace-workflows.md", + "references/providers.md", + "references/safety.md", + "references/troubleshooting.md", +} + +type agentInstall struct { + agent string + dir string // installed skill directory under the agent's temp install root +} + +var agentInstalls = []agentInstall{ + {agent: "codex", dir: filepath.Join(".agents", "skills", "devsy")}, + {agent: "claude-code", dir: filepath.Join(".claude", "skills", "devsy")}, +} + +// command is one external process invocation. +type command struct { + name string + args []string + dir string + env []string // nil inherits the parent environment + stdout io.Writer +} + +// runner executes a command. execCommand is the only real implementation. +type runner func(c command) error + +func execCommand(c command) error { + cmd := exec.Command(c.name, c.args...) + cmd.Dir = c.dir + cmd.Env = c.env + cmd.Stdout = c.stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +type validation struct { + repoRoot string + python string + goos string // selects the virtualenv layout + run runner + stdout io.Writer +} + +// missingSkillFile reports a required source skill file that is absent or empty. +type missingSkillFile string + +func (m missingSkillFile) Error() string { + return "Missing skill file: " + string(m) +} + +func main() { + os.Exit(runCLI(os.Args[1:], os.Stdout, os.Stderr, execCommand)) +} + +func runCLI(args []string, stdout, stderr io.Writer, run runner) int { + flags := flag.NewFlagSet("validate_agent_skill", flag.ContinueOnError) + flags.SetOutput(stderr) + repoRoot := flags.String("repo-root", ".", "repository root containing .agents/skills/devsy") + if err := flags.Parse(args); err != nil { + return 2 + } + root, err := filepath.Abs(*repoRoot) + if err != nil { + _, _ = fmt.Fprintln(stderr, err) + return 1 + } + err = validate(validation{ + repoRoot: root, + python: pythonBin(), + goos: runtime.GOOS, + run: run, + stdout: stdout, + }) + if err != nil { + _, _ = fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +func pythonBin() string { + if p := os.Getenv("PYTHON"); p != "" { + return p + } + return "python3" +} + +// validate runs the steps in order and stops at the first failure. +func validate(v validation) error { + skillDir := skillDirFor(v.repoRoot) + if err := checkSourceFiles(skillDir); err != nil { + return err + } + if err := v.run(command{name: v.python, args: []string{"-c", pythonVersionCheck}}); err != nil { + return fmt.Errorf("%s: %w", pythonRequired, err) + } + + tmp, err := os.MkdirTemp("", "validate-agent-skill-") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + + env := append(os.Environ(), "DISABLE_TELEMETRY=1", "NO_COLOR=1") + if err := validateWithSkillsRef(v, env, tmp, skillDir); err != nil { + return err + } + for _, in := range agentInstalls { + if err := checkInstall(v, env, tmp, in); err != nil { + return err + } + } + _, _ = fmt.Fprintln(v.stdout, successMessage) + return nil +} + +// validateWithSkillsRef creates a virtualenv with the pinned skills-ref +// validator and runs it against the source skill. +func validateWithSkillsRef(v validation, env []string, tmp, skillDir string) error { + venv := filepath.Join(tmp, "validator") + steps := []command{ + {name: v.python, args: []string{"-m", "venv", venv}}, + { + name: venvExecutable(venv, v.goos, "python"), + args: []string{ + "-m", "pip", "install", "--quiet", "--disable-pip-version-check", skillsRefSpec, + }, + }, + {name: venvExecutable(venv, v.goos, "skills-ref"), args: []string{"validate", skillDir}}, + } + for _, c := range steps { + c.env = env + c.stdout = v.stdout + if err := v.run(c); err != nil { + return fmt.Errorf("%s: %w", c.name, err) + } + } + return nil +} + +// checkInstall installs the skill for one agent with the pinned skills CLI and +// requires the installed copy to match the source tree. +func checkInstall(v validation, env []string, tmp string, in agentInstall) error { + dir := filepath.Join(tmp, in.agent) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + err := v.run(command{ + name: "npx", + args: []string{ + "--yes", "skills@" + skillsVersion, "add", v.repoRoot, + "--skill", "devsy", "--agent", in.agent, "--copy", "--yes", "--json", + }, + dir: dir, + env: env, + stdout: io.Discard, + }) + if err != nil { + return fmt.Errorf("%s install: %w", in.agent, err) + } + + installed := filepath.Join(dir, in.dir) + if !isNonEmptyFile(filepath.Join(installed, "SKILL.md")) { + return fmt.Errorf("%s install: SKILL.md missing or empty in %s", in.agent, installed) + } + if err := compareTrees(skillDirFor(v.repoRoot), installed); err != nil { + return fmt.Errorf("%s install: %w", in.agent, err) + } + return nil +} + +func skillDirFor(repoRoot string) string { + return filepath.Join(repoRoot, ".agents", "skills", "devsy") +} + +// venvExecutable returns the path of an executable in a virtualenv built for +// goos. Windows virtualenvs keep executables in Scripts with an .exe suffix. +func venvExecutable(venvDir, goos, name string) string { + if goos == "windows" { + return filepath.Join(venvDir, "Scripts", name+".exe") + } + return filepath.Join(venvDir, "bin", name) +} + +func checkSourceFiles(skillDir string) error { + for _, rel := range requiredSkillFiles { + if !isNonEmptyFile(filepath.Join(skillDir, filepath.FromSlash(rel))) { + return missingSkillFile(rel) + } + } + return nil +} + +func isNonEmptyFile(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() && info.Size() > 0 +} + +var errContentDiffers = errors.New("content differs") + +// compareTrees reports the first difference between the regular files under +// want and got: a file missing from got, a file whose bytes differ, or a file +// present only in got. Paths are compared relative to each root. +func compareTrees(want, got string) error { + wantFiles, err := listFiles(want) + if err != nil { + return err + } + gotFiles, err := listFiles(got) + if err != nil { + return err + } + for _, rel := range wantFiles { + if !slices.Contains(gotFiles, rel) { + return fmt.Errorf("installed copy missing %s", rel) + } + if err := sameBytes(filepath.Join(want, rel), filepath.Join(got, rel)); err != nil { + return fmt.Errorf("installed copy %s: %w", rel, err) + } + } + for _, rel := range gotFiles { + if !slices.Contains(wantFiles, rel) { + return fmt.Errorf("installed copy has extra file %s", rel) + } + } + return nil +} + +// listFiles returns the sorted slash-separated paths of regular files under root. +func listFiles(root string) ([]string, error) { + var files []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + slices.Sort(files) + return files, err +} + +func sameBytes(a, b string) error { + aData, err := os.ReadFile(a) + if err != nil { + return err + } + bData, err := os.ReadFile(b) + if err != nil { + return err + } + if !bytes.Equal(aData, bData) { + return errContentDiffers + } + return nil +} diff --git a/hack/validate_agent_skill/main_test.go b/hack/validate_agent_skill/main_test.go new file mode 100644 index 0000000000..9921f6b599 --- /dev/null +++ b/hack/validate_agent_skill/main_test.go @@ -0,0 +1,279 @@ +package main + +import ( + "bytes" + "errors" + "io/fs" + "maps" + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + testPython = "python-under-test" + cliRef = "references/cli.md" + safetyRef = "references/safety.md" + skillMD = "SKILL.md" +) + +var errBoom = errors.New("boom") + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// skillFixture returns a repository root holding a complete source skill. +func skillFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + for _, rel := range requiredSkillFiles { + writeFile(t, filepath.Join(skillDirFor(root), filepath.FromSlash(rel)), "content of "+rel) + } + return root +} + +func copyTree(t *testing.T, src, dst string) { + t.Helper() + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(filepath.Join(dst, rel)), 0o755); err != nil { + return err + } + return os.WriteFile(filepath.Join(dst, rel), data, 0o644) + }) + if err != nil { + t.Fatal(err) + } +} + +// fakeRunner records every command. failAt is the index of the call that +// returns errBoom (-1 for none). copyInstalls makes the fake npx install the +// source skill into its working directory, as the real installer does. +type fakeRunner struct { + t *testing.T + repoRoot string + failAt int + copyInstalls bool + calls []command +} + +func (f *fakeRunner) run(c command) error { + idx := len(f.calls) + f.calls = append(f.calls, c) + if idx == f.failAt { + return errBoom + } + if c.name == "npx" && f.copyInstalls { + agent := filepath.Base(c.dir) + for _, in := range agentInstalls { + if in.agent == agent { + copyTree(f.t, skillDirFor(f.repoRoot), filepath.Join(c.dir, in.dir)) + } + } + } + return nil +} + +func validationFor(root string, f *fakeRunner, out *bytes.Buffer) validation { + return validation{repoRoot: root, python: testPython, goos: "linux", run: f.run, stdout: out} +} + +func TestCheckSourceFiles(t *testing.T) { + t.Run("complete skill", func(t *testing.T) { + if err := checkSourceFiles(skillDirFor(skillFixture(t))); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("missing reference", func(t *testing.T) { + root := skillFixture(t) + safety := filepath.Join(skillDirFor(root), filepath.FromSlash(safetyRef)) + if err := os.Remove(safety); err != nil { + t.Fatal(err) + } + err := checkSourceFiles(skillDirFor(root)) + if err == nil || err.Error() != "Missing skill file: "+safetyRef { + t.Fatalf("got %v, want missing %s", err, safetyRef) + } + }) + + t.Run("empty SKILL.md", func(t *testing.T) { + root := skillFixture(t) + writeFile(t, filepath.Join(skillDirFor(root), skillMD), "") + err := checkSourceFiles(skillDirFor(root)) + if err == nil || err.Error() != "Missing skill file: "+skillMD { + t.Fatalf("got %v, want missing %s", err, skillMD) + } + }) +} + +func TestVenvExecutable(t *testing.T) { + venv := filepath.Join("tmp", "validator") + tests := []struct { + goos, name, want string + }{ + {"linux", "python", filepath.Join(venv, "bin", "python")}, + {"darwin", "skills-ref", filepath.Join(venv, "bin", "skills-ref")}, + {"windows", "python", filepath.Join(venv, "Scripts", "python.exe")}, + {"windows", "skills-ref", filepath.Join(venv, "Scripts", "skills-ref.exe")}, + } + for _, tt := range tests { + if got := venvExecutable(venv, tt.goos, tt.name); got != tt.want { + t.Errorf("venvExecutable(%s, %s) = %q, want %q", tt.goos, tt.name, got, tt.want) + } + } +} + +func baseTree() map[string]string { + return map[string]string{ + skillMD: "skill", + cliRef: "cli", + safetyRef: "safety", + } +} + +// tree returns baseTree with overrides applied and drop removed. +func tree(overrides map[string]string, drop string) map[string]string { + out := baseTree() + maps.Copy(out, overrides) + delete(out, drop) + return out +} + +func writeTree(t *testing.T, files map[string]string) string { + t.Helper() + root := t.TempDir() + for rel, content := range files { + writeFile(t, filepath.Join(root, filepath.FromSlash(rel)), content) + } + return root +} + +func assertErrContains(t *testing.T, err error, subs []string) { + t.Helper() + if len(subs) == 0 { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatal("expected error, got nil") + } + for _, sub := range subs { + if !strings.Contains(err.Error(), sub) { + t.Errorf("error %q does not contain %q", err, sub) + } + } +} + +func TestCompareTrees(t *testing.T) { + tests := []struct { + name string + got map[string]string + wantErr []string // substrings the error must contain; nil means no error + }{ + {name: "equal", got: tree(nil, "")}, + { + name: "changed bytes", + got: tree(map[string]string{cliRef: "changed"}, ""), + wantErr: []string{cliRef, "content differs"}, + }, + { + name: "missing file", + got: tree(nil, safetyRef), + wantErr: []string{"missing", safetyRef}, + }, + { + name: "extra file", + got: tree(map[string]string{"references/extra.md": "x"}, ""), + wantErr: []string{"extra", "references/extra.md"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := compareTrees(writeTree(t, baseTree()), writeTree(t, tt.got)) + assertErrContains(t, err, tt.wantErr) + }) + } +} + +// TestValidateStopsAtFirstFailure fails each step in turn and requires the +// error to surface and no later step to run. Call order: python version +// check, venv, pip install, skills-ref validate, codex install, claude install. +func TestValidateStopsAtFirstFailure(t *testing.T) { + steps := []struct { + name string + want string + }{ + {"python version check", pythonRequired}, + {"venv creation", ""}, + {"pip install", ""}, + {"skills-ref validate", ""}, + {"codex install", "codex install"}, + {"claude-code install", "claude-code install"}, + } + for i, step := range steps { + t.Run(step.name, func(t *testing.T) { + root := skillFixture(t) + f := &fakeRunner{t: t, repoRoot: root, failAt: i, copyInstalls: true} + var out bytes.Buffer + err := validate(validationFor(root, f, &out)) + if !errors.Is(err, errBoom) { + t.Fatalf("got %v, want wrapped errBoom", err) + } + if step.want != "" && !strings.Contains(err.Error(), step.want) { + t.Errorf("error %q does not contain %q", err, step.want) + } + if len(f.calls) != i+1 { + t.Errorf("ran %d commands, want %d (stop at failing step)", len(f.calls), i+1) + } + if out.Len() != 0 { + t.Errorf("success output written after failure: %q", out.String()) + } + }) + } +} + +func TestValidateInstalledSkillMissing(t *testing.T) { + root := skillFixture(t) + f := &fakeRunner{t: t, repoRoot: root, failAt: -1, copyInstalls: false} + var out bytes.Buffer + err := validate(validationFor(root, f, &out)) + if err == nil || !strings.Contains(err.Error(), "SKILL.md missing or empty") { + t.Fatalf("got %v, want installed SKILL.md failure", err) + } + if out.Len() != 0 { + t.Errorf("success output written after failure: %q", out.String()) + } +} + +func TestValidateSucceeds(t *testing.T) { + root := skillFixture(t) + f := &fakeRunner{t: t, repoRoot: root, failAt: -1, copyInstalls: true} + var out bytes.Buffer + if err := validate(validationFor(root, f, &out)); err != nil { + t.Fatalf("validate: %v", err) + } + if got := out.String(); got != successMessage+"\n" { + t.Errorf("stdout = %q, want %q", got, successMessage+"\n") + } +}