+
+
+The collection pipeline is the foundation; the payoff is task-specific skill retrieval before the agent acts.
+
+收集和策展是基础，真正的价值是 agent 执行前能拿到与任务匹配的技能。
diff --git a/docs/assets/pipeline.png b/docs/assets/pipeline.png deleted file mode 100644 index 43f3c81..0000000 Binary files a/docs/assets/pipeline.png and /dev/null differ diff --git a/docs/assets/taxonomy.png b/docs/assets/taxonomy.png deleted file mode 100644 index 7eb0cb0..0000000 Binary files a/docs/assets/taxonomy.png and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml index 17c66fd..bb6a7f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,5 +42,5 @@ eval = [ # Keeping the package name `skillcorpus` (not renamed) so existing # `python -m skillcorpus.*` invocations and absolute imports keep working. [tool.setuptools.packages.find] -include = ["skillcorpus*"] - +include = ["skillcorpus", "skillcorpus.*"] +exclude = ["skillcorpus.tests", "skillcorpus.tests.*"] diff --git a/scripts/check_file_sizes.py b/scripts/check_file_sizes.py new file mode 100644 index 0000000..7878d10 --- /dev/null +++ b/scripts/check_file_sizes.py @@ -0,0 +1,126 @@ +"""Block oversized files introduced or enlarged by the current change. + +This mirrors the local ``check-added-large-files`` hook, but covers modified +files too. The check compares the working tree to the merge base, so CI must +check out full history (``fetch-depth: 0``). +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import Path + +MAX_KB = 640 +DEFAULT_BASE = "origin/main" +EXEMPT_PREFIXES: tuple[str, ...] = () +EXEMPT_PATHS = frozenset() + + +class BaseRefError(RuntimeError): + """The comparison base could not be resolved.""" + + +@dataclass(frozen=True) +class Violation: + path: str + size_bytes: int + + +def _repo_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def _git(root: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=root, capture_output=True, text=True) + if result.returncode: + detail = result.stderr.strip() or "unknown error" + raise BaseRefError(f"`git {' '.join(args)}` failed: {detail}") + return result.stdout + + +def default_base_ref() -> str: + base = os.environ.get("GITHUB_BASE_REF", "").strip() + return f"origin/{base}" if base else DEFAULT_BASE + + +def changed_paths(root: Path, base_ref: str) -> list[str]: + merge_base = _git(root, "merge-base", base_ref, "HEAD").strip() + if not merge_base: + raise BaseRefError(f"no merge base between {base_ref} and HEAD") + + changed = _git( + root, + "diff", + "--name-only", + "-z", + "--diff-filter=ACMR", + merge_base, + ) + untracked = _git(root, "ls-files", "--others", "--exclude-standard", "-z") + paths = {path for path in changed.split("\0") if path} + paths.update(path for path in untracked.split("\0") if path) + return sorted(paths) + + +def is_exempt(path: str, prefixes: Sequence[str] = EXEMPT_PREFIXES) -> bool: + normalised = path.replace("\\", "/") + return normalised in EXEMPT_PATHS or any( + normalised.startswith(prefix) for prefix in prefixes + ) + + +def find_violations( + paths: Iterable[str], + *, + root: Path, + max_kb: int = MAX_KB, +) -> list[Violation]: + violations: list[Violation] = [] + for path in paths: + candidate = root / path + if is_exempt(path) or candidate.is_symlink() or not candidate.is_file(): + continue + if candidate.stat().st_size > max_kb * 1024: + violations.append(Violation(path=path, size_bytes=candidate.stat().st_size)) + return violations + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", help="ref to diff against") + args = parser.parse_args(argv) + + root = _repo_root() + base_ref = args.base or default_base_ref() + try: + paths = changed_paths(root, base_ref) + except BaseRefError as exc: + print( + f"Repository file-size check could not run: {exc}\n" + "Fetch the base branch and full history first; CI must use fetch-depth: 0." + ) + return 1 + + violations = find_violations(paths, root=root) + if not violations: + print( + f"Repository file-size check passed ({len(paths)} changed file(s) vs " + f"{base_ref}, ceiling {MAX_KB} KB)." + ) + return 0 + + print( + f"Repository file-size check failed: this change adds or grows files above " + f"{MAX_KB} KB. Store large payloads externally or as a release artifact." + ) + for violation in sorted(violations, key=lambda item: -item.size_bytes): + print(f"- {violation.path}: {violation.size_bytes / 1024:.1f} KB") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_package_contents.py b/scripts/check_package_contents.py new file mode 100644 index 0000000..cc627af --- /dev/null +++ b/scripts/check_package_contents.py @@ -0,0 +1,43 @@ +"""Verify that the root wheel contains only the root distribution's code.""" + +from __future__ import annotations + +import argparse +import zipfile +from collections.abc import Sequence +from pathlib import Path + +FORBIDDEN_PREFIXES = ("skillcorpus_plugin/", "skillcorpus/tests/") + + +def _check_wheel(path: Path) -> list[str]: + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + failures: list[str] = [] + if "skillcorpus/__init__.py" not in names: + failures.append(f"{path.name}: missing skillcorpus/__init__.py") + if not any(name.endswith(".dist-info/licenses/LICENSE") for name in names): + failures.append(f"{path.name}: missing packaged LICENSE") + for prefix in FORBIDDEN_PREFIXES: + leaked = sorted(name for name in names if name.startswith(prefix)) + if leaked: + preview = ", ".join(leaked[:3]) + failures.append(f"{path.name}: must not include {prefix} ({preview})") + return failures + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheel", nargs="+", type=Path) + args = parser.parse_args(argv) + failures = [failure for path in args.wheel for failure in _check_wheel(path)] + if failures: + print("Package contents check failed:") + print("\n".join(f"- {failure}" for failure in failures)) + return 1 + print(f"Package contents check passed ({len(args.wheel)} wheel(s)).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_project_inventory.py b/scripts/check_project_inventory.py new file mode 100644 index 0000000..8fe1db1 --- /dev/null +++ b/scripts/check_project_inventory.py @@ -0,0 +1,170 @@ +"""Keep the supported SkillCorpus projects and public host list in sync. + +The repository contains one core Python package and several independently +installable host integrations. Adding a new integration means updating this +inventory, its manifest, its guide, and both top-level README host tables in +the same review. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path + +import tomllib + +ROOT = Path(__file__).resolve().parent.parent +PLUGIN_ROOT = ROOT / "skillcorpus_plugin" +VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:[a-zA-Z0-9.+-]*)$") + + +@dataclass(frozen=True) +class Project: + label: str + manifest: Path + guide: Path + expected_name: str + format: str + + +PROJECTS = ( + Project( + "core pipeline", + Path("pyproject.toml"), + Path("README.md"), + "skillcorpus", + "toml", + ), + Project( + "Python retrieval engine", + Path("skillcorpus_plugin/engine-python/pyproject.toml"), + Path("skillcorpus_plugin/engine-python/README.md"), + "skillsearch", + "toml", + ), + Project( + "DeepSeek Harness", + Path("skillcorpus_plugin/engine-typescript/package.json"), + Path("skillcorpus_plugin/engine-typescript/README.md"), + "@evermind-ai/dsh-skill-search", + "json", + ), + Project( + "Hermes", + Path("skillcorpus_plugin/plugin-hermes/plugin.yaml"), + Path("skillcorpus_plugin/plugin-hermes/README.md"), + "skillsearch", + "yaml", + ), + Project( + "OpenClaw", + Path("skillcorpus_plugin/plugin-openclaw/package.json"), + Path("skillcorpus_plugin/plugin-openclaw/README.md"), + "@evermind-ai/openclaw-skillsearch", + "json", + ), + Project( + "Raven", + Path("skillcorpus_plugin/plugin-raven/pyproject.toml"), + Path("skillcorpus_plugin/plugin-raven/README.md"), + "skillsearch-raven", + "toml", + ), + Project( + "WorkBuddy", + Path("skillcorpus_plugin/plugin-workbuddy/package.json"), + Path("skillcorpus_plugin/plugin-workbuddy/README.md"), + "@evermind-ai/workbuddy-skillsearch", + "json", + ), +) +HOST_GUIDES = tuple(project.guide.as_posix() for project in PROJECTS[2:]) +EXPECTED_PLUGIN_DIRS = { + ".github", + "engine-python", + "engine-typescript", + "plugin-hermes", + "plugin-openclaw", + "plugin-raven", + "plugin-workbuddy", + "scripts", +} + + +def _read_manifest(project: Project) -> tuple[str | None, str | None, bool | None]: + content = (ROOT / project.manifest).read_text(encoding="utf-8") + if project.format == "toml": + data = tomllib.loads(content)["project"] + return data.get("name"), data.get("version"), None + if project.format == "json": + data = json.loads(content) + return data.get("name"), data.get("version"), data.get("private") + + name = re.search(r"^name:\s*[\"']?([^\"'\s]+)", content, re.M) + version = re.search(r"^version:\s*[\"']?([^\"'\s]+)", content, re.M) + return ( + name.group(1) if name else None, + version.group(1) if version else None, + None, + ) + + +def _check_projects() -> list[str]: + failures: list[str] = [] + for project in PROJECTS: + manifest = ROOT / project.manifest + guide = ROOT / project.guide + if not manifest.is_file(): + failures.append(f"{project.label}: missing manifest {project.manifest}") + continue + if not guide.is_file(): + failures.append(f"{project.label}: missing guide {project.guide}") + name, version, private = _read_manifest(project) + if name != project.expected_name: + failures.append( + f"{project.label}: manifest name {name!r} != {project.expected_name!r}" + ) + if not version or not VERSION_RE.fullmatch(version): + failures.append(f"{project.label}: invalid release version {version!r}") + if project.label == "DeepSeek Harness" and private is not True: + failures.append("DeepSeek Harness: package.json must remain private") + return failures + + +def _check_plugin_dirs() -> list[str]: + actual = {path.name for path in PLUGIN_ROOT.iterdir() if path.is_dir()} + unexpected = sorted(actual - EXPECTED_PLUGIN_DIRS) + missing = sorted(EXPECTED_PLUGIN_DIRS - actual) + failures = [ + f"plugin inventory: undocumented directory {name}" for name in unexpected + ] + failures.extend( + f"plugin inventory: expected directory missing: {name}" for name in missing + ) + return failures + + +def _check_top_level_readmes() -> list[str]: + failures: list[str] = [] + for readme in (ROOT / "README.md", ROOT / "README.zh-CN.md"): + text = readme.read_text(encoding="utf-8") + for guide in HOST_GUIDES: + if guide not in text: + failures.append(f"{readme.name}: host table missing {guide}") + return failures + + +def main() -> int: + failures = _check_projects() + _check_plugin_dirs() + _check_top_level_readmes() + if failures: + print("Project inventory check failed:") + print("\n".join(f"- {failure}" for failure in failures)) + return 1 + print(f"Project inventory check passed ({len(PROJECTS)} maintained projects).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_repo_assets.py b/scripts/check_repo_assets.py new file mode 100644 index 0000000..0d82c15 --- /dev/null +++ b/scripts/check_repo_assets.py @@ -0,0 +1,91 @@ +"""Reject committed images, videos, and asset-style directories. + +README media must be externally hosted (for example, on the project paper, +GitHub user content, or a release asset) and linked into the documentation. +Keeping it out of the source tree prevents repository bloat and makes the +large-file policy enforceable. +""" + +from __future__ import annotations + +import subprocess +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import PurePosixPath + +BLOCKED_DIR_NAMES = frozenset( + {"asset", "assets", "image", "images", "img", "media", "video", "videos"} +) +IMAGE_EXTENSIONS = frozenset( + { + ".avif", + ".bmp", + ".gif", + ".heic", + ".heif", + ".icns", + ".ico", + ".jpeg", + ".jpg", + ".png", + ".svg", + ".tif", + ".tiff", + ".webp", + } +) +VIDEO_EXTENSIONS = frozenset( + {".avi", ".flv", ".m4v", ".mkv", ".mov", ".mp4", ".mpeg", ".mpg", ".webm", ".wmv"} +) + + +@dataclass(frozen=True) +class Violation: + path: str + reason: str + + +def _violation_reason(path: str) -> str | None: + posix_path = PurePosixPath(path.replace("\\", "/")) + if any(part.lower() in BLOCKED_DIR_NAMES for part in posix_path.parts): + return "asset/media directory" + if posix_path.suffix.lower() in IMAGE_EXTENSIONS: + return "image file" + if posix_path.suffix.lower() in VIDEO_EXTENSIONS: + return "video file" + return None + + +def find_violations(paths: Iterable[str]) -> list[Violation]: + return [ + Violation(path=path, reason=reason) + for path in paths + if (reason := _violation_reason(path)) is not None + ] + + +def _tracked_paths() -> list[str]: + result = subprocess.run( + ["git", "ls-files", "-z"], check=True, capture_output=True, text=False + ) + return [raw.decode("utf-8") for raw in result.stdout.split(b"\0") if raw] + + +def main() -> int: + violations = find_violations(_tracked_paths()) + if not violations: + print("Repository asset/media check passed.") + return 0 + + print( + "Repository asset/media check failed.\n" + "Do not commit images, videos, or asset/media directories. Host media " + "externally or in a release artifact, then link to it from the docs.\n" + ) + for violation in violations: + print(f"- {violation.path}: {violation.reason}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())