Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ jobs:
# and runs. Locally a missing toolchain skips those tests; here it must fail
# them, or "the build scans the example on every commit" would quietly stop
# being true.
SENTINEL_REQUIRE_GO: "1"
TOOLGAUGE_REQUIRE_GO: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
# Direction 1: the broker must PASS.
- name: scan the broker
run: |
uv run sentinel scan \
uv run toolgauge scan \
--endpoint http://localhost:8080/mcp \
--token "$SCAN_TOKEN" \
--gate must --format sarif --out broker.sarif --no-color \
Expand All @@ -84,15 +84,15 @@ jobs:
# Without this the job proves only that the harness can say yes.
- name: the non-conformant fixture must fail the gate
run: |
uv run sentinel fixture serve --profile nonconformant &
uv run toolgauge fixture serve --profile nonconformant &
for _ in $(seq 1 30); do
curl -sf -o /dev/null http://127.0.0.1:9000/mcp \
-H 'Mcp-Method: server/discover' \
-d '{"jsonrpc":"2.0","id":1,"method":"server/discover"}' && break
sleep 1
done

if uv run sentinel scan --endpoint http://127.0.0.1:9000/mcp --gate must --no-color; then
if uv run toolgauge scan --endpoint http://127.0.0.1:9000/mcp --gate must --no-color; then
echo "::error::the non-conformant fixture PASSED the MUST gate — the harness is broken"
exit 1
fi
Expand All @@ -102,21 +102,21 @@ jobs:
# "the rules work" from "the rules reject everything".
- name: the conformant fixture must pass the gate
run: |
uv run sentinel fixture serve --profile conformant &
uv run toolgauge fixture serve --profile conformant &
for _ in $(seq 1 30); do
curl -sf -o /dev/null http://127.0.0.1:9001/mcp \
-H 'Mcp-Method: server/discover' \
-d '{"jsonrpc":"2.0","id":1,"method":"server/discover"}' && break
sleep 1
done
uv run sentinel scan --endpoint http://127.0.0.1:9001/mcp --gate must --no-color
uv run toolgauge scan --endpoint http://127.0.0.1:9001/mcp --gate must --no-color

# The deprecation inventory, in both directions too. It never fails a
# gate — a deprecated feature that still works is a migration to plan, not
# a defect — so the assertion is on the CONTENT rather than the exit code.
- name: the broker depends on no deprecated feature
run: |
uv run sentinel deprecations \
uv run toolgauge deprecations \
--endpoint http://localhost:8080/mcp \
--token "$SCAN_TOKEN" --format json --out broker-deprecations.json --no-color
IN_USE=$(python -c "import json,sys; d=json.load(open('broker-deprecations.json')); print(sum(1 for f in d['features'] if f['inUse']))")
Expand All @@ -128,7 +128,7 @@ jobs:

- name: the fixture's deprecations are all found
run: |
uv run sentinel deprecations \
uv run toolgauge deprecations \
--endpoint http://127.0.0.1:9000/mcp \
--format json --out fixture-deprecations.json --no-color
python - <<'PY'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/registry-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
run: |
args=(registry-scan --out-dir scan --concurrency 8 --timeout 10)
if [ -n "$LIMIT" ]; then args+=(--limit "$LIMIT"); fi
sentinel "${args[@]}"
toolgauge "${args[@]}"
cat scan/summary.md >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v4
with:
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# On a GitHub release this publishes the wheel to PyPI as `sentinel-mcp-conformance`,
# On a GitHub release this publishes the wheel to PyPI as `toolgauge`,
# pushes a multi-arch container image to GHCR, signs both with cosign (keyless, using
# this workflow's GitHub OIDC identity, so there is no signing key to leak) and
# attaches a CycloneDX SBOM of the harness to the release.
#
# Verify a release:
# cosign verify ghcr.io/patsypppe/sentinel:<version> \
# cosign verify ghcr.io/patsypppe/toolgauge:<version> \
# --certificate-identity-regexp 'https://github.com/patsypppe/sentinel/.github/workflows/release.yml@.*' \
# --certificate-oidc-issuer https://token.actions.githubusercontent.com
#
Expand Down Expand Up @@ -44,8 +44,8 @@ jobs:
run: |
uv venv /tmp/smoke --python 3.12
uv pip install --python /tmp/smoke dist/*.whl
/tmp/smoke/bin/sentinel version
/tmp/smoke/bin/sentinel-mcp-conformance catalog validate
/tmp/smoke/bin/toolgauge version
/tmp/smoke/bin/toolgauge catalog validate
- uses: actions/upload-artifact@v4
with:
name: dist
Expand Down Expand Up @@ -98,15 +98,15 @@ jobs:
push: true
provenance: true
tags: |
ghcr.io/${{ github.repository_owner }}/sentinel:${{ steps.meta.outputs.version }}
ghcr.io/${{ github.repository_owner }}/sentinel:latest
ghcr.io/${{ github.repository_owner }}/toolgauge:${{ steps.meta.outputs.version }}
ghcr.io/${{ github.repository_owner }}/toolgauge:latest
- name: sign the image by digest
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: cosign sign --yes "ghcr.io/${{ github.repository_owner }}/sentinel@${DIGEST}"
run: cosign sign --yes "ghcr.io/${{ github.repository_owner }}/toolgauge@${DIGEST}"
- name: the image runs
run: |
docker run --rm "ghcr.io/${{ github.repository_owner }}/sentinel:${{ steps.meta.outputs.version }}" version
docker run --rm "ghcr.io/${{ github.repository_owner }}/toolgauge:${{ steps.meta.outputs.version }}" version

attest:
name: sign the wheel, attach the SBOM
Expand Down Expand Up @@ -134,11 +134,11 @@ jobs:
with:
path: harness
format: cyclonedx-json
output-file: sentinel-harness.cdx.json
output-file: toolgauge-harness.cdx.json
upload-artifact: false
upload-release-assets: false
- name: attach to the release
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release upload "$GITHUB_REF_NAME" dist/*.sigstore.json sentinel-harness.cdx.json --clobber
gh release upload "$GITHUB_REF_NAME" dist/*.sigstore.json toolgauge-harness.cdx.json --clobber
22 changes: 11 additions & 11 deletions .pre-commit-hooks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,28 @@
# - repo: https://github.com/patsypppe/sentinel
# rev: v0.4.0
# hooks:
# - id: sentinel-scan
# - id: toolgauge-scan
#
# Point it at a server with a sentinel.toml in your repository (scan.endpoint) or
# the SENTINEL_ENDPOINT environment variable. It runs on pre-push by default: a
# Point it at a server with a toolgauge.toml in your repository (scan.endpoint) or
# the TOOLGAUGE_ENDPOINT environment variable. It runs on pre-push by default: a
# scan makes a few hundred requests and is not something to run on every commit.

# Runs the published image. Needs Docker, nothing else, and cannot drift from the
# version pinned by `rev`.
- id: sentinel-scan
name: sentinel MCP conformance scan
- id: toolgauge-scan
name: toolgauge MCP conformance scan
description: Grade a locally-running MCP server against the current MCP spec and fail on a MUST violation.
entry: ghcr.io/patsypppe/sentinel:latest scan --gate must --no-color
entry: ghcr.io/patsypppe/toolgauge:latest scan --gate must --no-color
language: docker_image
pass_filenames: false
always_run: true
stages: [pre-push, manual]

# For machines that already have `sentinel` installed (pip, pipx or uvx).
- id: sentinel-scan-local
name: sentinel MCP conformance scan (installed)
description: Same scan, using the sentinel command already on PATH.
entry: sentinel scan --gate must --no-color
# For machines that already have `toolgauge` installed (pip, pipx or uvx).
- id: toolgauge-scan-local
name: toolgauge MCP conformance scan (installed)
description: Same scan, using the toolgauge command already on PATH.
entry: toolgauge scan --gate must --no-color
language: system
pass_filenames: false
always_run: true
Expand Down
42 changes: 27 additions & 15 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@

## 0.4.0

### Renamed: Sentinel is now toolgauge

The project was called Sentinel. That name collided in three places: the PyPI package `sentinel`
installs a top-level `sentinel` module, HashiCorp's Sentinel policy tool installs a `sentinel` command,
and `mcp-sentinel` is a different MCP testing project. Everything user-facing is now `toolgauge`: the
PyPI distribution, the import package, the command, the rule namespace (`TOOLGAUGE/...` instead of
`SENTINEL/...`), the config file (`toolgauge.toml`), environment variables (`TOOLGAUGE_*`), the
history service (`toolgauge-server`), the Docker image and the GitHub Action. Baselines written by an
earlier build still load (`sentinelVersion` is read as a fallback); their `SENTINEL/` rule ids no
longer match, so recreate a baseline after upgrading. The reference server's internal names (database,
dev headers, `*.sentinel.local`) and the repository URL are unchanged.

The harness stops being a one-shot grader and becomes something a team runs.

### Measured: the whole MCP registry
Expand All @@ -18,11 +30,11 @@ aggregate as an annotation so it can be read without downloading an artifact.
refined, 7 of 12 after, on a fresh draw; specification findings: 15 of 15 correct). The rule now treats
enums, paging names, flag booleans and unit-suffixed numbers as self-explanatory and accepts quoted
parameter names in prose. Headline: 11% of parameters flagged, an estimated 6 to 7% genuinely missing.
`benchmarks/compare_cisco.py`: Sentinel against Cisco's `mcp-scanner` on a labelled poisoning corpus.
`benchmarks/compare_cisco.py`: toolgauge against Cisco's `mcp-scanner` on a labelled poisoning corpus.

### Fixed: the parameter-documentation rule over-reported

`SENTINEL/STYLE/parameters-are-described` counted only schema descriptions. It now also accepts a
`TOOLGAUGE/STYLE/parameters-are-described` counted only schema descriptions. It now also accepts a
parameter documented in the tool description (`name: ...`, as SDK docstring `Args:` blocks produce), an
object whose fields are all described, and self-evident plain-string names (`path`, `url`, ...).
The benchmark moved from 28% to 17% of parameters as a result.
Expand All @@ -32,11 +44,11 @@ The benchmark moved from 28% to 17% of parameters as a result.
The lint rules only read, so they share one child process instead of starting the server per rule:
4.4x to 6.7x faster with identical results. `benchmarks/chart.py` draws the 27-server result as an SVG.

### Added: `sentinel lint` and stdio servers
### Added: `toolgauge lint` and stdio servers

- `sentinel lint --command "npx -y <server>"` runs the manifest rules against a server started as a
- `toolgauge lint --command "npx -y <server>"` runs the manifest rules against a server started as a
child process (newline-delimited JSON over pipes), or `--endpoint` for HTTP. Most published MCP
servers are stdio programs, which Sentinel could not reach before.
servers are stdio programs, which toolgauge could not reach before.
- `benchmarks/popular_servers.py`: lints 29 published servers; 28 started, 362 tools, 1,349
parameters, 17% documented nowhere.

Expand All @@ -49,17 +61,17 @@ session and run, instead of every rule reporting the server unreachable.
### Added: benchmarks against servers this project did not write

- `benchmarks/official_negatives.py`: runs the broken fixtures from the official conformance repo and
reports which labelled violations Sentinel catches (7 of 7 gradable; one fixture is unusable and one violation has no rule, both listed).
reports which labelled violations toolgauge catches (7 of 7 gradable; one fixture is unusable and one violation has no rule, both listed).
- `benchmarks/sdk_readiness.py`: scans servers built with the official SDKs.
- Reports now say which protocol generation answered, and a scan of a legacy-only server says so at
the top instead of presenting a wall of failures as independent bugs.
- The stub-description threshold is 12 characters, not 20 ("Add two numbers." is a description).

### Added: `sentinel demo` and tool-description lint
### Added: `toolgauge demo` and tool-description lint

- `sentinel demo`: starts two bundled practice servers (one broken, one correct), scans both and
- `toolgauge demo`: starts two bundled practice servers (one broken, one correct), scans both and
prints the verdicts. Works from a plain `uvx` install because the practice servers ship in the wheel.
- `SENTINEL/STYLE/description-states-purpose` and `SENTINEL/STYLE/parameters-are-described`,
- `TOOLGAUGE/STYLE/description-states-purpose` and `TOOLGAUGE/STYLE/parameters-are-described`,
grounded in two 2026 studies of MCP tool descriptions (arXiv 2602.14878, 2602.18914).

### Fixed: rules that were stricter than the specification
Expand All @@ -72,7 +84,7 @@ deprecated and points at its successor.
`server/discover` from version negotiation, so it failed conformant servers.
- Demoted to SHOULD (new ids): `discover-reports-server-info`, `list-changed-advertised-truthfully`
(now INDETERMINATE when the probe cannot tell), `malformed-json-is-parse-error`.
- Moved out of the spec namespace to `SENTINEL/STYLE/`: `tools-have-descriptions`, and the
- Moved out of the spec namespace to `TOOLGAUGE/STYLE/`: `tools-have-descriptions`, and the
`initialize` / `ping` / `logging/setLevel` / `resources/(un)subscribe` removed-method checks. A
dual-era server MAY keep serving them. The `sampling` and `roots` checks were dropped: they remain
functional during their deprecation window.
Expand All @@ -89,8 +101,8 @@ deprecated and points at its successor.

### Added: operating it

- **`sentinel.toml`**: endpoint, gate, timeouts, rule toggles, severity overrides, waivers.
Precedence is CLI > `SENTINEL_*` > file > `extends` > default, tested cell by cell, and every
- **`toolgauge.toml`**: endpoint, gate, timeouts, rule toggles, severity overrides, waivers.
Precedence is CLI > `TOOLGAUGE_*` > file > `extends` > default, tested cell by cell, and every
resolved value records its source in the report.
- **Waivers** with a required `expires_at`. A malformed date is an error; an expired waiver stops
applying and the finding fails again, labelled.
Expand All @@ -115,9 +127,9 @@ deprecated and points at its successor.

`x-mcp-header` constraints (5), `tools-declare-valid-input-schema`, tool-name hygiene (2), tools
capability agreement, `tampered-requeststate-rejected`, `input-required-carries-requests-or-state`,
`input-requests-respect-declared-capabilities`, and the beyond-spec `SENTINEL/SECURITY/`
`input-requests-respect-declared-capabilities`, and the beyond-spec `TOOLGAUGE/SECURITY/`
`tool-description-injection`, `wildcard-scopes`, `manifest-drift`, `tls-posture` and
`SENTINEL/OPS/latency-budget`. A third fixture, `partial`, exercises every `NOT_APPLICABLE` branch.
`TOOLGAUGE/OPS/latency-budget`. A third fixture, `partial`, exercises every `NOT_APPLICABLE` branch.

Every `MCP/` rule cites a heading anchor, checked against the specification source. Three security rules had cited a page that does not exist (`basic/security_best_practices`; the page lives at `docs/2026-07-28/tutorials/security/`) and now point at the real one.

Expand All @@ -133,7 +145,7 @@ SBOM on each release.
- **`mcp/`**: the reusable packages extracted from `broker/internal/` into a module of their own,
with `server.New`, `server.InsecureAnonymous`, an in-memory audit sink, and `mcp/example`, a
one-tool server the build scans with `--gate must` and `--gate security`.
- **`fleet/` (`sentinel-server`)**: ingest, history, posture, drift, expiring waivers, an API and a
- **`fleet/` (`toolgauge-server`)**: ingest, history, posture, drift, expiring waivers, an API and a
server-rendered dashboard with a strict CSP. Memory or Postgres behind one contract suite.
`scan --publish` sends a report and never changes the exit code.

Expand Down
4 changes: 2 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# Sentinel — working instructions
# toolgauge — working instructions

Two products in one repository:

- **`broker/`** — a Go MCP server built natively on the **stateless** MCP `2026-07-28` specification.
- **`harness/`** — `sentinel`, a Python conformance harness that grades *any* MCP server.
- **`harness/`** — `toolgauge`, a Python conformance harness that grades *any* MCP server.

The spec is `docs/HANDOFF.md` (SN-HND-001). **Where this repository and the
[MCP specification](https://modelcontextprotocol.io/specification/2026-07-28/) disagree, the spec wins.**
Expand Down
6 changes: 3 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The scanner as a container: `docker run ghcr.io/patsypppe/sentinel scan --endpoint ...`.
# The scanner as a container: `docker run ghcr.io/patsypppe/toolgauge scan --endpoint ...`.
#
# Two stages, so the image carries the installed wheel and nothing that built it:
# no compiler, no pip cache, no source tree. It runs as an unprivileged user, and
Expand All @@ -18,11 +18,11 @@ FROM python:3.12-slim
LABEL org.opencontainers.image.source="https://github.com/patsypppe/sentinel" \
org.opencontainers.image.description="Conformance harness for MCP 2026-07-28" \
org.opencontainers.image.licenses="MIT"
RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin sentinel
RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin toolgauge
COPY --from=build /dist/*.whl /tmp/
RUN pip install --no-cache-dir /tmp/*.whl && rm -f /tmp/*.whl
USER 10001
WORKDIR /work
ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
ENTRYPOINT ["sentinel"]
ENTRYPOINT ["toolgauge"]
CMD ["--help"]
2 changes: 1 addition & 1 deletion MEASUREMENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ Generated in 7.7s.

**Result:** **100%** (52 of 52 seeded violations detected)

**Value:** MCP/MUST 43/43 (100%); MCP/SHOULD 6/6 (100%); SENTINEL/MUST 2/2 (100%); SENTINEL/SHOULD 1/1 (100%)
**Value:** MCP/MUST 43/43 (100%); MCP/SHOULD 6/6 (100%); TOOLGAUGE/MUST 2/2 (100%); TOOLGAUGE/SHOULD 1/1 (100%)

**Method:** Seeded violations detected ÷ seeded violations. The fixture TAGS each violation in its own source and lists it in `SEEDED_VIOLATIONS`, and a test asserts the two agree — so the denominator describes what the fixture actually does rather than what the scanner happened to find. A scanner supplying its own denominator would be grading its own homework. Reported per severity because a MUST failure and a beyond-spec opinion are not the same claim, and one number over both would let a strong figure hide a weak one.

Expand Down
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Sentinel — two products, one repository.
# toolgauge — two products, one repository.
# broker/ a Go MCP server built natively on MCP 2026-07-28
# harness/ `sentinel`, a conformance harness that grades any MCP server
# harness/ `toolgauge`, a conformance harness that grades any MCP server
#
# Go 1.23+ is required. Homebrew's Go is preferred when present because the
# system Go on some machines is older than the module's floor.
Expand Down Expand Up @@ -84,8 +84,8 @@ demo: ## Run the nine-step demo from docs/HANDOFF.md §13 (needs `make up`)

.PHONY: scan-broker
scan-broker: ## Scan the running broker; exits 0 when it passes the MUST gate
$(UV) run sentinel scan --endpoint http://localhost:8080/mcp --gate must --format text
$(UV) run toolgauge scan --endpoint http://localhost:8080/mcp --gate must --format text

.PHONY: scan-fixture
scan-fixture: ## Scan the non-conformant fixture; expected to exit 1
$(UV) run sentinel scan --endpoint http://localhost:9000/mcp --gate must --format text
$(UV) run toolgauge scan --endpoint http://localhost:9000/mcp --gate must --format text
Loading
Loading