diff --git a/.dockerignore b/.dockerignore index f94699d..52952e0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,10 @@ .git .github -target +.memoryproof .forgetproof -__pycache__ -*.pyc -.venv +target dist build +*.pyc +__pycache__ +conformance/evidence diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b506511..f96f8c2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,5 @@ # Default reviewer for project-wide changes. * @Hughhhhcoder +/.github/ @Hughhhhcoder +/crates/ @Hughhhhcoder +/python/ @Hughhhhcoder diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index dd4ae15..cfe49c5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -12,8 +12,8 @@ body: - type: input id: version attributes: - label: ForgetProof version / 版本 - placeholder: "0.1.0 or commit SHA" + label: MemoryProof version / 版本 + placeholder: "1.0.0, 0.1.0, or commit SHA" validations: required: true - type: input diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9843d7c..d18d219 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,17 @@ updates: directory: "/" schedule: interval: weekly +version: 2 +updates: + - package-ecosystem: cargo + directory: "/" + schedule: + interval: weekly + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1e36aa3..9c94b55 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,8 +9,8 @@ ## Validation / 验证 - [ ] `cargo fmt --all -- --check` -- [ ] `cargo clippy --all-targets --all-features -- -D warnings` -- [ ] `cargo test` +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` +- [ ] `cargo test --workspace` - [ ] `PYTHONPATH=python python -m unittest discover -s python/tests -v` - [ ] Documentation links and redaction boundaries checked / 已检查文档链接和脱敏边界 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b687d56..a6e736a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,40 +4,107 @@ on: push: pull_request: +permissions: + contents: read + jobs: - test: + rust-and-python: + name: Rust + Python contract tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v5 + with: + components: rustfmt, clippy + - uses: actions/setup-python@v7 with: python-version: "3.11" - - run: cargo fmt --all -- --check - - run: cargo clippy --all-targets --all-features -- -D warnings - - run: cargo test - - run: python -m compileall -q python - - run: PYTHONPATH=python python -m unittest discover -s python/tests -v - - name: Validate scenario schema JSON + - name: Check formatting + run: cargo fmt --all -- --check + - name: Lint Rust + run: cargo clippy --workspace --all-targets --all-features -- -D warnings + - name: Test Rust + run: cargo test --workspace -- --test-threads=2 + - name: Compile Python sources + shell: bash + run: | + set -euo pipefail + while IFS= read -r -d '' file; do python -m py_compile "$file"; done < <(find python -name '*.py' -print0) + - name: Test Python adapters + run: PYTHONPATH=python python -m unittest discover -s python/tests -v + - name: Validate scenario schema run: python -m json.tool schemas/scenario.schema.json >/dev/null - - name: Run passing reference scenario - run: cargo run --quiet -- run examples/reference-clean.yml --output .forgetproof/runs - - name: Verify evidence bundle + - name: Validate action metadata + run: ruby -e 'require "yaml"; YAML.load_file("action.yml")' + - name: Run passing reference scenarios shell: bash run: | - bundle=$(find .forgetproof/runs -mindepth 1 -maxdepth 1 -type d | head -1) + set -euo pipefail + cargo run --quiet -- run examples/reference-clean.yml --output .memoryproof/clean + cargo run --quiet -- run examples/isolation-reference.yml --output .memoryproof/isolation + - name: Verify a generated evidence bundle + shell: bash + run: | + set -euo pipefail + bundle=$(find .memoryproof/clean -mindepth 1 -maxdepth 1 -type d | head -1) cargo run --quiet -- verify "$bundle" - - name: Check leaky reference is detected + - name: Check deliberate failures are detected shell: bash run: | set +e - cargo run --quiet -- run examples/reference-leaky.yml --output .forgetproof/leaky-runs - code=$? - test "$code" -eq 1 + cargo run --quiet -- run examples/reference-leaky.yml --output .memoryproof/leaky + leaky=$? + cargo run --quiet -- run examples/reference-overdelete.yml --output .memoryproof/overdelete + overdelete=$? + set -e + test "$leaky" -eq 1 + test "$overdelete" -eq 1 + - name: Build public matrix + run: | + python scripts/build_matrix.py + python -m json.tool site/matrix.json >/dev/null - name: Upload evidence bundles if: always() uses: actions/upload-artifact@v4 with: - name: forgetproof-evidence - path: .forgetproof/ + name: memoryproof-evidence + path: .memoryproof/ if-no-files-found: ignore + + cross-platform: + name: Cross-platform build (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - run: cargo build --workspace --all-targets + - run: cargo test --workspace -- --test-threads=2 + + official-adapter-contracts: + name: Official adapter contracts (local mock) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + - name: Start deterministic provider mocks and run all official adapters + shell: bash + run: | + set -euo pipefail + python -u scripts/mock_remote_backend.py --provider mem0 --port 8888 >/tmp/mem0.log 2>&1 & mem0=$! + python -u scripts/mock_remote_backend.py --provider letta --port 8283 >/tmp/letta.log 2>&1 & letta=$! + python -u scripts/mock_remote_backend.py --provider zep --port 8000 >/tmp/zep.log 2>&1 & zep=$! + trap 'kill "$mem0" "$letta" "$zep" 2>/dev/null || true' EXIT + sleep 1 + NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \ + cargo run --quiet -- run examples/mem0.yml --allow-network --output .memoryproof/mem0 + NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \ + cargo run --quiet -- run examples/letta.yml --allow-network --output .memoryproof/letta + NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 \ + cargo run --quiet -- run examples/zep.yml --allow-network --output .memoryproof/zep diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..d0aa67a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,32 @@ +name: CodeQL / 代码安全分析 + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "23 3 * * 1" + +permissions: + contents: read + security-events: write + +jobs: + analyze: + name: Analyze ${{ matrix.language }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [rust, python] + steps: + - uses: actions/checkout@v7 + - uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + - uses: github/codeql-action/autobuild@v4 + if: matrix.language == 'rust' + - uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d0890a5..0607928 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,8 +1,9 @@ -name: Build conformance matrix / 构建认证矩阵 +name: Publish Memory Assurance Matrix / 发布记忆保证矩阵 on: workflow_dispatch: push: + branches: [main] paths: - "conformance/**" - "scripts/build_matrix.py" @@ -16,19 +17,24 @@ concurrency: cancel-in-progress: true jobs: - matrix: + build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.11" - - run: python scripts/build_matrix.py + - name: Build and verify matrix + run: | + python scripts/build_matrix.py + python -m json.tool site/matrix.json >/dev/null + - uses: actions/configure-pages@v5 - uses: actions/upload-pages-artifact@v3 with: path: site + deploy: - needs: matrix + needs: build runs-on: ubuntu-latest permissions: pages: write diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a142e6..89c72c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release / 发布 +name: Release MemoryProof / 发布 MemoryProof on: push: @@ -9,69 +9,84 @@ permissions: jobs: binaries: - name: Build ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build ${{ matrix.name }} strategy: + fail-fast: false matrix: include: - os: ubuntu-latest - archive: forgetproof-linux-x86_64 - binary: target/release/forgetproof - - os: macos-latest - archive: forgetproof-macos-arm64 - binary: target/release/forgetproof + name: linux-x86_64 + archive: memoryproof-linux-x86_64.tar.gz + shell: bash + - os: macos-13 + name: macos-x86_64 + archive: memoryproof-macos-x86_64.tar.gz + shell: bash + - os: macos-14 + name: macos-arm64 + archive: memoryproof-macos-arm64.tar.gz + shell: bash - os: windows-latest - archive: forgetproof-windows-x86_64 - binary: target/release/forgetproof.exe + name: windows-x86_64 + archive: memoryproof-windows-x86_64.zip + shell: pwsh + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable - - run: cargo build --release - - name: Package Unix binary + - run: cargo build --release --bins + - name: Package Unix binaries if: runner.os != 'Windows' - run: tar -czf ${{ matrix.archive }}.tar.gz -C target/release forgetproof - - name: Package Windows binary + run: tar -czf "${{ matrix.archive }}" -C target/release memoryproof forgetproof + - name: Package Windows binaries if: runner.os == 'Windows' shell: pwsh - run: Compress-Archive -Path target/release/forgetproof.exe -DestinationPath ${{ matrix.archive }}.zip + run: Compress-Archive -Path target/release/memoryproof.exe,target/release/forgetproof.exe -DestinationPath "${{ matrix.archive }}" - uses: actions/upload-artifact@v4 with: - name: ${{ matrix.archive }} - path: | - ${{ matrix.archive }}.tar.gz - ${{ matrix.archive }}.zip + name: binary-${{ matrix.name }} + path: ${{ matrix.archive }} + python: + name: Build Python distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.11" - run: python -m pip install --upgrade build - run: python -m build - uses: actions/upload-artifact@v4 with: - name: forgetproof-adapters-python + name: python-package path: dist/ - publish: - name: Publish GitHub release + + release: + name: Create GitHub release needs: [binaries, python] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v4 with: path: dist - pattern: forgetproof-* + pattern: binary-* merge-multiple: true - - name: Create release and attach artifacts + - uses: actions/download-artifact@v4 + with: + name: python-package + path: dist + - name: Create or update release env: GH_TOKEN: ${{ github.token }} run: | - gh release create "${GITHUB_REF_NAME}" \ - --repo "${GITHUB_REPOSITORY}" \ - --title "ForgetProof ${GITHUB_REF_NAME}" \ - --generate-notes \ - dist/* + if gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + gh release upload "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" dist/* --clobber + else + gh release create "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" \ + --title "MemoryProof ${GITHUB_REF_NAME}" --generate-notes dist/* + fi + container: name: Publish OCI image runs-on: ubuntu-latest @@ -79,18 +94,43 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 - - name: Log in to GHCR - uses: docker/login-action@v3 + - uses: actions/checkout@v7 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push image + - name: Build and push multi-architecture image uses: docker/build-push-action@v6 with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: | - ghcr.io/hughhhhcoder/forgetproof:${{ github.ref_name }} - ghcr.io/hughhhhcoder/forgetproof:latest + ghcr.io/hughhhhcoder/memoryproof:${{ github.ref_name }} + ghcr.io/hughhhhcoder/memoryproof:1 + ghcr.io/hughhhhcoder/memoryproof:latest + labels: | + org.opencontainers.image.source=https://github.com/Hughhhhcoder/MemoryProof + org.opencontainers.image.description=Evidence-driven memory assurance for AI agents + provenance: mode=max + sbom: true + + publish-pypi: + name: Publish PyPI package (opt-in) + if: vars.PUBLISH_PYPI == 'true' && startsWith(github.ref, 'refs/tags/v') + needs: python + runs-on: ubuntu-latest + permissions: + id-token: write + environment: + name: pypi + url: https://pypi.org/p/memoryproof-adapters + steps: + - uses: actions/download-artifact@v4 + with: + name: python-package + path: dist + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000..df59338 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,31 @@ +name: OpenSSF Scorecard + +on: + branch_protection_rule: + schedule: + - cron: "17 2 * * 1" + push: + branches: [main] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + security-events: write + id-token: write + contents: read + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.4.0 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + - uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: results.sarif diff --git a/.gitignore b/.gitignore index 3d03925..d1c77ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target/ /.forgetproof/ +/.memoryproof/ __pycache__/ *.py[cod] .venv/ diff --git a/CHANGELOG.md b/CHANGELOG.md index bbd1b98..001abc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,16 @@ [English](CHANGELOG.md) · [简体中文](CHANGELOG.zh-CN.md) -All notable changes to ForgetProof are documented here. +All notable changes to MemoryProof are documented here. + +## [Unreleased] + +### Changed + +- Rebranded the project as MemoryProof, with ForgetProof retained as the compatibility erasure suite. +- Added the Isolation suite, stable `memoryproof.dev/v1` scenario API, and `memoryproof.adapter/v1` protocol. +- Added target/control subject isolation, explicit `UNKNOWN` semantics, bundle compatibility, and a bilingual offline report. +- Added multi-platform release packaging, multi-architecture OCI builds with SBOM/provenance, CodeQL, Scorecard, and a verified static matrix. ## [0.1.0] - 2026-08-18 diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index 387a0fe..3e97e59 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -2,7 +2,16 @@ [English](CHANGELOG.md) · [简体中文](CHANGELOG.zh-CN.md) -这里记录 ForgetProof 的重要变化。 +这里记录 MemoryProof 的重要变化。 + +## [未发布] + +### 变更 + +- 项目升级为 MemoryProof 总品牌,并保留 ForgetProof 作为兼容的遗忘套件。 +- 增加 Isolation 隔离套件、稳定的 `memoryproof.dev/v1` 场景 API 和 `memoryproof.adapter/v1` 协议。 +- 增加目标/控制主体隔离、明确的 `UNKNOWN` 语义、证据包兼容和双语离线报告。 +- 增加跨平台发行、多架构 OCI 镜像及 SBOM/来源证明、CodeQL、Scorecard 和已验证的静态矩阵。 ## [0.1.0] - 2026-08-18 @@ -18,4 +27,4 @@ ### 安全边界 -ForgetProof 不会声称物理磁盘擦除、服务商日志删除、备份删除或模型权重反学习。无法观察的部分统一保持为 `UNKNOWN` 或 `OUT OF SCOPE`。 +MemoryProof 不会声称物理磁盘擦除、服务商日志删除、备份删除或模型权重反学习。无法观察的部分统一保持为 `UNKNOWN` 或 `OUT OF SCOPE`。 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 764e5a5..74224f4 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,4 +2,4 @@ [English](CODE_OF_CONDUCT.md) · [简体中文](CODE_OF_CONDUCT.zh-CN.md) -We are committed to making participation in ForgetProof a harassment-free experience for everyone. Be respectful, assume good faith, and focus criticism on ideas and evidence rather than people. +We are committed to making participation in MemoryProof a harassment-free experience for everyone. Be respectful, assume good faith, and focus criticism on ideas and evidence rather than people. diff --git a/CODE_OF_CONDUCT.zh-CN.md b/CODE_OF_CONDUCT.zh-CN.md index da92e7b..ca7afa1 100644 --- a/CODE_OF_CONDUCT.zh-CN.md +++ b/CODE_OF_CONDUCT.zh-CN.md @@ -2,4 +2,4 @@ [English](CODE_OF_CONDUCT.md) · [简体中文](CODE_OF_CONDUCT.zh-CN.md) -我们希望 ForgetProof 对所有参与者都保持友善和无骚扰。请尊重他人、善意理解不同观点,并把批评集中在想法和证据上,而不是针对个人。 +我们希望 MemoryProof 对所有参与者都保持友善和无骚扰。请尊重他人、善意理解不同观点,并把批评集中在想法和证据上,而不是针对个人。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3d87c6..2dd168e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,14 +1,81 @@ -# Contributing to ForgetProof +# Contributing to MemoryProof [English](CONTRIBUTING.md) · [简体中文](CONTRIBUTING.zh-CN.md) -The most valuable contributions are new adapters, reproducible erasure scenarios, and tests that expose a concrete residual-memory boundary. +MemoryProof is an assurance project: the most valuable contributions make an observable boundary clearer, safer, and easier to reproduce. New adapters, deterministic scenarios, and tests that expose a concrete residual-memory path are especially welcome. -Before opening a pull request: +## Before you start -1. Run `cargo fmt --all`, `cargo test`, and `python3 -m compileall python`. -2. Keep credentials and real user data out of scenarios and evidence bundles. -3. Add a reference or mocked test for every new adapter capability. -4. Make unsupported observability explicit as `UNKNOWN`; do not weaken a test to make a backend pass. +- Read the [security policy](SECURITY.md). Use synthetic canaries and isolated tenants only. +- Check existing issues and pull requests before starting a large change. +- For a new adapter, document the backend version, endpoint semantics, deletion scope, and which capabilities are genuinely observable. +- Never add API keys, customer data, private URLs, or unredacted response payloads to the repository. -By contributing, you agree to the Developer Certificate of Origin (DCO). Add `Signed-off-by: Your Name ` to commits. +## Local checks + +```bash +cargo fmt --all +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo clippy --workspace --all-targets --all-features -- -D warnings +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo test --workspace -- --test-threads=2 +PYTHONPATH=python python3 -m unittest discover -s python/tests -v +python3 scripts/build_matrix.py +``` + +Also run the relevant reference scenarios: + +```bash +cargo run -- run examples/reference-clean.yml +cargo run -- run examples/isolation-reference.yml +cargo run -- run examples/reference-leaky.yml # expected exit code: 1 +cargo run -- run examples/reference-overdelete.yml # expected exit code: 1 +``` + +## Design rules + +1. **Do not create false passes.** If a backend cannot expose a required boundary, report `UNKNOWN` or `SKIP`. +2. **Prove the precondition.** A delete test must first show that the target can be observed. +3. **Keep target and control fixtures separate.** A test that deletes the control subject is a failure. +4. **Keep the adapter protocol boring.** stdout is NDJSON frames only; diagnostics go to stderr. +5. **Make evidence deterministic.** Freeze generated probes, normalize JSON, and include hashes. +6. **Prefer additive changes.** Preserve the v0.1 loader and compatibility binary when practical. +7. **Keep documentation paired.** New user-facing material belongs in the English README first and in the Simplified Chinese README and relevant guide in the same change. + +## Adding an adapter + +Implement the versioned methods in `python/forgetproof_adapters/`, advertise capabilities honestly, and add a mocked contract test. The adapter must: + +- create resources with a unique `memoryproof` run marker; +- refuse destructive operations against resources it did not create; +- expose request IDs without exposing credentials; +- turn unsupported observability into `UNKNOWN`, never an invented success; +- tolerate asynchronous backends through `settle` rather than guessing that a write is stable. + +If an upstream API has ambiguous delete semantics, document the ambiguity in the adapter and report it in the evidence bundle. + +## Pull requests + +- Use a focused branch and a descriptive title. +- Include the problem, the observable behavior before/after, and the test commands you ran. +- Add or update a scenario when behavior changes. +- Keep generated evidence redacted and small; explain the backend version and reproduction command. +- Keep CI green. Maintainers may ask for a draft PR while the contract is being discussed. + +## Commit sign-off + +By contributing, you agree to the [Developer Certificate of Origin](DCO). Sign each commit with: + +```text +Signed-off-by: Your Name +``` + +For example: + +```bash +git commit -s -m "feat: add an isolation probe" +``` + +## License and conduct + +Contributions are licensed under Apache-2.0. Please follow the [Code of Conduct](CODE_OF_CONDUCT.md) and keep technical disagreement specific, respectful, and evidence-led. diff --git a/CONTRIBUTING.zh-CN.md b/CONTRIBUTING.zh-CN.md index 7405099..df9b1d5 100644 --- a/CONTRIBUTING.zh-CN.md +++ b/CONTRIBUTING.zh-CN.md @@ -1,14 +1,81 @@ -# 为 ForgetProof 贡献代码 +# 参与贡献 MemoryProof [English](CONTRIBUTING.md) · [简体中文](CONTRIBUTING.zh-CN.md) -最有价值的贡献包括:新的记忆后端适配器、可重复的遗忘场景,以及能够发现具体残留数据边界的测试。 +MemoryProof 是一个“记忆保证”项目:最有价值的贡献,是让某个可观察边界更清楚、更安全、更容易复现。欢迎提交新适配器、确定性的测试场景,以及能暴露具体残留记忆路径的测试。 -提交 Pull Request 前,请完成: +## 开始前 -1. 运行 `cargo fmt --all`、`cargo test` 和 `python3 -m compileall python`。 -2. 不要把凭据或真实用户数据放进场景文件和证据包。 -3. 每新增一种适配器能力,都要补充参考测试或模拟 API 测试。 -4. 无法观察的能力必须明确报告为 `UNKNOWN`,不能为了让某个后端通过而降低测试标准。 +- 先阅读[安全策略](SECURITY.zh-CN.md),只使用合成 canary 和隔离租户。 +- 开始大型改动前,先检查已有 Issue 和 Pull Request。 +- 新适配器需要记录后端版本、端点语义、删除范围,以及真正可以观察到的能力。 +- 不要把 API Key、客户数据、私有 URL 或未脱敏响应加入仓库。 -贡献代码即表示你同意 Developer Certificate of Origin(DCO)。请在提交信息中加入 `Signed-off-by: Your Name `。 +## 本地检查 + +```bash +cargo fmt --all +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo clippy --workspace --all-targets --all-features -- -D warnings +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo test --workspace -- --test-threads=2 +PYTHONPATH=python python3 -m unittest discover -s python/tests -v +python3 scripts/build_matrix.py +``` + +同时运行相关参考场景: + +```bash +cargo run -- run examples/reference-clean.yml +cargo run -- run examples/isolation-reference.yml +cargo run -- run examples/reference-leaky.yml # 预期退出码:1 +cargo run -- run examples/reference-overdelete.yml # 预期退出码:1 +``` + +## 设计规则 + +1. **不要制造假通过。** 后端无法暴露必需边界时,报告 `UNKNOWN` 或 `SKIP`。 +2. **先证明前置条件。** 删除测试必须先证明目标确实可以被观察到。 +3. **目标和控制 fixture 分开。** 控制主体被删除就是失败。 +4. **让适配器协议保持简单。** stdout 只允许 NDJSON 帧,诊断日志写 stderr。 +5. **让证据可确定性复现。** 冻结生成的探针、规范化 JSON 并加入哈希。 +6. **优先做向后兼容的增量改动。** 条件允许时保留 v0.1 加载器和兼容二进制。 +7. **双语同步。** 新的用户介绍先写英文 README,同时在中文 README 和相关指南中补上对应内容。 + +## 添加适配器 + +在 `python/forgetproof_adapters/` 中实现版本化方法,诚实声明能力,并添加模拟 API 的契约测试。适配器必须: + +- 使用唯一的 `memoryproof` 运行标记创建资源; +- 拒绝操作不是本次运行创建的资源; +- 记录请求 ID,但不暴露凭据; +- 无法观察的边界报告 `UNKNOWN`,不能伪造成功; +- 通过 `settle` 处理异步后端,不能猜测写入已经稳定。 + +如果上游 API 的删除语义有歧义,请在适配器和证据包中明确记录。 + +## Pull Request + +- 使用专注的分支和清晰的标题。 +- 写明问题、前后可观察行为,以及运行过的测试命令。 +- 行为发生变化时,添加或更新场景。 +- 生成的证据必须脱敏且尽量小,并说明后端版本和复现命令。 +- 保持 CI 通过。维护者可能会先要求以 Draft PR 讨论契约。 + +## 提交签署 + +贡献即表示同意 [Developer Certificate of Origin](DCO)。每次提交添加: + +```text +Signed-off-by: Your Name +``` + +例如: + +```bash +git commit -s -m "feat: add an isolation probe" +``` + +## 许可证与行为 + +贡献内容采用 Apache-2.0 许可证。请遵守[行为准则](CODE_OF_CONDUCT.zh-CN.md),以具体证据讨论技术分歧,并保持尊重。 diff --git a/Cargo.lock b/Cargo.lock index eeefbfb..15071a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,18 +154,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "forgetproof" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "serde", - "serde_json", - "serde_yaml", - "sha2", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -222,6 +210,18 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memoryproof" +version = "1.0.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", + "serde_yaml", + "sha2", +] + [[package]] name = "once_cell_polyfill" version = "1.70.2" diff --git a/Cargo.toml b/Cargo.toml index 7234895..aa21073 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,10 +3,12 @@ members = ["crates/forgetproof"] resolver = "2" [workspace.package] -version = "0.1.0" +version = "1.0.0" edition = "2021" license = "Apache-2.0" -repository = "https://github.com/Hughhhhcoder/ForgetProof" +repository = "https://github.com/Hughhhhcoder/MemoryProof" +homepage = "https://hughhhhcoder.github.io/MemoryProof/" +description = "Test what your AI remembers. Prove what it forgot." [workspace.dependencies] anyhow = "1.0" diff --git a/DCO b/DCO new file mode 100644 index 0000000..2bf3ea0 --- /dev/null +++ b/DCO @@ -0,0 +1,25 @@ +Developer Certificate of Origin +Version 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I have the + right to submit it under the open source license indicated in the file; or + +(b) The contribution is based upon previous work that, to the best of my + knowledge, is covered under an appropriate open source license and I have + the right under that license to submit that work with modifications, + whether created in whole or in part by me, under the same open source + license (unless I am permitted to submit under a different license), as + indicated in the file; or + +(c) The contribution was provided directly to me by some other person who + certified (a), (b) or (c) and I have not modified it. + +(d) I understand and agree that this project and the contribution are public + and that a record of the contribution (including all personal information + I submit with it, including my sign-off) is maintained indefinitely and + may be redistributed consistent with this project or the open source + license(s) involved. + +Signed-off-by: Your Name diff --git a/Dockerfile b/Dockerfile index 7bca4e7..4c28e9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,15 +4,21 @@ WORKDIR /src COPY Cargo.toml Cargo.lock ./ COPY crates ./crates COPY examples ./examples -RUN cargo build --release +RUN cargo build --release --bin memoryproof --bin forgetproof FROM python:3.11-slim -WORKDIR /opt/forgetproof +LABEL org.opencontainers.image.title="MemoryProof" +LABEL org.opencontainers.image.description="Test what your AI remembers. Prove what it forgot." +LABEL org.opencontainers.image.source="https://github.com/Hughhhhcoder/MemoryProof" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +WORKDIR /opt/memoryproof +COPY --from=builder /src/target/release/memoryproof /usr/local/bin/memoryproof COPY --from=builder /src/target/release/forgetproof /usr/local/bin/forgetproof COPY python ./python COPY schemas ./schemas COPY examples ./examples -ENV PYTHONPATH=/opt/forgetproof/python +ENV PYTHONPATH=/opt/memoryproof/python ENV PYTHONDONTWRITEBYTECODE=1 -ENTRYPOINT ["forgetproof"] +ENTRYPOINT ["memoryproof"] diff --git a/README.md b/README.md index 6f7e50b..3bb3ab1 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -# ForgetProof +# MemoryProof -> Prove your AI forgot. +> Prove what your AI forgot. [English](README.md) · [简体中文](README.zh-CN.md)

- CI status - Live conformance matrix - Latest release - Apache-2.0 license + CI status + Live MemoryProof matrix + Latest release + Apache-2.0 license

@@ -16,110 +16,104 @@ Python 3.11 or newer Local-first privacy SHA-256 evidence + v1 preview

- 🧪 Try it in 60 seconds · 🔍 See the proof · 🌐 Open the live matrix + 🧪 Try it in 60 seconds · 🔍 See the proof · 🧭 Open the live matrix

-![ForgetProof hero: a synthetic canary disappears from a memory graph while an evidence ledger verifies the change](assets/forgetproof-hero.png) +![MemoryProof: a synthetic canary leaves an observable memory graph while a verification ring and evidence ledger record the boundary](assets/memoryproof-hero.png) -ForgetProof is an open-source test runner for one uncomfortable question: +MemoryProof is an open-source **memory assurance** toolkit for AI agents. It tests a question that a successful `DELETE` response cannot answer: -> When an AI memory system says “deleted”, can you show that the information is no longer observable? +> Can the information still be observed through any path the backend exposes? -It creates isolated synthetic canaries, asks a memory or Agent backend to erase them, checks raw and derived storage boundaries, and writes evidence that can be rerun locally or in CI. +It creates isolated synthetic canaries, runs deterministic before/after probes, checks raw and derived boundaries, and emits an offline evidence bundle that developers can review, verify, and run in CI. > [!IMPORTANT] -> `DELETE 200 OK` is an API response. ForgetProof tests the stronger claim: **the canary is no longer observable through the paths you can inspect**. +> `DELETE 200 OK` is an API response. MemoryProof tests the stronger, observable claim: **the canary is no longer retrievable through the configured boundary**. -## Why this exists +## Why MemoryProof? -```text -DELETE 200 OK != the canary is no longer observable -``` - -Memory systems may keep information in more than one place: the original record, a summary, an embedding, a graph node, a cache, or an Agent’s working context. ForgetProof makes those boundaries visible and reports what was actually checked. - -It does not store your production memories. It tests a controlled namespace with synthetic data. +Agent memory is rarely one table. A single fact can be copied into a raw record, summary, embedding, graph node, cache, block, or working context. A delete endpoint may remove one representation while another still answers a search query. -## 🧠 The problem in one glance +MemoryProof makes that gap concrete with a controlled experiment: -| What a backend may say | What may still be alive | What ForgetProof checks | +| 🧾 What an API may report | 🕳️ What can remain | 🔬 What MemoryProof records | | --- | --- | --- | -| ✅ `DELETE 200 OK` | 📝 A summary still contains the canary | 🔎 Deterministic before/after probes | -| ✅ The raw row is gone | 🧭 An index, embedding, or graph edge still recalls it | 🧬 Derived-artifact inspection | -| ✅ The Agent cannot see one block | 💬 Another Agent path still leaks the fact | 🤖 Optional black-box Agent query | +| `DELETE 200 OK` | A summary still contains the canary | Deterministic pre/post probes | +| Raw row is gone | An index, vector, or graph edge still recalls it | Derived-artifact inspection | +| One block was detached | Another Agent path can still leak the fact | Optional black-box Agent query | +| “Delete all” was accepted | An unrelated control subject disappeared too | Isolation and scope assertions | -The goal is not to punish a backend for being incomplete. The goal is to make the boundary visible, reproducible, and honest. +The goal is not to assign a vendor a simplistic score. The goal is to make **what was checked, what passed, and what cannot be observed** explicit. -## How it works +## The proof in one screen ```mermaid flowchart LR - A["Scenario + canary"] --> B["Rust runner"] - B --> C["NDJSON adapter"] - C --> D["Mem0 / Letta / Zep"] + A["Scenario + synthetic canaries"] --> B["Rust runner"] + B --> C["Versioned NDJSON adapter"] + C --> D["Mem0 · Letta · Zep"] B --> E["Before probes"] - D --> F["Erase request"] - F --> G["After probes"] + D --> F["Erase / isolate"] + F --> G["Settle + after probes"] E --> H["Evidence bundle"] G --> H - H --> I["HTML / JUnit / CI"] + H --> I["HTML · JUnit · CI · Matrix"] ``` -The runner proves that the canary was visible before deletion, performs a scoped erase, waits for the backend to settle, and checks the target plus a control fixture afterward. Unsupported inspection is reported as `UNKNOWN`, never as a pass. +1. **Seed a canary** that is unique, synthetic, and safe to send to a test namespace. +2. **Prove the precondition**: the target canary is observable before deletion. +3. **Erase or isolate** only resources owned by this run. +4. **Wait for asynchronous writes** to settle; a timeout becomes `UNKNOWN`, not a false pass. +5. **Probe raw, derived, Agent, and control paths** with deterministic rules. +6. **Seal the evidence** with sorted SHA-256 checksums and a bundle hash. -## 🔍 See the difference in 30 seconds +## The difference it makes -| | Without ForgetProof | With ForgetProof | +| | ❌ “Trust the endpoint” | ✅ MemoryProof | | --- | --- | --- | -| **Evidence** | Trust a `200 OK` response | 📦 Keep a verifiable evidence bundle | -| **Coverage** | Check the object you deleted | 🧠 Check raw, summary, vector, graph, cache, and Agent paths that are observable | -| **Regression testing** | Run a one-off script | 🔁 Re-run the same locked scenario in CI | -| **Uncertainty** | Turn missing APIs into “probably fine” | ⚠️ Report `UNKNOWN` when the backend cannot support a claim | +| Evidence | A green HTTP response | A portable bundle with events, results, report, JUnit, and hashes | +| Coverage | The object named in the delete call | Every observable boundary advertised by the adapter | +| Safety | A script may delete the wrong data | Synthetic canaries plus target/control isolation | +| Regression testing | A one-off manual check | A locked scenario in local runs and pull requests | +| Uncertainty | Missing APIs become “probably fine” | `UNKNOWN`, `SKIP`, and `OUT OF SCOPE` stay visible | -```mermaid -flowchart LR - A["Before delete
canary is recallable"] --> B["DELETE 200 OK"] - B --> C{"After: deterministic probes"} - C -->|"clean"| D["✅ PASS
no observable path"] - C -->|"leaky"| E["❌ FAIL
summary / index / graph remains"] -``` +### A failure is a useful result -### A failure is useful - -The deliberately leaky reference backend is part of the demo. It deletes the raw item but leaves a derived artifact, so ForgetProof must fail at the derived layer: +The repository includes a deliberately leaky backend. It deletes the raw item but leaves a derived artifact. MemoryProof must fail at the derived boundary: ```text status: FAIL -profile: FP-Derived +profile: erasure.derived probe: target-derived-after reason: the unique canary is still observable in a derived artifact ``` -That is the product promise in miniature: a vague “memory problem” becomes a named, reviewable, reproducible failure. +This is the core experience: an ambiguous memory concern becomes a named, reproducible, reviewable regression. -## 🚀 Quickstart +## Quickstart Requirements: Rust stable and Python 3.11+. -Choose the path that fits your workflow: - -- 🛠️ **From source:** run the commands below with Rust stable. -- 📥 **Released binary:** download the archive for Linux, macOS, or Windows from [Releases](https://github.com/Hughhhhcoder/ForgetProof/releases). -- 🐳 **Docker:** `docker run --rm -v "$PWD":/workspace ghcr.io/hughhhhcoder/forgetproof:v0.1.0 run /workspace/examples/reference-clean.yml --output /workspace/.forgetproof/runs` +### Run from source ```bash +git clone https://github.com/Hughhhhcoder/MemoryProof.git +cd MemoryProof + cargo run -- adapters list cargo run -- run examples/reference-clean.yml ``` -The clean reference scenario exits `0` and writes a bundle under `.forgetproof/runs/`. Verify it: +The clean reference scenario exits `0` and writes a bundle under `.memoryproof/runs/`. Verify and open it offline: ```bash -cargo run -- verify .forgetproof/runs/ -open .forgetproof/runs//report.html +cargo run -- verify .memoryproof/runs/ +open .memoryproof/runs//report.html # macOS +# xdg-open .memoryproof/runs//report.html # Linux ``` Now run the intentionally leaky backend: @@ -128,105 +122,155 @@ Now run the intentionally leaky backend: cargo run -- run examples/reference-leaky.yml ``` -It exits `1` because the raw item is deleted while a derived artifact remains observable. The report identifies the failing probe and profile. +It exits `1`: the raw item disappears, but a derived artifact remains observable. That failure is expected and demonstrates the check is working. + +### Use the released binary or container + +Download a platform binary from [Releases](https://github.com/Hughhhhcoder/MemoryProof/releases), or run the public image: + +```bash +docker run --rm -v "$PWD":/workspace \ + ghcr.io/hughhhhcoder/memoryproof:1 \ + run /workspace/examples/reference-clean.yml \ + --output /workspace/.memoryproof/runs +``` + +For compatibility with the original v0.1 project, the `forgetproof` binary name and `forgetproof` Python import path remain available during the migration window. + +## Suites and profiles + +MemoryProof is an umbrella with two suites: + +| Suite | What it answers | Profiles | +| --- | --- | --- | +| 🧹 **Erasure** | Did an owned memory boundary stop exposing the target? | `erasure.object`, `erasure.scope`, `erasure.derived`, `erasure.agent` | +| 🧱 **Isolation** | Can one subject be read without leaking another subject’s memory? | `isolation.read`, `isolation.search`, `isolation.agent` | -## 🏅 Conformance profiles +The human-facing legacy names `FP-Object`, `FP-Scope`, `FP-Derived`, and `FP-Agent` are accepted when loading v0.1 scenarios. New scenarios use the stable names above. -| Profile | Plain-English meaning | +Every assertion is one of: + +| Status | Meaning | | --- | --- | -| `FP-Object` | The erased target is gone from the configured recall and list probes. | -| `FP-Scope` | The target is gone while an unrelated control fixture remains. | -| `FP-Derived` | Observable summaries, indexes, graph artifacts, or other derivatives are gone or invalidated. | -| `FP-Agent` | An optional Agent query no longer leaks the unique canary. | +| `PASS` | The required observable check passed. | +| `FAIL` | A probe found the target, a forbidden derivative, or a scope violation. | +| `SKIP` | The selected profile is not required or is intentionally not executed. | +| `UNKNOWN` | The adapter cannot expose enough information to make the claim. | +| `ERROR` | The scenario, protocol, precondition, or execution failed. | -Results are `PASS`, `FAIL`, `SKIP`, `UNKNOWN`, or `ERROR`. There is no misleading single score. +There is no aggregate score. A capability boundary should be readable, not averaged away. -## ✅ What ForgetProof can prove +## What it can—and cannot—prove -| It can show | It cannot claim | +| ✅ It can show | 🚫 It does not claim | | --- | --- | | A target was observable before erase. | Provider logs were deleted. | | A configured API no longer returns the target. | Backups or physical storage were wiped. | | A visible summary, graph, index, or Agent path still leaks it. | A model’s weights were unlearned. | -| A run’s evidence bundle was not modified after creation. | Anything outside the adapter’s observable boundary. | +| A control subject stayed intact—or was accidentally deleted. | Anything outside the adapter’s observable boundary. | +| The evidence bundle was not modified after creation. | The identity of the person who produced the bundle. | + +These boundaries are part of the product, not a footnote. Reports explicitly separate **proved**, **not observed**, and **out of scope**. + +## Supported adapters -## 🔌 Supported adapters +| Adapter | Modes | Coverage in v1 preview | +| --- | --- | --- | +| `reference-clean` | local | Full erasure and isolation reference behavior | +| `reference-leaky` | local | Deliberate derived-artifact residue | +| `reference-overdelete` | local | Deliberate control-subject deletion | +| `mem0` | OSS / platform / cloud | Object, scope, search, optional inspect | +| `letta` | self-hosted / cloud | Agent, archival passage, scope, optional query | +| `zep` | self-hosted / cloud | Episode, user scope, search, graph inspection | -The repository includes dependency-free Python adapters for Mem0, Letta, and Zep, plus clean and deliberately leaky reference backends. Remote access is opt-in: +Remote access is opt-in. Credentials are read from environment variables and are never written into scenarios or evidence: ```bash MEM0_BASE_URL=https://... MEM0_API_KEY=... \ cargo run -- run examples/mem0.yml --allow-network ``` -Credentials stay in environment variables. Endpoint paths can be overridden with `endpoint_*` adapter settings for self-hosted or version-specific deployments. +Adapters communicate with the Rust runner through the versioned `memoryproof.adapter/v1` NDJSON protocol. Third-party adapters can implement the same contract without linking to the Rust binary. -## 📦 Evidence bundle +## Evidence you can review in a pull request -Each run emits a small, offline-readable evidence package: +Each run emits an offline-readable package: ```text -manifest.json run metadata and protocol version -scenario.lock.json redacted scenario snapshot +manifest.json format, run, adapter, protocol, and bundle hash +scenario.lock.json redacted, frozen scenario snapshot events.ndjson ordered method-level journal -results.json machine-readable assertions -report.html single-file human report -junit.xml CI test report -checksums.sha256 per-file integrity hashes +results.json machine-readable assertions and profile states +report.html single-file bilingual report +junit.xml CI-native test report +checksums.sha256 per-file SHA-256 integrity list bundle.hash hash of the checksum manifest ``` -Payloads are redacted by default to hashes, lengths, and structural summaries. `--allow-network` does not change that privacy policy. +By default, payloads are reduced to hashes, lengths, types, and safe structural summaries. `--allow-network` authorizes a remote backend; it does not disable redaction. -## 🤖 Use it in CI +## GitHub Actions -The repository provides a Docker-based GitHub Action. A scenario can fail a pull request when an erasure regression is detected and upload the evidence bundle for review. +The repository ships a Docker-based action that uploads the evidence bundle even when the test fails: ```yaml -name: Memory erasure +name: Memory assurance -on: [pull_request] +on: + pull_request: jobs: - forgetproof: + memoryproof: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: Hughhhhcoder/ForgetProof@v0.1.0 + - uses: Hughhhhcoder/MemoryProof@v1 with: scenario: examples/reference-clean.yml ``` -## 🧩 Adapter protocol +Use a leaky or vendor-specific scenario in a separate job when you want the PR to fail on a known regression. The action exposes `bundle-path`, `status`, and `exit-code` outputs and writes a short job summary. -The Rust runner starts one adapter process per run and communicates over stdout using `forgetproof.adapter/v1alpha1` NDJSON. stdout is reserved for protocol frames; diagnostics go to stderr. +## Scenario and adapter contract -Supported methods are `hello`, `capabilities`, `prepare`, `ingest`, `settle`, `probe`, `erase`, `inspect`, `agent_query`, `cleanup`, and `close`. A third-party adapter only needs to implement this protocol and advertise its capabilities. +The stable scenario API is `memoryproof.dev/v1`. A scenario contains: -## 🗺️ Learn more +- an adapter and non-sensitive configuration references; +- isolated target and control subjects; +- synthetic target/control fixtures; +- settle policy for asynchronous backends; +- an erase intent such as `object_delete` or `subject_erase`; +- deterministic probes and selected profiles; +- a privacy policy for redacted evidence. -- [中文说明](README.zh-CN.md) -- [Scenario schema](schemas/scenario.schema.json) -- [Contributing guide](CONTRIBUTING.md) · [中文贡献指南](CONTRIBUTING.zh-CN.md) -- [Security policy](SECURITY.md) · [中文安全策略](SECURITY.zh-CN.md) -- [Code of conduct](CODE_OF_CONDUCT.md) · [中文行为准则](CODE_OF_CONDUCT.zh-CN.md) -- [Public conformance submissions](conformance/README.md) -- [Conformance matrix](site/index.html) -- [Changelog](CHANGELOG.md) -- [Container images on GHCR](https://github.com/Hughhhhcoder/ForgetProof/pkgs/container/forgetproof) +The adapter protocol methods are `hello`, `capabilities`, `prepare`, `ingest`, `settle`, `probe`, `erase`, `inspect`, `agent_query`, `cleanup`, and `close`. stdout is reserved for protocol frames; adapter logs go to stderr. A capability that is not implemented must become `SKIP` or `UNKNOWN`, never a fabricated pass. -## Development +## Build and contribute ```bash cargo fmt --all -cargo clippy --all-targets --all-features -- -D warnings -cargo test +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo clippy --workspace --all-targets --all-features -- -D warnings +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo test --workspace -- --test-threads=2 PYTHONPATH=python python3 -m unittest discover -s python/tests -v -python3 -m compileall python ``` -The default test suite is local and credential-free. Live Mem0, Letta, and Zep checks belong in a separately authorized workflow. +Please read [CONTRIBUTING.md](CONTRIBUTING.md), [SECURITY.md](SECURITY.md), and their [中文版本](CONTRIBUTING.zh-CN.md) before opening a pull request. MemoryProof is Apache-2.0 licensed and follows the DCO sign-off workflow. + +## Learn more + +- 🌐 [Live Memory Assurance Matrix](https://hughhhhcoder.github.io/MemoryProof/) +- 🧪 [Public conformance evidence](conformance/README.md) +- 📐 [Scenario schema](schemas/scenario.schema.json) +- 🏗️ [Architecture and trust boundaries](docs/architecture.md) · [中文架构说明](docs/architecture.zh-CN.md) +- 🧭 [中文说明](README.zh-CN.md) +- 🤝 [Contributing](CONTRIBUTING.md) · [中文贡献指南](CONTRIBUTING.zh-CN.md) +- 🛡️ [Security policy](SECURITY.md) · [中文安全策略](SECURITY.zh-CN.md) +- 📜 [Changelog](CHANGELOG.md) +- 📦 [Releases](https://github.com/Hughhhhcoder/MemoryProof/releases) +- 🐳 [Container packages](https://github.com/Hughhhhcoder/MemoryProof/pkgs/container/memoryproof) -## License +## Migration from ForgetProof -Apache-2.0. See [LICENSE](LICENSE). +MemoryProof is the new umbrella name for the project formerly published as ForgetProof. The v0.1 evidence format and compatibility binary remain readable, while new scenarios and releases use `memoryproof.dev/v1` and the `memoryproof` command. If you have an old GitHub Action reference, update `Hughhhhcoder/ForgetProof@v0.1.0` to `Hughhhhcoder/MemoryProof@v1`; GitHub does not redirect Action references after a repository rename. diff --git a/README.zh-CN.md b/README.zh-CN.md index 999596c..c85dd3e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,232 +1,276 @@ -# ForgetProof +# MemoryProof -> 证明你的 AI 真的忘了。 +> 用证据证明 AI 忘记了什么。 [English](README.md) · [简体中文](README.zh-CN.md)

- 持续集成状态 - 在线认证矩阵 - 最新版本 - Apache-2.0 许可证 + CI 状态 + 在线记忆保证矩阵 + 最新版本 + Apache-2.0 许可证

Rust stable - Python 3.11+ + Python 3.11 及以上 本地优先隐私 SHA-256 证据 + v1 预览

- 🧪 60 秒试用 · 🔍 查看证明效果 · 🌐 打开在线矩阵 + 🧪 60 秒上手 · 🔍 看懂证明过程 · 🧭 打开在线矩阵

-![ForgetProof 主视觉:合成 canary 从记忆图谱中消失,同时由证据账本确认变化](assets/forgetproof-hero.png) +![MemoryProof:合成 canary 离开可观察的记忆图谱,验证环和证据账本记录边界](assets/memoryproof-hero.png) -ForgetProof 是一个开源测试工具,用来回答一个容易被忽略的问题: +MemoryProof 是面向 AI Agent 的开源 **Memory Assurance(记忆保证)** 工具。它验证一个成功的 `DELETE` 响应无法回答的问题: -> 当 AI 记忆系统说“已删除”时,你能不能证明这条信息已经无法被观察到? +> 通过后端暴露的其他路径,还能不能观察到这条信息? -它会创建隔离的合成 canary,让记忆系统或 Agent 执行删除,再检查原始存储和衍生存储边界,最后生成可以在本地或 CI 中重复执行的证据。 +它创建隔离的合成 canary,执行确定性的删除前后探针,检查原始与衍生数据边界,并输出可以离线审阅、校验和在 CI 中重复运行的证据包。 > [!IMPORTANT] -> `DELETE 200 OK` 只是 API 的响应。ForgetProof 检查的是更强的结论:**在可观察的路径上,canary 是否真的已经无法被召回**。 +> `DELETE 200 OK` 只是 API 响应。MemoryProof 验证的是更强、但明确限定在可观察边界内的结论:**通过已配置的路径已经无法召回这条 canary。** -## 为什么需要它 +## 为什么需要 MemoryProof? -```text -DELETE 200 OK != canary 已经无法被召回 -``` - -记忆系统可能把信息保存在多个地方:原始记录、摘要、向量、知识图谱节点、缓存,甚至 Agent 当前上下文。ForgetProof 会把这些边界显式展示出来,并只报告实际检查过的内容。 - -它不会保存你的生产记忆,只会在隔离命名空间中使用合成数据进行测试。 +Agent 记忆很少只有一张表。一条事实可能被复制到原始记录、摘要、向量、图节点、缓存、block 或工作上下文里。删除接口可能只移除一种表示,而另一个搜索路径仍然可以回答相关问题。 -## 🧠 一眼看懂问题 +MemoryProof 用一次受控实验把这个差距变得清晰可见: -| 后端可能说 | 实际可能还存在 | ForgetProof 检查 | +| 🧾 API 可能报告 | 🕳️ 实际仍可能存在 | 🔬 MemoryProof 记录 | | --- | --- | --- | -| ✅ `DELETE 200 OK` | 📝 摘要里仍然包含 canary | 🔎 删除前后确定性探针 | -| ✅ 原始记录没了 | 🧭 索引、向量或图边仍然可以召回 | 🧬 衍生工件检查 | -| ✅ Agent 看不到某个 block | 💬 另一条 Agent 路径仍然泄露事实 | 🤖 可选黑盒 Agent 查询 | +| `DELETE 200 OK` | 摘要里仍然包含 canary | 确定性的删除前/后探针 | +| 原始行已经消失 | 索引、向量或图边仍能召回 | 衍生工件检查 | +| 某个 block 已 detach | 另一条 Agent 路径仍然泄露事实 | 可选黑盒 Agent 查询 | +| “全部删除”已被接受 | 不相关的控制主体也被误删 | 隔离与范围断言 | -目标不是惩罚能力不完整的后端,而是把边界变得清楚、可复现、诚实。 +目标不是给供应商打一个容易误导的总分,而是把**检查了什么、通过了什么、哪些无法观察**讲清楚。 -## 工作方式 +## 一屏看懂证明过程 ```mermaid flowchart LR - A["场景 + canary"] --> B["Rust 测试引擎"] - B --> C["NDJSON 适配器"] - C --> D["Mem0 / Letta / Zep"] + A["场景 + 合成 canary"] --> B["Rust 运行器"] + B --> C["版本化 NDJSON 适配器"] + C --> D["Mem0 · Letta · Zep"] B --> E["删除前探针"] - D --> F["删除请求"] - F --> G["删除后探针"] + D --> F["删除 / 隔离"] + F --> G["等待稳定 + 删除后探针"] E --> H["证据包"] G --> H - H --> I["HTML / JUnit / CI"] + H --> I["HTML · JUnit · CI · 矩阵"] ``` -测试引擎先证明 canary 在删除前确实可以被召回,然后执行范围受控的删除,等待后端稳定,最后检查目标 canary 和控制 canary。如果某个存储边界无法观察,结果会写成 `UNKNOWN`,不会被当成通过。 +1. **写入 canary**:它必须唯一、合成且适合发送到测试命名空间。 +2. **证明前置条件**:删除前确实能观察到目标 canary。 +3. **删除或隔离**:只操作本次运行拥有的资源。 +4. **等待异步写入稳定**:超时会变成 `UNKNOWN`,不会被伪装成通过。 +5. **检查原始、衍生、Agent 和控制路径**:使用确定性规则判定。 +6. **封存证据**:用排序后的 SHA-256 清单和 bundle hash 固化结果。 -## 🔍 30 秒看懂差异 +## 它带来的实际差别 -| | 没有 ForgetProof | 使用 ForgetProof | +| | ❌ “相信接口就好” | ✅ MemoryProof | | --- | --- | --- | -| **证据** | 相信 `200 OK` | 📦 保存可验证的证据包 | -| **覆盖范围** | 只检查被删除的对象 | 🧠 检查可观察的原始记录、摘要、向量、图谱、缓存和 Agent 路径 | -| **回归测试** | 写一次性脚本 | 🔁 在 CI 中反复执行同一个锁定场景 | -| **不确定性** | 把缺少 API 当成“应该没问题” | ⚠️ 能力不足时明确报告 `UNKNOWN` | +| 证据 | 一个绿色的 HTTP 响应 | 包含事件、结果、报告、JUnit 和哈希的可携带证据包 | +| 覆盖范围 | 删除接口参数里的那个对象 | 适配器声明的每个可观察边界 | +| 安全性 | 一段脚本可能误删真实数据 | 合成 canary + 目标/控制隔离 | +| 回归测试 | 一次性的手工检查 | 本地和 Pull Request 中可重复运行的锁定场景 | +| 不确定性 | 缺少 API 就“默认没问题” | 保留 `UNKNOWN`、`SKIP` 和 `OUT OF SCOPE` | -```mermaid -flowchart LR - A["删除前
canary 可以被召回"] --> B["DELETE 200 OK"] - B --> C{"删除后:确定性探针"} - C -->|"clean"| D["✅ PASS
没有可观察路径"] - C -->|"leaky"| E["❌ FAIL
摘要 / 索引 / 图谱仍存在"] -``` +### 失败也是有价值的结果 -### 失败本身就是价值 - -故意泄漏的参考后端是项目演示的一部分。它删除了原始记录,却留下衍生工件,因此 ForgetProof 必须在衍生层失败: +仓库包含一个故意泄漏的参考后端。它会删除原始项目,但留下衍生工件,因此 MemoryProof 必须在衍生边界失败: ```text status: FAIL -profile: FP-Derived +profile: erasure.derived probe: target-derived-after -reason: 唯一 canary 仍然可以从衍生工件中被观察到 +reason: the unique canary is still observable in a derived artifact ``` -这就是产品承诺的缩影:把模糊的“记忆问题”变成有名称、可审阅、可复现的失败。 - -## 🚀 快速开始 +这就是项目的核心体验:把模糊的“记忆可能没删干净”,变成可命名、可复现、可审阅的回归问题。 -需要 Rust stable 和 Python 3.11+。 +## 快速开始 -可以按你的工作方式选择入口: +要求:Rust stable、Python 3.11 及以上。 -- 🛠️ **源码运行:** 使用 Rust stable 执行下面的命令。 -- 📥 **下载二进制:** 从 [Releases](https://github.com/Hughhhhcoder/ForgetProof/releases) 下载 Linux、macOS 或 Windows 压缩包。 -- 🐳 **Docker:** `docker run --rm -v "$PWD":/workspace ghcr.io/hughhhhcoder/forgetproof:v0.1.0 run /workspace/examples/reference-clean.yml --output /workspace/.forgetproof/runs` +### 从源码运行 ```bash +git clone https://github.com/Hughhhhcoder/MemoryProof.git +cd MemoryProof + cargo run -- adapters list cargo run -- run examples/reference-clean.yml ``` -clean 参考后端会以退出码 `0` 结束,并在 `.forgetproof/runs/` 下生成证据包。验证它: +clean 参考场景会以 `0` 退出,并把证据包写入 `.memoryproof/runs/`。可以离线校验并打开报告: ```bash -cargo run -- verify .forgetproof/runs/ -open .forgetproof/runs//report.html +cargo run -- verify .memoryproof/runs/ +open .memoryproof/runs//report.html # macOS +# xdg-open .memoryproof/runs//report.html # Linux ``` -再运行故意留下衍生数据的后端: +再运行故意泄漏的后端: ```bash cargo run -- run examples/reference-leaky.yml ``` -它会以退出码 `1` 结束,因为原始记录虽然被删除,但衍生工件仍然可观察。报告会指出失败的探针和认证档案。 +它会以 `1` 退出:原始项目消失了,但衍生工件仍可观察。这个失败是预期的,说明检查确实捕捉到了问题。 + +### 使用发行版二进制或容器 + +可以从 [Releases](https://github.com/Hughhhhcoder/MemoryProof/releases) 下载 Linux、macOS 或 Windows 二进制,也可以运行公开容器: + +```bash +docker run --rm -v "$PWD":/workspace \ + ghcr.io/hughhhhcoder/memoryproof:1 \ + run /workspace/examples/reference-clean.yml \ + --output /workspace/.memoryproof/runs +``` + +为了兼容最初的 v0.1 项目,迁移窗口内仍保留 `forgetproof` 二进制名称和 `forgetproof` Python 导入路径。 + +## 套件与认证档案 + +MemoryProof 是一个包含两个套件的总项目: + +| 套件 | 它回答的问题 | 档案 | +| --- | --- | --- | +| 🧹 **Erasure(遗忘)** | 一个受本次运行拥有的记忆边界是否不再暴露目标? | `erasure.object`、`erasure.scope`、`erasure.derived`、`erasure.agent` | +| 🧱 **Isolation(隔离)** | 读取一个主体时,是否会泄露另一个主体的记忆? | `isolation.read`、`isolation.search`、`isolation.agent` | -## 🏅 认证档案 +加载 v0.1 场景时仍接受 `FP-Object`、`FP-Scope`、`FP-Derived`、`FP-Agent` 这些旧名称;新场景使用上面的稳定名称。 -| 档案 | 通俗解释 | +每个断言只会有以下状态: + +| 状态 | 含义 | | --- | --- | -| `FP-Object` | 目标数据已经无法通过配置好的召回和列表探针找到。 | -| `FP-Scope` | 目标数据消失,同时不相关的控制数据仍然存在。 | -| `FP-Derived` | 可观察的摘要、索引、图谱工件和其他衍生数据已经消失或失效。 | -| `FP-Agent` | 可选的 Agent 查询不再泄露唯一 canary。 | +| `PASS` | 必需的可观察检查通过。 | +| `FAIL` | 探针找到了目标、禁止的衍生数据,或发现范围违规。 | +| `SKIP` | 当前档案不是必需项,或被明确跳过。 | +| `UNKNOWN` | 适配器没有暴露足够信息作出结论。 | +| `ERROR` | 场景、协议、前置条件或执行失败。 | -结果统一为 `PASS`、`FAIL`、`SKIP`、`UNKNOWN` 或 `ERROR`,不使用容易误导的单一总分。 +这里没有综合分数。能力边界应该被读懂,而不是被平均掉。 -## ✅ ForgetProof 能证明什么 +## 它能证明什么,不能证明什么 -| 它可以展示 | 它不会声称 | +| ✅ 它可以说明 | 🚫 它不会声称 | | --- | --- | -| 目标数据在删除前确实可以被观察到。 | 服务商日志已经删除。 | -| 配置好的 API 不再返回目标数据。 | 备份或物理存储已经被擦除。 | -| 摘要、图谱、索引或 Agent 路径仍然泄露目标数据。 | 模型权重已经完成反学习。 | -| 证据包创建后没有被修改。 | 适配器观察边界之外的任何事情。 | +| 删除前确实能观察到目标。 | 供应商日志已经删除。 | +| 已配置 API 不再返回目标。 | 备份或物理存储已经擦除。 | +| 可观察的摘要、图、索引或 Agent 路径仍在泄漏。 | 模型权重已经完成反学习。 | +| 控制主体保持完整,或被意外误删。 | 适配器观察边界之外的任何事情。 | +| 证据包创建后没有被修改。 | 证据创建者的身份。 | + +这些边界不是脚注,而是产品的一部分。报告会明确区分**已证明、未观察到和范围之外**。 + +## 适配器 -## 🔌 支持的适配器 +| 适配器 | 模式 | v1 预览覆盖 | +| --- | --- | --- | +| `reference-clean` | 本地 | 完整遗忘和隔离参考行为 | +| `reference-leaky` | 本地 | 故意遗留衍生工件 | +| `reference-overdelete` | 本地 | 故意删除控制主体 | +| `mem0` | OSS / platform / cloud | 对象、范围、搜索、可选检查 | +| `letta` | self-hosted / cloud | Agent、archival passage、范围、可选查询 | +| `zep` | self-hosted / cloud | episode、user 范围、搜索、图检查 | -仓库包含 Mem0、Letta 和 Zep 的无额外依赖 Python 适配器,以及 clean/leaky 两个参考后端。远程访问默认关闭,必须显式开启: +访问远程后端必须显式授权。凭据只从环境变量读取,绝不会写入场景或证据包: ```bash MEM0_BASE_URL=https://... MEM0_API_KEY=... \ cargo run -- run examples/mem0.yml --allow-network ``` -凭据只从环境变量读取。对于自托管或版本不同的部署,可以通过适配器配置中的 `endpoint_*` 覆盖接口路径。 +适配器通过版本化的 `memoryproof.adapter/v1` NDJSON 协议与 Rust 运行器通信。第三方适配器无需链接 Rust 二进制,也可以实现同一契约。 -## 📦 证据包 +## 可以放进 Pull Request 审阅的证据 -每次运行都会生成一个可以离线打开的小型证据包: +每次运行都会生成一个可离线打开的包: ```text -manifest.json 运行元数据和协议版本 -scenario.lock.json 脱敏后的场景快照 -events.ndjson 按顺序记录的方法调用日志 -results.json 机器可读的断言结果 -report.html 单文件人类可读报告 -junit.xml CI 测试报告 -checksums.sha256 每个文件的完整性哈希 -bundle.hash 校验清单的哈希 +manifest.json 格式、运行、适配器、协议与 bundle hash +scenario.lock.json 脱敏后的冻结场景快照 +events.ndjson 按顺序记录的方法级事件 +results.json 机器可读的断言与档案状态 +report.html 单文件双语报告 +junit.xml CI 原生测试报告 +checksums.sha256 每个文件的 SHA-256 完整性清单 +bundle.hash 清单本身的哈希 ``` -默认只记录哈希、长度和结构摘要,不记录原始 payload。开启 `--allow-network` 不会改变这个隐私策略。 +默认情况下,内容会被缩减为哈希、长度、类型和安全的结构摘要。`--allow-network` 只授权访问远程后端,不会关闭脱敏策略。 -## 🤖 在 CI 中使用 +## GitHub Actions -仓库提供 Docker 版 GitHub Action。当遗忘回归测试失败时,它可以阻止 Pull Request,并上传证据包供审阅。 +仓库提供一个 Docker Action,即使测试失败也会上传证据包: ```yaml -name: Memory erasure +name: Memory assurance -on: [pull_request] +on: + pull_request: jobs: - forgetproof: + memoryproof: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: Hughhhhcoder/ForgetProof@v0.1.0 + - uses: Hughhhhcoder/MemoryProof@v1 with: scenario: examples/reference-clean.yml ``` -## 🧩 适配器协议 +如果希望某个已知回归阻断 PR,可以在独立 job 中运行 leaky 或供应商场景。Action 暴露 `bundle-path`、`status` 和 `exit-code` 输出,并在 Job Summary 中写入简要结果。 -Rust 测试引擎每次运行启动一个独立适配器进程,通过 stdout 使用 `forgetproof.adapter/v1alpha1` NDJSON 通信。stdout 只允许输出协议帧,诊断日志写入 stderr。 +## 场景与适配器契约 -支持的方法包括 `hello`、`capabilities`、`prepare`、`ingest`、`settle`、`probe`、`erase`、`inspect`、`agent_query`、`cleanup` 和 `close`。第三方适配器只需要实现该协议并声明自己的能力。 +稳定场景 API 是 `memoryproof.dev/v1`。一个场景包含: -## 🗺️ 继续阅读 +- 适配器与非敏感配置引用; +- 隔离的目标主体和控制主体; +- 合成目标/控制 fixture; +- 异步后端的 settle 策略; +- `object_delete`、`subject_erase` 等删除意图; +- 确定性的探针与所选档案; +- 证据脱敏策略。 -- [English README](README.md) -- [场景 Schema](schemas/scenario.schema.json) -- [贡献指南](CONTRIBUTING.zh-CN.md) · [English contributing guide](CONTRIBUTING.md) -- [安全策略](SECURITY.zh-CN.md) · [English security policy](SECURITY.md) -- [行为准则](CODE_OF_CONDUCT.zh-CN.md) · [English code of conduct](CODE_OF_CONDUCT.md) -- [公开认证结果提交说明](conformance/README.zh-CN.md) -- [认证矩阵](site/index.html) -- [变更记录](CHANGELOG.zh-CN.md) -- [GHCR 容器镜像](https://github.com/Hughhhhcoder/ForgetProof/pkgs/container/forgetproof) +适配器协议方法包括 `hello`、`capabilities`、`prepare`、`ingest`、`settle`、`probe`、`erase`、`inspect`、`agent_query`、`cleanup` 和 `close`。stdout 只允许协议帧,适配器日志必须写 stderr。没有实现的能力必须报告 `SKIP` 或 `UNKNOWN`,不能伪造通过。 -## 开发 +## 开发与贡献 ```bash cargo fmt --all -cargo clippy --all-targets --all-features -- -D warnings -cargo test +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo clippy --workspace --all-targets --all-features -- -D warnings +CARGO_NET_OFFLINE=true CARGO_TARGET_DIR=/tmp/memoryproof-target \ + cargo test --workspace -- --test-threads=2 PYTHONPATH=python python3 -m unittest discover -s python/tests -v -python3 -m compileall python ``` -默认测试完全在本地运行,不需要凭据。Mem0、Letta 和 Zep 的真实服务测试应该放在单独授权的工作流中。 +提交 Pull Request 前请阅读 [英文贡献指南](CONTRIBUTING.md)、[英文安全策略](SECURITY.md) 及其[中文版本](CONTRIBUTING.zh-CN.md)。MemoryProof 使用 Apache-2.0 许可证,并要求按 DCO 签署提交。 + +## 更多资料 + +- 🌐 [在线 Memory Assurance 矩阵](https://hughhhhcoder.github.io/MemoryProof/) +- 🧪 [公开认证证据](conformance/README.zh-CN.md) +- 📐 [场景 Schema](schemas/scenario.schema.json) +- 🏗️ [架构与信任边界](docs/architecture.zh-CN.md) · [English](docs/architecture.md) +- 🧭 [English README](README.md) +- 🤝 [贡献指南](CONTRIBUTING.zh-CN.md) · [English](CONTRIBUTING.md) +- 🛡️ [安全策略](SECURITY.zh-CN.md) · [English](SECURITY.md) +- 📜 [变更记录](CHANGELOG.zh-CN.md) +- 📦 [发行版](https://github.com/Hughhhhcoder/MemoryProof/releases) +- 🐳 [GHCR 容器包](https://github.com/Hughhhhcoder/MemoryProof/pkgs/container/memoryproof) -## 许可证 +## 从 ForgetProof 迁移 -Apache-2.0,详见 [LICENSE](LICENSE)。 +MemoryProof 是原 ForgetProof 项目的新总品牌。v0.1 证据格式和兼容二进制仍然可以读取;新场景和发行版使用 `memoryproof.dev/v1` 与 `memoryproof` 命令。如果之前使用过 GitHub Action,请把 `Hughhhhcoder/ForgetProof@v0.1.0` 更新为 `Hughhhhcoder/MemoryProof@v1`;GitHub 在仓库改名后不会自动重定向 Action 引用。 diff --git a/SECURITY.md b/SECURITY.md index 73cf36c..9efdd4c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,8 +2,29 @@ [English](SECURITY.md) · [简体中文](SECURITY.zh-CN.md) -ForgetProof can issue destructive deletion requests to a configured memory backend. Use isolated test tenants and synthetic canaries only. +MemoryProof can issue destructive deletion requests against a configured memory backend. Treat every remote run as a destructive test: use a dedicated tenant, synthetic canaries, and credentials with the smallest possible scope. -The runner refuses remote access unless `--allow-network` is explicit. Adapters must scope cleanup to resources created by the current run. Never put API keys in scenario files, issue logs, or committed evidence bundles. +## Safe operating rules -Please report security issues privately to the repository maintainers instead of opening a public issue with credentials or customer data. +- Remote access is denied unless `--allow-network` is explicitly supplied. +- Scenarios must use owned, temporary resources. Adapters must refuse cleanup or scope deletion without the current run’s ownership marker. +- Never put API keys, bearer tokens, customer data, private endpoint URLs, or raw production responses in a scenario, issue, pull request, or committed evidence bundle. +- Evidence is redacted by default. Review generated artifacts before sharing them publicly. +- `UNKNOWN` and `OUT OF SCOPE` are security boundaries, not successful deletion claims. + +## Reporting a vulnerability + +Please use [GitHub private vulnerability reporting](https://github.com/Hughhhhcoder/MemoryProof/security/advisories/new) when available. If the form is unavailable, contact the maintainer through the repository profile and do not include credentials or customer data in the first message. + +Include: + +- affected commit, release, or adapter; +- a minimal reproduction using synthetic data; +- impact and likely attack path; +- any mitigation that has already been tested. + +We will acknowledge a report when we can, coordinate a fix and disclosure timeline with the reporter, and credit the reporter unless they prefer to remain anonymous. Please do not publicly disclose an unpatched vulnerability. + +## Scope + +The security policy covers the MemoryProof CLI, adapter protocol, official adapters, Docker image, GitHub Action, evidence redaction, and release workflow. It does not make claims about the security of a third-party memory provider or prove that a provider deleted logs, backups, physical media, or model weights. diff --git a/SECURITY.zh-CN.md b/SECURITY.zh-CN.md index 32b95f7..d7d156b 100644 --- a/SECURITY.zh-CN.md +++ b/SECURITY.zh-CN.md @@ -2,8 +2,29 @@ [English](SECURITY.md) · [简体中文](SECURITY.zh-CN.md) -ForgetProof 可以向配置好的记忆后端发起具有破坏性的删除请求。请只使用隔离的测试租户和合成 canary。 +MemoryProof 可以向配置的记忆后端发起具有破坏性的删除请求。请把每次远程运行都当作破坏性测试:使用专用租户、合成 canary,以及权限最小化的凭据。 -除非显式传入 `--allow-network`,测试引擎不会访问远程地址。适配器必须只清理当前运行创建的资源。不要把 API Key 写入场景文件、日志或提交到仓库的证据包。 +## 安全运行规则 -如果发现安全问题,请私下联系仓库维护者,不要在公开 Issue 中发布凭据或客户数据。 +- 除非显式传入 `--allow-network`,否则禁止访问远程地址。 +- 场景只能使用本次运行拥有的临时资源。没有当前运行所有权标记时,适配器必须拒绝清理或范围删除。 +- 不要把 API Key、Bearer Token、客户数据、私有端点 URL 或生产响应原文放进场景、Issue、Pull Request 或提交的证据包。 +- 证据默认脱敏。公开分享前请人工检查生成的工件。 +- `UNKNOWN` 和 `OUT OF SCOPE` 是安全边界,不是删除成功的结论。 + +## 报告漏洞 + +如果可用,请使用 [GitHub 私密漏洞报告](https://github.com/Hughhhhcoder/MemoryProof/security/advisories/new)。如果表单暂时不可用,请通过仓库维护者主页私下联系,第一条消息不要包含凭据或客户数据。 + +请提供: + +- 受影响的 commit、发行版或适配器; +- 使用合成数据的最小复现; +- 影响和可能的攻击路径; +- 已经尝试过的缓解措施。 + +我们会在条件允许时确认收到报告,并与报告者协调修复和披露时间;除非报告者希望匿名,否则会在修复说明中致谢。请不要公开披露尚未修复的漏洞。 + +## 范围 + +本安全策略覆盖 MemoryProof CLI、适配器协议、官方适配器、Docker 镜像、GitHub Action、证据脱敏和发行流程。不对第三方记忆供应商的安全性作保证,也不证明供应商日志、备份、物理介质或模型权重已经删除。 diff --git a/action.yml b/action.yml index 3e7ad6f..3d63c62 100644 --- a/action.yml +++ b/action.yml @@ -1,23 +1,75 @@ -name: ForgetProof +name: MemoryProof author: Hughhhhcoder -description: Evidence-driven tests that prove AI memory was forgotten / 用证据验证 AI 记忆是否真的被遗忘。 +description: Test what your AI remembers and prove what it forgot / 测试 AI 记住了什么,并证明它忘记了什么。 inputs: scenario: - description: Path to a ForgetProof scenario relative to the repository / 相对于仓库根目录的 ForgetProof 场景路径。 + description: Path to a MemoryProof scenario relative to the repository / 相对于仓库根目录的 MemoryProof 场景路径。 required: true allow-network: description: Explicitly allow calls to non-loopback backends / 显式允许访问非本机后端。 required: false default: "false" + output: + description: Evidence output directory relative to the repository / 相对于仓库根目录的证据输出目录。 + required: false + default: ".memoryproof/runs" +outputs: + bundle-path: + description: Path to the generated evidence bundle / 生成的证据包路径。 + value: ${{ steps.run.outputs.bundle-path }} + exit-code: + description: MemoryProof exit code / MemoryProof 退出码。 + value: ${{ steps.run.outputs.exit-code }} runs: - using: docker - image: Dockerfile - args: - - run - - /github/workspace/${{ inputs.scenario }} - - --output - - /github/workspace/.forgetproof/runs - - --allow-network=${{ inputs.allow-network }} + using: composite + steps: + - id: run + name: Run MemoryProof + shell: bash + env: + MEMORYPROOF_SCENARIO: ${{ inputs.scenario }} + MEMORYPROOF_ALLOW_NETWORK: ${{ inputs.allow-network }} + MEMORYPROOF_OUTPUT: ${{ inputs.output }} + run: | + set +e + mkdir -p "$GITHUB_WORKSPACE/$MEMORYPROOF_OUTPUT" + docker run --rm \ + --user "$(id -u):$(id -g)" \ + -v "$GITHUB_WORKSPACE:/workspace" \ + ghcr.io/hughhhhcoder/memoryproof:1 \ + run "/workspace/$MEMORYPROOF_SCENARIO" \ + --output "/workspace/$MEMORYPROOF_OUTPUT" \ + --allow-network="$MEMORYPROOF_ALLOW_NETWORK" + code=$? + bundle=$(find "$GITHUB_WORKSPACE/$MEMORYPROOF_OUTPUT" -mindepth 1 -maxdepth 1 -type d -print -quit) + echo "bundle-path=${bundle#"$GITHUB_WORKSPACE/"}" >> "$GITHUB_OUTPUT" + echo "exit-code=$code" >> "$GITHUB_OUTPUT" + exit 0 + - name: Upload evidence bundle + if: always() + uses: actions/upload-artifact@v4 + with: + name: memoryproof-evidence + path: ${{ inputs.output }} + if-no-files-found: ignore + - name: Publish summary + if: always() + shell: bash + run: | + echo "## MemoryProof" >> "$GITHUB_STEP_SUMMARY" + echo "- Scenario: \`${{ inputs.scenario }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Bundle: \`${{ steps.run.outputs.bundle-path }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Exit code: \`${{ steps.run.outputs.exit-code }}\`" >> "$GITHUB_STEP_SUMMARY" + - name: Enforce MemoryProof result + if: always() + shell: bash + run: | + code="${{ steps.run.outputs.exit-code }}" + if [ -z "$code" ]; then + echo "MemoryProof did not produce an exit code" >&2 + exit 2 + fi + exit "$code" branding: icon: shield color: purple diff --git a/assets/memoryproof-hero.png b/assets/memoryproof-hero.png new file mode 100644 index 0000000..4e88b9f Binary files /dev/null and b/assets/memoryproof-hero.png differ diff --git a/conformance/README.md b/conformance/README.md index 0ad5246..43f4ff4 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -1,20 +1,45 @@ -# Public conformance submissions +# Public Memory Assurance Evidence [English](README.md) · [简体中文](README.zh-CN.md) -![ForgetProof evidence flow](../assets/forgetproof-hero.png) +![MemoryProof evidence flow](../assets/memoryproof-hero.png) -Each JSON file in this directory is a reviewed, redacted pointer to an evidence bundle. The Pages workflow converts it into the static matrix; it does not compute a score. +This directory contains small, reviewed, redacted evidence bundles that can be opened without a hosted service. The Pages workflow verifies each bundle’s checksums before adding its profiles to the public matrix. -```json -{ - "adapter": "reference-clean", - "backend": "forgetproof-reference@0.1.0", - "bundle": "https://example.invalid/bundles/reference-clean", - "profiles": [ - { "profile": "FP-Object", "status": "PASS", "evidence": "https://example.invalid/report.html" } - ] -} +## What a submission must contain + +Each directory under `conformance/evidence/` is a complete bundle produced by `memoryproof run` and must include: + +- `manifest.json`, `scenario.lock.json`, `events.ndjson`, and `results.json`; +- `report.html` and `junit.xml`; +- `checksums.sha256` and `bundle.hash`; +- synthetic fixtures only, with credentials and raw customer content removed. + +The matrix shows `PASS`, `FAIL`, `SKIP`, or `UNKNOWN` per profile. It never turns these states into a single score. A failing or unknown result is welcome when it is reproducible and honestly described. + +## Reproduce a local submission + +```bash +cargo run -- run examples/reference-clean.yml --output /tmp/memoryproof-evidence +python3 scripts/build_matrix.py +cargo run -- matrix validate site/matrix.json ``` -Before merging a submission, verify the referenced bundle with `forgetproof verify`, check that the scenario contains only synthetic data, and record whether the run was maintainer-reproduced or community-submitted. +For a remote provider, pass `--allow-network` only after checking the scenario, credentials, tenant, and cleanup behavior. Do not publish a remote bundle until it has been manually reviewed. + +## Review checklist + +1. Verify the bundle with `memoryproof verify `. +2. Confirm `scenario.lock.json` contains synthetic canaries only. +3. Confirm the backend name and version are recorded. +4. Confirm target/control scope and the exact failing or passing probe are visible. +5. Confirm all unsupported boundaries are `UNKNOWN`, `SKIP`, or explicitly `OUT OF SCOPE`. +6. Mark the source as maintainer-reproduced or community-submitted in the accompanying release note or pull request. + +The public matrix is an evidence index, not a vendor leaderboard. + +The `*-adapter-contract` bundles are maintainer-reproduced runs against the +deterministic local mock in `scripts/mock_remote_backend.py`. They verify the +official adapter contract without claiming that a particular hosted provider +has the same behavior. Provider conformance claims must include the provider +version and a separately reviewed, credentialed run. diff --git a/conformance/README.zh-CN.md b/conformance/README.zh-CN.md index d215bb8..0034cee 100644 --- a/conformance/README.zh-CN.md +++ b/conformance/README.zh-CN.md @@ -1,20 +1,44 @@ -# 公开认证结果提交说明 +# 公开 Memory Assurance 证据 [English](README.md) · [简体中文](README.zh-CN.md) -![ForgetProof 证据流程](../assets/forgetproof-hero.png) +![MemoryProof 证据流程](../assets/memoryproof-hero.png) -该目录中的每个 JSON 文件,都是一个经过审阅、已经脱敏的证据包引用。Pages 工作流会把它生成静态认证矩阵,不会计算一个综合分数。 +该目录包含经过审阅、已经脱敏、无需在线服务即可打开的小型证据包。Pages 工作流会先校验每个证据包的哈希,再把其中的档案加入公开矩阵。 -```json -{ - "adapter": "reference-clean", - "backend": "forgetproof-reference@0.1.0", - "bundle": "https://example.invalid/bundles/reference-clean", - "profiles": [ - { "profile": "FP-Object", "status": "PASS", "evidence": "https://example.invalid/report.html" } - ] -} +## 提交必须包含什么 + +`conformance/evidence/` 下的每个目录都应该是由 `memoryproof run` 生成的完整证据包,并包含: + +- `manifest.json`、`scenario.lock.json`、`events.ndjson` 和 `results.json`; +- `report.html` 和 `junit.xml`; +- `checksums.sha256` 和 `bundle.hash`; +- 只有合成 fixture,已移除凭据和客户原文。 + +矩阵会按档案展示 `PASS`、`FAIL`、`SKIP` 或 `UNKNOWN`,不会把它们压成一个综合分数。只要可复现且描述诚实,失败或未知结果同样欢迎提交。 + +## 复现本地提交 + +```bash +cargo run -- run examples/reference-clean.yml --output /tmp/memoryproof-evidence +python3 scripts/build_matrix.py +cargo run -- matrix validate site/matrix.json ``` -合并提交前,请使用 `forgetproof verify` 验证证据包,确认场景只包含合成数据,并记录该结果是维护者复现还是社区提交。 +对于远程供应商,只有在确认场景、凭据、租户和清理行为后,才传入 `--allow-network`。公开远程证据前必须人工审阅。 + +## 审阅清单 + +1. 使用 `memoryproof verify ` 校验证据包。 +2. 确认 `scenario.lock.json` 只包含合成 canary。 +3. 确认记录了后端名称和版本。 +4. 确认目标/控制范围,以及具体失败或通过的探针可见。 +5. 确认不支持的边界被标为 `UNKNOWN`、`SKIP` 或明确的 `OUT OF SCOPE`。 +6. 在配套 Release Note 或 Pull Request 中标明维护者复现还是社区提交。 + +公开矩阵是证据索引,不是供应商排行榜。 + +`*-adapter-contract` 证据包是维护者使用 +`scripts/mock_remote_backend.py` 中的确定性本地 mock 复现的结果。它们验证 +官方适配器契约,不声称某个托管供应商一定具有相同的行为。供应商认证结果必须 +记录供应商版本,并经过单独审阅的带凭据运行。 diff --git a/conformance/evidence/isolation-reference/bundle.hash b/conformance/evidence/isolation-reference/bundle.hash new file mode 100644 index 0000000..1cf8603 --- /dev/null +++ b/conformance/evidence/isolation-reference/bundle.hash @@ -0,0 +1 @@ +c9ae865a698903047bbfff8723d0acd4e4247ada1e24cfafd67b66bb44163ec8 diff --git a/conformance/evidence/isolation-reference/checksums.sha256 b/conformance/evidence/isolation-reference/checksums.sha256 new file mode 100644 index 0000000..3f95989 --- /dev/null +++ b/conformance/evidence/isolation-reference/checksums.sha256 @@ -0,0 +1,6 @@ +3619e19e504a5edde9121a32b3e4b5a76e79929ed2912242137d59fdd0d7c772 events.ndjson +06444b7cf30a1d97a041b4f8117116ee80de9f8a63c1f90340155383d259d52f junit.xml +8ac91e46ce97c204a844dc54193f55cc721ccbf4e65b37c0cd9bf46d29b5adeb manifest.json +52d557c12baea363683d2762331565e0bd3645fc6c8d1f248e7ae3e0df9abd13 report.html +6bbb5e6be1117ce4f3fbcb5f41ed3b952ef14b6c0867817157becc1bcb026328 results.json +a9c5bbcdf16f5b5210367bec28ffbd39452874ebf81a50a2e3cc319c8daef5bb scenario.lock.json diff --git a/conformance/evidence/isolation-reference/events.ndjson b/conformance/evidence/isolation-reference/events.ndjson new file mode 100644 index 0000000..615b9a9 --- /dev/null +++ b/conformance/evidence/isolation-reference/events.ndjson @@ -0,0 +1,13 @@ +{"seq":1,"at_ms":1787321604319,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321604319,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"reference-clean","backend":"memoryproof-reference","version":"1.0.0"}} +{"seq":3,"at_ms":1787321604319,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321604319,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321604319,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321604319,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321604319,"phase":"isolation","method":"probe","status":"PASS","details":{"found":"true","probe":"target-from-target"}} +{"seq":8,"at_ms":1787321604320,"phase":"isolation","method":"probe","status":"PASS","details":{"found":"false","probe":"target-from-control"}} +{"seq":9,"at_ms":1787321604320,"phase":"isolation","method":"probe","status":"PASS","details":{"found":"true","probe":"control-from-control"}} +{"seq":10,"at_ms":1787321604320,"phase":"isolation","method":"probe","status":"PASS","details":{"found":"false","probe":"control-from-target"}} +{"seq":11,"at_ms":1787321604320,"phase":"isolation","method":"probe","status":"PASS","details":{"found":"false","probe":"target-search-from-control"}} +{"seq":12,"at_ms":1787321604320,"phase":"isolation","method":"probe","status":"PASS","details":{"found":"false","probe":"control-search-from-target"}} +{"seq":13,"at_ms":1787321604320,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/isolation-reference/junit.xml b/conformance/evidence/isolation-reference/junit.xml new file mode 100644 index 0000000..82a8e17 --- /dev/null +++ b/conformance/evidence/isolation-reference/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/isolation-reference/manifest.json b/conformance/evidence/isolation-reference/manifest.json new file mode 100644 index 0000000..8e343eb --- /dev/null +++ b/conformance/evidence/isolation-reference/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321604254-79596", + "created_at_ms": 1787321604321, + "scenario_hash": "4040132e59f3cf18a13681923ff6949733677d12b3795a9352ee2ebbba4248d3", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/isolation-reference/report.html b/conformance/evidence/isolation-reference/report.html new file mode 100644 index 0000000..8e9d0cb --- /dev/null +++ b/conformance/evidence/isolation-reference/report.html @@ -0,0 +1,30 @@ + + + +MemoryProof · isolation-reference +
+
MEMORYPROOF · isolation

isolation-reference

reference-clean · memoryproof-reference · 1.0.0

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

PASS
+
Run / 运行run-1787321604254-79596
Scenario hash / 场景哈希4040132e59f3cf18a13681923ff6949733677d12b3795a9352ee2ebbba4248d3
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码0
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+
Profile / 档案Status / 状态Assertions / 断言
isolation.read
IsolationProof · Read
PASSbefore.target-from-target, before.target-from-control, before.control-from-control, before.control-from-target
isolation.search
IsolationProof · Search
PASSafter.target-search-from-control, after.control-search-from-target
+

Assertions / 断言明细

+ + + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-from-targetisolation.readnamespacePASSquery subject 'target-subject' expected fixture 'target' to be isolated, observed visibletrue / true
before.target-from-controlisolation.readnamespacePASSquery subject 'control-subject' expected fixture 'target' to be inaccessible, observed absentfalse / false
before.control-from-controlisolation.readnamespacePASSquery subject 'control-subject' expected fixture 'control' to be isolated, observed visibletrue / true
before.control-from-targetisolation.readnamespacePASSquery subject 'target-subject' expected fixture 'control' to be inaccessible, observed absentfalse / false
after.target-search-from-controlisolation.searchnamespacePASSquery subject 'control-subject' expected fixture 'target' to be inaccessible, observed absentfalse / false
after.control-search-from-targetisolation.searchnamespacePASSquery subject 'target-subject' expected fixture 'control' to be inaccessible, observed absentfalse / false
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/isolation-reference/results.json b/conformance/evidence/isolation-reference/results.json new file mode 100644 index 0000000..cd878b8 --- /dev/null +++ b/conformance/evidence/isolation-reference/results.json @@ -0,0 +1,128 @@ +{ + "run_id": "run-1787321604254-79596", + "suite": "isolation", + "scenario": "isolation-reference", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "4040132e59f3cf18a13681923ff6949733677d12b3795a9352ee2ebbba4248d3", + "status": "PASS", + "exit_code": 0, + "profiles": [ + { + "profile": "isolation.read", + "label": "IsolationProof · Read", + "status": "PASS", + "assertions": [ + "before.target-from-target", + "before.target-from-control", + "before.control-from-control", + "before.control-from-target" + ] + }, + { + "profile": "isolation.search", + "label": "IsolationProof · Search", + "status": "PASS", + "assertions": [ + "after.target-search-from-control", + "after.control-search-from-target" + ] + } + ], + "assertions": [ + { + "id": "before.target-from-target", + "profile": "isolation.read", + "status": "PASS", + "message": "query subject 'target-subject' expected fixture 'target' to be isolated, observed visible", + "expected": true, + "observed": true, + "artifact": "namespace", + "evidence": [ + "probe:target-from-target" + ] + }, + { + "id": "before.target-from-control", + "profile": "isolation.read", + "status": "PASS", + "message": "query subject 'control-subject' expected fixture 'target' to be inaccessible, observed absent", + "expected": false, + "observed": false, + "artifact": "namespace", + "evidence": [ + "probe:target-from-control" + ] + }, + { + "id": "before.control-from-control", + "profile": "isolation.read", + "status": "PASS", + "message": "query subject 'control-subject' expected fixture 'control' to be isolated, observed visible", + "expected": true, + "observed": true, + "artifact": "namespace", + "evidence": [ + "probe:control-from-control" + ] + }, + { + "id": "before.control-from-target", + "profile": "isolation.read", + "status": "PASS", + "message": "query subject 'target-subject' expected fixture 'control' to be inaccessible, observed absent", + "expected": false, + "observed": false, + "artifact": "namespace", + "evidence": [ + "probe:control-from-target" + ] + }, + { + "id": "after.target-search-from-control", + "profile": "isolation.search", + "status": "PASS", + "message": "query subject 'control-subject' expected fixture 'target' to be inaccessible, observed absent", + "expected": false, + "observed": false, + "artifact": "namespace", + "evidence": [ + "probe:target-search-from-control" + ] + }, + { + "id": "after.control-search-from-target", + "profile": "isolation.search", + "status": "PASS", + "message": "query subject 'target-subject' expected fixture 'control' to be inaccessible, observed absent", + "expected": false, + "observed": false, + "artifact": "namespace", + "evidence": [ + "probe:control-search-from-target" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "derived_delete", + "agent_query", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/isolation-reference/scenario.lock.json b/conformance/evidence/isolation-reference/scenario.lock.json new file mode 100644 index 0000000..6733294 --- /dev/null +++ b/conformance/evidence/isolation-reference/scenario.lock.json @@ -0,0 +1,108 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "Two subjects must not be able to observe one another's canaries.", + "name": "isolation-reference" + }, + "spec": { + "adapter": { + "config": {}, + "mode": "clean", + "name": "reference-clean" + }, + "erase": { + "intent": "object_delete", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:1db86affcda819079ec9bc5bf8fd6c8407956edcffb7402f99bfb34351868a1d len:35", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:5b84565ed2e8964ffb121bb4dbd05ee39cce47406fc9345ff1d3b6f0189d4154 len:36", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "control-subject", + "fixture": "target", + "id": "target-search-from-control", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "target-subject", + "fixture": "control", + "id": "control-search-from-target", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "target-subject", + "fixture": "target", + "id": "target-from-target", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "control-subject", + "fixture": "target", + "id": "target-from-control", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "control-subject", + "fixture": "control", + "id": "control-from-control", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "target-subject", + "fixture": "control", + "id": "control-from-target", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "isolation.read", + "isolation.search" + ], + "settle": { + "interval_ms": 25, + "timeout_ms": 5000 + }, + "suite": "isolation" + } +} \ No newline at end of file diff --git a/conformance/evidence/letta-adapter-contract/bundle.hash b/conformance/evidence/letta-adapter-contract/bundle.hash new file mode 100644 index 0000000..56e28b0 --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/bundle.hash @@ -0,0 +1 @@ +0a9cbd606722d2e40453648005f0ba52b601987aed37bcf0e70ac9abcf80fcdf diff --git a/conformance/evidence/letta-adapter-contract/checksums.sha256 b/conformance/evidence/letta-adapter-contract/checksums.sha256 new file mode 100644 index 0000000..cc29b29 --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/checksums.sha256 @@ -0,0 +1,6 @@ +d985ef8b00ee864dc6c09f2be8e5e6c6290e74bafce42f734adeff49945b6b79 events.ndjson +0c4090cc6e5a7ecc6602122cfc41cb0c28bd1aad7853cfbccb5a799df166d4c4 junit.xml +984af31b49fb404fb931cf13cb6992bbfdda38a28dae98f303747e1113040ee8 manifest.json +7d0e395050956eb8f1c4455c74e65e30773f74db8e0f9d55a04ae359f824d58f report.html +d450cc4fd87ab3910dbc6c4a37e4a3537c95e107c953d6114b5780e54dcbabff results.json +427c14ee7d88b27bd5770e0fa0ab7eda658b75d3344411be686d026cb61d58f0 scenario.lock.json diff --git a/conformance/evidence/letta-adapter-contract/events.ndjson b/conformance/evidence/letta-adapter-contract/events.ndjson new file mode 100644 index 0000000..ff3e492 --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/events.ndjson @@ -0,0 +1,15 @@ +{"seq":1,"at_ms":1787321624149,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321624150,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"letta","backend":"letta","version":"configured"}} +{"seq":3,"at_ms":1787321624170,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321624178,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321624184,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321624184,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321624191,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"target-before"}} +{"seq":8,"at_ms":1787321624197,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"control-before"}} +{"seq":9,"at_ms":1787321624204,"phase":"run","method":"erase","status":"PASS","details":{}} +{"seq":10,"at_ms":1787321624204,"phase":"after-erase","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":11,"at_ms":1787321624210,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"target-after"}} +{"seq":12,"at_ms":1787321624217,"phase":"after","method":"probe","status":"PASS","details":{"found":"true","probe":"control-after"}} +{"seq":13,"at_ms":1787321624224,"phase":"after","method":"inspect","status":"PASS","details":{"found":"false","probe":"target-derived-after"}} +{"seq":14,"at_ms":1787321624231,"phase":"after","method":"agent_query","status":"PASS","details":{"found":"false","probe":"target-agent-after"}} +{"seq":15,"at_ms":1787321624243,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/letta-adapter-contract/junit.xml b/conformance/evidence/letta-adapter-contract/junit.xml new file mode 100644 index 0000000..94eb28a --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/letta-adapter-contract/manifest.json b/conformance/evidence/letta-adapter-contract/manifest.json new file mode 100644 index 0000000..a5d5560 --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321620648-79664", + "created_at_ms": 1787321624245, + "scenario_hash": "f9a37596cbb8ce787eb02dede4561cd8e14b831187fa938abc2746cc5a647cbe", + "adapter": "letta", + "backend": "letta", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/letta-adapter-contract/report.html b/conformance/evidence/letta-adapter-contract/report.html new file mode 100644 index 0000000..696b082 --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/report.html @@ -0,0 +1,33 @@ + + + +MemoryProof · letta-erasure +
+
MEMORYPROOF · erasure

letta-erasure

letta · letta · configured

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

PASS
+
Run / 运行run-1787321620648-79664
Scenario hash / 场景哈希f9a37596cbb8ce787eb02dede4561cd8e14b831187fa938abc2746cc5a647cbe
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码0
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+ + +
Profile / 档案Status / 状态Assertions / 断言
erasure.object
MemoryProof · Erasure · Object
PASSbefore.target-before, before.control-before, after.target-after, after.control-after
erasure.scope
MemoryProof · Erasure · Scope
PASSscope.control-after
erasure.derived
MemoryProof · Erasure · Derived
PASSafter.target-derived-after
erasure.agent
MemoryProof · Erasure · Agent
PASSafter.target-agent-after
+

Assertions / 断言明细

+ + + + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-beforeerasure.objectsemanticPASSsemantic canary 'target' was observable before the erase operationtrue / true
before.control-beforeerasure.objectsemanticPASSsemantic canary 'control' was observable before the erase operationtrue / true
after.target-aftererasure.objectsemantic-indexPASSsemantic probe for fixture 'target' expected absent, observed absentfalse / false
after.control-aftererasure.objectsemantic-indexPASSsemantic probe for fixture 'control' expected present, observed presenttrue / true
scope.control-aftererasure.scopescopePASScontrol fixture 'control' remained observable after target erasetrue / true
after.target-derived-aftererasure.derivedderivedPASSinspect probe for fixture 'target' expected absent, observed absentfalse / false
after.target-agent-aftererasure.agentagentPASSagent probe for fixture 'target' expected absent, observed absentfalse / false
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/letta-adapter-contract/results.json b/conformance/evidence/letta-adapter-contract/results.json new file mode 100644 index 0000000..a63d434 --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/results.json @@ -0,0 +1,154 @@ +{ + "run_id": "run-1787321620648-79664", + "suite": "erasure", + "scenario": "letta-erasure", + "adapter": "letta", + "backend": "letta", + "backend_version": "configured", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "f9a37596cbb8ce787eb02dede4561cd8e14b831187fa938abc2746cc5a647cbe", + "status": "PASS", + "exit_code": 0, + "profiles": [ + { + "profile": "erasure.object", + "label": "MemoryProof · Erasure · Object", + "status": "PASS", + "assertions": [ + "before.target-before", + "before.control-before", + "after.target-after", + "after.control-after" + ] + }, + { + "profile": "erasure.scope", + "label": "MemoryProof · Erasure · Scope", + "status": "PASS", + "assertions": [ + "scope.control-after" + ] + }, + { + "profile": "erasure.derived", + "label": "MemoryProof · Erasure · Derived", + "status": "PASS", + "assertions": [ + "after.target-derived-after" + ] + }, + { + "profile": "erasure.agent", + "label": "MemoryProof · Erasure · Agent", + "status": "PASS", + "assertions": [ + "after.target-agent-after" + ] + } + ], + "assertions": [ + { + "id": "before.target-before", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic canary 'target' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "semantic", + "evidence": [ + "probe:target-before" + ] + }, + { + "id": "before.control-before", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic canary 'control' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "semantic", + "evidence": [ + "probe:control-before" + ] + }, + { + "id": "after.target-after", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "semantic-index", + "evidence": [ + "probe:target-after" + ] + }, + { + "id": "after.control-after", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic probe for fixture 'control' expected present, observed present", + "expected": true, + "observed": true, + "artifact": "semantic-index", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "scope.control-after", + "profile": "erasure.scope", + "status": "PASS", + "message": "control fixture 'control' remained observable after target erase", + "expected": true, + "observed": true, + "artifact": "scope", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "after.target-derived-after", + "profile": "erasure.derived", + "status": "PASS", + "message": "inspect probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "derived", + "evidence": [ + "probe:target-derived-after" + ] + }, + { + "id": "after.target-agent-after", + "profile": "erasure.agent", + "status": "PASS", + "message": "agent probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "agent", + "evidence": [ + "probe:target-agent-after" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "agent_query", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/letta-adapter-contract/scenario.lock.json b/conformance/evidence/letta-adapter-contract/scenario.lock.json new file mode 100644 index 0000000..e274ced --- /dev/null +++ b/conformance/evidence/letta-adapter-contract/scenario.lock.json @@ -0,0 +1,112 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "Letta archival passage and Agent erasure with owned temporary Agents.", + "name": "letta-erasure" + }, + "spec": { + "adapter": { + "config": { + "base_url": "http://localhost:8283" + }, + "mode": "self-hosted", + "name": "letta" + }, + "erase": { + "intent": "subject_erase", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:df94daba03c3f2f49bd5eb6ce64fa57395f7b309649d79da0f9309bb9d375938 len:38", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:f9e5cd398085acf2075cff253f91a4629ec9d5a0d10e7deaf3f670a061621f08 len:32", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-after", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-after", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-derived-after", + "kind": "inspect", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-agent-after", + "kind": "agent", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-before", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-before", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "erasure.object", + "erasure.scope", + "erasure.derived", + "erasure.agent" + ], + "settle": { + "interval_ms": 1000, + "timeout_ms": 30000 + }, + "suite": "erasure" + } +} \ No newline at end of file diff --git a/conformance/evidence/mem0-adapter-contract/bundle.hash b/conformance/evidence/mem0-adapter-contract/bundle.hash new file mode 100644 index 0000000..28f7c7c --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/bundle.hash @@ -0,0 +1 @@ +af8b543c1f708f227257598a8f681aa7ad2f3902bc1b496d243e719f272d2fd2 diff --git a/conformance/evidence/mem0-adapter-contract/checksums.sha256 b/conformance/evidence/mem0-adapter-contract/checksums.sha256 new file mode 100644 index 0000000..ba10ccd --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/checksums.sha256 @@ -0,0 +1,6 @@ +c49e6ebea5dea1a0dd712de8c4c6743c09138a4e5fb08c9da4c4c5fc926f5027 events.ndjson +fcd7415a65a28f82447dcdf7a10d10904bf5e8f07dd7a0017291b264e55dcddd junit.xml +7f45d2f7399fba648e5e7b5c8c1e8ddce2ff2adbd25404db6818ab0d5ac804a8 manifest.json +7745d6678010dd1672ae8bd3503ef2e179044d0f6f46f804999a50108b454e0b report.html +ff14980667823b237a770b90ca27c5897506d327aaf332ea751f7ce60fac7d6d results.json +f4d402ffe42b1c6378d6190bc17411e90617c82f8c930326487642a37edd255f scenario.lock.json diff --git a/conformance/evidence/mem0-adapter-contract/events.ndjson b/conformance/evidence/mem0-adapter-contract/events.ndjson new file mode 100644 index 0000000..2b9c82c --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/events.ndjson @@ -0,0 +1,14 @@ +{"seq":1,"at_ms":1787321618425,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321618425,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"mem0","backend":"mem0","version":"configured"}} +{"seq":3,"at_ms":1787321618425,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321618441,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321618449,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321618449,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321618457,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"target-before"}} +{"seq":8,"at_ms":1787321618464,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"control-before"}} +{"seq":9,"at_ms":1787321618471,"phase":"run","method":"erase","status":"PASS","details":{}} +{"seq":10,"at_ms":1787321618471,"phase":"after-erase","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":11,"at_ms":1787321618478,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"target-after"}} +{"seq":12,"at_ms":1787321618485,"phase":"after","method":"probe","status":"PASS","details":{"found":"true","probe":"control-after"}} +{"seq":13,"at_ms":1787321618491,"phase":"after","method":"inspect","status":"PASS","details":{"found":"false","probe":"target-derived-after"}} +{"seq":14,"at_ms":1787321618505,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/mem0-adapter-contract/junit.xml b/conformance/evidence/mem0-adapter-contract/junit.xml new file mode 100644 index 0000000..45b97e3 --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/mem0-adapter-contract/manifest.json b/conformance/evidence/mem0-adapter-contract/manifest.json new file mode 100644 index 0000000..5c5830d --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321611383-79632", + "created_at_ms": 1787321618506, + "scenario_hash": "2482de1c69be35d20f2f685adc8b8813e08c22984b24d159b2046c480ccfc4d9", + "adapter": "mem0", + "backend": "mem0", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/mem0-adapter-contract/report.html b/conformance/evidence/mem0-adapter-contract/report.html new file mode 100644 index 0000000..26f264e --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/report.html @@ -0,0 +1,31 @@ + + + +MemoryProof · mem0-erasure +
+
MEMORYPROOF · erasure

mem0-erasure

mem0 · mem0 · configured

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

PASS
+
Run / 运行run-1787321611383-79632
Scenario hash / 场景哈希2482de1c69be35d20f2f685adc8b8813e08c22984b24d159b2046c480ccfc4d9
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码0
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+ +
Profile / 档案Status / 状态Assertions / 断言
erasure.object
MemoryProof · Erasure · Object
PASSbefore.target-before, before.control-before, after.target-after, after.control-after
erasure.scope
MemoryProof · Erasure · Scope
PASSscope.control-after
erasure.derived
MemoryProof · Erasure · Derived
PASSafter.target-derived-after
+

Assertions / 断言明细

+ + + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-beforeerasure.objectexactPASSexact canary 'target' was observable before the erase operationtrue / true
before.control-beforeerasure.objectexactPASSexact canary 'control' was observable before the erase operationtrue / true
after.target-aftererasure.objectsemantic-indexPASSsemantic probe for fixture 'target' expected absent, observed absentfalse / false
after.control-aftererasure.objectsemantic-indexPASSsemantic probe for fixture 'control' expected present, observed presenttrue / true
scope.control-aftererasure.scopescopePASScontrol fixture 'control' remained observable after target erasetrue / true
after.target-derived-aftererasure.derivedderivedPASSinspect probe for fixture 'target' expected absent, observed absentfalse / false
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/mem0-adapter-contract/results.json b/conformance/evidence/mem0-adapter-contract/results.json new file mode 100644 index 0000000..51b0257 --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/results.json @@ -0,0 +1,133 @@ +{ + "run_id": "run-1787321611383-79632", + "suite": "erasure", + "scenario": "mem0-erasure", + "adapter": "mem0", + "backend": "mem0", + "backend_version": "configured", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "2482de1c69be35d20f2f685adc8b8813e08c22984b24d159b2046c480ccfc4d9", + "status": "PASS", + "exit_code": 0, + "profiles": [ + { + "profile": "erasure.object", + "label": "MemoryProof · Erasure · Object", + "status": "PASS", + "assertions": [ + "before.target-before", + "before.control-before", + "after.target-after", + "after.control-after" + ] + }, + { + "profile": "erasure.scope", + "label": "MemoryProof · Erasure · Scope", + "status": "PASS", + "assertions": [ + "scope.control-after" + ] + }, + { + "profile": "erasure.derived", + "label": "MemoryProof · Erasure · Derived", + "status": "PASS", + "assertions": [ + "after.target-derived-after" + ] + } + ], + "assertions": [ + { + "id": "before.target-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'target' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:target-before" + ] + }, + { + "id": "before.control-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'control' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:control-before" + ] + }, + { + "id": "after.target-after", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "semantic-index", + "evidence": [ + "probe:target-after" + ] + }, + { + "id": "after.control-after", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic probe for fixture 'control' expected present, observed present", + "expected": true, + "observed": true, + "artifact": "semantic-index", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "scope.control-after", + "profile": "erasure.scope", + "status": "PASS", + "message": "control fixture 'control' remained observable after target erase", + "expected": true, + "observed": true, + "artifact": "scope", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "after.target-derived-after", + "profile": "erasure.derived", + "status": "PASS", + "message": "inspect probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "derived", + "evidence": [ + "probe:target-derived-after" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/mem0-adapter-contract/scenario.lock.json b/conformance/evidence/mem0-adapter-contract/scenario.lock.json new file mode 100644 index 0000000..b888ed0 --- /dev/null +++ b/conformance/evidence/mem0-adapter-contract/scenario.lock.json @@ -0,0 +1,104 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "Mem0 deletion with separate target and control users.", + "name": "mem0-erasure" + }, + "spec": { + "adapter": { + "config": { + "base_url": "http://localhost:8888" + }, + "mode": "oss", + "name": "mem0" + }, + "erase": { + "intent": "subject_erase", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:417e0a03d4694a63943bd7c5f1ee3b879e5ad5e13081f03bcbdb4ffdd806eb95 len:37", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:fe547dd665771a091ae7fbe254b554f1d59c8447e90aa9a9e7e4008433c20e9b len:31", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-after", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-after", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-derived-after", + "kind": "inspect", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "erasure.object", + "erasure.scope", + "erasure.derived" + ], + "settle": { + "interval_ms": 1000, + "timeout_ms": 30000 + }, + "suite": "erasure" + } +} \ No newline at end of file diff --git a/conformance/evidence/reference-clean/bundle.hash b/conformance/evidence/reference-clean/bundle.hash new file mode 100644 index 0000000..0141ddd --- /dev/null +++ b/conformance/evidence/reference-clean/bundle.hash @@ -0,0 +1 @@ +c6feb6e3dbd230d29e439f8e2d7e735b910e68d5104e848ab2120397e9d7d028 diff --git a/conformance/evidence/reference-clean/checksums.sha256 b/conformance/evidence/reference-clean/checksums.sha256 new file mode 100644 index 0000000..2db6549 --- /dev/null +++ b/conformance/evidence/reference-clean/checksums.sha256 @@ -0,0 +1,6 @@ +b2a5d933c7a7389b3e91629b95678a785afdd0a8c6acd09f29c4b868f0f44e0d events.ndjson +0c4090cc6e5a7ecc6602122cfc41cb0c28bd1aad7853cfbccb5a799df166d4c4 junit.xml +bbb7c8ac9278de907b0d4f910948649f952e31491c6aec8aaa10cac4168328eb manifest.json +f452dcef9797e259eff9ab686bc443088a428438a0783baa0463d29c450f0f56 report.html +af704f1117a1cfa3ca5ae400365c4719cdd942cfa88c266336eaea41f84e51c4 results.json +19def784e418984d02e95d447bea05807abf420e541dcd9702148ec91884685e scenario.lock.json diff --git a/conformance/evidence/reference-clean/events.ndjson b/conformance/evidence/reference-clean/events.ndjson new file mode 100644 index 0000000..fe45e9c --- /dev/null +++ b/conformance/evidence/reference-clean/events.ndjson @@ -0,0 +1,15 @@ +{"seq":1,"at_ms":1787321601653,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321601653,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"reference-clean","backend":"memoryproof-reference","version":"1.0.0"}} +{"seq":3,"at_ms":1787321601654,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321601654,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321601654,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321601654,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321601654,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"target-before"}} +{"seq":8,"at_ms":1787321601654,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"control-before"}} +{"seq":9,"at_ms":1787321601654,"phase":"run","method":"erase","status":"PASS","details":{}} +{"seq":10,"at_ms":1787321601654,"phase":"after-erase","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":11,"at_ms":1787321601654,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"target-after"}} +{"seq":12,"at_ms":1787321601654,"phase":"after","method":"probe","status":"PASS","details":{"found":"true","probe":"control-after"}} +{"seq":13,"at_ms":1787321601654,"phase":"after","method":"inspect","status":"PASS","details":{"found":"false","probe":"target-derived-after"}} +{"seq":14,"at_ms":1787321601655,"phase":"after","method":"agent_query","status":"PASS","details":{"found":"false","probe":"target-agent-after"}} +{"seq":15,"at_ms":1787321601655,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/reference-clean/junit.xml b/conformance/evidence/reference-clean/junit.xml new file mode 100644 index 0000000..94eb28a --- /dev/null +++ b/conformance/evidence/reference-clean/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/reference-clean/manifest.json b/conformance/evidence/reference-clean/manifest.json new file mode 100644 index 0000000..8f75c2c --- /dev/null +++ b/conformance/evidence/reference-clean/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321596205-79581", + "created_at_ms": 1787321601655, + "scenario_hash": "60634e3baaf7b25c6e51612075ea1cffd6a67da00ea6c981e7645b1df84aa4bf", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/reference-clean/report.html b/conformance/evidence/reference-clean/report.html new file mode 100644 index 0000000..8369f7d --- /dev/null +++ b/conformance/evidence/reference-clean/report.html @@ -0,0 +1,33 @@ + + + +MemoryProof · reference-clean +
+
MEMORYPROOF · erasure

reference-clean

reference-clean · memoryproof-reference · 1.0.0

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

PASS
+
Run / 运行run-1787321596205-79581
Scenario hash / 场景哈希60634e3baaf7b25c6e51612075ea1cffd6a67da00ea6c981e7645b1df84aa4bf
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码0
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+ + +
Profile / 档案Status / 状态Assertions / 断言
erasure.object
MemoryProof · Erasure · Object
PASSbefore.target-before, before.control-before, after.target-after, after.control-after
erasure.scope
MemoryProof · Erasure · Scope
PASSscope.control-after
erasure.derived
MemoryProof · Erasure · Derived
PASSafter.target-derived-after
erasure.agent
MemoryProof · Erasure · Agent
PASSafter.target-agent-after
+

Assertions / 断言明细

+ + + + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-beforeerasure.objectexactPASSexact canary 'target' was observable before the erase operationtrue / true
before.control-beforeerasure.objectexactPASSexact canary 'control' was observable before the erase operationtrue / true
after.target-aftererasure.objectobjectPASSexact probe for fixture 'target' expected absent, observed absentfalse / false
after.control-aftererasure.objectobjectPASSexact probe for fixture 'control' expected present, observed presenttrue / true
scope.control-aftererasure.scopescopePASScontrol fixture 'control' remained observable after target erasetrue / true
after.target-derived-aftererasure.derivedderivedPASSinspect probe for fixture 'target' expected absent, observed absentfalse / false
after.target-agent-aftererasure.agentagentPASSagent probe for fixture 'target' expected absent, observed absentfalse / false
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/reference-clean/results.json b/conformance/evidence/reference-clean/results.json new file mode 100644 index 0000000..4f3e52b --- /dev/null +++ b/conformance/evidence/reference-clean/results.json @@ -0,0 +1,155 @@ +{ + "run_id": "run-1787321596205-79581", + "suite": "erasure", + "scenario": "reference-clean", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "60634e3baaf7b25c6e51612075ea1cffd6a67da00ea6c981e7645b1df84aa4bf", + "status": "PASS", + "exit_code": 0, + "profiles": [ + { + "profile": "erasure.object", + "label": "MemoryProof · Erasure · Object", + "status": "PASS", + "assertions": [ + "before.target-before", + "before.control-before", + "after.target-after", + "after.control-after" + ] + }, + { + "profile": "erasure.scope", + "label": "MemoryProof · Erasure · Scope", + "status": "PASS", + "assertions": [ + "scope.control-after" + ] + }, + { + "profile": "erasure.derived", + "label": "MemoryProof · Erasure · Derived", + "status": "PASS", + "assertions": [ + "after.target-derived-after" + ] + }, + { + "profile": "erasure.agent", + "label": "MemoryProof · Erasure · Agent", + "status": "PASS", + "assertions": [ + "after.target-agent-after" + ] + } + ], + "assertions": [ + { + "id": "before.target-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'target' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:target-before" + ] + }, + { + "id": "before.control-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'control' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:control-before" + ] + }, + { + "id": "after.target-after", + "profile": "erasure.object", + "status": "PASS", + "message": "exact probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "object", + "evidence": [ + "probe:target-after" + ] + }, + { + "id": "after.control-after", + "profile": "erasure.object", + "status": "PASS", + "message": "exact probe for fixture 'control' expected present, observed present", + "expected": true, + "observed": true, + "artifact": "object", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "scope.control-after", + "profile": "erasure.scope", + "status": "PASS", + "message": "control fixture 'control' remained observable after target erase", + "expected": true, + "observed": true, + "artifact": "scope", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "after.target-derived-after", + "profile": "erasure.derived", + "status": "PASS", + "message": "inspect probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "derived", + "evidence": [ + "probe:target-derived-after" + ] + }, + { + "id": "after.target-agent-after", + "profile": "erasure.agent", + "status": "PASS", + "message": "agent probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "agent", + "evidence": [ + "probe:target-agent-after" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "derived_delete", + "agent_query", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/reference-clean/scenario.lock.json b/conformance/evidence/reference-clean/scenario.lock.json new file mode 100644 index 0000000..0896ad7 --- /dev/null +++ b/conformance/evidence/reference-clean/scenario.lock.json @@ -0,0 +1,110 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "A deterministic clean backend removes raw and derived artifacts.", + "name": "reference-clean" + }, + "spec": { + "adapter": { + "config": {}, + "mode": "clean", + "name": "reference-clean" + }, + "erase": { + "intent": "subject_erase", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:9df585a469698301386fcf8fa937ed898fcb0bb0d2ecbae8501a37acd697d4d6 len:38", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:ec352e339e595077e80dc559d83350b185d517967d83202117e0e2986216783e len:38", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-after", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-after", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-derived-after", + "kind": "inspect", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-agent-after", + "kind": "agent", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "erasure.object", + "erasure.scope", + "erasure.derived", + "erasure.agent" + ], + "settle": { + "interval_ms": 25, + "timeout_ms": 5000 + }, + "suite": "erasure" + } +} \ No newline at end of file diff --git a/conformance/evidence/reference-leaky/bundle.hash b/conformance/evidence/reference-leaky/bundle.hash new file mode 100644 index 0000000..7d7f49e --- /dev/null +++ b/conformance/evidence/reference-leaky/bundle.hash @@ -0,0 +1 @@ +75a3632854a97861d6b2e44fa328415e32600907f7f321b82659a90c54505263 diff --git a/conformance/evidence/reference-leaky/checksums.sha256 b/conformance/evidence/reference-leaky/checksums.sha256 new file mode 100644 index 0000000..afc97c5 --- /dev/null +++ b/conformance/evidence/reference-leaky/checksums.sha256 @@ -0,0 +1,6 @@ +4bb59654ca14667dc7aa84b26b8b2ccb87b05359e1e999a1e47e349d4188cd10 events.ndjson +9c6db4c450336e8baf8d394bc92d74eb130eb5dd7e7582c99cb8b73a0396d6af junit.xml +a57f7bf0cd1a1aacf97e14bea315dce837b6e5868d53fae6fbee6fd8476c74c4 manifest.json +b4ee208d09915e158fe85330285b5576feb9ccb01a47619875328487b8b63f67 report.html +8cfa272fcc6f53235fce607379d037d3904b9ab1469a90755a85217e2cf2e765 results.json +ecdd9521795d33c7b522c66168a4d2f3aa1801fe64186d5bc72735bfeabe2519 scenario.lock.json diff --git a/conformance/evidence/reference-leaky/events.ndjson b/conformance/evidence/reference-leaky/events.ndjson new file mode 100644 index 0000000..54c36f6 --- /dev/null +++ b/conformance/evidence/reference-leaky/events.ndjson @@ -0,0 +1,15 @@ +{"seq":1,"at_ms":1787321604376,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321604376,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"reference-leaky","backend":"memoryproof-reference","version":"1.0.0"}} +{"seq":3,"at_ms":1787321604376,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321604376,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321604376,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321604376,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321604376,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"target-before"}} +{"seq":8,"at_ms":1787321604376,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"control-before"}} +{"seq":9,"at_ms":1787321604377,"phase":"run","method":"erase","status":"PASS","details":{}} +{"seq":10,"at_ms":1787321604377,"phase":"after-erase","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":11,"at_ms":1787321604377,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"target-after"}} +{"seq":12,"at_ms":1787321604377,"phase":"after","method":"probe","status":"PASS","details":{"found":"true","probe":"control-after"}} +{"seq":13,"at_ms":1787321604377,"phase":"after","method":"inspect","status":"PASS","details":{"found":"true","probe":"target-derived-after"}} +{"seq":14,"at_ms":1787321604377,"phase":"after","method":"agent_query","status":"PASS","details":{"found":"true","probe":"target-agent-after"}} +{"seq":15,"at_ms":1787321604377,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/reference-leaky/junit.xml b/conformance/evidence/reference-leaky/junit.xml new file mode 100644 index 0000000..9fc7270 --- /dev/null +++ b/conformance/evidence/reference-leaky/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/reference-leaky/manifest.json b/conformance/evidence/reference-leaky/manifest.json new file mode 100644 index 0000000..97aaa46 --- /dev/null +++ b/conformance/evidence/reference-leaky/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321604334-79610", + "created_at_ms": 1787321604378, + "scenario_hash": "1a1ebd045054ba001f87b2592698dd654151edd31e718e3eed62b18fee4d8af1", + "adapter": "reference-leaky", + "backend": "memoryproof-reference", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/reference-leaky/report.html b/conformance/evidence/reference-leaky/report.html new file mode 100644 index 0000000..e0fe343 --- /dev/null +++ b/conformance/evidence/reference-leaky/report.html @@ -0,0 +1,33 @@ + + + +MemoryProof · reference-leaky +
+
MEMORYPROOF · erasure

reference-leaky

reference-leaky · memoryproof-reference · 1.0.0

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

FAIL
+
Run / 运行run-1787321604334-79610
Scenario hash / 场景哈希1a1ebd045054ba001f87b2592698dd654151edd31e718e3eed62b18fee4d8af1
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码1
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+ + +
Profile / 档案Status / 状态Assertions / 断言
erasure.object
MemoryProof · Erasure · Object
PASSbefore.target-before, before.control-before, after.target-after, after.control-after
erasure.scope
MemoryProof · Erasure · Scope
PASSscope.control-after
erasure.derived
MemoryProof · Erasure · Derived
FAILafter.target-derived-after
erasure.agent
MemoryProof · Erasure · Agent
FAILafter.target-agent-after
+

Assertions / 断言明细

+ + + + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-beforeerasure.objectexactPASSexact canary 'target' was observable before the erase operationtrue / true
before.control-beforeerasure.objectexactPASSexact canary 'control' was observable before the erase operationtrue / true
after.target-aftererasure.objectobjectPASSexact probe for fixture 'target' expected absent, observed absentfalse / false
after.control-aftererasure.objectobjectPASSexact probe for fixture 'control' expected present, observed presenttrue / true
scope.control-aftererasure.scopescopePASScontrol fixture 'control' remained observable after target erasetrue / true
after.target-derived-aftererasure.derivedderivedFAILinspect probe for fixture 'target' expected absent, observed presentfalse / true
after.target-agent-aftererasure.agentagentFAILagent probe for fixture 'target' expected absent, observed presentfalse / true
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/reference-leaky/results.json b/conformance/evidence/reference-leaky/results.json new file mode 100644 index 0000000..8b079de --- /dev/null +++ b/conformance/evidence/reference-leaky/results.json @@ -0,0 +1,155 @@ +{ + "run_id": "run-1787321604334-79610", + "suite": "erasure", + "scenario": "reference-leaky", + "adapter": "reference-leaky", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "1a1ebd045054ba001f87b2592698dd654151edd31e718e3eed62b18fee4d8af1", + "status": "FAIL", + "exit_code": 1, + "profiles": [ + { + "profile": "erasure.object", + "label": "MemoryProof · Erasure · Object", + "status": "PASS", + "assertions": [ + "before.target-before", + "before.control-before", + "after.target-after", + "after.control-after" + ] + }, + { + "profile": "erasure.scope", + "label": "MemoryProof · Erasure · Scope", + "status": "PASS", + "assertions": [ + "scope.control-after" + ] + }, + { + "profile": "erasure.derived", + "label": "MemoryProof · Erasure · Derived", + "status": "FAIL", + "assertions": [ + "after.target-derived-after" + ] + }, + { + "profile": "erasure.agent", + "label": "MemoryProof · Erasure · Agent", + "status": "FAIL", + "assertions": [ + "after.target-agent-after" + ] + } + ], + "assertions": [ + { + "id": "before.target-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'target' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:target-before" + ] + }, + { + "id": "before.control-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'control' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:control-before" + ] + }, + { + "id": "after.target-after", + "profile": "erasure.object", + "status": "PASS", + "message": "exact probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "object", + "evidence": [ + "probe:target-after" + ] + }, + { + "id": "after.control-after", + "profile": "erasure.object", + "status": "PASS", + "message": "exact probe for fixture 'control' expected present, observed present", + "expected": true, + "observed": true, + "artifact": "object", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "scope.control-after", + "profile": "erasure.scope", + "status": "PASS", + "message": "control fixture 'control' remained observable after target erase", + "expected": true, + "observed": true, + "artifact": "scope", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "after.target-derived-after", + "profile": "erasure.derived", + "status": "FAIL", + "message": "inspect probe for fixture 'target' expected absent, observed present", + "expected": false, + "observed": true, + "artifact": "derived", + "evidence": [ + "probe:target-derived-after" + ] + }, + { + "id": "after.target-agent-after", + "profile": "erasure.agent", + "status": "FAIL", + "message": "agent probe for fixture 'target' expected absent, observed present", + "expected": false, + "observed": true, + "artifact": "agent", + "evidence": [ + "probe:target-agent-after" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "derived_delete", + "agent_query", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/reference-leaky/scenario.lock.json b/conformance/evidence/reference-leaky/scenario.lock.json new file mode 100644 index 0000000..7bbc03c --- /dev/null +++ b/conformance/evidence/reference-leaky/scenario.lock.json @@ -0,0 +1,110 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "A deliberately leaky backend leaves derived artifacts after deletion.", + "name": "reference-leaky" + }, + "spec": { + "adapter": { + "config": {}, + "mode": "leaky", + "name": "reference-leaky" + }, + "erase": { + "intent": "subject_erase", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:9df585a469698301386fcf8fa937ed898fcb0bb0d2ecbae8501a37acd697d4d6 len:38", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:ec352e339e595077e80dc559d83350b185d517967d83202117e0e2986216783e len:38", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-after", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-after", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-derived-after", + "kind": "inspect", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-agent-after", + "kind": "agent", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "erasure.object", + "erasure.scope", + "erasure.derived", + "erasure.agent" + ], + "settle": { + "interval_ms": 25, + "timeout_ms": 5000 + }, + "suite": "erasure" + } +} \ No newline at end of file diff --git a/conformance/evidence/reference-overdelete/bundle.hash b/conformance/evidence/reference-overdelete/bundle.hash new file mode 100644 index 0000000..4c0b1d9 --- /dev/null +++ b/conformance/evidence/reference-overdelete/bundle.hash @@ -0,0 +1 @@ +f0a0d36541832468cb67459319daf3617d99b0f11c3d9e7482f2a28bc4787b9a diff --git a/conformance/evidence/reference-overdelete/checksums.sha256 b/conformance/evidence/reference-overdelete/checksums.sha256 new file mode 100644 index 0000000..98ab0ca --- /dev/null +++ b/conformance/evidence/reference-overdelete/checksums.sha256 @@ -0,0 +1,6 @@ +2cda36d44f570ac716004b772a0e81fd45a38ae4f637902eee21320a9387ef7b events.ndjson +c815275adebffac0f1da33d35633f265d71a6bb8a03102e913135c738c3e7cc3 junit.xml +7a671dd1923b84a4196e53c27d6ae5beb0aa8909928613502a60296054898c05 manifest.json +5bdac8408e1149013b2d0af251799d916e439116e93e2d2c983dd45f0bb84a2c report.html +1834dec14f27eeedb49baeab6d216af1d8ffc8e4803aded14d6a3cedd0b0905c results.json +fd3b05d74b39d23ebae7b4cc8320576daa1a2c40f6bc597e638e23ce017a9dec scenario.lock.json diff --git a/conformance/evidence/reference-overdelete/events.ndjson b/conformance/evidence/reference-overdelete/events.ndjson new file mode 100644 index 0000000..ff20034 --- /dev/null +++ b/conformance/evidence/reference-overdelete/events.ndjson @@ -0,0 +1,13 @@ +{"seq":1,"at_ms":1787321607839,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321607839,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"reference-overdelete","backend":"memoryproof-reference","version":"1.0.0"}} +{"seq":3,"at_ms":1787321607839,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321607839,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321607839,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321607839,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321607840,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"target-before"}} +{"seq":8,"at_ms":1787321607840,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"control-before"}} +{"seq":9,"at_ms":1787321607840,"phase":"run","method":"erase","status":"PASS","details":{}} +{"seq":10,"at_ms":1787321607840,"phase":"after-erase","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":11,"at_ms":1787321607840,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"target-after"}} +{"seq":12,"at_ms":1787321607840,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"control-after"}} +{"seq":13,"at_ms":1787321607840,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/reference-overdelete/junit.xml b/conformance/evidence/reference-overdelete/junit.xml new file mode 100644 index 0000000..731ec4e --- /dev/null +++ b/conformance/evidence/reference-overdelete/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/reference-overdelete/manifest.json b/conformance/evidence/reference-overdelete/manifest.json new file mode 100644 index 0000000..5066ec8 --- /dev/null +++ b/conformance/evidence/reference-overdelete/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321607787-79618", + "created_at_ms": 1787321607841, + "scenario_hash": "67f27aa547f4003a605b67e46256a633e94dccb16beae79a307d86c3d0f521b7", + "adapter": "reference-overdelete", + "backend": "memoryproof-reference", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/reference-overdelete/report.html b/conformance/evidence/reference-overdelete/report.html new file mode 100644 index 0000000..91589ff --- /dev/null +++ b/conformance/evidence/reference-overdelete/report.html @@ -0,0 +1,29 @@ + + + +MemoryProof · reference-overdelete +
+
MEMORYPROOF · erasure

reference-overdelete

reference-overdelete · memoryproof-reference · 1.0.0

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

FAIL
+
Run / 运行run-1787321607787-79618
Scenario hash / 场景哈希67f27aa547f4003a605b67e46256a633e94dccb16beae79a307d86c3d0f521b7
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码1
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+
Profile / 档案Status / 状态Assertions / 断言
erasure.object
MemoryProof · Erasure · Object
FAILbefore.target-before, before.control-before, after.target-after, after.control-after
erasure.scope
MemoryProof · Erasure · Scope
FAILscope.control-after
+

Assertions / 断言明细

+ + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-beforeerasure.objectexactPASSexact canary 'target' was observable before the erase operationtrue / true
before.control-beforeerasure.objectexactPASSexact canary 'control' was observable before the erase operationtrue / true
after.target-aftererasure.objectobjectPASSexact probe for fixture 'target' expected absent, observed absentfalse / false
after.control-aftererasure.objectobjectFAILexact probe for fixture 'control' expected present, observed absenttrue / false
scope.control-aftererasure.scopescopeFAILcontrol fixture 'control' remained absent after target erasetrue / false
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/reference-overdelete/results.json b/conformance/evidence/reference-overdelete/results.json new file mode 100644 index 0000000..2d616a5 --- /dev/null +++ b/conformance/evidence/reference-overdelete/results.json @@ -0,0 +1,115 @@ +{ + "run_id": "run-1787321607787-79618", + "suite": "erasure", + "scenario": "reference-overdelete", + "adapter": "reference-overdelete", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "67f27aa547f4003a605b67e46256a633e94dccb16beae79a307d86c3d0f521b7", + "status": "FAIL", + "exit_code": 1, + "profiles": [ + { + "profile": "erasure.object", + "label": "MemoryProof · Erasure · Object", + "status": "FAIL", + "assertions": [ + "before.target-before", + "before.control-before", + "after.target-after", + "after.control-after" + ] + }, + { + "profile": "erasure.scope", + "label": "MemoryProof · Erasure · Scope", + "status": "FAIL", + "assertions": [ + "scope.control-after" + ] + } + ], + "assertions": [ + { + "id": "before.target-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'target' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:target-before" + ] + }, + { + "id": "before.control-before", + "profile": "erasure.object", + "status": "PASS", + "message": "exact canary 'control' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "exact", + "evidence": [ + "probe:control-before" + ] + }, + { + "id": "after.target-after", + "profile": "erasure.object", + "status": "PASS", + "message": "exact probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "object", + "evidence": [ + "probe:target-after" + ] + }, + { + "id": "after.control-after", + "profile": "erasure.object", + "status": "FAIL", + "message": "exact probe for fixture 'control' expected present, observed absent", + "expected": true, + "observed": false, + "artifact": "object", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "scope.control-after", + "profile": "erasure.scope", + "status": "FAIL", + "message": "control fixture 'control' remained absent after target erase", + "expected": true, + "observed": false, + "artifact": "scope", + "evidence": [ + "probe:control-after" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "derived_delete", + "agent_query", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/reference-overdelete/scenario.lock.json b/conformance/evidence/reference-overdelete/scenario.lock.json new file mode 100644 index 0000000..e8133d3 --- /dev/null +++ b/conformance/evidence/reference-overdelete/scenario.lock.json @@ -0,0 +1,94 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "A deliberately unsafe backend deletes the control subject too.", + "name": "reference-overdelete" + }, + "spec": { + "adapter": { + "config": {}, + "mode": "overdelete", + "name": "reference-overdelete" + }, + "erase": { + "intent": "subject_erase", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:d2d85e38b6a4e6310c553541490c2dd8ca431bfde350dcfdd9d61023621dba68 len:36", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:9a22732ab65e4317166170cc89b9ad57135c3c954fe76b092be85f4cdff673ee len:37", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-after", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-after", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-before", + "kind": "exact", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "erasure.object", + "erasure.scope" + ], + "settle": { + "interval_ms": 25, + "timeout_ms": 5000 + }, + "suite": "erasure" + } +} \ No newline at end of file diff --git a/conformance/evidence/zep-adapter-contract/bundle.hash b/conformance/evidence/zep-adapter-contract/bundle.hash new file mode 100644 index 0000000..64fbfc3 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/bundle.hash @@ -0,0 +1 @@ +0ba7ba7f701f49d1e3f2236300812db0a538c082f3b765caf661e76b2498bbc8 diff --git a/conformance/evidence/zep-adapter-contract/checksums.sha256 b/conformance/evidence/zep-adapter-contract/checksums.sha256 new file mode 100644 index 0000000..2885484 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/checksums.sha256 @@ -0,0 +1,6 @@ +13862a8c91981e0a6616d63b6729511740a01bef75099d9248c9a7872a720d7a events.ndjson +fcd7415a65a28f82447dcdf7a10d10904bf5e8f07dd7a0017291b264e55dcddd junit.xml +238dc915485c0048e9b99fa35ed11e47bff20683b0fa6c4e95bc608976230535 manifest.json +2e0bf62e127016eb26dbfc0dfabdc0690356bb241fe0544fe133846771097412 report.html +fd0523c591a98bac146ee043ed3d4d469ea6a67ffb7b0df191eae01f17dad468 results.json +6df490702d15da34c939eaddee6ee7120d47bddc1d5f5dc2824598d428dd4a56 scenario.lock.json diff --git a/conformance/evidence/zep-adapter-contract/events.ndjson b/conformance/evidence/zep-adapter-contract/events.ndjson new file mode 100644 index 0000000..4822421 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/events.ndjson @@ -0,0 +1,14 @@ +{"seq":1,"at_ms":1787321631505,"phase":"prepare","method":"hello","status":"PASS","details":{"protocol":"memoryproof.adapter/v1"}} +{"seq":2,"at_ms":1787321631505,"phase":"prepare","method":"capabilities","status":"PASS","details":{"adapter":"zep","backend":"zep","version":"configured"}} +{"seq":3,"at_ms":1787321631538,"phase":"run","method":"prepare","status":"PASS","details":{}} +{"seq":4,"at_ms":1787321631544,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":5,"at_ms":1787321631551,"phase":"run","method":"ingest","status":"PASS","details":{}} +{"seq":6,"at_ms":1787321631551,"phase":"after-ingest","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":7,"at_ms":1787321631558,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"target-before"}} +{"seq":8,"at_ms":1787321631565,"phase":"before","method":"probe","status":"PASS","details":{"found":"true","probe":"control-before"}} +{"seq":9,"at_ms":1787321631571,"phase":"run","method":"erase","status":"PASS","details":{}} +{"seq":10,"at_ms":1787321631571,"phase":"after-erase","method":"settle","status":"PASS","details":{"state":"stable"}} +{"seq":11,"at_ms":1787321631578,"phase":"after","method":"probe","status":"PASS","details":{"found":"false","probe":"target-after"}} +{"seq":12,"at_ms":1787321631584,"phase":"after","method":"probe","status":"PASS","details":{"found":"true","probe":"control-after"}} +{"seq":13,"at_ms":1787321631591,"phase":"after","method":"inspect","status":"PASS","details":{"found":"false","probe":"target-derived-after"}} +{"seq":14,"at_ms":1787321631603,"phase":"cleanup","method":"cleanup","status":"PASS","details":{}} diff --git a/conformance/evidence/zep-adapter-contract/junit.xml b/conformance/evidence/zep-adapter-contract/junit.xml new file mode 100644 index 0000000..45b97e3 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/conformance/evidence/zep-adapter-contract/manifest.json b/conformance/evidence/zep-adapter-contract/manifest.json new file mode 100644 index 0000000..170f23a --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/manifest.json @@ -0,0 +1,18 @@ +{ + "format": "memoryproof.bundle/v1", + "run_id": "run-1787321627919-79681", + "created_at_ms": 1787321631604, + "scenario_hash": "454c8931d265e9eb7d1705249091c31c1343869d930bb643b3ffec38a27bfb1b", + "adapter": "zep", + "backend": "zep", + "protocol": "memoryproof.adapter/v1", + "files": [ + "manifest.json", + "scenario.lock.json", + "events.ndjson", + "results.json", + "report.html", + "junit.xml" + ], + "bundle_hash": "" +} \ No newline at end of file diff --git a/conformance/evidence/zep-adapter-contract/report.html b/conformance/evidence/zep-adapter-contract/report.html new file mode 100644 index 0000000..01eadc1 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/report.html @@ -0,0 +1,31 @@ + + + +MemoryProof · zep-erasure +
+
MEMORYPROOF · erasure

zep-erasure

zep · zep · configured

Prove what was observable, what disappeared, and what remains unknown.
证明可观察到什么、什么已经消失,以及什么仍然未知。

PASS
+
Run / 运行run-1787321627919-79681
Scenario hash / 场景哈希454c8931d265e9eb7d1705249091c31c1343869d930bb643b3ffec38a27bfb1b
Protocol / 协议memoryproof.adapter/v1
Exit code / 退出码0
+

Evidence summary / 证据摘要

Proved / 已证明

Required assertions marked PASS passed deterministic observable checks.

Observed residue / 发现残留

FAIL means a probe still observed the target or a forbidden derivative.

Unknown / 未知

UNKNOWN means the backend did not expose enough evidence to claim more.

+

Profiles / 认证档案

+ +
Profile / 档案Status / 状态Assertions / 断言
erasure.object
MemoryProof · Erasure · Object
PASSbefore.target-before, before.control-before, after.target-after, after.control-after
erasure.scope
MemoryProof · Erasure · Scope
PASSscope.control-after
erasure.derived
MemoryProof · Erasure · Derived
PASSafter.target-derived-after
+

Assertions / 断言明细

+ + + + +
IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
before.target-beforeerasure.objectsemanticPASSsemantic canary 'target' was observable before the erase operationtrue / true
before.control-beforeerasure.objectsemanticPASSsemantic canary 'control' was observable before the erase operationtrue / true
after.target-aftererasure.objectsemantic-indexPASSsemantic probe for fixture 'target' expected absent, observed absentfalse / false
after.control-aftererasure.objectsemantic-indexPASSsemantic probe for fixture 'control' expected present, observed presenttrue / true
scope.control-aftererasure.scopescopePASScontrol fixture 'control' remained observable after target erasetrue / true
after.target-derived-aftererasure.derivedderivedPASSinspect probe for fixture 'target' expected absent, observed absentfalse / false
+

Warnings / 警告

  • None / 无
+

Out of scope / 无法证明

  • provider logs and backups
  • physical storage erasure
  • model-weight unlearning
  • unobservable artifacts outside the adapter boundary
+
MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
+
\ No newline at end of file diff --git a/conformance/evidence/zep-adapter-contract/results.json b/conformance/evidence/zep-adapter-contract/results.json new file mode 100644 index 0000000..9d10382 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/results.json @@ -0,0 +1,133 @@ +{ + "run_id": "run-1787321627919-79681", + "suite": "erasure", + "scenario": "zep-erasure", + "adapter": "zep", + "backend": "zep", + "backend_version": "configured", + "protocol": "memoryproof.adapter/v1", + "scenario_hash": "454c8931d265e9eb7d1705249091c31c1343869d930bb643b3ffec38a27bfb1b", + "status": "PASS", + "exit_code": 0, + "profiles": [ + { + "profile": "erasure.object", + "label": "MemoryProof · Erasure · Object", + "status": "PASS", + "assertions": [ + "before.target-before", + "before.control-before", + "after.target-after", + "after.control-after" + ] + }, + { + "profile": "erasure.scope", + "label": "MemoryProof · Erasure · Scope", + "status": "PASS", + "assertions": [ + "scope.control-after" + ] + }, + { + "profile": "erasure.derived", + "label": "MemoryProof · Erasure · Derived", + "status": "PASS", + "assertions": [ + "after.target-derived-after" + ] + } + ], + "assertions": [ + { + "id": "before.target-before", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic canary 'target' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "semantic", + "evidence": [ + "probe:target-before" + ] + }, + { + "id": "before.control-before", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic canary 'control' was observable before the erase operation", + "expected": true, + "observed": true, + "artifact": "semantic", + "evidence": [ + "probe:control-before" + ] + }, + { + "id": "after.target-after", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "semantic-index", + "evidence": [ + "probe:target-after" + ] + }, + { + "id": "after.control-after", + "profile": "erasure.object", + "status": "PASS", + "message": "semantic probe for fixture 'control' expected present, observed present", + "expected": true, + "observed": true, + "artifact": "semantic-index", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "scope.control-after", + "profile": "erasure.scope", + "status": "PASS", + "message": "control fixture 'control' remained observable after target erase", + "expected": true, + "observed": true, + "artifact": "scope", + "evidence": [ + "probe:control-after" + ] + }, + { + "id": "after.target-derived-after", + "profile": "erasure.derived", + "status": "PASS", + "message": "inspect probe for fixture 'target' expected absent, observed absent", + "expected": false, + "observed": false, + "artifact": "derived", + "evidence": [ + "probe:target-derived-after" + ] + } + ], + "capabilities": [ + "object_delete", + "scope_delete", + "probe", + "lexical_search", + "semantic_search", + "inspect", + "derived_inspect", + "async_settle", + "isolated_namespace" + ], + "out_of_scope": [ + "provider logs and backups", + "physical storage erasure", + "model-weight unlearning", + "unobservable artifacts outside the adapter boundary" + ], + "warnings": [] +} \ No newline at end of file diff --git a/conformance/evidence/zep-adapter-contract/scenario.lock.json b/conformance/evidence/zep-adapter-contract/scenario.lock.json new file mode 100644 index 0000000..faf4d27 --- /dev/null +++ b/conformance/evidence/zep-adapter-contract/scenario.lock.json @@ -0,0 +1,104 @@ +{ + "apiVersion": "memoryproof.dev/v1", + "kind": "AssuranceScenario", + "metadata": { + "description": "Zep episode and graph erasure with separate temporary users.", + "name": "zep-erasure" + }, + "spec": { + "adapter": { + "config": { + "base_url": "http://localhost:8000" + }, + "mode": "self-hosted", + "name": "zep" + }, + "erase": { + "intent": "subject_erase", + "scope": "", + "target": "target" + }, + "fixtures": [ + { + "content": "sha256:acc522fc044c565cb84eebf3fc180284403187eee97c341219e7d037e0cac3f5 len:36", + "id": "target", + "kind": "memory", + "namespace": "", + "role": "target", + "subject": "target-subject", + "target": false + }, + { + "content": "sha256:8aaf3b5f99ce306785b8143d1a1f5fa1b35777de55b6d41172e9146693290843 len:30", + "id": "control", + "kind": "memory", + "namespace": "", + "role": "control", + "subject": "control-subject", + "target": false + } + ], + "isolation": { + "agent": "", + "control_subject": "control-subject", + "scope": "memoryproof-run", + "subject": "", + "target_subject": "target-subject", + "thread": "" + }, + "privacy": { + "raw_payloads": false + }, + "probes": { + "after": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-after", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-after", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "target", + "id": "target-derived-after", + "kind": "inspect", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ], + "before": [ + { + "as_subject": "", + "fixture": "target", + "id": "target-before", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + }, + { + "as_subject": "", + "fixture": "control", + "id": "control-before", + "kind": "semantic", + "query": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 len:0" + } + ] + }, + "profiles": [ + "erasure.object", + "erasure.scope", + "erasure.derived" + ], + "settle": { + "interval_ms": 1000, + "timeout_ms": 30000 + }, + "suite": "erasure" + } +} \ No newline at end of file diff --git a/crates/forgetproof/Cargo.toml b/crates/forgetproof/Cargo.toml index 0522a72..a4d0a1f 100644 --- a/crates/forgetproof/Cargo.toml +++ b/crates/forgetproof/Cargo.toml @@ -1,9 +1,10 @@ [package] -name = "forgetproof" +name = "memoryproof" version.workspace = true edition.workspace = true license.workspace = true -description = "Evidence-driven conformance tests for AI memory erasure / 面向 AI 记忆遗忘的证据驱动认证测试" +description = "Assurance tests for AI agent memory / AI Agent 记忆保障测试" +default-run = "memoryproof" [dependencies] anyhow.workspace = true @@ -14,5 +15,9 @@ serde_yaml.workspace = true sha2.workspace = true [[bin]] -name = "forgetproof" +name = "memoryproof" path = "src/main.rs" + +[[bin]] +name = "forgetproof" +path = "src/forgetproof.rs" diff --git a/crates/forgetproof/src/cli.rs b/crates/forgetproof/src/cli.rs new file mode 100644 index 0000000..b691c3c --- /dev/null +++ b/crates/forgetproof/src/cli.rs @@ -0,0 +1,224 @@ +use crate::{evidence, model, runner}; +use anyhow::{bail, Context, Result}; +use clap::{Args, Parser, Subcommand}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::fs; +use std::path::PathBuf; +use std::process; + +#[derive(Debug, Parser)] +#[command( + name = "memoryproof", + version, + about = "Test what your AI remembers. Prove what it forgot." +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Create local configuration and reference scenarios. + Init { + #[arg(default_value = ".")] + path: PathBuf, + }, + /// Inspect an adapter without mutating a backend. + Doctor(AdapterArgs), + /// Manage registered adapters. + Adapters { + #[command(subcommand)] + command: AdapterCommand, + }, + /// Freeze deterministic lexical and semantic probe variants. + Expand { + input: PathBuf, + #[arg(short, long)] + output: PathBuf, + }, + /// Execute a MemoryProof scenario and write an evidence bundle. + Run { + scenario: PathBuf, + #[arg(long, default_value = ".memoryproof/runs")] + output: PathBuf, + #[arg(long, default_value_t = false, default_missing_value = "true", num_args = 0..=1)] + allow_network: bool, + }, + /// Verify an evidence bundle's checksums and format. + Verify { bundle: PathBuf }, + /// Regenerate reports and refresh the evidence checksums. + Report { bundle: PathBuf }, + /// Validate a generated static conformance matrix. + Matrix { + #[command(subcommand)] + command: MatrixCommand, + }, +} + +#[derive(Debug, Subcommand)] +enum AdapterCommand { + List, +} + +#[derive(Debug, Subcommand)] +enum MatrixCommand { + Validate { + #[arg(default_value = "site/matrix.json")] + path: PathBuf, + }, +} + +#[derive(Debug, Args)] +struct AdapterArgs { + #[arg(long)] + adapter: Option, + #[arg(long, default_value = "default")] + mode: String, + /// Repeat as --config key=value. Values are passed to the adapter process. + #[arg(long = "config", value_parser = parse_key_value)] + config: Vec<(String, String)>, +} + +fn parse_key_value(value: &str) -> Result<(String, String), String> { + value + .split_once('=') + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .ok_or_else(|| "expected key=value".to_owned()) +} + +pub fn run() -> Result<()> { + let cli = Cli::parse(); + match cli.command { + Command::Init { path } => { + runner::init_project(&path)?; + println!("initialized MemoryProof project at {}", path.display()); + } + Command::Doctor(args) => { + let config = args.config.into_iter().collect::>(); + let names = args.adapter.map_or_else( + || { + vec![ + "reference-clean".to_owned(), + "reference-leaky".to_owned(), + "reference-overdelete".to_owned(), + "mem0".to_owned(), + "letta".to_owned(), + "zep".to_owned(), + ] + }, + |name| vec![name], + ); + for name in names { + let capabilities = runner::doctor_adapter(&name, &args.mode, &config)?; + println!("adapter: {}", capabilities.adapter); + println!("backend: {}", capabilities.backend); + println!("protocol: {}", capabilities.protocol); + println!("version: {}", capabilities.version); + println!("capabilities: {}", capabilities.capabilities.join(", ")); + if !capabilities.modes.is_empty() { + println!("modes: {}", capabilities.modes.join(", ")); + } + } + } + Command::Adapters { command } => match command { + AdapterCommand::List => { + println!("NAME KIND PROTOCOL"); + for name in [ + "reference-clean", + "reference-leaky", + "reference-overdelete", + "reference-slow", + "reference-crash", + "reference-malformed", + ] { + println!("{name:<22} built-in {}", model::PROTOCOL_VERSION); + } + for name in ["mem0", "letta", "zep"] { + println!("{name:<22} python {}", model::PROTOCOL_VERSION); + } + } + }, + Command::Expand { input, output } => { + runner::expand_scenario(&input, &output)?; + println!("wrote frozen scenario to {}", output.display()); + } + Command::Run { + scenario, + output, + allow_network, + } => { + let outcome = runner::run_scenario(&scenario, &output, allow_network)?; + println!("status: {}", outcome.result.status); + println!("exit code: {}", outcome.result.exit_code); + println!("bundle: {}", outcome.directory.display()); + println!("bundle hash: {}", outcome.bundle_hash); + if !outcome.result.warnings.is_empty() { + println!("warnings:"); + for warning in outcome.result.warnings { + println!(" - {warning}"); + } + } + process::exit(outcome.result.exit_code); + } + Command::Verify { bundle } => { + let verification = evidence::verify_bundle(&bundle)?; + if verification.valid { + println!("valid bundle: {}", verification.bundle_hash); + } else { + println!("invalid bundle: {}", verification.bundle_hash); + for error in verification.errors { + println!(" - {error}"); + } + process::exit(1); + } + } + Command::Report { bundle } => { + let bundle_hash = evidence::refresh_bundle(&bundle)?; + println!( + "regenerated report.html and junit.xml in {}", + bundle.display() + ); + println!("bundle hash: {bundle_hash}"); + } + Command::Matrix { command } => match command { + MatrixCommand::Validate { path } => { + validate_matrix(&path)?; + println!("valid matrix: {}", path.display()); + } + }, + } + Ok(()) +} + +fn validate_matrix(path: &PathBuf) -> Result<()> { + let value: Value = serde_json::from_slice( + &fs::read(path).with_context(|| format!("failed to read {}", path.display()))?, + ) + .with_context(|| format!("invalid JSON in {}", path.display()))?; + let rows = value + .get("results") + .and_then(Value::as_array) + .ok_or_else(|| anyhow::anyhow!("matrix results must be an array"))?; + for row in rows { + let status = row + .get("status") + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("matrix row is missing status"))?; + if !["PASS", "FAIL", "SKIP", "UNKNOWN"].contains(&status) { + bail!("unsupported matrix status '{status}'"); + } + for field in ["adapter", "backend", "profile", "evidence"] { + if row + .get(field) + .and_then(Value::as_str) + .unwrap_or("") + .is_empty() + { + bail!("matrix row is missing {field}"); + } + } + } + Ok(()) +} diff --git a/crates/forgetproof/src/evidence.rs b/crates/forgetproof/src/evidence.rs index b1a8f98..3bb0d83 100644 --- a/crates/forgetproof/src/evidence.rs +++ b/crates/forgetproof/src/evidence.rs @@ -1,12 +1,14 @@ -use crate::model::{Assertion, Event, Manifest, ProfileResult, RunResult}; -use anyhow::{Context, Result}; +use crate::model::{ + normalize_value, Assertion, Event, Manifest, ProfileResult, RunResult, BUNDLE_FORMAT, +}; +use anyhow::{bail, Context, Result}; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File}; use std::io::{BufRead, BufReader, Write}; -use std::path::Path; +use std::path::{Component, Path}; pub fn sha256_hex(bytes: &[u8]) -> String { let digest = Sha256::digest(bytes); @@ -40,8 +42,10 @@ pub fn write_bundle( ) -> Result { fs::create_dir_all(directory)?; - let scenario_path = directory.join("scenario.lock.json"); - write_json(&scenario_path, &redact_scenario(scenario))?; + write_json( + &directory.join("scenario.lock.json"), + &redact_scenario(scenario), + )?; let events_path = directory.join("events.ndjson"); let mut events_file = File::create(&events_path)?; @@ -55,19 +59,31 @@ pub fn write_bundle( fs::write(directory.join("junit.xml"), render_junit(result))?; write_json(&directory.join("manifest.json"), manifest)?; - let files = [ - "manifest.json", - "scenario.lock.json", - "events.ndjson", - "results.json", - "report.html", - "junit.xml", - ]; - let mut checksums = String::new(); - for file in files { - let bytes = fs::read(directory.join(file))?; - checksums.push_str(&format!("{} {}\n", sha256_hex(&bytes), file)); - } + let files = manifest + .files + .iter() + .map(String::as_str) + .collect::>(); + let checksums = checksum_text(directory, &files)?; + fs::write(directory.join("checksums.sha256"), &checksums)?; + let bundle_hash = sha256_hex(checksums.as_bytes()); + fs::write(directory.join("bundle.hash"), format!("{bundle_hash}\n"))?; + Ok(bundle_hash) +} + +pub fn refresh_bundle(directory: &Path) -> Result { + let result = load_result(directory)?; + fs::write(directory.join("report.html"), render_html(&result))?; + fs::write(directory.join("junit.xml"), render_junit(&result))?; + let manifest: Manifest = serde_json::from_slice(&fs::read(directory.join("manifest.json"))?)?; + let checksums = checksum_text( + directory, + &manifest + .files + .iter() + .map(String::as_str) + .collect::>(), + )?; fs::write(directory.join("checksums.sha256"), &checksums)?; let bundle_hash = sha256_hex(checksums.as_bytes()); fs::write(directory.join("bundle.hash"), format!("{bundle_hash}\n"))?; @@ -75,10 +91,24 @@ pub fn write_bundle( } pub fn verify_bundle(directory: &Path) -> Result { + let manifest_path = directory.join("manifest.json"); + let manifest: Manifest = serde_json::from_slice( + &fs::read(&manifest_path) + .with_context(|| format!("missing {}", manifest_path.display()))?, + ) + .context("invalid manifest.json")?; let checksums_path = directory.join("checksums.sha256"); let file = File::open(&checksums_path) .with_context(|| format!("missing {}", checksums_path.display()))?; let mut errors = Vec::new(); + if manifest.format != BUNDLE_FORMAT && manifest.format != "forgetproof.bundle/v1alpha1" { + errors.push(format!( + "unsupported evidence format '{}', expected '{}'", + manifest.format, BUNDLE_FORMAT + )); + } + let expected_files = manifest.files.iter().cloned().collect::>(); + let mut seen_files = BTreeSet::new(); let mut checksum_text = String::new(); for line in BufReader::new(file).lines() { let line = line?; @@ -87,11 +117,26 @@ pub fn verify_bundle(directory: &Path) -> Result { let mut parts = line.splitn(2, " "); let expected = parts.next().unwrap_or_default(); let name = parts.next().unwrap_or_default(); - if expected.is_empty() || name.is_empty() { + if expected.len() != 64 + || !expected.chars().all(|c| c.is_ascii_hexdigit()) + || name.is_empty() + { errors.push(format!("invalid checksum line: {line}")); continue; } + if !seen_files.insert(name.to_owned()) { + errors.push(format!("duplicate checksum entry: {name}")); + } + if !expected_files.contains(name) { + errors.push(format!( + "checksum lists file not declared in manifest: {name}" + )); + } let path = directory.join(name); + if !safe_bundle_path(name) { + errors.push(format!("unsafe bundle path: {name}")); + continue; + } match fs::read(&path) { Ok(bytes) => { let actual = sha256_hex(&bytes); @@ -102,6 +147,13 @@ pub fn verify_bundle(directory: &Path) -> Result { Err(_) => errors.push(format!("missing bundle file: {name}")), } } + for expected in expected_files { + if !seen_files.contains(&expected) { + errors.push(format!( + "manifest file missing from checksum list: {expected}" + )); + } + } let expected_bundle = fs::read_to_string(directory.join("bundle.hash")) .unwrap_or_default() .trim() @@ -129,6 +181,32 @@ pub fn load_result(directory: &Path) -> Result { Ok(serde_json::from_slice(&bytes)?) } +fn checksum_text(directory: &Path, files: &[&str]) -> Result { + let mut ordered = files.to_vec(); + ordered.sort_unstable(); + let mut checksums = String::new(); + for file in ordered { + if !safe_bundle_path(file) { + bail!("unsafe bundle path: {file}"); + } + let bytes = fs::read(directory.join(file)) + .with_context(|| format!("missing bundle file: {file}"))?; + checksums.push_str(&format!("{} {file}\n", sha256_hex(&bytes))); + } + Ok(checksums) +} + +fn safe_bundle_path(name: &str) -> bool { + let path = Path::new(name); + !path.is_absolute() + && path.components().all(|component| { + !matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) +} + fn redact_scenario(scenario: &Value) -> Value { let raw_payloads = scenario .get("spec") @@ -137,22 +215,44 @@ fn redact_scenario(scenario: &Value) -> Value { .and_then(Value::as_bool) .unwrap_or(false); if raw_payloads { - return scenario.clone(); + return normalize_value(scenario); } - let mut safe = scenario.clone(); - if let Some(fixtures) = safe - .get_mut("spec") - .and_then(Value::as_object_mut) - .and_then(|spec| spec.get_mut("fixtures")) - .and_then(Value::as_array_mut) - { - for fixture in fixtures { - if let Some(object) = fixture.as_object_mut() { - if let Some(content) = object.get("content").and_then(Value::as_str) { - object.insert( - "content".to_owned(), - Value::String(redact_content(content, false)), - ); + let mut safe = normalize_value(scenario); + if let Some(spec) = safe.get_mut("spec").and_then(Value::as_object_mut) { + if let Some(fixtures) = spec.get_mut("fixtures").and_then(Value::as_array_mut) { + for fixture in fixtures { + if let Some(object) = fixture.as_object_mut() { + if let Some(content) = object.get("content").and_then(Value::as_str) { + object.insert( + "content".to_owned(), + Value::String(redact_content(content, false)), + ); + } + } + } + } + if let Some(probes) = spec.get_mut("probes").and_then(Value::as_object_mut) { + for phase in ["before", "after"] { + if let Some(items) = probes.get_mut(phase).and_then(Value::as_array_mut) { + for probe in items { + if let Some(object) = probe.as_object_mut() { + if let Some(query) = object.get("query").and_then(Value::as_str) { + object.insert( + "query".to_owned(), + Value::String(redact_content(query, false)), + ); + } + } + } + } + } + } + if let Some(adapter) = spec.get_mut("adapter").and_then(Value::as_object_mut) { + if let Some(config) = adapter.get_mut("config").and_then(Value::as_object_mut) { + for (key, value) in config.iter_mut() { + if is_sensitive_key(key) { + *value = Value::String("".to_owned()); + } } } } @@ -160,6 +260,20 @@ fn redact_scenario(scenario: &Value) -> Value { safe } +fn is_sensitive_key(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + [ + "key", + "token", + "secret", + "password", + "credential", + "authorization", + ] + .iter() + .any(|part| key.contains(part)) +} + pub fn render_html(result: &RunResult) -> String { let status_class = result.status.to_lowercase(); let profiles = result @@ -174,42 +288,70 @@ pub fn render_html(result: &RunResult) -> String { .map(render_assertion_row) .collect::>() .join("\n"); + let warnings = result + .warnings + .iter() + .map(|warning| format!("
  • {}
  • ", html_escape(warning))) + .collect::>() + .join(""); + let out_of_scope = result + .out_of_scope + .iter() + .map(|item| format!("
  • {}
  • ", html_escape(item))) + .collect::>() + .join(""); format!( r#" -ForgetProof · {scenario} + +MemoryProof · {scenario}
    -

    FORGETPROOF · Prove your AI forgot.

    {scenario}

    {adapter} · {backend}

    {status}
    -
    Run{run_id}
    Scenario hash{scenario_hash}
    Exit code{exit_code}
    Evidence policyredacted by default
    -

    Conformance profiles

    {profiles}
    ProfileStatusAssertions
    -

    Assertions

    {assertions}
    IDProfileStatusMessage
    -

    ForgetProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, or physical storage erasure.

    +
    MEMORYPROOF · {suite}

    {scenario}

    {adapter} · {backend} · {backend_version}

    Prove what was observable, what disappeared, and what remains unknown.
    证明可观察到什么、什么已经消失,以及什么仍然未知。

    {status}
    +
    Run / 运行{run_id}
    Scenario hash / 场景哈希{scenario_hash}
    Protocol / 协议{protocol}
    Exit code / 退出码{exit_code}
    +

    Evidence summary / 证据摘要

    Proved / 已证明

    Required assertions marked PASS passed deterministic observable checks.

    Observed residue / 发现残留

    FAIL means a probe still observed the target or a forbidden derivative.

    Unknown / 未知

    UNKNOWN means the backend did not expose enough evidence to claim more.

    +

    Profiles / 认证档案

    {profiles}
    Profile / 档案Status / 状态Assertions / 断言
    +

    Assertions / 断言明细

    {assertions}
    IDProfileArtifact / 边界StatusMessage / 说明Expected / Observed
    +

    Warnings / 警告

      {warnings}
    +

    Out of scope / 无法证明

      {out_of_scope}
    +
    MemoryProof reports controlled, observable evidence. It does not prove provider logs, backups, model-weight unlearning, physical storage erasure, or anything outside the adapter boundary.
    MemoryProof 只报告受控且可观察的证据,不证明服务商日志、备份、模型权重反学习、物理存储擦除或适配器边界之外的事情。
    "#, + suite = html_escape(&result.suite), scenario = html_escape(&result.scenario), adapter = html_escape(&result.adapter), backend = html_escape(&result.backend), + backend_version = html_escape(&result.backend_version), status = html_escape(&result.status), status_class = html_escape(&status_class), run_id = html_escape(&result.run_id), scenario_hash = html_escape(&result.scenario_hash), + protocol = html_escape(&result.protocol), exit_code = result.exit_code, profiles = profiles, assertions = assertions, + warnings = if warnings.is_empty() { + "
  • None / 无
  • ".to_owned() + } else { + warnings + }, + out_of_scope = out_of_scope, ) } fn render_profile_row(profile: &ProfileResult) -> String { format!( - "{}{}{}", + "{}
    {}{}{}", html_escape(&profile.profile), + html_escape(&profile.label), html_escape(&profile.status.to_lowercase()), html_escape(&profile.status), html_escape(&profile.assertions.join(", ")) @@ -217,13 +359,24 @@ fn render_profile_row(profile: &ProfileResult) -> String { } fn render_assertion_row(assertion: &Assertion) -> String { + let expected = assertion + .expected + .map(|value| value.to_string()) + .unwrap_or_else(|| "—".to_owned()); + let observed = assertion + .observed + .map(|value| value.to_string()) + .unwrap_or_else(|| "—".to_owned()); format!( - "{}{}{}{}", + "{}{}{}{}{}{} / {}", html_escape(&assertion.id), html_escape(&assertion.profile), + html_escape(&assertion.artifact), html_escape(&assertion.status.to_lowercase()), html_escape(&assertion.status), - html_escape(&assertion.message) + html_escape(&assertion.message), + html_escape(&expected), + html_escape(&observed), ) } @@ -248,7 +401,7 @@ pub fn render_junit(result: &RunResult) -> String { .collect::>() .join(""); format!( - "{}", + "{}", result.assertions.len(), cases ) @@ -270,3 +423,8 @@ fn xml_escape(value: &str) -> String { pub fn details(pairs: impl IntoIterator) -> BTreeMap { pairs.into_iter().collect() } + +#[allow(dead_code)] +fn _assertion_types_are_serializable(_assertion: &Assertion, _profile: &ProfileResult) -> Value { + Value::Null +} diff --git a/crates/forgetproof/src/forgetproof.rs b/crates/forgetproof/src/forgetproof.rs new file mode 100644 index 0000000..162d252 --- /dev/null +++ b/crates/forgetproof/src/forgetproof.rs @@ -0,0 +1,6 @@ +fn main() { + if let Err(error) = memoryproof::cli::run() { + eprintln!("error: {error:#}"); + std::process::exit(2); + } +} diff --git a/crates/forgetproof/src/lib.rs b/crates/forgetproof/src/lib.rs new file mode 100644 index 0000000..00e5115 --- /dev/null +++ b/crates/forgetproof/src/lib.rs @@ -0,0 +1,5 @@ +pub mod cli; +pub mod evidence; +pub mod model; +pub mod protocol; +pub mod runner; diff --git a/crates/forgetproof/src/main.rs b/crates/forgetproof/src/main.rs index f845c8a..162d252 100644 --- a/crates/forgetproof/src/main.rs +++ b/crates/forgetproof/src/main.rs @@ -1,168 +1,6 @@ -mod evidence; -mod model; -mod protocol; -mod runner; - -use anyhow::Result; -use clap::{Args, Parser, Subcommand}; -use std::collections::BTreeMap; -use std::path::PathBuf; -use std::process; - -#[derive(Debug, Parser)] -#[command(name = "forgetproof", version, about = "Prove your AI forgot.")] -struct Cli { - #[command(subcommand)] - command: Command, -} - -#[derive(Debug, Subcommand)] -enum Command { - /// Create local configuration and reference scenarios. - Init { - #[arg(default_value = ".")] - path: PathBuf, - }, - /// Inspect an adapter without mutating a backend. - Doctor(AdapterArgs), - /// Manage registered adapters. - Adapters { - #[command(subcommand)] - command: AdapterCommand, - }, - /// Freeze deterministic lexical and semantic probe variants. - Expand { - input: PathBuf, - #[arg(short, long)] - output: PathBuf, - }, - /// Execute an erasure scenario and write an evidence bundle. - Run { - scenario: PathBuf, - #[arg(long, default_value = ".forgetproof/runs")] - output: PathBuf, - #[arg(long, default_value_t = false, default_missing_value = "true", num_args = 0..=1)] - allow_network: bool, - }, - /// Verify an evidence bundle's checksums. - Verify { bundle: PathBuf }, - /// Regenerate HTML and JUnit reports from results.json. - Report { bundle: PathBuf }, -} - -#[derive(Debug, Subcommand)] -enum AdapterCommand { - List, -} - -#[derive(Debug, Args)] -struct AdapterArgs { - #[arg(long)] - adapter: Option, - #[arg(long, default_value = "default")] - mode: String, - /// Repeat as --config key=value. Values are passed to the adapter process. - #[arg(long = "config", value_parser = parse_key_value)] - config: Vec<(String, String)>, -} - -fn parse_key_value(value: &str) -> Result<(String, String), String> { - value - .split_once('=') - .map(|(key, value)| (key.to_owned(), value.to_owned())) - .ok_or_else(|| "expected key=value".to_owned()) -} - fn main() { - if let Err(error) = run_cli() { + if let Err(error) = memoryproof::cli::run() { eprintln!("error: {error:#}"); - process::exit(2); - } -} - -fn run_cli() -> Result<()> { - let cli = Cli::parse(); - match cli.command { - Command::Init { path } => { - runner::init_project(&path)?; - println!("initialized ForgetProof project at {}", path.display()); - } - Command::Doctor(args) => { - let config = args.config.into_iter().collect::>(); - let names = args.adapter.map_or_else( - || { - vec![ - "reference-clean".to_owned(), - "reference-leaky".to_owned(), - "mem0".to_owned(), - "letta".to_owned(), - "zep".to_owned(), - ] - }, - |name| vec![name], - ); - for name in names { - let capabilities = runner::doctor_adapter(&name, &args.mode, &config)?; - println!("adapter: {}", capabilities.adapter); - println!("backend: {}", capabilities.backend); - println!("protocol: {}", capabilities.protocol); - println!("version: {}", capabilities.version); - println!("capabilities: {}", capabilities.capabilities.join(", ")); - } - } - Command::Adapters { command } => match command { - AdapterCommand::List => { - println!("NAME KIND PROTOCOL"); - println!("reference-clean built-in {}", model::PROTOCOL_VERSION); - println!("reference-leaky built-in {}", model::PROTOCOL_VERSION); - println!("mem0 python {}", model::PROTOCOL_VERSION); - println!("letta python {}", model::PROTOCOL_VERSION); - println!("zep python {}", model::PROTOCOL_VERSION); - } - }, - Command::Expand { input, output } => { - runner::expand_scenario(&input, &output)?; - println!("wrote frozen scenario to {}", output.display()); - } - Command::Run { - scenario, - output, - allow_network, - } => { - let outcome = runner::run_scenario(&scenario, &output, allow_network)?; - println!("status: {}", outcome.result.status); - println!("exit code: {}", outcome.result.exit_code); - println!("bundle: {}", outcome.directory.display()); - println!("bundle hash: {}", outcome.bundle_hash); - if !outcome.result.warnings.is_empty() { - println!("warnings:"); - for warning in outcome.result.warnings { - println!(" - {warning}"); - } - } - process::exit(outcome.result.exit_code); - } - Command::Verify { bundle } => { - let verification = evidence::verify_bundle(&bundle)?; - if verification.valid { - println!("valid bundle: {}", verification.bundle_hash); - } else { - println!("invalid bundle: {}", verification.bundle_hash); - for error in verification.errors { - println!(" - {error}"); - } - process::exit(1); - } - } - Command::Report { bundle } => { - let result = evidence::load_result(&bundle)?; - std::fs::write(bundle.join("report.html"), evidence::render_html(&result))?; - std::fs::write(bundle.join("junit.xml"), evidence::render_junit(&result))?; - println!( - "regenerated report.html and junit.xml in {}", - bundle.display() - ); - } + std::process::exit(2); } - Ok(()) } diff --git a/crates/forgetproof/src/model.rs b/crates/forgetproof/src/model.rs index f3eecc6..d2f4584 100644 --- a/crates/forgetproof/src/model.rs +++ b/crates/forgetproof/src/model.rs @@ -1,8 +1,17 @@ use serde::{Deserialize, Serialize}; +use serde_json::Value; use std::collections::BTreeMap; -pub const PROTOCOL_VERSION: &str = "forgetproof.adapter/v1alpha1"; -pub const API_VERSION: &str = "forgetproof.dev/v1alpha1"; +pub const PROTOCOL_VERSION: &str = "memoryproof.adapter/v1"; +#[allow(dead_code)] +pub const LEGACY_PROTOCOL_VERSION: &str = "forgetproof.adapter/v1alpha1"; +pub const API_VERSION: &str = "memoryproof.dev/v1"; +pub const LEGACY_API_VERSION: &str = "forgetproof.dev/v1alpha1"; +pub const BUNDLE_FORMAT: &str = "memoryproof.bundle/v1"; + +fn default_suite() -> String { + "erasure".to_owned() +} fn default_scope() -> String { "run".to_owned() @@ -33,13 +42,21 @@ fn default_probe_kind() -> String { } fn default_profiles() -> Vec { - vec!["FP-Object".to_owned()] + vec!["erasure.object".to_owned()] } fn default_fixture_kind() -> String { "memory".to_owned() } +fn default_target_subject() -> String { + "target".to_owned() +} + +fn default_control_subject() -> String { + "control".to_owned() +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Scenario { #[serde(rename = "apiVersion")] @@ -59,6 +76,8 @@ pub struct Metadata { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ScenarioSpec { + #[serde(default = "default_suite")] + pub suite: String, pub adapter: AdapterSpec, #[serde(default)] pub isolation: IsolationSpec, @@ -89,6 +108,10 @@ pub struct AdapterSpec { pub struct IsolationSpec { #[serde(default = "default_scope")] pub scope: String, + #[serde(default = "default_target_subject")] + pub target_subject: String, + #[serde(default = "default_control_subject")] + pub control_subject: String, #[serde(default)] pub subject: String, #[serde(default)] @@ -101,6 +124,8 @@ impl Default for IsolationSpec { fn default() -> Self { Self { scope: default_scope(), + target_subject: default_target_subject(), + control_subject: default_control_subject(), subject: String::new(), agent: String::new(), thread: String::new(), @@ -113,11 +138,38 @@ pub struct Fixture { pub id: String, pub content: String, #[serde(default)] + pub role: String, + #[serde(default)] pub target: bool, + #[serde(default)] + pub subject: String, + #[serde(default)] + pub namespace: String, #[serde(default = "default_fixture_kind")] pub kind: String, } +impl Fixture { + pub fn is_target(&self) -> bool { + self.role == "target" || (self.role.is_empty() && self.target) + } + + pub fn is_control(&self) -> bool { + self.role == "control" || (self.role.is_empty() && !self.target) + } + + pub fn effective_subject(&self, isolation: &IsolationSpec) -> String { + if !self.subject.is_empty() { + return self.subject.clone(); + } + if self.is_target() { + isolation.target_subject.clone() + } else { + isolation.control_subject.clone() + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SettleSpec { #[serde(default = "default_timeout")] @@ -141,6 +193,8 @@ pub struct EraseSpec { pub intent: String, #[serde(default = "default_target")] pub target: String, + #[serde(default)] + pub scope: String, } impl Default for EraseSpec { @@ -148,6 +202,7 @@ impl Default for EraseSpec { Self { intent: default_intent(), target: default_target(), + scope: String::new(), } } } @@ -168,6 +223,8 @@ pub struct Probe { pub kind: String, #[serde(default)] pub query: String, + #[serde(default)] + pub as_subject: String, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -177,11 +234,15 @@ pub struct PrivacySpec { } impl Scenario { + pub fn is_legacy(&self) -> bool { + self.api_version == LEGACY_API_VERSION + } + pub fn validate(&self) -> Result<(), Vec> { let mut errors = Vec::new(); - if self.api_version != API_VERSION { + if self.api_version != API_VERSION && self.api_version != LEGACY_API_VERSION { errors.push(format!( - "apiVersion must be {API_VERSION}, got {}", + "apiVersion must be {API_VERSION} or {LEGACY_API_VERSION}, got {}", self.api_version )); } @@ -191,25 +252,61 @@ impl Scenario { if self.spec.adapter.name.trim().is_empty() { errors.push("spec.adapter.name must not be empty".to_owned()); } + if self.spec.suite != "erasure" && self.spec.suite != "isolation" { + errors.push(format!( + "spec.suite must be 'erasure' or 'isolation', got '{}'", + self.spec.suite + )); + } if self.spec.fixtures.is_empty() { errors.push("spec.fixtures must contain at least one fixture".to_owned()); } - if !self.spec.fixtures.iter().any(|fixture| fixture.target) { + let mut ids = std::collections::BTreeSet::new(); + for fixture in &self.spec.fixtures { + if fixture.id.trim().is_empty() { + errors.push("fixture id must not be empty".to_owned()); + } + if !ids.insert(fixture.id.clone()) { + errors.push(format!("duplicate fixture id '{}'", fixture.id)); + } + if fixture.content.trim().is_empty() { + errors.push(format!( + "fixture '{}' content must not be empty", + fixture.id + )); + } + if !self.is_legacy() && fixture.role != "target" && fixture.role != "control" { + errors.push(format!( + "fixture '{}' must declare role target or control", + fixture.id + )); + } + } + if !self.spec.fixtures.iter().any(Fixture::is_target) { errors.push("spec.fixtures must contain a target fixture".to_owned()); } - if !self - .spec - .fixtures - .iter() - .any(|fixture| fixture.id == self.spec.erase.target) + if self.spec.suite == "erasure" + && !self + .spec + .fixtures + .iter() + .any(|fixture| fixture.id == self.spec.erase.target && fixture.is_target()) { errors.push(format!( - "erase.target '{}' does not name a fixture", + "erase.target '{}' must name a target fixture", self.spec.erase.target )); } - if self.spec.probes.before.is_empty() || self.spec.probes.after.is_empty() { - errors.push("probes.before and probes.after must both contain probes".to_owned()); + if (self.spec.suite == "erasure" || self.spec.profiles.iter().any(|p| p.contains("scope"))) + && !self.spec.fixtures.iter().any(Fixture::is_control) + { + errors.push("scope and isolation checks require a control fixture".to_owned()); + } + if self.spec.probes.before.is_empty() { + errors.push("probes.before must contain probes".to_owned()); + } + if self.spec.suite == "erasure" && self.spec.probes.after.is_empty() { + errors.push("erasure scenarios require probes.after".to_owned()); } for probe in self .spec @@ -218,6 +315,9 @@ impl Scenario { .iter() .chain(self.spec.probes.after.iter()) { + if probe.id.trim().is_empty() { + errors.push("probe id must not be empty".to_owned()); + } if !self .spec .fixtures @@ -229,6 +329,24 @@ impl Scenario { probe.id, probe.fixture )); } + if !["exact", "lexical", "semantic", "inspect", "agent"].contains(&probe.kind.as_str()) + { + errors.push(format!( + "probe '{}' has unsupported kind '{}'", + probe.id, probe.kind + )); + } + } + if self.spec.profiles.is_empty() { + errors.push("spec.profiles must contain at least one profile".to_owned()); + } + for profile in &self.spec.profiles { + if !supported_profile(profile, &self.spec.suite) { + errors.push(format!( + "unsupported profile '{}' for suite '{}'", + profile, self.spec.suite + )); + } } if errors.is_empty() { Ok(()) @@ -236,6 +354,87 @@ impl Scenario { Err(errors) } } + + pub fn normalize_legacy(&mut self) { + if !self.is_legacy() { + return; + } + self.api_version = API_VERSION.to_owned(); + self.kind = if self.kind.is_empty() { + "AssuranceScenario".to_owned() + } else { + self.kind.clone() + }; + if self.spec.suite.is_empty() { + self.spec.suite = "erasure".to_owned(); + } + for fixture in &mut self.spec.fixtures { + if fixture.role.is_empty() { + fixture.role = if fixture.target { "target" } else { "control" }.to_owned(); + } + } + for profile in &mut self.spec.profiles { + *profile = legacy_profile(profile); + } + } + + pub fn probe_subject(&self, probe: &Probe) -> String { + if !probe.as_subject.is_empty() { + return probe.as_subject.clone(); + } + self.spec + .fixtures + .iter() + .find(|fixture| fixture.id == probe.fixture) + .map(|fixture| fixture.effective_subject(&self.spec.isolation)) + .unwrap_or_else(|| self.spec.isolation.target_subject.clone()) + } +} + +pub fn legacy_profile(profile: &str) -> String { + match profile { + "FP-Object" => "erasure.object".to_owned(), + "FP-Scope" => "erasure.scope".to_owned(), + "FP-Derived" => "erasure.derived".to_owned(), + "FP-Agent" => "erasure.agent".to_owned(), + other => other.to_owned(), + } +} + +pub fn display_profile(profile: &str) -> &str { + match profile { + "erasure.object" => "MemoryProof · Erasure · Object", + "erasure.scope" => "MemoryProof · Erasure · Scope", + "erasure.derived" => "MemoryProof · Erasure · Derived", + "erasure.agent" => "MemoryProof · Erasure · Agent", + "isolation.read" => "IsolationProof · Read", + "isolation.search" => "IsolationProof · Search", + "isolation.agent" => "IsolationProof · Agent", + other => other, + } +} + +pub fn supported_profile(profile: &str, suite: &str) -> bool { + match suite { + "erasure" => matches!( + profile, + "erasure.object" + | "erasure.scope" + | "erasure.derived" + | "erasure.agent" + | "FP-Object" + | "FP-Scope" + | "FP-Derived" + | "FP-Agent" + ), + "isolation" => { + matches!( + profile, + "isolation.read" | "isolation.search" | "isolation.agent" + ) + } + _ => false, + } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -248,6 +447,8 @@ pub struct Capabilities { pub version: String, #[serde(default)] pub capabilities: Vec, + #[serde(default)] + pub modes: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -261,12 +462,16 @@ pub struct Assertion { #[serde(default)] pub observed: Option, #[serde(default)] + pub artifact: String, + #[serde(default)] pub evidence: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ProfileResult { pub profile: String, + #[serde(default)] + pub label: String, pub status: String, pub assertions: Vec, } @@ -274,18 +479,32 @@ pub struct ProfileResult { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RunResult { pub run_id: String, + #[serde(default = "default_suite")] + pub suite: String, pub scenario: String, pub adapter: String, pub backend: String, + #[serde(default)] + pub backend_version: String, + #[serde(default = "default_protocol")] + pub protocol: String, pub scenario_hash: String, pub status: String, pub exit_code: i32, pub profiles: Vec, pub assertions: Vec, #[serde(default)] + pub capabilities: Vec, + #[serde(default)] + pub out_of_scope: Vec, + #[serde(default)] pub warnings: Vec, } +fn default_protocol() -> String { + LEGACY_PROTOCOL_VERSION.to_owned() +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Manifest { pub format: String, @@ -310,3 +529,61 @@ pub struct Event { #[serde(default)] pub details: BTreeMap, } + +pub fn profile_capabilities(profile: &str) -> &'static [&'static str] { + match profile { + "erasure.object" | "FP-Object" => &["object_delete", "probe"], + "erasure.scope" | "FP-Scope" => &["scope_delete", "probe"], + "erasure.derived" | "FP-Derived" => &["derived_inspect", "inspect"], + "erasure.agent" | "FP-Agent" => &["agent_query"], + "isolation.read" => &["isolated_namespace", "probe"], + "isolation.search" => &["isolated_namespace", "lexical_search", "semantic_search"], + "isolation.agent" => &["isolated_namespace", "agent_query"], + _ => &[], + } +} + +pub fn profile_for_probe(probe: &Probe, suite: &str) -> String { + if suite == "isolation" { + return match probe.kind.as_str() { + "agent" => "isolation.agent".to_owned(), + "lexical" | "semantic" => "isolation.search".to_owned(), + _ => "isolation.read".to_owned(), + }; + } + match probe.kind.as_str() { + "inspect" => "erasure.derived".to_owned(), + "agent" => "erasure.agent".to_owned(), + _ => "erasure.object".to_owned(), + } +} + +pub fn erase_capability(intent: &str) -> &'static str { + match intent { + "subject_erase" => "scope_delete", + "derived_purge" => "derived_delete", + "access_revoke" => "access_revoke", + _ => "object_delete", + } +} + +pub fn capability_available(capabilities: &Capabilities, capability: &str) -> bool { + capabilities + .capabilities + .iter() + .any(|item| item == capability) +} + +pub fn normalize_value(value: &Value) -> Value { + match value { + Value::Object(map) => { + let sorted = map + .iter() + .map(|(key, value)| (key.clone(), normalize_value(value))) + .collect::>(); + serde_json::to_value(sorted).unwrap_or(Value::Null) + } + Value::Array(values) => Value::Array(values.iter().map(normalize_value).collect()), + other => other.clone(), + } +} diff --git a/crates/forgetproof/src/protocol.rs b/crates/forgetproof/src/protocol.rs index b4d2566..d04c47c 100644 --- a/crates/forgetproof/src/protocol.rs +++ b/crates/forgetproof/src/protocol.rs @@ -17,21 +17,36 @@ pub struct AdapterClient { impl AdapterClient { pub fn spawn(name: &str, mode: &str, config: &BTreeMap) -> Result { - let python = std::env::var("FORGETPROOF_PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let python = std::env::var("MEMORYPROOF_PYTHON") + .or_else(|_| std::env::var("FORGETPROOF_PYTHON")) + .unwrap_or_else(|_| { + if cfg!(windows) { + "python".to_owned() + } else { + "python3".to_owned() + } + }); let module = match name { - "reference-clean" | "reference-leaky" => "forgetproof_adapters.reference", + "reference-clean" + | "reference-leaky" + | "reference-overdelete" + | "reference-slow" + | "reference-crash" + | "reference-malformed" => "forgetproof_adapters.reference", "mem0" => "forgetproof_adapters.mem0", "letta" => "forgetproof_adapters.letta", "zep" => "forgetproof_adapters.zep", - other => bail!("unknown registered adapter '{other}'"), + other => bail!("unknown registered adapter '{other}'; use 'memoryproof adapters list'"), }; - let effective_mode = if name == "reference-clean" { - "clean" - } else if name == "reference-leaky" { - "leaky" - } else { - mode + let effective_mode = match name { + "reference-clean" => "clean", + "reference-leaky" => "leaky", + "reference-overdelete" => "overdelete", + "reference-slow" => "slow", + "reference-crash" => "crash", + "reference-malformed" => "malformed", + _ => mode, }; let config_json = serde_json::to_string(config)?; @@ -39,22 +54,26 @@ impl AdapterClient { command .arg("-m") .arg(module) + .env("MEMORYPROOF_ADAPTER_MODE", effective_mode) .env("FORGETPROOF_ADAPTER_MODE", effective_mode) + .env("MEMORYPROOF_ADAPTER_NAME", name) .env("FORGETPROOF_ADAPTER_NAME", name) - .env("FORGETPROOF_CONFIG_JSON", config_json) + .env("MEMORYPROOF_CONFIG_JSON", &config_json) + .env("FORGETPROOF_CONFIG_JSON", &config_json) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()); + if let Ok(current_dir) = std::env::current_dir() { let package_root = current_dir.join("python"); if package_root.join("forgetproof_adapters").is_dir() { - let existing = std::env::var("PYTHONPATH").unwrap_or_default(); - let value = if existing.is_empty() { - package_root.display().to_string() - } else { - format!("{}:{}", package_root.display(), existing) - }; - command.env("PYTHONPATH", value); + let mut paths = vec![package_root]; + if let Some(existing) = std::env::var_os("PYTHONPATH") { + paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(paths) + .context("failed to construct adapter PYTHONPATH")?; + command.env("PYTHONPATH", joined); } } @@ -95,9 +114,14 @@ impl AdapterClient { "method": method, "params": params, }); - serde_json::to_writer(&mut self.stdin, &request)?; - self.stdin.write_all(b"\n")?; - self.stdin.flush()?; + serde_json::to_writer(&mut self.stdin, &request) + .with_context(|| format!("failed to write adapter request '{method}'"))?; + self.stdin + .write_all(b"\n") + .with_context(|| format!("failed to terminate adapter request '{method}'"))?; + self.stdin + .flush() + .with_context(|| format!("failed to flush adapter request '{method}'"))?; let line = self .lines @@ -110,10 +134,13 @@ impl AdapterClient { anyhow!("adapter exited while handling '{method}'") } })??; + if line.trim().is_empty() { + bail!("adapter emitted an empty response for '{method}'"); + } let response: Value = serde_json::from_str(&line) .with_context(|| format!("adapter emitted invalid JSON for '{method}'"))?; if response.get("id").and_then(Value::as_str) != Some(id.as_str()) { - bail!("adapter response id mismatch for '{method}'") + bail!("adapter response id mismatch for '{method}'"); } if response.get("ok").and_then(Value::as_bool) == Some(true) { Ok(response.get("result").cloned().unwrap_or(Value::Null)) @@ -127,17 +154,35 @@ impl AdapterClient { .get("message") .and_then(Value::as_str) .unwrap_or("adapter returned an error"); - bail!("{code}: {message}") + bail!("{code}: {message}"); } } pub fn hello(&mut self, timeout_ms: u64) -> Result { - self.call("hello", json!({ "protocol": PROTOCOL_VERSION }), timeout_ms) + let value = self.call("hello", json!({ "protocol": PROTOCOL_VERSION }), timeout_ms)?; + let protocol = value + .get("protocol") + .and_then(Value::as_str) + .ok_or_else(|| anyhow!("adapter hello response omitted protocol"))?; + if protocol != PROTOCOL_VERSION { + bail!( + "protocol mismatch: runner expects {PROTOCOL_VERSION}, adapter returned {protocol}" + ); + } + Ok(value) } pub fn capabilities(&mut self, timeout_ms: u64) -> Result { let value = self.call("capabilities", json!({}), timeout_ms)?; - serde_json::from_value(value).context("invalid capabilities response") + let capabilities: Capabilities = + serde_json::from_value(value).context("invalid capabilities response")?; + if capabilities.protocol != PROTOCOL_VERSION { + bail!( + "protocol mismatch in capabilities: expected {PROTOCOL_VERSION}, got {}", + capabilities.protocol + ); + } + Ok(capabilities) } pub fn close(&mut self) { diff --git a/crates/forgetproof/src/runner.rs b/crates/forgetproof/src/runner.rs index 9c71b4a..6856c17 100644 --- a/crates/forgetproof/src/runner.rs +++ b/crates/forgetproof/src/runner.rs @@ -1,6 +1,8 @@ use crate::evidence::{details, hash_text, write_bundle}; use crate::model::{ - Assertion, Capabilities, Event, Manifest, Probe, ProfileResult, RunResult, Scenario, + capability_available, display_profile, erase_capability, profile_capabilities, + profile_for_probe, Assertion, Capabilities, Event, Fixture, Manifest, Probe, ProfileResult, + RunResult, Scenario, BUNDLE_FORMAT, PROTOCOL_VERSION, }; use crate::protocol::AdapterClient; use anyhow::{anyhow, bail, Context, Result}; @@ -26,12 +28,17 @@ pub fn load_scenario(path: &Path) -> Result<(Scenario, Value)> { } else { serde_yaml::from_str(&source).context("invalid YAML scenario")? }; - let scenario: Scenario = - serde_json::from_value(value.clone()).context("invalid scenario schema")?; + let mut scenario: Scenario = + serde_json::from_value(value).context("invalid scenario schema")?; scenario .validate() .map_err(|errors| anyhow!(errors.join("; ")))?; - Ok((scenario, value)) + scenario.normalize_legacy(); + scenario + .validate() + .map_err(|errors| anyhow!(errors.join("; ")))?; + let normalized = serde_json::to_value(&scenario)?; + Ok((scenario, normalized)) } pub fn run_scenario( @@ -45,27 +52,31 @@ pub fn run_scenario( let run_id = format!("run-{}-{}", now_ms(), std::process::id()); let directory = output_root.join(&run_id); fs::create_dir_all(&directory)?; - let scenario_hash = hash_text(&serde_json::to_string(&scenario_value)?); - let timeout = scenario.spec.settle.timeout_ms.max(1_000); + let normalized_scenario = serde_json::to_string(&scenario_value)?; + let scenario_hash = hash_text(&normalized_scenario); + // Python startup can be materially slower on macOS/Windows runners when + // several conformance tests start adapters concurrently. The scenario + // settle timeout still controls backend convergence; this bound only + // prevents a healthy adapter handshake from being classified as a crash. + let call_timeout = scenario.spec.settle.timeout_ms.max(30_000); let mut events = Vec::new(); + let mut warnings = Vec::new(); + let mut assertions = Vec::new(); let mut client = AdapterClient::spawn( &scenario.spec.adapter.name, &scenario.spec.adapter.mode, &scenario.spec.adapter.config, )?; - client.hello(timeout)?; + client.hello(call_timeout)?; record_event( &mut events, "prepare", "hello", "PASS", - details([( - String::from("protocol"), - crate::model::PROTOCOL_VERSION.to_owned(), - )]), + details([(String::from("protocol"), PROTOCOL_VERSION.to_owned())]), ); - let capabilities = client.capabilities(timeout)?; + let capabilities = client.capabilities(call_timeout)?; record_event( &mut events, "prepare", @@ -74,25 +85,16 @@ pub fn run_scenario( details([ (String::from("adapter"), capabilities.adapter.clone()), (String::from("backend"), capabilities.backend.clone()), + (String::from("version"), capabilities.version.clone()), ]), ); - - let mut assertions = Vec::new(); - let mut warnings = Vec::new(); add_capability_assertions(&scenario, &capabilities, &mut assertions); let fixture_payloads = scenario .spec .fixtures .iter() - .map(|fixture| { - json!({ - "id": fixture.id, - "content": fixture.content, - "target": fixture.target, - "kind": fixture.kind, - }) - }) + .map(|fixture| fixture_payload(&scenario, fixture)) .collect::>(); call_checked( &mut client, @@ -100,10 +102,11 @@ pub fn run_scenario( "prepare", json!({ "run_id": run_id, + "suite": scenario.spec.suite, "isolation": scenario.spec.isolation, "fixtures": fixture_payloads, }), - timeout, + call_timeout, )?; for fixture in &scenario.spec.fixtures { @@ -113,225 +116,301 @@ pub fn run_scenario( "ingest", json!({ "run_id": run_id, - "fixture": { - "id": fixture.id, - "content": fixture.content, - "target": fixture.target, - "kind": fixture.kind, - } + "fixture": fixture_payload(&scenario, fixture), }), - timeout, + call_timeout, )?; } - settle(&mut client, &mut events, &scenario, timeout)?; - - for probe in &scenario.spec.probes.before { - let fixture = fixture_by_id(&scenario, &probe.fixture)?; - let observed = perform_probe(&mut client, &mut events, &run_id, probe, "before", timeout)?; - if !observed { - bail!( - "precondition failed: fixture '{}' was not observable before erase", - fixture.id - ); - } - assertions.push(Assertion { - id: format!("before.{}", probe.id), - profile: "PRECONDITION".to_owned(), - status: "PASS".to_owned(), - message: format!("fixture '{}' was observable before erase", fixture.id), - expected: Some(true), - observed: Some(observed), - evidence: vec![format!("probe:{}", probe.id)], - }); - } - let erase_performed = - has_capability(&capabilities, erase_capability(&scenario.spec.erase.intent)); - if erase_performed { - call_checked( - &mut client, - &mut events, - "erase", - json!({ - "run_id": run_id, - "intent": scenario.spec.erase.intent, - "target": scenario.spec.erase.target, - }), - timeout, - )?; - } else { - warnings.push(format!( - "erase intent '{}' is not supported by adapter", - scenario.spec.erase.intent - )); - } - settle(&mut client, &mut events, &scenario, timeout)?; + let initial_stable = settle( + &mut client, + &mut events, + &scenario, + call_timeout, + &mut warnings, + "after-ingest", + )?; - for probe in &scenario.spec.probes.after { - let fixture = fixture_by_id(&scenario, &probe.fixture)?; - let observed = perform_probe(&mut client, &mut events, &run_id, probe, "after", timeout)?; - let expected = fixture.id != scenario.spec.erase.target; - let profile = profile_for_probe(probe); - assertions.push(if erase_performed { - assertion_for_probe( - &format!("after.{}", probe.id), - &profile, - expected, - observed, - fixture.id.as_str(), + let mut precondition_failed = false; + if scenario.spec.suite == "erasure" { + for probe in &scenario.spec.probes.before { + let fixture = fixture_by_id(&scenario, &probe.fixture)?; + let profile = profile_for_probe(probe, &scenario.spec.suite); + let observed = run_probe_if_supported( + &mut client, + &mut events, + &scenario, + &run_id, probe, - ) - } else { - unknown_assertion_for_probe( - &format!("after.{}", probe.id), - &profile, - "erase operation was not available; post-erase observation is inconclusive", + "before", + call_timeout, + &capabilities, + &mut assertions, + )?; + match observed { + Some(true) if initial_stable => assertions.push(assertion( + &format!("before.{}", probe.id), + &profile, + "PASS", + format!( + "{} canary '{}' was observable before the erase operation", + probe.kind, fixture.id + ), + Some(true), + Some(true), + &probe.kind, + format!("probe:{}", probe.id), + )), + Some(false) => { + precondition_failed = true; + assertions.push(assertion( + &format!("before.{}", probe.id), + &profile, + "ERROR", + format!( + "canary '{}' was not observable before the erase operation", + fixture.id + ), + Some(true), + Some(false), + &probe.kind, + format!("probe:{}", probe.id), + )); + } + Some(true) => { + precondition_failed = true; + assertions.push(unknown_assertion( + &format!("before.{}", probe.id), + &profile, + "backend did not reach a stable state before the precondition probe", + &probe.kind, + format!("probe:{}", probe.id), + )); + } + None => precondition_failed = true, + } + } + } else { + for probe in &scenario.spec.probes.before { + let fixture = fixture_by_id(&scenario, &probe.fixture)?; + let profile = profile_for_probe(probe, &scenario.spec.suite); + let expected = scenario.probe_subject(probe) + == fixture.effective_subject(&scenario.spec.isolation); + let observed = run_probe_if_supported( + &mut client, + &mut events, + &scenario, + &run_id, probe, - ) - }); - if profile == "FP-Object" - && selected(&scenario, "FP-Scope") - && fixture.id != scenario.spec.erase.target - { - assertions.push(if erase_performed { - assertion_for_probe( - &format!("scope.{}", probe.id), - "FP-Scope", - true, - observed, - fixture.id.as_str(), - probe, - ) - } else { - unknown_assertion_for_probe( - &format!("scope.{}", probe.id), - "FP-Scope", - "erase operation was not available; scope observation is inconclusive", - probe, - ) - }); + "isolation", + call_timeout, + &capabilities, + &mut assertions, + )?; + if let Some(observed) = observed { + assertions.push(assertion( + &format!("before.{}", probe.id), + &profile, + if initial_stable && expected == observed { + "PASS" + } else if !initial_stable { + "UNKNOWN" + } else { + "FAIL" + }, + format!( + "query subject '{}' expected fixture '{}' to be {}, observed {}", + scenario.probe_subject(probe), + fixture.id, + if expected { "isolated" } else { "inaccessible" }, + if observed { "visible" } else { "absent" } + ), + Some(expected), + Some(observed), + "namespace", + format!("probe:{}", probe.id), + )); + } } } - if selected(&scenario, "FP-Scope") - && !scenario.spec.probes.after.iter().any(|probe| { - profile_for_probe(probe) == "FP-Object" && probe.fixture != scenario.spec.erase.target - }) - { - let control = scenario - .spec - .fixtures - .iter() - .find(|fixture| !fixture.target) - .ok_or_else(|| anyhow!("FP-Scope requires a non-target control fixture"))?; - let probe = Probe { - id: "scope.control-preserved".to_owned(), - fixture: control.id.clone(), - kind: "exact".to_owned(), - query: String::new(), - }; - let observed = perform_probe(&mut client, &mut events, &run_id, &probe, "after", timeout)?; - assertions.push(if erase_performed { - assertion_for_probe( - "scope.control-preserved", - "FP-Scope", - true, - observed, - &control.id, - &probe, - ) + if scenario.spec.suite == "erasure" && !precondition_failed { + let erase_capability = erase_capability(&scenario.spec.erase.intent); + if capability_available(&capabilities, erase_capability) { + call_checked( + &mut client, + &mut events, + "erase", + json!({ + "run_id": run_id, + "intent": scenario.spec.erase.intent, + "target": scenario.spec.erase.target, + "target_subject": target_subject(&scenario), + "control_subject": scenario.spec.isolation.control_subject, + "scope": scenario.spec.erase.scope, + }), + call_timeout, + )?; } else { - unknown_assertion_for_probe( - "scope.control-preserved", - "FP-Scope", - "erase operation was not available; scope observation is inconclusive", - &probe, - ) - }); - } + warnings.push(format!( + "erase intent '{}' requires unsupported capability '{}'", + scenario.spec.erase.intent, erase_capability + )); + assertions.push(unknown_assertion( + "operation.erase", + "erasure.object", + "the adapter did not advertise the selected erase capability", + "operation", + "capabilities".to_owned(), + )); + } - if selected(&scenario, "FP-Derived") - && !scenario - .spec - .probes - .after - .iter() - .any(|probe| profile_for_probe(probe) == "FP-Derived") - && has_capability(&capabilities, "derived_inspect") - { - let target = &scenario.spec.erase.target; - let probe = Probe { - id: "derived.target-absent".to_owned(), - fixture: target.clone(), - kind: "inspect".to_owned(), - query: String::new(), - }; - let observed = perform_probe(&mut client, &mut events, &run_id, &probe, "after", timeout)?; - assertions.push(if erase_performed { - assertion_for_probe( - "derived.target-absent", - "FP-Derived", - false, - observed, - target, - &probe, - ) - } else { - unknown_assertion_for_probe( - "derived.target-absent", - "FP-Derived", - "erase operation was not available; derived observation is inconclusive", - &probe, - ) - }); + let stable = settle( + &mut client, + &mut events, + &scenario, + call_timeout, + &mut warnings, + "after-erase", + )?; + for probe in &scenario.spec.probes.after { + let fixture = fixture_by_id(&scenario, &probe.fixture)?; + let profile = profile_for_probe(probe, &scenario.spec.suite); + let expected = !fixture.is_target(); + let observed = run_probe_if_supported( + &mut client, + &mut events, + &scenario, + &run_id, + probe, + "after", + call_timeout, + &capabilities, + &mut assertions, + )?; + if let Some(observed) = observed { + assertions.push(assertion( + &format!("after.{}", probe.id), + &profile, + if !stable { + "UNKNOWN" + } else if expected == observed { + "PASS" + } else { + "FAIL" + }, + format!( + "{} probe for fixture '{}' expected {}, observed {}", + probe.kind, + fixture.id, + if expected { "present" } else { "absent" }, + if observed { "present" } else { "absent" } + ), + Some(expected), + Some(observed), + artifact_for_probe(probe), + format!("probe:{}", probe.id), + )); + if fixture.is_control() + && scenario + .spec + .profiles + .iter() + .any(|profile| crate::model::legacy_profile(profile) == "erasure.scope") + { + assertions.push(assertion( + &format!("scope.{}", probe.id), + "erasure.scope", + if !stable { + "UNKNOWN" + } else if observed { + "PASS" + } else { + "FAIL" + }, + format!( + "control fixture '{}' remained {} after target erase", + fixture.id, + if observed { "observable" } else { "absent" } + ), + Some(true), + Some(observed), + "scope", + format!("probe:{}", probe.id), + )); + } + } + } + } else if scenario.spec.suite == "erasure" { + warnings + .push("erase operation was skipped because the precondition was not proven".to_owned()); + } else { + for probe in &scenario.spec.probes.after { + let fixture = fixture_by_id(&scenario, &probe.fixture)?; + let profile = profile_for_probe(probe, &scenario.spec.suite); + let expected = scenario.probe_subject(probe) + == fixture.effective_subject(&scenario.spec.isolation); + let observed = run_probe_if_supported( + &mut client, + &mut events, + &scenario, + &run_id, + probe, + "isolation", + call_timeout, + &capabilities, + &mut assertions, + )?; + if let Some(observed) = observed { + assertions.push(assertion( + &format!("after.{}", probe.id), + &profile, + if expected == observed { "PASS" } else { "FAIL" }, + format!( + "query subject '{}' expected fixture '{}' to be {}, observed {}", + scenario.probe_subject(probe), + fixture.id, + if expected { "isolated" } else { "inaccessible" }, + if observed { "visible" } else { "absent" } + ), + Some(expected), + Some(observed), + "namespace", + format!("probe:{}", probe.id), + )); + } + } } - if selected(&scenario, "FP-Agent") - && !scenario - .spec - .probes - .after - .iter() - .any(|probe| profile_for_probe(probe) == "FP-Agent") - && has_capability(&capabilities, "agent_query") - { - let target = &scenario.spec.erase.target; - let probe = Probe { - id: "agent.target-absent".to_owned(), - fixture: target.clone(), - kind: "agent".to_owned(), - query: String::new(), - }; - let observed = perform_probe(&mut client, &mut events, &run_id, &probe, "after", timeout)?; - assertions.push(if erase_performed { - assertion_for_probe( - "agent.target-absent", - "FP-Agent", - false, - observed, - target, - &probe, - ) - } else { - unknown_assertion_for_probe( - "agent.target-absent", - "FP-Agent", - "erase operation was not available; Agent observation is inconclusive", - &probe, - ) - }); + if let Err(error) = client.call("cleanup", json!({ "run_id": run_id }), call_timeout) { + warnings.push(format!("cleanup failed: {error:#}")); + record_event( + &mut events, + "cleanup", + "cleanup", + "ERROR", + details([(String::from("error"), "cleanup failed".to_owned())]), + ); + } else { + record_event(&mut events, "cleanup", "cleanup", "PASS", BTreeMap::new()); } + client.close(); let profiles = summarize_profiles(&scenario.spec.profiles, &assertions); - let status = overall_status(&profiles, &assertions); + let status = if precondition_failed && assertions.iter().any(|item| item.status == "ERROR") { + "ERROR".to_owned() + } else { + overall_status(&profiles, &assertions) + }; let exit_code = match status.as_str() { "PASS" => 0, "FAIL" => 1, - "INCOMPLETE" => 3, + "UNKNOWN" | "SKIP" => 3, _ => 2, }; let result = RunResult { run_id: run_id.clone(), + suite: scenario.spec.suite.clone(), scenario: scenario.metadata.name.clone(), adapter: scenario.spec.adapter.name.clone(), backend: if capabilities.backend.is_empty() { @@ -339,25 +418,31 @@ pub fn run_scenario( } else { capabilities.backend.clone() }, + backend_version: capabilities.version.clone(), + protocol: capabilities.protocol.clone(), scenario_hash: scenario_hash.clone(), status, exit_code, profiles, assertions, + capabilities: capabilities.capabilities.clone(), + out_of_scope: vec![ + "provider logs and backups".to_owned(), + "physical storage erasure".to_owned(), + "model-weight unlearning".to_owned(), + "unobservable artifacts outside the adapter boundary".to_owned(), + ], warnings, }; - let _ = client.call("cleanup", json!({ "run_id": run_id }), timeout); - client.close(); - let manifest = Manifest { - format: "forgetproof.bundle/v1alpha1".to_owned(), + format: BUNDLE_FORMAT.to_owned(), run_id: result.run_id.clone(), created_at_ms: now_ms(), scenario_hash, adapter: result.adapter.clone(), backend: result.backend.clone(), - protocol: crate::model::PROTOCOL_VERSION.to_owned(), + protocol: result.protocol.clone(), files: vec![ "manifest.json".to_owned(), "scenario.lock.json".to_owned(), @@ -389,17 +474,21 @@ pub fn doctor_adapter( } pub fn init_project(root: &Path) -> Result<()> { - fs::create_dir_all(root.join(".forgetproof/runs"))?; + fs::create_dir_all(root.join(".memoryproof/runs"))?; fs::create_dir_all(root.join("scenarios"))?; - let config = r#"# ForgetProof local configuration -# Adapter commands are built in; credentials stay in environment variables. + let config = r#"# MemoryProof local configuration +# Adapter commands are registered by the CLI; credentials stay in environment variables. [project] -protocol = "forgetproof.adapter/v1alpha1" +protocol = "memoryproof.adapter/v1" [security] allow_network = false "#; - fs::write(root.join("forgetproof.toml"), config)?; + fs::write(root.join("memoryproof.toml"), config)?; + let gitignore = root.join(".gitignore"); + if !gitignore.exists() { + fs::write(&gitignore, "/.memoryproof/\n")?; + } fs::write( root.join("scenarios/reference-clean.yml"), include_str!("../../../examples/reference-clean.yml"), @@ -413,7 +502,9 @@ allow_network = false pub fn expand_scenario(input: &Path, output: &Path) -> Result<()> { let (mut scenario, _) = load_scenario(input)?; - if std::env::var("FORGETPROOF_LLM_BASE_URL").is_ok() || std::env::var("OPENAI_BASE_URL").is_ok() + if std::env::var("MEMORYPROOF_LLM_BASE_URL").is_ok() + || std::env::var("FORGETPROOF_LLM_BASE_URL").is_ok() + || std::env::var("OPENAI_BASE_URL").is_ok() { scenario = expand_with_compatible_llm(&scenario)?; } @@ -451,23 +542,32 @@ pub fn expand_scenario(input: &Path, output: &Path) -> Result<()> { fixture: probe.fixture.clone(), kind: kind.to_owned(), query, + as_subject: probe.as_subject.clone(), }); } } } } scenario.spec.probes.after.extend(additions); - scenario.kind = "ErasureScenarioLock".to_owned(); + scenario.kind = "AssuranceScenarioLock".to_owned(); let yaml = serde_yaml::to_string(&scenario)?; fs::write( output, - format!("# Generated by forgetproof expand. Probes are frozen before execution.\n{yaml}"), + format!("# Generated by memoryproof expand. Probes are frozen before execution.\n{yaml}"), )?; Ok(()) } fn expand_with_compatible_llm(scenario: &Scenario) -> Result { - let python = std::env::var("FORGETPROOF_PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let python = std::env::var("MEMORYPROOF_PYTHON") + .or_else(|_| std::env::var("FORGETPROOF_PYTHON")) + .unwrap_or_else(|_| { + if cfg!(windows) { + "python".to_owned() + } else { + "python3".to_owned() + } + }); let mut command = Command::new(python); command .args(["-m", "forgetproof_adapters.expand"]) @@ -477,13 +577,11 @@ fn expand_with_compatible_llm(scenario: &Scenario) -> Result { if let Ok(current_dir) = std::env::current_dir() { let package_root = current_dir.join("python"); if package_root.join("forgetproof_adapters").is_dir() { - let existing = std::env::var("PYTHONPATH").unwrap_or_default(); - let value = if existing.is_empty() { - package_root.display().to_string() - } else { - format!("{}:{}", package_root.display(), existing) - }; - command.env("PYTHONPATH", value); + let mut paths = vec![package_root]; + if let Some(existing) = std::env::var_os("PYTHONPATH") { + paths.extend(std::env::split_paths(&existing)); + } + command.env("PYTHONPATH", std::env::join_paths(paths)?); } } let mut child = command @@ -504,26 +602,36 @@ fn expand_with_compatible_llm(scenario: &Scenario) -> Result { } fn enforce_network_policy(scenario: &Scenario, allow_network: bool) -> Result<()> { - if matches!( - scenario.spec.adapter.name.as_str(), - "reference-clean" | "reference-leaky" - ) { + if scenario.spec.adapter.name.starts_with("reference-") { return Ok(()); } - if allow_network || std::env::var("FORGETPROOF_ALLOW_NETWORK").as_deref() == Ok("1") { + if allow_network + || std::env::var("MEMORYPROOF_ALLOW_NETWORK").as_deref() == Ok("1") + || std::env::var("FORGETPROOF_ALLOW_NETWORK").as_deref() == Ok("1") + { return Ok(()); } - let local = scenario + let base_url = scenario .spec .adapter .config .get("base_url") - .map(|url| { - url.starts_with("http://127.0.0.1") - || url.starts_with("http://localhost") - || url.starts_with("http://[::1]") + .cloned() + .or_else(|| { + let env_name = match scenario.spec.adapter.name.as_str() { + "mem0" => "MEM0_BASE_URL", + "letta" => "LETTA_BASE_URL", + "zep" => "ZEP_BASE_URL", + _ => "", + }; + (!env_name.is_empty()) + .then(|| std::env::var(env_name).ok()) + .flatten() }) - .unwrap_or(false); + .unwrap_or_default(); + let local = base_url.starts_with("http://127.0.0.1") + || base_url.starts_with("http://localhost") + || base_url.starts_with("http://[::1]"); if !local { bail!( "network access is disabled; pass --allow-network for '{}' or configure a loopback base_url", @@ -533,6 +641,28 @@ fn enforce_network_policy(scenario: &Scenario, allow_network: bool) -> Result<() Ok(()) } +fn fixture_payload(scenario: &Scenario, fixture: &Fixture) -> Value { + json!({ + "id": fixture.id, + "content": fixture.content, + "role": if fixture.is_target() { "target" } else { "control" }, + "target": fixture.is_target(), + "subject": fixture.effective_subject(&scenario.spec.isolation), + "namespace": if fixture.namespace.is_empty() { scenario.spec.isolation.scope.clone() } else { fixture.namespace.clone() }, + "kind": fixture.kind, + }) +} + +fn target_subject(scenario: &Scenario) -> String { + scenario + .spec + .fixtures + .iter() + .find(|fixture| fixture.id == scenario.spec.erase.target) + .map(|fixture| fixture.effective_subject(&scenario.spec.isolation)) + .unwrap_or_else(|| scenario.spec.isolation.target_subject.clone()) +} + fn call_checked( client: &mut AdapterClient, events: &mut Vec, @@ -550,10 +680,10 @@ fn settle( events: &mut Vec, scenario: &Scenario, timeout: u64, -) -> Result<()> { - let value = call_checked( - client, - events, + warnings: &mut Vec, + phase: &str, +) -> Result { + let value = client.call( "settle", json!({ "timeout_ms": scenario.spec.settle.timeout_ms, @@ -561,25 +691,66 @@ fn settle( }), timeout, )?; - if value.get("stable").and_then(Value::as_bool) == Some(false) { - bail!("adapter did not reach a stable state before timeout") + let state = value + .get("state") + .and_then(Value::as_str) + .unwrap_or_else(|| { + if value.get("stable").and_then(Value::as_bool) == Some(true) { + "stable" + } else { + "unknown" + } + }); + let stable = state == "stable"; + record_event( + events, + phase, + "settle", + if stable { "PASS" } else { "UNKNOWN" }, + details([(String::from("state"), state.to_owned())]), + ); + if !stable { + warnings.push(format!( + "backend did not report stable state during {phase}: {state}" + )); } - Ok(()) + Ok(stable) } -fn perform_probe( +#[allow(clippy::too_many_arguments)] +fn run_probe_if_supported( client: &mut AdapterClient, events: &mut Vec, + scenario: &Scenario, run_id: &str, probe: &Probe, phase: &str, timeout: u64, -) -> Result { - let method = match probe.kind.as_str() { - "inspect" => "inspect", - "agent" => "agent_query", - _ => "probe", - }; + capabilities: &Capabilities, + assertions: &mut Vec, +) -> Result> { + let required = probe_capability(probe); + if !capability_available(capabilities, required) { + assertions.push(unknown_assertion( + &format!("{phase}.{}.capability", probe.id), + &profile_for_probe(probe, &scenario.spec.suite), + &format!("adapter does not advertise capability '{required}'"), + artifact_for_probe(probe), + "capabilities".to_owned(), + )); + record_event( + events, + phase, + method_for_probe(probe), + "UNKNOWN", + details([ + (String::from("probe"), probe.id.clone()), + (String::from("missing_capability"), required.to_owned()), + ]), + ); + return Ok(None); + } + let method = method_for_probe(probe); let value = client.call( method, json!({ @@ -590,6 +761,7 @@ fn perform_probe( "fixture": probe.fixture, "kind": probe.kind, "query": probe.query, + "as_subject": scenario.probe_subject(probe), } }), timeout, @@ -608,113 +780,108 @@ fn perform_probe( (String::from("found"), found.to_string()), ]), ); - Ok(found) + Ok(Some(found)) } -fn add_capability_assertions( - scenario: &Scenario, - capabilities: &Capabilities, - assertions: &mut Vec, -) { - for profile in &scenario.spec.profiles { - for capability in profile_capabilities(profile) { - if !has_capability(capabilities, capability) { - assertions.push(Assertion { - id: format!("capability.{profile}.{capability}"), - profile: profile.clone(), - status: "UNKNOWN".to_owned(), - message: format!("adapter does not advertise capability '{capability}'"), - expected: None, - observed: None, - evidence: vec!["capabilities".to_owned()], - }); - } - } +fn probe_capability(probe: &Probe) -> &'static str { + match probe.kind.as_str() { + "inspect" => "derived_inspect", + "agent" => "agent_query", + "lexical" => "lexical_search", + "semantic" => "semantic_search", + _ => "probe", } } -fn profile_capabilities(profile: &str) -> &'static [&'static str] { - match profile { - "FP-Object" => &["object_delete", "probe"], - "FP-Scope" => &["scope_delete", "probe"], - "FP-Derived" => &["derived_inspect", "inspect"], - "FP-Agent" => &["agent_query"], - _ => &[], +fn method_for_probe(probe: &Probe) -> &'static str { + match probe.kind.as_str() { + "inspect" => "inspect", + "agent" => "agent_query", + _ => "probe", } } -fn erase_capability(intent: &str) -> &str { - match intent { - "subject_erase" => "scope_delete", - "derived_purge" => "derived_delete", - "access_revoke" => "access_revoke", - _ => "object_delete", +fn artifact_for_probe(probe: &Probe) -> &'static str { + match probe.kind.as_str() { + "inspect" => "derived", + "agent" => "agent", + "lexical" => "lexical-index", + "semantic" => "semantic-index", + _ => "object", } } -fn has_capability(capabilities: &Capabilities, capability: &str) -> bool { - capabilities - .capabilities - .iter() - .any(|item| item == capability) -} - -fn selected(scenario: &Scenario, profile: &str) -> bool { - scenario.spec.profiles.iter().any(|item| item == profile) -} - -fn profile_for_probe(probe: &Probe) -> String { - match probe.kind.as_str() { - "inspect" => "FP-Derived".to_owned(), - "agent" => "FP-Agent".to_owned(), - _ => "FP-Object".to_owned(), +fn add_capability_assertions( + scenario: &Scenario, + capabilities: &Capabilities, + assertions: &mut Vec, +) { + for profile in &scenario.spec.profiles { + let profile = crate::model::legacy_profile(profile); + for capability in profile_capabilities(&profile) { + if !capability_available(capabilities, capability) { + assertions.push(unknown_assertion( + &format!("capability.{profile}.{capability}"), + &profile, + &format!("adapter does not advertise capability '{capability}'"), + "capability", + "capabilities".to_owned(), + )); + } + } } } -fn assertion_for_probe( +#[allow(clippy::too_many_arguments)] +fn assertion( id: &str, profile: &str, - expected: bool, - observed: bool, - fixture: &str, - probe: &Probe, + status: &str, + message: String, + expected: Option, + observed: Option, + artifact: &str, + evidence: String, ) -> Assertion { - let status = if expected == observed { "PASS" } else { "FAIL" }; - let expectation = if expected { "present" } else { "absent" }; - let actual = if observed { "present" } else { "absent" }; Assertion { id: id.to_owned(), - profile: profile.to_owned(), + profile: crate::model::legacy_profile(profile), status: status.to_owned(), - message: format!( - "{} probe for fixture '{}' expected {}, observed {}", - probe.kind, fixture, expectation, actual - ), - expected: Some(expected), - observed: Some(observed), - evidence: vec![format!("probe:{}", probe.id)], + message, + expected, + observed, + artifact: artifact.to_owned(), + evidence: vec![evidence], } } -fn unknown_assertion_for_probe(id: &str, profile: &str, message: &str, probe: &Probe) -> Assertion { - Assertion { - id: id.to_owned(), - profile: profile.to_owned(), - status: "UNKNOWN".to_owned(), - message: message.to_owned(), - expected: None, - observed: None, - evidence: vec![format!("probe:{}", probe.id)], - } +fn unknown_assertion( + id: &str, + profile: &str, + message: &str, + artifact: &str, + evidence: String, +) -> Assertion { + assertion( + id, + profile, + "UNKNOWN", + message.to_owned(), + None, + None, + artifact, + evidence, + ) } fn summarize_profiles(profiles: &[String], assertions: &[Assertion]) -> Vec { profiles .iter() - .map(|profile| { + .map(|raw_profile| { + let profile = crate::model::legacy_profile(raw_profile); let relevant = assertions .iter() - .filter(|assertion| assertion.profile == *profile) + .filter(|item| item.profile == profile) .collect::>(); let status = if relevant.iter().any(|item| item.status == "ERROR") { "ERROR" @@ -725,12 +892,13 @@ fn summarize_profiles(profiles: &[String], assertions: &[Assertion]) -> Vec Strin "ERROR".to_owned() } else if profiles.iter().any(|item| item.status == "FAIL") { "FAIL".to_owned() - } else if profiles.iter().any(|item| item.status == "INCOMPLETE") { - "INCOMPLETE".to_owned() + } else if profiles + .iter() + .any(|item| matches!(item.status.as_str(), "UNKNOWN" | "SKIP")) + { + "UNKNOWN".to_owned() } else { "PASS".to_owned() } } -fn fixture_by_id<'a>(scenario: &'a Scenario, id: &str) -> Result<&'a crate::model::Fixture> { +fn fixture_by_id<'a>(scenario: &'a Scenario, id: &str) -> Result<&'a Fixture> { scenario .spec .fixtures @@ -764,7 +935,7 @@ fn record_event( phase: &str, method: &str, status: &str, - details: BTreeMap, + event_details: BTreeMap, ) { events.push(Event { seq: events.len() as u64 + 1, @@ -772,7 +943,7 @@ fn record_event( phase: phase.to_owned(), method: method.to_owned(), status: status.to_owned(), - details, + details: event_details, }); } diff --git a/crates/forgetproof/tests/cli.rs b/crates/forgetproof/tests/cli.rs index 7516fc1..a997cb9 100644 --- a/crates/forgetproof/tests/cli.rs +++ b/crates/forgetproof/tests/cli.rs @@ -1,34 +1,42 @@ use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; fn project_root() -> &'static Path { Path::new(env!("CARGO_MANIFEST_DIR")).join("../..").leak() } -fn temp_root(label: &str) -> std::path::PathBuf { +fn temp_root(label: &str) -> PathBuf { let path = - std::env::temp_dir().join(format!("forgetproof-test-{label}-{}", std::process::id())); + std::env::temp_dir().join(format!("memoryproof-test-{label}-{}", std::process::id())); let _ = fs::remove_dir_all(&path); fs::create_dir_all(&path).unwrap(); path } -#[test] -fn clean_backend_passes_and_bundle_verifies() { - let root = project_root(); - let output = temp_root("clean"); - let run = Command::new(env!("CARGO_BIN_EXE_forgetproof")) - .current_dir(root) +fn run(binary: &str, scenario: &str, output: &Path) -> std::process::Output { + Command::new(binary) + .current_dir(project_root()) .args([ "run", - "examples/reference-clean.yml", + scenario, "--output", output.to_str().unwrap(), "--allow-network=false", ]) .output() - .unwrap(); + .unwrap() +} + +#[test] +fn clean_backend_passes_and_bundle_verifies() { + let root = project_root(); + let output = temp_root("clean"); + let run = run( + env!("CARGO_BIN_EXE_memoryproof"), + "examples/reference-clean.yml", + &output, + ); assert_eq!( run.status.code(), Some(0), @@ -41,7 +49,7 @@ fn clean_backend_passes_and_bundle_verifies() { .unwrap() .unwrap() .path(); - let verify = Command::new(env!("CARGO_BIN_EXE_forgetproof")) + let verify = Command::new(env!("CARGO_BIN_EXE_memoryproof")) .current_dir(root) .args(["verify", bundle.to_str().unwrap()]) .output() @@ -55,22 +63,18 @@ fn clean_backend_passes_and_bundle_verifies() { let scenario = fs::read_to_string(bundle.join("scenario.lock.json")).unwrap(); assert!(!scenario.contains("target alpha 7f3e9d")); assert!(scenario.contains("sha256:")); + let manifest = fs::read_to_string(bundle.join("manifest.json")).unwrap(); + assert!(manifest.contains("memoryproof.bundle/v1")); } #[test] -fn leaky_backend_fails_with_a_report() { - let root = project_root(); +fn leaky_backend_fails_at_observable_derived_boundary() { let output = temp_root("leaky"); - let run = Command::new(env!("CARGO_BIN_EXE_forgetproof")) - .current_dir(root) - .args([ - "run", - "examples/reference-leaky.yml", - "--output", - output.to_str().unwrap(), - ]) - .output() - .unwrap(); + let run = run( + env!("CARGO_BIN_EXE_memoryproof"), + "examples/reference-leaky.yml", + &output, + ); assert_eq!( run.status.code(), Some(1), @@ -84,6 +88,83 @@ fn leaky_backend_fails_with_a_report() { .unwrap() .path(); let results = fs::read_to_string(bundle.join("results.json")).unwrap(); - assert!(results.contains("\"status\": \"FAIL\"")); + assert!(results.contains(r#""status": "FAIL""#)); assert!(results.contains("target-derived-after")); } + +#[test] +fn overdelete_backend_fails_scope_without_hiding_the_control_regression() { + let output = temp_root("overdelete"); + let run = run( + env!("CARGO_BIN_EXE_memoryproof"), + "examples/reference-overdelete.yml", + &output, + ); + assert_eq!( + run.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&run.stdout) + ); + let bundle = fs::read_dir(&output) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let results = fs::read_to_string(bundle.join("results.json")).unwrap(); + assert!(results.contains(r#""profile": "erasure.scope""#)); + assert!(results.contains("control fixture 'control' remained absent")); +} + +#[test] +fn isolation_backend_passes_cross_subject_checks() { + let output = temp_root("isolation"); + let run = run( + env!("CARGO_BIN_EXE_memoryproof"), + "examples/isolation-reference.yml", + &output, + ); + assert_eq!( + run.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&run.stdout) + ); +} + +#[test] +fn tampering_is_detected_and_report_refreshes_checksums() { + let output = temp_root("tamper"); + let run = run( + env!("CARGO_BIN_EXE_memoryproof"), + "examples/reference-clean.yml", + &output, + ); + assert_eq!(run.status.code(), Some(0)); + let bundle = fs::read_dir(&output) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + fs::write(bundle.join("report.html"), "tampered").unwrap(); + let verify = Command::new(env!("CARGO_BIN_EXE_memoryproof")) + .current_dir(project_root()) + .args(["verify", bundle.to_str().unwrap()]) + .output() + .unwrap(); + assert_eq!(verify.status.code(), Some(1)); + let report = Command::new(env!("CARGO_BIN_EXE_memoryproof")) + .current_dir(project_root()) + .args(["report", bundle.to_str().unwrap()]) + .output() + .unwrap(); + assert_eq!(report.status.code(), Some(0)); + let verify_again = Command::new(env!("CARGO_BIN_EXE_memoryproof")) + .current_dir(project_root()) + .args(["verify", bundle.to_str().unwrap()]) + .output() + .unwrap(); + assert_eq!(verify_again.status.code(), Some(0)); +} diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..f07078d --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,69 @@ +# MemoryProof architecture + +[English](architecture.md) · [简体中文](architecture.zh-CN.md) + +MemoryProof has a small trust boundary on purpose. The Rust process owns scenario parsing, capability negotiation, deterministic assertions, redaction, and evidence hashing. Provider-specific behavior stays behind an isolated Python process. + +## Runtime flow + +```mermaid +sequenceDiagram + participant S as Scenario + participant R as Rust runner + participant A as Adapter process + participant B as Memory backend + participant E as Evidence bundle + S->>R: load, normalize, validate + R->>A: hello + capabilities + R->>A: prepare owned namespace + R->>A: ingest target + control canaries + A->>B: provider API calls + R->>A: settle + before probes + R->>A: erase or isolation probes + R->>A: settle + after probes + R->>A: cleanup owned resources + R->>E: results, report, JUnit, SHA-256 manifest +``` + +## Trust boundaries + +| Boundary | Owner | Rule | +| --- | --- | --- | +| Scenario and assertion engine | Rust | No LLM output participates in the final decision. | +| Adapter process | Python | stdout is protocol-only; stderr is diagnostic. | +| Provider API | Adapter | Credentials come from the environment; request IDs may be recorded, secrets may not. | +| Evidence bundle | Rust filesystem | Paths are validated and checksums are sorted before the bundle hash is written. | +| Public matrix | CI + reviewed bundles | A bundle must verify before a profile is indexed. | + +## Stable contracts + +The scenario API is `memoryproof.dev/v1`; the adapter protocol is `memoryproof.adapter/v1`; the evidence format is `memoryproof.bundle/v1`. Every frame is one JSON object per line: + +```json +{"protocol":"memoryproof.adapter/v1","id":"7","method":"probe","params":{"probe":{"fixture":"target","kind":"semantic"}}} +``` + +```json +{"protocol":"memoryproof.adapter/v1","id":"7","ok":true,"result":{"found":false}} +``` + +The response ID must match. A process crash, timeout, malformed JSON, protocol mismatch, or unsupported method becomes a standardized error. A missing backend capability becomes `SKIP` or `UNKNOWN`, never a fabricated `PASS`. + +## Why the control fixture exists + +The target fixture proves that deletion happened to something observable. The control fixture proves that the test did not accidentally delete the whole tenant, user, Agent, thread, or namespace. Scope is a first-class assertion, not an optional convenience. + +## What the evidence hash means + +`checksums.sha256` lists the declared bundle files in sorted order. `bundle.hash` is the SHA-256 hash of that exact checksum text. This proves post-run byte integrity; v1 does not claim signer identity, physical erasure, provider-log deletion, backup deletion, or model-weight unlearning. + +## Adding a provider adapter + +1. Add a dependency-free module under `python/forgetproof_adapters/`. +2. Advertise only capabilities that the provider exposes in the configured mode. +3. Create temporary resources with a unique run marker. +4. Refuse cleanup and scope deletion without ownership. +5. Add mock contract tests and a redacted public bundle. +6. Document provider-version semantics and the observable boundary in both README languages. + +See [CONTRIBUTING.md](../CONTRIBUTING.md) for the contributor workflow and [SECURITY.md](../SECURITY.md) for remote-run safety. diff --git a/docs/architecture.zh-CN.md b/docs/architecture.zh-CN.md new file mode 100644 index 0000000..ed1432a --- /dev/null +++ b/docs/architecture.zh-CN.md @@ -0,0 +1,69 @@ +# MemoryProof 架构 + +[English](architecture.md) · [简体中文](architecture.zh-CN.md) + +MemoryProof 刻意保持较小的信任边界。Rust 进程负责场景解析、能力协商、确定性断言、脱敏和证据哈希;供应商差异全部放在隔离的 Python 子进程之后。 + +## 运行流程 + +```mermaid +sequenceDiagram + participant S as 场景 + participant R as Rust 运行器 + participant A as 适配器进程 + participant B as 记忆后端 + participant E as 证据包 + S->>R: 加载、规范化、校验 + R->>A: hello + capabilities + R->>A: 创建自有隔离命名空间 + R->>A: 写入目标 + 控制 canary + A->>B: 调用供应商 API + R->>A: settle + 删除前探针 + R->>A: 删除或隔离探针 + R->>A: settle + 删除后探针 + R->>A: 清理自有资源 + R->>E: 结果、报告、JUnit、SHA-256 清单 +``` + +## 信任边界 + +| 边界 | 负责者 | 规则 | +| --- | --- | --- | +| 场景与断言引擎 | Rust | LLM 输出不能参与最终判定。 | +| 适配器进程 | Python | stdout 只允许协议帧,stderr 只写诊断日志。 | +| 供应商 API | 适配器 | 凭据来自环境变量,可以记录请求 ID,但不能记录密钥。 | +| 证据包 | Rust 文件系统 | 校验路径,并在写 bundle hash 前排序哈希清单。 | +| 公开矩阵 | CI + 审阅后的证据包 | 证据包必须先通过 verify,才能进入索引。 | + +## 稳定契约 + +场景 API 是 `memoryproof.dev/v1`,适配器协议是 `memoryproof.adapter/v1`,证据格式是 `memoryproof.bundle/v1`。每一帧都是一行 JSON: + +```json +{"protocol":"memoryproof.adapter/v1","id":"7","method":"probe","params":{"probe":{"fixture":"target","kind":"semantic"}}} +``` + +```json +{"protocol":"memoryproof.adapter/v1","id":"7","ok":true,"result":{"found":false}} +``` + +响应 ID 必须匹配。进程崩溃、超时、畸形 JSON、协议不匹配或不支持的方法都会变成标准错误;后端没有的能力会变为 `SKIP` 或 `UNKNOWN`,绝不会伪造 `PASS`。 + +## 为什么需要控制 fixture + +目标 fixture 证明删除之前确实存在可观察数据。控制 fixture 证明测试没有误删整个租户、用户、Agent、线程或命名空间。范围是一级断言,不是可有可无的附加项。 + +## 证据哈希代表什么 + +`checksums.sha256` 按排序顺序列出证据包声明的文件。`bundle.hash` 是这段完整清单文本的 SHA-256 哈希。它证明运行后文件字节没有被修改;v1 不声称证明签名者身份、物理擦除、供应商日志删除、备份删除或模型权重反学习。 + +## 添加供应商适配器 + +1. 在 `python/forgetproof_adapters/` 下添加无依赖模块。 +2. 只声明当前配置模式真正暴露的能力。 +3. 使用唯一运行标记创建临时资源。 +4. 没有所有权时拒绝清理和范围删除。 +5. 添加 mock 契约测试和脱敏的公开证据包。 +6. 用中英文记录供应商版本语义和可观察边界。 + +贡献流程见[贡献指南](../CONTRIBUTING.zh-CN.md),远程运行安全见[安全策略](../SECURITY.zh-CN.md)。 diff --git a/examples/isolation-reference.yml b/examples/isolation-reference.yml new file mode 100644 index 0000000..2d155e3 --- /dev/null +++ b/examples/isolation-reference.yml @@ -0,0 +1,58 @@ +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario +metadata: + name: isolation-reference + description: Two subjects must not be able to observe one another's canaries. +spec: + suite: isolation + adapter: + name: reference-clean + mode: clean + isolation: + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject + fixtures: + - id: target + role: target + subject: target-subject + content: "MemoryProof isolation target 4a7d91" + - id: control + role: control + subject: control-subject + content: "MemoryProof isolation control 9c2b60" + settle: + timeout_ms: 5000 + interval_ms: 25 + probes: + before: + - id: target-from-target + fixture: target + as_subject: target-subject + kind: exact + - id: target-from-control + fixture: target + as_subject: control-subject + kind: exact + - id: control-from-control + fixture: control + as_subject: control-subject + kind: exact + - id: control-from-target + fixture: control + as_subject: target-subject + kind: exact + after: + - id: target-search-from-control + fixture: target + as_subject: control-subject + kind: semantic + - id: control-search-from-target + fixture: control + as_subject: target-subject + kind: semantic + profiles: + - isolation.read + - isolation.search + privacy: + raw_payloads: false diff --git a/examples/letta.yml b/examples/letta.yml index 7239c88..9121b2f 100644 --- a/examples/letta.yml +++ b/examples/letta.yml @@ -1,23 +1,28 @@ -apiVersion: forgetproof.dev/v1alpha1 -kind: ErasureScenario +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario metadata: name: letta-erasure - description: Letta archival passage and Agent erasure with a temporary Agent. + description: Letta archival passage and Agent erasure with owned temporary Agents. spec: + suite: erasure adapter: name: letta - mode: default + mode: self-hosted config: base_url: http://localhost:8283 isolation: - scope: run + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject fixtures: - id: target - content: "ForgetProof Letta canary target 31a9e2" - target: true + role: target + subject: target-subject + content: "MemoryProof Letta canary target 31a9e2" - id: control - content: "ForgetProof Letta control 8c2d10" - target: false + role: control + subject: control-subject + content: "MemoryProof Letta control 8c2d10" settle: timeout_ms: 30000 interval_ms: 1000 @@ -45,4 +50,8 @@ spec: - id: target-agent-after fixture: target kind: agent - profiles: [FP-Object, FP-Scope, FP-Derived, FP-Agent] + profiles: + - erasure.object + - erasure.scope + - erasure.derived + - erasure.agent diff --git a/examples/mem0.yml b/examples/mem0.yml index 17307ed..4997a11 100644 --- a/examples/mem0.yml +++ b/examples/mem0.yml @@ -1,23 +1,28 @@ -apiVersion: forgetproof.dev/v1alpha1 -kind: ErasureScenario +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario metadata: name: mem0-erasure - description: Mem0 OSS or platform memory deletion with isolated user_id. + description: Mem0 deletion with separate target and control users. spec: + suite: erasure adapter: name: mem0 mode: oss config: base_url: http://localhost:8888 isolation: - scope: run + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject fixtures: - id: target - content: "ForgetProof Mem0 canary target 31a9e2" - target: true + role: target + subject: target-subject + content: "MemoryProof Mem0 canary target 31a9e2" - id: control - content: "ForgetProof Mem0 control 8c2d10" - target: false + role: control + subject: control-subject + content: "MemoryProof Mem0 control 8c2d10" settle: timeout_ms: 30000 interval_ms: 1000 @@ -42,4 +47,7 @@ spec: - id: target-derived-after fixture: target kind: inspect - profiles: [FP-Object, FP-Scope, FP-Derived] + profiles: + - erasure.object + - erasure.scope + - erasure.derived diff --git a/examples/reference-clean.yml b/examples/reference-clean.yml index 22ae1cf..f33a931 100644 --- a/examples/reference-clean.yml +++ b/examples/reference-clean.yml @@ -1,21 +1,26 @@ -apiVersion: forgetproof.dev/v1alpha1 -kind: ErasureScenario +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario metadata: name: reference-clean - description: A deterministic clean backend should remove raw and derived artifacts. + description: A deterministic clean backend removes raw and derived artifacts. spec: + suite: erasure adapter: name: reference-clean mode: clean isolation: - scope: run + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject fixtures: - id: target - content: "ForgetProof canary: target alpha 7f3e9d" - target: true + role: target + subject: target-subject + content: "MemoryProof canary target alpha 7f3e9d" - id: control - content: "ForgetProof canary: control beta 91c4a2" - target: false + role: control + subject: control-subject + content: "MemoryProof canary control beta 91c4a2" settle: timeout_ms: 5000 interval_ms: 25 @@ -44,9 +49,9 @@ spec: fixture: target kind: agent profiles: - - FP-Object - - FP-Scope - - FP-Derived - - FP-Agent + - erasure.object + - erasure.scope + - erasure.derived + - erasure.agent privacy: raw_payloads: false diff --git a/examples/reference-leaky.yml b/examples/reference-leaky.yml index 814d8ab..c851241 100644 --- a/examples/reference-leaky.yml +++ b/examples/reference-leaky.yml @@ -1,21 +1,26 @@ -apiVersion: forgetproof.dev/v1alpha1 -kind: ErasureScenario +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario metadata: name: reference-leaky description: A deliberately leaky backend leaves derived artifacts after deletion. spec: + suite: erasure adapter: name: reference-leaky mode: leaky isolation: - scope: run + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject fixtures: - id: target - content: "ForgetProof canary: target alpha 7f3e9d" - target: true + role: target + subject: target-subject + content: "MemoryProof canary target alpha 7f3e9d" - id: control - content: "ForgetProof canary: control beta 91c4a2" - target: false + role: control + subject: control-subject + content: "MemoryProof canary control beta 91c4a2" settle: timeout_ms: 5000 interval_ms: 25 @@ -44,9 +49,9 @@ spec: fixture: target kind: agent profiles: - - FP-Object - - FP-Scope - - FP-Derived - - FP-Agent + - erasure.object + - erasure.scope + - erasure.derived + - erasure.agent privacy: raw_payloads: false diff --git a/examples/reference-overdelete.yml b/examples/reference-overdelete.yml new file mode 100644 index 0000000..545aee7 --- /dev/null +++ b/examples/reference-overdelete.yml @@ -0,0 +1,47 @@ +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario +metadata: + name: reference-overdelete + description: A deliberately unsafe backend deletes the control subject too. +spec: + suite: erasure + adapter: + name: reference-overdelete + mode: overdelete + isolation: + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject + fixtures: + - id: target + role: target + subject: target-subject + content: "MemoryProof overdelete target 5f1e8c" + - id: control + role: control + subject: control-subject + content: "MemoryProof overdelete control 0b4d72" + settle: + timeout_ms: 5000 + interval_ms: 25 + erase: + intent: subject_erase + target: target + probes: + before: + - id: target-before + fixture: target + kind: exact + - id: control-before + fixture: control + kind: exact + after: + - id: target-after + fixture: target + kind: exact + - id: control-after + fixture: control + kind: exact + profiles: + - erasure.object + - erasure.scope diff --git a/examples/zep.yml b/examples/zep.yml index 1c6d7ef..ab559be 100644 --- a/examples/zep.yml +++ b/examples/zep.yml @@ -1,23 +1,28 @@ -apiVersion: forgetproof.dev/v1alpha1 -kind: ErasureScenario +apiVersion: memoryproof.dev/v1 +kind: AssuranceScenario metadata: name: zep-erasure - description: Zep episode/thread graph erasure with node and summary inspection. + description: Zep episode and graph erasure with separate temporary users. spec: + suite: erasure adapter: name: zep - mode: default + mode: self-hosted config: base_url: http://localhost:8000 isolation: - scope: run + scope: memoryproof-run + target_subject: target-subject + control_subject: control-subject fixtures: - id: target - content: "ForgetProof Zep canary target 31a9e2" - target: true + role: target + subject: target-subject + content: "MemoryProof Zep canary target 31a9e2" - id: control - content: "ForgetProof Zep control 8c2d10" - target: false + role: control + subject: control-subject + content: "MemoryProof Zep control 8c2d10" settle: timeout_ms: 30000 interval_ms: 1000 @@ -42,4 +47,7 @@ spec: - id: target-derived-after fixture: target kind: inspect - profiles: [FP-Object, FP-Scope, FP-Derived] + profiles: + - erasure.object + - erasure.scope + - erasure.derived diff --git a/pyproject.toml b/pyproject.toml index 0388ab2..0234a69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,27 @@ requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" [project] -name = "forgetproof-adapters" -version = "0.1.0" -description = "Python NDJSON adapters for ForgetProof / ForgetProof 的 Python NDJSON 适配器" +name = "memoryproof-adapters" +version = "1.0.0" +description = "Python NDJSON adapters for MemoryProof / MemoryProof 的 Python NDJSON 适配器" readme = "README.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } -authors = [{ name = "ForgetProof contributors" }] +authors = [{ name = "MemoryProof contributors" }] +keywords = ["ai", "agents", "memory", "privacy", "testing", "erasure"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Topic :: Software Development :: Testing", +] dependencies = [] +[project.urls] +Homepage = "https://hughhhhcoder.github.io/MemoryProof/" +Repository = "https://github.com/Hughhhhcoder/MemoryProof" +Documentation = "https://hughhhhcoder.github.io/MemoryProof/" + [tool.setuptools.packages.find] where = ["python"] diff --git a/python/forgetproof_adapters/__init__.py b/python/forgetproof_adapters/__init__.py index e9cfaba..91af7dc 100644 --- a/python/forgetproof_adapters/__init__.py +++ b/python/forgetproof_adapters/__init__.py @@ -1,3 +1,7 @@ -"""Official and reference adapters for ForgetProof.""" +"""Official and reference adapters for MemoryProof. -PROTOCOL_VERSION = "forgetproof.adapter/v1alpha1" +The import path remains forgetproof_adapters for v0.x compatibility; the +published distribution is memoryproof-adapters. +""" + +PROTOCOL_VERSION = "memoryproof.adapter/v1" diff --git a/python/forgetproof_adapters/expand.py b/python/forgetproof_adapters/expand.py index a37e2b1..aa0d8b8 100644 --- a/python/forgetproof_adapters/expand.py +++ b/python/forgetproof_adapters/expand.py @@ -10,15 +10,27 @@ def main() -> None: scenario = json.load(sys.stdin) - base_url = os.environ.get("FORGETPROOF_LLM_BASE_URL") or os.environ.get("OPENAI_BASE_URL") - api_key = os.environ.get("FORGETPROOF_LLM_API_KEY") or os.environ.get("OPENAI_API_KEY", "") - model = os.environ.get("FORGETPROOF_LLM_MODEL") or os.environ.get("OPENAI_MODEL", "gpt-4o-mini") + base_url = ( + os.environ.get("MEMORYPROOF_LLM_BASE_URL") + or os.environ.get("FORGETPROOF_LLM_BASE_URL") + or os.environ.get("OPENAI_BASE_URL") + ) + api_key = ( + os.environ.get("MEMORYPROOF_LLM_API_KEY") + or os.environ.get("FORGETPROOF_LLM_API_KEY") + or os.environ.get("OPENAI_API_KEY", "") + ) + model = ( + os.environ.get("MEMORYPROOF_LLM_MODEL") + or os.environ.get("FORGETPROOF_LLM_MODEL") + or os.environ.get("OPENAI_MODEL", "gpt-4o-mini") + ) if not base_url: json.dump(scenario, sys.stdout, separators=(",", ":")) return prompt = { - "instruction": "Generate additional deterministic recall probes for this ForgetProof scenario.", + "instruction": "Generate additional deterministic recall probes for this MemoryProof scenario.", "rules": [ "Return a JSON array only.", "Each item must contain id, fixture, kind, query.", @@ -75,6 +87,7 @@ def main() -> None: "fixture": str(variant["fixture"]), "kind": str(variant["kind"]), "query": str(variant["query"]), + "as_subject": str(variant.get("as_subject", "")), } ) existing.add(str(variant["id"])) diff --git a/python/forgetproof_adapters/letta.py b/python/forgetproof_adapters/letta.py index fd5b2b0..ec68e44 100644 --- a/python/forgetproof_adapters/letta.py +++ b/python/forgetproof_adapters/letta.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Any from urllib.parse import quote @@ -9,7 +10,7 @@ class LettaAdapter(RemoteAdapter): backend = "letta" - version = "api-compatible" + version = "configured" api_key_env = "LETTA_API_KEY" base_url_env = "LETTA_BASE_URL" default_base_url = "http://localhost:8283" @@ -21,80 +22,126 @@ class LettaAdapter(RemoteAdapter): "semantic_search", "inspect", "derived_inspect", - "derived_delete", "agent_query", "async_settle", "isolated_namespace", ) - - def __init__(self) -> None: - super().__init__() - self.agent_id = "" + modes = ("self-hosted", "cloud") def prepare_remote(self, params: dict[str, Any]) -> dict[str, Any]: - requested = self.config.get("agent_id", "") - if requested: - self.agent_id = requested - else: + if self.config.get("agent_id"): + raise AdapterError( + "refusing to delete a configured existing Agent; omit agent_id so MemoryProof can own temporary Agents", + "ownership_required", + ) + subjects = sorted( + { + str(item.get("subject")) + for item in params.get("fixtures", []) + if isinstance(item, dict) and item.get("subject") + } + ) + for subject in subjects: result = self.request( "POST", self.endpoint("agent_create", "/v1/agents"), - {"name": f"forgetproof-{self.run_id}", "description": "ForgetProof isolated test agent"}, + { + "name": f"memoryproof-{self.run_id}-{_safe(subject)}", + "description": "Temporary MemoryProof isolation test Agent", + }, ) - self.agent_id = extract_id(result) - if not self.agent_id: - raise AdapterError("Letta agent creation did not return an id", "invalid_response") - return {"namespace": self.agent_id, "agent_id": self.agent_id} + agent_id = extract_id(result) + if not agent_id: + raise AdapterError("Letta Agent creation did not return an id", "invalid_response") + self.subject_ids[subject] = agent_id + self.created_resources.append(agent_id) + return { + "namespace": self.run_id, + "owned_agents": self.subject_ids, + "ownership_token": f"memoryproof:{self.run_id}", + } def ingest_remote(self, fixture: dict[str, Any]) -> Any: + agent_id = self._agent_for(str(fixture["subject"])) return self.request( "POST", - self.endpoint("archival_insert", f"/v1/agents/{self.agent_id}/archival-memory"), + self.endpoint("archival_insert", f"/v1/agents/{agent_id}/archival-memory"), { "text": fixture["content"], - "metadata": {"forgetproof_run": self.run_id, "forgetproof_fixture": fixture["id"]}, + "metadata": { + "memoryproof_run": self.run_id, + "memoryproof_fixture": fixture["id"], + "memoryproof_subject": fixture["subject"], + }, }, ) - def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: - endpoint = self.endpoint("archival_search", f"/v1/agents/{self.agent_id}/archival-memory/search") - return self.request( - "GET", - endpoint + "?query=" + quote(probe.get("query") or fixture["content"]) + "&top_k=20", + def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + agent_id = self._agent_for(subject) + endpoint = self.endpoint( + "archival_search", + f"/v1/agents/{agent_id}/archival-memory/search", ) + query = quote(str(probe.get("query") or fixture["content"])) + return self.request("GET", f"{endpoint}?query={query}&top_k=20") - def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: + def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + agent_id = self._agent_for(subject) return self.request( "GET", - self.endpoint("archival_list", f"/v1/agents/{self.agent_id}/archival-memory"), + self.endpoint("archival_list", f"/v1/agents/{agent_id}/archival-memory"), ) - def handle_agent_query(self, params: dict[str, Any]) -> dict[str, Any]: - probe = params.get("probe") or {} - fixture = self.fixtures.get(str(probe.get("fixture", ""))) - if fixture is None: - raise AdapterError("unknown fixture", "unknown_fixture") - result = self.request( + def agent_query_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + agent_id = self._agent_for(subject) + query = str(probe.get("query") or "What do you remember that is unique to this test?") + return self.request( "POST", - self.endpoint("agent_message", f"/v1/agents/{self.agent_id}/messages"), - {"messages": [{"role": "user", "content": f"Recall: {fixture['content']}"}]}, + self.endpoint("agent_message", f"/v1/agents/{agent_id}/messages"), + {"messages": [{"role": "user", "content": query}]}, ) - return {"found": fixture["content"] in str(result), "response": "redacted"} - def erase_remote(self, fixture: dict[str, Any], intent: str) -> Any: + def erase_remote(self, fixture: dict[str, Any], intent: str, subject: str) -> Any: + agent_id = self._agent_for(subject) if intent in {"subject_erase", "derived_purge"}: - endpoint = self.endpoint("agent_delete", f"/v1/agents/{self.agent_id}") - return self.request("DELETE", endpoint) - remote_id = self.remote_ids.get(fixture["id"], fixture["id"]) + if intent == "derived_purge": + raise AdapterError( + "Letta requires a configured derived-artifact deletion mapping for derived_purge", + "unsupported_capability", + ) + return self.request("DELETE", self.endpoint("agent_delete", f"/v1/agents/{agent_id}")) + remote_id = self.remote_ids.get(fixture["id"]) + if not remote_id: + raise AdapterError("archival insert response did not return a passage id", "invalid_response") return self.request( "DELETE", - self.endpoint("archival_delete", f"/v1/agents/{self.agent_id}/archival-memory/{remote_id}"), + self.endpoint( + "archival_delete", + f"/v1/agents/{agent_id}/archival-memory/{remote_id}", + ), ) def cleanup_remote(self, params: dict[str, Any]) -> dict[str, Any]: - if self.agent_id: - self.request("DELETE", self.endpoint("agent_delete", f"/v1/agents/{self.agent_id}")) - return {"cleaned": True, "agent_id": self.agent_id} + failures: list[str] = [] + for agent_id in set(self.subject_ids.values()): + try: + self.request("DELETE", self.endpoint("agent_delete", f"/v1/agents/{agent_id}")) + except AdapterError as exc: + if "HTTP 404" not in str(exc): + failures.append(str(exc)) + if failures: + raise AdapterError("; ".join(failures), "cleanup_error") + return {"cleaned": True, "owned_agents": sorted(self.subject_ids.values())} + + def _agent_for(self, subject: str) -> str: + agent_id = self.subject_ids.get(subject) + if not agent_id: + raise AdapterError(f"no owned Letta Agent for subject '{subject}'", "ownership_required") + return agent_id + + +def _safe(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_-]+", "-", value).strip("-") or "subject" def main() -> None: diff --git a/python/forgetproof_adapters/mem0.py b/python/forgetproof_adapters/mem0.py index 942c601..af52cd2 100644 --- a/python/forgetproof_adapters/mem0.py +++ b/python/forgetproof_adapters/mem0.py @@ -1,13 +1,16 @@ from __future__ import annotations +import re from typing import Any +from urllib.parse import urlencode +from .protocol import AdapterError from .remote import RemoteAdapter class Mem0Adapter(RemoteAdapter): backend = "mem0" - version = "api-compatible" + version = "configured" api_key_env = "MEM0_API_KEY" base_url_env = "MEM0_BASE_URL" default_base_url = "http://localhost:8888" @@ -19,73 +22,102 @@ class Mem0Adapter(RemoteAdapter): "semantic_search", "inspect", "derived_inspect", - "derived_delete", "async_settle", "isolated_namespace", ) + modes = ("oss", "platform", "cloud") def endpoint(self, key: str, default: str) -> str: configured = self.config.get(f"endpoint_{key}") if configured: return configured if self.mode in {"platform", "cloud"}: - platform_defaults = { + return { "add": "/v1/memories", "search": "/v1/memories/search", "list": "/v1/memories", "delete": "/v1/memories/{memory_id}", "scope_delete": "/v1/memories", - } - return platform_defaults.get(key, default) + }.get(key, default) return default + def prepare_remote(self, params: dict[str, Any]) -> dict[str, Any]: + fixtures = params.get("fixtures") or [] + for item in fixtures: + subject = str(item.get("subject", "")) + if subject: + self.subject_ids[subject] = self._subject_id(subject) + return { + "namespace": self.run_id, + "owned_subjects": sorted(self.subject_ids.values()), + "ownership_token": f"memoryproof:{self.run_id}", + } + def ingest_remote(self, fixture: dict[str, Any]) -> Any: - return self.request( - "POST", - self.endpoint("add", "/memories"), - { - "messages": [{"role": "user", "content": fixture["content"]}], - "user_id": self.run_id, - "metadata": {"forgetproof_run": self.run_id, "forgetproof_fixture": fixture["id"]}, + subject_id = self.subject_ids.get(str(fixture["subject"]), self._subject_id(str(fixture["subject"]))) + body: dict[str, Any] = { + "messages": [{"role": "user", "content": fixture["content"]}], + "user_id": subject_id, + "metadata": { + "memoryproof_run": self.run_id, + "memoryproof_fixture": fixture["id"], + "memoryproof_subject": fixture["subject"], }, - ) + } + infer = self.config.get("infer") + if infer is not None: + body["infer"] = infer.lower() == "true" + return self.request("POST", self.endpoint("add", "/memories"), body) - def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: - kind = str(probe.get("kind", "exact")) - endpoint = self.endpoint("search", "/search") + def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + subject_id = self.subject_ids.get(subject, self._subject_id(subject)) + query = probe.get("query") or fixture["content"] return self.request( "POST", - endpoint, - { - "query": probe.get("query") or fixture["content"], - "user_id": self.run_id, - "limit": 20, - "mode": kind, - }, + self.endpoint("search", "/search"), + {"query": query, "user_id": subject_id, "limit": 20}, ) - def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: + def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + subject_id = self.subject_ids.get(subject, self._subject_id(subject)) endpoint = self.endpoint("list", "/memories") separator = "&" if "?" in endpoint else "?" - return self.request("GET", f"{endpoint}{separator}user_id={self.run_id}") + return self.request("GET", f"{endpoint}{separator}{urlencode({'user_id': subject_id})}") - def erase_remote(self, fixture: dict[str, Any], intent: str) -> Any: - remote_id = self.remote_ids.get(fixture["id"], fixture["id"]) - if intent in {"subject_erase", "derived_purge"}: + def erase_remote(self, fixture: dict[str, Any], intent: str, subject: str) -> Any: + subject_id = self.subject_ids.get(subject, self._subject_id(subject)) + if intent in {"subject_erase"}: endpoint = self.endpoint("scope_delete", "/memories") separator = "&" if "?" in endpoint else "?" - return self.request("DELETE", f"{endpoint}{separator}user_id={self.run_id}") - endpoint = self.endpoint("delete", f"/memories/{remote_id}") - endpoint = endpoint.replace("{memory_id}", remote_id) + return self.request("DELETE", f"{endpoint}{separator}{urlencode({'user_id': subject_id})}") + if intent == "derived_purge": + raise AdapterError( + "Mem0 does not expose a separately observable derived-delete endpoint", + "unsupported_capability", + ) + remote_id = self.remote_ids.get(fixture["id"]) + if not remote_id: + raise AdapterError("memory create response did not return an id", "invalid_response") + endpoint = self.endpoint("delete", f"/memories/{remote_id}").replace("{memory_id}", remote_id) return self.request("DELETE", endpoint) def cleanup_remote(self, params: dict[str, Any]) -> dict[str, Any]: - # The namespace is generated per run. Cleanup is intentionally scoped - # to the synthetic user_id and never issues a provider-wide reset. - endpoint = self.endpoint("scope_delete", "/memories") - separator = "&" if "?" in endpoint else "?" - self.request("DELETE", f"{endpoint}{separator}user_id={self.run_id}") - return {"cleaned": True, "scope": self.run_id} + failures: list[str] = [] + for subject_id in set(self.subject_ids.values()): + endpoint = self.endpoint("scope_delete", "/memories") + separator = "&" if "?" in endpoint else "?" + try: + self.request("DELETE", f"{endpoint}{separator}{urlencode({'user_id': subject_id})}") + except AdapterError as exc: + if "HTTP 404" not in str(exc): + failures.append(str(exc)) + if failures: + raise AdapterError("; ".join(failures), "cleanup_error") + return {"cleaned": True, "owned_subjects": sorted(self.subject_ids.values())} + + def _subject_id(self, subject: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_-]+", "-", subject).strip("-") or "subject" + return f"memoryproof-{self.run_id}-{safe}" def main() -> None: diff --git a/python/forgetproof_adapters/protocol.py b/python/forgetproof_adapters/protocol.py index ace6f91..889d74a 100644 --- a/python/forgetproof_adapters/protocol.py +++ b/python/forgetproof_adapters/protocol.py @@ -6,8 +6,9 @@ import sys from typing import Any -PROTOCOL_VERSION = "forgetproof.adapter/v1alpha1" -log = logging.getLogger("forgetproof.adapter") +PROTOCOL_VERSION = "memoryproof.adapter/v1" +LEGACY_PROTOCOL_VERSION = "forgetproof.adapter/v1alpha1" +log = logging.getLogger("memoryproof.adapter") class AdapterError(Exception): @@ -19,24 +20,36 @@ def __init__(self, message: str, code: str = "adapter_error") -> None: class AdapterServer: - """Small dependency-free JSON-lines server used by every adapter. + """Dependency-free JSON-lines server used by every official adapter. stdout is reserved for protocol frames. Diagnostic logs always go to stderr. """ adapter_name = "unknown" backend = "unknown" - version = "0.1" + version = "1.0.0" capabilities: tuple[str, ...] = () + modes: tuple[str, ...] = () def __init__(self) -> None: - self.mode = os.environ.get("FORGETPROOF_ADAPTER_MODE", "default") - self.adapter_name = os.environ.get("FORGETPROOF_ADAPTER_NAME", self.adapter_name) - raw_config = os.environ.get("FORGETPROOF_CONFIG_JSON", "{}") + self.mode = os.environ.get( + "MEMORYPROOF_ADAPTER_MODE", + os.environ.get("FORGETPROOF_ADAPTER_MODE", "default"), + ) + self.adapter_name = os.environ.get( + "MEMORYPROOF_ADAPTER_NAME", + os.environ.get("FORGETPROOF_ADAPTER_NAME", self.adapter_name), + ) + raw_config = os.environ.get( + "MEMORYPROOF_CONFIG_JSON", + os.environ.get("FORGETPROOF_CONFIG_JSON", "{}"), + ) try: self.config: dict[str, str] = json.loads(raw_config) except json.JSONDecodeError as exc: - raise AdapterError(f"invalid FORGETPROOF_CONFIG_JSON: {exc}", "invalid_config") + raise AdapterError(f"invalid adapter configuration: {exc}", "invalid_config") from exc + if not isinstance(self.config, dict): + raise AdapterError("adapter configuration must be a JSON object", "invalid_config") self.closed = False def serve(self) -> None: @@ -45,17 +58,20 @@ def serve(self) -> None: line = raw_line.strip() if not line: continue + request: dict[str, Any] | None = None try: request = json.loads(line) response = self.dispatch(request) except json.JSONDecodeError as exc: response = { + "protocol": PROTOCOL_VERSION, "id": None, "ok": False, "error": {"code": "invalid_json", "message": str(exc)}, } except AdapterError as exc: response = { + "protocol": PROTOCOL_VERSION, "id": request.get("id") if isinstance(request, dict) else None, "ok": False, "error": {"code": exc.code, "message": str(exc)}, @@ -63,6 +79,7 @@ def serve(self) -> None: except Exception as exc: # pragma: no cover - defensive process boundary log.exception("adapter request failed") response = { + "protocol": PROTOCOL_VERSION, "id": request.get("id") if isinstance(request, dict) else None, "ok": False, "error": {"code": "internal_error", "message": str(exc)}, @@ -78,6 +95,12 @@ def dispatch(self, request: dict[str, Any]) -> dict[str, Any]: request_id = request.get("id") if not isinstance(request_id, str): raise AdapterError("request id must be a string", "invalid_request") + requested_protocol = request.get("protocol", PROTOCOL_VERSION) + if requested_protocol != PROTOCOL_VERSION: + raise AdapterError( + f"protocol mismatch: expected {PROTOCOL_VERSION}, got {requested_protocol}", + "protocol_mismatch", + ) method = request.get("method") if not isinstance(method, str): raise AdapterError("method must be a string", "invalid_request") @@ -88,7 +111,7 @@ def dispatch(self, request: dict[str, Any]) -> dict[str, Any]: if handler is None: raise AdapterError(f"unsupported method: {method}", "unsupported_method") result = handler(params) - return {"id": request_id, "ok": True, "result": result} + return {"protocol": PROTOCOL_VERSION, "id": request_id, "ok": True, "result": result} def handle_hello(self, params: dict[str, Any]) -> dict[str, Any]: requested = params.get("protocol", PROTOCOL_VERSION) @@ -106,6 +129,7 @@ def handle_capabilities(self, params: dict[str, Any]) -> dict[str, Any]: "backend": self.backend, "version": self.version, "capabilities": list(self.capabilities), + "modes": list(self.modes), } def handle_close(self, params: dict[str, Any]) -> dict[str, Any]: @@ -119,7 +143,7 @@ def handle_ingest(self, params: dict[str, Any]) -> dict[str, Any]: raise AdapterError("ingest is not implemented", "unsupported_method") def handle_settle(self, params: dict[str, Any]) -> dict[str, Any]: - return {"stable": True} + return {"state": "stable", "stable": True, "observations": 1} def handle_probe(self, params: dict[str, Any]) -> dict[str, Any]: raise AdapterError("probe is not implemented", "unsupported_method") diff --git a/python/forgetproof_adapters/reference.py b/python/forgetproof_adapters/reference.py index c81e265..dcf46b5 100644 --- a/python/forgetproof_adapters/reference.py +++ b/python/forgetproof_adapters/reference.py @@ -1,14 +1,15 @@ from __future__ import annotations import argparse +import os from typing import Any -from .protocol import AdapterServer +from .protocol import AdapterError, AdapterServer class ReferenceAdapter(AdapterServer): - backend = "forgetproof-reference" - version = "0.1.0" + backend = "memoryproof-reference" + version = "1.0.0" capabilities = ( "object_delete", "scope_delete", @@ -22,75 +23,174 @@ class ReferenceAdapter(AdapterServer): "async_settle", "isolated_namespace", ) + modes = ("clean", "leaky", "overdelete", "slow", "crash", "malformed") def __init__(self) -> None: super().__init__() self.leaky = self.mode == "leaky" + self.overdelete = self.mode == "overdelete" + self.slow = self.mode == "slow" + self.crash = self.mode == "crash" + self.malformed = self.mode == "malformed" self.run_id = "" - self.raw: dict[str, str] = {} - self.derived: dict[str, str] = {} - self.targets: set[str] = set() + self.raw: dict[tuple[str, str], str] = {} + self.derived: dict[tuple[str, str], str] = {} + self.fixtures: dict[str, dict[str, Any]] = {} + self.owned_subjects: set[str] = set() + + def serve(self) -> None: + if self.malformed: + for _line in __import__("sys").stdin: + __import__("sys").stdout.write("this is not a protocol frame\n") + __import__("sys").stdout.flush() + return + super().serve() + + def _maybe_crash(self) -> None: + if self.crash: + os._exit(17) def handle_prepare(self, params: dict[str, Any]) -> dict[str, Any]: + self._maybe_crash() self.run_id = str(params.get("run_id", "")) self.raw.clear() self.derived.clear() - self.targets.clear() - return {"namespace": f"reference:{self.run_id}"} + self.fixtures = { + str(item.get("id")): item + for item in params.get("fixtures", []) + if isinstance(item, dict) and item.get("id") + } + self.owned_subjects = { + str(item.get("subject")) + for item in self.fixtures.values() + if item.get("subject") + } + return { + "namespace": f"reference:{self.run_id}", + "owned_subjects": sorted(self.owned_subjects), + "ownership_token": f"memoryproof:{self.run_id}", + } def handle_ingest(self, params: dict[str, Any]) -> dict[str, Any]: + self._maybe_crash() fixture = params.get("fixture") or {} fixture_id = str(fixture.get("id", "")) content = str(fixture.get("content", "")) - if not fixture_id or not content: - raise ValueError("fixture requires id and content") - self.raw[fixture_id] = content - self.derived[fixture_id] = content - if fixture.get("target"): - self.targets.add(fixture_id) - return {"stored": True, "fixture": fixture_id} + subject = str(fixture.get("subject", "")) + if not fixture_id or not content or not subject: + raise AdapterError("fixture requires id, content, and subject", "invalid_fixture") + self.fixtures[fixture_id] = fixture + self.owned_subjects.add(subject) + key = (subject, fixture_id) + self.raw[key] = content + self.derived[key] = content + return {"stored": True, "fixture": fixture_id, "subject": subject} + + def handle_settle(self, params: dict[str, Any]) -> dict[str, Any]: + if self.slow: + return {"state": "timeout", "stable": False, "observations": 1} + return {"state": "stable", "stable": True, "observations": 1} def handle_probe(self, params: dict[str, Any]) -> dict[str, Any]: + self._maybe_crash() probe = params.get("probe") or {} fixture_id = str(probe.get("fixture", "")) + subject = self._probe_subject(probe, fixture_id) kind = str(probe.get("kind", "exact")) - found = fixture_id in self.raw - if kind in {"lexical", "semantic"}: - found = fixture_id in self.raw or fixture_id in self.derived - if kind == "exact": - found = fixture_id in self.raw or (self.leaky and fixture_id in self.derived) - return {"found": found, "scope": "reference"} + key = (subject, fixture_id) + if subject in getattr(self, "revoked_subjects", set()): + found = False + elif kind in {"lexical", "semantic"}: + found = key in self.raw or key in self.derived + else: + # Exact probes inspect only the raw object boundary. The leaky + # mode should therefore fail at derived/agent boundaries. + found = key in self.raw + return {"found": found, "subject": subject, "artifact": "raw"} def handle_inspect(self, params: dict[str, Any]) -> dict[str, Any]: - fixture_id = str((params.get("probe") or {}).get("fixture", "")) - # Inspect represents observable derived artifacts, not raw memory. - return {"found": fixture_id in self.derived, "artifact_types": ["summary", "index"]} + self._maybe_crash() + probe = params.get("probe") or {} + fixture_id = str(probe.get("fixture", "")) + subject = self._probe_subject(probe, fixture_id) + return { + "found": (subject, fixture_id) in self.derived, + "subject": subject, + "artifact_types": ["summary", "index"], + } def handle_agent_query(self, params: dict[str, Any]) -> dict[str, Any]: - fixture_id = str((params.get("probe") or {}).get("fixture", "")) - found = fixture_id in self.raw or fixture_id in self.derived - return {"found": found, "response_contains_canary": found} + self._maybe_crash() + probe = params.get("probe") or {} + fixture_id = str(probe.get("fixture", "")) + subject = self._probe_subject(probe, fixture_id) + key = (subject, fixture_id) + found = key in self.raw or key in self.derived + return {"found": found, "subject": subject, "response_contains_canary": found} def handle_erase(self, params: dict[str, Any]) -> dict[str, Any]: + self._maybe_crash() target = str(params.get("target", "")) intent = str(params.get("intent", "object_delete")) + target_subject = str(params.get("target_subject", "")) + if not target or not target_subject: + raise AdapterError("erase requires target and target_subject", "invalid_erase") + if target_subject not in self.owned_subjects: + raise AdapterError("target subject is not owned by this run", "ownership_required") if intent == "access_revoke": - return {"revoked": True, "deleted": False} - self.raw.pop(target, None) - if not self.leaky or intent == "derived_purge": - self.derived.pop(target, None) - return {"deleted": True, "target": target} + if not hasattr(self, "revoked_subjects"): + self.revoked_subjects: set[str] = set() + self.revoked_subjects.add(target_subject) + return {"revoked": True, "deleted": False, "subject": target_subject} + + subjects = set(self.owned_subjects) if self.overdelete else {target_subject} + keys = [key for key in self.raw if key[0] in subjects and (self.overdelete or key[1] == target)] + for key in keys: + self.raw.pop(key, None) + if not self.leaky or intent == "derived_purge": + self.derived.pop(key, None) + if intent in {"subject_erase", "derived_purge"}: + for key in list(self.raw): + if key[0] in subjects: + self.raw.pop(key, None) + if not self.leaky or intent == "derived_purge": + for key in list(self.derived): + if key[0] in subjects: + self.derived.pop(key, None) + return { + "deleted": True, + "target": target, + "target_subject": target_subject, + "affected_subjects": sorted(subjects), + } def handle_cleanup(self, params: dict[str, Any]) -> dict[str, Any]: + # The in-memory backend has no external side effects. The ownership + # check mirrors what remote adapters must enforce before deletion. + requested = str(params.get("run_id", "")) + if requested and requested != self.run_id: + raise AdapterError("cleanup run_id does not match owner", "ownership_required") self.raw.clear() self.derived.clear() - self.targets.clear() - return {"cleaned": True} + self.fixtures.clear() + self.owned_subjects.clear() + return {"cleaned": True, "ownership_token": f"memoryproof:{self.run_id}"} + + def _probe_subject(self, probe: dict[str, Any], fixture_id: str) -> str: + explicit = str(probe.get("as_subject", "")) + if explicit: + return explicit + fixture = self.fixtures.get(fixture_id) or {} + return str(fixture.get("subject", "")) def main() -> None: - parser = argparse.ArgumentParser(description="ForgetProof reference adapter") - parser.add_argument("--mode", choices=["clean", "leaky"], default=None) + parser = argparse.ArgumentParser(description="MemoryProof reference adapter") + parser.add_argument( + "--mode", + choices=["clean", "leaky", "overdelete", "slow", "crash", "malformed"], + default=None, + ) parser.parse_args() ReferenceAdapter().serve() diff --git a/python/forgetproof_adapters/remote.py b/python/forgetproof_adapters/remote.py index 876e569..2e5e58f 100644 --- a/python/forgetproof_adapters/remote.py +++ b/python/forgetproof_adapters/remote.py @@ -1,7 +1,10 @@ from __future__ import annotations +import hashlib import json import os +import re +import time import urllib.error import urllib.parse import urllib.request @@ -11,15 +14,12 @@ class RemoteAdapter(AdapterServer): - """Dependency-free HTTP adapter base. - - Endpoint paths are intentionally configurable. Hosted and self-hosted - deployments often expose the same concepts under different prefixes. - """ + """Dependency-free HTTP adapter base with ownership and redaction guards.""" api_key_env = "" base_url_env = "" default_base_url = "" + request_id_headers = ("x-request-id", "x-correlation-id", "traceparent") def __init__(self) -> None: super().__init__() @@ -31,6 +31,11 @@ def __init__(self) -> None: self.run_id = "" self.fixtures: dict[str, dict[str, Any]] = {} self.remote_ids: dict[str, str] = {} + self.subject_ids: dict[str, str] = {} + self.created_resources: list[str] = [] + self.request_ids: list[str] = [] + self.pending_operations: list[dict[str, Any]] = [] + self.owned = False def endpoint(self, key: str, default: str) -> str: return self.config.get(f"endpoint_{key}", default) @@ -38,116 +43,229 @@ def endpoint(self, key: str, default: str) -> str: def _headers(self) -> dict[str, str]: headers = {"Accept": "application/json", "Content-Type": "application/json"} env_name = self.config.get("api_key_env", self.api_key_env) - if env_name: - key = os.environ.get(env_name, "") - if key: - headers["Authorization"] = f"Bearer {key}" - headers["X-API-Key"] = key + key = os.environ.get(env_name, "") if env_name else "" + if key: + auth_header = self.config.get("api_key_header", "Authorization") + headers[auth_header] = ( + f"Bearer {key}" if auth_header.lower() == "authorization" else key + ) return headers def request(self, method: str, path: str, body: Any | None = None) -> Any: if not self.base_url: raise AdapterError( - "no base_url configured; set it in scenario adapter.config or the adapter environment", + "no base_url configured; set adapter.config.base_url or the adapter environment", "missing_base_url", ) - if path.startswith("http://") or path.startswith("https://"): - url = path - else: - url = f"{self.base_url}/{path.lstrip('/')}" + url = path if path.startswith(("http://", "https://")) else f"{self.base_url}/{path.lstrip('/')}" data = None if body is None else json.dumps(body).encode("utf-8") request = urllib.request.Request(url, data=data, method=method, headers=self._headers()) + timeout = float(self.config.get("request_timeout_sec", "30")) + open_request = urllib.request.urlopen + hostname = urllib.parse.urlparse(url).hostname + if hostname in {"localhost", "127.0.0.1", "::1"}: + # Local self-hosted deployments and the deterministic contract + # mock must not be routed through a workstation HTTP proxy. + open_request = urllib.request.build_opener(urllib.request.ProxyHandler({})).open try: - with urllib.request.urlopen(request, timeout=30) as response: + with open_request(request, timeout=timeout) as response: raw = response.read() + for name in self.request_id_headers: + request_id = response.headers.get(name) + if request_id: + self.request_ids.append(request_id[:200]) + break + status = response.status except urllib.error.HTTPError as exc: - detail = exc.read().decode("utf-8", errors="replace")[:500] - raise AdapterError(f"HTTP {exc.code} from {method} {path}: {detail}", "http_error") from exc + detail = exc.read().decode("utf-8", errors="replace")[:300] + raise AdapterError( + f"HTTP {exc.code} from {method} {path}: {detail}", + "http_error", + ) from exc except urllib.error.URLError as exc: - raise AdapterError(f"request failed for {method} {path}: {exc.reason}", "network_error") from exc + raise AdapterError( + f"request failed for {method} {path}: {exc.reason}", + "network_error", + ) from exc if not raw: - return {} + return {"accepted": status < 400, "status": status} try: return json.loads(raw.decode("utf-8")) - except json.JSONDecodeError: - return {"text": raw.decode("utf-8", errors="replace")[:500]} + except json.JSONDecodeError as exc: + raise AdapterError( + f"backend returned non-JSON data for {method} {path}", + "invalid_response", + ) from exc def handle_prepare(self, params: dict[str, Any]) -> dict[str, Any]: self.run_id = str(params.get("run_id", "")) + if not self.run_id: + raise AdapterError("prepare requires run_id", "invalid_prepare") self.fixtures.clear() self.remote_ids.clear() - return self.prepare_remote(params) + self.subject_ids.clear() + self.created_resources.clear() + self.pending_operations.clear() + self.request_ids.clear() + self.owned = True + result = self.prepare_remote(params) + result.setdefault("ownership_token", f"memoryproof:{self.run_id}") + result.setdefault("request_ids", list(self.request_ids)) + return result def handle_ingest(self, params: dict[str, Any]) -> dict[str, Any]: fixture = params.get("fixture") or {} fixture_id = str(fixture.get("id", "")) - if not fixture_id or not fixture.get("content"): - raise AdapterError("fixture requires id and content", "invalid_fixture") + if not fixture_id or not fixture.get("content") or not fixture.get("subject"): + raise AdapterError("fixture requires id, content, and subject", "invalid_fixture") + if not self.owned: + raise AdapterError("run does not own a prepared namespace", "ownership_required") self.fixtures[fixture_id] = fixture result = self.ingest_remote(fixture) remote_id = extract_id(result) if remote_id: self.remote_ids[fixture_id] = remote_id - return {"stored": True, "fixture": fixture_id, "remote_id": remote_id, "response": summarize(result)} + self.created_resources.append(remote_id) + self._register_pending(result) + return { + "stored": True, + "fixture": fixture_id, + "subject": fixture["subject"], + "remote_id": remote_id, + "request_ids": list(self.request_ids), + "response": summarize(result), + } def handle_probe(self, params: dict[str, Any]) -> dict[str, Any]: - probe = params.get("probe") or {} - fixture_id = str(probe.get("fixture", "")) - fixture = self.fixtures.get(fixture_id) - if fixture is None: - raise AdapterError(f"unknown fixture: {fixture_id}", "unknown_fixture") - result = self.probe_remote(fixture, probe) - return {"found": extract_found(result, fixture), "response": summarize(result)} + fixture_id, fixture, probe = self._fixture_probe(params) + result = self.probe_remote(fixture, probe, str(probe.get("as_subject") or fixture["subject"])) + return { + "found": extract_found(result, fixture), + "request_ids": list(self.request_ids), + "response": summarize(result), + } def handle_inspect(self, params: dict[str, Any]) -> dict[str, Any]: - probe = params.get("probe") or {} - fixture_id = str(probe.get("fixture", "")) - fixture = self.fixtures.get(fixture_id) - if fixture is None: - raise AdapterError(f"unknown fixture: {fixture_id}", "unknown_fixture") - result = self.inspect_remote(fixture, probe) - return {"found": extract_found(result, fixture), "response": summarize(result)} + fixture_id, fixture, probe = self._fixture_probe(params) + result = self.inspect_remote(fixture, probe, str(probe.get("as_subject") or fixture["subject"])) + return { + "found": extract_found(result, fixture), + "request_ids": list(self.request_ids), + "response": summarize(result), + } + + def handle_agent_query(self, params: dict[str, Any]) -> dict[str, Any]: + fixture_id, fixture, probe = self._fixture_probe(params) + result = self.agent_query_remote(fixture, probe, str(probe.get("as_subject") or fixture["subject"])) + return { + "found": extract_found(result, fixture), + "request_ids": list(self.request_ids), + "response": summarize(result), + } def handle_erase(self, params: dict[str, Any]) -> dict[str, Any]: target = str(params.get("target", "")) intent = str(params.get("intent", "object_delete")) + target_subject = str(params.get("target_subject", "")) fixture = self.fixtures.get(target) if fixture is None: raise AdapterError(f"unknown fixture: {target}", "unknown_fixture") - result = self.erase_remote(fixture, intent) - return {"deleted": True, "target": target, "response": summarize(result)} + if target_subject != fixture.get("subject"): + raise AdapterError("erase target subject does not match owned fixture", "ownership_required") + if not self.owned: + raise AdapterError("run does not own a prepared namespace", "ownership_required") + result = self.erase_remote(fixture, intent, target_subject) + self._register_pending(result) + return { + "accepted": True, + "target": target, + "target_subject": target_subject, + "request_ids": list(self.request_ids), + "response": summarize(result), + } def handle_settle(self, params: dict[str, Any]) -> dict[str, Any]: return self.settle_remote(params) def handle_cleanup(self, params: dict[str, Any]) -> dict[str, Any]: - return self.cleanup_remote(params) + requested = str(params.get("run_id", "")) + if requested and requested != self.run_id: + raise AdapterError("cleanup run_id does not match owner", "ownership_required") + if not self.owned: + return {"cleaned": True, "owned": False} + result = self.cleanup_remote(params) + self.owned = False + result.setdefault("request_ids", list(self.request_ids)) + return result + + def _fixture_probe(self, params: dict[str, Any]) -> tuple[str, dict[str, Any], dict[str, Any]]: + probe = params.get("probe") or {} + fixture_id = str(probe.get("fixture", "")) + fixture = self.fixtures.get(fixture_id) + if fixture is None: + raise AdapterError(f"unknown fixture: {fixture_id}", "unknown_fixture") + return fixture_id, fixture, probe + + def _register_pending(self, result: Any) -> None: + if not isinstance(result, dict): + return + state = str(result.get("status") or result.get("state") or "").lower() + operation_id = result.get("event_id") or result.get("job_id") or result.get("operation_id") + if operation_id or state in {"pending", "processing", "queued", "accepted"}: + self.pending_operations.append( + {"id": str(operation_id or ""), "state": state, "result": result} + ) def prepare_remote(self, params: dict[str, Any]) -> dict[str, Any]: - return {"namespace": self.run_id} + return {"namespace": self.run_id, "owned": True} def ingest_remote(self, fixture: dict[str, Any]) -> Any: raise AdapterError("ingest mapping is not implemented", "unsupported_method") - def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: + def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: raise AdapterError("probe mapping is not implemented", "unsupported_method") - def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: + def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: raise AdapterError("inspect mapping is not implemented", "unsupported_method") - def erase_remote(self, fixture: dict[str, Any], intent: str) -> Any: + def agent_query_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + raise AdapterError("agent query mapping is not implemented", "unsupported_method") + + def erase_remote(self, fixture: dict[str, Any], intent: str, subject: str) -> Any: raise AdapterError("erase mapping is not implemented", "unsupported_method") def settle_remote(self, params: dict[str, Any]) -> dict[str, Any]: - return {"stable": True} + if not self.pending_operations: + return {"state": "stable", "stable": True, "observations": 1} + endpoint = self.config.get("settle_endpoint") + if not endpoint: + return {"state": "unknown", "stable": False, "observations": 1} + timeout = max(1, int(params.get("timeout_ms", 10_000))) / 1000 + interval = max(0.01, int(params.get("interval_ms", 250)) / 1000) + deadline = time.monotonic() + timeout + observations = 0 + while time.monotonic() < deadline: + observations += 1 + result = self.request( + "GET", + endpoint.replace("{run_id}", urllib.parse.quote(self.run_id)), + ) + state = str(result.get("state") or result.get("status") or "unknown").lower() + if state in {"stable", "complete", "completed", "done", "success"}: + self.pending_operations.clear() + return {"state": "stable", "stable": True, "observations": observations} + if state in {"failed", "error"}: + return {"state": "unknown", "stable": False, "observations": observations} + time.sleep(interval) + return {"state": "timeout", "stable": False, "observations": observations} def cleanup_remote(self, params: dict[str, Any]) -> dict[str, Any]: - return {"cleaned": True} + return {"cleaned": True, "owned": True} def extract_id(value: Any) -> str: if isinstance(value, dict): - for key in ("id", "memory_id", "uuid", "message_id", "node_id"): + for key in ("id", "memory_id", "uuid", "message_id", "node_id", "episode_id", "agent_id"): candidate = value.get(key) if candidate: return str(candidate) @@ -163,31 +281,47 @@ def extract_id(value: Any) -> str: return "" +def _markers(content: str) -> list[str]: + markers = re.findall(r"[A-Za-z0-9][A-Za-z0-9_-]{5,}", content) + return list(dict.fromkeys(markers)) + + def extract_found(value: Any, fixture: dict[str, Any]) -> bool: - if isinstance(value, dict): - if isinstance(value.get("found"), bool): - return value["found"] - for key in ("results", "memories", "data", "facts", "nodes", "messages", "edges"): - if key in value and extract_found(value[key], fixture): - return True - fixture_id = str(fixture.get("id", "")) - content = str(fixture.get("content", "")) - for key in ("id", "memory_id", "uuid", "fixture_id", "text", "memory", "content", "summary"): - candidate = value.get(key) - if candidate is not None and (str(candidate) == fixture_id or content in str(candidate)): - return True - elif isinstance(value, list): - return any(extract_found(child, fixture) for child in value) - elif isinstance(value, str): - return str(fixture.get("content", "")) in value - return False + content = str(fixture.get("content", "")) + fixture_id = str(fixture.get("id", "")) + markers = _markers(content) + + def visit(item: Any) -> bool: + if isinstance(item, dict): + if isinstance(item.get("found"), bool): + return bool(item["found"]) + for key in ("results", "memories", "data", "facts", "nodes", "messages", "edges", "episodes"): + if key in item and visit(item[key]): + return True + for key in ("fixture_id", "text", "memory", "content", "summary", "fact", "name"): + candidate = item.get(key) + if candidate is not None and (content in str(candidate) or any(marker in str(candidate) for marker in markers)): + return True + candidate_id = item.get("id") or item.get("memory_id") or item.get("uuid") + return str(candidate_id) == fixture_id if candidate_id is not None else False + if isinstance(item, list): + return any(visit(child) for child in item) + if isinstance(item, str): + return content in item or any(marker in item for marker in markers) + return False + + return visit(value) def summarize(value: Any) -> Any: if isinstance(value, dict): - return {str(key): summarize(child) for key, child in list(value.items())[:20] if key not in {"content", "text", "memory", "messages"}} + return { + str(key): summarize(child) + for key, child in list(value.items())[:30] + if key.lower() not in {"content", "text", "memory", "messages", "headers", "authorization"} + } if isinstance(value, list): return {"count": len(value)} if isinstance(value, str): - return {"sha256": __import__("hashlib").sha256(value.encode()).hexdigest(), "length": len(value)} + return {"sha256": hashlib.sha256(value.encode()).hexdigest(), "length": len(value)} return value diff --git a/python/forgetproof_adapters/zep.py b/python/forgetproof_adapters/zep.py index 2402611..d1771a0 100644 --- a/python/forgetproof_adapters/zep.py +++ b/python/forgetproof_adapters/zep.py @@ -1,14 +1,16 @@ from __future__ import annotations +import re from typing import Any -from urllib.parse import quote +from urllib.parse import urlencode -from .remote import RemoteAdapter +from .protocol import AdapterError +from .remote import RemoteAdapter, extract_id class ZepAdapter(RemoteAdapter): backend = "zep" - version = "api-compatible" + version = "configured" api_key_env = "ZEP_API_KEY" base_url_env = "ZEP_BASE_URL" default_base_url = "http://localhost:8000" @@ -20,67 +22,126 @@ class ZepAdapter(RemoteAdapter): "semantic_search", "inspect", "derived_inspect", - "derived_delete", "async_settle", "isolated_namespace", ) + modes = ("self-hosted", "cloud") def __init__(self) -> None: super().__init__() - self.user_id = "" - self.thread_id = "" + self.thread_ids: dict[str, str] = {} def prepare_remote(self, params: dict[str, Any]) -> dict[str, Any]: - self.user_id = f"forgetproof-{self.run_id}" - user_result = self.request( - "POST", - self.endpoint("user_create", "/api/v2/users"), - {"user_id": self.user_id, "first_name": "ForgetProof", "last_name": "Test"}, - ) - self.request( - "POST", - self.endpoint("thread_create", "/api/v2/threads"), - {"thread_id": self.user_id, "user_id": self.user_id}, + subjects = sorted( + { + str(item.get("subject")) + for item in params.get("fixtures", []) + if isinstance(item, dict) and item.get("subject") + } ) - self.thread_id = self.user_id - return {"namespace": self.user_id, "user_id": self.user_id, "user_response": bool(user_result)} + for subject in subjects: + user_id = f"memoryproof-{self.run_id}-{_safe(subject)}" + user_result = self.request( + "POST", + self.endpoint("user_create", "/api/v2/users"), + {"user_id": user_id, "first_name": "MemoryProof", "last_name": "Test"}, + ) + self.subject_ids[subject] = str( + user_result.get("user_id") if isinstance(user_result, dict) else user_id + ) + thread_id = f"{user_id}-thread" + self.request( + "POST", + self.endpoint("thread_create", "/api/v2/threads"), + {"thread_id": thread_id, "user_id": self.subject_ids[subject]}, + ) + self.thread_ids[subject] = thread_id + self.created_resources.extend([self.subject_ids[subject], thread_id]) + return { + "namespace": self.run_id, + "owned_users": self.subject_ids, + "owned_threads": self.thread_ids, + "ownership_token": f"memoryproof:{self.run_id}", + } def ingest_remote(self, fixture: dict[str, Any]) -> Any: + thread_id = self._thread_for(str(fixture["subject"])) return self.request( "POST", - self.endpoint("episode_add", f"/api/v2/threads/{self.thread_id}/messages"), + self.endpoint("episode_add", f"/api/v2/threads/{thread_id}/messages"), { "messages": [{"role": "user", "content": fixture["content"]}], - "metadata": {"forgetproof_run": self.run_id, "forgetproof_fixture": fixture["id"]}, + "metadata": { + "memoryproof_run": self.run_id, + "memoryproof_fixture": fixture["id"], + "memoryproof_subject": fixture["subject"], + }, }, ) - def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: + def probe_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + thread_id = self._thread_for(subject) + query = str(probe.get("query") or fixture["content"]) return self.request( "GET", - self.endpoint("search", f"/api/v2/threads/{self.thread_id}/search") - + "?" + "query=" + quote(probe.get("query") or fixture["content"]), + f"{self.endpoint('search', f'/api/v2/threads/{thread_id}/search')}?{urlencode({'query': query})}", ) - def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any]) -> Any: + def inspect_remote(self, fixture: dict[str, Any], probe: dict[str, Any], subject: str) -> Any: + user_id = self._user_for(subject) return self.request( "GET", - self.endpoint("graph", f"/api/v2/users/{self.user_id}/graph"), + self.endpoint("graph", f"/api/v2/users/{user_id}/graph"), ) - def erase_remote(self, fixture: dict[str, Any], intent: str) -> Any: - if intent in {"subject_erase", "derived_purge"}: - return self.request("DELETE", self.endpoint("user_delete", f"/api/v2/users/{self.user_id}")) - remote_id = self.remote_ids.get(fixture["id"], fixture["id"]) + def erase_remote(self, fixture: dict[str, Any], intent: str, subject: str) -> Any: + user_id = self._user_for(subject) + if intent in {"subject_erase"}: + return self.request("DELETE", self.endpoint("user_delete", f"/api/v2/users/{user_id}")) + if intent == "derived_purge": + raise AdapterError( + "Zep requires a configured graph-derived deletion mapping for derived_purge", + "unsupported_capability", + ) + remote_id = self.remote_ids.get(fixture["id"]) + if not remote_id: + raise AdapterError("episode response did not return an id", "invalid_response") + thread_id = self._thread_for(subject) return self.request( "DELETE", - self.endpoint("episode_delete", f"/api/v2/threads/{self.thread_id}/messages/{remote_id}"), + self.endpoint( + "episode_delete", + f"/api/v2/threads/{thread_id}/messages/{remote_id}", + ), ) def cleanup_remote(self, params: dict[str, Any]) -> dict[str, Any]: - if self.user_id: - self.request("DELETE", self.endpoint("user_delete", f"/api/v2/users/{self.user_id}")) - return {"cleaned": True, "user_id": self.user_id} + failures: list[str] = [] + for user_id in set(self.subject_ids.values()): + try: + self.request("DELETE", self.endpoint("user_delete", f"/api/v2/users/{user_id}")) + except AdapterError as exc: + if "HTTP 404" not in str(exc): + failures.append(str(exc)) + if failures: + raise AdapterError("; ".join(failures), "cleanup_error") + return {"cleaned": True, "owned_users": sorted(self.subject_ids.values())} + + def _user_for(self, subject: str) -> str: + user_id = self.subject_ids.get(subject) + if not user_id: + raise AdapterError(f"no owned Zep user for subject '{subject}'", "ownership_required") + return user_id + + def _thread_for(self, subject: str) -> str: + thread_id = self.thread_ids.get(subject) + if not thread_id: + raise AdapterError(f"no owned Zep thread for subject '{subject}'", "ownership_required") + return thread_id + + +def _safe(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_-]+", "-", value).strip("-") or "subject" def main() -> None: diff --git a/python/tests/test_adapters.py b/python/tests/test_adapters.py index be8711e..0803941 100644 --- a/python/tests/test_adapters.py +++ b/python/tests/test_adapters.py @@ -1,35 +1,101 @@ import os import unittest +from forgetproof_adapters.protocol import AdapterError, PROTOCOL_VERSION from forgetproof_adapters.reference import ReferenceAdapter from forgetproof_adapters.remote import extract_found +def fixture_payload(fixture_id: str, subject: str, content: str, role: str) -> dict: + return { + "id": fixture_id, + "subject": subject, + "content": content, + "role": role, + "target": role == "target", + } + + class ReferenceAdapterTests(unittest.TestCase): def setUp(self): - os.environ["FORGETPROOF_ADAPTER_NAME"] = "reference-clean" - os.environ["FORGETPROOF_ADAPTER_MODE"] = "clean" + os.environ["MEMORYPROOF_ADAPTER_NAME"] = "reference-clean" + os.environ["MEMORYPROOF_ADAPTER_MODE"] = "clean" self.adapter = ReferenceAdapter() - self.adapter.handle_prepare({"run_id": "test-run"}) - self.fixture = {"id": "target", "content": "unique canary", "target": True} - self.control = {"id": "control", "content": "control canary", "target": False} - self.adapter.handle_ingest({"fixture": self.fixture}) + self.target = fixture_payload("target", "target-subject", "unique target canary 7f3e9d", "target") + self.control = fixture_payload("control", "control-subject", "unique control canary 91c4a2", "control") + self.adapter.handle_prepare( + { + "run_id": "test-run", + "fixtures": [self.target, self.control], + } + ) + self.adapter.handle_ingest({"fixture": self.target}) self.adapter.handle_ingest({"fixture": self.control}) - def test_clean_removes_raw_and_derived(self): - self.assertTrue(self.adapter.handle_probe({"probe": {"fixture": "target", "kind": "exact"}})["found"]) - self.adapter.handle_erase({"target": "target", "intent": "subject_erase"}) - self.assertFalse(self.adapter.handle_probe({"probe": {"fixture": "target", "kind": "exact"}})["found"]) - self.assertFalse(self.adapter.handle_inspect({"probe": {"fixture": "target"}})["found"]) - self.assertTrue(self.adapter.handle_probe({"probe": {"fixture": "control", "kind": "exact"}})["found"]) + def probe(self, fixture_id: str, subject: str, kind: str = "exact") -> bool: + return self.adapter.handle_probe( + {"probe": {"fixture": fixture_id, "as_subject": subject, "kind": kind}} + )["found"] + + def test_clean_removes_target_scope_and_preserves_control(self): + self.assertTrue(self.probe("target", "target-subject")) + self.assertTrue(self.probe("control", "control-subject")) + self.adapter.handle_erase( + {"target": "target", "target_subject": "target-subject", "intent": "subject_erase"} + ) + self.assertFalse(self.probe("target", "target-subject")) + self.assertFalse( + self.adapter.handle_inspect( + {"probe": {"fixture": "target", "as_subject": "target-subject"}} + )["found"] + ) + self.assertTrue(self.probe("control", "control-subject")) + + def test_isolation_rejects_cross_subject_reads(self): + self.assertTrue(self.probe("target", "target-subject")) + self.assertFalse(self.probe("target", "control-subject")) + self.assertTrue(self.probe("control", "control-subject")) + self.assertFalse(self.probe("control", "target-subject")) + + def test_leaky_mode_leaves_only_derived_boundary(self): + os.environ["MEMORYPROOF_ADAPTER_MODE"] = "leaky" + adapter = ReferenceAdapter() + adapter.handle_prepare({"run_id": "leaky", "fixtures": [self.target, self.control]}) + adapter.handle_ingest({"fixture": self.target}) + adapter.handle_ingest({"fixture": self.control}) + adapter.handle_erase( + {"target": "target", "target_subject": "target-subject", "intent": "subject_erase"} + ) + self.assertFalse( + adapter.handle_probe( + {"probe": {"fixture": "target", "as_subject": "target-subject", "kind": "exact"}} + )["found"] + ) + self.assertTrue( + adapter.handle_inspect( + {"probe": {"fixture": "target", "as_subject": "target-subject"}} + )["found"] + ) + + def test_cleanup_requires_matching_run_owner(self): + with self.assertRaises(AdapterError): + self.adapter.handle_cleanup({"run_id": "another-run"}) class ResponseParsingTests(unittest.TestCase): def test_structured_search_response(self): - fixture = {"id": "target", "content": "unique canary"} - response = {"results": [{"id": "remote-1", "content": "unique canary"}]} + fixture = {"id": "target", "content": "unique canary 7f3e9d"} + response = {"results": [{"id": "remote-1", "content": "unique canary 7f3e9d"}]} self.assertTrue(extract_found(response, fixture)) + def test_unrelated_response_is_not_a_match(self): + fixture = {"id": "target", "content": "unique canary 7f3e9d"} + response = {"results": [{"id": "remote-1", "content": "another memory"}]} + self.assertFalse(extract_found(response, fixture)) + + def test_protocol_constant_is_stable(self): + self.assertEqual(PROTOCOL_VERSION, "memoryproof.adapter/v1") + if __name__ == "__main__": unittest.main() diff --git a/schemas/scenario.schema.json b/schemas/scenario.schema.json index 09f81c2..e7aac17 100644 --- a/schemas/scenario.schema.json +++ b/schemas/scenario.schema.json @@ -1,12 +1,13 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://forgetproof.dev/schema/scenario/v1alpha1.json", - "title": "ForgetProof Erasure Scenario", + "$id": "https://memoryproof.dev/schema/scenario/v1.json", + "title": "MemoryProof Assurance Scenario", + "description": "A deterministic scenario for erasure or tenant-isolation assurance checks.", "type": "object", - "required": ["apiVersion", "metadata", "spec"], + "required": ["apiVersion", "kind", "metadata", "spec"], "properties": { - "apiVersion": { "const": "forgetproof.dev/v1alpha1" }, - "kind": { "type": "string" }, + "apiVersion": { "const": "memoryproof.dev/v1" }, + "kind": { "const": "AssuranceScenario" }, "metadata": { "type": "object", "required": ["name"], @@ -18,8 +19,9 @@ }, "spec": { "type": "object", - "required": ["adapter", "fixtures", "erase", "probes"], + "required": ["suite", "adapter", "fixtures", "settle", "probes", "profiles"], "properties": { + "suite": { "enum": ["erasure", "isolation"] }, "adapter": { "type": "object", "required": ["name"], @@ -33,7 +35,9 @@ "isolation": { "type": "object", "properties": { - "scope": { "type": "string" }, + "scope": { "type": "string", "minLength": 1 }, + "target_subject": { "type": "string", "minLength": 1 }, + "control_subject": { "type": "string", "minLength": 1 }, "subject": { "type": "string" }, "agent": { "type": "string" }, "thread": { "type": "string" } @@ -42,14 +46,16 @@ }, "fixtures": { "type": "array", - "minItems": 1, + "minItems": 2, "items": { "type": "object", - "required": ["id", "content"], + "required": ["id", "role", "content"], "properties": { "id": { "type": "string", "minLength": 1 }, + "role": { "enum": ["target", "control"] }, "content": { "type": "string", "minLength": 1 }, - "target": { "type": "boolean" }, + "subject": { "type": "string", "minLength": 1 }, + "namespace": { "type": "string" }, "kind": { "type": "string" } }, "additionalProperties": false @@ -67,7 +73,8 @@ "type": "object", "properties": { "intent": { "enum": ["object_delete", "subject_erase", "access_revoke", "derived_purge"] }, - "target": { "type": "string" } + "target": { "type": "string", "minLength": 1 }, + "scope": { "type": "string" } }, "additionalProperties": false }, @@ -75,35 +82,56 @@ "type": "object", "required": ["before", "after"], "properties": { - "before": { "$ref": "#/$defs/probes" }, - "after": { "$ref": "#/$defs/probes" } + "before": { "$ref": "#/$defs/probeList" }, + "after": { "$ref": "#/$defs/probeList" } }, "additionalProperties": false }, "profiles": { "type": "array", - "items": { "enum": ["FP-Object", "FP-Scope", "FP-Derived", "FP-Agent"] } + "minItems": 1, + "items": { + "enum": [ + "erasure.object", + "erasure.scope", + "erasure.derived", + "erasure.agent", + "isolation.read", + "isolation.search", + "isolation.agent" + ] + } }, "privacy": { "type": "object", - "properties": { "raw_payloads": { "type": "boolean", "default": false } }, + "properties": { + "raw_payloads": { "type": "boolean", "default": false } + }, "additionalProperties": false } }, - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { + "if": { "properties": { "suite": { "const": "erasure" } } }, + "then": { "required": ["erase"] } + } + ] } }, "$defs": { - "probes": { + "probeList": { "type": "array", + "minItems": 1, "items": { "type": "object", "required": ["id", "fixture"], - "properties": { + "properties": { "id": { "type": "string", "minLength": 1 }, "fixture": { "type": "string", "minLength": 1 }, "kind": { "enum": ["exact", "lexical", "semantic", "inspect", "agent"] }, - "query": { "type": "string" } + "query": { "type": "string" }, + "as_subject": { "type": "string" } }, "additionalProperties": false } diff --git a/scripts/build_matrix.py b/scripts/build_matrix.py index a1f7c61..0b98b07 100644 --- a/scripts/build_matrix.py +++ b/scripts/build_matrix.py @@ -1,38 +1,107 @@ +"""Build the static MemoryProof assurance matrix from reviewed evidence bundles. + +The script deliberately verifies each local bundle before exposing it in the +public matrix. A result without a valid checksum manifest is never published. +""" + from __future__ import annotations +import hashlib import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -SOURCE = ROOT / "conformance" +SOURCE = ROOT / "conformance" / "evidence" OUTPUT = ROOT / "site" / "matrix.json" ALLOWED_STATUSES = {"PASS", "FAIL", "SKIP", "UNKNOWN"} +ALLOWED_FORMATS = {"memoryproof.bundle/v1", "forgetproof.bundle/v1alpha1"} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def verify_bundle(directory: Path, manifest: dict) -> str: + if manifest.get("format") not in ALLOWED_FORMATS: + raise SystemExit(f"{directory}: unsupported bundle format {manifest.get('format')!r}") + checksums_path = directory / "checksums.sha256" + if not checksums_path.is_file(): + raise SystemExit(f"{directory}: missing checksums.sha256") + expected = {} + for line in checksums_path.read_text().splitlines(): + digest, separator, name = line.partition(" ") + if not separator or len(digest) != 64: + raise SystemExit(f"{directory}: malformed checksum line {line!r}") + expected[name] = digest + listed = set(manifest.get("files", [])) + if set(expected) != listed: + raise SystemExit(f"{directory}: manifest/checksum file list differs") + for name, digest in expected.items(): + path = directory / name + if not path.is_file() or sha256(path) != digest: + raise SystemExit(f"{directory}: checksum mismatch for {name}") + bundle_hash = hashlib.sha256((checksums_path.read_text()).encode()).hexdigest() + recorded = (directory / "bundle.hash").read_text().strip() if (directory / "bundle.hash").is_file() else "" + if recorded and recorded != bundle_hash: + raise SystemExit(f"{directory}: bundle.hash does not match checksums.sha256") + return bundle_hash def main() -> None: - results = [] - for path in sorted(SOURCE.glob("*.json")): - if path.name.startswith("_"): - continue - data = json.loads(path.read_text()) - bundle = data.get("bundle", path.stem) - for profile in data.get("profiles", []): - status = profile.get("status", "UNKNOWN") - if status not in ALLOWED_STATUSES: - raise SystemExit(f"{path}: unsupported status {status!r}") - if not profile.get("evidence", bundle): - raise SystemExit(f"{path}: profile is missing evidence") - results.append( - { - "adapter": data.get("adapter", "unknown"), - "backend": data.get("backend", "unknown"), - "profile": profile.get("profile", "unknown"), - "status": status, - "evidence": profile.get("evidence", bundle), - } - ) - OUTPUT.write_text(json.dumps({"generated_by": "forgetproof", "results": results}, indent=2) + "\n") + rows: list[dict] = [] + if SOURCE.exists(): + for directory in sorted(path for path in SOURCE.iterdir() if path.is_dir()): + manifest_path = directory / "manifest.json" + results_path = directory / "results.json" + if not manifest_path.is_file() or not results_path.is_file(): + raise SystemExit(f"{directory}: public evidence needs manifest.json and results.json") + manifest = json.loads(manifest_path.read_text()) + result = json.loads(results_path.read_text()) + bundle_hash = verify_bundle(directory, manifest) + statuses = {item.get("status", "UNKNOWN") for item in result.get("profiles", [])} + if not statuses: + statuses = {result.get("status", "UNKNOWN")} + for status in statuses: + if status not in ALLOWED_STATUSES: + raise SystemExit(f"{directory}: unsupported status {status!r}") + for profile in result.get("profiles", []): + status = profile.get("status", "UNKNOWN") + if status not in ALLOWED_STATUSES: + raise SystemExit(f"{directory}: unsupported profile status {status!r}") + rows.append( + { + "suite": result.get("suite", "erasure"), + "adapter": result.get("adapter", "unknown"), + "backend": result.get("backend", "unknown"), + "backend_version": result.get("backend_version", "unknown"), + "profile": profile.get("profile", "unknown"), + "status": status, + "evidence_kind": ( + "official adapter contract · local mock" + if directory.name.endswith("-adapter-contract") + else "maintainer reference scenario" + ), + "evidence": f"./evidence/{directory.name}/report.html", + "bundle_hash": bundle_hash[:16], + "source": manifest.get("source", "maintainer-reproduced"), + } + ) + OUTPUT.write_text( + json.dumps( + { + "format": "memoryproof.matrix/v1", + "generated_by": "scripts/build_matrix.py", + "results": rows, + }, + indent=2, + ) + + "\n" + ) if __name__ == "__main__": diff --git a/scripts/mock_remote_backend.py b/scripts/mock_remote_backend.py new file mode 100644 index 0000000..ced3430 --- /dev/null +++ b/scripts/mock_remote_backend.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Small deterministic HTTP backend for official adapter contract tests. + +It is intentionally not a provider emulator. It implements only the narrow +request/response shapes used by the dependency-free Mem0, Letta, and Zep +adapters so contributors can reproduce adapter behavior without credentials. +It stores everything in memory and exits without writing user data to disk. +""" + +from __future__ import annotations + +import argparse +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse + + +class State: + def __init__(self, provider: str) -> None: + self.provider = provider + self.lock = threading.Lock() + self.counter = 0 + self.records: dict[str, dict] = {} + self.agents: dict[str, dict] = {} + self.users: dict[str, dict] = {} + self.threads: dict[str, dict] = {} + + def new_id(self, prefix: str) -> str: + with self.lock: + self.counter += 1 + return f"mock-{prefix}-{self.counter}" + + +def _json(handler: BaseHTTPRequestHandler) -> dict: + length = int(handler.headers.get("Content-Length", "0")) + if not length: + return {} + try: + value = json.loads(handler.rfile.read(length)) + except json.JSONDecodeError: + return {} + return value if isinstance(value, dict) else {} + + +def _contains(query: str, content: str) -> bool: + if not query: + return True + markers = re.findall(r"[A-Za-z0-9][A-Za-z0-9_-]{5,}", content) + return query in content or any(marker in query for marker in markers) + + +class Handler(BaseHTTPRequestHandler): + server_version = "MemoryProofMock/1" + + @property + def state(self) -> State: + return self.server.state # type: ignore[attr-defined] + + def log_message(self, fmt: str, *args: object) -> None: + # Keep CI output quiet and deterministic; the adapter owns request IDs. + return + + def send_json(self, value: object, status: int = 200) -> None: + raw = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.send_header("X-Request-Id", f"mock-{self.state.provider}") + self.end_headers() + self.wfile.write(raw) + + def do_GET(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + query = parse_qs(parsed.query) + path = parsed.path + if path == "/health": + self.send_json({"status": "ok", "provider": self.state.provider}) + return + if self.state.provider == "mem0": + self.mem0_get(path, query) + elif self.state.provider == "letta": + self.letta_get(path, query) + else: + self.zep_get(path, query) + + def do_POST(self) -> None: # noqa: N802 + body = _json(self) + path = urlparse(self.path).path + if self.state.provider == "mem0": + self.mem0_post(path, body) + elif self.state.provider == "letta": + self.letta_post(path, body) + else: + self.zep_post(path, body) + + def do_DELETE(self) -> None: # noqa: N802 + parsed = urlparse(self.path) + query = parse_qs(parsed.query) + path = parsed.path + if self.state.provider == "mem0": + self.mem0_delete(path, query) + elif self.state.provider == "letta": + self.letta_delete(path) + else: + self.zep_delete(path) + + def mem0_post(self, path: str, body: dict) -> None: + if path in {"/memories", "/v1/memories"}: + user = str(body.get("user_id", "")) + messages = body.get("messages") or [] + content = str(messages[0].get("content", "")) if messages else str(body.get("text", "")) + item_id = self.state.new_id("memory") + self.state.records[item_id] = {"id": item_id, "user_id": user, "content": content} + self.send_json({"id": item_id, "status": "created"}) + return + if path.endswith("/search"): + self.send_json(self._mem0_results(body.get("user_id", ""), body.get("query", ""))) + return + self.send_json({"ok": True}) + + def mem0_get(self, path: str, query: dict[str, list[str]]) -> None: + if path in {"/memories", "/v1/memories"}: + user = (query.get("user_id") or [""])[0] + self.send_json(self._mem0_results(user, "")) + return + self.send_json({"results": []}) + + def mem0_delete(self, path: str, query: dict[str, list[str]]) -> None: + user = (query.get("user_id") or [""])[0] + if path in {"/memories", "/v1/memories"} and user: + for item_id in list(self.state.records): + if self.state.records[item_id].get("user_id") == user: + del self.state.records[item_id] + self.send_json({"deleted": True}) + return + item_id = path.rsplit("/", 1)[-1] + self.state.records.pop(item_id, None) + self.send_json({"deleted": True}) + + def _mem0_results(self, user: object, query: object) -> dict: + user = str(user) + query = str(query) + return {"results": [{"id": item_id, "memory": item["content"], "user_id": user} for item_id, item in self.state.records.items() if item.get("user_id") == user and _contains(query, item["content"])]} + + def letta_post(self, path: str, body: dict) -> None: + if path == "/v1/agents": + agent_id = self.state.new_id("agent") + self.state.agents[agent_id] = {"deleted": False, "records": {}} + self.send_json({"id": agent_id}) + return + match = re.fullmatch(r"/v1/agents/([^/]+)/archival-memory", path) + if match: + agent = self.state.agents.setdefault(match.group(1), {"deleted": False, "records": {}}) + passage_id = self.state.new_id("passage") + agent["records"][passage_id] = str(body.get("text", "")) + self.send_json({"id": passage_id}) + return + match = re.fullmatch(r"/v1/agents/([^/]+)/messages", path) + if match: + agent = self.state.agents.get(match.group(1), {"deleted": True, "records": {}}) + texts = list(agent.get("records", {}).values()) if not agent.get("deleted") else [] + self.send_json({"messages": [{"role": "assistant", "content": text} for text in texts]}) + return + self.send_json({"ok": True}) + + def letta_get(self, path: str, query: dict[str, list[str]]) -> None: + match = re.fullmatch(r"/v1/agents/([^/]+)/archival-memory/search", path) + if match: + agent = self.state.agents.get(match.group(1), {"deleted": True, "records": {}}) + wanted = (query.get("query") or [""])[0] + self.send_json({"results": [{"id": item_id, "text": text} for item_id, text in agent.get("records", {}).items() if not agent.get("deleted") and _contains(wanted, text)]}) + return + match = re.fullmatch(r"/v1/agents/([^/]+)/archival-memory", path) + if match: + agent = self.state.agents.get(match.group(1), {"deleted": True, "records": {}}) + self.send_json({"results": [{"id": item_id, "text": text} for item_id, text in agent.get("records", {}).items()] if not agent.get("deleted") else {"results": []}}) + return + self.send_json({"results": []}) + + def letta_delete(self, path: str) -> None: + match = re.fullmatch(r"/v1/agents/([^/]+)(?:/archival-memory/([^/]+))?", path) + if match: + agent = self.state.agents.get(match.group(1)) + if agent is not None: + if match.group(2): + agent["records"].pop(match.group(2), None) + else: + agent["deleted"] = True + agent["records"].clear() + self.send_json({"deleted": True}) + return + self.send_json({"deleted": True}) + + def zep_post(self, path: str, body: dict) -> None: + if path == "/api/v2/users": + user_id = str(body.get("user_id") or self.state.new_id("user")) + self.state.users[user_id] = {"deleted": False, "threads": []} + self.send_json({"user_id": user_id}) + return + if path == "/api/v2/threads": + thread_id = str(body.get("thread_id") or self.state.new_id("thread")) + self.state.threads[thread_id] = {"user_id": str(body.get("user_id", "")), "deleted": False, "records": {}} + self.send_json({"thread_id": thread_id}) + return + match = re.fullmatch(r"/api/v2/threads/([^/]+)/messages", path) + if match: + thread = self.state.threads.setdefault(match.group(1), {"user_id": "", "deleted": False, "records": {}}) + messages = body.get("messages") or [] + content = str(messages[0].get("content", "")) if messages else "" + episode_id = self.state.new_id("episode") + thread["records"][episode_id] = content + self.send_json({"id": episode_id, "episode_id": episode_id}) + return + self.send_json({"ok": True}) + + def zep_get(self, path: str, query: dict[str, list[str]]) -> None: + match = re.fullmatch(r"/api/v2/threads/([^/]+)/search", path) + if match: + thread = self.state.threads.get(match.group(1), {"deleted": True, "records": {}}) + wanted = (query.get("query") or [""])[0] + self.send_json({"messages": [{"id": item_id, "content": text} for item_id, text in thread.get("records", {}).items() if not thread.get("deleted") and _contains(wanted, text)]}) + return + match = re.fullmatch(r"/api/v2/users/([^/]+)/graph", path) + if match: + user_id = match.group(1) + nodes = [] + for thread in self.state.threads.values(): + if thread.get("user_id") == user_id and not thread.get("deleted"): + nodes.extend({"id": item_id, "name": text} for item_id, text in thread["records"].items()) + self.send_json({"nodes": nodes, "edges": []}) + return + self.send_json({"results": []}) + + def zep_delete(self, path: str) -> None: + match = re.fullmatch(r"/api/v2/users/([^/]+)", path) + if match: + user_id = match.group(1) + user = self.state.users.setdefault(user_id, {"deleted": False, "threads": []}) + user["deleted"] = True + for thread in self.state.threads.values(): + if thread.get("user_id") == user_id: + thread["deleted"] = True + thread["records"].clear() + self.send_json({"deleted": True}) + return + match = re.fullmatch(r"/api/v2/threads/([^/]+)/messages/([^/]+)", path) + if match: + thread = self.state.threads.get(match.group(1)) + if thread: + thread["records"].pop(match.group(2), None) + self.send_json({"deleted": True}) + return + self.send_json({"deleted": True}) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--provider", choices=["mem0", "letta", "zep"], required=True) + parser.add_argument("--port", type=int, required=True) + args = parser.parse_args() + server = ThreadingHTTPServer(("127.0.0.1", args.port), Handler) + server.state = State(args.provider) # type: ignore[attr-defined] + print(f"MemoryProof mock {args.provider} listening on 127.0.0.1:{args.port}", flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/site/assets/memoryproof-hero.png b/site/assets/memoryproof-hero.png new file mode 100644 index 0000000..4e88b9f Binary files /dev/null and b/site/assets/memoryproof-hero.png differ diff --git a/site/favicon.svg b/site/favicon.svg index 32b56f8..e6c587f 100644 --- a/site/favicon.svg +++ b/site/favicon.svg @@ -1,6 +1,7 @@ - + - - - + + + + diff --git a/site/index.html b/site/index.html index f623c3b..9e655f2 100644 --- a/site/index.html +++ b/site/index.html @@ -3,177 +3,68 @@ - - - + + + + - ForgetProof · Conformance Matrix + MemoryProof · Memory Assurance Matrix
    -
    - FORGETPROOF · PROVE YOUR AI FORGOT - -
    +
    MEMORYPROOF · MEMORY ASSURANCE
    -
    -

    Conformance
    for forgetting.

    -

    A practical way to test whether an AI memory system really erased a piece of information.

    -
    DELETE 200 OK ≠ the canary is no longer observable.
    -
    -
    -

    给“遗忘”
    做认证。

    -

    用一套可重复的测试,检查 AI 记忆系统是否真的删除了一条信息。

    -
    DELETE 200 OK ≠ canary 已经无法被召回。
    -
    - A synthetic canary disappearing from a memory graph while an evidence ledger verifies the change +

    Make forgetting
    observable.

    A practical conformance layer for AI memory systems. Test the claim behind “deleted” before it reaches production.

    DELETE 200 OK ≠ the canary is no longer observable.
    +

    让“遗忘”
    可以被观察。

    面向 AI 记忆系统的实用认证层。在进入生产前,验证“已删除”背后的真实结论。

    DELETE 200 OK ≠ canary 已经无法被召回。
    + A synthetic canary leaves an observable memory graph while evidence records the boundary
    -
    -

    From “trust me” to evidence

    -
    -
    BEFORE

    DELETE 200 OK

    The API accepted the request. You still do not know whether a summary, index, graph, or Agent path can recall the canary.

    ⚠️ Claim not proven
    -
    AFTER

    Evidence bundle

    Before/after probes, control fixtures, capability limits, and a SHA-256 manifest make the result reviewable.

    ✅ PASS · FAIL · UNKNOWN
    -
    -
    -
    -

    从“相信我”到证据

    -
    -
    之前

    DELETE 200 OK

    API 接受了删除请求,但你仍不知道摘要、索引、图谱或 Agent 路径是否还能召回 canary。

    ⚠️ 结论还没有被证明
    -
    之后

    证据包

    删除前后探针、控制 canary、能力边界和 SHA-256 清单,让结果可以审阅和复现。

    ✅ PASS · FAIL · UNKNOWN
    -
    -
    +

    From an API promise to a proof package

    BEFORE
    ✅ 200 OK

    The request was accepted. You still do not know whether a summary, vector, graph, cache, or Agent path can recall the canary.

    ⚠ claim not proven
    AFTER
    📦 Evidence bundle

    Before/after probes, control fixtures, capability limits, and a SHA-256 manifest make the result reviewable and reproducible.

    ✓ PASS · FAIL · UNKNOWN
    +

    从 API 承诺到可审阅的证明包

    之前
    ✅ 200 OK

    请求被接受了,但你仍不知道摘要、向量、图、缓存或 Agent 路径能不能召回这条 canary。

    ⚠ 结论尚未证明
    之后
    📦 证据包

    删除前后探针、控制 fixture、能力边界和 SHA-256 清单,让结果可审阅、可复现。

    ✓ PASS · FAIL · UNKNOWN
    -
    -

    What this matrix means

    -
    -

    PASS

    Every required observable check passed.

    -

    FAIL

    A probe still found the target or a forbidden derivative.

    -

    UNKNOWN

    The backend did not expose enough information to make a claim.

    -
    -
    -
    -

    如何理解这张矩阵

    -
    -

    PASS

    所有必需的可观察检查都通过了。

    -

    FAIL

    某个探针仍然找到了目标数据或被禁止的衍生数据。

    -

    UNKNOWN

    后端没有公开足够信息,无法作出更强结论。

    -
    -
    +

    What the matrix says

    PASS
    The required path is clear.

    Every selected observable assertion passed.

    ✓ proved within boundary
    FAIL
    A path still leaks.

    A target, derivative, or unrelated control fixture was found.

    × regression found
    UNKNOWN
    The boundary is not visible.

    The adapter did not expose enough information to make a stronger claim.

    ? honest uncertainty
    +

    如何理解这张矩阵

    PASS
    必需路径已经清空。

    所选的所有可观察断言都通过了。

    ✓ 在边界内已证明
    FAIL
    仍然存在泄漏路径。

    找到了目标、衍生数据或不相关的控制 fixture。

    × 发现回归
    UNKNOWN
    边界不可见。

    适配器没有暴露足够信息,不能作出更强结论。

    ? 诚实的不确定
    -
    -
    -

    Public conformance results

    -

    Results are evidence-linked and versioned. There is no aggregate score. Unknown means unknown.

    -
    -
    -

    公开认证结果

    -

    每条结果都关联证据包并记录后端版本。这里没有综合分数;Unknown 就是未知。

    -
    -
    Loading verified bundles…
    -
    +

    Public assurance results

    Evidence-linked and backend-versioned. There is no aggregate score: unknown means unknown.

    公开保证结果

    每条结果都链接证据并记录后端版本。没有综合分数:未知就是未知。

    Loading verified bundles…
    -
    -

    What ForgetProof does not claim

    -

    It does not prove provider logs, backups, physical storage erasure, model-weight unlearning, or anything outside the adapter’s observable boundary.

    -
    -
    -

    ForgetProof 不会声称什么

    -

    它不会证明服务商日志、备份、物理存储已经删除,也不会证明模型权重完成反学习,更不会对适配器观察边界之外的事情作保证。

    -
    - +

    Designed for safe experiments

    🧪 Synthetic first

    Unique canaries and control fixtures keep production memories out of the test.

    🔐 Owned resources

    Cleanup refuses resources without the current run’s ownership marker.

    🧭 Honest boundaries

    Provider logs, backups, physical storage, and model weights stay explicitly out of scope.

    +

    为安全实验而设计

    🧪 合成数据优先

    唯一 canary 和控制 fixture 让生产记忆不会进入测试。

    🔐 只操作自有资源

    没有本次运行所有权标记的资源,清理阶段会拒绝操作。

    🧭 诚实的边界

    供应商日志、备份、物理存储和模型权重都会明确标为范围之外。

    + +

    Try the reference proof

    Run cargo run -- run examples/reference-clean.yml for a passing baseline, then cargo run -- run examples/reference-leaky.yml to see a deterministic derived-residue failure.

    +

    运行参考证明

    运行 cargo run -- run examples/reference-clean.yml 查看通过基线,再运行 cargo run -- run examples/reference-leaky.yml 查看确定性的衍生残留失败。

    + +
    diff --git a/site/matrix.json b/site/matrix.json index 567c2bd..33204f2 100644 --- a/site/matrix.json +++ b/site/matrix.json @@ -1,4 +1,270 @@ { - "generated_by": "forgetproof", - "results": [] + "format": "memoryproof.matrix/v1", + "generated_by": "scripts/build_matrix.py", + "results": [ + { + "suite": "isolation", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "isolation.read", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/isolation-reference/report.html", + "bundle_hash": "c9ae865a69890304", + "source": "maintainer-reproduced" + }, + { + "suite": "isolation", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "isolation.search", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/isolation-reference/report.html", + "bundle_hash": "c9ae865a69890304", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "letta", + "backend": "letta", + "backend_version": "configured", + "profile": "erasure.object", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/letta-adapter-contract/report.html", + "bundle_hash": "0a9cbd606722d2e4", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "letta", + "backend": "letta", + "backend_version": "configured", + "profile": "erasure.scope", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/letta-adapter-contract/report.html", + "bundle_hash": "0a9cbd606722d2e4", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "letta", + "backend": "letta", + "backend_version": "configured", + "profile": "erasure.derived", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/letta-adapter-contract/report.html", + "bundle_hash": "0a9cbd606722d2e4", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "letta", + "backend": "letta", + "backend_version": "configured", + "profile": "erasure.agent", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/letta-adapter-contract/report.html", + "bundle_hash": "0a9cbd606722d2e4", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "mem0", + "backend": "mem0", + "backend_version": "configured", + "profile": "erasure.object", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/mem0-adapter-contract/report.html", + "bundle_hash": "af8b543c1f708f22", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "mem0", + "backend": "mem0", + "backend_version": "configured", + "profile": "erasure.scope", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/mem0-adapter-contract/report.html", + "bundle_hash": "af8b543c1f708f22", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "mem0", + "backend": "mem0", + "backend_version": "configured", + "profile": "erasure.derived", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/mem0-adapter-contract/report.html", + "bundle_hash": "af8b543c1f708f22", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.object", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-clean/report.html", + "bundle_hash": "c6feb6e3dbd230d2", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.scope", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-clean/report.html", + "bundle_hash": "c6feb6e3dbd230d2", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.derived", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-clean/report.html", + "bundle_hash": "c6feb6e3dbd230d2", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-clean", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.agent", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-clean/report.html", + "bundle_hash": "c6feb6e3dbd230d2", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-leaky", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.object", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-leaky/report.html", + "bundle_hash": "75a3632854a97861", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-leaky", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.scope", + "status": "PASS", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-leaky/report.html", + "bundle_hash": "75a3632854a97861", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-leaky", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.derived", + "status": "FAIL", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-leaky/report.html", + "bundle_hash": "75a3632854a97861", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-leaky", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.agent", + "status": "FAIL", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-leaky/report.html", + "bundle_hash": "75a3632854a97861", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-overdelete", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.object", + "status": "FAIL", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-overdelete/report.html", + "bundle_hash": "f0a0d36541832468", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "reference-overdelete", + "backend": "memoryproof-reference", + "backend_version": "1.0.0", + "profile": "erasure.scope", + "status": "FAIL", + "evidence_kind": "maintainer reference scenario", + "evidence": "./evidence/reference-overdelete/report.html", + "bundle_hash": "f0a0d36541832468", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "zep", + "backend": "zep", + "backend_version": "configured", + "profile": "erasure.object", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/zep-adapter-contract/report.html", + "bundle_hash": "0ba7ba7f701f49d1", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "zep", + "backend": "zep", + "backend_version": "configured", + "profile": "erasure.scope", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/zep-adapter-contract/report.html", + "bundle_hash": "0ba7ba7f701f49d1", + "source": "maintainer-reproduced" + }, + { + "suite": "erasure", + "adapter": "zep", + "backend": "zep", + "backend_version": "configured", + "profile": "erasure.derived", + "status": "PASS", + "evidence_kind": "official adapter contract \u00b7 local mock", + "evidence": "./evidence/zep-adapter-contract/report.html", + "bundle_hash": "0ba7ba7f701f49d1", + "source": "maintainer-reproduced" + } + ] }